From a04b88e714e5a05d5193a60144f346a3d154da4e Mon Sep 17 00:00:00 2001 From: DevelopmentCats Date: Mon, 16 Nov 2026 16:30:44 +0000 Subject: [PATCH 1/5] engine: declare offline settings and hub cache variables Engines say how their server stays off the network and which variable sets its hub cache, so no code has to name an engine to do it. --- catalogue/engines/README.md | 1 + catalogue/engines/llamacpp.json | 5 +++ catalogue/engines/mlx.json | 3 ++ catalogue/engines/vllm.json | 3 ++ internal/engine/engine.go | 72 +++++++++++++++++++++++++++++++++ internal/engine/engine_test.go | 57 ++++++++++++++++++++++++++ internal/engine/registry.go | 6 +++ 7 files changed, 147 insertions(+) diff --git a/catalogue/engines/README.md b/catalogue/engines/README.md index 39fcd44..a0c2a8b 100644 --- a/catalogue/engines/README.md +++ b/catalogue/engines/README.md @@ -6,6 +6,7 @@ Each file here is one inference server llm-helper can launch, named after the en To add an engine, copy the closest file to `.json` and change it. No Go code is needed: shared code acts on the engine's `capabilities`, never on its name. `chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; leave it out if the engine has none. +`hub_cache_env` names the variable that sets the hub cache directly, and `offline` says how to keep the server off the network (`env`, `args`, and `probe_args` to check the installed server accepts them); leave either out if the engine has no such setting. `notes.hybrid` is a sentence `inspect` prints for a model whose layers are not all attention; leave it out if there is nothing to say. The file is checked when it loads, and the tests in `internal/engine` render every file here for every rank. diff --git a/catalogue/engines/llamacpp.json b/catalogue/engines/llamacpp.json index a3040d1..97d4aec 100644 --- a/catalogue/engines/llamacpp.json +++ b/catalogue/engines/llamacpp.json @@ -31,6 +31,11 @@ "with_key": "0.0.0.0", "without_key": "127.0.0.1" }, + "hub_cache_env": "LLAMA_CACHE", + "offline": { + "args": ["--offline"], + "probe_args": ["--help"] + }, "api_key": "env:LLAMA_API_KEY", "health": { "path": "/health", diff --git a/catalogue/engines/mlx.json b/catalogue/engines/mlx.json index 39aec0b..20f2508 100644 --- a/catalogue/engines/mlx.json +++ b/catalogue/engines/mlx.json @@ -13,6 +13,9 @@ "names": ["mlx_lm.server"] }, "serve": ["{bin}", "--model", "{model}", "--port", "{port}"], + "offline": { + "env": { "HF_HUB_OFFLINE": "1" } + }, "api_key": "none", "health": { "path": "/health", diff --git a/catalogue/engines/vllm.json b/catalogue/engines/vllm.json index baf3e3c..f7470e8 100644 --- a/catalogue/engines/vllm.json +++ b/catalogue/engines/vllm.json @@ -26,6 +26,9 @@ "--enable-prompt-tokens-details" ], "optional_args": [["--kv-cache-memory-bytes", "{kv_cache_bytes}"]], + "offline": { + "env": { "HF_HUB_OFFLINE": "1" } + }, "api_key": "flag:--api-key", "health": { "path": "/health", diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 80cb08c..e0f454b 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -79,6 +79,12 @@ type Engine struct { Caps Capabilities `json:"capabilities"` // Notes are sentences inspect prints about serving some kinds of model. Notes Notes `json:"notes,omitempty"` + // HubCacheEnv names a variable that sets the server's Hugging Face hub + // cache directly, ahead of the HF_* variables. + HubCacheEnv string `json:"hub_cache_env,omitempty"` + // Offline is how the server is kept off the network. Absent means the + // engine has no setting for it. + Offline *Offline `json:"offline,omitempty"` // Source is the engines.d file a local definition came from, and "" for // one shipped in the binary. @@ -205,6 +211,18 @@ type Capabilities struct { ChatTemplateKwargsFlag string `json:"chat_template_kwargs_flag,omitempty"` } +// Offline is what a profile gets so that its server never reaches the Hub. +type Offline struct { + // Env is added to the profile's environment. + Env map[string]string `json:"env,omitempty"` + // Args are added to the profile's EXTRA_ARGS. + Args []string `json:"args,omitempty"` + // ProbeArgs, when set, are run after the server's binary, and Args are + // added only when the output names Args[0]. A release that predates the + // flag would reject it and fail to start. + ProbeArgs []string `json:"probe_args,omitempty"` +} + // Notes are sentences about how the engine serves a kind of model. type Notes struct { // Hybrid is shown for models whose layers are not all attention. @@ -314,6 +332,28 @@ func (e Engine) Capabilities() Capabilities { // HybridNote is what inspect says about serving a hybrid model. func (e Engine) HybridNote() string { return e.Notes.Hybrid } +// HubCacheVar is the variable that sets the server's hub cache directly, or +// "" when it has none. +func (e Engine) HubCacheVar() string { return e.HubCacheEnv } + +// OfflineSetting is how the server is kept off the network, or nil. +func (e Engine) OfflineSetting() *Offline { return e.Offline } + +// EnvLines is the KEY=value entries, sorted by name. +func (o Offline) EnvLines() []string { + out := make([]string, 0, len(o.Env)) + for _, k := range slices.Sorted(maps.Keys(o.Env)) { + out = append(out, k+"="+o.Env[k]) + } + return out +} + +// ArgsAccepted reports whether the server takes Args, given the output of +// running it with ProbeArgs. Without ProbeArgs it always does. +func (o Offline) ArgsAccepted(probeOutput string) bool { + return len(o.ProbeArgs) == 0 || (len(o.Args) > 0 && strings.Contains(probeOutput, o.Args[0])) +} + // InstallHint says how to get the server on goos. func (e Engine) InstallHint(goos string) string { if h := e.Install[goos]; h != "" { @@ -804,6 +844,13 @@ func validate(e Engine, hw *hardware.Registry) []error { add("capabilities.chat_template_kwargs_flag %q is not a flag", f) } + if k := e.HubCacheEnv; k != "" && !envNamePattern.MatchString(k) { + add("hub_cache_env %q is not a variable name", k) + } + if o := e.Offline; o != nil { + validateOffline(o, e.Has(RuntimeProcess), add) + } + params := map[string]bool{} for _, p := range e.Params { switch { @@ -911,6 +958,31 @@ func validate(e Engine, hw *hardware.Registry) []error { return errs } +func validateOffline(o *Offline, process bool, add func(string, ...any)) { + switch { + case !process: + add("offline belongs to the process runtime, which this engine does not declare") + case len(o.Env) == 0 && len(o.Args) == 0: + add("offline sets neither env nor args") + } + for _, k := range slices.Sorted(maps.Keys(o.Env)) { + if !envNamePattern.MatchString(k) { + add("offline.env: %q is not a variable name", k) + } + } + if len(o.Args) > 0 && !capFlagPattern.MatchString(o.Args[0]) { + add("offline.args must start with a flag, not %q", o.Args[0]) + } + if len(o.ProbeArgs) > 0 && len(o.Args) == 0 { + add("offline.probe_args needs offline.args, which it checks for") + } + for _, a := range slices.Concat(o.Args, o.ProbeArgs) { + if a == "" || placeholderRE.MatchString(a) { + add("offline: %q is empty or a placeholder; these are literal", a) + } + } +} + func checkEnv(where string, env map[string]string, ok map[string]bool, check func(string, []string, map[string]bool), add func(string, ...any)) { for _, k := range slices.Sorted(maps.Keys(env)) { if !envNamePattern.MatchString(k) { diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index 8ecd8cb..4b3e521 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -175,6 +175,13 @@ func TestParseIsStrict(t *testing.T) { {"process engine without install hint", toyProcess, `"install": { "any": "pip install toy" },`, ``, "install hint"}, {"trailing data", toyContainer, ``, ``, "after the top-level"}, {"kwargs flag that is not a flag", toyContainer, `"capabilities": {}`, `"capabilities": {"chat_template_kwargs_flag": "kwargs"}`, "chat_template_kwargs_flag"}, + {"offline that sets nothing", toyProcess, `"capabilities": {}`, `"offline": {}, "capabilities": {}`, "neither env nor args"}, + {"offline env that is not a name", toyProcess, `"capabilities": {}`, `"offline": {"env": {"A B": "1"}}, "capabilities": {}`, "not a variable name"}, + {"offline arg that is not a flag", toyProcess, `"capabilities": {}`, `"offline": {"args": ["offline"]}, "capabilities": {}`, "must start with a flag"}, + {"probe without args", toyProcess, `"capabilities": {}`, `"offline": {"env": {"A": "1"}, "probe_args": ["--help"]}, "capabilities": {}`, "needs offline.args"}, + {"offline arg that is a placeholder", toyProcess, `"capabilities": {}`, `"offline": {"args": ["--offline", "{port}"]}, "capabilities": {}`, "placeholder"}, + {"offline on a container-only engine", toyContainer, `"capabilities": {}`, `"offline": {"env": {"A": "1"}}, "capabilities": {}`, "process runtime"}, + {"hub cache variable that is not a name", toyProcess, `"capabilities": {}`, `"hub_cache_env": "A-B", "capabilities": {}`, "hub_cache_env"}, } { data := strings.Replace(tc.base, tc.old, tc.new, 1) if tc.name == "trailing data" { @@ -338,3 +345,53 @@ func TestUseLocalSkipsAndReportsBadFiles(t *testing.T) { t.Error("UseLocal kept errors from a directory it no longer reads") } } + +// How each shipped engine is kept off the network is data in its file, and the +// llama.cpp flag is used only when the installed server lists it. +func TestShippedEnginesDeclareHowToGoOffline(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + id string + env, args []string + probe []string + cacheVar string + }{ + {id: engine.VLLM, env: []string{"HF_HUB_OFFLINE=1"}}, + {id: engine.MLX, env: []string{"HF_HUB_OFFLINE=1"}}, + {id: engine.LlamaCPP, args: []string{"--offline"}, probe: []string{"--help"}, cacheVar: "LLAMA_CACHE"}, + } { + p, err := engine.Lookup(tc.id) + if err != nil { + t.Fatal(err) + } + o := p.OfflineSetting() + if o == nil { + t.Fatalf("%s: no offline setting", tc.id) + } + if got := o.EnvLines(); !slices.Equal(got, tc.env) && len(got)+len(tc.env) > 0 { + t.Errorf("%s: env = %q, want %q", tc.id, got, tc.env) + } + if !slices.Equal(o.Args, tc.args) || !slices.Equal(o.ProbeArgs, tc.probe) { + t.Errorf("%s: args = %q, probe = %q", tc.id, o.Args, o.ProbeArgs) + } + if p.HubCacheVar() != tc.cacheVar { + t.Errorf("%s: hub_cache_env = %q, want %q", tc.id, p.HubCacheVar(), tc.cacheVar) + } + } +} + +func TestOfflineArgsAreProbed(t *testing.T) { + t.Parallel() + p, _ := engine.Lookup(engine.LlamaCPP) + o := p.OfflineSetting() + if !o.ArgsAccepted(" --offline Offline mode: forces use of cache") { + t.Error("a --help that lists --offline was refused") + } + if o.ArgsAccepted("--host HOST") || o.ArgsAccepted("") { + t.Error("a --help that does not list --offline was accepted") + } + v, _ := engine.Lookup(engine.VLLM) + if !v.OfflineSetting().ArgsAccepted("") { + t.Error("an offline setting with no probe must always be accepted") + } +} diff --git a/internal/engine/registry.go b/internal/engine/registry.go index 0e00a12..f935c17 100644 --- a/internal/engine/registry.go +++ b/internal/engine/registry.go @@ -61,6 +61,12 @@ type Provider interface { // HybridNote is what inspect says about serving a hybrid model, if // anything. HybridNote() string + // HubCacheVar is the variable that sets the server's hub cache directly, + // "" when it has none. + HubCacheVar() string + // OfflineSetting is how the server is kept off the network, nil when it + // has no setting for it. + OfflineSetting() *Offline // Has reports whether the engine can run in a runtime. Has(runtime string) bool // InstallHint says how to get the server on an OS, by GOOS. From 0cf7cdd9a95b1c713cd7e5a87456aa7fcd94422e Mon Sep 17 00:00:00 2001 From: DevelopmentCats Date: Mon, 16 Nov 2026 16:30:45 +0000 Subject: [PATCH 2/5] hfcache: add hub cache scan, import and export Reads the cache layout all engines share, and copies complete models between caches with every file checked against its hash. --- internal/hfcache/copy.go | 367 +++++++++++ internal/hfcache/hfcache.go | 139 ++++ internal/hfcache/hfcache_internal_test.go | 731 ++++++++++++++++++++++ internal/hfcache/scan.go | 532 ++++++++++++++++ internal/hfcache/space_darwin.go | 29 + internal/hfcache/space_linux.go | 22 + internal/hfcache/space_other.go | 11 + 7 files changed, 1831 insertions(+) create mode 100644 internal/hfcache/copy.go create mode 100644 internal/hfcache/hfcache.go create mode 100644 internal/hfcache/hfcache_internal_test.go create mode 100644 internal/hfcache/scan.go create mode 100644 internal/hfcache/space_darwin.go create mode 100644 internal/hfcache/space_linux.go create mode 100644 internal/hfcache/space_other.go diff --git a/internal/hfcache/copy.go b/internal/hfcache/copy.go new file mode 100644 index 0000000..b468d0c --- /dev/null +++ b/internal/hfcache/copy.go @@ -0,0 +1,367 @@ +package hfcache + +import ( + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "os" + "path/filepath" + "strings" +) + +// partialSuffix ends a file that is still being copied. Caches ignore +// *.incomplete, so an interrupted copy leaves nothing a loader would read. +const partialSuffix = ".llm-helper.incomplete" + +// lfsThreshold is the size above which the Hub stores a file with LFS, so its +// blob is named by sha256 rather than by git blob id. +const lfsThreshold = 10 << 20 + +// Progress is told of each file as it is copied or found already there. +type Progress func(path string, size int64, skipped bool) + +// Result is what copying one model did. +type Result struct { + Copied int + Skipped int + Bytes int64 + // Links is whether snapshot files are links to blobs, rather than files. + Links bool + // RefKept is the revision refs/main already named, when it differs from + // the one copied. It is left alone, so the new revision is not the one an + // offline load reads. + RefKept string +} + +// newSum hashes a file the way its blob is named: the sha256 of an LFS file, +// and the git blob id of any other. etag says which when the source records +// it; without one, the size does. +func newSum(etag string, size int64) hash.Hash { + if len(etag) == 64 || (etag == "" && size > lfsThreshold) { + return sha256.New() + } + h := sha1.New() + fmt.Fprintf(h, "blob %d\x00", size) + return h +} + +// copyFile copies src to tmp, hashing as it goes, and returns the blob name. +// A recorded etag that the content does not match removes tmp and fails. +func copyFile(src, tmp, etag string, size int64) (string, error) { + in, err := os.Open(src) + if err != nil { + return "", err + } + defer in.Close() + if st, err := in.Stat(); err != nil || st.Size() != size { + return "", fmt.Errorf("%s changed size while being read", src) + } + out, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + return "", err + } + h := newSum(etag, size) + n, err := io.CopyBuffer(io.MultiWriter(out, h), in, make([]byte, 1<<20)) + if err == nil && n != size { + err = fmt.Errorf("read %d of %d bytes from %s", n, size, src) + } + if err == nil { + err = out.Sync() + } + if cerr := out.Close(); err == nil { + err = cerr + } + if err != nil { + os.Remove(tmp) + return "", err + } + got := hex.EncodeToString(h.Sum(nil)) + if etag != "" && got != etag { + os.Remove(tmp) + return "", &CorruptError{Path: src, Want: etag, Got: got} + } + return got, nil +} + +// CorruptError is a file whose content does not match the hash recorded for +// it. +type CorruptError struct { + Path, Want, Got string +} + +func (e *CorruptError) Error() string { + return fmt.Sprintf("%s does not match its recorded hash (want %s, got %s): the copy it came from may be damaged", e.Path, e.Want, e.Got) +} + +// partialPath is where the file at rel is copied before it is renamed into +// place. +func partialPath(repoDir, rel string) string { + sum := sha256.Sum256([]byte(rel)) + return filepath.Join(repoDir, "blobs", hex.EncodeToString(sum[:8])+partialSuffix) +} + +// sizeIs reports whether p is a file of the given size. +func sizeIs(p string, size int64) bool { + st, err := os.Stat(p) + return err == nil && st.Mode().IsRegular() && st.Size() == size +} + +// symlink is os.Symlink, so tests can take links away. +var symlink = os.Symlink + +// canLink reports whether dir takes symbolic links. +func canLink(dir string) bool { + p := filepath.Join(dir, ".llm-helper-link-test") + os.Remove(p) + if symlink("x", p) != nil { + return false + } + os.Remove(p) + return true +} + +// place copies a complete model into hub. With links, files become blobs named +// by their hash and snapshot entries relative links to them, as +// huggingface_hub writes them; without, the snapshot holds the files. +// Whatever is already there is kept and never overwritten. +func place(m Model, hub string, links bool, progress Progress) (Result, error) { + res := Result{Links: links} + if !m.Complete() { + return res, fmt.Errorf("%s is incomplete: %s", m.Repo, m.Problems[0]) + } + repoDir := RepoDir(hub, m.Repo) + snap := filepath.Join(repoDir, "snapshots", m.Revision) + for _, d := range []string{filepath.Join(repoDir, "blobs"), snap} { + if err := os.MkdirAll(d, 0o755); err != nil { + return res, err + } + } + for _, f := range m.Files { + dst := filepath.Join(snap, filepath.FromSlash(f.Path)) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return res, err + } + var copied bool + var err error + if links { + copied, err = placeBlobLink(repoDir, dst, f) + } else { + copied, err = placeFile(repoDir, dst, f) + } + if err != nil { + return res, fmt.Errorf("%s: %w", f.Path, err) + } + if copied { + res.Copied++ + res.Bytes += f.Size + } else { + res.Skipped++ + } + if progress != nil { + progress(f.Path, f.Size, !copied) + } + } + kept, err := writeRef(repoDir, m.Revision) + if err != nil { + return res, err + } + res.RefKept = kept + return res, writeTree(repoDir, m.Revision, m.tree) +} + +// placeFile puts f at dst as a regular file. +func placeFile(repoDir, dst string, f File) (copied bool, err error) { + if _, err := os.Lstat(dst); err == nil { + if sizeIs(dst, f.Size) { + return false, nil + } + return false, fmt.Errorf("%s exists with different content", dst) + } + tmp := partialPath(repoDir, f.Path) + if _, err := copyFile(f.Abs, tmp, f.Etag, f.Size); err != nil { + return false, err + } + return true, os.Rename(tmp, dst) +} + +// placeBlobLink puts f's content in blobs/ and links dst to it. +func placeBlobLink(repoDir, dst string, f File) (copied bool, err error) { + name := f.Etag + if name == "" || !sizeIs(filepath.Join(repoDir, "blobs", name), f.Size) { + tmp := partialPath(repoDir, f.Path) + got, err := copyFile(f.Abs, tmp, f.Etag, f.Size) + if err != nil { + return false, err + } + name = got + blob := filepath.Join(repoDir, "blobs", name) + switch _, err := os.Lstat(blob); { + case err != nil: + if err := os.Rename(tmp, blob); err != nil { + return false, err + } + copied = true + case sizeIs(blob, f.Size): + os.Remove(tmp) + default: + os.Remove(tmp) + return false, fmt.Errorf("blob %s exists with a different size", name) + } + } + if _, err := os.Lstat(dst); err == nil { + if sizeIs(dst, f.Size) { + return copied, nil + } + return false, fmt.Errorf("%s exists with different content", dst) + } + up := strings.Repeat("../", 2+strings.Count(f.Path, "/")) + return copied, symlink(up+"blobs/"+name, dst) +} + +// writeRef makes refs/main name rev when the cache has none. A ref that names +// another revision is left as it is, and returned. +func writeRef(repoDir, rev string) (kept string, err error) { + p := filepath.Join(repoDir, "refs", "main") + if b, err := os.ReadFile(p); err == nil { + if have := strings.TrimSpace(string(b)); have != rev { + return have, nil + } + return "", nil + } + return "", writeAtomic(p, []byte(rev)) +} + +// writeTree records the source's listing for rev, which names exactly the +// files that were copied. An existing listing is kept. +func writeTree(repoDir, rev string, raw []byte) error { + if len(raw) == 0 { + return nil + } + p := filepath.Join(repoDir, "trees", rev+".json") + if _, err := os.Stat(p); err == nil { + return nil + } + return writeAtomic(p, raw) +} + +func writeAtomic(p string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + return err + } + tmp := p + partialSuffix + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, p) +} + +// Import copies a complete model into the hub cache at hub, so a server finds +// it as if it had been downloaded. Files already there are skipped, which +// lets an interrupted import resume. Every file is hashed as it is copied and +// checked against its recorded hash. +func Import(m Model, hub string, progress Progress) (Result, error) { + if err := os.MkdirAll(hub, 0o755); err != nil { + return Result{}, err + } + return place(m, hub, canLink(hub), progress) +} + +// Export copies a complete model onto a drive as a cache with no links, which +// every filesystem a drive is likely to have can hold, and which is itself a +// cache that can be read offline. +func Export(m Model, hub string, progress Progress) (Result, error) { + if err := os.MkdirAll(hub, 0o755); err != nil { + return Result{}, err + } + return place(m, hub, false, progress) +} + +// ErrNoSpace is wrapped by the error a copy that would not fit returns. +var ErrNoSpace = errors.New("not enough free space") + +// Need is the bytes copying the models into hub would write. It counts a file +// as already there when it is, either as a blob or in the snapshot. +func Need(models []Model, hub string) int64 { + var n int64 + for _, m := range models { + snap := filepath.Join(RepoDir(hub, m.Repo), "snapshots", m.Revision) + for _, f := range m.Files { + if sizeIs(filepath.Join(snap, filepath.FromSlash(f.Path)), f.Size) { + continue + } + if f.Etag != "" && sizeIs(filepath.Join(RepoDir(hub, m.Repo), "blobs", f.Etag), f.Size) { + continue + } + n += f.Size + } + } + return n +} + +// Free is the bytes free where dir is, or would be created. ok is false when +// that cannot be told. +func Free(dir string) (free int64, ok bool) { + probe := dir + for { + if _, err := os.Stat(probe); err == nil { + break + } + next := filepath.Dir(probe) + if next == probe { + return 0, false + } + probe = next + } + free, err := freeBytes(probe) + return free, err == nil +} + +// CheckSpace fails when need bytes do not fit where dir is, or will be +// created. +func CheckSpace(dir string, need int64) error { + free, ok := Free(dir) + if !ok || need <= free { + return nil + } + return fmt.Errorf("%w on %s: need %s, %s free", ErrNoSpace, dir, FormatBytes(need), FormatBytes(free)) +} + +// isFAT is onFAT, so tests can name a drive FAT32. +var isFAT = onFAT + +// fatFileLimit is the largest file FAT32 holds. +const fatFileLimit = 4<<30 - 1 + +// CheckDrive fails before an export starts when the models cannot be written +// to the drive at dir: FAT32 cannot hold a file over 4 GiB, and the rest must +// fit in the free space. +func CheckDrive(models []Model, dir string) error { + if isFAT(dir) { + for _, m := range models { + for _, f := range m.Files { + if f.Size > fatFileLimit { + return fmt.Errorf("%s is %s, and %s is formatted as FAT32, which cannot hold a file over 4 GiB: reformat the drive as exFAT (or use ext4 or APFS)", f.Path, FormatBytes(f.Size), dir) + } + } + } + } + return CheckSpace(dir, Need(models, filepath.Join(dir, "hub"))) +} + +// FormatBytes writes n the way a person reads a disk: 14.5 GB. +func FormatBytes(n int64) string { + const unit = 1000 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for m := n / unit; m >= unit; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(n)/float64(div), "kMGTPE"[exp]) +} diff --git a/internal/hfcache/hfcache.go b/internal/hfcache/hfcache.go new file mode 100644 index 0000000..d417dda --- /dev/null +++ b/internal/hfcache/hfcache.go @@ -0,0 +1,139 @@ +// Package hfcache locates, reads and writes Hugging Face hub caches, so models +// can move between machines on a drive without a download. +// +// It speaks the layout huggingface_hub, vLLM, mlx-lm and llama.cpp share: +// models--org--name/{blobs, snapshots/, refs/main, trees/.json}. +package hfcache + +import ( + "os" + "path/filepath" + "regexp" + "strings" +) + +// Env looks up one environment variable, "" when unset. +type Env func(key string) string + +// OSEnv reads the process environment. +func OSEnv(key string) string { return os.Getenv(key) } + +// Dir is the hub cache directory a server reads, given the environment it +// starts with. The first of extra that is set wins, then HF_HUB_CACHE, +// HUGGINGFACE_HUB_CACHE, $HF_HOME/hub and $XDG_CACHE_HOME/huggingface/hub, and +// last ~/.cache/huggingface/hub under home. +func Dir(env Env, home string, extra ...string) string { + get := func(k string) string { return expandHome(strings.TrimSpace(env(k)), home) } + for _, k := range extra { + if v := get(k); v != "" { + return v + } + } + for _, k := range []string{"HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE"} { + if v := get(k); v != "" { + return v + } + } + if v := get("HF_HOME"); v != "" { + return filepath.Join(v, "hub") + } + if v := get("XDG_CACHE_HOME"); v != "" { + return filepath.Join(v, "huggingface", "hub") + } + return filepath.Join(home, ".cache", "huggingface", "hub") +} + +// expandHome resolves a leading ~ and $HOME, which a sourced env file would +// have expanded. +func expandHome(v, home string) string { + switch { + case v == "~": + return home + case strings.HasPrefix(v, "~/"): + return filepath.Join(home, v[2:]) + } + v = strings.ReplaceAll(v, "${HOME}", home) + return strings.ReplaceAll(v, "$HOME", home) +} + +// Offline reports whether HF_HUB_OFFLINE or TRANSFORMERS_OFFLINE is set to a +// true value, as huggingface_hub reads them. +func Offline(env Env) bool { + for _, k := range []string{"HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"} { + switch strings.ToUpper(strings.TrimSpace(env(k))) { + case "1", "ON", "YES", "TRUE": + return true + } + } + return false +} + +var ( + repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) + revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + hexPattern = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`) +) + +const repoPrefix = "models--" + +// ValidRepo reports whether repo is an org/name Hub id. +func ValidRepo(repo string) bool { + return repoPattern.MatchString(repo) && !strings.Contains(repo, "--") && strings.Trim(repo[strings.IndexByte(repo, '/')+1:], ".") != "" +} + +// RepoDir is where a repo lives under a hub cache. +func RepoDir(hub, repo string) string { + return filepath.Join(hub, repoPrefix+strings.ReplaceAll(repo, "/", "--")) +} + +// repoFromDir is the repo a models--org--name folder holds. +func repoFromDir(name string) (string, bool) { + rest, ok := strings.CutPrefix(name, repoPrefix) + if !ok { + return "", false + } + org, model, ok := strings.Cut(rest, "--") + if !ok { + return "", false + } + repo := org + "/" + model + return repo, ValidRepo(repo) +} + +// Snapshot is the directory holding repo's refs/main revision in a hub cache, +// the one an offline load reads. +func Snapshot(hub, repo string) (dir, revision string, ok bool) { + rev, err := mainRevision(RepoDir(hub, repo)) + if err != nil { + return "", "", false + } + dir = filepath.Join(RepoDir(hub, repo), "snapshots", rev) + if st, err := os.Stat(dir); err != nil || !st.IsDir() { + return "", "", false + } + return dir, rev, true +} + +// mainRevision reads refs/main, or names the one snapshot a repo has when it +// has no ref. +func mainRevision(repoDir string) (string, error) { + if b, err := os.ReadFile(filepath.Join(repoDir, "refs", "main")); err == nil { + if rev := strings.TrimSpace(string(b)); revisionPattern.MatchString(rev) { + return rev, nil + } + } + entries, err := os.ReadDir(filepath.Join(repoDir, "snapshots")) + if err != nil { + return "", err + } + var revs []string + for _, e := range entries { + if e.IsDir() && revisionPattern.MatchString(e.Name()) { + revs = append(revs, e.Name()) + } + } + if len(revs) != 1 { + return "", os.ErrNotExist + } + return revs[0], nil +} diff --git a/internal/hfcache/hfcache_internal_test.go b/internal/hfcache/hfcache_internal_test.go new file mode 100644 index 0000000..3adf443 --- /dev/null +++ b/internal/hfcache/hfcache_internal_test.go @@ -0,0 +1,731 @@ +package hfcache + +import ( + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +const rev = "0123456789abcdef0123456789abcdef01234567" + +func gitID(content string) string { + h := sha1.New() + fmt.Fprintf(h, "blob %d\x00%s", len(content), content) + return hex.EncodeToString(h.Sum(nil)) +} + +func sha(content string) string { + s := sha256.Sum256([]byte(content)) + return hex.EncodeToString(s[:]) +} + +func write(t *testing.T, p, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func read(t *testing.T, p string) string { + t.Helper() + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// cacheOpts shape a cache fixture. +type cacheOpts struct { + // noLinks writes real files in the snapshot and no blobs, as a cache on + // exFAT does. + noLinks bool + // tree writes trees/.json. + tree bool + // lfs names the files stored by sha256 rather than git blob id. + lfs []string + // shared puts blob content in the cache-wide store, as newer caches do. + shared bool +} + +var modelFiles = map[string]string{ + "config.json": `{"model_type":"toy"}`, + "model.safetensors.index.json": `{"weight_map":{"a":"model-1.safetensors","b":"sub/model-2.safetensors"}}`, + "model-1.safetensors": "weights one", + "sub/model-2.safetensors": "weights two", +} + +// makeCache writes a hub cache holding repo, and returns the hub directory. +func makeCache(t *testing.T, repo string, files map[string]string, o cacheOpts) string { + t.Helper() + hub := filepath.Join(t.TempDir(), "hub") + repoDir := RepoDir(hub, repo) + tree := map[string]treeFile{} + for _, name := range sortedKeys(files) { + content := files[name] + etag, tf := gitID(content), treeFile{Size: int64(len(content)), BlobID: gitID(content)} + if slices.Contains(o.lfs, name) { + etag = sha(content) + tf.LFSSha256 = etag + tf.BlobID = gitID("pointer " + name) + } + tree[name] = tf + dst := filepath.Join(repoDir, "snapshots", rev, filepath.FromSlash(name)) + if o.noLinks { + write(t, dst, content) + continue + } + blob := filepath.Join(repoDir, "blobs", etag) + if o.shared { + store := filepath.Join(hub, "blobs", etag[:2], etag) + write(t, store, content) + if err := os.MkdirAll(filepath.Dir(blob), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join("..", "..", "blobs", etag[:2], etag), blob); err != nil { + t.Fatal(err) + } + } else { + write(t, blob, content) + } + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + t.Fatal(err) + } + up := strings.Repeat("../", 2+strings.Count(name, "/")) + if err := os.Symlink(up+"blobs/"+etag, dst); err != nil { + t.Fatal(err) + } + } + write(t, filepath.Join(repoDir, "refs", "main"), rev) + if o.tree { + raw, _ := json.Marshal(map[string]any{"format_version": 1, "files": tree}) + write(t, filepath.Join(repoDir, "trees", rev+".json"), string(raw)) + } + return hub +} + +func scanOne(t *testing.T, hub, repo string) Model { + t.Helper() + m, err := ScanRepo(hub, repo) + if err != nil { + t.Fatal(err) + } + return m +} + +func TestDirPrecedence(t *testing.T) { + t.Parallel() + home := "/home/u" + for _, tc := range []struct { + name string + env map[string]string + extra []string + want string + }{ + {"default", nil, nil, "/home/u/.cache/huggingface/hub"}, + {"xdg", map[string]string{"XDG_CACHE_HOME": "/xdg"}, nil, "/xdg/huggingface/hub"}, + {"hf home beats xdg", map[string]string{"XDG_CACHE_HOME": "/xdg", "HF_HOME": "/hf"}, nil, "/hf/hub"}, + {"old hub cache beats hf home", map[string]string{"HF_HOME": "/hf", "HUGGINGFACE_HUB_CACHE": "/old"}, nil, "/old"}, + {"hub cache beats old", map[string]string{"HUGGINGFACE_HUB_CACHE": "/old", "HF_HUB_CACHE": "/new"}, nil, "/new"}, + {"server variable beats all", map[string]string{"HF_HUB_CACHE": "/new", "LLAMA_CACHE": "/llama"}, []string{"LLAMA_CACHE"}, "/llama"}, + {"server variable unset", map[string]string{"HF_HUB_CACHE": "/new"}, []string{"LLAMA_CACHE"}, "/new"}, + {"tilde", map[string]string{"HF_HOME": "~/models"}, nil, "/home/u/models/hub"}, + {"$HOME", map[string]string{"HF_HUB_CACHE": "$HOME/hub"}, nil, "/home/u/hub"}, + {"blank is unset", map[string]string{"HF_HUB_CACHE": " ", "HF_HOME": "/hf"}, nil, "/hf/hub"}, + } { + got := Dir(func(k string) string { return tc.env[k] }, home, tc.extra...) + if got != tc.want { + t.Errorf("%s: Dir = %q, want %q", tc.name, got, tc.want) + } + } +} + +func TestOffline(t *testing.T) { + t.Parallel() + for v, want := range map[string]bool{"1": true, "true": true, "TRUE": true, "on": true, "yes": true, "0": false, "": false, "false": false, "no": false} { + for _, key := range []string{"HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"} { + if got := Offline(func(k string) string { + if k == key { + return v + } + return "" + }); got != want { + t.Errorf("%s=%q: Offline = %v, want %v", key, v, got, want) + } + } + } +} + +func TestScanReadsALinkedCache(t *testing.T) { + t.Parallel() + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{tree: true, lfs: []string{"model-1.safetensors"}}) + m := scanOne(t, hub, "org/toy") + if !m.Complete() || m.Revision != rev || len(m.Files) != 4 { + t.Fatalf("model = %+v", m) + } + for _, f := range m.Files { + want := gitID(modelFiles[f.Path]) + if f.Path == "model-1.safetensors" { + want = sha(modelFiles[f.Path]) + } + if f.Etag != want { + t.Errorf("%s etag = %s, want %s", f.Path, f.Etag, want) + } + } + if m.Size() != int64(len(modelFiles["config.json"])+len(modelFiles["model.safetensors.index.json"])+len("weights one")+len("weights two")) { + t.Errorf("Size = %d", m.Size()) + } +} + +func TestScanReadsACacheWithoutLinks(t *testing.T) { + t.Parallel() + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{noLinks: true, tree: true, lfs: []string{"model-1.safetensors"}}) + m := scanOne(t, hub, "org/toy") + if !m.Complete() { + t.Fatalf("problems: %v", m.Problems) + } + // With no blob names to read, the listing says what each file is called. + for _, f := range m.Files { + if f.Path == "model-1.safetensors" && f.Etag != sha("weights one") { + t.Errorf("etag = %q, want the listing's lfs_sha256", f.Etag) + } + if f.Etag == "" { + t.Errorf("%s has no etag", f.Path) + } + } + hub = makeCache(t, "org/toy", modelFiles, cacheOpts{noLinks: true}) + for _, f := range scanOne(t, hub, "org/toy").Files { + if f.Etag != "" { + t.Errorf("%s: etag %q with nothing to say it", f.Path, f.Etag) + } + } +} + +func TestScanFindsTheCacheWhereverItSits(t *testing.T) { + t.Parallel() + for _, sub := range []string{"", "hub", "huggingface/hub", ".cache/huggingface/hub"} { + root := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + if sub != "" { + root = t.TempDir() + dst := filepath.Join(root, filepath.FromSlash(sub)) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Rename(makeCache(t, "org/toy", modelFiles, cacheOpts{}), dst); err != nil { + t.Fatal(err) + } + } + found, err := Scan(root, ScanOptions{}) + if err != nil || found.Kind != KindCache || len(found.Models) != 1 || found.Models[0].Repo != "org/toy" { + t.Errorf("%q: Scan = %+v, %v", sub, found, err) + } + } + // One copied models--org--name folder is enough. + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + found, err := Scan(RepoDir(hub, "org/toy"), ScanOptions{}) + if err != nil || len(found.Models) != 1 { + t.Errorf("single repo folder: %+v, %v", found, err) + } +} + +func TestScanSkipsWhatIsNotAModel(t *testing.T) { + t.Parallel() + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + for _, junk := range []string{"datasets--org--data", ".locks", "models--bad", "CACHEDIR.TAG"} { + write(t, filepath.Join(hub, junk, "x"), "x") + } + write(t, filepath.Join(RepoDir(hub, "org/toy"), "blobs", "abc.incomplete"), "partial") + models, err := ScanCache(hub) + if err != nil || len(models) != 1 || !models[0].Complete() { + t.Fatalf("ScanCache = %+v, %v", models, err) + } +} + +func TestScanFollowsTheSharedBlobStore(t *testing.T) { + t.Parallel() + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{shared: true}) + m := scanOne(t, hub, "org/toy") + if !m.Complete() { + t.Fatalf("problems: %v", m.Problems) + } + // The name is the repo's blob link, not the store's path. + for _, f := range m.Files { + if f.Etag != gitID(modelFiles[f.Path]) { + t.Errorf("%s etag = %s", f.Path, f.Etag) + } + } + drive := t.TempDir() + if _, err := Export(m, filepath.Join(drive, "hub"), nil); err != nil { + t.Fatal(err) + } + p := filepath.Join(RepoDir(filepath.Join(drive, "hub"), "org/toy"), "snapshots", rev, "model-1.safetensors") + if st, err := os.Lstat(p); err != nil || !st.Mode().IsRegular() || read(t, p) != "weights one" { + t.Errorf("exported file: %v, %v", st, err) + } +} + +func TestScanFlagsIncompleteModels(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + mutate func(t *testing.T, hub string) + want string + }{ + {"dangling link", func(t *testing.T, hub string) { + os.Remove(filepath.Join(RepoDir(hub, "org/toy"), "blobs", gitID("weights two"))) + }, "dangling link: sub/model-2.safetensors"}, + {"a file the listing names is missing", func(t *testing.T, hub string) { + os.Remove(filepath.Join(RepoDir(hub, "org/toy"), "snapshots", rev, "config.json")) + }, "missing config.json"}, + {"size differs from the listing", func(t *testing.T, hub string) { + write(t, filepath.Join(RepoDir(hub, "org/toy"), "blobs", gitID("weights one")), "short") + }, "the listing says"}, + {"a shard the index names is missing", func(t *testing.T, hub string) { + os.Remove(filepath.Join(RepoDir(hub, "org/toy"), "snapshots", rev, "model-1.safetensors")) + }, "missing model-1.safetensors"}, + } { + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{tree: tc.name != "a shard the index names is missing"}) + tc.mutate(t, hub) + m := scanOne(t, hub, "org/toy") + if m.Complete() || !strings.Contains(strings.Join(m.Problems, "; "), tc.want) { + t.Errorf("%s: problems = %v, want %q", tc.name, m.Problems, tc.want) + } + } + // Config only is what an interrupted first download leaves. + hub := makeCache(t, "org/toy", map[string]string{"config.json": "{}"}, cacheOpts{}) + if m := scanOne(t, hub, "org/toy"); m.Complete() || m.Problems[0] != "no weight files" { + t.Errorf("config only: %v", m.Problems) + } + // Two snapshots and no ref leave nothing to choose. + hub = makeCache(t, "org/toy", modelFiles, cacheOpts{}) + os.Remove(filepath.Join(RepoDir(hub, "org/toy"), "refs", "main")) + if err := os.MkdirAll(filepath.Join(RepoDir(hub, "org/toy"), "snapshots", strings.Repeat("b", 40)), 0o755); err != nil { + t.Fatal(err) + } + if m := scanOne(t, hub, "org/toy"); m.Complete() { + t.Error("a repo with two snapshots and no refs/main was complete") + } +} + +func TestImportWritesTheCacheHuggingFaceReads(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{noLinks: true, tree: true, lfs: []string{"model-1.safetensors"}}) + dst := filepath.Join(t.TempDir(), "hub") + var seen []string + res, err := Import(scanOne(t, src, "org/toy"), dst, func(p string, _ int64, skipped bool) { + seen = append(seen, fmt.Sprint(p, skipped)) + }) + if err != nil || !res.Links || res.Copied != 4 || res.Skipped != 0 || len(seen) != 4 { + t.Fatalf("Import = %+v, %v, %v", res, err, seen) + } + repoDir := RepoDir(dst, "org/toy") + for name, content := range modelFiles { + p := filepath.Join(repoDir, "snapshots", rev, filepath.FromSlash(name)) + target, err := os.Readlink(p) + if err != nil { + t.Fatalf("%s is not a link: %v", name, err) + } + want := gitID(content) + if name == "model-1.safetensors" { + want = sha(content) + } + wantTarget := strings.Repeat("../", 2+strings.Count(name, "/")) + "blobs/" + want + if target != wantTarget || filepath.IsAbs(target) { + t.Errorf("%s -> %s, want %s", name, target, wantTarget) + } + if read(t, p) != content { + t.Errorf("%s content differs", name) + } + } + if got := read(t, filepath.Join(repoDir, "refs", "main")); got != rev { + t.Errorf("refs/main = %q, want the bare commit", got) + } + if read(t, filepath.Join(repoDir, "trees", rev+".json")) != read(t, filepath.Join(RepoDir(src, "org/toy"), "trees", rev+".json")) { + t.Error("the tree listing was not carried over") + } + if m := scanOne(t, dst, "org/toy"); !m.Complete() { + t.Errorf("what was imported does not scan complete: %v", m.Problems) + } + entries, _ := os.ReadDir(filepath.Join(repoDir, "blobs")) + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".incomplete") { + t.Errorf("left %s behind", e.Name()) + } + } +} + +func TestImportResumes(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + m := scanOne(t, src, "org/toy") + dst := filepath.Join(t.TempDir(), "hub") + if _, err := Import(m, dst, nil); err != nil { + t.Fatal(err) + } + res, err := Import(m, dst, nil) + if err != nil || res.Copied != 0 || res.Skipped != 4 { + t.Errorf("second import = %+v, %v; want everything skipped", res, err) + } + + // An interrupted import: one blob and its link gone, a partial left. + repoDir := RepoDir(dst, "org/toy") + os.Remove(filepath.Join(repoDir, "snapshots", rev, "model-1.safetensors")) + os.Remove(filepath.Join(repoDir, "blobs", gitID("weights one"))) + write(t, partialPath(repoDir, "model-1.safetensors"), "half") + res, err = Import(m, dst, nil) + if err != nil || res.Copied != 1 || res.Skipped != 3 { + t.Errorf("resumed import = %+v, %v; want one file copied", res, err) + } + if read(t, filepath.Join(repoDir, "snapshots", rev, "model-1.safetensors")) != "weights one" { + t.Error("the resumed file is wrong") + } +} + +func TestImportCatchesADamagedSource(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + // Same length, same name, different bytes. + write(t, filepath.Join(RepoDir(src, "org/toy"), "blobs", gitID("weights two")), "weights twX") + dst := filepath.Join(t.TempDir(), "hub") + _, err := Import(scanOne(t, src, "org/toy"), dst, nil) + var corrupt *CorruptError + if !errors.As(err, &corrupt) { + t.Fatalf("Import = %v, want a CorruptError", err) + } + repoDir := RepoDir(dst, "org/toy") + if _, err := os.Stat(filepath.Join(repoDir, "blobs", gitID("weights twX"))); err == nil { + t.Error("the damaged file was kept") + } + if _, err := os.Lstat(filepath.Join(repoDir, "snapshots", rev, "sub", "model-2.safetensors")); err == nil { + t.Error("a link to the damaged file was made") + } + if _, err := os.Stat(filepath.Join(repoDir, "refs", "main")); err == nil { + t.Error("refs/main was written for a model that did not finish") + } +} + +func TestImportFallsBackToFilesWhenLinksFail(t *testing.T) { + src := makeCache(t, "org/toy", modelFiles, cacheOpts{tree: true}) + symlink = func(string, string) error { return errors.New("operation not permitted") } + t.Cleanup(func() { symlink = os.Symlink }) + dst := filepath.Join(t.TempDir(), "hub") + res, err := Import(scanOne(t, src, "org/toy"), dst, nil) + if err != nil || res.Links { + t.Fatalf("Import = %+v, %v", res, err) + } + p := filepath.Join(RepoDir(dst, "org/toy"), "snapshots", rev, "sub", "model-2.safetensors") + if st, err := os.Lstat(p); err != nil || !st.Mode().IsRegular() || read(t, p) != "weights two" { + t.Errorf("snapshot file: %v, %v", st, err) + } + if m := scanOne(t, dst, "org/toy"); !m.Complete() { + t.Errorf("problems: %v", m.Problems) + } +} + +func TestImportRefusesIncompleteModels(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + os.Remove(filepath.Join(RepoDir(src, "org/toy"), "snapshots", rev, "model-1.safetensors")) + dst := filepath.Join(t.TempDir(), "hub") + if _, err := Import(scanOne(t, src, "org/toy"), dst, nil); err == nil || !strings.Contains(err.Error(), "incomplete") { + t.Errorf("Import = %v", err) + } + if _, err := os.Stat(RepoDir(dst, "org/toy")); err == nil { + t.Error("an incomplete model left files behind") + } +} + +func TestImportKeepsWhatIsThere(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{tree: true}) + dst := filepath.Join(t.TempDir(), "hub") + repoDir := RepoDir(dst, "org/toy") + other := strings.Repeat("c", 40) + write(t, filepath.Join(repoDir, "refs", "main"), other) + write(t, filepath.Join(repoDir, "trees", rev+".json"), `{"format_version":1,"files":{}}`) + res, err := Import(scanOne(t, src, "org/toy"), dst, nil) + if err != nil || res.RefKept != other { + t.Fatalf("Import = %+v, %v; want refs/main kept at %s", res, err, other) + } + if read(t, filepath.Join(repoDir, "refs", "main")) != other { + t.Error("refs/main was overwritten") + } + if !strings.Contains(read(t, filepath.Join(repoDir, "trees", rev+".json")), `"files":{}`) { + t.Error("an existing tree listing was overwritten") + } + + // A blob of the right name and the wrong size is not ours to replace. + dst2 := filepath.Join(t.TempDir(), "hub") + write(t, filepath.Join(RepoDir(dst2, "org/toy"), "blobs", gitID("weights two")), "not what its name says") + if _, err := Import(scanOne(t, src, "org/toy"), dst2, nil); err == nil || !strings.Contains(err.Error(), "different size") { + t.Errorf("Import over a bad blob = %v", err) + } +} + +func TestExportWritesAReadableCacheWithoutLinks(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{tree: true}) + drive := filepath.Join(t.TempDir(), "hub") + if _, err := Export(scanOne(t, src, "org/toy"), drive, nil); err != nil { + t.Fatal(err) + } + filepath.WalkDir(drive, func(p string, d os.DirEntry, err error) error { + if err == nil && d.Type()&os.ModeSymlink != 0 { + t.Errorf("%s is a link", p) + } + return nil + }) + m := scanOne(t, drive, "org/toy") + if !m.Complete() || len(m.Files) != 4 || m.Revision != rev { + t.Fatalf("exported model = %+v", m) + } + if read(t, filepath.Join(RepoDir(drive, "org/toy"), "refs", "main")) != rev { + t.Error("refs/main") + } + + // And it imports into a linked cache, files verified against the tree. + dst := filepath.Join(t.TempDir(), "hub") + res, err := Import(m, dst, nil) + if err != nil || !res.Links || res.Copied != 4 { + t.Fatalf("Import = %+v, %v", res, err) + } + if _, err := os.Stat(filepath.Join(RepoDir(dst, "org/toy"), "blobs", gitID("weights two"))); err != nil { + t.Error("blobs are not named by the etag the listing gave") + } +} + +func TestExportRefusesIncompleteModels(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + os.Remove(filepath.Join(RepoDir(src, "org/toy"), "blobs", gitID("weights one"))) + drive := filepath.Join(t.TempDir(), "hub") + if _, err := Export(scanOne(t, src, "org/toy"), drive, nil); err == nil { + t.Error("an incomplete model was exported") + } +} + +func TestExportCatchesADamagedCache(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + write(t, filepath.Join(RepoDir(src, "org/toy"), "blobs", gitID("weights one")), "weights onX") + _, err := Export(scanOne(t, src, "org/toy"), filepath.Join(t.TempDir(), "hub"), nil) + var corrupt *CorruptError + if !errors.As(err, &corrupt) { + t.Errorf("Export = %v, want a CorruptError", err) + } +} + +func TestCheckSpaceRefusesWhatWillNotFit(t *testing.T) { + prev := freeBytes + freeBytes = func(string) (int64, error) { return 100, nil } + t.Cleanup(func() { freeBytes = prev }) + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + m := scanOne(t, src, "org/toy") + dst := filepath.Join(t.TempDir(), "not", "yet", "hub") + need := Need([]Model{m}, dst) + if need != m.Size() { + t.Fatalf("Need = %d, want %d for an empty cache", need, m.Size()) + } + if err := CheckSpace(dst, 101); !errors.Is(err, ErrNoSpace) { + t.Errorf("CheckSpace(101 of 100) = %v", err) + } + if err := CheckSpace(dst, 100); err != nil { + t.Errorf("CheckSpace(100 of 100) = %v", err) + } + // What an import already holds is not counted again. + if _, err := Import(m, dst, nil); err != nil { + t.Fatal(err) + } + if n := Need([]Model{m}, dst); n != 0 { + t.Errorf("Need after importing = %d, want 0", n) + } + freeBytes = func(string) (int64, error) { return 0, errors.New("unknown") } + if err := CheckSpace(dst, 1<<40); err != nil { + t.Errorf("an unreadable filesystem was refused: %v", err) + } +} + +func TestCheckDriveRefusesBigFilesOnFAT(t *testing.T) { + prevFAT, prevFree := isFAT, freeBytes + isFAT = func(string) bool { return true } + freeBytes = func(string) (int64, error) { return 1 << 50, nil } + t.Cleanup(func() { isFAT, freeBytes = prevFAT, prevFree }) + big := Model{Repo: "org/toy", Files: []File{{Path: "model.safetensors", Size: 5 << 30}}} + err := CheckDrive([]Model{big}, t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "FAT32") || !strings.Contains(err.Error(), "model.safetensors") { + t.Errorf("CheckDrive = %v", err) + } + small := Model{Repo: "org/toy", Files: []File{{Path: "model.safetensors", Size: 3 << 30}}} + if err := CheckDrive([]Model{small}, t.TempDir()); err != nil { + t.Errorf("files under 4 GiB on FAT: %v", err) + } + isFAT = func(string) bool { return false } + if err := CheckDrive([]Model{big}, t.TempDir()); err != nil { + t.Errorf("a big file on exFAT: %v", err) + } +} + +func writeLocalDir(t *testing.T, commits ...string) string { + t.Helper() + dir := t.TempDir() + for i, name := range []string{"config.json", "model.safetensors"} { + content := modelFiles[name] + if name == "model.safetensors" { + content = "local weights" + } + write(t, filepath.Join(dir, name), content) + etag := gitID(content) + commit := commits[0] + if i < len(commits) { + commit = commits[i] + } + write(t, filepath.Join(dir, ".cache", "huggingface", "download", name+".metadata"), commit+"\n"+etag+"\n1700000000.0\n") + } + return dir +} + +func TestScanReadsAnHFLocalDir(t *testing.T) { + t.Parallel() + dir := writeLocalDir(t, rev) + if _, err := Scan(dir, ScanOptions{}); err == nil || !strings.Contains(err.Error(), "--repo") { + t.Errorf("without --repo: %v", err) + } + found, err := Scan(dir, ScanOptions{Repo: "org/toy"}) + if err != nil || found.Kind != KindLocalDir || len(found.Models) != 1 { + t.Fatalf("Scan = %+v, %v", found, err) + } + m := found.Models[0] + if m.Revision != rev || !m.Complete() || len(m.Files) != 2 { + t.Fatalf("model = %+v", m) + } + for _, f := range m.Files { + if f.Path == "model.safetensors" && f.Etag != gitID("local weights") { + t.Errorf("etag = %q, want the metadata's", f.Etag) + } + } + + other := strings.Repeat("d", 40) + if _, err := Scan(writeLocalDir(t, rev, other), ScanOptions{Repo: "org/toy"}); err == nil || !strings.Contains(err.Error(), "different commits") { + t.Errorf("mixed commits: %v", err) + } + if _, err := Scan(dir, ScanOptions{Repo: "org/toy", Revision: other}); err == nil || !strings.Contains(err.Error(), "--revision") { + t.Errorf("a --revision that disagrees: %v", err) + } + write(t, filepath.Join(dir, ".cache", "huggingface", "download", "model.safetensors.abc.incomplete"), "x") + found, _ = Scan(dir, ScanOptions{Repo: "org/toy"}) + if found.Models[0].Complete() { + t.Error("a partial download was complete") + } +} + +func TestScanReadsAPlainFolder(t *testing.T) { + t.Parallel() + dir := t.TempDir() + write(t, filepath.Join(dir, "config.json"), "{}") + write(t, filepath.Join(dir, "model.safetensors"), "w") + if _, err := Scan(dir, ScanOptions{}); err == nil || !strings.Contains(err.Error(), "--repo") { + t.Errorf("without --repo: %v", err) + } + if _, err := Scan(dir, ScanOptions{Repo: "org/toy"}); err == nil || !strings.Contains(err.Error(), "--revision") { + t.Errorf("without a commit it must not invent one: %v", err) + } + if _, err := Scan(dir, ScanOptions{Repo: "org/toy", Revision: "main"}); err == nil { + t.Error("a branch name was taken for a commit") + } + if _, err := Scan(dir, ScanOptions{Repo: "toy"}); err == nil { + t.Error("a repo with no org was accepted") + } + found, err := Scan(dir, ScanOptions{Repo: "org/toy", Revision: rev}) + if err != nil || found.Kind != KindFolder || found.Models[0].Revision != rev || len(found.Models[0].Files) != 2 { + t.Fatalf("Scan = %+v, %v", found, err) + } + // Blobs of an unrecorded etag are named as the Hub names them. + dst := filepath.Join(t.TempDir(), "hub") + if _, err := Import(found.Models[0], dst, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(RepoDir(dst, "org/toy"), "blobs", gitID("w"))); err != nil { + t.Errorf("a small file is not named by its git blob id: %v", err) + } +} + +func TestScanTakesAGitCloneRevisionFromHEAD(t *testing.T) { + t.Parallel() + for name, setup := range map[string]func(t *testing.T, dir string){ + "detached": func(t *testing.T, dir string) { write(t, filepath.Join(dir, ".git", "HEAD"), rev+"\n") }, + "branch": func(t *testing.T, dir string) { + write(t, filepath.Join(dir, ".git", "HEAD"), "ref: refs/heads/main\n") + write(t, filepath.Join(dir, ".git", "refs", "heads", "main"), rev+"\n") + }, + "packed": func(t *testing.T, dir string) { + write(t, filepath.Join(dir, ".git", "HEAD"), "ref: refs/heads/main\n") + write(t, filepath.Join(dir, ".git", "packed-refs"), "# pack-refs\n"+rev+" refs/heads/main\n") + }, + } { + dir := t.TempDir() + write(t, filepath.Join(dir, "config.json"), "{}") + write(t, filepath.Join(dir, "model.safetensors"), "w") + write(t, filepath.Join(dir, ".git", "objects", "x"), "git internals") + setup(t, dir) + found, err := Scan(dir, ScanOptions{Repo: "org/toy"}) + if err != nil || found.Models[0].Revision != rev || len(found.Models[0].Files) != 2 { + t.Errorf("%s: %+v, %v", name, found, err) + } + } +} + +func TestScanFlagsGitLFSPointers(t *testing.T) { + t.Parallel() + dir := t.TempDir() + write(t, filepath.Join(dir, "config.json"), "{}") + write(t, filepath.Join(dir, "model.safetensors"), "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 123\n") + found, err := Scan(dir, ScanOptions{Repo: "org/toy", Revision: rev}) + if err != nil { + t.Fatal(err) + } + if m := found.Models[0]; m.Complete() || !strings.Contains(m.Problems[0], "git lfs pull") { + t.Errorf("problems = %v", m.Problems) + } +} + +func TestScanFindsNothingInAnEmptyFolder(t *testing.T) { + t.Parallel() + if _, err := Scan(t.TempDir(), ScanOptions{Repo: "org/toy"}); err == nil || !strings.Contains(err.Error(), "no models") { + t.Errorf("Scan = %v", err) + } + if _, err := Scan(filepath.Join(t.TempDir(), "missing"), ScanOptions{}); err == nil { + t.Error("a missing path was accepted") + } +} + +func TestSnapshotFollowsRefsMain(t *testing.T) { + t.Parallel() + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + dir, got, ok := Snapshot(hub, "org/toy") + if !ok || got != rev || !strings.HasSuffix(dir, filepath.Join("snapshots", rev)) { + t.Fatalf("Snapshot = %q, %q, %v", dir, got, ok) + } + if _, _, ok := Snapshot(hub, "org/other"); ok { + t.Error("a repo that is not cached has a snapshot") + } +} + +func TestFormatBytes(t *testing.T) { + t.Parallel() + for n, want := range map[int64]string{0: "0 B", 999: "999 B", 1500: "1.5 kB", 14_500_000_000: "14.5 GB"} { + if got := FormatBytes(n); got != want { + t.Errorf("FormatBytes(%d) = %q, want %q", n, got, want) + } + } +} diff --git a/internal/hfcache/scan.go b/internal/hfcache/scan.go new file mode 100644 index 0000000..4f1f203 --- /dev/null +++ b/internal/hfcache/scan.go @@ -0,0 +1,532 @@ +package hfcache + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" +) + +// Kind is the layout a source was read from. +type Kind string + +// The layouts Scan reads. +const ( + // KindCache is a hub cache: models--org--name folders. + KindCache Kind = "Hugging Face cache" + // KindLocalDir is an `hf download --local-dir` folder. + KindLocalDir Kind = "hf download --local-dir folder" + // KindFolder is a plain model folder, such as a git clone. + KindFolder Kind = "model folder" +) + +// File is one file of a model. +type File struct { + // Path is the file's path in the repo, slash-separated. + Path string + Size int64 + // Etag is the blob's name in a cache: the sha256 of an LFS file or the git + // blob hash of any other. It is "" when the source does not record it. + Etag string + // Abs is where the content is read from. + Abs string +} + +// Model is one revision of one repo found in a source. +type Model struct { + Repo string + Revision string + Files []File + // Problems say why the model cannot be copied, and are empty when it is + // complete. + Problems []string + Kind Kind + // Root is the source the model was read from. + Root string + + // tree is the source's trees/.json, copied alongside the files it + // lists. + tree []byte +} + +// Size is the total bytes of the model's files. +func (m Model) Size() int64 { + var n int64 + for _, f := range m.Files { + n += f.Size + } + return n +} + +// Complete reports whether the model has every file it needs. +func (m Model) Complete() bool { return len(m.Problems) == 0 } + +// ScanOptions name what a source cannot say about itself. +type ScanOptions struct { + // Repo and Revision identify a folder that is not a cache. + Repo string + Revision string +} + +// Found is what Scan read. +type Found struct { + Kind Kind + Models []Model +} + +// Scan reads the models at path. A Hugging Face cache is read at path itself, +// or at hub/, huggingface/hub/ or .cache/huggingface/hub/ under it, or from a +// single models--org--name folder. Otherwise path is an `hf download +// --local-dir` folder or a plain model folder, and o names its repo. +func Scan(path string, o ScanOptions) (Found, error) { + st, err := os.Stat(path) + if err != nil { + return Found{}, fmt.Errorf("read %s: %w", path, err) + } + if !st.IsDir() { + return Found{}, fmt.Errorf("%s is not a directory", path) + } + path = filepath.Clean(path) + if _, ok := repoFromDir(filepath.Base(path)); ok { + models, err := scanCache(filepath.Dir(path), filepath.Base(path)) + return Found{Kind: KindCache, Models: models}, err + } + var models []Model + seen := map[string]bool{} + for _, hub := range []string{path, filepath.Join(path, "hub"), filepath.Join(path, "huggingface", "hub"), filepath.Join(path, ".cache", "huggingface", "hub")} { + if !isHub(hub) { + continue + } + found, err := scanCache(hub, "") + if err != nil { + return Found{}, err + } + for _, m := range found { + if key := m.Repo + "@" + m.Revision; !seen[key] { + seen[key] = true + models = append(models, m) + } + } + } + if len(models) > 0 { + return Found{Kind: KindCache, Models: models}, nil + } + + if _, err := os.Stat(filepath.Join(path, ".cache", "huggingface", "download")); err == nil { + if o.Repo == "" { + return Found{}, fmt.Errorf("%s was written by `hf download --local-dir`, which does not record the repo: pass --repo org/model", path) + } + m, err := scanLocalDir(path, o) + return Found{Kind: KindLocalDir, Models: []Model{m}}, err + } + if hasFiles(path) { + if o.Repo == "" { + return Found{}, fmt.Errorf("no Hugging Face cache found at %s; to import it as a model folder, pass --repo org/model (and --revision unless it is a git clone)", path) + } + m, err := scanFolder(path, o) + return Found{Kind: KindFolder, Models: []Model{m}}, err + } + return Found{}, fmt.Errorf("found no models at %s", path) +} + +// ScanCache lists every model in a hub cache. +func ScanCache(hub string) ([]Model, error) { return scanCache(hub, "") } + +// ScanRepo reads one repo's refs/main revision from a hub cache. +func ScanRepo(hub, repo string) (Model, error) { + models, err := scanCache(hub, repoPrefix+strings.ReplaceAll(repo, "/", "--")) + if err != nil { + return Model{}, err + } + if len(models) == 0 { + return Model{}, fmt.Errorf("%s is not in the cache at %s", repo, hub) + } + return models[0], nil +} + +// Has reports whether the hub cache at hub holds any repo. +func Has(hub string) bool { return isHub(hub) } + +func isHub(dir string) bool { + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + for _, e := range entries { + if _, ok := repoFromDir(e.Name()); ok && e.IsDir() { + return true + } + } + return false +} + +func hasFiles(dir string) bool { + found := false + _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err == nil && d.IsDir() && d.Name() == ".git" { + return filepath.SkipDir + } + if err == nil && !d.IsDir() { + found = true + return filepath.SkipAll + } + return nil + }) + return found +} + +// scanCache reads the repos under hub, or only the folder named only. +func scanCache(hub, only string) ([]Model, error) { + entries, err := os.ReadDir(hub) + if err != nil { + return nil, fmt.Errorf("read %s: %w", hub, err) + } + var out []Model + for _, e := range entries { + repo, ok := repoFromDir(e.Name()) + if !ok || !e.IsDir() || (only != "" && e.Name() != only) { + continue + } + out = append(out, scanSnapshot(hub, repo)) + } + sort.Slice(out, func(i, j int) bool { return out[i].Repo < out[j].Repo }) + return out, nil +} + +// weightSuffixes are the files a model is useless without. +var weightSuffixes = []string{".safetensors", ".bin", ".gguf"} + +func scanSnapshot(hub, repo string) Model { + repoDir := RepoDir(hub, repo) + m := Model{Repo: repo, Kind: KindCache, Root: hub} + rev, err := mainRevision(repoDir) + if err != nil { + m.Problems = append(m.Problems, "no refs/main, and not exactly one snapshot to choose") + return m + } + m.Revision = rev + snap := filepath.Join(repoDir, "snapshots", rev) + if _, err := os.Stat(snap); err != nil { + m.Problems = append(m.Problems, "snapshot "+rev+" is missing") + return m + } + _ = filepath.WalkDir(snap, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + rel, _ := filepath.Rel(snap, p) + rel = filepath.ToSlash(rel) + st, err := os.Stat(p) + if err != nil { + m.Problems = append(m.Problems, "dangling link: "+rel) + return nil + } + f := File{Path: rel, Size: st.Size(), Abs: p} + if d.Type()&fs.ModeSymlink != 0 { + if target, err := os.Readlink(p); err == nil { + if name := path.Base(filepath.ToSlash(target)); hexPattern.MatchString(name) { + f.Etag = name + } + } + } + m.Files = append(m.Files, f) + return nil + }) + if raw, err := os.ReadFile(filepath.Join(repoDir, "trees", rev+".json")); err == nil { + m.tree = raw + m.applyTree(raw) + } + m.checkComplete() + return m +} + +// treeFile is one entry of trees/.json. +type treeFile struct { + Size int64 `json:"size"` + BlobID string `json:"blob_id"` + LFSSha256 string `json:"lfs_sha256"` +} + +func (t treeFile) etag() string { + if hexPattern.MatchString(t.LFSSha256) { + return t.LFSSha256 + } + if hexPattern.MatchString(t.BlobID) { + return t.BlobID + } + return "" +} + +// applyTree checks the files against the listing the cache recorded for the +// revision: huggingface_hub refuses an offline load that misses any of them. +func (m *Model) applyTree(raw []byte) { + var t struct { + Files map[string]treeFile `json:"files"` + } + if json.Unmarshal(raw, &t) != nil { + m.Problems = append(m.Problems, "trees/"+m.Revision+".json is unreadable") + return + } + have := map[string]int{} + for i, f := range m.Files { + have[f.Path] = i + } + for _, p := range sortedKeys(t.Files) { + want := t.Files[p] + i, ok := have[p] + switch { + case !ok: + m.Problems = append(m.Problems, "missing "+p) + case want.Size > 0 && m.Files[i].Size != want.Size: + m.Problems = append(m.Problems, fmt.Sprintf("%s is %d bytes, the listing says %d", p, m.Files[i].Size, want.Size)) + case m.Files[i].Etag == "": + m.Files[i].Etag = want.etag() + } + } +} + +// checkComplete adds a problem for each way the files cannot serve a model: +// none at all, no weights, or a shard an index names that is not there. +func (m *Model) checkComplete() { + if len(m.Files) == 0 { + m.Problems = append(m.Problems, "no files") + return + } + have := map[string]bool{} + weights := false + for _, f := range m.Files { + have[f.Path] = true + for _, s := range weightSuffixes { + weights = weights || strings.HasSuffix(strings.ToLower(f.Path), s) + } + } + if !weights { + m.Problems = append(m.Problems, "no weight files") + } + for _, f := range m.Files { + if !strings.HasSuffix(f.Path, ".index.json") { + continue + } + raw, err := os.ReadFile(f.Abs) + if err != nil { + continue + } + var idx struct { + WeightMap map[string]string `json:"weight_map"` + } + if json.Unmarshal(raw, &idx) != nil { + continue + } + dir := path.Dir(f.Path) + for _, name := range uniqueSorted(idx.WeightMap) { + if !have[path.Join(dir, name)] { + m.Problems = append(m.Problems, "missing "+path.Join(dir, name)+", which "+f.Path+" names") + } + } + } +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func uniqueSorted(m map[string]string) []string { + set := map[string]bool{} + for _, v := range m { + set[v] = true + } + return sortedKeys(set) +} + +// scanLocalDir reads an `hf download --local-dir` folder, whose +// .cache/huggingface/download/.metadata files record the commit and etag +// of each download: three lines, commit, etag, timestamp. +func scanLocalDir(dir string, o ScanOptions) (Model, error) { + m := Model{Repo: o.Repo, Kind: KindLocalDir, Root: dir} + if !ValidRepo(o.Repo) { + return m, fmt.Errorf("--repo %q is not an org/name Hub id", o.Repo) + } + meta := filepath.Join(dir, ".cache", "huggingface", "download") + commits := map[string]bool{} + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if p == filepath.Join(dir, ".cache") { + return filepath.SkipDir + } + return nil + } + rel, _ := filepath.Rel(dir, p) + rel = filepath.ToSlash(rel) + st, err := os.Stat(p) + if err != nil { + m.Problems = append(m.Problems, "dangling link: "+rel) + return nil + } + f := File{Path: rel, Size: st.Size(), Abs: p} + if commit, etag, ok := readMetadata(filepath.Join(meta, filepath.FromSlash(rel)+".metadata")); ok { + if revisionPattern.MatchString(commit) { + commits[commit] = true + } + if hexPattern.MatchString(etag) { + f.Etag = etag + } + } + m.Files = append(m.Files, f) + return nil + }) + if err != nil { + return m, fmt.Errorf("read %s: %w", dir, err) + } + _ = filepath.WalkDir(meta, func(p string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && strings.HasSuffix(d.Name(), ".incomplete") { + m.Problems = append(m.Problems, "partial download: "+d.Name()) + } + return nil + }) + if len(commits) > 1 { + return m, fmt.Errorf("%s holds files from %d different commits; download one revision at a time", dir, len(commits)) + } + for c := range commits { + m.Revision = c + } + if err := m.useRevision(o.Revision); err != nil { + return m, err + } + m.checkComplete() + return m, nil +} + +// useRevision settles a folder's commit, which the caller's flag may state +// and the source may also record. It never makes one up. +func (m *Model) useRevision(given string) error { + if given != "" && !revisionPattern.MatchString(given) { + return fmt.Errorf("--revision %q is not a 40-character commit hash", given) + } + switch { + case m.Revision != "" && given != "" && m.Revision != given: + return fmt.Errorf("the folder records commit %s, but --revision says %s", m.Revision, given) + case m.Revision == "": + m.Revision = given + } + if m.Revision == "" { + return errors.New("the folder does not record its commit: pass --revision <40-character commit hash>") + } + return nil +} + +func readMetadata(file string) (commit, etag string, ok bool) { + f, err := os.Open(file) + if err != nil { + return "", "", false + } + defer f.Close() + sc := bufio.NewScanner(f) + var lines []string + for sc.Scan() && len(lines) < 2 { + lines = append(lines, strings.TrimSpace(sc.Text())) + } + if len(lines) < 2 { + return "", "", false + } + return lines[0], strings.Trim(lines[1], `"`), true +} + +// scanFolder reads a plain model folder. The commit comes from --revision, or +// from the folder's git HEAD. +func scanFolder(dir string, o ScanOptions) (Model, error) { + m := Model{Repo: o.Repo, Kind: KindFolder, Root: dir} + if !ValidRepo(o.Repo) { + return m, fmt.Errorf("--repo %q is not an org/name Hub id", o.Repo) + } + m.Revision = gitHead(dir) + if err := m.useRevision(o.Revision); err != nil { + return m, err + } + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == ".git" { + return filepath.SkipDir + } + return nil + } + rel, _ := filepath.Rel(dir, p) + rel = filepath.ToSlash(rel) + st, err := os.Stat(p) + if err != nil { + m.Problems = append(m.Problems, "dangling link: "+rel) + return nil + } + if isLFSPointer(p, st.Size()) { + m.Problems = append(m.Problems, rel+" is a git-lfs pointer, not the file: run git lfs pull") + } + m.Files = append(m.Files, File{Path: rel, Size: st.Size(), Abs: p}) + return nil + }) + if err != nil { + return m, fmt.Errorf("read %s: %w", dir, err) + } + m.checkComplete() + return m, nil +} + +const lfsPointerHead = "version https://git-lfs.github.com/spec/v1" + +func isLFSPointer(file string, size int64) bool { + if size > 1024 { + return false + } + f, err := os.Open(file) + if err != nil { + return false + } + defer f.Close() + head := make([]byte, len(lfsPointerHead)) + n, _ := io.ReadFull(f, head) + return string(head[:n]) == lfsPointerHead +} + +// gitHead is the commit a clone has checked out, "" when dir is not one. +func gitHead(dir string) string { + read := func(p string) string { + b, err := os.ReadFile(filepath.Join(dir, ".git", filepath.FromSlash(p))) + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) + } + head := read("HEAD") + ref, ok := strings.CutPrefix(head, "ref: ") + if !ok { + if revisionPattern.MatchString(head) { + return head + } + return "" + } + if c := read(ref); revisionPattern.MatchString(c) { + return c + } + for line := range strings.SplitSeq(read("packed-refs"), "\n") { + if c, name, ok := strings.Cut(line, " "); ok && name == ref && revisionPattern.MatchString(c) { + return c + } + } + return "" +} diff --git a/internal/hfcache/space_darwin.go b/internal/hfcache/space_darwin.go new file mode 100644 index 0000000..fef685c --- /dev/null +++ b/internal/hfcache/space_darwin.go @@ -0,0 +1,29 @@ +package hfcache + +import "syscall" + +// freeBytes is what a user can still write to the filesystem holding dir. It +// is a variable so tests can fix it. +var freeBytes = func(dir string) (int64, error) { + var st syscall.Statfs_t + if err := syscall.Statfs(dir, &st); err != nil { + return 0, err + } + return int64(st.Bavail) * int64(st.Bsize), nil +} + +// onFAT reports whether dir is on a FAT filesystem, which macOS calls msdos. +func onFAT(dir string) bool { + var st syscall.Statfs_t + if syscall.Statfs(dir, &st) != nil { + return false + } + var name []byte + for _, c := range st.Fstypename { + if c == 0 { + break + } + name = append(name, byte(c)) + } + return string(name) == "msdos" +} diff --git a/internal/hfcache/space_linux.go b/internal/hfcache/space_linux.go new file mode 100644 index 0000000..4aee98b --- /dev/null +++ b/internal/hfcache/space_linux.go @@ -0,0 +1,22 @@ +package hfcache + +import "syscall" + +// msdosSuperMagic is the filesystem type of FAT12, FAT16, FAT32 and VFAT. +const msdosSuperMagic = 0x4d44 + +// freeBytes is what a user can still write to the filesystem holding dir. It +// is a variable so tests can fix it. +var freeBytes = func(dir string) (int64, error) { + var st syscall.Statfs_t + if err := syscall.Statfs(dir, &st); err != nil { + return 0, err + } + return int64(st.Bavail) * int64(st.Bsize), nil +} + +// onFAT reports whether dir is on a FAT filesystem. +func onFAT(dir string) bool { + var st syscall.Statfs_t + return syscall.Statfs(dir, &st) == nil && int64(st.Type) == msdosSuperMagic +} diff --git a/internal/hfcache/space_other.go b/internal/hfcache/space_other.go new file mode 100644 index 0000000..d6bc150 --- /dev/null +++ b/internal/hfcache/space_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin + +package hfcache + +import "errors" + +// freeBytes is unknown here, which CheckSpace reads as nothing to check. +var freeBytes = func(string) (int64, error) { return 0, errors.New("free space is not known on this platform") } + +// onFAT is false where the filesystem cannot be read. +func onFAT(string) bool { return false } From a0c1280e857d56e7f4498a1c3622076d7ecb3790 Mon Sep 17 00:00:00 2001 From: DevelopmentCats Date: Mon, 16 Nov 2026 16:30:45 +0000 Subject: [PATCH 3/5] hf: read the local cache when offline new, inspect and search answer from the cache when the Hub is not used or does not answer, instead of failing on the network. --- internal/hf/arch.go | 2 +- internal/hf/hf.go | 28 +++- internal/hf/local.go | 142 +++++++++++++++++++ internal/hf/local_internal_test.go | 213 +++++++++++++++++++++++++++++ 4 files changed, 380 insertions(+), 5 deletions(-) create mode 100644 internal/hf/local.go create mode 100644 internal/hf/local_internal_test.go diff --git a/internal/hf/arch.go b/internal/hf/arch.go index 1abdd80..f767f0f 100644 --- a/internal/hf/arch.go +++ b/internal/hf/arch.go @@ -167,7 +167,7 @@ func (a Arch) MoE() bool { // the repo has none: GGUF repos typically ship none, which is a fact about the // repo rather than a failure. func Config(ctx context.Context, repo string) (Arch, error) { - resp, err := get(ctx, fmt.Sprintf("%s/%s/raw/main/config.json", baseURL, repo)) + resp, err := rawGet(ctx, repo, "config.json") if err != nil { return Arch{}, err } diff --git a/internal/hf/hf.go b/internal/hf/hf.go index a2b0e3b..2b7fe8e 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -101,8 +101,14 @@ func (m Model) QuantTags() []string { // Lookup fetches one model's metadata by repo id. func Lookup(ctx context.Context, repo string) (Model, error) { var m Model + if hub := local(); hub != "" { + return localModel(hub, repo) + } resp, err := get(ctx, baseURL+"/api/models/"+repo) if err != nil { + if hub := canFallBack(ctx, err); hub != "" { + return localModel(hub, repo) + } return m, err } defer resp.Body.Close() @@ -151,12 +157,20 @@ func WhoAmI(ctx context.Context, token string) (string, error) { } // Search queries the Hub for text-generation models, most-downloaded first. +// +// With the Hub not in use it lists the cached models that match instead. func Search(ctx context.Context, query string, limit int) ([]Model, error) { + if hub := local(); hub != "" { + return localSearch(hub, query, limit), nil + } u := fmt.Sprintf( baseURL+"/api/models?search=%s&filter=text-generation&sort=downloads&direction=-1&limit=%d", url.QueryEscape(query), limit) resp, err := get(ctx, u) if err != nil { + if hub := canFallBack(ctx, err); hub != "" { + return localSearch(hub, query, limit), nil + } return nil, err } defer resp.Body.Close() @@ -615,12 +629,18 @@ func isWeightFile(path string) bool { // undercounts, and an undercount is the one wrong answer that tells someone a // model fits when it does not. func repoTree(ctx context.Context, repo string) ([]treeEntry, error) { + if hub := local(); hub != "" { + return localTree(hub, repo) + } next := baseURL + "/api/models/" + repo + "/tree/main?recursive=true" var all []treeEntry seen := map[string]bool{} for page := 0; page < maxTreePages; page++ { entries, link, err := treePage(ctx, next) if err != nil { + if hub := canFallBack(ctx, err); hub != "" && page == 0 { + return localTree(hub, repo) + } return nil, err } for _, e := range entries { @@ -706,7 +726,7 @@ type DetectedFlags struct { // someone else's deployment, not the model. func DetectFlags(ctx context.Context, repo string) (DetectedFlags, error) { var d DetectedFlags - resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/README.md") + resp, err := rawGet(ctx, repo, "README.md") if err != nil { return d, err } @@ -746,7 +766,7 @@ func DetectFlags(ctx context.Context, repo string) (DetectedFlags, error) { // the ceiling vLLM validates --max-model-len against. Returns 0 when the // field is absent. func ContextLimit(ctx context.Context, repo string) (int, error) { - resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/config.json") + resp, err := rawGet(ctx, repo, "config.json") if err != nil { return 0, err } @@ -780,7 +800,7 @@ func clampContext(value string, limit int) string { // ChatTemplate fetches a repo's chat_template.jinja. found is false when the // repo has no separate template file. func ChatTemplate(ctx context.Context, repo string) (text string, found bool, err error) { - resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/chat_template.jinja") + resp, err := rawGet(ctx, repo, "chat_template.jinja") if err != nil { return "", false, err } @@ -802,7 +822,7 @@ func TemplateText(ctx context.Context, repo string) (string, error) { if text, found, err := ChatTemplate(ctx, repo); err != nil || found { return text, err } - resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/tokenizer_config.json") + resp, err := rawGet(ctx, repo, "tokenizer_config.json") if err != nil { return "", err } diff --git a/internal/hf/local.go b/internal/hf/local.go new file mode 100644 index 0000000..2e06e65 --- /dev/null +++ b/internal/hf/local.go @@ -0,0 +1,142 @@ +package hf + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/coder/llm-helper/internal/hfcache" +) + +// ErrOffline is returned for a model that is not in the local cache while the +// Hub is not being used. +var ErrOffline = errors.New("not in the local cache, and the Hub is not reachable") + +// cache is where the Hub readers look when the Hub cannot answer. +var cache struct { + sync.Mutex + hub string + offline bool + // downUntil is when the Hub is tried again after a request failed to + // reach it, so a panel that outlives a network drop recovers. + downUntil time.Time +} + +// retryHubAfter is how long the cache answers after the Hub did not. +const retryHubAfter = time.Minute + +// UseCache points the readers at the hub cache a server reads. With offline +// set they never ask the Hub; without it they fall back to the cache once a +// request fails to reach the Hub at all. An empty hub turns the fallback off. +func UseCache(hub string, offline bool) { + cache.Lock() + defer cache.Unlock() + cache.hub, cache.offline, cache.downUntil = hub, offline, time.Time{} +} + +// Offline reports whether the readers are answering from the cache: by +// choice, or because the Hub did not answer. +func Offline() bool { + cache.Lock() + defer cache.Unlock() + return cache.hub != "" && (cache.offline || time.Now().Before(cache.downUntil)) +} + +// local is the cache to read, "" when the Hub is the source. +func local() string { + cache.Lock() + defer cache.Unlock() + if cache.hub != "" && (cache.offline || time.Now().Before(cache.downUntil)) { + return cache.hub + } + return "" +} + +// canFallBack is the cache to read after a request failed to reach the Hub, +// and notes that it did not. A cache with no models is no fallback: the +// request's own error says more. +func canFallBack(ctx context.Context, err error) string { + if err == nil || ctx.Err() != nil { + return "" + } + cache.Lock() + defer cache.Unlock() + if cache.hub == "" || !hfcache.Has(cache.hub) { + return "" + } + cache.downUntil = time.Now().Add(retryHubAfter) + return cache.hub +} + +// rawGet reads one file of a repo at main, from the Hub or, when that is not +// being used, from the cached snapshot. A cached repo without the file is a +// 404, as the Hub would say. +func rawGet(ctx context.Context, repo, name string) (*http.Response, error) { + hub := local() + if hub == "" { + resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/"+name) + if hub = canFallBack(ctx, err); hub == "" { + return resp, err + } + } + dir, _, ok := hfcache.Snapshot(hub, repo) + if !ok { + return nil, fmt.Errorf("%s: %w", repo, ErrOffline) + } + f, err := os.Open(filepath.Join(dir, filepath.FromSlash(name))) + if err != nil { + return &http.Response{StatusCode: http.StatusNotFound, Body: http.NoBody}, nil + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(f), ContentLength: -1}, nil +} + +// localModel is Lookup's answer for a cached repo. +func localModel(hub, repo string) (Model, error) { + if _, _, ok := hfcache.Snapshot(hub, repo); !ok { + return Model{}, fmt.Errorf("%s: %w", repo, ErrOffline) + } + return Model{ID: repo}, nil +} + +// localTree is repoTree for a cached repo: the files its snapshot holds. +func localTree(hub, repo string) ([]treeEntry, error) { + m, err := hfcache.ScanRepo(hub, repo) + if err != nil || len(m.Files) == 0 { + return nil, fmt.Errorf("%s: %w", repo, ErrOffline) + } + out := make([]treeEntry, 0, len(m.Files)) + for _, f := range m.Files { + out = append(out, treeEntry{Type: "file", Path: f.Path, Size: f.Size}) + } + return out, nil +} + +// localSearch lists the cached models whose id holds every word of query. +func localSearch(hub, query string, limit int) []Model { + models, _ := hfcache.ScanCache(hub) + words := strings.Fields(strings.ToLower(query)) + var out []Model + for _, m := range models { + id := strings.ToLower(m.Repo) + match := true + for _, w := range words { + match = match && strings.Contains(id, w) + } + if match { + out = append(out, Model{ID: m.Repo}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return out +} diff --git a/internal/hf/local_internal_test.go b/internal/hf/local_internal_test.go new file mode 100644 index 0000000..ae44d8d --- /dev/null +++ b/internal/hf/local_internal_test.go @@ -0,0 +1,213 @@ +package hf + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +const rev = "0123456789abcdef0123456789abcdef01234567" + +// cachedRepo writes a repo into a hub cache the way huggingface_hub does, +// with each snapshot file a link to a blob. +func cachedRepo(t *testing.T, hub, repoID string, files map[string]string) { + t.Helper() + root := filepath.Join(hub, "models--"+strings.ReplaceAll(repoID, "/", "--")) + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(os.MkdirAll(filepath.Join(root, "refs"), 0o755)) + must(os.WriteFile(filepath.Join(root, "refs", "main"), []byte(rev), 0o644)) + i := 0 + for name, body := range files { + i++ + blob := filepath.Join(root, "blobs", strings.Repeat(string(rune('a'+i)), 40)) + must(os.MkdirAll(filepath.Dir(blob), 0o755)) + must(os.WriteFile(blob, []byte(body), 0o644)) + link := filepath.Join(root, "snapshots", rev, filepath.FromSlash(name)) + must(os.MkdirAll(filepath.Dir(link), 0o755)) + rel, err := filepath.Rel(filepath.Dir(link), blob) + must(err) + must(os.Symlink(rel, link)) + } +} + +func useCache(t *testing.T, hub string, offline bool) { + t.Helper() + UseCache(hub, offline) + t.Cleanup(func() { UseCache("", false) }) +} + +// deadHub points the client at a server that is already closed. +func deadHub(t *testing.T) { + t.Helper() + srv := httptest.NewServer(http.NotFoundHandler()) + url := srv.URL + srv.Close() + old := baseURL + baseURL = url + t.Cleanup(func() { baseURL = old }) +} + +func TestOfflineReadsTheCachedSnapshot(t *testing.T) { + hub := t.TempDir() + cachedRepo(t, hub, repo, map[string]string{ + "config.json": `{"model_type":"qwen3","max_position_embeddings":40960}`, + "chat_template.jinja": "{{ messages }}", + "tokenizer_config.json": `{"chat_template":"x"}`, + "model.safetensors": strings.Repeat("w", 64), + }) + newHub(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("offline asked the Hub for %s", r.URL) + })) + useCache(t, hub, true) + ctx := context.Background() + + if !Offline() { + t.Fatal("Offline() is false") + } + arch, err := Config(ctx, repo) + if err != nil || arch.ModelType != "qwen3" { + t.Fatalf("Config = %+v, %v", arch, err) + } + if n, err := ContextLimit(ctx, repo); err != nil || n != 40960 { + t.Fatalf("ContextLimit = %d, %v", n, err) + } + if text, found, err := ChatTemplate(ctx, repo); err != nil || !found || text != "{{ messages }}" { + t.Fatalf("ChatTemplate = %q, %v, %v", text, found, err) + } + if m, err := Lookup(ctx, repo); err != nil || m.ID != repo { + t.Fatalf("Lookup = %+v, %v", m, err) + } + tree, err := repoTree(ctx, repo) + if err != nil || len(tree) != 4 { + t.Fatalf("repoTree = %+v, %v", tree, err) + } + found, err := Search(ctx, "qwen3 fp8", 10) + if err != nil || len(found) != 1 || found[0].ID != repo { + t.Fatalf("Search = %+v, %v", found, err) + } + if found, _ := Search(ctx, "llama", 10); len(found) != 0 { + t.Fatalf("Search matched an uncached model: %+v", found) + } +} + +func TestOfflineFileMissingFromACachedRepoIsNotFound(t *testing.T) { + hub := t.TempDir() + cachedRepo(t, hub, repo, map[string]string{"model.safetensors": "w"}) + useCache(t, hub, true) + + arch, err := Config(context.Background(), repo) + if err != nil || arch.ModelType != "" { + t.Fatalf("Config = %+v, %v; want an empty Arch like a repo with no config.json", arch, err) + } + if _, found, err := ChatTemplate(context.Background(), repo); err != nil || found { + t.Fatalf("ChatTemplate found=%v err=%v", found, err) + } +} + +func TestOfflineUncachedRepoSaysSo(t *testing.T) { + useCache(t, t.TempDir(), true) + ctx := context.Background() + if _, err := Config(ctx, repo); !errors.Is(err, ErrOffline) { + t.Fatalf("Config err = %v", err) + } + if _, err := Lookup(ctx, repo); !errors.Is(err, ErrOffline) { + t.Fatalf("Lookup err = %v", err) + } + if _, err := repoTree(ctx, repo); !errors.Is(err, ErrOffline) { + t.Fatalf("repoTree err = %v", err) + } +} + +func TestUnreachableHubFallsBackToTheCache(t *testing.T) { + hub := t.TempDir() + cachedRepo(t, hub, repo, map[string]string{"config.json": `{"model_type":"qwen3"}`}) + deadHub(t) + useCache(t, hub, false) + + if Offline() { + t.Fatal("Offline() is true before any request failed") + } + arch, err := Config(context.Background(), repo) + if err != nil || arch.ModelType != "qwen3" { + t.Fatalf("Config = %+v, %v", arch, err) + } + if !Offline() { + t.Fatal("Offline() is false after the Hub did not answer") + } +} + +func TestHubIsUsedWhenItAnswers(t *testing.T) { + hub := t.TempDir() + cachedRepo(t, hub, repo, map[string]string{"config.json": `{"model_type":"cached"}`}) + newHub(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"model_type":"hub"}`)) + })) + useCache(t, hub, false) + + arch, err := Config(context.Background(), repo) + if err != nil || arch.ModelType != "hub" { + t.Fatalf("Config = %+v, %v", arch, err) + } + if Offline() { + t.Fatal("Offline() is true while the Hub answers") + } +} + +func TestNoCacheMeansNoFallback(t *testing.T) { + deadHub(t) + UseCache("", false) + if _, err := Config(context.Background(), repo); err == nil || errors.Is(err, ErrOffline) { + t.Fatalf("Config err = %v; want the transport error", err) + } +} + +func TestACancelledRequestDoesNotMarkTheHubUnreachable(t *testing.T) { + hub := t.TempDir() + deadHub(t) + useCache(t, hub, false) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, _ = Config(ctx, repo) + if Offline() { + t.Fatal("a cancelled request marked the Hub unreachable") + } +} + +func TestHubIsTriedAgainAfterAWhile(t *testing.T) { + hub := t.TempDir() + cachedRepo(t, hub, repo, map[string]string{"config.json": `{"model_type":"cached"}`}) + newHub(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"model_type":"hub"}`)) + })) + useCache(t, hub, false) + cache.Lock() + cache.downUntil = time.Now().Add(-time.Second) + cache.Unlock() + + arch, err := Config(context.Background(), repo) + if err != nil || arch.ModelType != "hub" { + t.Fatalf("Config = %+v, %v", arch, err) + } +} + +func TestAnEmptyCacheIsNoFallback(t *testing.T) { + deadHub(t) + useCache(t, t.TempDir(), false) + if _, err := Config(context.Background(), repo); err == nil || errors.Is(err, ErrOffline) { + t.Fatalf("Config err = %v; want the transport error", err) + } + if Offline() { + t.Fatal("Offline() is true with nothing to read") + } +} From 02d9f928e50f90df1112b10e1f00d60f25e8f488 Mon Sep 17 00:00:00 2001 From: DevelopmentCats Date: Mon, 16 Nov 2026 16:30:45 +0000 Subject: [PATCH 4/5] cli: add import and export, and offline new, search and doctor import and export move models on a drive. A new profile made offline tells its server to stay off the network, and doctor shows which cache the server reads. --- docs/manifest.json | 10 + docs/reference/cli.md | 2 + docs/reference/cli/export.md | 39 +++ docs/reference/cli/import.md | 75 +++++ internal/cli/detectcmds.go | 44 +++ internal/cli/doctor_internal_test.go | 33 ++ internal/cli/engines_internal_test.go | 2 +- internal/cli/hfcache.go | 344 +++++++++++++++++++++ internal/cli/hfcache_test.go | 291 +++++++++++++++++ internal/cli/profiles.go | 1 + internal/cli/recipelaunch.go | 4 +- internal/cli/root.go | 10 +- internal/cli/root_internal_test.go | 2 +- internal/cli/search.go | 8 + internal/control/hubcache.go | 56 ++++ internal/control/hubcache_internal_test.go | 148 +++++++++ internal/resolve/offline.go | 43 +++ internal/resolve/offline_internal_test.go | 91 ++++++ internal/web/models.go | 1 + 19 files changed, 1199 insertions(+), 5 deletions(-) create mode 100644 docs/reference/cli/export.md create mode 100644 docs/reference/cli/import.md create mode 100644 internal/cli/hfcache.go create mode 100644 internal/cli/hfcache_test.go create mode 100644 internal/control/hubcache.go create mode 100644 internal/control/hubcache_internal_test.go create mode 100644 internal/resolve/offline.go create mode 100644 internal/resolve/offline_internal_test.go diff --git a/docs/manifest.json b/docs/manifest.json index 0cd09ff..184c48d 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -133,6 +133,11 @@ } ] }, + { + "title": "llm-helper export", + "description": "Copy models from the Hugging Face cache onto an external drive", + "path": "./reference/cli/export.md" + }, { "title": "llm-helper hardware", "description": "List hardware definitions and report this machine's hardware", @@ -150,6 +155,11 @@ } ] }, + { + "title": "llm-helper import", + "description": "Copy models from an external drive into the Hugging Face cache", + "path": "./reference/cli/import.md" + }, { "title": "llm-helper inspect", "description": "Show a model's license, weight size against this machine, and detected flags", diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 4a50162..3dd6bcb 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -31,7 +31,9 @@ sharding, replica pools, recipes, optional Coder Agents sync). | [`down`](cli/down.md) | Stop serving a profile (the only one running when none is named) | | [`edit`](cli/edit.md) | Open a profile in $EDITOR and validate the result | | [`engines`](cli/engines.md) | List the inference engines a profile or recipe can run on | +| [`export`](cli/export.md) | Copy models from the Hugging Face cache onto an external drive | | [`hardware`](cli/hardware.md) | List hardware definitions and report this machine's hardware | +| [`import`](cli/import.md) | Copy models from an external drive into the Hugging Face cache | | [`inspect`](cli/inspect.md) | Show a model's license, weight size against this machine, and detected flags | | [`install-gguf`](cli/install-gguf.md) | Install the plugin that lets vLLM load GGUF models | | [`list`](cli/list.md) | List saved model profiles | diff --git a/docs/reference/cli/export.md b/docs/reference/cli/export.md new file mode 100644 index 0000000..4f697b0 --- /dev/null +++ b/docs/reference/cli/export.md @@ -0,0 +1,39 @@ +--- +# Code generated by go run ./scripts/gendocs. DO NOT EDIT. +title: "llm-helper export" +description: "Copy models from the Hugging Face cache onto an external drive" +--- + + + +Copy models from the Hugging Face cache the server reads onto an external drive, to +carry to a machine with no internet. The drive gets a plain copy of each file in a +hub/ folder, which any filesystem holds, and 'llm-helper import' reads it back. + +Only complete models are copied, and each file is checked against its recorded +hash as it is copied. FAT32 cannot hold a file over 4 GiB; format the drive as +exFAT. + +## Usage + +```console +llm-helper export ... [flags] +``` + +## Options + +### --cache + +| | | +| ---- | -------- | +| Type | `string` | + +hub cache directory to copy from (default: the one the server reads) + +### --dry-run + +| | | +| ---- | ------ | +| Type | `bool` | + +show what would be copied and stop diff --git a/docs/reference/cli/import.md b/docs/reference/cli/import.md new file mode 100644 index 0000000..684e33c --- /dev/null +++ b/docs/reference/cli/import.md @@ -0,0 +1,75 @@ +--- +# Code generated by go run ./scripts/gendocs. DO NOT EDIT. +title: "llm-helper import" +description: "Copy models from an external drive into the Hugging Face cache" +--- + + + +Copy models from an external drive, or any folder, into the Hugging Face cache the +server reads, so 'llm-helper new ' loads them with no download. + +The drive can hold a Hugging Face cache (also one made by 'llm-helper export'), an +'hf download --local-dir' folder, or a model folder. The last two do not record +which repo they are, so name it with --repo (and --revision unless the folder is +a git clone). + +Every file is checked against its recorded hash as it is copied, an interrupted +import resumes where it stopped, and nothing already in the cache is overwritten. +With --recipe the models go where that recipe's engine reads them. + +## Usage + +```console +llm-helper import [org/model...] [flags] +``` + +## Options + +### --all + +| | | +| ---- | ------ | +| Type | `bool` | + +import every complete model on the drive + +### --cache + +| | | +| ---- | -------- | +| Type | `string` | + +hub cache directory to import into (default: the one the server reads) + +### --dry-run + +| | | +| ---- | ------ | +| Type | `bool` | + +show what would be copied and stop + +### --recipe + +| | | +| ---- | -------- | +| Type | `string` | + +import the models a tested recipe pins, into the cache its engine reads + +### --repo + +| | | +| ---- | -------- | +| Type | `string` | + +org/model of a folder that is not a Hugging Face cache + +### --revision + +| | | +| ---- | -------- | +| Type | `string` | + +commit of a folder that is not a Hugging Face cache or git clone diff --git a/internal/cli/detectcmds.go b/internal/cli/detectcmds.go index 1ef0b23..b9b2195 100644 --- a/internal/cli/detectcmds.go +++ b/internal/cli/detectcmds.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "strings" "github.com/spf13/cobra" @@ -15,6 +16,7 @@ import ( "github.com/coder/llm-helper/internal/engine" "github.com/coder/llm-helper/internal/hardware" "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/hfcache" "github.com/coder/llm-helper/internal/modeltype" "github.com/coder/llm-helper/internal/plugin" "github.com/coder/llm-helper/internal/profile" @@ -34,6 +36,7 @@ type doctorReport struct { Toolchain []toolchain.Check `json:"toolchain"` Coder coderStatus `json:"coder_sync"` HFToken hfTokenStatus `json:"hf_token"` + HubCache hubCacheStatus `json:"hub_cache"` // WiredLimit is the macOS GPU wired memory cap, nil unless the Mac // serves on the GPU through vllm-metal. WiredLimit *wiredLimit `json:"wired_limit,omitempty"` @@ -69,6 +72,17 @@ type doctorInstance struct { Restarts int `json:"restarts"` } +// hubCacheStatus is the model cache the server reads. +type hubCacheStatus struct { + Path string `json:"path"` + FreeBytes int64 `json:"free_bytes,omitempty"` + Offline bool `json:"offline"` + // ShellPath is the cache this shell would use when it is not the + // server's: models downloaded from here land somewhere the server never + // looks. + ShellPath string `json:"shell_path,omitempty"` +} + type coderStatus struct { Configured bool `json:"configured"` URL string `json:"url,omitempty"` @@ -167,6 +181,12 @@ func gatherDoctor() doctorReport { r.Toolchain = toolchain.Gather(vllmBin) r.Coder = gatherCoderStatus(cfg) r.HFToken = gatherHFToken(cfg) + r.HubCache = gatherHubCache(cfg) + if r.HubCache.ShellPath != "" { + r.Warnings = append(r.Warnings, fmt.Sprintf( + "The server reads models from %s, but this shell's HF_HOME or HF_HUB_CACHE points at %s, so 'hf download' from here is not seen by it. Set HF_HOME in config.env, or download with HF_HUB_CACHE=%s.", + r.HubCache.Path, r.HubCache.ShellPath, r.HubCache.Path)) + } r.WiredLimit = gatherWiredLimit(cfg, r.Hardware) running := 0 if r.Config != nil { @@ -269,6 +289,20 @@ func pluginNamed(name string) plugin.Plugin { return nil } +// gatherHubCache reports the cache the server reads, which comes from +// config.env rather than this shell. +func gatherHubCache(cfg *config.Config) hubCacheStatus { + st := hubCacheStatus{Path: fallbackHub(), Offline: hfcache.Offline(hfcache.OSEnv)} + if cfg != nil { + st.Path, st.Offline = control.HubCache(cfg), control.HubOffline(cfg) + } + st.FreeBytes, _ = hfcache.Free(st.Path) + if shell := fallbackHub(); cfg != nil && filepath.Clean(shell) != filepath.Clean(st.Path) { + st.ShellPath = shell + } + return st +} + // gatherHFToken reports whether a HuggingFace token is available (gated // models can't be pulled without one) and whether the Hub accepts it. func gatherHFToken(cfg *config.Config) hfTokenStatus { @@ -535,6 +569,16 @@ func renderDoctor(r doctorReport, yes bool) { fmt.Println() } + fmt.Println("MODEL CACHE") + fmt.Printf(" path: %s\n", r.HubCache.Path) + if r.HubCache.FreeBytes > 0 { + fmt.Printf(" free: %s\n", hfcache.FormatBytes(r.HubCache.FreeBytes)) + } + if r.HubCache.Offline { + fmt.Println(" offline: yes -- models load from the cache and the Hub is not asked") + } + fmt.Println() + fmt.Println("RUNNING SERVERS") if len(r.Servers) == 0 && len(r.Containers) == 0 { fmt.Println(" no vLLM processes or containers found") diff --git a/internal/cli/doctor_internal_test.go b/internal/cli/doctor_internal_test.go index e77816f..716a1a8 100644 --- a/internal/cli/doctor_internal_test.go +++ b/internal/cli/doctor_internal_test.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strconv" "strings" "testing" @@ -718,3 +719,35 @@ func TestOverlapWarningsNameTheIgnoredDefinition(t *testing.T) { t.Errorf("an overlap on a PCI device: %q", w) } } + +// The server sources config.env, not the login shell, so doctor reports the +// cache the server reads and says when this shell would use another. +func TestGatherDoctorReportsTheServersModelCache(t *testing.T) { + dirs := newTree(t) + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("HF_HUB_CACHE", "") + t.Setenv("HF_HUB_OFFLINE", "") + t.Setenv("HF_HOME", filepath.Join(t.TempDir(), "shell")) + models := filepath.Join(t.TempDir(), "models") + writeConfig(t, dirs, "PORT=9101\nHF_HOME="+models+"\nHF_HUB_OFFLINE=1\n") + + r := gatherDoctor() + if want := filepath.Join(models, "hub"); r.HubCache.Path != want { + t.Errorf("path = %q, want %q", r.HubCache.Path, want) + } + if !r.HubCache.Offline { + t.Error("HF_HUB_OFFLINE in config.env was not reported") + } + if want := filepath.Join(os.Getenv("HF_HOME"), "hub"); r.HubCache.ShellPath != want { + t.Errorf("shell path = %q, want %q", r.HubCache.ShellPath, want) + } + if !slices.ContainsFunc(r.Warnings, func(w string) bool { return strings.Contains(w, "this shell") }) { + t.Errorf("no warning about the shell's cache: %v", r.Warnings) + } + + writeConfig(t, dirs, "PORT=9101\nHF_HOME="+os.Getenv("HF_HOME")+"\n") + if r := gatherDoctor(); r.HubCache.ShellPath != "" { + t.Errorf("shell path = %q when it is the server's", r.HubCache.ShellPath) + } +} diff --git a/internal/cli/engines_internal_test.go b/internal/cli/engines_internal_test.go index c33982f..478eaed 100644 --- a/internal/cli/engines_internal_test.go +++ b/internal/cli/engines_internal_test.go @@ -81,7 +81,7 @@ func TestCheckStagedNamesWhatIsMissing(t *testing.T) { t.Fatal("a missing drafter passed the staging check") } msg := err.Error() - for _, want := range []string{drafter.ID + "@" + drafter.Revision, "node head", "hf download " + drafter.ID + " --revision " + drafter.Revision} { + for _, want := range []string{drafter.ID + "@" + drafter.Revision, "node head", "hf download " + drafter.ID + " --revision " + drafter.Revision, "llm-helper import --recipe " + r.ID} { if !strings.Contains(msg, want) { t.Errorf("error %q does not mention %q", msg, want) } diff --git a/internal/cli/hfcache.go b/internal/cli/hfcache.go new file mode 100644 index 0000000..2764f24 --- /dev/null +++ b/internal/cli/hfcache.go @@ -0,0 +1,344 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/spf13/cobra" + + "github.com/coder/llm-helper/internal/control" + "github.com/coder/llm-helper/internal/hfcache" + "github.com/coder/llm-helper/internal/recipe" +) + +type importOptions struct { + all bool + dryRun bool + cache string + recipeID string + repo string + revision string +} + +func newImportCmd() *cobra.Command { + var o importOptions + c := &cobra.Command{ + Use: "import [org/model...]", + Short: "Copy models from an external drive into the Hugging Face cache", + Long: "Copy models from an external drive, or any folder, into the Hugging Face cache the\n" + + "server reads, so 'llm-helper new ' loads them with no download.\n\n" + + "The drive can hold a Hugging Face cache (also one made by 'llm-helper export'), an\n" + + "'hf download --local-dir' folder, or a model folder. The last two do not record\n" + + "which repo they are, so name it with --repo (and --revision unless the folder is\n" + + "a git clone).\n\n" + + "Every file is checked against its recorded hash as it is copied, an interrupted\n" + + "import resumes where it stopped, and nothing already in the cache is overwritten.\n" + + "With --recipe the models go where that recipe's engine reads them.", + Example: " llm-helper import /media/usb\n" + + " llm-helper import /media/usb Qwen/Qwen3-8B-FP8\n" + + " llm-helper import /media/usb --recipe ", + GroupID: groupModels, + Args: cobra.MinimumNArgs(1), + RunE: func(_ *cobra.Command, args []string) error { return runImport(args[0], args[1:], o) }, + } + f := c.Flags() + f.BoolVar(&o.all, "all", false, "import every complete model on the drive") + f.BoolVar(&o.dryRun, "dry-run", false, "show what would be copied and stop") + f.StringVar(&o.cache, "cache", "", "hub cache directory to import into (default: the one the server reads)") + f.StringVar(&o.recipeID, "recipe", "", "import the models a tested recipe pins, into the cache its engine reads") + f.StringVar(&o.repo, "repo", "", "org/model of a folder that is not a Hugging Face cache") + f.StringVar(&o.revision, "revision", "", "commit of a folder that is not a Hugging Face cache or git clone") + return c +} + +func newExportCmd() *cobra.Command { + var cache string + var dryRun bool + c := &cobra.Command{ + Use: "export ... ", + Short: "Copy models from the Hugging Face cache onto an external drive", + Long: "Copy models from the Hugging Face cache the server reads onto an external drive, to\n" + + "carry to a machine with no internet. The drive gets a plain copy of each file in a\n" + + "hub/ folder, which any filesystem holds, and 'llm-helper import' reads it back.\n\n" + + "Only complete models are copied, and each file is checked against its recorded\n" + + "hash as it is copied. FAT32 cannot hold a file over 4 GiB; format the drive as\n" + + "exFAT.", + Example: " llm-helper export Qwen/Qwen3-8B-FP8 /media/usb", + GroupID: groupModels, + Args: cobra.MinimumNArgs(2), + RunE: func(_ *cobra.Command, args []string) error { + return runExport(args[:len(args)-1], args[len(args)-1], cache, dryRun) + }, + } + c.Flags().StringVar(&cache, "cache", "", "hub cache directory to copy from (default: the one the server reads)") + c.Flags().BoolVar(&dryRun, "dry-run", false, "show what would be copied and stop") + return c +} + +// serverHub is the cache a command works on when --cache is not given. +func serverHub() string { + if cfg, err := loadConfig(); err == nil { + return control.HubCache(cfg) + } + return fallbackHub() +} + +// fallbackHub is the cache of a machine with no config yet. +func fallbackHub() string { + home, _ := os.UserHomeDir() + return hfcache.Dir(hfcache.OSEnv, home) +} + +// recipeTarget is the models a recipe pins and the hub cache its engine reads. +func recipeTarget(id string) (models []recipe.Model, hub string, err error) { + tested, err := recipe.LoadTested() + if err != nil { + return nil, "", err + } + r, ok := tested.Get(id) + if !ok { + return nil, "", fmt.Errorf("no tested recipe %q", id) + } + cache := r.Engine().HFCacheHostPath(r.Params) + if cache == "" || len(r.Models) == 0 { + return nil, "", fmt.Errorf("recipe %s loads no models from a Hugging Face cache", id) + } + home, err := os.UserHomeDir() + if err != nil { + return nil, "", err + } + return r.Models, filepath.Join(home, cache, "hub"), nil +} + +func runImport(drive string, names []string, o importOptions) error { + if o.recipeID != "" && (o.cache != "" || o.repo != "") { + return errors.New("--recipe picks the models and the cache itself, so it cannot be combined with --cache or --repo") + } + for _, n := range names { + if !hfcache.ValidRepo(n) { + return fmt.Errorf("%q is not an org/model id", n) + } + } + hub := o.cache + var pinned []recipe.Model + switch { + case o.recipeID != "": + var err error + if pinned, hub, err = recipeTarget(o.recipeID); err != nil { + return err + } + case hub == "": + hub = serverHub() + } + + found, err := hfcache.Scan(drive, hfcache.ScanOptions{Repo: o.repo, Revision: o.revision}) + if err != nil { + return err + } + if len(found.Models) == 0 { + return fmt.Errorf("found no models at %s", drive) + } + + var chosen []hfcache.Model + switch { + case o.recipeID != "": + if chosen, err = pickPinned(found.Models, pinned); err != nil { + return err + } + case len(names) > 0: + if chosen, err = pickNamed(found.Models, names); err != nil { + return err + } + default: + listModels(found.Models) + if chosen, err = pickInteractive(found.Models, o.all); err != nil { + return err + } + } + if len(chosen) == 0 { + fmt.Println("Nothing selected.") + return nil + } + + need := hfcache.Need(chosen, hub) + fmt.Printf("Importing %d model(s) into %s: %s to copy.\n", len(chosen), hub, hfcache.FormatBytes(need)) + if err := hfcache.CheckSpace(hub, need); err != nil { + return err + } + if o.dryRun { + for _, m := range chosen { + fmt.Printf(" would import %s (%s)\n", m.Repo, hfcache.FormatBytes(m.Size())) + } + return nil + } + return copyModels(chosen, hub, hfcache.Import, "import", func(m hfcache.Model) { + switch { + case o.recipeID != "": + fmt.Printf("%s is staged for recipe %s.\n", m.Repo, o.recipeID) + default: + fmt.Printf("Run 'llm-helper new %s' to serve it.\n", m.Repo) + } + }) +} + +func runExport(repos []string, drive, cache string, dryRun bool) error { + for _, n := range repos { + if !hfcache.ValidRepo(n) { + return fmt.Errorf("%q is not an org/model id", n) + } + } + if st, err := os.Stat(drive); err != nil || !st.IsDir() { + return fmt.Errorf("%s is not a directory: mount the drive first", drive) + } + if cache == "" { + cache = serverHub() + } + var models []hfcache.Model + for _, repo := range repos { + m, err := hfcache.ScanRepo(cache, repo) + if err != nil { + return err + } + if !m.Complete() { + return fmt.Errorf("%s is incomplete in %s, so it would not load on the other machine: %s", repo, cache, strings.Join(m.Problems, "; ")) + } + models = append(models, m) + } + if err := hfcache.CheckDrive(models, drive); err != nil { + return err + } + dest := filepath.Join(drive, "hub") + fmt.Printf("Exporting %d model(s) to %s: %s to copy.\n", len(models), dest, hfcache.FormatBytes(hfcache.Need(models, dest))) + if dryRun { + for _, m := range models { + fmt.Printf(" would export %s (%s)\n", m.Repo, hfcache.FormatBytes(m.Size())) + } + return nil + } + return copyModels(models, dest, hfcache.Export, "export", func(m hfcache.Model) { + fmt.Printf("On the other machine: llm-helper import %s %s\n", drive, m.Repo) + }) +} + +// copyModels runs one copy per model and reports each. +func copyModels(models []hfcache.Model, dest string, do func(hfcache.Model, string, hfcache.Progress) (hfcache.Result, error), verb string, done func(hfcache.Model)) error { + for _, m := range models { + fmt.Printf("%s %s\n", strings.ToUpper(verb[:1])+verb[1:], m.Repo) + progress := func(path string, size int64, skipped bool) { + if !skipped { + fmt.Printf(" %s (%s)\n", path, hfcache.FormatBytes(size)) + } + } + res, err := do(m, dest, progress) + if err != nil { + var corrupt *hfcache.CorruptError + if errors.As(err, &corrupt) { + return fmt.Errorf("%s %s: %w (the source is damaged; nothing was kept from that file)", verb, m.Repo, err) + } + return fmt.Errorf("%s %s: %w (run it again to resume)", verb, m.Repo, err) + } + fmt.Printf(" %d file(s) copied, %d already there, %s written\n", res.Copied, res.Skipped, hfcache.FormatBytes(res.Bytes)) + if res.RefKept != "" { + fmt.Printf(" note: refs/main still names %s, so the copy is not what an offline load of %s reads\n", res.RefKept, m.Repo) + } + done(m) + } + return nil +} + +func listModels(models []hfcache.Model) { + fmt.Println() + for i, m := range models { + state := "complete" + if !m.Complete() { + state = "INCOMPLETE: " + strings.Join(m.Problems, "; ") + } + fmt.Printf(" %2d) %-52s %9s %.7s %s\n", i+1, m.Repo, hfcache.FormatBytes(m.Size()), m.Revision, state) + } + fmt.Println() +} + +// pickNamed selects the models the user named, which must all be importable. +func pickNamed(found []hfcache.Model, names []string) ([]hfcache.Model, error) { + var out []hfcache.Model + for _, n := range names { + m, ok := findRepo(found, n) + if !ok { + return nil, fmt.Errorf("%s is not on the drive", n) + } + if !m.Complete() { + return nil, fmt.Errorf("%s is incomplete on the drive: %s", n, strings.Join(m.Problems, "; ")) + } + out = append(out, m) + } + return out, nil +} + +// pickPinned selects the models a recipe pins, each at its pinned revision. +func pickPinned(found []hfcache.Model, pinned []recipe.Model) ([]hfcache.Model, error) { + var out []hfcache.Model + for _, p := range pinned { + m, ok := findRepo(found, p.ID) + switch { + case !ok: + return nil, fmt.Errorf("%s is not on the drive", p.ID) + case m.Revision != p.Revision: + return nil, fmt.Errorf("the recipe pins %s at %s, but the drive has %s", p.ID, p.Revision, m.Revision) + case !m.Complete(): + return nil, fmt.Errorf("%s is incomplete on the drive: %s", p.ID, strings.Join(m.Problems, "; ")) + } + out = append(out, m) + } + return out, nil +} + +// pickInteractive selects with --all, or asks, and skips incomplete models. +func pickInteractive(found []hfcache.Model, all bool) ([]hfcache.Model, error) { + var ok []hfcache.Model + for _, m := range found { + if m.Complete() { + ok = append(ok, m) + } + } + if len(ok) == 0 { + return nil, errors.New("no complete model on the drive") + } + if all { + return ok, nil + } + if !stdinIsTTY() { + return nil, errors.New("name the models to import, or pass --all") + } + answer := ask("Import which? (numbers like 1,3, or 'all'; Enter to cancel)", "") + if answer == "" { + return nil, nil + } + if answer == "all" { + return ok, nil + } + var out []hfcache.Model + for _, field := range strings.Split(answer, ",") { + n, err := strconv.Atoi(strings.TrimSpace(field)) + if err != nil || n < 1 || n > len(found) { + return nil, fmt.Errorf("%q is not a number from the list", field) + } + if !found[n-1].Complete() { + return nil, fmt.Errorf("%s is incomplete on the drive", found[n-1].Repo) + } + out = append(out, found[n-1]) + } + return out, nil +} + +func findRepo(models []hfcache.Model, repo string) (hfcache.Model, bool) { + for _, m := range models { + if strings.EqualFold(m.Repo, repo) { + return m, true + } + } + return hfcache.Model{}, false +} diff --git a/internal/cli/hfcache_test.go b/internal/cli/hfcache_test.go new file mode 100644 index 0000000..c800180 --- /dev/null +++ b/internal/cli/hfcache_test.go @@ -0,0 +1,291 @@ +package cli_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/recipe" +) + +const testRev = "0123456789abcdef0123456789abcdef01234567" + +// seedHub writes a repo into a hub cache the way huggingface_hub does: blobs +// named by hash and snapshot files linked to them. +func seedHub(t *testing.T, hub, repo, rev string, files map[string]string) { + t.Helper() + root := filepath.Join(hub, "models--"+strings.ReplaceAll(repo, "/", "--")) + mustWrite := func(p, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + mustWrite(filepath.Join(root, "refs", "main"), rev) + for name, body := range files { + sum := sha256.Sum256([]byte(body)) + blob := filepath.Join(root, "blobs", hex.EncodeToString(sum[:])) + mustWrite(blob, body) + link := filepath.Join(root, "snapshots", rev, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { + t.Fatal(err) + } + rel, err := filepath.Rel(filepath.Dir(link), blob) + if err != nil { + t.Fatal(err) + } + if err := os.Symlink(rel, link); err != nil { + t.Fatal(err) + } + } +} + +var qwenFiles = map[string]string{ + "config.json": `{"model_type":"qwen3"}`, + "model.safetensors": strings.Repeat("weights", 100), +} + +func snapshotFile(hub, repo, rev, name string) string { + return filepath.Join(hub, "models--"+strings.ReplaceAll(repo, "/", "--"), "snapshots", rev, name) +} + +func TestImportCopiesANamedModelIntoTheCache(t *testing.T) { + newTree(t) + drive, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, filepath.Join(drive, "hub"), "Qwen/Qwen3-8B", testRev, qwenFiles) + + code, out, errOut := runCLI(t, "import", drive, "Qwen/Qwen3-8B", "--cache", hub) + if code != 0 { + t.Fatalf("exit %d\n%s\n%s", code, out, errOut) + } + got, err := os.ReadFile(snapshotFile(hub, "Qwen/Qwen3-8B", testRev, "model.safetensors")) + if err != nil || string(got) != qwenFiles["model.safetensors"] { + t.Fatalf("imported weights: %v", err) + } + if ref, _ := os.ReadFile(filepath.Join(hub, "models--Qwen--Qwen3-8B", "refs", "main")); string(ref) != testRev { + t.Errorf("refs/main = %q", ref) + } + if !strings.Contains(out, "llm-helper new Qwen/Qwen3-8B") { + t.Errorf("output does not say how to serve it:\n%s", out) + } +} + +func TestImportNeedsNamesOrAllWithoutATerminal(t *testing.T) { + newTree(t) + drive, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, drive, "Qwen/Qwen3-8B", testRev, qwenFiles) + seedHub(t, drive, "org/other", testRev, qwenFiles) + + code, out, errOut := runCLI(t, "import", drive, "--cache", hub) + if code == 0 || !strings.Contains(errOut, "--all") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } + if !strings.Contains(out, "Qwen/Qwen3-8B") || !strings.Contains(out, "org/other") { + t.Errorf("the models on the drive were not listed:\n%s", out) + } + if _, err := os.Stat(hub); err == nil { + t.Error("the cache was written without a choice") + } + + if code, out, errOut := runCLI(t, "import", drive, "--all", "--cache", hub); code != 0 { + t.Fatalf("--all: exit %d\n%s\n%s", code, out, errOut) + } + for _, repo := range []string{"Qwen/Qwen3-8B", "org/other"} { + if _, err := os.Stat(snapshotFile(hub, repo, testRev, "config.json")); err != nil { + t.Errorf("%s was not imported: %v", repo, err) + } + } +} + +func TestImportDryRunCopiesNothing(t *testing.T) { + newTree(t) + drive, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, drive, "Qwen/Qwen3-8B", testRev, qwenFiles) + + code, out, _ := runCLI(t, "import", drive, "Qwen/Qwen3-8B", "--cache", hub, "--dry-run") + if code != 0 || !strings.Contains(out, "would import Qwen/Qwen3-8B") { + t.Fatalf("exit %d\n%s", code, out) + } + if _, err := os.Stat(hub); err == nil { + t.Error("a dry run wrote the cache") + } +} + +func TestImportRefusesAnIncompleteModel(t *testing.T) { + newTree(t) + drive, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, drive, "Qwen/Qwen3-8B", testRev, qwenFiles) + repoDir := filepath.Join(drive, "models--Qwen--Qwen3-8B") + entries, _ := os.ReadDir(filepath.Join(repoDir, "blobs")) + for _, e := range entries { + if err := os.Remove(filepath.Join(repoDir, "blobs", e.Name())); err != nil { + t.Fatal(err) + } + } + + code, _, errOut := runCLI(t, "import", drive, "Qwen/Qwen3-8B", "--cache", hub) + if code == 0 || !strings.Contains(errOut, "incomplete") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } +} + +func TestImportFailsOnADamagedFile(t *testing.T) { + newTree(t) + drive, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, drive, "Qwen/Qwen3-8B", testRev, qwenFiles) + snap := snapshotFile(drive, "Qwen/Qwen3-8B", testRev, "model.safetensors") + if err := os.WriteFile(snap, []byte(strings.Repeat("WEIGHTS", 100)), 0o644); err != nil { + t.Fatal(err) + } + + code, _, errOut := runCLI(t, "import", drive, "Qwen/Qwen3-8B", "--cache", hub) + if code == 0 { + t.Fatal("a damaged file was imported") + } + if _, err := os.Stat(snapshotFile(hub, "Qwen/Qwen3-8B", testRev, "model.safetensors")); err == nil { + t.Errorf("the damaged file reached the snapshot (stderr %q)", errOut) + } +} + +func TestExportThenImportRoundTrips(t *testing.T) { + newTree(t) + src, drive, dst := t.TempDir(), t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, src, "Qwen/Qwen3-8B", testRev, qwenFiles) + + code, out, errOut := runCLI(t, "export", "Qwen/Qwen3-8B", drive, "--cache", src) + if code != 0 { + t.Fatalf("export: exit %d\n%s\n%s", code, out, errOut) + } + _ = filepath.WalkDir(drive, func(p string, d os.DirEntry, _ error) error { + if d != nil && d.Type()&os.ModeSymlink != 0 { + t.Errorf("the drive holds a link: %s", p) + } + return nil + }) + + code, out, errOut = runCLI(t, "import", drive, "--all", "--cache", dst) + if code != 0 { + t.Fatalf("import: exit %d\n%s\n%s", code, out, errOut) + } + got, err := os.ReadFile(snapshotFile(dst, "Qwen/Qwen3-8B", testRev, "config.json")) + if err != nil || string(got) != qwenFiles["config.json"] { + t.Fatalf("round trip lost config.json: %v", err) + } +} + +func TestExportRefusesWhatIsNotComplete(t *testing.T) { + newTree(t) + src, drive := t.TempDir(), t.TempDir() + seedHub(t, src, "Qwen/Qwen3-8B", testRev, map[string]string{"config.json": "{}"}) + + code, _, errOut := runCLI(t, "export", "Qwen/Qwen3-8B", drive, "--cache", src) + if code == 0 || !strings.Contains(errOut, "incomplete") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } + code, _, errOut = runCLI(t, "export", "Qwen/Missing", drive, "--cache", src) + if code == 0 || !strings.Contains(errOut, "not in the cache") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } + code, _, errOut = runCLI(t, "export", "Qwen/Qwen3-8B", filepath.Join(drive, "nope"), "--cache", src) + if code == 0 || !strings.Contains(errOut, "mount the drive") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } +} + +func TestImportTakesAFolderWithARepoName(t *testing.T) { + newTree(t) + folder, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + for name, body := range qwenFiles { + if err := os.WriteFile(filepath.Join(folder, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + code, _, errOut := runCLI(t, "import", folder, "--cache", hub, "--all") + if code == 0 || !strings.Contains(errOut, "--repo") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } + code, _, errOut = runCLI(t, "import", folder, "--cache", hub, "--all", "--repo", "Qwen/Qwen3-8B") + if code == 0 || !strings.Contains(errOut, "revision") { + t.Fatalf("a folder with no commit was imported: exit %d, stderr %q", code, errOut) + } + code, out, errOut := runCLI(t, "import", folder, "--cache", hub, "--all", "--repo", "Qwen/Qwen3-8B", "--revision", testRev) + if code != 0 { + t.Fatalf("exit %d\n%s\n%s", code, out, errOut) + } + if _, err := os.Stat(snapshotFile(hub, "Qwen/Qwen3-8B", testRev, "model.safetensors")); err != nil { + t.Error(err) + } +} + +func TestImportForARecipeStagesWhereItsEngineReads(t *testing.T) { + newTree(t) + tested, err := recipe.LoadTested() + if err != nil { + t.Fatal(err) + } + var r recipe.TestedRecipe + for _, c := range tested.Recipes { + if c.Engine().HFCacheHostPath(c.Params) != "" && len(c.Models) > 0 { + r = c + break + } + } + if r.ID == "" { + t.Skip("no recipe pins models in a host cache") + } + home := t.TempDir() + t.Setenv("HOME", home) + drive := t.TempDir() + for _, m := range r.Models { + seedHub(t, filepath.Join(drive, "hub"), m.ID, m.Revision, qwenFiles) + } + + code, out, errOut := runCLI(t, "import", drive, "--recipe", r.ID) + if code != 0 { + t.Fatalf("exit %d\n%s\n%s", code, out, errOut) + } + for _, m := range r.Models { + dir := filepath.Join(home, engine.SnapshotPath(r.Engine().HFCacheHostPath(r.Params), engine.ModelRef{ID: m.ID, Revision: m.Revision})) + if _, err := os.Stat(filepath.Join(dir, "model.safetensors")); err != nil { + t.Errorf("%s is not where the recipe reads it: %v", m.ID, err) + } + } + if !strings.Contains(out, "staged for recipe "+r.ID) { + t.Errorf("output:\n%s", out) + } + + other := t.TempDir() + seedHub(t, filepath.Join(other, "hub"), r.Models[0].ID, testRev, qwenFiles) + if code, _, errOut := runCLI(t, "import", other, "--recipe", r.ID); code == 0 || !strings.Contains(errOut, "pins") { + t.Errorf("a revision the recipe does not pin was imported: exit %d, stderr %q", code, errOut) + } +} + +func TestSearchOfflineListsTheCachedModels(t *testing.T) { + configDir, _ := newTree(t) + hub := filepath.Join(t.TempDir(), "hub") + seedHub(t, hub, "Qwen/Qwen3-8B", testRev, qwenFiles) + cfg, err := os.ReadFile(filepath.Join(configDir, "config.env")) + if err != nil { + t.Fatal(err) + } + body := string(cfg) + "HF_HUB_CACHE=" + hub + "\nHF_HUB_OFFLINE=1\n" + if err := os.WriteFile(filepath.Join(configDir, "config.env"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + code, out, errOut := runCLI(t, "search", "qwen3", "--json") + if code != 0 { + t.Fatalf("exit %d\n%s\n%s", code, out, errOut) + } + if !strings.Contains(out, "Qwen/Qwen3-8B") || !strings.Contains(errOut, "local cache") { + t.Errorf("stdout:\n%s\nstderr:\n%s", out, errOut) + } +} diff --git a/internal/cli/profiles.go b/internal/cli/profiles.go index eaa5ab1..c06162d 100644 --- a/internal/cli/profiles.go +++ b/internal/cli/profiles.go @@ -255,6 +255,7 @@ func pullAndUp(ctx context.Context, cfg *config.Config, repo string, choice *res if err := plan.SetThinking(thinking, prov, caps); err != nil { return err } + control.KeepOffline(ctx, cfg, prov, &plan) for _, n := range plan.Notes { fmt.Println(n) } diff --git a/internal/cli/recipelaunch.go b/internal/cli/recipelaunch.go index 96295d9..590c73f 100644 --- a/internal/cli/recipelaunch.go +++ b/internal/cli/recipelaunch.go @@ -698,8 +698,8 @@ func checkStaged(r recipe.TestedRecipe, members []recipeMember, key string) erro id, rev, _ := strings.Cut(ref, "@") hint = append(hint, fmt.Sprintf(" HF_HOME=\"$HOME/%s\" hf download %s --revision %s", cache, id, rev)) } - return fmt.Errorf("%s needs %s on node %s, which is missing or incomplete in its Hugging Face cache; download it there once with:\n%s", - r.ID, strings.Join(missing, ", "), m.node.Name, strings.Join(hint, "\n")) + return fmt.Errorf("%s needs %s on node %s, which is missing or incomplete in its Hugging Face cache; download it there once with:\n%s\nor, from a drive that has it, run on that node:\n llm-helper import --recipe %s", + r.ID, strings.Join(missing, ", "), m.node.Name, strings.Join(hint, "\n"), r.ID) } return nil } diff --git a/internal/cli/root.go b/internal/cli/root.go index 2b44ec5..462fa0d 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -18,6 +18,8 @@ import ( "github.com/coder/llm-helper/internal/control" "github.com/coder/llm-helper/internal/engine" "github.com/coder/llm-helper/internal/hardware" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/hfcache" "github.com/coder/llm-helper/internal/modeltype" "github.com/coder/llm-helper/internal/plugin" "github.com/coder/llm-helper/internal/profile" @@ -83,11 +85,15 @@ func newRootCmd() *cobra.Command { // Recipes load last, because a local recipe may run on a local // engine. recipe.UseLocal(filepath.Join(config.ResolveDirs().Config, recipe.LocalDir)) - if cfg, err := loadConfig(); err == nil { + cfg, err := loadConfig() + if err != nil { + hf.UseCache(fallbackHub(), hfcache.Offline(hfcache.OSEnv)) + } else { useMetalProbe(cfg.VLLMBin()) // config.env can set CUDA_VISIBLE_DEVICES, and sizing every GPU // for a server allowed one of them overstates the budget. control.UseServerGPUs(cfg) + hf.UseCache(control.HubCache(cfg), control.HubOffline(cfg)) } } root.SetVersionTemplate(cmdName + " {{.Version}}\n") @@ -113,6 +119,8 @@ func newRootCmd() *cobra.Command { newSearchCmd(), newInspectCmd(), newInstallGGUFCmd(), + newImportCmd(), + newExportCmd(), newEditCmd(), newSetCmd(), newRemoveCmd(), diff --git a/internal/cli/root_internal_test.go b/internal/cli/root_internal_test.go index abd4abe..8e7b91b 100644 --- a/internal/cli/root_internal_test.go +++ b/internal/cli/root_internal_test.go @@ -13,7 +13,7 @@ func TestTheCommandSetIsPinned(t *testing.T) { // help is not here: cobra attaches it during Execute, not registration. // completion is not here either, because the root disables it. want := []string{ - "adopt", "coder-sync", "default", "doctor", "down", "edit", "engines", "hardware", "inspect", "install-gguf", "list", "logs", + "adopt", "coder-sync", "default", "doctor", "down", "edit", "engines", "export", "hardware", "import", "inspect", "install-gguf", "list", "logs", "metal-limit", "migrate", "model-types", "new", "node", "pool", "recipes", "remove", "restart", "router", "search", "serve", "set", "setup", "shard", "spark", "start", "status", "switch", "up", "upgrade", "version", "warmup", diff --git a/internal/cli/search.go b/internal/cli/search.go index 84807ea..5d68e90 100644 --- a/internal/cli/search.go +++ b/internal/cli/search.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "strconv" "strings" @@ -46,6 +47,9 @@ func runSearch(ctx context.Context, args []string, jsonOut bool) error { if err != nil { return fmt.Errorf("search request failed (network or rate limit): %w", err) } + if hf.Offline() { + fmt.Fprintln(os.Stderr, "Not asking the Hub (offline, or it did not answer): these are the matching models in the local cache.") + } if jsonOut { out := make([]resolve.Result, 0, len(models)) for _, m := range models { @@ -54,6 +58,10 @@ func runSearch(ctx context.Context, args []string, jsonOut bool) error { return printJSON(out) } if len(models) == 0 { + if hf.Offline() { + fmt.Println("No cached model matches. 'llm-helper import ' copies models from a drive.") + return nil + } fmt.Println("No text-generation models found. Try different terms.") return nil } diff --git a/internal/control/hubcache.go b/internal/control/hubcache.go new file mode 100644 index 0000000..5e174ff --- /dev/null +++ b/internal/control/hubcache.go @@ -0,0 +1,56 @@ +package control + +import ( + "context" + "os" + "os/exec" + "time" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/hfcache" + "github.com/coder/llm-helper/internal/resolve" +) + +// HubCache is the hub cache directory the server reads. The service sources +// config.env and secrets.env but not a login shell, so only those count. +func HubCache(cfg *config.Config) string { + home, _ := os.UserHomeDir() + var extra []string + if p, err := engine.LookupEngine(cfg.Lookup(engine.Key, "")); err == nil && p.HubCacheVar() != "" { + extra = append(extra, p.HubCacheVar()) + } + return hfcache.Dir(func(k string) string { return cfg.Lookup(k, "") }, home, extra...) +} + +// HubOffline reports whether HF_HUB_OFFLINE is on for the server, or for this +// process, which is what a command run by hand sees. +func HubOffline(cfg *config.Config) bool { + return hfcache.Offline(func(k string) string { return cfg.Lookup(k, "") }) || + hfcache.Offline(hfcache.OSEnv) +} + +// KeepOffline sets a new profile up to load from the cache when the Hub is +// not being used, asking the installed server what it accepts. +func KeepOffline(ctx context.Context, cfg *config.Config, prov engine.Provider, plan *resolve.Plan) { + if hf.Offline() { + plan.SetOffline(prov, func(args []string) string { return probeServer(ctx, cfg, prov, args) }) + } +} + +// probeServer runs the installed server with args and returns what it +// printed, "" when it cannot be run. +func probeServer(ctx context.Context, cfg *config.Config, prov engine.Provider, args []string) string { + bin := cfg.Lookup(prov.BinKey(), "") + if bin == "" { + bin, _ = exec.LookPath(prov.BinName()) + } + if bin == "" { + return "" + } + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + out, _ := exec.CommandContext(ctx, bin, args...).CombinedOutput() + return string(out) +} diff --git a/internal/control/hubcache_internal_test.go b/internal/control/hubcache_internal_test.go new file mode 100644 index 0000000..d802662 --- /dev/null +++ b/internal/control/hubcache_internal_test.go @@ -0,0 +1,148 @@ +package control + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/profile" + "github.com/coder/llm-helper/internal/resolve" +) + +func configWith(t *testing.T, body string) *config.Config { + t.Helper() + path := filepath.Join(t.TempDir(), "config.env") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + f, err := config.ParseEnvFile(path) + if err != nil { + t.Fatal(err) + } + return &config.Config{File: f} +} + +func TestHubCacheIsWhatTheServerSees(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("HF_HOME", "/shell/only") + t.Setenv("HF_HUB_CACHE", "") + + cases := []struct { + name, env, want string + }{ + {"the shell's HF_HOME is not the server's", "", filepath.Join(home, ".cache", "huggingface", "hub")}, + {"HF_HOME in config.env", "HF_HOME=/mnt/models\n", "/mnt/models/hub"}, + {"HF_HUB_CACHE beats HF_HOME", "HF_HOME=/a\nHF_HUB_CACHE=/b\n", "/b"}, + {"llama.cpp reads LLAMA_CACHE first", "PROVIDER=llamacpp\nHF_HUB_CACHE=/b\nLLAMA_CACHE=/c\n", "/c"}, + {"LLAMA_CACHE is ignored by other engines", "PROVIDER=vllm\nHF_HUB_CACHE=/b\nLLAMA_CACHE=/c\n", "/b"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := HubCache(configWith(t, c.env)); got != c.want { + t.Fatalf("HubCache = %q, want %q", got, c.want) + } + }) + } +} + +func TestHubOfflineReadsTheServerAndTheProcess(t *testing.T) { + t.Setenv("HF_HUB_OFFLINE", "") + t.Setenv("TRANSFORMERS_OFFLINE", "") + if HubOffline(configWith(t, "PORT=8001\n")) { + t.Fatal("offline with nothing set") + } + if !HubOffline(configWith(t, "HF_HUB_OFFLINE=1\n")) { + t.Fatal("config.env HF_HUB_OFFLINE=1 not seen") + } + t.Setenv("HF_HUB_OFFLINE", "1") + if !HubOffline(configWith(t, "PORT=8001\n")) { + t.Fatal("process HF_HUB_OFFLINE=1 not seen") + } +} + +// fakeServer is a stand-in for an installed server whose --help says help. +func fakeServer(t *testing.T, help string) string { + t.Helper() + bin := filepath.Join(t.TempDir(), "server") + script := "#!/bin/sh\ncat <<'EOF'\n" + help + "\nEOF\n" + if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return bin +} + +func TestKeepOfflineAsksTheInstalledServer(t *testing.T) { + hub := t.TempDir() + hf.UseCache(hub, true) + t.Cleanup(func() { hf.UseCache("", false) }) + prov, err := engine.Lookup("llamacpp") + if err != nil { + t.Fatal(err) + } + + for _, c := range []struct { + name, help, want string + }{ + {"lists the flag", " --offline no network", "--offline"}, + {"does not list it", " --port N", ""}, + } { + t.Run(c.name, func(t *testing.T) { + cfg := configWith(t, prov.BinKey()+"="+fakeServer(t, c.help)+"\n") + var plan resolve.Plan + KeepOffline(context.Background(), cfg, prov, &plan) + if plan.Opts.ExtraArgs != c.want { + t.Errorf("ExtraArgs = %q, want %q", plan.Opts.ExtraArgs, c.want) + } + }) + } +} + +func TestKeepOfflineDoesNothingOnline(t *testing.T) { + hf.UseCache("", false) + prov, _ := engine.Lookup("vllm") + var plan resolve.Plan + KeepOffline(context.Background(), configWith(t, "PORT=1\n"), prov, &plan) + if len(plan.Opts.Env) != 0 || plan.Opts.ExtraArgs != "" || len(plan.Notes) != 0 { + t.Errorf("plan changed: %+v", plan) + } +} + +func TestKeepOfflineWritesTheProfileEveryEngineNeeds(t *testing.T) { + hf.UseCache(t.TempDir(), true) + t.Cleanup(func() { hf.UseCache("", false) }) + for _, id := range []string{"vllm", "mlx", "llamacpp"} { + prov, err := engine.Lookup(id) + if err != nil { + t.Fatal(err) + } + cfg := configWith(t, prov.BinKey()+"="+fakeServer(t, "--offline")+"\n") + plan := resolve.Plan{Opts: profile.CreateOpts{Repo: "org/model", Provider: id, Source: "test"}} + KeepOffline(context.Background(), cfg, prov, &plan) + + dirs := config.Dirs{Config: t.TempDir(), Profiles: t.TempDir()} + if _, err := profile.Create(dirs, plan.Opts); err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(dirs.ProfileFile("model")) + if err != nil { + t.Fatal(err) + } + off := prov.OfflineSetting() + for _, line := range off.EnvLines() { + if !strings.Contains(string(body), line) { + t.Errorf("%s: profile lacks %q:\n%s", id, line, body) + } + } + for _, a := range off.Args { + if !strings.Contains(string(body), a) { + t.Errorf("%s: profile lacks %q:\n%s", id, a, body) + } + } + } +} diff --git a/internal/resolve/offline.go b/internal/resolve/offline.go new file mode 100644 index 0000000..a0e0442 --- /dev/null +++ b/internal/resolve/offline.go @@ -0,0 +1,43 @@ +package resolve + +import ( + "fmt" + "slices" + "strings" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/engine" +) + +// SetOffline keeps the profile's server off the network, the way its engine +// declares: environment lines, and arguments the installed server is asked +// about first. probe runs the server with the given arguments and returns its +// output. +func (p *Plan) SetOffline(prov engine.Provider, probe func(args []string) string) { + off := prov.OfflineSetting() + if off == nil { + p.Notes = append(p.Notes, fmt.Sprintf("%s has no offline setting; set what it needs in the profile.", prov.Name())) + return + } + for _, line := range off.EnvLines() { + key, _, _ := strings.Cut(line, "=") + if !slices.ContainsFunc(p.Opts.Env, func(l string) bool { return strings.HasPrefix(l, key+"=") }) { + p.Opts.Env = append(p.Opts.Env, line) + } + } + if len(off.Args) == 0 { + return + } + var out string + if len(off.ProbeArgs) > 0 && probe != nil { + out = probe(off.ProbeArgs) + } + if !off.ArgsAccepted(out) { + p.Notes = append(p.Notes, fmt.Sprintf("The installed %s does not list %s, so it was not added; a newer build can load a cached model offline.", prov.Name(), off.Args[0])) + return + } + if words, err := config.SplitArgs(p.Opts.ExtraArgs); err == nil && slices.Contains(words, off.Args[0]) { + return + } + p.Opts.ExtraArgs = strings.TrimSpace(p.Opts.ExtraArgs + " " + strings.Join(off.Args, " ")) +} diff --git a/internal/resolve/offline_internal_test.go b/internal/resolve/offline_internal_test.go new file mode 100644 index 0000000..a39dc52 --- /dev/null +++ b/internal/resolve/offline_internal_test.go @@ -0,0 +1,91 @@ +package resolve + +import ( + "slices" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/engine" +) + +func offlinePlan(t *testing.T, id, extra string, probeOut string) (Plan, []string) { + t.Helper() + prov, err := engine.Lookup(id) + if err != nil { + t.Fatal(err) + } + p := Plan{} + p.Opts.ExtraArgs = extra + var probed []string + p.SetOffline(prov, func(args []string) string { + probed = args + return probeOut + }) + return p, probed +} + +func TestSetOfflineUsesWhatEachEngineDeclares(t *testing.T) { + for _, id := range engine.ProcessNames() { + prov, _ := engine.Lookup(id) + off := prov.OfflineSetting() + if off == nil { + continue + } + help := strings.Join(off.Args, " ") + " listed" + p, _ := offlinePlan(t, id, "--port 1", help) + for _, line := range off.EnvLines() { + if !slices.Contains(p.Opts.Env, line) { + t.Errorf("%s: env %q missing from %v", id, line, p.Opts.Env) + } + } + for _, a := range off.Args { + if !strings.Contains(p.Opts.ExtraArgs, a) { + t.Errorf("%s: %q missing from %q", id, a, p.Opts.ExtraArgs) + } + } + if !strings.HasPrefix(p.Opts.ExtraArgs, "--port 1") { + t.Errorf("%s: existing args lost: %q", id, p.Opts.ExtraArgs) + } + } +} + +func TestSetOfflineSetsHFHubOfflineForPythonEngines(t *testing.T) { + for _, id := range []string{"vllm", "mlx"} { + p, probed := offlinePlan(t, id, "", "") + if !slices.Contains(p.Opts.Env, "HF_HUB_OFFLINE=1") { + t.Errorf("%s: env = %v", id, p.Opts.Env) + } + if p.Opts.ExtraArgs != "" || probed != nil { + t.Errorf("%s: args %q probed %v", id, p.Opts.ExtraArgs, probed) + } + } +} + +func TestSetOfflineAddsTheFlagOnlyWhenTheServerListsIt(t *testing.T) { + p, probed := offlinePlan(t, "llamacpp", "-ngl 99", " --offline work offline\n --port N") + if p.Opts.ExtraArgs != "-ngl 99 --offline" || !slices.Equal(probed, []string{"--help"}) { + t.Errorf("args %q probed %v", p.Opts.ExtraArgs, probed) + } + + p, _ = offlinePlan(t, "llamacpp", "-ngl 99", " --port N") + if p.Opts.ExtraArgs != "-ngl 99" { + t.Errorf("flag added to a server that does not list it: %q", p.Opts.ExtraArgs) + } + if len(p.Notes) != 1 || !strings.Contains(p.Notes[0], "--offline") { + t.Errorf("notes = %v", p.Notes) + } +} + +func TestSetOfflineDoesNotRepeatWhatIsThere(t *testing.T) { + p, _ := offlinePlan(t, "llamacpp", "--offline", "--offline") + if p.Opts.ExtraArgs != "--offline" { + t.Errorf("args = %q", p.Opts.ExtraArgs) + } + prov, _ := engine.Lookup("vllm") + q := Plan{} + q.Opts.Env = []string{"HF_HUB_OFFLINE=0"} + q.SetOffline(prov, nil) + if len(q.Opts.Env) != 1 || q.Opts.Env[0] != "HF_HUB_OFFLINE=0" { + t.Errorf("env = %v; a setting already in the profile must win", q.Opts.Env) + } +} diff --git a/internal/web/models.go b/internal/web/models.go index d5fce49..aca8c30 100644 --- a/internal/web/models.go +++ b/internal/web/models.go @@ -301,6 +301,7 @@ func (s *Server) handlePull(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } + control.KeepOffline(r.Context(), s.cfg, prov, &plan) out.Notes = append(out.Notes, plan.Notes...) if _, err := profile.Create(s.cfg.Dirs, plan.Opts); err != nil { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) From feed37ab0e119e9588d746bed8ac42e3071d1470 Mon Sep 17 00:00:00 2001 From: DevelopmentCats Date: Mon, 16 Nov 2026 16:30:45 +0000 Subject: [PATCH 5/5] docs: add offline and external drive guide Explains moving models on a drive, which cache the server reads, and what a profile made offline sets. --- CHANGELOG.md | 16 ++++++++ docs/manifest.json | 5 +++ docs/offline.md | 96 ++++++++++++++++++++++++++++++++++++++++++++++ docs/providers.md | 2 + 4 files changed, 119 insertions(+) create mode 100644 docs/offline.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f7c38..79cbac3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,22 @@ the fact. ### Added +- Models from an external drive: `llm-helper export ... ` + copies models from the Hugging Face cache onto a drive, and + `llm-helper import ` copies them into the cache the server + reads, from a drive written by `export`, a copy of a cache, an + `hf download --local-dir` folder or a model folder. Files are verified + against their hashes as they are copied, an interrupted copy resumes, + incomplete models and a drive that is too small or FAT32 are refused, and + `import --recipe ` stages the models a tested recipe pins. See the new + "Offline machines and external drives" page. +- Offline machines: with `HF_HUB_OFFLINE=1` in `config.env`, or once the Hub + does not answer, `new`, `inspect` and `search` read the local cache instead + of the Hub, and a new profile tells its server to stay off the network + (`HF_HUB_OFFLINE=1` for vLLM and MLX, `--offline` for a llama.cpp that lists + it). Engines declare this in `offline` and `hub_cache_env`. +- `doctor` prints the model cache the server reads and its free space, and + warns when the shell's `HF_HOME` or `HF_HUB_CACHE` points elsewhere. - Warm-up: a profile's `WARMUP_FILE` names a chat request that llm-helper sends when `up`, `restart` or `switch` sees the server healthy, and that the router sends when a server comes back, so the server has the client's fixed prompt cached diff --git a/docs/manifest.json b/docs/manifest.json index 184c48d..66bb198 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -36,6 +36,11 @@ "description": "Finding a model that fits this machine, and choosing a quant.", "path": "./models.md" }, + { + "title": "Offline machines and external drives", + "description": "Move models between machines on a drive, and serve a machine that cannot reach the Hugging Face Hub.", + "path": "./offline.md" + }, { "title": "Model types", "description": "What a model's architecture needs to run correctly, and how to add or change it.", diff --git a/docs/offline.md b/docs/offline.md new file mode 100644 index 0000000..f79b7b8 --- /dev/null +++ b/docs/offline.md @@ -0,0 +1,96 @@ +--- +title: Offline machines and external drives +description: Move models between machines on a drive, and serve a machine that cannot reach the Hugging Face Hub. +--- + +A machine with no route to the Hugging Face Hub can still serve any model that is in its Hugging Face cache. +`llm-helper export` copies models from a machine that has them onto a drive, and `llm-helper import` copies them into the cache of the machine that does not. + +## Move a model on a drive + +On a machine that has the model, copy it onto the mounted drive: + +```sh +llm-helper export Qwen/Qwen3-8B-FP8 /media/usb +``` + +On the offline machine, copy it into the cache the server reads, then create the profile: + +```sh +llm-helper import /media/usb Qwen/Qwen3-8B-FP8 +llm-helper new Qwen/Qwen3-8B-FP8 +``` + +`import` lists the models it finds when you name none, and asks which to copy. +`--all` copies every complete model, and `--dry-run` shows what would be copied and stops. + +Both commands: + +- Refuse a model that is incomplete, because it would not load on the other machine. A missing file, a link to a blob that is not there, no weights, or a shard an index names but the folder lacks all count. +- Check every file against its recorded hash as it is copied, and fail on a damaged one without keeping it. +- Resume after an interruption, skip files already there, and never overwrite what is in the cache. +- Check the free space first, and say how much is needed. + +`export` writes a plain copy of each file under `hub/` on the drive, with no links, so any filesystem can hold it. +FAT32 cannot hold a file over 4 GiB, so `export` refuses a model with one: format the drive as exFAT, or use ext4 or APFS. + +## What `import` reads + +`import` finds a Hugging Face cache at the path you give, or under it in `hub/`, `huggingface/hub/` or `.cache/huggingface/hub/`. +That covers a drive written by `export`, and a copy of `~/.cache/huggingface`. +A cache that shares one blob between several snapshots is read as it is. + +Two other layouts record no repo id, so name it: + +```sh +llm-helper import /media/usb/qwen --repo Qwen/Qwen3-8B-FP8 --revision +``` + +- A folder written by `hf download --local-dir` needs `--repo`. Its commit is read from the folder. +- A plain model folder needs `--repo` and a commit. A git clone supplies its own, and any other folder needs `--revision`, because llm-helper does not invent one. + +## Which cache the server reads + +`import` and `export` work on the cache the server reads, which is the one `llm-helper doctor` prints under `MODEL CACHE`. +The service sources `config.env` and `secrets.env`, not your login shell, so an `HF_HOME` exported in a shell is not the server's. +In order, the first of these that `config.env` sets wins: + +1. `LLAMA_CACHE`, for llama.cpp only. +2. `HF_HUB_CACHE`, then `HUGGINGFACE_HUB_CACHE`. +3. `HF_HOME`, with `/hub` added. +4. `XDG_CACHE_HOME`, with `/huggingface/hub` added. +5. `~/.cache/huggingface/hub`. + +`doctor` warns when your shell's `HF_HOME` or `HF_HUB_CACHE` points somewhere else. +`--cache ` names a different cache for one command. + +## Serve with no network + +Set `HF_HUB_OFFLINE=1` in `config.env` on a machine that never reaches the Hub. +`llm-helper new`, `inspect` and `search` then read the cache instead of the Hub: + +- `new` and `inspect` read the model's config, chat template and file list from the cached snapshot. +- `search` lists the cached models that match your terms. +- A model that is not in the cache is reported as not cached, instead of failing on a network error. + +Without the setting, the same happens once a request cannot reach the Hub, and the Hub is tried again a minute later. + +A profile made while offline tells its server to stay off the network, as the engine defines it: + +| Engine | Setting in the new profile | +| --------- | ---------------------------------------------------------------------- | +| vLLM | `HF_HUB_OFFLINE=1` | +| MLX | `HF_HUB_OFFLINE=1` | +| llama.cpp | `--offline` in `EXTRA_ARGS`, only when the installed `--help` lists it | + +For llama.cpp, set `LLAMA_CACHE` in `config.env` if the cache is not the default one, because `llama-server -hf` reads that first. +A build that does not list `--offline` is left without it, and `new` says so. +Existing profiles are not changed: add the setting to one with `llm-helper edit `. + +## Tested recipes + +A recipe launches containers that load a pinned revision from each node's cache and refuse to start when it is missing. +`llm-helper import --recipe ` copies exactly the models the recipe pins into the cache its engine reads, and fails when the drive has another revision. +Run it on each node. + +See [Models and quantizations](./models.md) for choosing a model, and [Recipes](./recipes.md) for the launch configurations. diff --git a/docs/providers.md b/docs/providers.md index bbebfdd..7ef1196 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -283,6 +283,8 @@ A definition holds `schema_version` and the engine's fields, and says: - `health`: the path that answers 200, and whether it does so only once the model is loaded (`ready`) or as soon as the process listens (`live`). - `capabilities`: the few differences shared code acts on, such as `vllm_args` for vLLM's sizing and flag checks. Launching and health checks act on a capability, never on an engine's name. `chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; see [Model types](./model-types.md). +- `hub_cache_env`: the variable that sets the hub cache directly, which the server reads before `HF_HUB_CACHE`. `llm-helper import` and `doctor` look there first. +- `offline`: how to keep the server off the network when a new profile is made for a machine that does not use the Hub, as `env` entries and `args`. `probe_args` is run against the installed server first, and `args` are added only when its output lists the first of them. See [Offline machines and external drives](./offline.md). - `notes`: optional sentences `inspect` prints for some kinds of model. `hybrid` is shown for a model whose layers are not all attention, such as one with linear-attention or Mamba layers.