diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f7c38..79cbac3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,22 @@ the fact. ### Added +- Models from an external drive: `llm-helper export ... ` + copies models from the Hugging Face cache onto a drive, and + `llm-helper import ` copies them into the cache the server + reads, from a drive written by `export`, a copy of a cache, an + `hf download --local-dir` folder or a model folder. Files are verified + against their hashes as they are copied, an interrupted copy resumes, + incomplete models and a drive that is too small or FAT32 are refused, and + `import --recipe ` stages the models a tested recipe pins. See the new + "Offline machines and external drives" page. +- Offline machines: with `HF_HUB_OFFLINE=1` in `config.env`, or once the Hub + does not answer, `new`, `inspect` and `search` read the local cache instead + of the Hub, and a new profile tells its server to stay off the network + (`HF_HUB_OFFLINE=1` for vLLM and MLX, `--offline` for a llama.cpp that lists + it). Engines declare this in `offline` and `hub_cache_env`. +- `doctor` prints the model cache the server reads and its free space, and + warns when the shell's `HF_HOME` or `HF_HUB_CACHE` points elsewhere. - Warm-up: a profile's `WARMUP_FILE` names a chat request that llm-helper sends when `up`, `restart` or `switch` sees the server healthy, and that the router sends when a server comes back, so the server has the client's fixed prompt cached diff --git a/catalogue/engines/README.md b/catalogue/engines/README.md index 39fcd44..a0c2a8b 100644 --- a/catalogue/engines/README.md +++ b/catalogue/engines/README.md @@ -6,6 +6,7 @@ Each file here is one inference server llm-helper can launch, named after the en To add an engine, copy the closest file to `.json` and change it. No Go code is needed: shared code acts on the engine's `capabilities`, never on its name. `chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; leave it out if the engine has none. +`hub_cache_env` names the variable that sets the hub cache directly, and `offline` says how to keep the server off the network (`env`, `args`, and `probe_args` to check the installed server accepts them); leave either out if the engine has no such setting. `notes.hybrid` is a sentence `inspect` prints for a model whose layers are not all attention; leave it out if there is nothing to say. The file is checked when it loads, and the tests in `internal/engine` render every file here for every rank. diff --git a/catalogue/engines/llamacpp.json b/catalogue/engines/llamacpp.json index a3040d1..97d4aec 100644 --- a/catalogue/engines/llamacpp.json +++ b/catalogue/engines/llamacpp.json @@ -31,6 +31,11 @@ "with_key": "0.0.0.0", "without_key": "127.0.0.1" }, + "hub_cache_env": "LLAMA_CACHE", + "offline": { + "args": ["--offline"], + "probe_args": ["--help"] + }, "api_key": "env:LLAMA_API_KEY", "health": { "path": "/health", diff --git a/catalogue/engines/mlx.json b/catalogue/engines/mlx.json index 39aec0b..20f2508 100644 --- a/catalogue/engines/mlx.json +++ b/catalogue/engines/mlx.json @@ -13,6 +13,9 @@ "names": ["mlx_lm.server"] }, "serve": ["{bin}", "--model", "{model}", "--port", "{port}"], + "offline": { + "env": { "HF_HUB_OFFLINE": "1" } + }, "api_key": "none", "health": { "path": "/health", diff --git a/catalogue/engines/vllm.json b/catalogue/engines/vllm.json index baf3e3c..f7470e8 100644 --- a/catalogue/engines/vllm.json +++ b/catalogue/engines/vllm.json @@ -26,6 +26,9 @@ "--enable-prompt-tokens-details" ], "optional_args": [["--kv-cache-memory-bytes", "{kv_cache_bytes}"]], + "offline": { + "env": { "HF_HUB_OFFLINE": "1" } + }, "api_key": "flag:--api-key", "health": { "path": "/health", diff --git a/docs/manifest.json b/docs/manifest.json index 0cd09ff..66bb198 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -36,6 +36,11 @@ "description": "Finding a model that fits this machine, and choosing a quant.", "path": "./models.md" }, + { + "title": "Offline machines and external drives", + "description": "Move models between machines on a drive, and serve a machine that cannot reach the Hugging Face Hub.", + "path": "./offline.md" + }, { "title": "Model types", "description": "What a model's architecture needs to run correctly, and how to add or change it.", @@ -133,6 +138,11 @@ } ] }, + { + "title": "llm-helper export", + "description": "Copy models from the Hugging Face cache onto an external drive", + "path": "./reference/cli/export.md" + }, { "title": "llm-helper hardware", "description": "List hardware definitions and report this machine's hardware", @@ -150,6 +160,11 @@ } ] }, + { + "title": "llm-helper import", + "description": "Copy models from an external drive into the Hugging Face cache", + "path": "./reference/cli/import.md" + }, { "title": "llm-helper inspect", "description": "Show a model's license, weight size against this machine, and detected flags", diff --git a/docs/offline.md b/docs/offline.md new file mode 100644 index 0000000..f79b7b8 --- /dev/null +++ b/docs/offline.md @@ -0,0 +1,96 @@ +--- +title: Offline machines and external drives +description: Move models between machines on a drive, and serve a machine that cannot reach the Hugging Face Hub. +--- + +A machine with no route to the Hugging Face Hub can still serve any model that is in its Hugging Face cache. +`llm-helper export` copies models from a machine that has them onto a drive, and `llm-helper import` copies them into the cache of the machine that does not. + +## Move a model on a drive + +On a machine that has the model, copy it onto the mounted drive: + +```sh +llm-helper export Qwen/Qwen3-8B-FP8 /media/usb +``` + +On the offline machine, copy it into the cache the server reads, then create the profile: + +```sh +llm-helper import /media/usb Qwen/Qwen3-8B-FP8 +llm-helper new Qwen/Qwen3-8B-FP8 +``` + +`import` lists the models it finds when you name none, and asks which to copy. +`--all` copies every complete model, and `--dry-run` shows what would be copied and stops. + +Both commands: + +- Refuse a model that is incomplete, because it would not load on the other machine. A missing file, a link to a blob that is not there, no weights, or a shard an index names but the folder lacks all count. +- Check every file against its recorded hash as it is copied, and fail on a damaged one without keeping it. +- Resume after an interruption, skip files already there, and never overwrite what is in the cache. +- Check the free space first, and say how much is needed. + +`export` writes a plain copy of each file under `hub/` on the drive, with no links, so any filesystem can hold it. +FAT32 cannot hold a file over 4 GiB, so `export` refuses a model with one: format the drive as exFAT, or use ext4 or APFS. + +## What `import` reads + +`import` finds a Hugging Face cache at the path you give, or under it in `hub/`, `huggingface/hub/` or `.cache/huggingface/hub/`. +That covers a drive written by `export`, and a copy of `~/.cache/huggingface`. +A cache that shares one blob between several snapshots is read as it is. + +Two other layouts record no repo id, so name it: + +```sh +llm-helper import /media/usb/qwen --repo Qwen/Qwen3-8B-FP8 --revision +``` + +- A folder written by `hf download --local-dir` needs `--repo`. Its commit is read from the folder. +- A plain model folder needs `--repo` and a commit. A git clone supplies its own, and any other folder needs `--revision`, because llm-helper does not invent one. + +## Which cache the server reads + +`import` and `export` work on the cache the server reads, which is the one `llm-helper doctor` prints under `MODEL CACHE`. +The service sources `config.env` and `secrets.env`, not your login shell, so an `HF_HOME` exported in a shell is not the server's. +In order, the first of these that `config.env` sets wins: + +1. `LLAMA_CACHE`, for llama.cpp only. +2. `HF_HUB_CACHE`, then `HUGGINGFACE_HUB_CACHE`. +3. `HF_HOME`, with `/hub` added. +4. `XDG_CACHE_HOME`, with `/huggingface/hub` added. +5. `~/.cache/huggingface/hub`. + +`doctor` warns when your shell's `HF_HOME` or `HF_HUB_CACHE` points somewhere else. +`--cache ` names a different cache for one command. + +## Serve with no network + +Set `HF_HUB_OFFLINE=1` in `config.env` on a machine that never reaches the Hub. +`llm-helper new`, `inspect` and `search` then read the cache instead of the Hub: + +- `new` and `inspect` read the model's config, chat template and file list from the cached snapshot. +- `search` lists the cached models that match your terms. +- A model that is not in the cache is reported as not cached, instead of failing on a network error. + +Without the setting, the same happens once a request cannot reach the Hub, and the Hub is tried again a minute later. + +A profile made while offline tells its server to stay off the network, as the engine defines it: + +| Engine | Setting in the new profile | +| --------- | ---------------------------------------------------------------------- | +| vLLM | `HF_HUB_OFFLINE=1` | +| MLX | `HF_HUB_OFFLINE=1` | +| llama.cpp | `--offline` in `EXTRA_ARGS`, only when the installed `--help` lists it | + +For llama.cpp, set `LLAMA_CACHE` in `config.env` if the cache is not the default one, because `llama-server -hf` reads that first. +A build that does not list `--offline` is left without it, and `new` says so. +Existing profiles are not changed: add the setting to one with `llm-helper edit `. + +## Tested recipes + +A recipe launches containers that load a pinned revision from each node's cache and refuse to start when it is missing. +`llm-helper import --recipe ` copies exactly the models the recipe pins into the cache its engine reads, and fails when the drive has another revision. +Run it on each node. + +See [Models and quantizations](./models.md) for choosing a model, and [Recipes](./recipes.md) for the launch configurations. diff --git a/docs/providers.md b/docs/providers.md index bbebfdd..7ef1196 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -283,6 +283,8 @@ A definition holds `schema_version` and the engine's fields, and says: - `health`: the path that answers 200, and whether it does so only once the model is loaded (`ready`) or as soon as the process listens (`live`). - `capabilities`: the few differences shared code acts on, such as `vllm_args` for vLLM's sizing and flag checks. Launching and health checks act on a capability, never on an engine's name. `chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; see [Model types](./model-types.md). +- `hub_cache_env`: the variable that sets the hub cache directly, which the server reads before `HF_HUB_CACHE`. `llm-helper import` and `doctor` look there first. +- `offline`: how to keep the server off the network when a new profile is made for a machine that does not use the Hub, as `env` entries and `args`. `probe_args` is run against the installed server first, and `args` are added only when its output lists the first of them. See [Offline machines and external drives](./offline.md). - `notes`: optional sentences `inspect` prints for some kinds of model. `hybrid` is shown for a model whose layers are not all attention, such as one with linear-attention or Mamba layers. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 4a50162..3dd6bcb 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -31,7 +31,9 @@ sharding, replica pools, recipes, optional Coder Agents sync). | [`down`](cli/down.md) | Stop serving a profile (the only one running when none is named) | | [`edit`](cli/edit.md) | Open a profile in $EDITOR and validate the result | | [`engines`](cli/engines.md) | List the inference engines a profile or recipe can run on | +| [`export`](cli/export.md) | Copy models from the Hugging Face cache onto an external drive | | [`hardware`](cli/hardware.md) | List hardware definitions and report this machine's hardware | +| [`import`](cli/import.md) | Copy models from an external drive into the Hugging Face cache | | [`inspect`](cli/inspect.md) | Show a model's license, weight size against this machine, and detected flags | | [`install-gguf`](cli/install-gguf.md) | Install the plugin that lets vLLM load GGUF models | | [`list`](cli/list.md) | List saved model profiles | diff --git a/docs/reference/cli/export.md b/docs/reference/cli/export.md new file mode 100644 index 0000000..4f697b0 --- /dev/null +++ b/docs/reference/cli/export.md @@ -0,0 +1,39 @@ +--- +# Code generated by go run ./scripts/gendocs. DO NOT EDIT. +title: "llm-helper export" +description: "Copy models from the Hugging Face cache onto an external drive" +--- + + + +Copy models from the Hugging Face cache the server reads onto an external drive, to +carry to a machine with no internet. The drive gets a plain copy of each file in a +hub/ folder, which any filesystem holds, and 'llm-helper import' reads it back. + +Only complete models are copied, and each file is checked against its recorded +hash as it is copied. FAT32 cannot hold a file over 4 GiB; format the drive as +exFAT. + +## Usage + +```console +llm-helper export ... [flags] +``` + +## Options + +### --cache + +| | | +| ---- | -------- | +| Type | `string` | + +hub cache directory to copy from (default: the one the server reads) + +### --dry-run + +| | | +| ---- | ------ | +| Type | `bool` | + +show what would be copied and stop diff --git a/docs/reference/cli/import.md b/docs/reference/cli/import.md new file mode 100644 index 0000000..684e33c --- /dev/null +++ b/docs/reference/cli/import.md @@ -0,0 +1,75 @@ +--- +# Code generated by go run ./scripts/gendocs. DO NOT EDIT. +title: "llm-helper import" +description: "Copy models from an external drive into the Hugging Face cache" +--- + + + +Copy models from an external drive, or any folder, into the Hugging Face cache the +server reads, so 'llm-helper new ' loads them with no download. + +The drive can hold a Hugging Face cache (also one made by 'llm-helper export'), an +'hf download --local-dir' folder, or a model folder. The last two do not record +which repo they are, so name it with --repo (and --revision unless the folder is +a git clone). + +Every file is checked against its recorded hash as it is copied, an interrupted +import resumes where it stopped, and nothing already in the cache is overwritten. +With --recipe the models go where that recipe's engine reads them. + +## Usage + +```console +llm-helper import [org/model...] [flags] +``` + +## Options + +### --all + +| | | +| ---- | ------ | +| Type | `bool` | + +import every complete model on the drive + +### --cache + +| | | +| ---- | -------- | +| Type | `string` | + +hub cache directory to import into (default: the one the server reads) + +### --dry-run + +| | | +| ---- | ------ | +| Type | `bool` | + +show what would be copied and stop + +### --recipe + +| | | +| ---- | -------- | +| Type | `string` | + +import the models a tested recipe pins, into the cache its engine reads + +### --repo + +| | | +| ---- | -------- | +| Type | `string` | + +org/model of a folder that is not a Hugging Face cache + +### --revision + +| | | +| ---- | -------- | +| Type | `string` | + +commit of a folder that is not a Hugging Face cache or git clone diff --git a/internal/cli/detectcmds.go b/internal/cli/detectcmds.go index 1ef0b23..b9b2195 100644 --- a/internal/cli/detectcmds.go +++ b/internal/cli/detectcmds.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "strings" "github.com/spf13/cobra" @@ -15,6 +16,7 @@ import ( "github.com/coder/llm-helper/internal/engine" "github.com/coder/llm-helper/internal/hardware" "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/hfcache" "github.com/coder/llm-helper/internal/modeltype" "github.com/coder/llm-helper/internal/plugin" "github.com/coder/llm-helper/internal/profile" @@ -34,6 +36,7 @@ type doctorReport struct { Toolchain []toolchain.Check `json:"toolchain"` Coder coderStatus `json:"coder_sync"` HFToken hfTokenStatus `json:"hf_token"` + HubCache hubCacheStatus `json:"hub_cache"` // WiredLimit is the macOS GPU wired memory cap, nil unless the Mac // serves on the GPU through vllm-metal. WiredLimit *wiredLimit `json:"wired_limit,omitempty"` @@ -69,6 +72,17 @@ type doctorInstance struct { Restarts int `json:"restarts"` } +// hubCacheStatus is the model cache the server reads. +type hubCacheStatus struct { + Path string `json:"path"` + FreeBytes int64 `json:"free_bytes,omitempty"` + Offline bool `json:"offline"` + // ShellPath is the cache this shell would use when it is not the + // server's: models downloaded from here land somewhere the server never + // looks. + ShellPath string `json:"shell_path,omitempty"` +} + type coderStatus struct { Configured bool `json:"configured"` URL string `json:"url,omitempty"` @@ -167,6 +181,12 @@ func gatherDoctor() doctorReport { r.Toolchain = toolchain.Gather(vllmBin) r.Coder = gatherCoderStatus(cfg) r.HFToken = gatherHFToken(cfg) + r.HubCache = gatherHubCache(cfg) + if r.HubCache.ShellPath != "" { + r.Warnings = append(r.Warnings, fmt.Sprintf( + "The server reads models from %s, but this shell's HF_HOME or HF_HUB_CACHE points at %s, so 'hf download' from here is not seen by it. Set HF_HOME in config.env, or download with HF_HUB_CACHE=%s.", + r.HubCache.Path, r.HubCache.ShellPath, r.HubCache.Path)) + } r.WiredLimit = gatherWiredLimit(cfg, r.Hardware) running := 0 if r.Config != nil { @@ -269,6 +289,20 @@ func pluginNamed(name string) plugin.Plugin { return nil } +// gatherHubCache reports the cache the server reads, which comes from +// config.env rather than this shell. +func gatherHubCache(cfg *config.Config) hubCacheStatus { + st := hubCacheStatus{Path: fallbackHub(), Offline: hfcache.Offline(hfcache.OSEnv)} + if cfg != nil { + st.Path, st.Offline = control.HubCache(cfg), control.HubOffline(cfg) + } + st.FreeBytes, _ = hfcache.Free(st.Path) + if shell := fallbackHub(); cfg != nil && filepath.Clean(shell) != filepath.Clean(st.Path) { + st.ShellPath = shell + } + return st +} + // gatherHFToken reports whether a HuggingFace token is available (gated // models can't be pulled without one) and whether the Hub accepts it. func gatherHFToken(cfg *config.Config) hfTokenStatus { @@ -535,6 +569,16 @@ func renderDoctor(r doctorReport, yes bool) { fmt.Println() } + fmt.Println("MODEL CACHE") + fmt.Printf(" path: %s\n", r.HubCache.Path) + if r.HubCache.FreeBytes > 0 { + fmt.Printf(" free: %s\n", hfcache.FormatBytes(r.HubCache.FreeBytes)) + } + if r.HubCache.Offline { + fmt.Println(" offline: yes -- models load from the cache and the Hub is not asked") + } + fmt.Println() + fmt.Println("RUNNING SERVERS") if len(r.Servers) == 0 && len(r.Containers) == 0 { fmt.Println(" no vLLM processes or containers found") diff --git a/internal/cli/doctor_internal_test.go b/internal/cli/doctor_internal_test.go index e77816f..716a1a8 100644 --- a/internal/cli/doctor_internal_test.go +++ b/internal/cli/doctor_internal_test.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strconv" "strings" "testing" @@ -718,3 +719,35 @@ func TestOverlapWarningsNameTheIgnoredDefinition(t *testing.T) { t.Errorf("an overlap on a PCI device: %q", w) } } + +// The server sources config.env, not the login shell, so doctor reports the +// cache the server reads and says when this shell would use another. +func TestGatherDoctorReportsTheServersModelCache(t *testing.T) { + dirs := newTree(t) + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("HF_HUB_CACHE", "") + t.Setenv("HF_HUB_OFFLINE", "") + t.Setenv("HF_HOME", filepath.Join(t.TempDir(), "shell")) + models := filepath.Join(t.TempDir(), "models") + writeConfig(t, dirs, "PORT=9101\nHF_HOME="+models+"\nHF_HUB_OFFLINE=1\n") + + r := gatherDoctor() + if want := filepath.Join(models, "hub"); r.HubCache.Path != want { + t.Errorf("path = %q, want %q", r.HubCache.Path, want) + } + if !r.HubCache.Offline { + t.Error("HF_HUB_OFFLINE in config.env was not reported") + } + if want := filepath.Join(os.Getenv("HF_HOME"), "hub"); r.HubCache.ShellPath != want { + t.Errorf("shell path = %q, want %q", r.HubCache.ShellPath, want) + } + if !slices.ContainsFunc(r.Warnings, func(w string) bool { return strings.Contains(w, "this shell") }) { + t.Errorf("no warning about the shell's cache: %v", r.Warnings) + } + + writeConfig(t, dirs, "PORT=9101\nHF_HOME="+os.Getenv("HF_HOME")+"\n") + if r := gatherDoctor(); r.HubCache.ShellPath != "" { + t.Errorf("shell path = %q when it is the server's", r.HubCache.ShellPath) + } +} diff --git a/internal/cli/engines_internal_test.go b/internal/cli/engines_internal_test.go index c33982f..478eaed 100644 --- a/internal/cli/engines_internal_test.go +++ b/internal/cli/engines_internal_test.go @@ -81,7 +81,7 @@ func TestCheckStagedNamesWhatIsMissing(t *testing.T) { t.Fatal("a missing drafter passed the staging check") } msg := err.Error() - for _, want := range []string{drafter.ID + "@" + drafter.Revision, "node head", "hf download " + drafter.ID + " --revision " + drafter.Revision} { + for _, want := range []string{drafter.ID + "@" + drafter.Revision, "node head", "hf download " + drafter.ID + " --revision " + drafter.Revision, "llm-helper import --recipe " + r.ID} { if !strings.Contains(msg, want) { t.Errorf("error %q does not mention %q", msg, want) } diff --git a/internal/cli/hfcache.go b/internal/cli/hfcache.go new file mode 100644 index 0000000..2764f24 --- /dev/null +++ b/internal/cli/hfcache.go @@ -0,0 +1,344 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/spf13/cobra" + + "github.com/coder/llm-helper/internal/control" + "github.com/coder/llm-helper/internal/hfcache" + "github.com/coder/llm-helper/internal/recipe" +) + +type importOptions struct { + all bool + dryRun bool + cache string + recipeID string + repo string + revision string +} + +func newImportCmd() *cobra.Command { + var o importOptions + c := &cobra.Command{ + Use: "import [org/model...]", + Short: "Copy models from an external drive into the Hugging Face cache", + Long: "Copy models from an external drive, or any folder, into the Hugging Face cache the\n" + + "server reads, so 'llm-helper new ' loads them with no download.\n\n" + + "The drive can hold a Hugging Face cache (also one made by 'llm-helper export'), an\n" + + "'hf download --local-dir' folder, or a model folder. The last two do not record\n" + + "which repo they are, so name it with --repo (and --revision unless the folder is\n" + + "a git clone).\n\n" + + "Every file is checked against its recorded hash as it is copied, an interrupted\n" + + "import resumes where it stopped, and nothing already in the cache is overwritten.\n" + + "With --recipe the models go where that recipe's engine reads them.", + Example: " llm-helper import /media/usb\n" + + " llm-helper import /media/usb Qwen/Qwen3-8B-FP8\n" + + " llm-helper import /media/usb --recipe ", + GroupID: groupModels, + Args: cobra.MinimumNArgs(1), + RunE: func(_ *cobra.Command, args []string) error { return runImport(args[0], args[1:], o) }, + } + f := c.Flags() + f.BoolVar(&o.all, "all", false, "import every complete model on the drive") + f.BoolVar(&o.dryRun, "dry-run", false, "show what would be copied and stop") + f.StringVar(&o.cache, "cache", "", "hub cache directory to import into (default: the one the server reads)") + f.StringVar(&o.recipeID, "recipe", "", "import the models a tested recipe pins, into the cache its engine reads") + f.StringVar(&o.repo, "repo", "", "org/model of a folder that is not a Hugging Face cache") + f.StringVar(&o.revision, "revision", "", "commit of a folder that is not a Hugging Face cache or git clone") + return c +} + +func newExportCmd() *cobra.Command { + var cache string + var dryRun bool + c := &cobra.Command{ + Use: "export ... ", + Short: "Copy models from the Hugging Face cache onto an external drive", + Long: "Copy models from the Hugging Face cache the server reads onto an external drive, to\n" + + "carry to a machine with no internet. The drive gets a plain copy of each file in a\n" + + "hub/ folder, which any filesystem holds, and 'llm-helper import' reads it back.\n\n" + + "Only complete models are copied, and each file is checked against its recorded\n" + + "hash as it is copied. FAT32 cannot hold a file over 4 GiB; format the drive as\n" + + "exFAT.", + Example: " llm-helper export Qwen/Qwen3-8B-FP8 /media/usb", + GroupID: groupModels, + Args: cobra.MinimumNArgs(2), + RunE: func(_ *cobra.Command, args []string) error { + return runExport(args[:len(args)-1], args[len(args)-1], cache, dryRun) + }, + } + c.Flags().StringVar(&cache, "cache", "", "hub cache directory to copy from (default: the one the server reads)") + c.Flags().BoolVar(&dryRun, "dry-run", false, "show what would be copied and stop") + return c +} + +// serverHub is the cache a command works on when --cache is not given. +func serverHub() string { + if cfg, err := loadConfig(); err == nil { + return control.HubCache(cfg) + } + return fallbackHub() +} + +// fallbackHub is the cache of a machine with no config yet. +func fallbackHub() string { + home, _ := os.UserHomeDir() + return hfcache.Dir(hfcache.OSEnv, home) +} + +// recipeTarget is the models a recipe pins and the hub cache its engine reads. +func recipeTarget(id string) (models []recipe.Model, hub string, err error) { + tested, err := recipe.LoadTested() + if err != nil { + return nil, "", err + } + r, ok := tested.Get(id) + if !ok { + return nil, "", fmt.Errorf("no tested recipe %q", id) + } + cache := r.Engine().HFCacheHostPath(r.Params) + if cache == "" || len(r.Models) == 0 { + return nil, "", fmt.Errorf("recipe %s loads no models from a Hugging Face cache", id) + } + home, err := os.UserHomeDir() + if err != nil { + return nil, "", err + } + return r.Models, filepath.Join(home, cache, "hub"), nil +} + +func runImport(drive string, names []string, o importOptions) error { + if o.recipeID != "" && (o.cache != "" || o.repo != "") { + return errors.New("--recipe picks the models and the cache itself, so it cannot be combined with --cache or --repo") + } + for _, n := range names { + if !hfcache.ValidRepo(n) { + return fmt.Errorf("%q is not an org/model id", n) + } + } + hub := o.cache + var pinned []recipe.Model + switch { + case o.recipeID != "": + var err error + if pinned, hub, err = recipeTarget(o.recipeID); err != nil { + return err + } + case hub == "": + hub = serverHub() + } + + found, err := hfcache.Scan(drive, hfcache.ScanOptions{Repo: o.repo, Revision: o.revision}) + if err != nil { + return err + } + if len(found.Models) == 0 { + return fmt.Errorf("found no models at %s", drive) + } + + var chosen []hfcache.Model + switch { + case o.recipeID != "": + if chosen, err = pickPinned(found.Models, pinned); err != nil { + return err + } + case len(names) > 0: + if chosen, err = pickNamed(found.Models, names); err != nil { + return err + } + default: + listModels(found.Models) + if chosen, err = pickInteractive(found.Models, o.all); err != nil { + return err + } + } + if len(chosen) == 0 { + fmt.Println("Nothing selected.") + return nil + } + + need := hfcache.Need(chosen, hub) + fmt.Printf("Importing %d model(s) into %s: %s to copy.\n", len(chosen), hub, hfcache.FormatBytes(need)) + if err := hfcache.CheckSpace(hub, need); err != nil { + return err + } + if o.dryRun { + for _, m := range chosen { + fmt.Printf(" would import %s (%s)\n", m.Repo, hfcache.FormatBytes(m.Size())) + } + return nil + } + return copyModels(chosen, hub, hfcache.Import, "import", func(m hfcache.Model) { + switch { + case o.recipeID != "": + fmt.Printf("%s is staged for recipe %s.\n", m.Repo, o.recipeID) + default: + fmt.Printf("Run 'llm-helper new %s' to serve it.\n", m.Repo) + } + }) +} + +func runExport(repos []string, drive, cache string, dryRun bool) error { + for _, n := range repos { + if !hfcache.ValidRepo(n) { + return fmt.Errorf("%q is not an org/model id", n) + } + } + if st, err := os.Stat(drive); err != nil || !st.IsDir() { + return fmt.Errorf("%s is not a directory: mount the drive first", drive) + } + if cache == "" { + cache = serverHub() + } + var models []hfcache.Model + for _, repo := range repos { + m, err := hfcache.ScanRepo(cache, repo) + if err != nil { + return err + } + if !m.Complete() { + return fmt.Errorf("%s is incomplete in %s, so it would not load on the other machine: %s", repo, cache, strings.Join(m.Problems, "; ")) + } + models = append(models, m) + } + if err := hfcache.CheckDrive(models, drive); err != nil { + return err + } + dest := filepath.Join(drive, "hub") + fmt.Printf("Exporting %d model(s) to %s: %s to copy.\n", len(models), dest, hfcache.FormatBytes(hfcache.Need(models, dest))) + if dryRun { + for _, m := range models { + fmt.Printf(" would export %s (%s)\n", m.Repo, hfcache.FormatBytes(m.Size())) + } + return nil + } + return copyModels(models, dest, hfcache.Export, "export", func(m hfcache.Model) { + fmt.Printf("On the other machine: llm-helper import %s %s\n", drive, m.Repo) + }) +} + +// copyModels runs one copy per model and reports each. +func copyModels(models []hfcache.Model, dest string, do func(hfcache.Model, string, hfcache.Progress) (hfcache.Result, error), verb string, done func(hfcache.Model)) error { + for _, m := range models { + fmt.Printf("%s %s\n", strings.ToUpper(verb[:1])+verb[1:], m.Repo) + progress := func(path string, size int64, skipped bool) { + if !skipped { + fmt.Printf(" %s (%s)\n", path, hfcache.FormatBytes(size)) + } + } + res, err := do(m, dest, progress) + if err != nil { + var corrupt *hfcache.CorruptError + if errors.As(err, &corrupt) { + return fmt.Errorf("%s %s: %w (the source is damaged; nothing was kept from that file)", verb, m.Repo, err) + } + return fmt.Errorf("%s %s: %w (run it again to resume)", verb, m.Repo, err) + } + fmt.Printf(" %d file(s) copied, %d already there, %s written\n", res.Copied, res.Skipped, hfcache.FormatBytes(res.Bytes)) + if res.RefKept != "" { + fmt.Printf(" note: refs/main still names %s, so the copy is not what an offline load of %s reads\n", res.RefKept, m.Repo) + } + done(m) + } + return nil +} + +func listModels(models []hfcache.Model) { + fmt.Println() + for i, m := range models { + state := "complete" + if !m.Complete() { + state = "INCOMPLETE: " + strings.Join(m.Problems, "; ") + } + fmt.Printf(" %2d) %-52s %9s %.7s %s\n", i+1, m.Repo, hfcache.FormatBytes(m.Size()), m.Revision, state) + } + fmt.Println() +} + +// pickNamed selects the models the user named, which must all be importable. +func pickNamed(found []hfcache.Model, names []string) ([]hfcache.Model, error) { + var out []hfcache.Model + for _, n := range names { + m, ok := findRepo(found, n) + if !ok { + return nil, fmt.Errorf("%s is not on the drive", n) + } + if !m.Complete() { + return nil, fmt.Errorf("%s is incomplete on the drive: %s", n, strings.Join(m.Problems, "; ")) + } + out = append(out, m) + } + return out, nil +} + +// pickPinned selects the models a recipe pins, each at its pinned revision. +func pickPinned(found []hfcache.Model, pinned []recipe.Model) ([]hfcache.Model, error) { + var out []hfcache.Model + for _, p := range pinned { + m, ok := findRepo(found, p.ID) + switch { + case !ok: + return nil, fmt.Errorf("%s is not on the drive", p.ID) + case m.Revision != p.Revision: + return nil, fmt.Errorf("the recipe pins %s at %s, but the drive has %s", p.ID, p.Revision, m.Revision) + case !m.Complete(): + return nil, fmt.Errorf("%s is incomplete on the drive: %s", p.ID, strings.Join(m.Problems, "; ")) + } + out = append(out, m) + } + return out, nil +} + +// pickInteractive selects with --all, or asks, and skips incomplete models. +func pickInteractive(found []hfcache.Model, all bool) ([]hfcache.Model, error) { + var ok []hfcache.Model + for _, m := range found { + if m.Complete() { + ok = append(ok, m) + } + } + if len(ok) == 0 { + return nil, errors.New("no complete model on the drive") + } + if all { + return ok, nil + } + if !stdinIsTTY() { + return nil, errors.New("name the models to import, or pass --all") + } + answer := ask("Import which? (numbers like 1,3, or 'all'; Enter to cancel)", "") + if answer == "" { + return nil, nil + } + if answer == "all" { + return ok, nil + } + var out []hfcache.Model + for _, field := range strings.Split(answer, ",") { + n, err := strconv.Atoi(strings.TrimSpace(field)) + if err != nil || n < 1 || n > len(found) { + return nil, fmt.Errorf("%q is not a number from the list", field) + } + if !found[n-1].Complete() { + return nil, fmt.Errorf("%s is incomplete on the drive", found[n-1].Repo) + } + out = append(out, found[n-1]) + } + return out, nil +} + +func findRepo(models []hfcache.Model, repo string) (hfcache.Model, bool) { + for _, m := range models { + if strings.EqualFold(m.Repo, repo) { + return m, true + } + } + return hfcache.Model{}, false +} diff --git a/internal/cli/hfcache_test.go b/internal/cli/hfcache_test.go new file mode 100644 index 0000000..c800180 --- /dev/null +++ b/internal/cli/hfcache_test.go @@ -0,0 +1,291 @@ +package cli_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/recipe" +) + +const testRev = "0123456789abcdef0123456789abcdef01234567" + +// seedHub writes a repo into a hub cache the way huggingface_hub does: blobs +// named by hash and snapshot files linked to them. +func seedHub(t *testing.T, hub, repo, rev string, files map[string]string) { + t.Helper() + root := filepath.Join(hub, "models--"+strings.ReplaceAll(repo, "/", "--")) + mustWrite := func(p, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + mustWrite(filepath.Join(root, "refs", "main"), rev) + for name, body := range files { + sum := sha256.Sum256([]byte(body)) + blob := filepath.Join(root, "blobs", hex.EncodeToString(sum[:])) + mustWrite(blob, body) + link := filepath.Join(root, "snapshots", rev, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { + t.Fatal(err) + } + rel, err := filepath.Rel(filepath.Dir(link), blob) + if err != nil { + t.Fatal(err) + } + if err := os.Symlink(rel, link); err != nil { + t.Fatal(err) + } + } +} + +var qwenFiles = map[string]string{ + "config.json": `{"model_type":"qwen3"}`, + "model.safetensors": strings.Repeat("weights", 100), +} + +func snapshotFile(hub, repo, rev, name string) string { + return filepath.Join(hub, "models--"+strings.ReplaceAll(repo, "/", "--"), "snapshots", rev, name) +} + +func TestImportCopiesANamedModelIntoTheCache(t *testing.T) { + newTree(t) + drive, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, filepath.Join(drive, "hub"), "Qwen/Qwen3-8B", testRev, qwenFiles) + + code, out, errOut := runCLI(t, "import", drive, "Qwen/Qwen3-8B", "--cache", hub) + if code != 0 { + t.Fatalf("exit %d\n%s\n%s", code, out, errOut) + } + got, err := os.ReadFile(snapshotFile(hub, "Qwen/Qwen3-8B", testRev, "model.safetensors")) + if err != nil || string(got) != qwenFiles["model.safetensors"] { + t.Fatalf("imported weights: %v", err) + } + if ref, _ := os.ReadFile(filepath.Join(hub, "models--Qwen--Qwen3-8B", "refs", "main")); string(ref) != testRev { + t.Errorf("refs/main = %q", ref) + } + if !strings.Contains(out, "llm-helper new Qwen/Qwen3-8B") { + t.Errorf("output does not say how to serve it:\n%s", out) + } +} + +func TestImportNeedsNamesOrAllWithoutATerminal(t *testing.T) { + newTree(t) + drive, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, drive, "Qwen/Qwen3-8B", testRev, qwenFiles) + seedHub(t, drive, "org/other", testRev, qwenFiles) + + code, out, errOut := runCLI(t, "import", drive, "--cache", hub) + if code == 0 || !strings.Contains(errOut, "--all") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } + if !strings.Contains(out, "Qwen/Qwen3-8B") || !strings.Contains(out, "org/other") { + t.Errorf("the models on the drive were not listed:\n%s", out) + } + if _, err := os.Stat(hub); err == nil { + t.Error("the cache was written without a choice") + } + + if code, out, errOut := runCLI(t, "import", drive, "--all", "--cache", hub); code != 0 { + t.Fatalf("--all: exit %d\n%s\n%s", code, out, errOut) + } + for _, repo := range []string{"Qwen/Qwen3-8B", "org/other"} { + if _, err := os.Stat(snapshotFile(hub, repo, testRev, "config.json")); err != nil { + t.Errorf("%s was not imported: %v", repo, err) + } + } +} + +func TestImportDryRunCopiesNothing(t *testing.T) { + newTree(t) + drive, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, drive, "Qwen/Qwen3-8B", testRev, qwenFiles) + + code, out, _ := runCLI(t, "import", drive, "Qwen/Qwen3-8B", "--cache", hub, "--dry-run") + if code != 0 || !strings.Contains(out, "would import Qwen/Qwen3-8B") { + t.Fatalf("exit %d\n%s", code, out) + } + if _, err := os.Stat(hub); err == nil { + t.Error("a dry run wrote the cache") + } +} + +func TestImportRefusesAnIncompleteModel(t *testing.T) { + newTree(t) + drive, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, drive, "Qwen/Qwen3-8B", testRev, qwenFiles) + repoDir := filepath.Join(drive, "models--Qwen--Qwen3-8B") + entries, _ := os.ReadDir(filepath.Join(repoDir, "blobs")) + for _, e := range entries { + if err := os.Remove(filepath.Join(repoDir, "blobs", e.Name())); err != nil { + t.Fatal(err) + } + } + + code, _, errOut := runCLI(t, "import", drive, "Qwen/Qwen3-8B", "--cache", hub) + if code == 0 || !strings.Contains(errOut, "incomplete") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } +} + +func TestImportFailsOnADamagedFile(t *testing.T) { + newTree(t) + drive, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, drive, "Qwen/Qwen3-8B", testRev, qwenFiles) + snap := snapshotFile(drive, "Qwen/Qwen3-8B", testRev, "model.safetensors") + if err := os.WriteFile(snap, []byte(strings.Repeat("WEIGHTS", 100)), 0o644); err != nil { + t.Fatal(err) + } + + code, _, errOut := runCLI(t, "import", drive, "Qwen/Qwen3-8B", "--cache", hub) + if code == 0 { + t.Fatal("a damaged file was imported") + } + if _, err := os.Stat(snapshotFile(hub, "Qwen/Qwen3-8B", testRev, "model.safetensors")); err == nil { + t.Errorf("the damaged file reached the snapshot (stderr %q)", errOut) + } +} + +func TestExportThenImportRoundTrips(t *testing.T) { + newTree(t) + src, drive, dst := t.TempDir(), t.TempDir(), filepath.Join(t.TempDir(), "hub") + seedHub(t, src, "Qwen/Qwen3-8B", testRev, qwenFiles) + + code, out, errOut := runCLI(t, "export", "Qwen/Qwen3-8B", drive, "--cache", src) + if code != 0 { + t.Fatalf("export: exit %d\n%s\n%s", code, out, errOut) + } + _ = filepath.WalkDir(drive, func(p string, d os.DirEntry, _ error) error { + if d != nil && d.Type()&os.ModeSymlink != 0 { + t.Errorf("the drive holds a link: %s", p) + } + return nil + }) + + code, out, errOut = runCLI(t, "import", drive, "--all", "--cache", dst) + if code != 0 { + t.Fatalf("import: exit %d\n%s\n%s", code, out, errOut) + } + got, err := os.ReadFile(snapshotFile(dst, "Qwen/Qwen3-8B", testRev, "config.json")) + if err != nil || string(got) != qwenFiles["config.json"] { + t.Fatalf("round trip lost config.json: %v", err) + } +} + +func TestExportRefusesWhatIsNotComplete(t *testing.T) { + newTree(t) + src, drive := t.TempDir(), t.TempDir() + seedHub(t, src, "Qwen/Qwen3-8B", testRev, map[string]string{"config.json": "{}"}) + + code, _, errOut := runCLI(t, "export", "Qwen/Qwen3-8B", drive, "--cache", src) + if code == 0 || !strings.Contains(errOut, "incomplete") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } + code, _, errOut = runCLI(t, "export", "Qwen/Missing", drive, "--cache", src) + if code == 0 || !strings.Contains(errOut, "not in the cache") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } + code, _, errOut = runCLI(t, "export", "Qwen/Qwen3-8B", filepath.Join(drive, "nope"), "--cache", src) + if code == 0 || !strings.Contains(errOut, "mount the drive") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } +} + +func TestImportTakesAFolderWithARepoName(t *testing.T) { + newTree(t) + folder, hub := t.TempDir(), filepath.Join(t.TempDir(), "hub") + for name, body := range qwenFiles { + if err := os.WriteFile(filepath.Join(folder, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + code, _, errOut := runCLI(t, "import", folder, "--cache", hub, "--all") + if code == 0 || !strings.Contains(errOut, "--repo") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } + code, _, errOut = runCLI(t, "import", folder, "--cache", hub, "--all", "--repo", "Qwen/Qwen3-8B") + if code == 0 || !strings.Contains(errOut, "revision") { + t.Fatalf("a folder with no commit was imported: exit %d, stderr %q", code, errOut) + } + code, out, errOut := runCLI(t, "import", folder, "--cache", hub, "--all", "--repo", "Qwen/Qwen3-8B", "--revision", testRev) + if code != 0 { + t.Fatalf("exit %d\n%s\n%s", code, out, errOut) + } + if _, err := os.Stat(snapshotFile(hub, "Qwen/Qwen3-8B", testRev, "model.safetensors")); err != nil { + t.Error(err) + } +} + +func TestImportForARecipeStagesWhereItsEngineReads(t *testing.T) { + newTree(t) + tested, err := recipe.LoadTested() + if err != nil { + t.Fatal(err) + } + var r recipe.TestedRecipe + for _, c := range tested.Recipes { + if c.Engine().HFCacheHostPath(c.Params) != "" && len(c.Models) > 0 { + r = c + break + } + } + if r.ID == "" { + t.Skip("no recipe pins models in a host cache") + } + home := t.TempDir() + t.Setenv("HOME", home) + drive := t.TempDir() + for _, m := range r.Models { + seedHub(t, filepath.Join(drive, "hub"), m.ID, m.Revision, qwenFiles) + } + + code, out, errOut := runCLI(t, "import", drive, "--recipe", r.ID) + if code != 0 { + t.Fatalf("exit %d\n%s\n%s", code, out, errOut) + } + for _, m := range r.Models { + dir := filepath.Join(home, engine.SnapshotPath(r.Engine().HFCacheHostPath(r.Params), engine.ModelRef{ID: m.ID, Revision: m.Revision})) + if _, err := os.Stat(filepath.Join(dir, "model.safetensors")); err != nil { + t.Errorf("%s is not where the recipe reads it: %v", m.ID, err) + } + } + if !strings.Contains(out, "staged for recipe "+r.ID) { + t.Errorf("output:\n%s", out) + } + + other := t.TempDir() + seedHub(t, filepath.Join(other, "hub"), r.Models[0].ID, testRev, qwenFiles) + if code, _, errOut := runCLI(t, "import", other, "--recipe", r.ID); code == 0 || !strings.Contains(errOut, "pins") { + t.Errorf("a revision the recipe does not pin was imported: exit %d, stderr %q", code, errOut) + } +} + +func TestSearchOfflineListsTheCachedModels(t *testing.T) { + configDir, _ := newTree(t) + hub := filepath.Join(t.TempDir(), "hub") + seedHub(t, hub, "Qwen/Qwen3-8B", testRev, qwenFiles) + cfg, err := os.ReadFile(filepath.Join(configDir, "config.env")) + if err != nil { + t.Fatal(err) + } + body := string(cfg) + "HF_HUB_CACHE=" + hub + "\nHF_HUB_OFFLINE=1\n" + if err := os.WriteFile(filepath.Join(configDir, "config.env"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + code, out, errOut := runCLI(t, "search", "qwen3", "--json") + if code != 0 { + t.Fatalf("exit %d\n%s\n%s", code, out, errOut) + } + if !strings.Contains(out, "Qwen/Qwen3-8B") || !strings.Contains(errOut, "local cache") { + t.Errorf("stdout:\n%s\nstderr:\n%s", out, errOut) + } +} diff --git a/internal/cli/profiles.go b/internal/cli/profiles.go index eaa5ab1..c06162d 100644 --- a/internal/cli/profiles.go +++ b/internal/cli/profiles.go @@ -255,6 +255,7 @@ func pullAndUp(ctx context.Context, cfg *config.Config, repo string, choice *res if err := plan.SetThinking(thinking, prov, caps); err != nil { return err } + control.KeepOffline(ctx, cfg, prov, &plan) for _, n := range plan.Notes { fmt.Println(n) } diff --git a/internal/cli/recipelaunch.go b/internal/cli/recipelaunch.go index 96295d9..590c73f 100644 --- a/internal/cli/recipelaunch.go +++ b/internal/cli/recipelaunch.go @@ -698,8 +698,8 @@ func checkStaged(r recipe.TestedRecipe, members []recipeMember, key string) erro id, rev, _ := strings.Cut(ref, "@") hint = append(hint, fmt.Sprintf(" HF_HOME=\"$HOME/%s\" hf download %s --revision %s", cache, id, rev)) } - return fmt.Errorf("%s needs %s on node %s, which is missing or incomplete in its Hugging Face cache; download it there once with:\n%s", - r.ID, strings.Join(missing, ", "), m.node.Name, strings.Join(hint, "\n")) + return fmt.Errorf("%s needs %s on node %s, which is missing or incomplete in its Hugging Face cache; download it there once with:\n%s\nor, from a drive that has it, run on that node:\n llm-helper import --recipe %s", + r.ID, strings.Join(missing, ", "), m.node.Name, strings.Join(hint, "\n"), r.ID) } return nil } diff --git a/internal/cli/root.go b/internal/cli/root.go index 2b44ec5..462fa0d 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -18,6 +18,8 @@ import ( "github.com/coder/llm-helper/internal/control" "github.com/coder/llm-helper/internal/engine" "github.com/coder/llm-helper/internal/hardware" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/hfcache" "github.com/coder/llm-helper/internal/modeltype" "github.com/coder/llm-helper/internal/plugin" "github.com/coder/llm-helper/internal/profile" @@ -83,11 +85,15 @@ func newRootCmd() *cobra.Command { // Recipes load last, because a local recipe may run on a local // engine. recipe.UseLocal(filepath.Join(config.ResolveDirs().Config, recipe.LocalDir)) - if cfg, err := loadConfig(); err == nil { + cfg, err := loadConfig() + if err != nil { + hf.UseCache(fallbackHub(), hfcache.Offline(hfcache.OSEnv)) + } else { useMetalProbe(cfg.VLLMBin()) // config.env can set CUDA_VISIBLE_DEVICES, and sizing every GPU // for a server allowed one of them overstates the budget. control.UseServerGPUs(cfg) + hf.UseCache(control.HubCache(cfg), control.HubOffline(cfg)) } } root.SetVersionTemplate(cmdName + " {{.Version}}\n") @@ -113,6 +119,8 @@ func newRootCmd() *cobra.Command { newSearchCmd(), newInspectCmd(), newInstallGGUFCmd(), + newImportCmd(), + newExportCmd(), newEditCmd(), newSetCmd(), newRemoveCmd(), diff --git a/internal/cli/root_internal_test.go b/internal/cli/root_internal_test.go index abd4abe..8e7b91b 100644 --- a/internal/cli/root_internal_test.go +++ b/internal/cli/root_internal_test.go @@ -13,7 +13,7 @@ func TestTheCommandSetIsPinned(t *testing.T) { // help is not here: cobra attaches it during Execute, not registration. // completion is not here either, because the root disables it. want := []string{ - "adopt", "coder-sync", "default", "doctor", "down", "edit", "engines", "hardware", "inspect", "install-gguf", "list", "logs", + "adopt", "coder-sync", "default", "doctor", "down", "edit", "engines", "export", "hardware", "import", "inspect", "install-gguf", "list", "logs", "metal-limit", "migrate", "model-types", "new", "node", "pool", "recipes", "remove", "restart", "router", "search", "serve", "set", "setup", "shard", "spark", "start", "status", "switch", "up", "upgrade", "version", "warmup", diff --git a/internal/cli/search.go b/internal/cli/search.go index 84807ea..5d68e90 100644 --- a/internal/cli/search.go +++ b/internal/cli/search.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "strconv" "strings" @@ -46,6 +47,9 @@ func runSearch(ctx context.Context, args []string, jsonOut bool) error { if err != nil { return fmt.Errorf("search request failed (network or rate limit): %w", err) } + if hf.Offline() { + fmt.Fprintln(os.Stderr, "Not asking the Hub (offline, or it did not answer): these are the matching models in the local cache.") + } if jsonOut { out := make([]resolve.Result, 0, len(models)) for _, m := range models { @@ -54,6 +58,10 @@ func runSearch(ctx context.Context, args []string, jsonOut bool) error { return printJSON(out) } if len(models) == 0 { + if hf.Offline() { + fmt.Println("No cached model matches. 'llm-helper import ' copies models from a drive.") + return nil + } fmt.Println("No text-generation models found. Try different terms.") return nil } diff --git a/internal/control/hubcache.go b/internal/control/hubcache.go new file mode 100644 index 0000000..5e174ff --- /dev/null +++ b/internal/control/hubcache.go @@ -0,0 +1,56 @@ +package control + +import ( + "context" + "os" + "os/exec" + "time" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/hfcache" + "github.com/coder/llm-helper/internal/resolve" +) + +// HubCache is the hub cache directory the server reads. The service sources +// config.env and secrets.env but not a login shell, so only those count. +func HubCache(cfg *config.Config) string { + home, _ := os.UserHomeDir() + var extra []string + if p, err := engine.LookupEngine(cfg.Lookup(engine.Key, "")); err == nil && p.HubCacheVar() != "" { + extra = append(extra, p.HubCacheVar()) + } + return hfcache.Dir(func(k string) string { return cfg.Lookup(k, "") }, home, extra...) +} + +// HubOffline reports whether HF_HUB_OFFLINE is on for the server, or for this +// process, which is what a command run by hand sees. +func HubOffline(cfg *config.Config) bool { + return hfcache.Offline(func(k string) string { return cfg.Lookup(k, "") }) || + hfcache.Offline(hfcache.OSEnv) +} + +// KeepOffline sets a new profile up to load from the cache when the Hub is +// not being used, asking the installed server what it accepts. +func KeepOffline(ctx context.Context, cfg *config.Config, prov engine.Provider, plan *resolve.Plan) { + if hf.Offline() { + plan.SetOffline(prov, func(args []string) string { return probeServer(ctx, cfg, prov, args) }) + } +} + +// probeServer runs the installed server with args and returns what it +// printed, "" when it cannot be run. +func probeServer(ctx context.Context, cfg *config.Config, prov engine.Provider, args []string) string { + bin := cfg.Lookup(prov.BinKey(), "") + if bin == "" { + bin, _ = exec.LookPath(prov.BinName()) + } + if bin == "" { + return "" + } + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + out, _ := exec.CommandContext(ctx, bin, args...).CombinedOutput() + return string(out) +} diff --git a/internal/control/hubcache_internal_test.go b/internal/control/hubcache_internal_test.go new file mode 100644 index 0000000..d802662 --- /dev/null +++ b/internal/control/hubcache_internal_test.go @@ -0,0 +1,148 @@ +package control + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/profile" + "github.com/coder/llm-helper/internal/resolve" +) + +func configWith(t *testing.T, body string) *config.Config { + t.Helper() + path := filepath.Join(t.TempDir(), "config.env") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + f, err := config.ParseEnvFile(path) + if err != nil { + t.Fatal(err) + } + return &config.Config{File: f} +} + +func TestHubCacheIsWhatTheServerSees(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("HF_HOME", "/shell/only") + t.Setenv("HF_HUB_CACHE", "") + + cases := []struct { + name, env, want string + }{ + {"the shell's HF_HOME is not the server's", "", filepath.Join(home, ".cache", "huggingface", "hub")}, + {"HF_HOME in config.env", "HF_HOME=/mnt/models\n", "/mnt/models/hub"}, + {"HF_HUB_CACHE beats HF_HOME", "HF_HOME=/a\nHF_HUB_CACHE=/b\n", "/b"}, + {"llama.cpp reads LLAMA_CACHE first", "PROVIDER=llamacpp\nHF_HUB_CACHE=/b\nLLAMA_CACHE=/c\n", "/c"}, + {"LLAMA_CACHE is ignored by other engines", "PROVIDER=vllm\nHF_HUB_CACHE=/b\nLLAMA_CACHE=/c\n", "/b"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := HubCache(configWith(t, c.env)); got != c.want { + t.Fatalf("HubCache = %q, want %q", got, c.want) + } + }) + } +} + +func TestHubOfflineReadsTheServerAndTheProcess(t *testing.T) { + t.Setenv("HF_HUB_OFFLINE", "") + t.Setenv("TRANSFORMERS_OFFLINE", "") + if HubOffline(configWith(t, "PORT=8001\n")) { + t.Fatal("offline with nothing set") + } + if !HubOffline(configWith(t, "HF_HUB_OFFLINE=1\n")) { + t.Fatal("config.env HF_HUB_OFFLINE=1 not seen") + } + t.Setenv("HF_HUB_OFFLINE", "1") + if !HubOffline(configWith(t, "PORT=8001\n")) { + t.Fatal("process HF_HUB_OFFLINE=1 not seen") + } +} + +// fakeServer is a stand-in for an installed server whose --help says help. +func fakeServer(t *testing.T, help string) string { + t.Helper() + bin := filepath.Join(t.TempDir(), "server") + script := "#!/bin/sh\ncat <<'EOF'\n" + help + "\nEOF\n" + if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return bin +} + +func TestKeepOfflineAsksTheInstalledServer(t *testing.T) { + hub := t.TempDir() + hf.UseCache(hub, true) + t.Cleanup(func() { hf.UseCache("", false) }) + prov, err := engine.Lookup("llamacpp") + if err != nil { + t.Fatal(err) + } + + for _, c := range []struct { + name, help, want string + }{ + {"lists the flag", " --offline no network", "--offline"}, + {"does not list it", " --port N", ""}, + } { + t.Run(c.name, func(t *testing.T) { + cfg := configWith(t, prov.BinKey()+"="+fakeServer(t, c.help)+"\n") + var plan resolve.Plan + KeepOffline(context.Background(), cfg, prov, &plan) + if plan.Opts.ExtraArgs != c.want { + t.Errorf("ExtraArgs = %q, want %q", plan.Opts.ExtraArgs, c.want) + } + }) + } +} + +func TestKeepOfflineDoesNothingOnline(t *testing.T) { + hf.UseCache("", false) + prov, _ := engine.Lookup("vllm") + var plan resolve.Plan + KeepOffline(context.Background(), configWith(t, "PORT=1\n"), prov, &plan) + if len(plan.Opts.Env) != 0 || plan.Opts.ExtraArgs != "" || len(plan.Notes) != 0 { + t.Errorf("plan changed: %+v", plan) + } +} + +func TestKeepOfflineWritesTheProfileEveryEngineNeeds(t *testing.T) { + hf.UseCache(t.TempDir(), true) + t.Cleanup(func() { hf.UseCache("", false) }) + for _, id := range []string{"vllm", "mlx", "llamacpp"} { + prov, err := engine.Lookup(id) + if err != nil { + t.Fatal(err) + } + cfg := configWith(t, prov.BinKey()+"="+fakeServer(t, "--offline")+"\n") + plan := resolve.Plan{Opts: profile.CreateOpts{Repo: "org/model", Provider: id, Source: "test"}} + KeepOffline(context.Background(), cfg, prov, &plan) + + dirs := config.Dirs{Config: t.TempDir(), Profiles: t.TempDir()} + if _, err := profile.Create(dirs, plan.Opts); err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(dirs.ProfileFile("model")) + if err != nil { + t.Fatal(err) + } + off := prov.OfflineSetting() + for _, line := range off.EnvLines() { + if !strings.Contains(string(body), line) { + t.Errorf("%s: profile lacks %q:\n%s", id, line, body) + } + } + for _, a := range off.Args { + if !strings.Contains(string(body), a) { + t.Errorf("%s: profile lacks %q:\n%s", id, a, body) + } + } + } +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 80cb08c..e0f454b 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -79,6 +79,12 @@ type Engine struct { Caps Capabilities `json:"capabilities"` // Notes are sentences inspect prints about serving some kinds of model. Notes Notes `json:"notes,omitempty"` + // HubCacheEnv names a variable that sets the server's Hugging Face hub + // cache directly, ahead of the HF_* variables. + HubCacheEnv string `json:"hub_cache_env,omitempty"` + // Offline is how the server is kept off the network. Absent means the + // engine has no setting for it. + Offline *Offline `json:"offline,omitempty"` // Source is the engines.d file a local definition came from, and "" for // one shipped in the binary. @@ -205,6 +211,18 @@ type Capabilities struct { ChatTemplateKwargsFlag string `json:"chat_template_kwargs_flag,omitempty"` } +// Offline is what a profile gets so that its server never reaches the Hub. +type Offline struct { + // Env is added to the profile's environment. + Env map[string]string `json:"env,omitempty"` + // Args are added to the profile's EXTRA_ARGS. + Args []string `json:"args,omitempty"` + // ProbeArgs, when set, are run after the server's binary, and Args are + // added only when the output names Args[0]. A release that predates the + // flag would reject it and fail to start. + ProbeArgs []string `json:"probe_args,omitempty"` +} + // Notes are sentences about how the engine serves a kind of model. type Notes struct { // Hybrid is shown for models whose layers are not all attention. @@ -314,6 +332,28 @@ func (e Engine) Capabilities() Capabilities { // HybridNote is what inspect says about serving a hybrid model. func (e Engine) HybridNote() string { return e.Notes.Hybrid } +// HubCacheVar is the variable that sets the server's hub cache directly, or +// "" when it has none. +func (e Engine) HubCacheVar() string { return e.HubCacheEnv } + +// OfflineSetting is how the server is kept off the network, or nil. +func (e Engine) OfflineSetting() *Offline { return e.Offline } + +// EnvLines is the KEY=value entries, sorted by name. +func (o Offline) EnvLines() []string { + out := make([]string, 0, len(o.Env)) + for _, k := range slices.Sorted(maps.Keys(o.Env)) { + out = append(out, k+"="+o.Env[k]) + } + return out +} + +// ArgsAccepted reports whether the server takes Args, given the output of +// running it with ProbeArgs. Without ProbeArgs it always does. +func (o Offline) ArgsAccepted(probeOutput string) bool { + return len(o.ProbeArgs) == 0 || (len(o.Args) > 0 && strings.Contains(probeOutput, o.Args[0])) +} + // InstallHint says how to get the server on goos. func (e Engine) InstallHint(goos string) string { if h := e.Install[goos]; h != "" { @@ -804,6 +844,13 @@ func validate(e Engine, hw *hardware.Registry) []error { add("capabilities.chat_template_kwargs_flag %q is not a flag", f) } + if k := e.HubCacheEnv; k != "" && !envNamePattern.MatchString(k) { + add("hub_cache_env %q is not a variable name", k) + } + if o := e.Offline; o != nil { + validateOffline(o, e.Has(RuntimeProcess), add) + } + params := map[string]bool{} for _, p := range e.Params { switch { @@ -911,6 +958,31 @@ func validate(e Engine, hw *hardware.Registry) []error { return errs } +func validateOffline(o *Offline, process bool, add func(string, ...any)) { + switch { + case !process: + add("offline belongs to the process runtime, which this engine does not declare") + case len(o.Env) == 0 && len(o.Args) == 0: + add("offline sets neither env nor args") + } + for _, k := range slices.Sorted(maps.Keys(o.Env)) { + if !envNamePattern.MatchString(k) { + add("offline.env: %q is not a variable name", k) + } + } + if len(o.Args) > 0 && !capFlagPattern.MatchString(o.Args[0]) { + add("offline.args must start with a flag, not %q", o.Args[0]) + } + if len(o.ProbeArgs) > 0 && len(o.Args) == 0 { + add("offline.probe_args needs offline.args, which it checks for") + } + for _, a := range slices.Concat(o.Args, o.ProbeArgs) { + if a == "" || placeholderRE.MatchString(a) { + add("offline: %q is empty or a placeholder; these are literal", a) + } + } +} + func checkEnv(where string, env map[string]string, ok map[string]bool, check func(string, []string, map[string]bool), add func(string, ...any)) { for _, k := range slices.Sorted(maps.Keys(env)) { if !envNamePattern.MatchString(k) { diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index 8ecd8cb..4b3e521 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -175,6 +175,13 @@ func TestParseIsStrict(t *testing.T) { {"process engine without install hint", toyProcess, `"install": { "any": "pip install toy" },`, ``, "install hint"}, {"trailing data", toyContainer, ``, ``, "after the top-level"}, {"kwargs flag that is not a flag", toyContainer, `"capabilities": {}`, `"capabilities": {"chat_template_kwargs_flag": "kwargs"}`, "chat_template_kwargs_flag"}, + {"offline that sets nothing", toyProcess, `"capabilities": {}`, `"offline": {}, "capabilities": {}`, "neither env nor args"}, + {"offline env that is not a name", toyProcess, `"capabilities": {}`, `"offline": {"env": {"A B": "1"}}, "capabilities": {}`, "not a variable name"}, + {"offline arg that is not a flag", toyProcess, `"capabilities": {}`, `"offline": {"args": ["offline"]}, "capabilities": {}`, "must start with a flag"}, + {"probe without args", toyProcess, `"capabilities": {}`, `"offline": {"env": {"A": "1"}, "probe_args": ["--help"]}, "capabilities": {}`, "needs offline.args"}, + {"offline arg that is a placeholder", toyProcess, `"capabilities": {}`, `"offline": {"args": ["--offline", "{port}"]}, "capabilities": {}`, "placeholder"}, + {"offline on a container-only engine", toyContainer, `"capabilities": {}`, `"offline": {"env": {"A": "1"}}, "capabilities": {}`, "process runtime"}, + {"hub cache variable that is not a name", toyProcess, `"capabilities": {}`, `"hub_cache_env": "A-B", "capabilities": {}`, "hub_cache_env"}, } { data := strings.Replace(tc.base, tc.old, tc.new, 1) if tc.name == "trailing data" { @@ -338,3 +345,53 @@ func TestUseLocalSkipsAndReportsBadFiles(t *testing.T) { t.Error("UseLocal kept errors from a directory it no longer reads") } } + +// How each shipped engine is kept off the network is data in its file, and the +// llama.cpp flag is used only when the installed server lists it. +func TestShippedEnginesDeclareHowToGoOffline(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + id string + env, args []string + probe []string + cacheVar string + }{ + {id: engine.VLLM, env: []string{"HF_HUB_OFFLINE=1"}}, + {id: engine.MLX, env: []string{"HF_HUB_OFFLINE=1"}}, + {id: engine.LlamaCPP, args: []string{"--offline"}, probe: []string{"--help"}, cacheVar: "LLAMA_CACHE"}, + } { + p, err := engine.Lookup(tc.id) + if err != nil { + t.Fatal(err) + } + o := p.OfflineSetting() + if o == nil { + t.Fatalf("%s: no offline setting", tc.id) + } + if got := o.EnvLines(); !slices.Equal(got, tc.env) && len(got)+len(tc.env) > 0 { + t.Errorf("%s: env = %q, want %q", tc.id, got, tc.env) + } + if !slices.Equal(o.Args, tc.args) || !slices.Equal(o.ProbeArgs, tc.probe) { + t.Errorf("%s: args = %q, probe = %q", tc.id, o.Args, o.ProbeArgs) + } + if p.HubCacheVar() != tc.cacheVar { + t.Errorf("%s: hub_cache_env = %q, want %q", tc.id, p.HubCacheVar(), tc.cacheVar) + } + } +} + +func TestOfflineArgsAreProbed(t *testing.T) { + t.Parallel() + p, _ := engine.Lookup(engine.LlamaCPP) + o := p.OfflineSetting() + if !o.ArgsAccepted(" --offline Offline mode: forces use of cache") { + t.Error("a --help that lists --offline was refused") + } + if o.ArgsAccepted("--host HOST") || o.ArgsAccepted("") { + t.Error("a --help that does not list --offline was accepted") + } + v, _ := engine.Lookup(engine.VLLM) + if !v.OfflineSetting().ArgsAccepted("") { + t.Error("an offline setting with no probe must always be accepted") + } +} diff --git a/internal/engine/registry.go b/internal/engine/registry.go index 0e00a12..f935c17 100644 --- a/internal/engine/registry.go +++ b/internal/engine/registry.go @@ -61,6 +61,12 @@ type Provider interface { // HybridNote is what inspect says about serving a hybrid model, if // anything. HybridNote() string + // HubCacheVar is the variable that sets the server's hub cache directly, + // "" when it has none. + HubCacheVar() string + // OfflineSetting is how the server is kept off the network, nil when it + // has no setting for it. + OfflineSetting() *Offline // Has reports whether the engine can run in a runtime. Has(runtime string) bool // InstallHint says how to get the server on an OS, by GOOS. diff --git a/internal/hf/arch.go b/internal/hf/arch.go index 1abdd80..f767f0f 100644 --- a/internal/hf/arch.go +++ b/internal/hf/arch.go @@ -167,7 +167,7 @@ func (a Arch) MoE() bool { // the repo has none: GGUF repos typically ship none, which is a fact about the // repo rather than a failure. func Config(ctx context.Context, repo string) (Arch, error) { - resp, err := get(ctx, fmt.Sprintf("%s/%s/raw/main/config.json", baseURL, repo)) + resp, err := rawGet(ctx, repo, "config.json") if err != nil { return Arch{}, err } diff --git a/internal/hf/hf.go b/internal/hf/hf.go index a2b0e3b..2b7fe8e 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -101,8 +101,14 @@ func (m Model) QuantTags() []string { // Lookup fetches one model's metadata by repo id. func Lookup(ctx context.Context, repo string) (Model, error) { var m Model + if hub := local(); hub != "" { + return localModel(hub, repo) + } resp, err := get(ctx, baseURL+"/api/models/"+repo) if err != nil { + if hub := canFallBack(ctx, err); hub != "" { + return localModel(hub, repo) + } return m, err } defer resp.Body.Close() @@ -151,12 +157,20 @@ func WhoAmI(ctx context.Context, token string) (string, error) { } // Search queries the Hub for text-generation models, most-downloaded first. +// +// With the Hub not in use it lists the cached models that match instead. func Search(ctx context.Context, query string, limit int) ([]Model, error) { + if hub := local(); hub != "" { + return localSearch(hub, query, limit), nil + } u := fmt.Sprintf( baseURL+"/api/models?search=%s&filter=text-generation&sort=downloads&direction=-1&limit=%d", url.QueryEscape(query), limit) resp, err := get(ctx, u) if err != nil { + if hub := canFallBack(ctx, err); hub != "" { + return localSearch(hub, query, limit), nil + } return nil, err } defer resp.Body.Close() @@ -615,12 +629,18 @@ func isWeightFile(path string) bool { // undercounts, and an undercount is the one wrong answer that tells someone a // model fits when it does not. func repoTree(ctx context.Context, repo string) ([]treeEntry, error) { + if hub := local(); hub != "" { + return localTree(hub, repo) + } next := baseURL + "/api/models/" + repo + "/tree/main?recursive=true" var all []treeEntry seen := map[string]bool{} for page := 0; page < maxTreePages; page++ { entries, link, err := treePage(ctx, next) if err != nil { + if hub := canFallBack(ctx, err); hub != "" && page == 0 { + return localTree(hub, repo) + } return nil, err } for _, e := range entries { @@ -706,7 +726,7 @@ type DetectedFlags struct { // someone else's deployment, not the model. func DetectFlags(ctx context.Context, repo string) (DetectedFlags, error) { var d DetectedFlags - resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/README.md") + resp, err := rawGet(ctx, repo, "README.md") if err != nil { return d, err } @@ -746,7 +766,7 @@ func DetectFlags(ctx context.Context, repo string) (DetectedFlags, error) { // the ceiling vLLM validates --max-model-len against. Returns 0 when the // field is absent. func ContextLimit(ctx context.Context, repo string) (int, error) { - resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/config.json") + resp, err := rawGet(ctx, repo, "config.json") if err != nil { return 0, err } @@ -780,7 +800,7 @@ func clampContext(value string, limit int) string { // ChatTemplate fetches a repo's chat_template.jinja. found is false when the // repo has no separate template file. func ChatTemplate(ctx context.Context, repo string) (text string, found bool, err error) { - resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/chat_template.jinja") + resp, err := rawGet(ctx, repo, "chat_template.jinja") if err != nil { return "", false, err } @@ -802,7 +822,7 @@ func TemplateText(ctx context.Context, repo string) (string, error) { if text, found, err := ChatTemplate(ctx, repo); err != nil || found { return text, err } - resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/tokenizer_config.json") + resp, err := rawGet(ctx, repo, "tokenizer_config.json") if err != nil { return "", err } diff --git a/internal/hf/local.go b/internal/hf/local.go new file mode 100644 index 0000000..2e06e65 --- /dev/null +++ b/internal/hf/local.go @@ -0,0 +1,142 @@ +package hf + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/coder/llm-helper/internal/hfcache" +) + +// ErrOffline is returned for a model that is not in the local cache while the +// Hub is not being used. +var ErrOffline = errors.New("not in the local cache, and the Hub is not reachable") + +// cache is where the Hub readers look when the Hub cannot answer. +var cache struct { + sync.Mutex + hub string + offline bool + // downUntil is when the Hub is tried again after a request failed to + // reach it, so a panel that outlives a network drop recovers. + downUntil time.Time +} + +// retryHubAfter is how long the cache answers after the Hub did not. +const retryHubAfter = time.Minute + +// UseCache points the readers at the hub cache a server reads. With offline +// set they never ask the Hub; without it they fall back to the cache once a +// request fails to reach the Hub at all. An empty hub turns the fallback off. +func UseCache(hub string, offline bool) { + cache.Lock() + defer cache.Unlock() + cache.hub, cache.offline, cache.downUntil = hub, offline, time.Time{} +} + +// Offline reports whether the readers are answering from the cache: by +// choice, or because the Hub did not answer. +func Offline() bool { + cache.Lock() + defer cache.Unlock() + return cache.hub != "" && (cache.offline || time.Now().Before(cache.downUntil)) +} + +// local is the cache to read, "" when the Hub is the source. +func local() string { + cache.Lock() + defer cache.Unlock() + if cache.hub != "" && (cache.offline || time.Now().Before(cache.downUntil)) { + return cache.hub + } + return "" +} + +// canFallBack is the cache to read after a request failed to reach the Hub, +// and notes that it did not. A cache with no models is no fallback: the +// request's own error says more. +func canFallBack(ctx context.Context, err error) string { + if err == nil || ctx.Err() != nil { + return "" + } + cache.Lock() + defer cache.Unlock() + if cache.hub == "" || !hfcache.Has(cache.hub) { + return "" + } + cache.downUntil = time.Now().Add(retryHubAfter) + return cache.hub +} + +// rawGet reads one file of a repo at main, from the Hub or, when that is not +// being used, from the cached snapshot. A cached repo without the file is a +// 404, as the Hub would say. +func rawGet(ctx context.Context, repo, name string) (*http.Response, error) { + hub := local() + if hub == "" { + resp, err := get(ctx, baseURL+"/"+repo+"/raw/main/"+name) + if hub = canFallBack(ctx, err); hub == "" { + return resp, err + } + } + dir, _, ok := hfcache.Snapshot(hub, repo) + if !ok { + return nil, fmt.Errorf("%s: %w", repo, ErrOffline) + } + f, err := os.Open(filepath.Join(dir, filepath.FromSlash(name))) + if err != nil { + return &http.Response{StatusCode: http.StatusNotFound, Body: http.NoBody}, nil + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(f), ContentLength: -1}, nil +} + +// localModel is Lookup's answer for a cached repo. +func localModel(hub, repo string) (Model, error) { + if _, _, ok := hfcache.Snapshot(hub, repo); !ok { + return Model{}, fmt.Errorf("%s: %w", repo, ErrOffline) + } + return Model{ID: repo}, nil +} + +// localTree is repoTree for a cached repo: the files its snapshot holds. +func localTree(hub, repo string) ([]treeEntry, error) { + m, err := hfcache.ScanRepo(hub, repo) + if err != nil || len(m.Files) == 0 { + return nil, fmt.Errorf("%s: %w", repo, ErrOffline) + } + out := make([]treeEntry, 0, len(m.Files)) + for _, f := range m.Files { + out = append(out, treeEntry{Type: "file", Path: f.Path, Size: f.Size}) + } + return out, nil +} + +// localSearch lists the cached models whose id holds every word of query. +func localSearch(hub, query string, limit int) []Model { + models, _ := hfcache.ScanCache(hub) + words := strings.Fields(strings.ToLower(query)) + var out []Model + for _, m := range models { + id := strings.ToLower(m.Repo) + match := true + for _, w := range words { + match = match && strings.Contains(id, w) + } + if match { + out = append(out, Model{ID: m.Repo}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return out +} diff --git a/internal/hf/local_internal_test.go b/internal/hf/local_internal_test.go new file mode 100644 index 0000000..ae44d8d --- /dev/null +++ b/internal/hf/local_internal_test.go @@ -0,0 +1,213 @@ +package hf + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +const rev = "0123456789abcdef0123456789abcdef01234567" + +// cachedRepo writes a repo into a hub cache the way huggingface_hub does, +// with each snapshot file a link to a blob. +func cachedRepo(t *testing.T, hub, repoID string, files map[string]string) { + t.Helper() + root := filepath.Join(hub, "models--"+strings.ReplaceAll(repoID, "/", "--")) + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(os.MkdirAll(filepath.Join(root, "refs"), 0o755)) + must(os.WriteFile(filepath.Join(root, "refs", "main"), []byte(rev), 0o644)) + i := 0 + for name, body := range files { + i++ + blob := filepath.Join(root, "blobs", strings.Repeat(string(rune('a'+i)), 40)) + must(os.MkdirAll(filepath.Dir(blob), 0o755)) + must(os.WriteFile(blob, []byte(body), 0o644)) + link := filepath.Join(root, "snapshots", rev, filepath.FromSlash(name)) + must(os.MkdirAll(filepath.Dir(link), 0o755)) + rel, err := filepath.Rel(filepath.Dir(link), blob) + must(err) + must(os.Symlink(rel, link)) + } +} + +func useCache(t *testing.T, hub string, offline bool) { + t.Helper() + UseCache(hub, offline) + t.Cleanup(func() { UseCache("", false) }) +} + +// deadHub points the client at a server that is already closed. +func deadHub(t *testing.T) { + t.Helper() + srv := httptest.NewServer(http.NotFoundHandler()) + url := srv.URL + srv.Close() + old := baseURL + baseURL = url + t.Cleanup(func() { baseURL = old }) +} + +func TestOfflineReadsTheCachedSnapshot(t *testing.T) { + hub := t.TempDir() + cachedRepo(t, hub, repo, map[string]string{ + "config.json": `{"model_type":"qwen3","max_position_embeddings":40960}`, + "chat_template.jinja": "{{ messages }}", + "tokenizer_config.json": `{"chat_template":"x"}`, + "model.safetensors": strings.Repeat("w", 64), + }) + newHub(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("offline asked the Hub for %s", r.URL) + })) + useCache(t, hub, true) + ctx := context.Background() + + if !Offline() { + t.Fatal("Offline() is false") + } + arch, err := Config(ctx, repo) + if err != nil || arch.ModelType != "qwen3" { + t.Fatalf("Config = %+v, %v", arch, err) + } + if n, err := ContextLimit(ctx, repo); err != nil || n != 40960 { + t.Fatalf("ContextLimit = %d, %v", n, err) + } + if text, found, err := ChatTemplate(ctx, repo); err != nil || !found || text != "{{ messages }}" { + t.Fatalf("ChatTemplate = %q, %v, %v", text, found, err) + } + if m, err := Lookup(ctx, repo); err != nil || m.ID != repo { + t.Fatalf("Lookup = %+v, %v", m, err) + } + tree, err := repoTree(ctx, repo) + if err != nil || len(tree) != 4 { + t.Fatalf("repoTree = %+v, %v", tree, err) + } + found, err := Search(ctx, "qwen3 fp8", 10) + if err != nil || len(found) != 1 || found[0].ID != repo { + t.Fatalf("Search = %+v, %v", found, err) + } + if found, _ := Search(ctx, "llama", 10); len(found) != 0 { + t.Fatalf("Search matched an uncached model: %+v", found) + } +} + +func TestOfflineFileMissingFromACachedRepoIsNotFound(t *testing.T) { + hub := t.TempDir() + cachedRepo(t, hub, repo, map[string]string{"model.safetensors": "w"}) + useCache(t, hub, true) + + arch, err := Config(context.Background(), repo) + if err != nil || arch.ModelType != "" { + t.Fatalf("Config = %+v, %v; want an empty Arch like a repo with no config.json", arch, err) + } + if _, found, err := ChatTemplate(context.Background(), repo); err != nil || found { + t.Fatalf("ChatTemplate found=%v err=%v", found, err) + } +} + +func TestOfflineUncachedRepoSaysSo(t *testing.T) { + useCache(t, t.TempDir(), true) + ctx := context.Background() + if _, err := Config(ctx, repo); !errors.Is(err, ErrOffline) { + t.Fatalf("Config err = %v", err) + } + if _, err := Lookup(ctx, repo); !errors.Is(err, ErrOffline) { + t.Fatalf("Lookup err = %v", err) + } + if _, err := repoTree(ctx, repo); !errors.Is(err, ErrOffline) { + t.Fatalf("repoTree err = %v", err) + } +} + +func TestUnreachableHubFallsBackToTheCache(t *testing.T) { + hub := t.TempDir() + cachedRepo(t, hub, repo, map[string]string{"config.json": `{"model_type":"qwen3"}`}) + deadHub(t) + useCache(t, hub, false) + + if Offline() { + t.Fatal("Offline() is true before any request failed") + } + arch, err := Config(context.Background(), repo) + if err != nil || arch.ModelType != "qwen3" { + t.Fatalf("Config = %+v, %v", arch, err) + } + if !Offline() { + t.Fatal("Offline() is false after the Hub did not answer") + } +} + +func TestHubIsUsedWhenItAnswers(t *testing.T) { + hub := t.TempDir() + cachedRepo(t, hub, repo, map[string]string{"config.json": `{"model_type":"cached"}`}) + newHub(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"model_type":"hub"}`)) + })) + useCache(t, hub, false) + + arch, err := Config(context.Background(), repo) + if err != nil || arch.ModelType != "hub" { + t.Fatalf("Config = %+v, %v", arch, err) + } + if Offline() { + t.Fatal("Offline() is true while the Hub answers") + } +} + +func TestNoCacheMeansNoFallback(t *testing.T) { + deadHub(t) + UseCache("", false) + if _, err := Config(context.Background(), repo); err == nil || errors.Is(err, ErrOffline) { + t.Fatalf("Config err = %v; want the transport error", err) + } +} + +func TestACancelledRequestDoesNotMarkTheHubUnreachable(t *testing.T) { + hub := t.TempDir() + deadHub(t) + useCache(t, hub, false) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, _ = Config(ctx, repo) + if Offline() { + t.Fatal("a cancelled request marked the Hub unreachable") + } +} + +func TestHubIsTriedAgainAfterAWhile(t *testing.T) { + hub := t.TempDir() + cachedRepo(t, hub, repo, map[string]string{"config.json": `{"model_type":"cached"}`}) + newHub(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"model_type":"hub"}`)) + })) + useCache(t, hub, false) + cache.Lock() + cache.downUntil = time.Now().Add(-time.Second) + cache.Unlock() + + arch, err := Config(context.Background(), repo) + if err != nil || arch.ModelType != "hub" { + t.Fatalf("Config = %+v, %v", arch, err) + } +} + +func TestAnEmptyCacheIsNoFallback(t *testing.T) { + deadHub(t) + useCache(t, t.TempDir(), false) + if _, err := Config(context.Background(), repo); err == nil || errors.Is(err, ErrOffline) { + t.Fatalf("Config err = %v; want the transport error", err) + } + if Offline() { + t.Fatal("Offline() is true with nothing to read") + } +} diff --git a/internal/hfcache/copy.go b/internal/hfcache/copy.go new file mode 100644 index 0000000..b468d0c --- /dev/null +++ b/internal/hfcache/copy.go @@ -0,0 +1,367 @@ +package hfcache + +import ( + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "os" + "path/filepath" + "strings" +) + +// partialSuffix ends a file that is still being copied. Caches ignore +// *.incomplete, so an interrupted copy leaves nothing a loader would read. +const partialSuffix = ".llm-helper.incomplete" + +// lfsThreshold is the size above which the Hub stores a file with LFS, so its +// blob is named by sha256 rather than by git blob id. +const lfsThreshold = 10 << 20 + +// Progress is told of each file as it is copied or found already there. +type Progress func(path string, size int64, skipped bool) + +// Result is what copying one model did. +type Result struct { + Copied int + Skipped int + Bytes int64 + // Links is whether snapshot files are links to blobs, rather than files. + Links bool + // RefKept is the revision refs/main already named, when it differs from + // the one copied. It is left alone, so the new revision is not the one an + // offline load reads. + RefKept string +} + +// newSum hashes a file the way its blob is named: the sha256 of an LFS file, +// and the git blob id of any other. etag says which when the source records +// it; without one, the size does. +func newSum(etag string, size int64) hash.Hash { + if len(etag) == 64 || (etag == "" && size > lfsThreshold) { + return sha256.New() + } + h := sha1.New() + fmt.Fprintf(h, "blob %d\x00", size) + return h +} + +// copyFile copies src to tmp, hashing as it goes, and returns the blob name. +// A recorded etag that the content does not match removes tmp and fails. +func copyFile(src, tmp, etag string, size int64) (string, error) { + in, err := os.Open(src) + if err != nil { + return "", err + } + defer in.Close() + if st, err := in.Stat(); err != nil || st.Size() != size { + return "", fmt.Errorf("%s changed size while being read", src) + } + out, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + return "", err + } + h := newSum(etag, size) + n, err := io.CopyBuffer(io.MultiWriter(out, h), in, make([]byte, 1<<20)) + if err == nil && n != size { + err = fmt.Errorf("read %d of %d bytes from %s", n, size, src) + } + if err == nil { + err = out.Sync() + } + if cerr := out.Close(); err == nil { + err = cerr + } + if err != nil { + os.Remove(tmp) + return "", err + } + got := hex.EncodeToString(h.Sum(nil)) + if etag != "" && got != etag { + os.Remove(tmp) + return "", &CorruptError{Path: src, Want: etag, Got: got} + } + return got, nil +} + +// CorruptError is a file whose content does not match the hash recorded for +// it. +type CorruptError struct { + Path, Want, Got string +} + +func (e *CorruptError) Error() string { + return fmt.Sprintf("%s does not match its recorded hash (want %s, got %s): the copy it came from may be damaged", e.Path, e.Want, e.Got) +} + +// partialPath is where the file at rel is copied before it is renamed into +// place. +func partialPath(repoDir, rel string) string { + sum := sha256.Sum256([]byte(rel)) + return filepath.Join(repoDir, "blobs", hex.EncodeToString(sum[:8])+partialSuffix) +} + +// sizeIs reports whether p is a file of the given size. +func sizeIs(p string, size int64) bool { + st, err := os.Stat(p) + return err == nil && st.Mode().IsRegular() && st.Size() == size +} + +// symlink is os.Symlink, so tests can take links away. +var symlink = os.Symlink + +// canLink reports whether dir takes symbolic links. +func canLink(dir string) bool { + p := filepath.Join(dir, ".llm-helper-link-test") + os.Remove(p) + if symlink("x", p) != nil { + return false + } + os.Remove(p) + return true +} + +// place copies a complete model into hub. With links, files become blobs named +// by their hash and snapshot entries relative links to them, as +// huggingface_hub writes them; without, the snapshot holds the files. +// Whatever is already there is kept and never overwritten. +func place(m Model, hub string, links bool, progress Progress) (Result, error) { + res := Result{Links: links} + if !m.Complete() { + return res, fmt.Errorf("%s is incomplete: %s", m.Repo, m.Problems[0]) + } + repoDir := RepoDir(hub, m.Repo) + snap := filepath.Join(repoDir, "snapshots", m.Revision) + for _, d := range []string{filepath.Join(repoDir, "blobs"), snap} { + if err := os.MkdirAll(d, 0o755); err != nil { + return res, err + } + } + for _, f := range m.Files { + dst := filepath.Join(snap, filepath.FromSlash(f.Path)) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return res, err + } + var copied bool + var err error + if links { + copied, err = placeBlobLink(repoDir, dst, f) + } else { + copied, err = placeFile(repoDir, dst, f) + } + if err != nil { + return res, fmt.Errorf("%s: %w", f.Path, err) + } + if copied { + res.Copied++ + res.Bytes += f.Size + } else { + res.Skipped++ + } + if progress != nil { + progress(f.Path, f.Size, !copied) + } + } + kept, err := writeRef(repoDir, m.Revision) + if err != nil { + return res, err + } + res.RefKept = kept + return res, writeTree(repoDir, m.Revision, m.tree) +} + +// placeFile puts f at dst as a regular file. +func placeFile(repoDir, dst string, f File) (copied bool, err error) { + if _, err := os.Lstat(dst); err == nil { + if sizeIs(dst, f.Size) { + return false, nil + } + return false, fmt.Errorf("%s exists with different content", dst) + } + tmp := partialPath(repoDir, f.Path) + if _, err := copyFile(f.Abs, tmp, f.Etag, f.Size); err != nil { + return false, err + } + return true, os.Rename(tmp, dst) +} + +// placeBlobLink puts f's content in blobs/ and links dst to it. +func placeBlobLink(repoDir, dst string, f File) (copied bool, err error) { + name := f.Etag + if name == "" || !sizeIs(filepath.Join(repoDir, "blobs", name), f.Size) { + tmp := partialPath(repoDir, f.Path) + got, err := copyFile(f.Abs, tmp, f.Etag, f.Size) + if err != nil { + return false, err + } + name = got + blob := filepath.Join(repoDir, "blobs", name) + switch _, err := os.Lstat(blob); { + case err != nil: + if err := os.Rename(tmp, blob); err != nil { + return false, err + } + copied = true + case sizeIs(blob, f.Size): + os.Remove(tmp) + default: + os.Remove(tmp) + return false, fmt.Errorf("blob %s exists with a different size", name) + } + } + if _, err := os.Lstat(dst); err == nil { + if sizeIs(dst, f.Size) { + return copied, nil + } + return false, fmt.Errorf("%s exists with different content", dst) + } + up := strings.Repeat("../", 2+strings.Count(f.Path, "/")) + return copied, symlink(up+"blobs/"+name, dst) +} + +// writeRef makes refs/main name rev when the cache has none. A ref that names +// another revision is left as it is, and returned. +func writeRef(repoDir, rev string) (kept string, err error) { + p := filepath.Join(repoDir, "refs", "main") + if b, err := os.ReadFile(p); err == nil { + if have := strings.TrimSpace(string(b)); have != rev { + return have, nil + } + return "", nil + } + return "", writeAtomic(p, []byte(rev)) +} + +// writeTree records the source's listing for rev, which names exactly the +// files that were copied. An existing listing is kept. +func writeTree(repoDir, rev string, raw []byte) error { + if len(raw) == 0 { + return nil + } + p := filepath.Join(repoDir, "trees", rev+".json") + if _, err := os.Stat(p); err == nil { + return nil + } + return writeAtomic(p, raw) +} + +func writeAtomic(p string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + return err + } + tmp := p + partialSuffix + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, p) +} + +// Import copies a complete model into the hub cache at hub, so a server finds +// it as if it had been downloaded. Files already there are skipped, which +// lets an interrupted import resume. Every file is hashed as it is copied and +// checked against its recorded hash. +func Import(m Model, hub string, progress Progress) (Result, error) { + if err := os.MkdirAll(hub, 0o755); err != nil { + return Result{}, err + } + return place(m, hub, canLink(hub), progress) +} + +// Export copies a complete model onto a drive as a cache with no links, which +// every filesystem a drive is likely to have can hold, and which is itself a +// cache that can be read offline. +func Export(m Model, hub string, progress Progress) (Result, error) { + if err := os.MkdirAll(hub, 0o755); err != nil { + return Result{}, err + } + return place(m, hub, false, progress) +} + +// ErrNoSpace is wrapped by the error a copy that would not fit returns. +var ErrNoSpace = errors.New("not enough free space") + +// Need is the bytes copying the models into hub would write. It counts a file +// as already there when it is, either as a blob or in the snapshot. +func Need(models []Model, hub string) int64 { + var n int64 + for _, m := range models { + snap := filepath.Join(RepoDir(hub, m.Repo), "snapshots", m.Revision) + for _, f := range m.Files { + if sizeIs(filepath.Join(snap, filepath.FromSlash(f.Path)), f.Size) { + continue + } + if f.Etag != "" && sizeIs(filepath.Join(RepoDir(hub, m.Repo), "blobs", f.Etag), f.Size) { + continue + } + n += f.Size + } + } + return n +} + +// Free is the bytes free where dir is, or would be created. ok is false when +// that cannot be told. +func Free(dir string) (free int64, ok bool) { + probe := dir + for { + if _, err := os.Stat(probe); err == nil { + break + } + next := filepath.Dir(probe) + if next == probe { + return 0, false + } + probe = next + } + free, err := freeBytes(probe) + return free, err == nil +} + +// CheckSpace fails when need bytes do not fit where dir is, or will be +// created. +func CheckSpace(dir string, need int64) error { + free, ok := Free(dir) + if !ok || need <= free { + return nil + } + return fmt.Errorf("%w on %s: need %s, %s free", ErrNoSpace, dir, FormatBytes(need), FormatBytes(free)) +} + +// isFAT is onFAT, so tests can name a drive FAT32. +var isFAT = onFAT + +// fatFileLimit is the largest file FAT32 holds. +const fatFileLimit = 4<<30 - 1 + +// CheckDrive fails before an export starts when the models cannot be written +// to the drive at dir: FAT32 cannot hold a file over 4 GiB, and the rest must +// fit in the free space. +func CheckDrive(models []Model, dir string) error { + if isFAT(dir) { + for _, m := range models { + for _, f := range m.Files { + if f.Size > fatFileLimit { + return fmt.Errorf("%s is %s, and %s is formatted as FAT32, which cannot hold a file over 4 GiB: reformat the drive as exFAT (or use ext4 or APFS)", f.Path, FormatBytes(f.Size), dir) + } + } + } + } + return CheckSpace(dir, Need(models, filepath.Join(dir, "hub"))) +} + +// FormatBytes writes n the way a person reads a disk: 14.5 GB. +func FormatBytes(n int64) string { + const unit = 1000 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for m := n / unit; m >= unit; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(n)/float64(div), "kMGTPE"[exp]) +} diff --git a/internal/hfcache/hfcache.go b/internal/hfcache/hfcache.go new file mode 100644 index 0000000..d417dda --- /dev/null +++ b/internal/hfcache/hfcache.go @@ -0,0 +1,139 @@ +// Package hfcache locates, reads and writes Hugging Face hub caches, so models +// can move between machines on a drive without a download. +// +// It speaks the layout huggingface_hub, vLLM, mlx-lm and llama.cpp share: +// models--org--name/{blobs, snapshots/, refs/main, trees/.json}. +package hfcache + +import ( + "os" + "path/filepath" + "regexp" + "strings" +) + +// Env looks up one environment variable, "" when unset. +type Env func(key string) string + +// OSEnv reads the process environment. +func OSEnv(key string) string { return os.Getenv(key) } + +// Dir is the hub cache directory a server reads, given the environment it +// starts with. The first of extra that is set wins, then HF_HUB_CACHE, +// HUGGINGFACE_HUB_CACHE, $HF_HOME/hub and $XDG_CACHE_HOME/huggingface/hub, and +// last ~/.cache/huggingface/hub under home. +func Dir(env Env, home string, extra ...string) string { + get := func(k string) string { return expandHome(strings.TrimSpace(env(k)), home) } + for _, k := range extra { + if v := get(k); v != "" { + return v + } + } + for _, k := range []string{"HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE"} { + if v := get(k); v != "" { + return v + } + } + if v := get("HF_HOME"); v != "" { + return filepath.Join(v, "hub") + } + if v := get("XDG_CACHE_HOME"); v != "" { + return filepath.Join(v, "huggingface", "hub") + } + return filepath.Join(home, ".cache", "huggingface", "hub") +} + +// expandHome resolves a leading ~ and $HOME, which a sourced env file would +// have expanded. +func expandHome(v, home string) string { + switch { + case v == "~": + return home + case strings.HasPrefix(v, "~/"): + return filepath.Join(home, v[2:]) + } + v = strings.ReplaceAll(v, "${HOME}", home) + return strings.ReplaceAll(v, "$HOME", home) +} + +// Offline reports whether HF_HUB_OFFLINE or TRANSFORMERS_OFFLINE is set to a +// true value, as huggingface_hub reads them. +func Offline(env Env) bool { + for _, k := range []string{"HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"} { + switch strings.ToUpper(strings.TrimSpace(env(k))) { + case "1", "ON", "YES", "TRUE": + return true + } + } + return false +} + +var ( + repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) + revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + hexPattern = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`) +) + +const repoPrefix = "models--" + +// ValidRepo reports whether repo is an org/name Hub id. +func ValidRepo(repo string) bool { + return repoPattern.MatchString(repo) && !strings.Contains(repo, "--") && strings.Trim(repo[strings.IndexByte(repo, '/')+1:], ".") != "" +} + +// RepoDir is where a repo lives under a hub cache. +func RepoDir(hub, repo string) string { + return filepath.Join(hub, repoPrefix+strings.ReplaceAll(repo, "/", "--")) +} + +// repoFromDir is the repo a models--org--name folder holds. +func repoFromDir(name string) (string, bool) { + rest, ok := strings.CutPrefix(name, repoPrefix) + if !ok { + return "", false + } + org, model, ok := strings.Cut(rest, "--") + if !ok { + return "", false + } + repo := org + "/" + model + return repo, ValidRepo(repo) +} + +// Snapshot is the directory holding repo's refs/main revision in a hub cache, +// the one an offline load reads. +func Snapshot(hub, repo string) (dir, revision string, ok bool) { + rev, err := mainRevision(RepoDir(hub, repo)) + if err != nil { + return "", "", false + } + dir = filepath.Join(RepoDir(hub, repo), "snapshots", rev) + if st, err := os.Stat(dir); err != nil || !st.IsDir() { + return "", "", false + } + return dir, rev, true +} + +// mainRevision reads refs/main, or names the one snapshot a repo has when it +// has no ref. +func mainRevision(repoDir string) (string, error) { + if b, err := os.ReadFile(filepath.Join(repoDir, "refs", "main")); err == nil { + if rev := strings.TrimSpace(string(b)); revisionPattern.MatchString(rev) { + return rev, nil + } + } + entries, err := os.ReadDir(filepath.Join(repoDir, "snapshots")) + if err != nil { + return "", err + } + var revs []string + for _, e := range entries { + if e.IsDir() && revisionPattern.MatchString(e.Name()) { + revs = append(revs, e.Name()) + } + } + if len(revs) != 1 { + return "", os.ErrNotExist + } + return revs[0], nil +} diff --git a/internal/hfcache/hfcache_internal_test.go b/internal/hfcache/hfcache_internal_test.go new file mode 100644 index 0000000..3adf443 --- /dev/null +++ b/internal/hfcache/hfcache_internal_test.go @@ -0,0 +1,731 @@ +package hfcache + +import ( + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +const rev = "0123456789abcdef0123456789abcdef01234567" + +func gitID(content string) string { + h := sha1.New() + fmt.Fprintf(h, "blob %d\x00%s", len(content), content) + return hex.EncodeToString(h.Sum(nil)) +} + +func sha(content string) string { + s := sha256.Sum256([]byte(content)) + return hex.EncodeToString(s[:]) +} + +func write(t *testing.T, p, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func read(t *testing.T, p string) string { + t.Helper() + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// cacheOpts shape a cache fixture. +type cacheOpts struct { + // noLinks writes real files in the snapshot and no blobs, as a cache on + // exFAT does. + noLinks bool + // tree writes trees/.json. + tree bool + // lfs names the files stored by sha256 rather than git blob id. + lfs []string + // shared puts blob content in the cache-wide store, as newer caches do. + shared bool +} + +var modelFiles = map[string]string{ + "config.json": `{"model_type":"toy"}`, + "model.safetensors.index.json": `{"weight_map":{"a":"model-1.safetensors","b":"sub/model-2.safetensors"}}`, + "model-1.safetensors": "weights one", + "sub/model-2.safetensors": "weights two", +} + +// makeCache writes a hub cache holding repo, and returns the hub directory. +func makeCache(t *testing.T, repo string, files map[string]string, o cacheOpts) string { + t.Helper() + hub := filepath.Join(t.TempDir(), "hub") + repoDir := RepoDir(hub, repo) + tree := map[string]treeFile{} + for _, name := range sortedKeys(files) { + content := files[name] + etag, tf := gitID(content), treeFile{Size: int64(len(content)), BlobID: gitID(content)} + if slices.Contains(o.lfs, name) { + etag = sha(content) + tf.LFSSha256 = etag + tf.BlobID = gitID("pointer " + name) + } + tree[name] = tf + dst := filepath.Join(repoDir, "snapshots", rev, filepath.FromSlash(name)) + if o.noLinks { + write(t, dst, content) + continue + } + blob := filepath.Join(repoDir, "blobs", etag) + if o.shared { + store := filepath.Join(hub, "blobs", etag[:2], etag) + write(t, store, content) + if err := os.MkdirAll(filepath.Dir(blob), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join("..", "..", "blobs", etag[:2], etag), blob); err != nil { + t.Fatal(err) + } + } else { + write(t, blob, content) + } + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + t.Fatal(err) + } + up := strings.Repeat("../", 2+strings.Count(name, "/")) + if err := os.Symlink(up+"blobs/"+etag, dst); err != nil { + t.Fatal(err) + } + } + write(t, filepath.Join(repoDir, "refs", "main"), rev) + if o.tree { + raw, _ := json.Marshal(map[string]any{"format_version": 1, "files": tree}) + write(t, filepath.Join(repoDir, "trees", rev+".json"), string(raw)) + } + return hub +} + +func scanOne(t *testing.T, hub, repo string) Model { + t.Helper() + m, err := ScanRepo(hub, repo) + if err != nil { + t.Fatal(err) + } + return m +} + +func TestDirPrecedence(t *testing.T) { + t.Parallel() + home := "/home/u" + for _, tc := range []struct { + name string + env map[string]string + extra []string + want string + }{ + {"default", nil, nil, "/home/u/.cache/huggingface/hub"}, + {"xdg", map[string]string{"XDG_CACHE_HOME": "/xdg"}, nil, "/xdg/huggingface/hub"}, + {"hf home beats xdg", map[string]string{"XDG_CACHE_HOME": "/xdg", "HF_HOME": "/hf"}, nil, "/hf/hub"}, + {"old hub cache beats hf home", map[string]string{"HF_HOME": "/hf", "HUGGINGFACE_HUB_CACHE": "/old"}, nil, "/old"}, + {"hub cache beats old", map[string]string{"HUGGINGFACE_HUB_CACHE": "/old", "HF_HUB_CACHE": "/new"}, nil, "/new"}, + {"server variable beats all", map[string]string{"HF_HUB_CACHE": "/new", "LLAMA_CACHE": "/llama"}, []string{"LLAMA_CACHE"}, "/llama"}, + {"server variable unset", map[string]string{"HF_HUB_CACHE": "/new"}, []string{"LLAMA_CACHE"}, "/new"}, + {"tilde", map[string]string{"HF_HOME": "~/models"}, nil, "/home/u/models/hub"}, + {"$HOME", map[string]string{"HF_HUB_CACHE": "$HOME/hub"}, nil, "/home/u/hub"}, + {"blank is unset", map[string]string{"HF_HUB_CACHE": " ", "HF_HOME": "/hf"}, nil, "/hf/hub"}, + } { + got := Dir(func(k string) string { return tc.env[k] }, home, tc.extra...) + if got != tc.want { + t.Errorf("%s: Dir = %q, want %q", tc.name, got, tc.want) + } + } +} + +func TestOffline(t *testing.T) { + t.Parallel() + for v, want := range map[string]bool{"1": true, "true": true, "TRUE": true, "on": true, "yes": true, "0": false, "": false, "false": false, "no": false} { + for _, key := range []string{"HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"} { + if got := Offline(func(k string) string { + if k == key { + return v + } + return "" + }); got != want { + t.Errorf("%s=%q: Offline = %v, want %v", key, v, got, want) + } + } + } +} + +func TestScanReadsALinkedCache(t *testing.T) { + t.Parallel() + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{tree: true, lfs: []string{"model-1.safetensors"}}) + m := scanOne(t, hub, "org/toy") + if !m.Complete() || m.Revision != rev || len(m.Files) != 4 { + t.Fatalf("model = %+v", m) + } + for _, f := range m.Files { + want := gitID(modelFiles[f.Path]) + if f.Path == "model-1.safetensors" { + want = sha(modelFiles[f.Path]) + } + if f.Etag != want { + t.Errorf("%s etag = %s, want %s", f.Path, f.Etag, want) + } + } + if m.Size() != int64(len(modelFiles["config.json"])+len(modelFiles["model.safetensors.index.json"])+len("weights one")+len("weights two")) { + t.Errorf("Size = %d", m.Size()) + } +} + +func TestScanReadsACacheWithoutLinks(t *testing.T) { + t.Parallel() + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{noLinks: true, tree: true, lfs: []string{"model-1.safetensors"}}) + m := scanOne(t, hub, "org/toy") + if !m.Complete() { + t.Fatalf("problems: %v", m.Problems) + } + // With no blob names to read, the listing says what each file is called. + for _, f := range m.Files { + if f.Path == "model-1.safetensors" && f.Etag != sha("weights one") { + t.Errorf("etag = %q, want the listing's lfs_sha256", f.Etag) + } + if f.Etag == "" { + t.Errorf("%s has no etag", f.Path) + } + } + hub = makeCache(t, "org/toy", modelFiles, cacheOpts{noLinks: true}) + for _, f := range scanOne(t, hub, "org/toy").Files { + if f.Etag != "" { + t.Errorf("%s: etag %q with nothing to say it", f.Path, f.Etag) + } + } +} + +func TestScanFindsTheCacheWhereverItSits(t *testing.T) { + t.Parallel() + for _, sub := range []string{"", "hub", "huggingface/hub", ".cache/huggingface/hub"} { + root := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + if sub != "" { + root = t.TempDir() + dst := filepath.Join(root, filepath.FromSlash(sub)) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Rename(makeCache(t, "org/toy", modelFiles, cacheOpts{}), dst); err != nil { + t.Fatal(err) + } + } + found, err := Scan(root, ScanOptions{}) + if err != nil || found.Kind != KindCache || len(found.Models) != 1 || found.Models[0].Repo != "org/toy" { + t.Errorf("%q: Scan = %+v, %v", sub, found, err) + } + } + // One copied models--org--name folder is enough. + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + found, err := Scan(RepoDir(hub, "org/toy"), ScanOptions{}) + if err != nil || len(found.Models) != 1 { + t.Errorf("single repo folder: %+v, %v", found, err) + } +} + +func TestScanSkipsWhatIsNotAModel(t *testing.T) { + t.Parallel() + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + for _, junk := range []string{"datasets--org--data", ".locks", "models--bad", "CACHEDIR.TAG"} { + write(t, filepath.Join(hub, junk, "x"), "x") + } + write(t, filepath.Join(RepoDir(hub, "org/toy"), "blobs", "abc.incomplete"), "partial") + models, err := ScanCache(hub) + if err != nil || len(models) != 1 || !models[0].Complete() { + t.Fatalf("ScanCache = %+v, %v", models, err) + } +} + +func TestScanFollowsTheSharedBlobStore(t *testing.T) { + t.Parallel() + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{shared: true}) + m := scanOne(t, hub, "org/toy") + if !m.Complete() { + t.Fatalf("problems: %v", m.Problems) + } + // The name is the repo's blob link, not the store's path. + for _, f := range m.Files { + if f.Etag != gitID(modelFiles[f.Path]) { + t.Errorf("%s etag = %s", f.Path, f.Etag) + } + } + drive := t.TempDir() + if _, err := Export(m, filepath.Join(drive, "hub"), nil); err != nil { + t.Fatal(err) + } + p := filepath.Join(RepoDir(filepath.Join(drive, "hub"), "org/toy"), "snapshots", rev, "model-1.safetensors") + if st, err := os.Lstat(p); err != nil || !st.Mode().IsRegular() || read(t, p) != "weights one" { + t.Errorf("exported file: %v, %v", st, err) + } +} + +func TestScanFlagsIncompleteModels(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + mutate func(t *testing.T, hub string) + want string + }{ + {"dangling link", func(t *testing.T, hub string) { + os.Remove(filepath.Join(RepoDir(hub, "org/toy"), "blobs", gitID("weights two"))) + }, "dangling link: sub/model-2.safetensors"}, + {"a file the listing names is missing", func(t *testing.T, hub string) { + os.Remove(filepath.Join(RepoDir(hub, "org/toy"), "snapshots", rev, "config.json")) + }, "missing config.json"}, + {"size differs from the listing", func(t *testing.T, hub string) { + write(t, filepath.Join(RepoDir(hub, "org/toy"), "blobs", gitID("weights one")), "short") + }, "the listing says"}, + {"a shard the index names is missing", func(t *testing.T, hub string) { + os.Remove(filepath.Join(RepoDir(hub, "org/toy"), "snapshots", rev, "model-1.safetensors")) + }, "missing model-1.safetensors"}, + } { + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{tree: tc.name != "a shard the index names is missing"}) + tc.mutate(t, hub) + m := scanOne(t, hub, "org/toy") + if m.Complete() || !strings.Contains(strings.Join(m.Problems, "; "), tc.want) { + t.Errorf("%s: problems = %v, want %q", tc.name, m.Problems, tc.want) + } + } + // Config only is what an interrupted first download leaves. + hub := makeCache(t, "org/toy", map[string]string{"config.json": "{}"}, cacheOpts{}) + if m := scanOne(t, hub, "org/toy"); m.Complete() || m.Problems[0] != "no weight files" { + t.Errorf("config only: %v", m.Problems) + } + // Two snapshots and no ref leave nothing to choose. + hub = makeCache(t, "org/toy", modelFiles, cacheOpts{}) + os.Remove(filepath.Join(RepoDir(hub, "org/toy"), "refs", "main")) + if err := os.MkdirAll(filepath.Join(RepoDir(hub, "org/toy"), "snapshots", strings.Repeat("b", 40)), 0o755); err != nil { + t.Fatal(err) + } + if m := scanOne(t, hub, "org/toy"); m.Complete() { + t.Error("a repo with two snapshots and no refs/main was complete") + } +} + +func TestImportWritesTheCacheHuggingFaceReads(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{noLinks: true, tree: true, lfs: []string{"model-1.safetensors"}}) + dst := filepath.Join(t.TempDir(), "hub") + var seen []string + res, err := Import(scanOne(t, src, "org/toy"), dst, func(p string, _ int64, skipped bool) { + seen = append(seen, fmt.Sprint(p, skipped)) + }) + if err != nil || !res.Links || res.Copied != 4 || res.Skipped != 0 || len(seen) != 4 { + t.Fatalf("Import = %+v, %v, %v", res, err, seen) + } + repoDir := RepoDir(dst, "org/toy") + for name, content := range modelFiles { + p := filepath.Join(repoDir, "snapshots", rev, filepath.FromSlash(name)) + target, err := os.Readlink(p) + if err != nil { + t.Fatalf("%s is not a link: %v", name, err) + } + want := gitID(content) + if name == "model-1.safetensors" { + want = sha(content) + } + wantTarget := strings.Repeat("../", 2+strings.Count(name, "/")) + "blobs/" + want + if target != wantTarget || filepath.IsAbs(target) { + t.Errorf("%s -> %s, want %s", name, target, wantTarget) + } + if read(t, p) != content { + t.Errorf("%s content differs", name) + } + } + if got := read(t, filepath.Join(repoDir, "refs", "main")); got != rev { + t.Errorf("refs/main = %q, want the bare commit", got) + } + if read(t, filepath.Join(repoDir, "trees", rev+".json")) != read(t, filepath.Join(RepoDir(src, "org/toy"), "trees", rev+".json")) { + t.Error("the tree listing was not carried over") + } + if m := scanOne(t, dst, "org/toy"); !m.Complete() { + t.Errorf("what was imported does not scan complete: %v", m.Problems) + } + entries, _ := os.ReadDir(filepath.Join(repoDir, "blobs")) + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".incomplete") { + t.Errorf("left %s behind", e.Name()) + } + } +} + +func TestImportResumes(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + m := scanOne(t, src, "org/toy") + dst := filepath.Join(t.TempDir(), "hub") + if _, err := Import(m, dst, nil); err != nil { + t.Fatal(err) + } + res, err := Import(m, dst, nil) + if err != nil || res.Copied != 0 || res.Skipped != 4 { + t.Errorf("second import = %+v, %v; want everything skipped", res, err) + } + + // An interrupted import: one blob and its link gone, a partial left. + repoDir := RepoDir(dst, "org/toy") + os.Remove(filepath.Join(repoDir, "snapshots", rev, "model-1.safetensors")) + os.Remove(filepath.Join(repoDir, "blobs", gitID("weights one"))) + write(t, partialPath(repoDir, "model-1.safetensors"), "half") + res, err = Import(m, dst, nil) + if err != nil || res.Copied != 1 || res.Skipped != 3 { + t.Errorf("resumed import = %+v, %v; want one file copied", res, err) + } + if read(t, filepath.Join(repoDir, "snapshots", rev, "model-1.safetensors")) != "weights one" { + t.Error("the resumed file is wrong") + } +} + +func TestImportCatchesADamagedSource(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + // Same length, same name, different bytes. + write(t, filepath.Join(RepoDir(src, "org/toy"), "blobs", gitID("weights two")), "weights twX") + dst := filepath.Join(t.TempDir(), "hub") + _, err := Import(scanOne(t, src, "org/toy"), dst, nil) + var corrupt *CorruptError + if !errors.As(err, &corrupt) { + t.Fatalf("Import = %v, want a CorruptError", err) + } + repoDir := RepoDir(dst, "org/toy") + if _, err := os.Stat(filepath.Join(repoDir, "blobs", gitID("weights twX"))); err == nil { + t.Error("the damaged file was kept") + } + if _, err := os.Lstat(filepath.Join(repoDir, "snapshots", rev, "sub", "model-2.safetensors")); err == nil { + t.Error("a link to the damaged file was made") + } + if _, err := os.Stat(filepath.Join(repoDir, "refs", "main")); err == nil { + t.Error("refs/main was written for a model that did not finish") + } +} + +func TestImportFallsBackToFilesWhenLinksFail(t *testing.T) { + src := makeCache(t, "org/toy", modelFiles, cacheOpts{tree: true}) + symlink = func(string, string) error { return errors.New("operation not permitted") } + t.Cleanup(func() { symlink = os.Symlink }) + dst := filepath.Join(t.TempDir(), "hub") + res, err := Import(scanOne(t, src, "org/toy"), dst, nil) + if err != nil || res.Links { + t.Fatalf("Import = %+v, %v", res, err) + } + p := filepath.Join(RepoDir(dst, "org/toy"), "snapshots", rev, "sub", "model-2.safetensors") + if st, err := os.Lstat(p); err != nil || !st.Mode().IsRegular() || read(t, p) != "weights two" { + t.Errorf("snapshot file: %v, %v", st, err) + } + if m := scanOne(t, dst, "org/toy"); !m.Complete() { + t.Errorf("problems: %v", m.Problems) + } +} + +func TestImportRefusesIncompleteModels(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + os.Remove(filepath.Join(RepoDir(src, "org/toy"), "snapshots", rev, "model-1.safetensors")) + dst := filepath.Join(t.TempDir(), "hub") + if _, err := Import(scanOne(t, src, "org/toy"), dst, nil); err == nil || !strings.Contains(err.Error(), "incomplete") { + t.Errorf("Import = %v", err) + } + if _, err := os.Stat(RepoDir(dst, "org/toy")); err == nil { + t.Error("an incomplete model left files behind") + } +} + +func TestImportKeepsWhatIsThere(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{tree: true}) + dst := filepath.Join(t.TempDir(), "hub") + repoDir := RepoDir(dst, "org/toy") + other := strings.Repeat("c", 40) + write(t, filepath.Join(repoDir, "refs", "main"), other) + write(t, filepath.Join(repoDir, "trees", rev+".json"), `{"format_version":1,"files":{}}`) + res, err := Import(scanOne(t, src, "org/toy"), dst, nil) + if err != nil || res.RefKept != other { + t.Fatalf("Import = %+v, %v; want refs/main kept at %s", res, err, other) + } + if read(t, filepath.Join(repoDir, "refs", "main")) != other { + t.Error("refs/main was overwritten") + } + if !strings.Contains(read(t, filepath.Join(repoDir, "trees", rev+".json")), `"files":{}`) { + t.Error("an existing tree listing was overwritten") + } + + // A blob of the right name and the wrong size is not ours to replace. + dst2 := filepath.Join(t.TempDir(), "hub") + write(t, filepath.Join(RepoDir(dst2, "org/toy"), "blobs", gitID("weights two")), "not what its name says") + if _, err := Import(scanOne(t, src, "org/toy"), dst2, nil); err == nil || !strings.Contains(err.Error(), "different size") { + t.Errorf("Import over a bad blob = %v", err) + } +} + +func TestExportWritesAReadableCacheWithoutLinks(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{tree: true}) + drive := filepath.Join(t.TempDir(), "hub") + if _, err := Export(scanOne(t, src, "org/toy"), drive, nil); err != nil { + t.Fatal(err) + } + filepath.WalkDir(drive, func(p string, d os.DirEntry, err error) error { + if err == nil && d.Type()&os.ModeSymlink != 0 { + t.Errorf("%s is a link", p) + } + return nil + }) + m := scanOne(t, drive, "org/toy") + if !m.Complete() || len(m.Files) != 4 || m.Revision != rev { + t.Fatalf("exported model = %+v", m) + } + if read(t, filepath.Join(RepoDir(drive, "org/toy"), "refs", "main")) != rev { + t.Error("refs/main") + } + + // And it imports into a linked cache, files verified against the tree. + dst := filepath.Join(t.TempDir(), "hub") + res, err := Import(m, dst, nil) + if err != nil || !res.Links || res.Copied != 4 { + t.Fatalf("Import = %+v, %v", res, err) + } + if _, err := os.Stat(filepath.Join(RepoDir(dst, "org/toy"), "blobs", gitID("weights two"))); err != nil { + t.Error("blobs are not named by the etag the listing gave") + } +} + +func TestExportRefusesIncompleteModels(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + os.Remove(filepath.Join(RepoDir(src, "org/toy"), "blobs", gitID("weights one"))) + drive := filepath.Join(t.TempDir(), "hub") + if _, err := Export(scanOne(t, src, "org/toy"), drive, nil); err == nil { + t.Error("an incomplete model was exported") + } +} + +func TestExportCatchesADamagedCache(t *testing.T) { + t.Parallel() + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + write(t, filepath.Join(RepoDir(src, "org/toy"), "blobs", gitID("weights one")), "weights onX") + _, err := Export(scanOne(t, src, "org/toy"), filepath.Join(t.TempDir(), "hub"), nil) + var corrupt *CorruptError + if !errors.As(err, &corrupt) { + t.Errorf("Export = %v, want a CorruptError", err) + } +} + +func TestCheckSpaceRefusesWhatWillNotFit(t *testing.T) { + prev := freeBytes + freeBytes = func(string) (int64, error) { return 100, nil } + t.Cleanup(func() { freeBytes = prev }) + src := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + m := scanOne(t, src, "org/toy") + dst := filepath.Join(t.TempDir(), "not", "yet", "hub") + need := Need([]Model{m}, dst) + if need != m.Size() { + t.Fatalf("Need = %d, want %d for an empty cache", need, m.Size()) + } + if err := CheckSpace(dst, 101); !errors.Is(err, ErrNoSpace) { + t.Errorf("CheckSpace(101 of 100) = %v", err) + } + if err := CheckSpace(dst, 100); err != nil { + t.Errorf("CheckSpace(100 of 100) = %v", err) + } + // What an import already holds is not counted again. + if _, err := Import(m, dst, nil); err != nil { + t.Fatal(err) + } + if n := Need([]Model{m}, dst); n != 0 { + t.Errorf("Need after importing = %d, want 0", n) + } + freeBytes = func(string) (int64, error) { return 0, errors.New("unknown") } + if err := CheckSpace(dst, 1<<40); err != nil { + t.Errorf("an unreadable filesystem was refused: %v", err) + } +} + +func TestCheckDriveRefusesBigFilesOnFAT(t *testing.T) { + prevFAT, prevFree := isFAT, freeBytes + isFAT = func(string) bool { return true } + freeBytes = func(string) (int64, error) { return 1 << 50, nil } + t.Cleanup(func() { isFAT, freeBytes = prevFAT, prevFree }) + big := Model{Repo: "org/toy", Files: []File{{Path: "model.safetensors", Size: 5 << 30}}} + err := CheckDrive([]Model{big}, t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "FAT32") || !strings.Contains(err.Error(), "model.safetensors") { + t.Errorf("CheckDrive = %v", err) + } + small := Model{Repo: "org/toy", Files: []File{{Path: "model.safetensors", Size: 3 << 30}}} + if err := CheckDrive([]Model{small}, t.TempDir()); err != nil { + t.Errorf("files under 4 GiB on FAT: %v", err) + } + isFAT = func(string) bool { return false } + if err := CheckDrive([]Model{big}, t.TempDir()); err != nil { + t.Errorf("a big file on exFAT: %v", err) + } +} + +func writeLocalDir(t *testing.T, commits ...string) string { + t.Helper() + dir := t.TempDir() + for i, name := range []string{"config.json", "model.safetensors"} { + content := modelFiles[name] + if name == "model.safetensors" { + content = "local weights" + } + write(t, filepath.Join(dir, name), content) + etag := gitID(content) + commit := commits[0] + if i < len(commits) { + commit = commits[i] + } + write(t, filepath.Join(dir, ".cache", "huggingface", "download", name+".metadata"), commit+"\n"+etag+"\n1700000000.0\n") + } + return dir +} + +func TestScanReadsAnHFLocalDir(t *testing.T) { + t.Parallel() + dir := writeLocalDir(t, rev) + if _, err := Scan(dir, ScanOptions{}); err == nil || !strings.Contains(err.Error(), "--repo") { + t.Errorf("without --repo: %v", err) + } + found, err := Scan(dir, ScanOptions{Repo: "org/toy"}) + if err != nil || found.Kind != KindLocalDir || len(found.Models) != 1 { + t.Fatalf("Scan = %+v, %v", found, err) + } + m := found.Models[0] + if m.Revision != rev || !m.Complete() || len(m.Files) != 2 { + t.Fatalf("model = %+v", m) + } + for _, f := range m.Files { + if f.Path == "model.safetensors" && f.Etag != gitID("local weights") { + t.Errorf("etag = %q, want the metadata's", f.Etag) + } + } + + other := strings.Repeat("d", 40) + if _, err := Scan(writeLocalDir(t, rev, other), ScanOptions{Repo: "org/toy"}); err == nil || !strings.Contains(err.Error(), "different commits") { + t.Errorf("mixed commits: %v", err) + } + if _, err := Scan(dir, ScanOptions{Repo: "org/toy", Revision: other}); err == nil || !strings.Contains(err.Error(), "--revision") { + t.Errorf("a --revision that disagrees: %v", err) + } + write(t, filepath.Join(dir, ".cache", "huggingface", "download", "model.safetensors.abc.incomplete"), "x") + found, _ = Scan(dir, ScanOptions{Repo: "org/toy"}) + if found.Models[0].Complete() { + t.Error("a partial download was complete") + } +} + +func TestScanReadsAPlainFolder(t *testing.T) { + t.Parallel() + dir := t.TempDir() + write(t, filepath.Join(dir, "config.json"), "{}") + write(t, filepath.Join(dir, "model.safetensors"), "w") + if _, err := Scan(dir, ScanOptions{}); err == nil || !strings.Contains(err.Error(), "--repo") { + t.Errorf("without --repo: %v", err) + } + if _, err := Scan(dir, ScanOptions{Repo: "org/toy"}); err == nil || !strings.Contains(err.Error(), "--revision") { + t.Errorf("without a commit it must not invent one: %v", err) + } + if _, err := Scan(dir, ScanOptions{Repo: "org/toy", Revision: "main"}); err == nil { + t.Error("a branch name was taken for a commit") + } + if _, err := Scan(dir, ScanOptions{Repo: "toy"}); err == nil { + t.Error("a repo with no org was accepted") + } + found, err := Scan(dir, ScanOptions{Repo: "org/toy", Revision: rev}) + if err != nil || found.Kind != KindFolder || found.Models[0].Revision != rev || len(found.Models[0].Files) != 2 { + t.Fatalf("Scan = %+v, %v", found, err) + } + // Blobs of an unrecorded etag are named as the Hub names them. + dst := filepath.Join(t.TempDir(), "hub") + if _, err := Import(found.Models[0], dst, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(RepoDir(dst, "org/toy"), "blobs", gitID("w"))); err != nil { + t.Errorf("a small file is not named by its git blob id: %v", err) + } +} + +func TestScanTakesAGitCloneRevisionFromHEAD(t *testing.T) { + t.Parallel() + for name, setup := range map[string]func(t *testing.T, dir string){ + "detached": func(t *testing.T, dir string) { write(t, filepath.Join(dir, ".git", "HEAD"), rev+"\n") }, + "branch": func(t *testing.T, dir string) { + write(t, filepath.Join(dir, ".git", "HEAD"), "ref: refs/heads/main\n") + write(t, filepath.Join(dir, ".git", "refs", "heads", "main"), rev+"\n") + }, + "packed": func(t *testing.T, dir string) { + write(t, filepath.Join(dir, ".git", "HEAD"), "ref: refs/heads/main\n") + write(t, filepath.Join(dir, ".git", "packed-refs"), "# pack-refs\n"+rev+" refs/heads/main\n") + }, + } { + dir := t.TempDir() + write(t, filepath.Join(dir, "config.json"), "{}") + write(t, filepath.Join(dir, "model.safetensors"), "w") + write(t, filepath.Join(dir, ".git", "objects", "x"), "git internals") + setup(t, dir) + found, err := Scan(dir, ScanOptions{Repo: "org/toy"}) + if err != nil || found.Models[0].Revision != rev || len(found.Models[0].Files) != 2 { + t.Errorf("%s: %+v, %v", name, found, err) + } + } +} + +func TestScanFlagsGitLFSPointers(t *testing.T) { + t.Parallel() + dir := t.TempDir() + write(t, filepath.Join(dir, "config.json"), "{}") + write(t, filepath.Join(dir, "model.safetensors"), "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 123\n") + found, err := Scan(dir, ScanOptions{Repo: "org/toy", Revision: rev}) + if err != nil { + t.Fatal(err) + } + if m := found.Models[0]; m.Complete() || !strings.Contains(m.Problems[0], "git lfs pull") { + t.Errorf("problems = %v", m.Problems) + } +} + +func TestScanFindsNothingInAnEmptyFolder(t *testing.T) { + t.Parallel() + if _, err := Scan(t.TempDir(), ScanOptions{Repo: "org/toy"}); err == nil || !strings.Contains(err.Error(), "no models") { + t.Errorf("Scan = %v", err) + } + if _, err := Scan(filepath.Join(t.TempDir(), "missing"), ScanOptions{}); err == nil { + t.Error("a missing path was accepted") + } +} + +func TestSnapshotFollowsRefsMain(t *testing.T) { + t.Parallel() + hub := makeCache(t, "org/toy", modelFiles, cacheOpts{}) + dir, got, ok := Snapshot(hub, "org/toy") + if !ok || got != rev || !strings.HasSuffix(dir, filepath.Join("snapshots", rev)) { + t.Fatalf("Snapshot = %q, %q, %v", dir, got, ok) + } + if _, _, ok := Snapshot(hub, "org/other"); ok { + t.Error("a repo that is not cached has a snapshot") + } +} + +func TestFormatBytes(t *testing.T) { + t.Parallel() + for n, want := range map[int64]string{0: "0 B", 999: "999 B", 1500: "1.5 kB", 14_500_000_000: "14.5 GB"} { + if got := FormatBytes(n); got != want { + t.Errorf("FormatBytes(%d) = %q, want %q", n, got, want) + } + } +} diff --git a/internal/hfcache/scan.go b/internal/hfcache/scan.go new file mode 100644 index 0000000..4f1f203 --- /dev/null +++ b/internal/hfcache/scan.go @@ -0,0 +1,532 @@ +package hfcache + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" +) + +// Kind is the layout a source was read from. +type Kind string + +// The layouts Scan reads. +const ( + // KindCache is a hub cache: models--org--name folders. + KindCache Kind = "Hugging Face cache" + // KindLocalDir is an `hf download --local-dir` folder. + KindLocalDir Kind = "hf download --local-dir folder" + // KindFolder is a plain model folder, such as a git clone. + KindFolder Kind = "model folder" +) + +// File is one file of a model. +type File struct { + // Path is the file's path in the repo, slash-separated. + Path string + Size int64 + // Etag is the blob's name in a cache: the sha256 of an LFS file or the git + // blob hash of any other. It is "" when the source does not record it. + Etag string + // Abs is where the content is read from. + Abs string +} + +// Model is one revision of one repo found in a source. +type Model struct { + Repo string + Revision string + Files []File + // Problems say why the model cannot be copied, and are empty when it is + // complete. + Problems []string + Kind Kind + // Root is the source the model was read from. + Root string + + // tree is the source's trees/.json, copied alongside the files it + // lists. + tree []byte +} + +// Size is the total bytes of the model's files. +func (m Model) Size() int64 { + var n int64 + for _, f := range m.Files { + n += f.Size + } + return n +} + +// Complete reports whether the model has every file it needs. +func (m Model) Complete() bool { return len(m.Problems) == 0 } + +// ScanOptions name what a source cannot say about itself. +type ScanOptions struct { + // Repo and Revision identify a folder that is not a cache. + Repo string + Revision string +} + +// Found is what Scan read. +type Found struct { + Kind Kind + Models []Model +} + +// Scan reads the models at path. A Hugging Face cache is read at path itself, +// or at hub/, huggingface/hub/ or .cache/huggingface/hub/ under it, or from a +// single models--org--name folder. Otherwise path is an `hf download +// --local-dir` folder or a plain model folder, and o names its repo. +func Scan(path string, o ScanOptions) (Found, error) { + st, err := os.Stat(path) + if err != nil { + return Found{}, fmt.Errorf("read %s: %w", path, err) + } + if !st.IsDir() { + return Found{}, fmt.Errorf("%s is not a directory", path) + } + path = filepath.Clean(path) + if _, ok := repoFromDir(filepath.Base(path)); ok { + models, err := scanCache(filepath.Dir(path), filepath.Base(path)) + return Found{Kind: KindCache, Models: models}, err + } + var models []Model + seen := map[string]bool{} + for _, hub := range []string{path, filepath.Join(path, "hub"), filepath.Join(path, "huggingface", "hub"), filepath.Join(path, ".cache", "huggingface", "hub")} { + if !isHub(hub) { + continue + } + found, err := scanCache(hub, "") + if err != nil { + return Found{}, err + } + for _, m := range found { + if key := m.Repo + "@" + m.Revision; !seen[key] { + seen[key] = true + models = append(models, m) + } + } + } + if len(models) > 0 { + return Found{Kind: KindCache, Models: models}, nil + } + + if _, err := os.Stat(filepath.Join(path, ".cache", "huggingface", "download")); err == nil { + if o.Repo == "" { + return Found{}, fmt.Errorf("%s was written by `hf download --local-dir`, which does not record the repo: pass --repo org/model", path) + } + m, err := scanLocalDir(path, o) + return Found{Kind: KindLocalDir, Models: []Model{m}}, err + } + if hasFiles(path) { + if o.Repo == "" { + return Found{}, fmt.Errorf("no Hugging Face cache found at %s; to import it as a model folder, pass --repo org/model (and --revision unless it is a git clone)", path) + } + m, err := scanFolder(path, o) + return Found{Kind: KindFolder, Models: []Model{m}}, err + } + return Found{}, fmt.Errorf("found no models at %s", path) +} + +// ScanCache lists every model in a hub cache. +func ScanCache(hub string) ([]Model, error) { return scanCache(hub, "") } + +// ScanRepo reads one repo's refs/main revision from a hub cache. +func ScanRepo(hub, repo string) (Model, error) { + models, err := scanCache(hub, repoPrefix+strings.ReplaceAll(repo, "/", "--")) + if err != nil { + return Model{}, err + } + if len(models) == 0 { + return Model{}, fmt.Errorf("%s is not in the cache at %s", repo, hub) + } + return models[0], nil +} + +// Has reports whether the hub cache at hub holds any repo. +func Has(hub string) bool { return isHub(hub) } + +func isHub(dir string) bool { + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + for _, e := range entries { + if _, ok := repoFromDir(e.Name()); ok && e.IsDir() { + return true + } + } + return false +} + +func hasFiles(dir string) bool { + found := false + _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err == nil && d.IsDir() && d.Name() == ".git" { + return filepath.SkipDir + } + if err == nil && !d.IsDir() { + found = true + return filepath.SkipAll + } + return nil + }) + return found +} + +// scanCache reads the repos under hub, or only the folder named only. +func scanCache(hub, only string) ([]Model, error) { + entries, err := os.ReadDir(hub) + if err != nil { + return nil, fmt.Errorf("read %s: %w", hub, err) + } + var out []Model + for _, e := range entries { + repo, ok := repoFromDir(e.Name()) + if !ok || !e.IsDir() || (only != "" && e.Name() != only) { + continue + } + out = append(out, scanSnapshot(hub, repo)) + } + sort.Slice(out, func(i, j int) bool { return out[i].Repo < out[j].Repo }) + return out, nil +} + +// weightSuffixes are the files a model is useless without. +var weightSuffixes = []string{".safetensors", ".bin", ".gguf"} + +func scanSnapshot(hub, repo string) Model { + repoDir := RepoDir(hub, repo) + m := Model{Repo: repo, Kind: KindCache, Root: hub} + rev, err := mainRevision(repoDir) + if err != nil { + m.Problems = append(m.Problems, "no refs/main, and not exactly one snapshot to choose") + return m + } + m.Revision = rev + snap := filepath.Join(repoDir, "snapshots", rev) + if _, err := os.Stat(snap); err != nil { + m.Problems = append(m.Problems, "snapshot "+rev+" is missing") + return m + } + _ = filepath.WalkDir(snap, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + rel, _ := filepath.Rel(snap, p) + rel = filepath.ToSlash(rel) + st, err := os.Stat(p) + if err != nil { + m.Problems = append(m.Problems, "dangling link: "+rel) + return nil + } + f := File{Path: rel, Size: st.Size(), Abs: p} + if d.Type()&fs.ModeSymlink != 0 { + if target, err := os.Readlink(p); err == nil { + if name := path.Base(filepath.ToSlash(target)); hexPattern.MatchString(name) { + f.Etag = name + } + } + } + m.Files = append(m.Files, f) + return nil + }) + if raw, err := os.ReadFile(filepath.Join(repoDir, "trees", rev+".json")); err == nil { + m.tree = raw + m.applyTree(raw) + } + m.checkComplete() + return m +} + +// treeFile is one entry of trees/.json. +type treeFile struct { + Size int64 `json:"size"` + BlobID string `json:"blob_id"` + LFSSha256 string `json:"lfs_sha256"` +} + +func (t treeFile) etag() string { + if hexPattern.MatchString(t.LFSSha256) { + return t.LFSSha256 + } + if hexPattern.MatchString(t.BlobID) { + return t.BlobID + } + return "" +} + +// applyTree checks the files against the listing the cache recorded for the +// revision: huggingface_hub refuses an offline load that misses any of them. +func (m *Model) applyTree(raw []byte) { + var t struct { + Files map[string]treeFile `json:"files"` + } + if json.Unmarshal(raw, &t) != nil { + m.Problems = append(m.Problems, "trees/"+m.Revision+".json is unreadable") + return + } + have := map[string]int{} + for i, f := range m.Files { + have[f.Path] = i + } + for _, p := range sortedKeys(t.Files) { + want := t.Files[p] + i, ok := have[p] + switch { + case !ok: + m.Problems = append(m.Problems, "missing "+p) + case want.Size > 0 && m.Files[i].Size != want.Size: + m.Problems = append(m.Problems, fmt.Sprintf("%s is %d bytes, the listing says %d", p, m.Files[i].Size, want.Size)) + case m.Files[i].Etag == "": + m.Files[i].Etag = want.etag() + } + } +} + +// checkComplete adds a problem for each way the files cannot serve a model: +// none at all, no weights, or a shard an index names that is not there. +func (m *Model) checkComplete() { + if len(m.Files) == 0 { + m.Problems = append(m.Problems, "no files") + return + } + have := map[string]bool{} + weights := false + for _, f := range m.Files { + have[f.Path] = true + for _, s := range weightSuffixes { + weights = weights || strings.HasSuffix(strings.ToLower(f.Path), s) + } + } + if !weights { + m.Problems = append(m.Problems, "no weight files") + } + for _, f := range m.Files { + if !strings.HasSuffix(f.Path, ".index.json") { + continue + } + raw, err := os.ReadFile(f.Abs) + if err != nil { + continue + } + var idx struct { + WeightMap map[string]string `json:"weight_map"` + } + if json.Unmarshal(raw, &idx) != nil { + continue + } + dir := path.Dir(f.Path) + for _, name := range uniqueSorted(idx.WeightMap) { + if !have[path.Join(dir, name)] { + m.Problems = append(m.Problems, "missing "+path.Join(dir, name)+", which "+f.Path+" names") + } + } + } +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func uniqueSorted(m map[string]string) []string { + set := map[string]bool{} + for _, v := range m { + set[v] = true + } + return sortedKeys(set) +} + +// scanLocalDir reads an `hf download --local-dir` folder, whose +// .cache/huggingface/download/.metadata files record the commit and etag +// of each download: three lines, commit, etag, timestamp. +func scanLocalDir(dir string, o ScanOptions) (Model, error) { + m := Model{Repo: o.Repo, Kind: KindLocalDir, Root: dir} + if !ValidRepo(o.Repo) { + return m, fmt.Errorf("--repo %q is not an org/name Hub id", o.Repo) + } + meta := filepath.Join(dir, ".cache", "huggingface", "download") + commits := map[string]bool{} + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if p == filepath.Join(dir, ".cache") { + return filepath.SkipDir + } + return nil + } + rel, _ := filepath.Rel(dir, p) + rel = filepath.ToSlash(rel) + st, err := os.Stat(p) + if err != nil { + m.Problems = append(m.Problems, "dangling link: "+rel) + return nil + } + f := File{Path: rel, Size: st.Size(), Abs: p} + if commit, etag, ok := readMetadata(filepath.Join(meta, filepath.FromSlash(rel)+".metadata")); ok { + if revisionPattern.MatchString(commit) { + commits[commit] = true + } + if hexPattern.MatchString(etag) { + f.Etag = etag + } + } + m.Files = append(m.Files, f) + return nil + }) + if err != nil { + return m, fmt.Errorf("read %s: %w", dir, err) + } + _ = filepath.WalkDir(meta, func(p string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && strings.HasSuffix(d.Name(), ".incomplete") { + m.Problems = append(m.Problems, "partial download: "+d.Name()) + } + return nil + }) + if len(commits) > 1 { + return m, fmt.Errorf("%s holds files from %d different commits; download one revision at a time", dir, len(commits)) + } + for c := range commits { + m.Revision = c + } + if err := m.useRevision(o.Revision); err != nil { + return m, err + } + m.checkComplete() + return m, nil +} + +// useRevision settles a folder's commit, which the caller's flag may state +// and the source may also record. It never makes one up. +func (m *Model) useRevision(given string) error { + if given != "" && !revisionPattern.MatchString(given) { + return fmt.Errorf("--revision %q is not a 40-character commit hash", given) + } + switch { + case m.Revision != "" && given != "" && m.Revision != given: + return fmt.Errorf("the folder records commit %s, but --revision says %s", m.Revision, given) + case m.Revision == "": + m.Revision = given + } + if m.Revision == "" { + return errors.New("the folder does not record its commit: pass --revision <40-character commit hash>") + } + return nil +} + +func readMetadata(file string) (commit, etag string, ok bool) { + f, err := os.Open(file) + if err != nil { + return "", "", false + } + defer f.Close() + sc := bufio.NewScanner(f) + var lines []string + for sc.Scan() && len(lines) < 2 { + lines = append(lines, strings.TrimSpace(sc.Text())) + } + if len(lines) < 2 { + return "", "", false + } + return lines[0], strings.Trim(lines[1], `"`), true +} + +// scanFolder reads a plain model folder. The commit comes from --revision, or +// from the folder's git HEAD. +func scanFolder(dir string, o ScanOptions) (Model, error) { + m := Model{Repo: o.Repo, Kind: KindFolder, Root: dir} + if !ValidRepo(o.Repo) { + return m, fmt.Errorf("--repo %q is not an org/name Hub id", o.Repo) + } + m.Revision = gitHead(dir) + if err := m.useRevision(o.Revision); err != nil { + return m, err + } + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == ".git" { + return filepath.SkipDir + } + return nil + } + rel, _ := filepath.Rel(dir, p) + rel = filepath.ToSlash(rel) + st, err := os.Stat(p) + if err != nil { + m.Problems = append(m.Problems, "dangling link: "+rel) + return nil + } + if isLFSPointer(p, st.Size()) { + m.Problems = append(m.Problems, rel+" is a git-lfs pointer, not the file: run git lfs pull") + } + m.Files = append(m.Files, File{Path: rel, Size: st.Size(), Abs: p}) + return nil + }) + if err != nil { + return m, fmt.Errorf("read %s: %w", dir, err) + } + m.checkComplete() + return m, nil +} + +const lfsPointerHead = "version https://git-lfs.github.com/spec/v1" + +func isLFSPointer(file string, size int64) bool { + if size > 1024 { + return false + } + f, err := os.Open(file) + if err != nil { + return false + } + defer f.Close() + head := make([]byte, len(lfsPointerHead)) + n, _ := io.ReadFull(f, head) + return string(head[:n]) == lfsPointerHead +} + +// gitHead is the commit a clone has checked out, "" when dir is not one. +func gitHead(dir string) string { + read := func(p string) string { + b, err := os.ReadFile(filepath.Join(dir, ".git", filepath.FromSlash(p))) + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) + } + head := read("HEAD") + ref, ok := strings.CutPrefix(head, "ref: ") + if !ok { + if revisionPattern.MatchString(head) { + return head + } + return "" + } + if c := read(ref); revisionPattern.MatchString(c) { + return c + } + for line := range strings.SplitSeq(read("packed-refs"), "\n") { + if c, name, ok := strings.Cut(line, " "); ok && name == ref && revisionPattern.MatchString(c) { + return c + } + } + return "" +} diff --git a/internal/hfcache/space_darwin.go b/internal/hfcache/space_darwin.go new file mode 100644 index 0000000..fef685c --- /dev/null +++ b/internal/hfcache/space_darwin.go @@ -0,0 +1,29 @@ +package hfcache + +import "syscall" + +// freeBytes is what a user can still write to the filesystem holding dir. It +// is a variable so tests can fix it. +var freeBytes = func(dir string) (int64, error) { + var st syscall.Statfs_t + if err := syscall.Statfs(dir, &st); err != nil { + return 0, err + } + return int64(st.Bavail) * int64(st.Bsize), nil +} + +// onFAT reports whether dir is on a FAT filesystem, which macOS calls msdos. +func onFAT(dir string) bool { + var st syscall.Statfs_t + if syscall.Statfs(dir, &st) != nil { + return false + } + var name []byte + for _, c := range st.Fstypename { + if c == 0 { + break + } + name = append(name, byte(c)) + } + return string(name) == "msdos" +} diff --git a/internal/hfcache/space_linux.go b/internal/hfcache/space_linux.go new file mode 100644 index 0000000..4aee98b --- /dev/null +++ b/internal/hfcache/space_linux.go @@ -0,0 +1,22 @@ +package hfcache + +import "syscall" + +// msdosSuperMagic is the filesystem type of FAT12, FAT16, FAT32 and VFAT. +const msdosSuperMagic = 0x4d44 + +// freeBytes is what a user can still write to the filesystem holding dir. It +// is a variable so tests can fix it. +var freeBytes = func(dir string) (int64, error) { + var st syscall.Statfs_t + if err := syscall.Statfs(dir, &st); err != nil { + return 0, err + } + return int64(st.Bavail) * int64(st.Bsize), nil +} + +// onFAT reports whether dir is on a FAT filesystem. +func onFAT(dir string) bool { + var st syscall.Statfs_t + return syscall.Statfs(dir, &st) == nil && int64(st.Type) == msdosSuperMagic +} diff --git a/internal/hfcache/space_other.go b/internal/hfcache/space_other.go new file mode 100644 index 0000000..d6bc150 --- /dev/null +++ b/internal/hfcache/space_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin + +package hfcache + +import "errors" + +// freeBytes is unknown here, which CheckSpace reads as nothing to check. +var freeBytes = func(string) (int64, error) { return 0, errors.New("free space is not known on this platform") } + +// onFAT is false where the filesystem cannot be read. +func onFAT(string) bool { return false } diff --git a/internal/resolve/offline.go b/internal/resolve/offline.go new file mode 100644 index 0000000..a0e0442 --- /dev/null +++ b/internal/resolve/offline.go @@ -0,0 +1,43 @@ +package resolve + +import ( + "fmt" + "slices" + "strings" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/engine" +) + +// SetOffline keeps the profile's server off the network, the way its engine +// declares: environment lines, and arguments the installed server is asked +// about first. probe runs the server with the given arguments and returns its +// output. +func (p *Plan) SetOffline(prov engine.Provider, probe func(args []string) string) { + off := prov.OfflineSetting() + if off == nil { + p.Notes = append(p.Notes, fmt.Sprintf("%s has no offline setting; set what it needs in the profile.", prov.Name())) + return + } + for _, line := range off.EnvLines() { + key, _, _ := strings.Cut(line, "=") + if !slices.ContainsFunc(p.Opts.Env, func(l string) bool { return strings.HasPrefix(l, key+"=") }) { + p.Opts.Env = append(p.Opts.Env, line) + } + } + if len(off.Args) == 0 { + return + } + var out string + if len(off.ProbeArgs) > 0 && probe != nil { + out = probe(off.ProbeArgs) + } + if !off.ArgsAccepted(out) { + p.Notes = append(p.Notes, fmt.Sprintf("The installed %s does not list %s, so it was not added; a newer build can load a cached model offline.", prov.Name(), off.Args[0])) + return + } + if words, err := config.SplitArgs(p.Opts.ExtraArgs); err == nil && slices.Contains(words, off.Args[0]) { + return + } + p.Opts.ExtraArgs = strings.TrimSpace(p.Opts.ExtraArgs + " " + strings.Join(off.Args, " ")) +} diff --git a/internal/resolve/offline_internal_test.go b/internal/resolve/offline_internal_test.go new file mode 100644 index 0000000..a39dc52 --- /dev/null +++ b/internal/resolve/offline_internal_test.go @@ -0,0 +1,91 @@ +package resolve + +import ( + "slices" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/engine" +) + +func offlinePlan(t *testing.T, id, extra string, probeOut string) (Plan, []string) { + t.Helper() + prov, err := engine.Lookup(id) + if err != nil { + t.Fatal(err) + } + p := Plan{} + p.Opts.ExtraArgs = extra + var probed []string + p.SetOffline(prov, func(args []string) string { + probed = args + return probeOut + }) + return p, probed +} + +func TestSetOfflineUsesWhatEachEngineDeclares(t *testing.T) { + for _, id := range engine.ProcessNames() { + prov, _ := engine.Lookup(id) + off := prov.OfflineSetting() + if off == nil { + continue + } + help := strings.Join(off.Args, " ") + " listed" + p, _ := offlinePlan(t, id, "--port 1", help) + for _, line := range off.EnvLines() { + if !slices.Contains(p.Opts.Env, line) { + t.Errorf("%s: env %q missing from %v", id, line, p.Opts.Env) + } + } + for _, a := range off.Args { + if !strings.Contains(p.Opts.ExtraArgs, a) { + t.Errorf("%s: %q missing from %q", id, a, p.Opts.ExtraArgs) + } + } + if !strings.HasPrefix(p.Opts.ExtraArgs, "--port 1") { + t.Errorf("%s: existing args lost: %q", id, p.Opts.ExtraArgs) + } + } +} + +func TestSetOfflineSetsHFHubOfflineForPythonEngines(t *testing.T) { + for _, id := range []string{"vllm", "mlx"} { + p, probed := offlinePlan(t, id, "", "") + if !slices.Contains(p.Opts.Env, "HF_HUB_OFFLINE=1") { + t.Errorf("%s: env = %v", id, p.Opts.Env) + } + if p.Opts.ExtraArgs != "" || probed != nil { + t.Errorf("%s: args %q probed %v", id, p.Opts.ExtraArgs, probed) + } + } +} + +func TestSetOfflineAddsTheFlagOnlyWhenTheServerListsIt(t *testing.T) { + p, probed := offlinePlan(t, "llamacpp", "-ngl 99", " --offline work offline\n --port N") + if p.Opts.ExtraArgs != "-ngl 99 --offline" || !slices.Equal(probed, []string{"--help"}) { + t.Errorf("args %q probed %v", p.Opts.ExtraArgs, probed) + } + + p, _ = offlinePlan(t, "llamacpp", "-ngl 99", " --port N") + if p.Opts.ExtraArgs != "-ngl 99" { + t.Errorf("flag added to a server that does not list it: %q", p.Opts.ExtraArgs) + } + if len(p.Notes) != 1 || !strings.Contains(p.Notes[0], "--offline") { + t.Errorf("notes = %v", p.Notes) + } +} + +func TestSetOfflineDoesNotRepeatWhatIsThere(t *testing.T) { + p, _ := offlinePlan(t, "llamacpp", "--offline", "--offline") + if p.Opts.ExtraArgs != "--offline" { + t.Errorf("args = %q", p.Opts.ExtraArgs) + } + prov, _ := engine.Lookup("vllm") + q := Plan{} + q.Opts.Env = []string{"HF_HUB_OFFLINE=0"} + q.SetOffline(prov, nil) + if len(q.Opts.Env) != 1 || q.Opts.Env[0] != "HF_HUB_OFFLINE=0" { + t.Errorf("env = %v; a setting already in the profile must win", q.Opts.Env) + } +} diff --git a/internal/web/models.go b/internal/web/models.go index d5fce49..aca8c30 100644 --- a/internal/web/models.go +++ b/internal/web/models.go @@ -301,6 +301,7 @@ func (s *Server) handlePull(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } + control.KeepOffline(r.Context(), s.cfg, prov, &plan) out.Notes = append(out.Notes, plan.Notes...) if _, err := profile.Create(s.cfg.Dirs, plan.Opts); err != nil { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})