diff --git a/laiser/llm_models/model_loader.py b/laiser/llm_models/model_loader.py index fe1245a..e7ac76f 100644 --- a/laiser/llm_models/model_loader.py +++ b/laiser/llm_models/model_loader.py @@ -76,10 +76,15 @@ def load_model_from_transformer(model_id: str = None, token: str = "", use_gpu: on_gpu = bool(use_gpu) and torch.cuda.is_available() def _load(name): - load_kwargs = {"use_auth_token": token} + # transformers 5 removed `use_auth_token`; an unrecognised keyword is passed on + # to the model's constructor, which rejects it. `token` is the current + # argument. Send it only when a token was actually given, so public models + # load the same way on every transformers version. + auth = {"token": token} if token else {} + load_kwargs = dict(auth) if on_gpu: load_kwargs.update(quantization_config=BitsAndBytesConfig(load_in_8bit=True), device_map="auto") - tokenizer = AutoTokenizer.from_pretrained(name, use_auth_token=token, revision="main") # nosec B615 + tokenizer = AutoTokenizer.from_pretrained(name, revision="main", **auth) # nosec B615 model = AutoModelForCausalLM.from_pretrained(name, revision="main", **load_kwargs) # nosec B615 return tokenizer, model diff --git a/tests/test_llm_router.py b/tests/test_llm_router.py index 8875595..56850dc 100644 --- a/tests/test_llm_router.py +++ b/tests/test_llm_router.py @@ -137,6 +137,45 @@ def failing_from_pretrained(name, **kwargs): assert attempted == ["missing/model"] +def _record_from_pretrained(monkeypatch): + calls = {"tokenizer": [], "model": []} + + def tokenizer_from_pretrained(name, **kwargs): + calls["tokenizer"].append(kwargs) + return FakeTokenizer() + + def model_from_pretrained(name, **kwargs): + calls["model"].append(kwargs) + return FakeModel() + + monkeypatch.setattr(model_loader.AutoTokenizer, "from_pretrained", tokenizer_from_pretrained) + monkeypatch.setattr(model_loader.AutoModelForCausalLM, "from_pretrained", model_from_pretrained) + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + return calls + + +def test_loader_sends_token_not_use_auth_token(monkeypatch): + """transformers 5 removed use_auth_token and forwards unknown keywords to the model, which rejects them.""" + calls = _record_from_pretrained(monkeypatch) + + model_loader.load_model_from_transformer("some/model", token="hf_example", use_gpu=False) + + for kwargs in calls["tokenizer"] + calls["model"]: + assert "use_auth_token" not in kwargs + assert calls["tokenizer"][0]["token"] == "hf_example" + assert calls["model"][0]["token"] == "hf_example" + + +@pytest.mark.parametrize("token", [None, ""]) +def test_loader_sends_no_token_for_public_models(monkeypatch, token): + calls = _record_from_pretrained(monkeypatch) + + model_loader.load_model_from_transformer("some/model", token=token, use_gpu=False) + + assert "token" not in calls["tokenizer"][0] + assert "token" not in calls["model"][0] + + # The tests below go through LLMRouter's constructor, which is what # SkillExtractorRefactored builds, rather than the loader alone.