Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions laiser/llm_models/model_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,15 @@ def load_model_from_transformer(model_id: str = None, token: str = "", use_gpu:
on_gpu = bool(use_gpu) and torch.cuda.is_available()

def _load(name):
load_kwargs = {"use_auth_token": token}
# transformers 5 removed `use_auth_token`; an unrecognised keyword is passed on
# to the model's constructor, which rejects it. `token` is the current
# argument. Send it only when a token was actually given, so public models
# load the same way on every transformers version.
auth = {"token": token} if token else {}
load_kwargs = dict(auth)
if on_gpu:
load_kwargs.update(quantization_config=BitsAndBytesConfig(load_in_8bit=True), device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(name, use_auth_token=token, revision="main") # nosec B615
tokenizer = AutoTokenizer.from_pretrained(name, revision="main", **auth) # nosec B615
model = AutoModelForCausalLM.from_pretrained(name, revision="main", **load_kwargs) # nosec B615
return tokenizer, model

Expand Down
39 changes: 39 additions & 0 deletions tests/test_llm_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,45 @@ def failing_from_pretrained(name, **kwargs):
assert attempted == ["missing/model"]


def _record_from_pretrained(monkeypatch):
calls = {"tokenizer": [], "model": []}

def tokenizer_from_pretrained(name, **kwargs):
calls["tokenizer"].append(kwargs)
return FakeTokenizer()

def model_from_pretrained(name, **kwargs):
calls["model"].append(kwargs)
return FakeModel()

monkeypatch.setattr(model_loader.AutoTokenizer, "from_pretrained", tokenizer_from_pretrained)
monkeypatch.setattr(model_loader.AutoModelForCausalLM, "from_pretrained", model_from_pretrained)
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
return calls


def test_loader_sends_token_not_use_auth_token(monkeypatch):
"""transformers 5 removed use_auth_token and forwards unknown keywords to the model, which rejects them."""
calls = _record_from_pretrained(monkeypatch)

model_loader.load_model_from_transformer("some/model", token="hf_example", use_gpu=False)

for kwargs in calls["tokenizer"] + calls["model"]:
assert "use_auth_token" not in kwargs
assert calls["tokenizer"][0]["token"] == "hf_example"
assert calls["model"][0]["token"] == "hf_example"


@pytest.mark.parametrize("token", [None, ""])
def test_loader_sends_no_token_for_public_models(monkeypatch, token):
calls = _record_from_pretrained(monkeypatch)

model_loader.load_model_from_transformer("some/model", token=token, use_gpu=False)

assert "token" not in calls["tokenizer"][0]
assert "token" not in calls["model"][0]


# The tests below go through LLMRouter's constructor, which is what
# SkillExtractorRefactored builds, rather than the loader alone.

Expand Down
Loading