diff --git a/fastembed/common/preprocessor_utils.py b/fastembed/common/preprocessor_utils.py index c08658db..fcd8acaa 100644 --- a/fastembed/common/preprocessor_utils.py +++ b/fastembed/common/preprocessor_utils.py @@ -95,7 +95,13 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: tokens_map = load_special_tokens(model_dir) tokenizer = Tokenizer.from_file(str(tokenizer_path)) - tokenizer.enable_truncation(max_length=max_context) + # enable_truncation resets the direction to right unless it is passed. The direction is + # resolved as in transformers: tokenizer_config.json, then tokenizer.json, then right. + truncation = tokenizer.truncation or {} + tokenizer.enable_truncation( + max_length=max_context, + direction=tokenizer_config.get("truncation_side") or truncation.get("direction", "right"), + ) # Registered before the padding is resolved: the map may name a pad token that # tokenizer.json does not carry, and it only gets an id once it is added. diff --git a/tests/test_preprocessor_utils.py b/tests/test_preprocessor_utils.py index 0b928f50..b9f718a8 100644 --- a/tests/test_preprocessor_utils.py +++ b/tests/test_preprocessor_utils.py @@ -210,6 +210,32 @@ def test_max_context_resolution(make_model_dir, model_max_length, max_length, ex assert tokenizer.truncation["max_length"] == expected +@pytest.mark.parametrize( + "serialized_direction,truncation_side,expected", + [ + (None, None, "right"), + ("left", None, "left"), + (None, "left", "left"), + ("left", "right", "right"), # tokenizer_config.json wins, as in transformers + ], +) +def test_truncation_direction_resolution( + make_model_dir, serialized_direction, truncation_side, expected +) -> None: + model_dir = make_model_dir(tokenizer_config={"truncation_side": truncation_side}) + tokenizer_path = model_dir / "tokenizer.json" + serialized = Tokenizer.from_file(str(tokenizer_path)) + if serialized_direction is None: + serialized.no_truncation() + else: + serialized.enable_truncation(max_length=512, direction=serialized_direction) + serialized.save(str(tokenizer_path)) + + tokenizer, _ = load_tokenizer(model_dir) + + assert tokenizer.truncation["direction"] == expected + + @pytest.mark.parametrize( "model_max_length,max_length", [