fix: preserve serialized tokenizer truncation direction

fastembed/common/preprocessor_utils.py

@@ -95,7 +95,13 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]:
     tokens_map = load_special_tokens(model_dir)
 
     tokenizer = Tokenizer.from_file(str(tokenizer_path))
-    tokenizer.enable_truncation(max_length=max_context)
+    # enable_truncation resets the direction to right unless it is passed. The direction is
+    # resolved as in transformers: tokenizer_config.json, then tokenizer.json, then right.
+    truncation = tokenizer.truncation or {}
+    tokenizer.enable_truncation(
+        max_length=max_context,
+        direction=tokenizer_config.get("truncation_side") or truncation.get("direction", "right"),
+    )
 
     # Registered before the padding is resolved: the map may name a pad token that
     # tokenizer.json does not carry, and it only gets an id once it is added.

tests/test_preprocessor_utils.py

@@ -210,6 +210,32 @@ def test_max_context_resolution(make_model_dir, model_max_length, max_length, ex
     assert tokenizer.truncation["max_length"] == expected
 
 
+@pytest.mark.parametrize(
+    "serialized_direction,truncation_side,expected",
+    [
+        (None, None, "right"),
+        ("left", None, "left"),
+        (None, "left", "left"),
+        ("left", "right", "right"),  # tokenizer_config.json wins, as in transformers
+    ],
+)
+def test_truncation_direction_resolution(
+    make_model_dir, serialized_direction, truncation_side, expected
+) -> None:
+    model_dir = make_model_dir(tokenizer_config={"truncation_side": truncation_side})
+    tokenizer_path = model_dir / "tokenizer.json"
+    serialized = Tokenizer.from_file(str(tokenizer_path))
+    if serialized_direction is None:
+        serialized.no_truncation()
+    else:
+        serialized.enable_truncation(max_length=512, direction=serialized_direction)
+    serialized.save(str(tokenizer_path))
+
+    tokenizer, _ = load_tokenizer(model_dir)
+
+    assert tokenizer.truncation["direction"] == expected
+
+
 @pytest.mark.parametrize(
     "model_max_length,max_length",
     [