diff --git a/tests/misc/logo.png b/tests/misc/logo.png new file mode 100644 index 000000000..8805f2b98 Binary files /dev/null and b/tests/misc/logo.png differ diff --git a/tests/test_image_onnx_embeddings.py b/tests/test_image_onnx_embeddings.py index 2e5288118..5a9e64229 100644 --- a/tests/test_image_onnx_embeddings.py +++ b/tests/test_image_onnx_embeddings.py @@ -5,7 +5,6 @@ import numpy as np import pytest -import requests from PIL import Image from fastembed import ImageEmbedding @@ -29,8 +28,15 @@ "nomic-ai/nomic-embed-vision-v1.5": np.array( [0.0048, -0.0254, 0.0067, -0.0296, -0.0435, -0.0123, 0.0024, -0.0361, -0.0703, -0.0186] ), - "nomic-ai/nomic-embed-vision-v1.5-Q": np.array( - [-0.0011, -0.0477, 0.0024, -0.049, -0.0458, -0.0314, 0.017, -0.0383, -0.0537, -0.021] + # onnxruntime's avx2 int8 kernel saturates on this model's full-range int8 weights, while vnni + # and amx compute them exactly, ci runners come with either + "nomic-ai/nomic-embed-vision-v1.5-Q": ( + np.array( # x86-64 avx2 + [-0.0006, -0.0446, 0.0069, -0.0553, -0.0459, -0.0261, 0.0058, -0.0463, -0.0606, -0.019] + ), + np.array( # x86-64 vnni + [0.0020, -0.0376, 0.0094, -0.0402, -0.0457, -0.0216, 0.0100, -0.0378, -0.0650, -0.0205] + ), ), "google/siglip2-base-patch16-224": np.array( [-0.02095927, -0.0075177, -0.00144479, -0.0080948, 0.05031789] @@ -82,21 +88,26 @@ def test_embedding(model_cache, model_name: str) -> None: dim = model_desc.dim with model_cache(model_desc.model) as model: + # quantized models scale activations over the whole batch, so the first embedding + # depends on every image here, keep them local files images = [ TEST_MISC_DIR / "image.jpeg", str(TEST_MISC_DIR / "small_image.jpeg"), Image.open((TEST_MISC_DIR / "small_image.jpeg")), - Image.open(BytesIO(requests.get("https://qdrant.tech/img/logo.png").content)), + Image.open(BytesIO((TEST_MISC_DIR / "logo.png").read_bytes())), ] embeddings = list(model.embed(images)) embeddings = np.stack(embeddings, axis=0) assert embeddings.shape == (len(images), dim) canonical_vector = CANONICAL_VECTOR_VALUES[model_desc.model] - - assert np.allclose( - embeddings[0, : canonical_vector.shape[0]], canonical_vector, atol=1e-3 - ), model_desc.model + canonical_variants = ( + canonical_vector if isinstance(canonical_vector, tuple) else (canonical_vector,) + ) + prefix = embeddings[0, : canonical_variants[0].shape[0]] + assert any( + np.allclose(prefix, expected, atol=1e-3) for expected in canonical_variants + ), f"{model_desc.model}: {prefix}" assert np.allclose(embeddings[1], embeddings[2]), model_desc.model diff --git a/tests/test_text_cross_encoder.py b/tests/test_text_cross_encoder.py index 85988ca8f..b00cf862b 100644 --- a/tests/test_text_cross_encoder.py +++ b/tests/test_text_cross_encoder.py @@ -13,8 +13,9 @@ "BAAI/bge-reranker-base": np.array([6.15733337, -3.65939403]), "BAAI/bge-reranker-v2-m3": np.array([8.78182220, -5.46485329]), "BAAI/bge-reranker-v2-m3-int8": ( - np.array([8.59202957, -5.39362288]), # ONNX Runtime 1.27.0 - np.array([8.51154804, -5.56898642]), # ONNX Runtime 1.30.0 + np.array([8.59202957, -5.39362288]), # arm64, ONNX Runtime 1.27.0 + np.array([8.51154804, -5.56898642]), # arm64, ONNX Runtime 1.30.0 + np.array([8.85104847, -5.73784494]), # x86-64 avx2 and vnni, ONNX Runtime 1.30.0 ), "jinaai/jina-reranker-v1-tiny-en": np.array([2.5911, 0.1122]), "jinaai/jina-reranker-v1-turbo-en": np.array([1.8295, -2.8908]), diff --git a/tests/test_text_onnx_embeddings.py b/tests/test_text_onnx_embeddings.py index 632bb70cd..1d8e243e5 100644 --- a/tests/test_text_onnx_embeddings.py +++ b/tests/test_text_onnx_embeddings.py @@ -56,8 +56,11 @@ "nomic-ai/nomic-embed-text-v1.5": np.array( [-0.15407836, -0.03053198, -3.9138033, 0.1910364, 0.13224715] ), - "nomic-ai/nomic-embed-text-v1.5-Q": np.array( - [0.0802303, 0.3700881, -4.3053818, 0.4431803, -0.271572] + # onnxruntime's avx2 int8 kernel saturates on this model's full-range int8 weights, while vnni + # and amx compute them exactly, ci runners come with either + "nomic-ai/nomic-embed-text-v1.5-Q": ( + np.array([0.0802303, 0.3700881, -4.3053818, 0.4431803, -0.271572]), # x86-64 avx2 + np.array([0.12422204, 0.14779393, -3.8240886, 0.47303754, -0.14925367]), # x86-64 vnni ), "thenlper/gte-large": np.array( [-0.00986551, -0.00018734, 0.00605892, -0.03289612, -0.0387564], @@ -188,9 +191,13 @@ def test_embedding(model_cache, model_name: str) -> None: assert embeddings.shape == (2, dim) canonical_vector = CANONICAL_VECTOR_VALUES[model_desc.model] - assert np.allclose( - embeddings[0, : canonical_vector.shape[0]], canonical_vector, atol=1e-3 - ), model_desc.model + canonical_variants = ( + canonical_vector if isinstance(canonical_vector, tuple) else (canonical_vector,) + ) + prefix = embeddings[0, : canonical_variants[0].shape[0]] + assert any( + np.allclose(prefix, expected, atol=1e-3) for expected in canonical_variants + ), f"{model_desc.model}: {prefix}" def test_query_embedding(model_cache) -> None: