diff --git a/.github/workflows/weekly-tests.yml b/.github/workflows/weekly-tests.yml new file mode 100644 index 00000000..f84900b4 --- /dev/null +++ b/.github/workflows/weekly-tests.yml @@ -0,0 +1,41 @@ +name: Weekly tests + +# PR runs test one representative model per family, scheduled and manual runs test every model, +# see is_manual_run in tests/utils.py +on: + schedule: + - cron: '5 9 * * 1' # Mondays, 09:05 UTC + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + + # linux runners have 16 GB of RAM, enough for every model tested in ci (colpali is skipped in ci, + # gpu tests are skipped everywhere) + runs-on: ubuntu-latest + timeout-minutes: 180 + + name: Python 3.14 weekly test + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14.x' + - name: Install dependencies + run: | + python -m pip install poetry + poetry config virtualenvs.create false + poetry install --no-interaction --no-ansi --without dev,docs + + - name: Run pytest + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + poetry run pytest --durations=20 diff --git a/tests/test_image_onnx_embeddings.py b/tests/test_image_onnx_embeddings.py index 7774055b..2e528811 100644 --- a/tests/test_image_onnx_embeddings.py +++ b/tests/test_image_onnx_embeddings.py @@ -10,7 +10,7 @@ from fastembed import ImageEmbedding from tests.config import TEST_MISC_DIR -from tests.utils import delete_model_cache, should_test_model +from tests.utils import delete_model_cache, is_manual_run, should_test_model CANONICAL_VECTOR_VALUES = { "Qdrant/clip-ViT-B-32-vision": np.array([-0.0098, 0.0128, -0.0274, 0.002, -0.0059]), @@ -70,7 +70,7 @@ def get_model(model_name: str): def test_embedding(model_cache, model_name: str) -> None: is_ci = os.getenv("CI") is_mac = platform.system() == "Darwin" - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() for model_desc in ImageEmbedding._list_supported_models(): # quantized int8 ops diverge on macOS; canonical vector is generated on linux/amd64 (CI) diff --git a/tests/test_late_interaction_embeddings.py b/tests/test_late_interaction_embeddings.py index 0e433fce..a9863aa7 100644 --- a/tests/test_late_interaction_embeddings.py +++ b/tests/test_late_interaction_embeddings.py @@ -7,7 +7,7 @@ from fastembed.late_interaction.late_interaction_text_embedding import ( LateInteractionTextEmbedding, ) -from tests.utils import delete_model_cache, should_test_model +from tests.utils import delete_model_cache, is_manual_run, should_test_model # vectors are abridged and rounded for brevity CANONICAL_COLUMN_VALUES = { @@ -211,7 +211,7 @@ def test_batch_inference_size_same_as_single_inference(model_cache, model_name: @pytest.mark.parametrize("model_name", ["answerdotai/answerai-colbert-small-v1"]) def test_single_embedding(model_cache, model_name: str): is_ci = os.getenv("CI") - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() docs_to_embed = docs for model_desc in LateInteractionTextEmbedding._list_supported_models(): @@ -231,7 +231,7 @@ def test_single_embedding(model_cache, model_name: str): @pytest.mark.parametrize("model_name", ["answerdotai/answerai-colbert-small-v1"]) def test_single_embedding_query(model_cache, model_name: str): is_ci = os.getenv("CI") - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() queries_to_embed = docs for model_desc in LateInteractionTextEmbedding._list_supported_models(): diff --git a/tests/test_sparse_embeddings.py b/tests/test_sparse_embeddings.py index 1a0e2df6..3aafbe32 100644 --- a/tests/test_sparse_embeddings.py +++ b/tests/test_sparse_embeddings.py @@ -6,7 +6,7 @@ from fastembed.sparse.bm25 import Bm25 from fastembed.sparse.sparse_text_embedding import SparseTextEmbedding -from tests.utils import delete_model_cache, should_test_model +from tests.utils import delete_model_cache, is_manual_run, should_test_model CANONICAL_COLUMN_VALUES = { @@ -175,7 +175,7 @@ def test_batch_embedding(model_cache, model_name: str) -> None: def test_single_embedding(model_cache) -> None: is_ci = os.getenv("CI") - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() for model_desc in SparseTextEmbedding._list_supported_models(): if ( diff --git a/tests/test_text_cross_encoder.py b/tests/test_text_cross_encoder.py index a29fe9d1..85988ca8 100644 --- a/tests/test_text_cross_encoder.py +++ b/tests/test_text_cross_encoder.py @@ -5,7 +5,7 @@ import pytest from fastembed.rerank.cross_encoder import TextCrossEncoder -from tests.utils import delete_model_cache, should_test_model +from tests.utils import delete_model_cache, is_manual_run, should_test_model CANONICAL_SCORE_VALUES = { "Xenova/ms-marco-MiniLM-L-6-v2": np.array([8.500708, -2.541011]), @@ -54,7 +54,7 @@ def get_model(model_name: str): @pytest.mark.parametrize("model_name", ["Xenova/ms-marco-MiniLM-L-6-v2"]) def test_rerank(model_cache, model_name: str) -> None: is_ci = os.getenv("CI") - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() for model_desc in TextCrossEncoder._list_supported_models(): if not should_test_model(model_desc, model_name, is_ci, is_manual): diff --git a/tests/test_text_multitask_embeddings.py b/tests/test_text_multitask_embeddings.py index 886a7125..fa12c24e 100644 --- a/tests/test_text_multitask_embeddings.py +++ b/tests/test_text_multitask_embeddings.py @@ -5,7 +5,7 @@ from fastembed import TextEmbedding from fastembed.text.multitask_embedding import JinaEmbeddingV3, Task -from tests.utils import delete_model_cache +from tests.utils import delete_model_cache, is_manual_run CANONICAL_VECTOR_VALUES = { @@ -63,7 +63,7 @@ @pytest.mark.parametrize("dim,model_name", [(1024, "jinaai/jina-embeddings-v3")]) def test_batch_embedding(dim: int, model_name: str): is_ci = os.getenv("CI") - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() if is_ci and not is_manual: pytest.skip("Skipping multitask models in CI non-manual mode") @@ -88,7 +88,7 @@ def test_batch_embedding(dim: int, model_name: str): def test_single_embedding(): is_ci = os.getenv("CI") - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() if is_ci and not is_manual: pytest.skip("Skipping multitask models in CI non-manual mode") @@ -134,7 +134,7 @@ def test_single_embedding(): def test_single_embedding_query(): is_ci = os.getenv("CI") - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() if is_ci and not is_manual: pytest.skip("Skipping multitask models in CI non-manual mode") @@ -165,7 +165,7 @@ def test_single_embedding_query(): def test_single_embedding_passage(): is_ci = os.getenv("CI") - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() if is_ci and not is_manual: pytest.skip("Skipping multitask models in CI non-manual mode") @@ -198,7 +198,7 @@ def test_single_embedding_passage(): @pytest.mark.parametrize("dim,model_name", [(1024, "jinaai/jina-embeddings-v3")]) def test_parallel_processing(dim: int, model_name: str): is_ci = os.getenv("CI") - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() if is_ci and not is_manual: pytest.skip("Skipping in CI non-manual mode") @@ -226,7 +226,7 @@ def test_parallel_processing(dim: int, model_name: str): @pytest.mark.parametrize("model_name", ["jinaai/jina-embeddings-v3"]) def test_lazy_load(model_name: str): is_ci = os.getenv("CI") - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() if is_ci and not is_manual: pytest.skip("Skipping in CI non-manual mode") diff --git a/tests/test_text_onnx_embeddings.py b/tests/test_text_onnx_embeddings.py index 5110a300..632bb70c 100644 --- a/tests/test_text_onnx_embeddings.py +++ b/tests/test_text_onnx_embeddings.py @@ -10,7 +10,7 @@ from fastembed.text.last_token_normalized_embedding import LastTokenNormalizedEmbedding from fastembed.text.onnx_embedding import OnnxTextEmbedding from fastembed.text.text_embedding import TextEmbedding -from tests.utils import delete_model_cache, should_test_model +from tests.utils import delete_model_cache, is_manual_run, should_test_model CANONICAL_VECTOR_VALUES = { "BAAI/bge-small-en": np.array([-0.0232, -0.0255, 0.0174, -0.0639, -0.0006]), @@ -166,7 +166,7 @@ def get_model(model_name: str): def test_embedding(model_cache, model_name: str) -> None: is_ci = os.getenv("CI") is_mac = platform.system() == "Darwin" - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() for model_desc in TextEmbedding._list_supported_models(): if model_desc.model in MULTI_TASK_MODELS or ( @@ -196,7 +196,7 @@ def test_embedding(model_cache, model_name: str) -> None: def test_query_embedding(model_cache) -> None: is_ci = os.getenv("CI") is_mac = platform.system() == "Darwin" - is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" + is_manual = is_manual_run() for model_desc in TextEmbedding._list_supported_models(): if model_desc.model in MULTI_TASK_MODELS or ( diff --git a/tests/utils.py b/tests/utils.py index 481c0504..181c3a0f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,3 +1,4 @@ +import os import shutil import traceback @@ -39,6 +40,11 @@ def on_error( shutil.rmtree(model_dir, onerror=on_error) +def is_manual_run() -> bool: + """Whether ci runs the heavyweight tests: on a manual dispatch or on the weekly schedule""" + return os.getenv("GITHUB_EVENT_NAME") in ("workflow_dispatch", "schedule") + + def should_test_model( model_desc: BaseModelDescription, autotest_model_name: str, @@ -55,6 +61,7 @@ def should_test_model( 2) Run heavyweight (manual) tests in ci: - test all models Running tests in ci each time is too expensive, however, it's fine to run it one time with a manual dispatch + or weekly on a schedule 3) Run tests locally: - test all models, which are not too heavy, since network speed might be a bottleneck