From 46e112a6e183dc5aacfc188f84a53d281ed4f8c4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Le Borgne Date: Tue, 27 Oct 2026 02:55:40 +0000 Subject: [PATCH] test(crewai): keep the tests off the network Since 2026-10-26, 13 of 36 crewai CI jobs failed, each on one or two tests: with a real 401 from DashScope for the fake key (litellm.AuthenticationError), with "OpenAIException - Connection error." (litellm.InternalServerError), or, in an error test, with no ERROR span. Before that, all 80 crewai jobs of the 20 runs since 2026-09-22 passed. crewAI exports anonymous telemetry to telemetry.crewai.com from a background thread, and the conftest listed that host in ignore_hosts, so vcrpy sent the exports to the real host. It sends such http.client requests inside vcr.patch.force_reset(), which puts back the original methods of every HTTP client it patches, httpx included, for the whole process while it connects and sends: 0.3 to 0.5 s for the TLS connection to telemetry.crewai.com. An LLM call that crewAI made on its worker thread in that window bypassed the cassette and reached the provider. When the window spanned the end of a test, the patches went back out of order and the next test was replayed from the previous test's cassette, which had no match for its requests (the connection error) or answered an error test with a success. The exports only go out since vcrpy 8.5.0, which made filter_post_data_parameters leave binary bodies alone. Before, it failed on their protobuf body with UnicodeDecodeError, which crewAI's exporter swallowed, so nothing was sent. Turn off crewAI's telemetry and its PyPI version check, which runs for verbose crews, and drop ignore_hosts, so that VCR passes nothing through to the network. Mark the crewai tests with pytest-recording's block_network, so that a request getting past VCR fails with "Network is disabled" instead of reaching a real host. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GwUvhmzgfkiZ6vo4uGyesN --- .../CHANGELOG.md | 5 +++ .../tests/conftest.py | 44 ++++++++++++++----- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-crewai/CHANGELOG.md b/instrumentation-loongsuite/loongsuite-instrumentation-crewai/CHANGELOG.md index 78fcfe435..69b06d97a 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-crewai/CHANGELOG.md +++ b/instrumentation-loongsuite/loongsuite-instrumentation-crewai/CHANGELOG.md @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Keep incidental provider requests out of the CrewAI VCR cassettes so a first-use model-metadata fetch cannot fail the recorded tests. ([#270](https://github.com/alibaba/loongsuite-python/pull/270)) +- Keep the CrewAI tests off the network: turn off crewAI's telemetry and + version check, stop VCR from passing requests through to real hosts, and + block real connections. A telemetry export in flight let concurrent LLM + calls skip their cassettes, so tests failed at random. + ([#287](https://github.com/alibaba/loongsuite-python/pull/287)) ## Version 0.9.0 (2026-09-07) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-crewai/tests/conftest.py b/instrumentation-loongsuite/loongsuite-instrumentation-crewai/tests/conftest.py index c89b9b2a8..7dabfbe10 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-crewai/tests/conftest.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-crewai/tests/conftest.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from pathlib import Path import pytest @@ -24,6 +25,35 @@ # ``litellm.litellm_core_utils.get_model_cost_map``. os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") +# crewAI exports anonymous telemetry to telemetry.crewai.com from a background +# thread, and checks PyPI for a newer release when a verbose crew starts. +# Neither belongs in these offline tests. Passing the telemetry requests +# through VCR (``ignore_hosts``) made them flaky: while vcrpy sends such a +# request to the real host, ``vcr.patch.force_reset`` undoes its patches for +# every HTTP client in the process, so an LLM call crewAI made meanwhile on its +# worker thread skipped the cassette and reached the real provider. When that +# window spanned the end of a test, the patches went back out of order and the +# next test was replayed from the previous test's cassette. +os.environ["CREWAI_DISABLE_TELEMETRY"] = "true" +os.environ["CREWAI_DISABLE_VERSION_CHECK"] = "true" + +_TESTS_DIR = Path(__file__).parent + + +def pytest_collection_modifyitems(items): + """Block real network access in these tests. + + Every response they need is recorded in the cassettes, so a connection + attempt means a request got past VCR: pytest-recording's + ``block_network`` mark makes it fail with "Network is disabled" instead + of reaching a real endpoint. As with ``--block-network``, recording new + cassettes (``--record-mode`` other than ``none``) still has network + access. + """ + for item in items: + if item.path.is_relative_to(_TESTS_DIR): + item.add_marker(pytest.mark.block_network) + def scrub_response_headers(response): """Clean up common sensitive response headers before recording.""" @@ -58,15 +88,7 @@ def vcr_config(): ], "before_record_response": scrub_response_headers, "decode_compressed_response": True, - # Requests incidental to the code under test must not reach the - # cassettes, or they fail playback under ``record_mode=none``. The - # litellm package's own conftest ignores its telemetry hosts for the - # same reason. - "ignore_hosts": [ - "raw.githubusercontent.com", - "us.i.posthog.com", - "app.posthog.com", - "api.litellm.ai", - "telemetry.crewai.com", - ], + # No ``ignore_hosts``: vcrpy sends requests for ignored hosts to the + # real network, and while it does, concurrent LLM calls skip the + # cassette (see the telemetry switches above). }