diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-crewai/CHANGELOG.md b/instrumentation-loongsuite/loongsuite-instrumentation-crewai/CHANGELOG.md index 78fcfe435..69b06d97a 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-crewai/CHANGELOG.md +++ b/instrumentation-loongsuite/loongsuite-instrumentation-crewai/CHANGELOG.md @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Keep incidental provider requests out of the CrewAI VCR cassettes so a first-use model-metadata fetch cannot fail the recorded tests. ([#270](https://github.com/alibaba/loongsuite-python/pull/270)) +- Keep the CrewAI tests off the network: turn off crewAI's telemetry and + version check, stop VCR from passing requests through to real hosts, and + block real connections. A telemetry export in flight let concurrent LLM + calls skip their cassettes, so tests failed at random. + ([#287](https://github.com/alibaba/loongsuite-python/pull/287)) ## Version 0.9.0 (2026-09-07) diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-crewai/tests/conftest.py b/instrumentation-loongsuite/loongsuite-instrumentation-crewai/tests/conftest.py index c89b9b2a8..7dabfbe10 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-crewai/tests/conftest.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-crewai/tests/conftest.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from pathlib import Path import pytest @@ -24,6 +25,35 @@ # ``litellm.litellm_core_utils.get_model_cost_map``. os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") +# crewAI exports anonymous telemetry to telemetry.crewai.com from a background +# thread, and checks PyPI for a newer release when a verbose crew starts. +# Neither belongs in these offline tests. Passing the telemetry requests +# through VCR (``ignore_hosts``) made them flaky: while vcrpy sends such a +# request to the real host, ``vcr.patch.force_reset`` undoes its patches for +# every HTTP client in the process, so an LLM call crewAI made meanwhile on its +# worker thread skipped the cassette and reached the real provider. When that +# window spanned the end of a test, the patches went back out of order and the +# next test was replayed from the previous test's cassette. +os.environ["CREWAI_DISABLE_TELEMETRY"] = "true" +os.environ["CREWAI_DISABLE_VERSION_CHECK"] = "true" + +_TESTS_DIR = Path(__file__).parent + + +def pytest_collection_modifyitems(items): + """Block real network access in these tests. + + Every response they need is recorded in the cassettes, so a connection + attempt means a request got past VCR: pytest-recording's + ``block_network`` mark makes it fail with "Network is disabled" instead + of reaching a real endpoint. As with ``--block-network``, recording new + cassettes (``--record-mode`` other than ``none``) still has network + access. + """ + for item in items: + if item.path.is_relative_to(_TESTS_DIR): + item.add_marker(pytest.mark.block_network) + def scrub_response_headers(response): """Clean up common sensitive response headers before recording.""" @@ -58,15 +88,7 @@ def vcr_config(): ], "before_record_response": scrub_response_headers, "decode_compressed_response": True, - # Requests incidental to the code under test must not reach the - # cassettes, or they fail playback under ``record_mode=none``. The - # litellm package's own conftest ignores its telemetry hosts for the - # same reason. - "ignore_hosts": [ - "raw.githubusercontent.com", - "us.i.posthog.com", - "app.posthog.com", - "api.litellm.ai", - "telemetry.crewai.com", - ], + # No ``ignore_hosts``: vcrpy sends requests for ignored hosts to the + # real network, and while it does, concurrent LLM calls skip the + # cassette (see the telemetry switches above). }