diff --git a/tests/docker/validate-image.mjs b/tests/docker/validate-image.mjs index 4d587aa..3d59f1d 100644 --- a/tests/docker/validate-image.mjs +++ b/tests/docker/validate-image.mjs @@ -196,6 +196,17 @@ function gracefulProGate(result) { ); } +// The docs tool's answer when its external search service is down (a timeout, a network error +// or a 5xx), as opposed to our request being wrong. +function docsServiceDown(result) { + return ( + /Docs Search Unavailable/.test(result.text) && + /(Request timed out|Connection refused|HTTP Error: 5\d\d|fetch failed|ECONNRESET|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|socket hang up)/i.test( + result.text + ) + ); +} + function recordToolResult(key, name, result, predicate = (r) => !r.isError, note) { if (TOKEN_REAL) { record(key, name, predicate(result), snip(result.text, 600), note); @@ -281,7 +292,8 @@ async function main() { } } - // 3. docs (token-only; calls an external API, so retry once for transient blips) + // 3. docs (token-only; calls an external API, so retry once for transient blips). An outage of + // that service itself is a warning: it says nothing about this image. if (!SKIP.has("docs")) { try { const r = await callToolUntil( @@ -294,11 +306,14 @@ async function main() { ok: (x) => !x.isError && /LocalStack Docs/i.test(x.text), } ); + const ok = !r.isError && /LocalStack Docs/i.test(r.text); + const down = !ok && docsServiceDown(r); record( "docs", "localstack-docs returns snippets", - !r.isError && /LocalStack Docs/i.test(r.text), - snip(r.text) + ok ? true : down ? "warn" : false, + snip(r.text), + down ? "the docs search service is down, which says nothing about this image" : undefined ); } catch (e) { record("docs", "localstack-docs", false, String(e.message)); diff --git a/tests/mcp/direct.spec.mjs b/tests/mcp/direct.spec.mjs index 4fff497..d64da11 100644 --- a/tests/mcp/direct.spec.mjs +++ b/tests/mcp/direct.spec.mjs @@ -61,6 +61,12 @@ test("smoke tests the infrastructure tester prompt", async ({ mcp }) => { expect(result.messages[0].content.text).toContain("`./infra`"); }); +// The docs tool asks an external search service. When that service itself is down (a timeout, a +// network error or a 5xx), the test is skipped with the tool's answer, since nothing in this +// server is at fault. Any other answer, a 4xx included, must hold the snippets. +const DOCS_SERVICE_DOWN = + /Request timed out|Connection refused|HTTP Error: 5\d\d|fetch failed|ECONNRESET|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|socket hang up/i; + test("docs tool returns useful documentation snippets", async ({ mcp }) => { requireEnv("LOCALSTACK_AUTH_TOKEN"); @@ -68,6 +74,11 @@ test("docs tool returns useful documentation snippets", async ({ mcp }) => { query: "How to start LocalStack and configure auth token", limit: 2, }); + const text = (result?.content ?? []).map((c) => c.text ?? "").join("\n"); + test.skip( + text.includes("Docs Search Unavailable") && DOCS_SERVICE_DOWN.test(text), + `the docs search service is unavailable: ${text.slice(0, 200)}` + ); expect(result).not.toBeToolError(); expect(result).toContainToolText("LocalStack Docs"); diff --git a/yarn.lock b/yarn.lock index 7433752..a8b0a64 100644 --- a/yarn.lock +++ b/yarn.lock @@ -678,9 +678,9 @@ ai "^6.0.97" "@grpc/grpc-js@^1.11.1": - version "1.14.4" - resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.14.4.tgz#e73ff57d97802f063999545f43ebb2b1eca65d9d" - integrity sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ== + version "1.14.5" + resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.14.5.tgz#12f97bd9f716358e377ffaa768195d1219eb76a3" + integrity sha512-7VZM+SVdEcUUqSQeNI3zM8Qs/BhQKZndPo2h5VkYkAM8Iz0wJIa8mKV5ekQGqG8UUsnkQ0NMxIxwkIHYvj0qOw== dependencies: "@grpc/proto-loader" "^0.8.0" "@js-sdsl/ordered-map" "^4.4.2" @@ -2616,9 +2616,9 @@ fast-string-width@^3.0.2: fast-string-truncated-width "^3.0.2" fast-uri@^3.0.1: - version "3.1.5" - resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.1.5.tgz#610f37419a030270430cecd68d74e3d4d96725d0" - integrity sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw== + version "3.1.8" + resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.1.8.tgz#f7db8d942e20eead3cfe93b0beeb4eb0169e938b" + integrity sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg== fast-wrap-ansi@^0.2.0: version "0.2.2"