diff --git a/docs/contributing/search-events-api-patterns.md b/docs/contributing/search-events-api-patterns.md index b6299c234..5fe627927 100644 --- a/docs/contributing/search-events-api-patterns.md +++ b/docs/contributing/search-events-api-patterns.md @@ -114,6 +114,7 @@ https://us.sentry.io/api/0/organizations/sentry/events/?dataset=spans&field=ai.m - Does NOT support timestamp filters in query (use `statsPeriod` instead) - Severity levels: fatal, error, warning, info, debug, trace - Common aggregate functions: `count()`, `epm()` +- Regex filters on string attributes: `message://timeout after \d+ms//` (RE2, unquoted, max 64 characters); prefer wildcards for plain substrings, prefixes, and suffixes #### Metrics Dataset - Represents newer span metrics, including counters, gauges, and distributions diff --git a/packages/mcp-core/src/tools/support/search-events/config.ts b/packages/mcp-core/src/tools/support/search-events/config.ts index dab182818..fc6d8299a 100644 --- a/packages/mcp-core/src/tools/support/search-events/config.ts +++ b/packages/mcp-core/src/tools/support/search-events/config.ts @@ -123,6 +123,21 @@ REPLAY SEARCH RULES: - If the user asks about replays they have viewed, prefer viewed_by_me:true - If the user asks about replay users and says "me", use whoami and translate to user.email: +LOGS TEXT MATCHING (LOGS DATASET ONLY): +- Plain word, phrase, prefix, or suffix: use wildcards. message:"*database*" contains, message:"database*" starts with, message:"*database" ends with. Do NOT use a regex when a wildcard is enough +- Any of several plain words or phrases: use a wildcard list, e.g. message:["*ConnectionReset*","*ReadTimeout*"] +- Use a regex filter key://pattern// when wildcards cannot express the request: number shapes (\\d+), character classes ([0-9a-f]), structured values like IPs, UUIDs, and status codes, or ignoring case (wildcards are case sensitive). Use alternation (a|b) and anchors (^ or $) only alongside one of these + - Example: message://request took \\d+ms// + - Negate with a leading !: !message://worker \\d+ ready// + - There is no regex list form (key:[//a//,//b//] is a literal list); put alternatives inside one pattern: message://(upload|download) of \\d+ bytes failed// + - For an exact set of values, keep the list: severity:[error,fatal], not severity://error|fatal// +- Regex on message is for text inside the message. When the user asks about a value that has its own attribute (e.g. response status), filter that attribute instead +- NEVER quote a regex: message:"//...//" is a literal string match, not a regex +- The pattern ends at the first // followed by a space, ) or the end of the query. Spaces, parentheses, and a // followed by anything else (https?://host) are fine unquoted; write \\/\\/ for a literal // followed by a space or ) +- Regex uses RE2 syntax (no lookarounds or backreferences), matches anywhere unless anchored, and is case sensitive; prefix the pattern with (?i) to ignore case +- Patterns are limited to 64 characters (an escape like \\d counts as one), so keep them short: leave out \\b unless the request needs it, and match either case with (?i)[0-9a-f] rather than [0-9a-fA-F] +- Regex only works on string attributes in the logs dataset. NEVER write key://pattern// for errors, spans, metrics, or any other dataset: they match //...// literally and return nothing. Approximate the shape with wildcards there, e.g. message:"*retry*failed*" + MATHEMATICAL QUERY PATTERNS: When user asks mathematical questions like "how many X", "total Y used", "sum of Z": - Identify the appropriate dataset based on context @@ -745,6 +760,33 @@ export const DATASET_EXAMPLES: Record< sort: "-timestamp", }, }, + { + description: + "logs whose message reports a retry count like 'retry 3 of 5'", + output: { + query: "message://retry \\d+ of \\d+//", + fields: ["timestamp", "message", "severity", "trace"], + sort: "-timestamp", + }, + }, + { + description: + "error logs whose message contains a hex error code like 'err=0x8007000e'", + output: { + query: "severity:error AND message://err=0x[0-9a-f]+//", + fields: ["timestamp", "message", "severity", "trace"], + sort: "-timestamp", + }, + }, + { + description: + "logs excluding heartbeat messages like 'heartbeat seq=1042'", + output: { + query: "!message://^heartbeat seq=\\d+$//", + fields: ["timestamp", "message", "severity", "trace"], + sort: "-timestamp", + }, + }, { description: "error logs excluding health checks", output: { diff --git a/packages/mcp-core/src/tools/support/search-events/utils.test.ts b/packages/mcp-core/src/tools/support/search-events/utils.test.ts index e7e84857b..b156e46f9 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.test.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.test.ts @@ -290,8 +290,25 @@ describe("search query helpers", () => { ); }); + it("should detect regex filters as structured Sentry search syntax", () => { + expect( + looksLikeSentrySearchSyntax(String.raw`message://request took \d+ms//`), + ).toBe(true); + expect( + looksLikeSentrySearchSyntax(String.raw`!message://worker \d+ ready//`), + ).toBe(true); + expect( + looksLikeSentrySearchSyntax( + String.raw`message://https?://api\.example\.com//`, + ), + ).toBe(true); + }); + it("should ignore common natural language colon patterns", () => { expect(looksLikeSentrySearchSyntax("open http://example.com")).toBe(false); + expect(looksLikeSentrySearchSyntax("see http://example.com/a/")).toBe( + false, + ); expect(looksLikeSentrySearchSyntax("started at 10:30")).toBe(false); expect(looksLikeSentrySearchSyntax("Note: show slow spans")).toBe(false); expect(looksLikeSentrySearchSyntax("ERROR: service is down")).toBe(false); @@ -392,6 +409,41 @@ describe("search query helpers", () => { ); expect(isSemanticFilterDowngrade("id:1", 'message:"error 1"')).toBe(true); }); + + it("detects message regex downgrades of structured filters", () => { + expect( + isSemanticFilterDowngrade( + "http.status_code:500", + "message://status 500//", + ), + ).toBe(true); + expect( + isSemanticFilterDowngrade( + "user.id:42 severity:error", + String.raw`severity:error message://(?i)user 42 failed//`, + ), + ).toBe(true); + + expect( + isSemanticFilterDowngrade( + "http.status_code:500", + String.raw`http.status_code:500 message://took \d+ms//`, + ), + ).toBe(false); + + // A key:value inside a regex is pattern text, not a kept structured filter. + expect( + isSemanticFilterDowngrade("custom:foo", "message://custom:foo//"), + ).toBe(true); + + // A quote inside a regex must not hide the filters after it. + expect( + isSemanticFilterDowngrade( + "custom:foo", + String.raw`message://it's \d+// message:"*foo*"`, + ), + ).toBe(true); + }); }); describe("fetchCustomAttributes", () => { diff --git a/packages/mcp-core/src/tools/support/search-events/utils.ts b/packages/mcp-core/src/tools/support/search-events/utils.ts index a8d699d7c..3e6b67daf 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.ts @@ -30,8 +30,14 @@ export type FlexibleEventData = Record; const DEFAULT_MAX_VALUE_LENGTH = 200; const DEFAULT_MAX_ARRAY_ITEMS = 20; -const SENTRY_SEARCH_TOKEN_PATTERN = - /(^|\s)!?([A-Za-z_][A-Za-z0-9_.[\],-]*):(?=\S)(?!\/\/)/g; +// Mirrors regex_value in Sentry's event_search.py grammar, which is what +// separates a key://pattern// filter from prose like http://example.com. +const REGEX_FILTER_VALUE_SOURCE = String.raw`\/\/(?!\/\/(?:[\t\n )]|$))[^\n]{1,1024}?\/\/(?=[\t\n )]|$)`; +const REGEX_FILTER_VALUE_PATTERN = new RegExp(`^${REGEX_FILTER_VALUE_SOURCE}`); +const SENTRY_SEARCH_TOKEN_PATTERN = new RegExp( + String.raw`(^|\s)!?([A-Za-z_][A-Za-z0-9_.[\],-]*):(?=\S)(?:(?!\/\/)|(?=${REGEX_FILTER_VALUE_SOURCE}))`, + "g", +); const KNOWN_SENTRY_SEARCH_KEYS = new Set([ "browser", "device", @@ -122,8 +128,9 @@ export function looksLikeSentrySearchSyntax(query?: string): boolean { const FULL_TEXT_SEARCH_KEYS = new Set(["message", "log.body"]); /** - * Replace quoted regions with same-length placeholders so colons inside quotes - * are not treated as filter-key separators (e.g. transaction:"handle message:hello"). + * Replace quoted regions and regex filter values with same-length placeholders + * so colons inside them are not treated as filter-key separators + * (e.g. transaction:"handle message:hello" or message://key:\d+//). * Length is preserved so offsets still map back to the original query. */ function maskQuotedRegions(query: string): string { @@ -131,7 +138,17 @@ function maskQuotedRegions(query: string): string { let quote: '"' | "'" | null = null; let escaped = false; - for (const char of query) { + for (let i = 0; i < query.length; i += 1) { + const char = query[i] ?? ""; + if (!quote && !escaped && char === "/" && query[i - 1] === ":") { + const regexValue = query.slice(i).match(REGEX_FILTER_VALUE_PATTERN)?.[0]; + if (regexValue) { + // Keep the // delimiters so the token pattern still sees a regex value. + out += `//${regexValue.slice(2, -2).replace(/[\s:"']/g, "x")}//`; + i += regexValue.length - 1; + continue; + } + } if (escaped) { escaped = false; out += quote ? "x" : char; @@ -169,6 +186,10 @@ type SearchFilterOccurrence = { }; function normalizeFilterValue(rawValue: string): string { + if (REGEX_FILTER_VALUE_PATTERN.test(rawValue)) { + return rawValue.slice(2, -2).trim().toLowerCase(); + } + return rawValue .replace(/^['"]|['"]$/g, "") .replace(/^\*+|\*+$/g, "") @@ -178,7 +199,8 @@ function normalizeFilterValue(rawValue: string): string { /** * Read one filter value starting at `valueStart` in the original query. - * Quoted values keep interior whitespace; unquoted values stop at whitespace. + * Quoted and regex values keep interior whitespace; other values stop at + * whitespace. */ function readRawFilterValue( query: string, @@ -188,6 +210,13 @@ function readRawFilterValue( return undefined; } + const regexValue = query + .slice(valueStart) + .match(REGEX_FILTER_VALUE_PATTERN)?.[0]; + if (regexValue) { + return regexValue; + } + const first = query[valueStart]; if (first === '"' || first === "'") { let escaped = false;