fix: encode every URL token in a Lucene search, not just the first (#2764)

Author: shuvamk

.changeset/lucene-encode-all-url-tokens.md

@@ -0,0 +1,11 @@
+---
+'@hyperdx/common-utils': patch
+---
+
+fix: Encode every `http://`, `https://` and `localhost:<port>` in a search, not
+just the first
+
+A search naming two or more URLs left the later colons unescaped, so Lucene read
+them as field queries. `http://a.com http://b.com` compiled the second URL to
+`http ILIKE '%//b.com%'` — a predicate on a bare `http` identifier rather than a
+search of the log body.

packages/common-utils/src/__tests__/queryParser.test.ts

@@ -450,6 +450,24 @@ describe('CustomSchemaSQLSerializerV2 - json', () => {
       sql: "(NOT (hasToken(lower(Body), lower('red'))) AND NOT (hasToken(lower(Body), lower('blue'))))",
       english: 'NOT event has whole word red AND NOT event has whole word blue',
     },
+    {
+      lucene: 'http://a.com http://b.com',
+      sql: "((hasToken(lower(Body), lower('http')) AND hasToken(lower(Body), lower('a')) AND hasToken(lower(Body), lower('com')) AND (lower(Body) LIKE lower('%http://a.com%'))) AND (hasToken(lower(Body), lower('http')) AND hasToken(lower(Body), lower('b')) AND hasToken(lower(Body), lower('com')) AND (lower(Body) LIKE lower('%http://b.com%'))))",
+      english:
+        'event has whole word http://a.com AND event has whole word http://b.com',
+    },
+    {
+      lucene: 'https://a.com https://b.com',
+      sql: "((hasToken(lower(Body), lower('https')) AND hasToken(lower(Body), lower('a')) AND hasToken(lower(Body), lower('com')) AND (lower(Body) LIKE lower('%https://a.com%'))) AND (hasToken(lower(Body), lower('https')) AND hasToken(lower(Body), lower('b')) AND hasToken(lower(Body), lower('com')) AND (lower(Body) LIKE lower('%https://b.com%'))))",
+      english:
+        'event has whole word https://a.com AND event has whole word https://b.com',
+    },
+    {
+      lucene: 'localhost:3000 localhost:4000',
+      sql: "((hasToken(lower(Body), lower('localhost')) AND hasToken(lower(Body), lower('3000')) AND (lower(Body) LIKE lower('%localhost:3000%'))) AND (hasToken(lower(Body), lower('localhost')) AND hasToken(lower(Body), lower('4000')) AND (lower(Body) LIKE lower('%localhost:4000%'))))",
+      english:
+        'event has whole word localhost:3000 AND event has whole word localhost:4000',
+    },
   ];
 
   it.each(testCases)(

packages/common-utils/src/queryParser.ts

@@ -32,18 +32,18 @@ const HAS_ALL_TOKENS_CHUNK_SIZE = 50;
 function encodeSpecialTokens(query: string): string {
   return query
     .replace(/\\\\/g, 'HDX_BACKSLASH_LITERAL')
-    .replace('http://', 'http_COLON_//')
-    .replace('https://', 'https_COLON_//')
-    .replace(/localhost:(\d{1,5})/, 'localhost_COLON_$1')
+    .replace(/http:\/\//g, 'http_COLON_//')
+    .replace(/https:\/\//g, 'https_COLON_//')
+    .replace(/localhost:(\d{1,5})/g, 'localhost_COLON_$1')
     .replace(/\\:/g, 'HDX_COLON');
 }
 function decodeSpecialTokens(query: string): string {
   return query
     .replace(/\\"/g, '"')
     .replace(/HDX_BACKSLASH_LITERAL/g, '\\')
-    .replace('http_COLON_//', 'http://')
-    .replace('https_COLON_//', 'https://')
-    .replace(/localhost_COLON_(\d{1,5})/, 'localhost:$1')
+    .replace(/http_COLON_\/\//g, 'http://')
+    .replace(/https_COLON_\/\//g, 'https://')
+    .replace(/localhost_COLON_(\d{1,5})/g, 'localhost:$1')
     .replace(/HDX_COLON/g, ':');
 }