Skip to content

Commit b30dfe0

Browse files
feat: support text index on lower(Body) with no preprocessor (#2326)
## Summary When a text index is defined as `INDEX lower(Body) TYPE text(tokenizer='splitByNonAlpha')` (no preprocessor), the Lucene-to-ClickHouse transpiler now generates `hasAllTokens(lower(Body), lower(...))` conditions instead of `hasAllTokens(Body, ...)`. This ensures our queries use the text index. ### Screenshots <details> <summary>DDL for lower(Body) Index</summary> ```sql CREATE TABLE default.otel_logs ( `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), `TraceId` String CODEC(ZSTD(1)), `SpanId` String CODEC(ZSTD(1)), `TraceFlags` UInt8, `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), `SeverityNumber` UInt8, `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), `Body` String CODEC(ZSTD(1)), `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), `ScopeName` String CODEC(ZSTD(1)), `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), `EventName` String CODEC(ZSTD(1)), `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.namespace.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.node.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 100000000 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) TTL toDateTime(Timestamp) + toIntervalDay(1) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, enable_block_number_column = 1, enable_block_offset_column = 1 ``` </details> Given this schema, we generate hasAllTokens(lower(Body), '...') conditions: <img width="953" height="634" alt="Screenshot 2026-05-22 at 7 37 41 AM" src="https://github.com/user-attachments/assets/34bdb121-d51f-4562-88ac-9c24e915231d" /> Which utilizes the text index: <img width="1525" height="552" alt="Screenshot 2026-05-22 at 7 40 34 AM" src="https://github.com/user-attachments/assets/53872419-7c06-465b-a4cd-fc48c9df6c21" /> <details> <summary>DDL for Body Index w/ pre-processor</summary> ```sql CREATE TABLE default.otel_logs_preprocessor ( `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), `TraceId` String CODEC(ZSTD(1)), `SpanId` String CODEC(ZSTD(1)), `TraceFlags` UInt8, `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), `SeverityNumber` UInt8, `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), `Body` String CODEC(ZSTD(1)), `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), `ScopeName` String CODEC(ZSTD(1)), `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), `EventName` String CODEC(ZSTD(1)), `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.namespace.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.node.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array') GRANULARITY 100000000, INDEX idx_lower_body Body TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(Body)) GRANULARITY 100000000 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) TTL toDateTime(Timestamp) + toIntervalDay(1) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, enable_block_number_column = 1, enable_block_offset_column = 1 ``` </details> Given this schema, we generate hasAllTokens(Body, '...') conditions: <img width="957" height="633" alt="Screenshot 2026-05-22 at 7 41 59 AM" src="https://github.com/user-attachments/assets/06680173-e2d9-4622-a186-fcc8e670849a" /> Which utilizes the text index: <img width="1505" height="545" alt="Screenshot 2026-05-22 at 7 42 57 AM" src="https://github.com/user-attachments/assets/ee585b76-2cff-4278-8dcf-5074c840bf03" /> ### How to test on Vercel preview You'd have to test this locally, by creating the schemas listed above and issuing a lucene implicit column search ### References - Linear Issue: Closes HDX-4320 Co-authored-by: Cursor Agent <199161495+cursoragent@users.noreply.github.com>
1 parent 8938b05 commit b30dfe0

3 files changed

Lines changed: 231 additions & 11 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hyperdx/common-utils": patch
3+
---
4+
5+
fix: support text index on lower(Body) with no preprocessor

‎packages/common-utils/src/__tests__/queryParser.test.ts‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1177,6 +1177,161 @@ describe('CustomSchemaSQLSerializerV2 - text indices', () => {
11771177
expect(sql).toBe("((hasToken(lower(Body), lower('foo'))))");
11781178
});
11791179

1180+
describe('lower(Body) text index (no preprocessor)', () => {
1181+
it('should use hasAllTokens(lower(Body), lower(...)) when index expression is lower(Body)', async () => {
1182+
metadata.getSkipIndices = jest.fn().mockResolvedValue([
1183+
{
1184+
name: 'idx_lower_body',
1185+
type: 'text',
1186+
typeFull: "text(tokenizer = 'splitByNonAlpha')",
1187+
expression: 'lower(Body)',
1188+
granularity: '8',
1189+
},
1190+
]);
1191+
1192+
const serializer = new CustomSchemaSQLSerializerV2({
1193+
metadata,
1194+
databaseName,
1195+
tableName,
1196+
connectionId,
1197+
implicitColumnExpression: 'Body',
1198+
});
1199+
1200+
const builder = new SearchQueryBuilder('Foo', serializer);
1201+
const sql = await builder.build();
1202+
1203+
expect(sql).toBe("((hasAllTokens(lower(Body), lower('Foo'))))");
1204+
});
1205+
1206+
it('should use hasAllTokens(lower(Body), lower(...)) for multi-token terms', async () => {
1207+
metadata.getSkipIndices = jest.fn().mockResolvedValue([
1208+
{
1209+
name: 'idx_lower_body',
1210+
type: 'text',
1211+
typeFull: "text(tokenizer = 'splitByNonAlpha')",
1212+
expression: 'lower(Body)',
1213+
granularity: '8',
1214+
},
1215+
]);
1216+
1217+
const serializer = new CustomSchemaSQLSerializerV2({
1218+
metadata,
1219+
databaseName,
1220+
tableName,
1221+
connectionId,
1222+
implicitColumnExpression: 'Body',
1223+
});
1224+
1225+
const builder = new SearchQueryBuilder('"Foo Bar"', serializer);
1226+
const sql = await builder.build();
1227+
1228+
expect(sql).toContain("hasAllTokens(lower(Body), lower('Foo Bar'))");
1229+
expect(sql).toContain("(lower(Body) LIKE lower('%Foo Bar%'))");
1230+
});
1231+
1232+
it('should handle negated searches with lower(Body) index', async () => {
1233+
metadata.getSkipIndices = jest.fn().mockResolvedValue([
1234+
{
1235+
name: 'idx_lower_body',
1236+
type: 'text',
1237+
typeFull: "text(tokenizer = 'splitByNonAlpha')",
1238+
expression: 'lower(Body)',
1239+
granularity: '8',
1240+
},
1241+
]);
1242+
1243+
const serializer = new CustomSchemaSQLSerializerV2({
1244+
metadata,
1245+
databaseName,
1246+
tableName,
1247+
connectionId,
1248+
implicitColumnExpression: 'Body',
1249+
});
1250+
1251+
const builder = new SearchQueryBuilder('-Foo', serializer);
1252+
const sql = await builder.build();
1253+
1254+
expect(sql).toBe("((NOT hasAllTokens(lower(Body), lower('Foo'))))");
1255+
});
1256+
1257+
it('should NOT use lower() when index is directly on Body', async () => {
1258+
metadata.getSkipIndices = jest.fn().mockResolvedValue([
1259+
{
1260+
name: 'idx_body_text',
1261+
type: 'text',
1262+
typeFull: 'text(tokenizer=splitByNonAlpha)',
1263+
expression: 'Body',
1264+
granularity: '8',
1265+
},
1266+
]);
1267+
1268+
const serializer = new CustomSchemaSQLSerializerV2({
1269+
metadata,
1270+
databaseName,
1271+
tableName,
1272+
connectionId,
1273+
implicitColumnExpression: 'Body',
1274+
});
1275+
1276+
const builder = new SearchQueryBuilder('Foo', serializer);
1277+
const sql = await builder.build();
1278+
1279+
expect(sql).toBe("((hasAllTokens(Body, 'Foo')))");
1280+
});
1281+
1282+
it('should batch tokens with lower() when index is on lower(Body)', async () => {
1283+
metadata.getSkipIndices = jest.fn().mockResolvedValue([
1284+
{
1285+
name: 'idx_lower_body',
1286+
type: 'text',
1287+
typeFull: "text(tokenizer = 'splitByNonAlpha')",
1288+
expression: 'lower(Body)',
1289+
granularity: '8',
1290+
},
1291+
]);
1292+
1293+
const serializer = new CustomSchemaSQLSerializerV2({
1294+
metadata,
1295+
databaseName,
1296+
tableName,
1297+
connectionId,
1298+
implicitColumnExpression: 'Body',
1299+
});
1300+
1301+
const builder = new SearchQueryBuilder('FOO NOT BAR BAZ', serializer);
1302+
const sql = await builder.build();
1303+
1304+
expect(sql).toContain("hasAllTokens(lower(Body), lower('FOO'))");
1305+
expect(sql).toContain("NOT (hasAllTokens(lower(Body), lower('BAR')))");
1306+
expect(sql).toContain("hasAllTokens(lower(Body), lower('BAZ'))");
1307+
});
1308+
1309+
it('should detect LOWER(Body) case-insensitively', async () => {
1310+
metadata.getSkipIndices = jest.fn().mockResolvedValue([
1311+
{
1312+
name: 'idx_lower_body',
1313+
type: 'text',
1314+
typeFull: "text(tokenizer = 'splitByNonAlpha')",
1315+
expression: 'LOWER(Body)',
1316+
granularity: '8',
1317+
},
1318+
]);
1319+
1320+
const serializer = new CustomSchemaSQLSerializerV2({
1321+
metadata,
1322+
databaseName,
1323+
tableName,
1324+
connectionId,
1325+
implicitColumnExpression: 'Body',
1326+
});
1327+
1328+
const builder = new SearchQueryBuilder('foo', serializer);
1329+
const sql = await builder.build();
1330+
1331+
expect(sql).toBe("((hasAllTokens(lower(Body), lower('foo'))))");
1332+
});
1333+
});
1334+
11801335
describe('useTextIndexForImplicitColumn source preference', () => {
11811336
it('Auto preserves the existing detection behavior when a text index is found', async () => {
11821337
metadata.getSkipIndices = jest.fn().mockResolvedValue([
@@ -1312,6 +1467,30 @@ describe('CustomSchemaSQLSerializerV2 - text indices', () => {
13121467
expect(sql).toBe("((hasAll(tokens(lower(Body)), tokens(lower('foo')))))");
13131468
});
13141469

1470+
it('Enabled uses lower() wrapping when a lower(Body) text index exists', async () => {
1471+
metadata.getSkipIndices = jest.fn().mockResolvedValue([
1472+
{
1473+
name: 'idx_lower_body',
1474+
type: 'text',
1475+
typeFull: "text(tokenizer = 'splitByNonAlpha')",
1476+
expression: 'lower(Body)',
1477+
granularity: '8',
1478+
},
1479+
]);
1480+
1481+
const serializer = new CustomSchemaSQLSerializerV2({
1482+
metadata,
1483+
databaseName,
1484+
tableName,
1485+
connectionId,
1486+
implicitColumnExpression: 'Body',
1487+
useTextIndexForImplicitColumn: UseTextIndex.Enabled,
1488+
});
1489+
1490+
const sql = await new SearchQueryBuilder('foo', serializer).build();
1491+
expect(sql).toBe("((hasAllTokens(lower(Body), lower('foo'))))");
1492+
});
1493+
13151494
it('Enabled with a multi-token term still falls back to LIKE when separators exist', async () => {
13161495
metadata.getSkipIndices = jest.fn().mockResolvedValue([]);
13171496

‎packages/common-utils/src/queryParser.ts‎

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1273,20 +1273,28 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer {
12731273
// - enabled: force hasAllTokens(), even if no text index is detected
12741274
// - disabled: skip the text-index branch entirely
12751275
let useHasAllTokens = false;
1276+
let textIndexHasLower = false;
12761277
if (this.useTextIndexForImplicitColumn === UseTextIndex.Enabled) {
12771278
useHasAllTokens = true;
1279+
const textIndexResult = await this.findTextIndex(column);
1280+
if (textIndexResult) {
1281+
textIndexHasLower = textIndexResult.indexHasLower;
1282+
}
12781283
} else if (this.useTextIndexForImplicitColumn === UseTextIndex.Auto) {
12791284
// Note: We check that enable_full_text_index = 1, otherwise hasAllTokens() errors
12801285
const isTextIndexEnabled = await this.enableTextIndexPromise;
1281-
const textIndex = isTextIndexEnabled
1286+
const textIndexResult = isTextIndexEnabled
12821287
? await this.findTextIndex(column)
12831288
: undefined;
12841289

1285-
if (textIndex) {
1286-
const tokenizer = parseTokenizerFromTextIndex(textIndex);
1290+
if (textIndexResult) {
1291+
const tokenizer = parseTokenizerFromTextIndex(
1292+
textIndexResult.index,
1293+
);
12871294
// HDX-3259: Support other tokenizers by overriding tokenizeTerm, termHasSeparators, and batching logic
12881295
if (tokenizer?.type === 'splitByNonAlpha') {
12891296
useHasAllTokens = true;
1297+
textIndexHasLower = textIndexResult.indexHasLower;
12901298
}
12911299
}
12921300
}
@@ -1295,13 +1303,24 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer {
12951303
const tokens = this.tokenizeTerm(term);
12961304
const hasSeparators = this.termHasSeparators(term);
12971305

1306+
// When the text index is on lower(column), we must pass lower(column)
1307+
// as the first argument and wrap the tokens in lower() to match.
1308+
const hasAllTokensColumn = textIndexHasLower
1309+
? `lower(${column})`
1310+
: column;
1311+
12981312
// Batch tokens to avoid exceeding hasAllTokens limit (64)
12991313
const tokenBatches = chunk(tokens, HAS_ALL_TOKENS_CHUNK_SIZE);
13001314
const hasAllTokensExpressions = tokenBatches.map(batch =>
1301-
SqlString.format(`hasAllTokens(?, ?)`, [
1302-
SqlString.raw(column),
1303-
batch.join(' '),
1304-
]),
1315+
textIndexHasLower
1316+
? SqlString.format(`hasAllTokens(?, lower(?))`, [
1317+
SqlString.raw(hasAllTokensColumn),
1318+
batch.join(' '),
1319+
])
1320+
: SqlString.format(`hasAllTokens(?, ?)`, [
1321+
SqlString.raw(hasAllTokensColumn),
1322+
batch.join(' '),
1323+
]),
13051324
);
13061325

13071326
if (hasSeparators || tokenBatches.length > 1) {
@@ -1324,7 +1343,9 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer {
13241343
const bloomIndex = await this.findBloomFilterTokensIndex(column);
13251344

13261345
if (bloomIndex.found) {
1327-
const indexHasLower = /\blower\s*\(/.test(bloomIndex.indexExpression);
1346+
const indexHasLower = this.isLowerExpression(
1347+
bloomIndex.indexExpression,
1348+
);
13281349
const termTokensExpression = indexHasLower
13291350
? SqlString.format('tokens(lower(?))', [term])
13301351
: SqlString.format('tokens(?)', [term]);
@@ -1531,21 +1552,36 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer {
15311552
// throw new Error(`Column not found: ${field}`);
15321553
}
15331554

1555+
private isLowerExpression(expr: string): boolean {
1556+
return /\blower\s*\(/i.test(expr);
1557+
}
1558+
15341559
private async findTextIndex(
15351560
columnExpression: string,
1536-
): Promise<SkipIndexMetadata | undefined> {
1561+
): Promise<{ index: SkipIndexMetadata; indexHasLower: boolean } | undefined> {
15371562
const skipIndices = await this.skipIndicesPromise;
15381563

15391564
if (!skipIndices || skipIndices.length === 0) {
15401565
return undefined;
15411566
}
15421567

1543-
// Note: Text index expressions should not be wrapped in tokens() or preprocessing functions like lower().
1544-
return skipIndices.find(
1568+
const idx = skipIndices.find(
15451569
idx =>
15461570
idx.type === 'text' &&
15471571
this.indexCoversColumn(idx.expression, columnExpression),
15481572
);
1573+
1574+
if (!idx) {
1575+
return undefined;
1576+
}
1577+
1578+
const normalizedExpr = normalizeChExpression(idx.expression);
1579+
const normalizedCol = normalizeChExpression(columnExpression);
1580+
const indexHasLower =
1581+
normalizedExpr !== normalizedCol &&
1582+
this.isLowerExpression(idx.expression);
1583+
1584+
return { index: idx, indexHasLower };
15491585
}
15501586

15511587
/**

0 commit comments

Comments
 (0)