Skip to content

Commit 7a4ad98

Browse files
knudttyknudtty
authored
feat: rework getAllKeyValues and add filters populated from text indices (#2643)
## Summary This is a mammoth of a change, but brings something much needed to our filters logic: order. 1. Deprecates `useAllFieldsAndValues` and `getAllFieldsAndValues` - these are fairly useless. 2. Adds a router to `getAllKeyValues` to intelligently apply the optimal strategy to fetch key values 3. Adds support for querying Text Indices for filter values. This is true for the map text indices and native columns (eg. TraceId) 4. Reuses `useFetchFacets` in autocomplete Strategy order: 1. Query from text indices directly 2. Query from Metadata MVs 3. Query from the table ### References - Linear Issue: Closes HDX-4229
1 parent 01508d1 commit 7a4ad98

29 files changed

Lines changed: 2736 additions & 1076 deletions

‎.changeset/sharp-fishes-yell.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@hyperdx/common-utils": minor
3+
"@hyperdx/app": minor
4+
---
5+
6+
feat: upgrade filters and autocomplete to intelligently route queries through the best rollup

‎docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql‎

Lines changed: 1 addition & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
-- +goose Up
22

3-
-- Map Attributes Key Value Rollup (all map columns in one table)
3+
-- Key Value Rollup (many columns in one table)
44
CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs_kv_rollup_15m
55
(
66
`Timestamp` DateTime,
@@ -17,58 +17,9 @@ ORDER BY (ColumnIdentifier, Key, Timestamp, Value)
1717
TTL Timestamp + ${TABLES_TTL}
1818
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
1919

20-
-- Key Only Rollup (derived from KV rollup)
21-
CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs_key_rollup_15m
22-
(
23-
`Timestamp` DateTime,
24-
`ColumnIdentifier` LowCardinality(String),
25-
`Key` LowCardinality(String),
26-
`count` UInt64,
27-
INDEX idx_count_minmax count TYPE minmax GRANULARITY 1,
28-
INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1
29-
)
30-
ENGINE = SummingMergeTree
31-
PARTITION BY toDate(Timestamp)
32-
ORDER BY (ColumnIdentifier, Key, Timestamp)
33-
TTL Timestamp + ${TABLES_TTL}
34-
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
35-
36-
CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_logs_key_rollup_15m_mv TO ${DATABASE}.otel_logs_key_rollup_15m
37-
AS SELECT
38-
Timestamp,
39-
ColumnIdentifier,
40-
Key,
41-
sum(count) as count
42-
FROM ${DATABASE}.otel_logs_kv_rollup_15m
43-
GROUP BY ColumnIdentifier, Key, Timestamp;
44-
4520
-- Single MV: CTE with UNION ALL across all columns, then aggregate
4621
CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_logs_attr_kv_rollup_15m_mv TO ${DATABASE}.otel_logs_kv_rollup_15m
4722
AS WITH elements AS (
48-
SELECT
49-
'ResourceAttributes' AS ColumnIdentifier,
50-
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
51-
replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
52-
CAST(entry.2 AS String) AS Value
53-
FROM ${DATABASE}.otel_logs
54-
ARRAY JOIN ResourceAttributes AS entry
55-
UNION ALL
56-
SELECT
57-
'LogAttributes' AS ColumnIdentifier,
58-
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
59-
replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
60-
CAST(entry.2 AS String) AS Value
61-
FROM ${DATABASE}.otel_logs
62-
ARRAY JOIN LogAttributes AS entry
63-
UNION ALL
64-
SELECT
65-
'ScopeAttributes' AS ColumnIdentifier,
66-
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
67-
replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
68-
CAST(entry.2 AS String) AS Value
69-
FROM ${DATABASE}.otel_logs
70-
ARRAY JOIN ScopeAttributes AS entry
71-
UNION ALL
7223
SELECT
7324
'NativeColumn' AS ColumnIdentifier,
7425
toStartOfFifteenMinutes(Timestamp) AS Timestamp,

‎docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql‎

Lines changed: 1 addition & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
-- +goose Up
22

3-
-- Key Value Rollup (all columns in one table)
3+
-- Key Value Rollup (many columns in one table)
44
CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces_kv_rollup_15m
55
(
66
`Timestamp` DateTime,
@@ -17,51 +17,9 @@ ORDER BY (ColumnIdentifier, Key, Timestamp, Value)
1717
TTL Timestamp + ${TABLES_TTL}
1818
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
1919

20-
-- Key Only Rollup (derived from KV rollup)
21-
CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces_key_rollup_15m
22-
(
23-
`Timestamp` DateTime,
24-
`ColumnIdentifier` LowCardinality(String),
25-
`Key` LowCardinality(String),
26-
`count` UInt64,
27-
INDEX idx_count_minmax count TYPE minmax GRANULARITY 1,
28-
INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1
29-
)
30-
ENGINE = SummingMergeTree
31-
PARTITION BY toDate(Timestamp)
32-
ORDER BY (ColumnIdentifier, Key, Timestamp)
33-
TTL Timestamp + ${TABLES_TTL}
34-
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
35-
36-
-- Key only rollup MV
37-
CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_traces_key_rollup_15m_mv TO ${DATABASE}.otel_traces_key_rollup_15m
38-
AS SELECT
39-
Timestamp,
40-
ColumnIdentifier,
41-
Key,
42-
sum(count) AS count
43-
FROM ${DATABASE}.otel_traces_kv_rollup_15m
44-
GROUP BY ColumnIdentifier, Key, Timestamp;
45-
4620
-- Single MV: CTE with UNION ALL across all columns, then aggregate
4721
CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_traces_kv_rollup_15m_mv TO ${DATABASE}.otel_traces_kv_rollup_15m
4822
AS WITH elements AS (
49-
SELECT
50-
'ResourceAttributes' AS ColumnIdentifier,
51-
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
52-
replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
53-
CAST(entry.2 AS String) AS Value
54-
FROM ${DATABASE}.otel_traces
55-
ARRAY JOIN ResourceAttributes AS entry
56-
UNION ALL
57-
SELECT
58-
'SpanAttributes' AS ColumnIdentifier,
59-
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
60-
replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
61-
CAST(entry.2 AS String) AS Value
62-
FROM ${DATABASE}.otel_traces
63-
ARRAY JOIN SpanAttributes AS entry
64-
UNION ALL
6523
SELECT
6624
'NativeColumn' AS ColumnIdentifier,
6725
toStartOfFifteenMinutes(Timestamp) AS Timestamp,

‎packages/api/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
"promised-handlebars": "^2.0.1",
5252
"protobufjs": "^7.6.3",
5353
"serialize-error": "^8.1.0",
54+
"sqlstring": "^2.3.3",
5455
"uuid": "^8.3.2",
5556
"zod": "3.25",
5657
"zod-express-middleware": "^1.4.0"
@@ -66,6 +67,7 @@
6667
"@types/minimist": "^1.2.2",
6768
"@types/ms": "^0.7.31",
6869
"@types/passport-local": "^1.0.34",
70+
"@types/sqlstring": "^2.3.0",
6971
"@types/supertest": "^2.0.12",
7072
"@types/swagger-jsdoc": "^6",
7173
"@types/uuid": "^8.3.4",

‎packages/api/src/mcp/tools/sources/describeSource.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
1010
import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
1111
import { type MetricTable, SourceKind } from '@hyperdx/common-utils/dist/types';
12+
import SqlString from 'sqlstring';
1213
import { z } from 'zod';
1314

1415
import { getConnectionById } from '@/controllers/connection';
@@ -114,7 +115,7 @@ async function sampleMetricNamesForKind({
114115
timestampValueExpression,
115116
signal,
116117
});
117-
const names = nameResults[0]?.value ?? [];
118+
const names = nameResults[0]?.value.map(v => v.toString()) ?? [];
118119
if (names.length === 0) return [];
119120

120121
// Best-effort enrichment with unit + description. One small query
@@ -451,7 +452,7 @@ async function describeSourceSchema(
451452
});
452453
for (const { key, value } of results) {
453454
if (value.length > 0) {
454-
lowCardinalityValues[key] = value;
455+
lowCardinalityValues[key] = value.map(v => v.toString());
455456
}
456457
}
457458
} catch {
@@ -473,7 +474,11 @@ async function describeSourceSchema(
473474
const keyExprs: string[] = [];
474475
for (const [colName, keys] of Object.entries(mapKeysResults)) {
475476
for (const key of keys.slice(0, MAX_MAP_KEYS_TO_SAMPLE)) {
476-
keyExprs.push(`${colName}['${key}']`);
477+
// Map keys come from ClickHouse data (customer telemetry) so they can
478+
// contain arbitrary characters, including single quotes. Escape as a
479+
// SQL string literal — `SqlString.escape` returns a fully-quoted,
480+
// safely-escaped value — before embedding in the key expression.
481+
keyExprs.push(`${colName}[${SqlString.escape(key)}]`);
477482
}
478483
}
479484

@@ -491,7 +496,7 @@ async function describeSourceSchema(
491496
});
492497
for (const { key, value } of results) {
493498
if (value.length > 0) {
494-
mapAttributeValues[key] = value;
499+
mapAttributeValues[key] = value.map(v => v.toString());
495500
}
496501
}
497502
} catch {

‎packages/app/src/components/DBSearchPageFilters.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1171,7 +1171,7 @@ const DBSearchPageFiltersComponent = ({
11711171

11721172
const [showMoreFields, setShowMoreFields] = useState(false);
11731173
const {
1174-
data: facets,
1174+
data: fetchFacetsData,
11751175
isLoading: isFacetsLoading,
11761176
isFetching: isFacetsFetching,
11771177
error,
@@ -1186,6 +1186,7 @@ const DBSearchPageFiltersComponent = ({
11861186
filterState,
11871187
showMoreFields,
11881188
});
1189+
const facets = fetchFacetsData.keyValues;
11891190

11901191
useEffect(() => {
11911192
if (error) {

0 commit comments

Comments
 (0)