Skip to content

Commit 1d83beb

Browse files
pulpdrewpulpdrew
authored
feat: Add support for dashboard filters on Raw SQL Charts (#1924)
## Summary This PR updates Raw SQL charts with support for dashboard filters, using the $__filters macro. Lucene global filters require a Source to be included in the ChartConfig, for schema introspection and the `implicitColumnExpression` value. To support Lucene filters, this PR also updates the RawSqlChartConfig type to include optional `source`, `implicitColumnExpression`, and `from` properties. Only `source` is saved in the database. The external API has been updated to accept the `source` field for raw SQL charts as well. Dashboard import/export has also been updated to support source mapping for raw sql charts with sources. ### Screenshots or video Both the global filter and the drop-down filters are applied: <img width="683" height="574" alt="Screenshot 2026-03-17 at 10 57 36 AM" src="https://github.com/user-attachments/assets/280ba0b5-55f7-4107-a55c-eeb1497ac7de" /> To render Lucene conditions, we require Source information (the `from` and `implicitColumnExpression` fields). When a source is not set, filters are therefore not applied: <img width="782" height="618" alt="Screenshot 2026-03-17 at 10 54 41 AM" src="https://github.com/user-attachments/assets/3ad19ea7-12ee-4334-abe2-8985a0be952c" /> Similarly, if the `$__filters` macro is not used, then the filters are not applied <img width="704" height="292" alt="Screenshot 2026-03-17 at 10 56 33 AM" src="https://github.com/user-attachments/assets/e1169e4a-2f64-4cd2-bc05-f699fecef8c1" /> Import: <img width="993" height="669" alt="Screenshot 2026-03-17 at 3 35 16 PM" src="https://github.com/user-attachments/assets/6ebe20c0-19e2-4e90-95d0-8b02c2af0612" /> ### How to test locally or on Vercel This can be tested in the preview environment. 1. Create a saved dashboard and add some filters. 2. Create a raw sql tile, with or without the $__filters macro 3. Try selecting filters, deselecting filters, and applying global SQL and Lucene filters 4. Inspect the generated queries in the console. ### References - Linear Issue: Closes HDX-3694 - Related PRs:
1 parent 031ca83 commit 1d83beb

29 files changed

Lines changed: 827 additions & 219 deletions

‎.changeset/silver-carrots-change.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@hyperdx/common-utils": patch
3+
"@hyperdx/app": patch
4+
---
5+
6+
feat: Add support for dashboard filters on Raw SQL Charts

‎packages/api/openapi.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1389,6 +1389,11 @@
13891389
"description": "SQL query template to execute. Supports HyperDX template variables.",
13901390
"example": "SELECT count() FROM otel_logs WHERE timestamp > now() - INTERVAL 1 HOUR"
13911391
},
1392+
"sourceId": {
1393+
"type": "string",
1394+
"description": "Optional ID of the data source associated with this Raw SQL chart. Used for applying dashboard filters.",
1395+
"example": "65f5e4a3b9e77c001a567890"
1396+
},
13921397
"numberFormat": {
13931398
"$ref": "#/components/schemas/NumberFormat",
13941399
"description": "Number formatting options for displayed values."

‎packages/api/src/routers/external-api/__tests__/dashboards.test.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2311,6 +2311,7 @@ describe('External API v2 Dashboards - new format', () => {
23112311

23122312
it('can round-trip all raw SQL chart config types', async () => {
23132313
const connectionId = connection._id.toString();
2314+
const sourceId = traceSource._id.toString();
23142315
const sqlTemplate = 'SELECT count() FROM otel_logs WHERE {timeFilter}';
23152316

23162317
const lineRawSql: ExternalDashboardTile = {
@@ -2324,6 +2325,7 @@ describe('External API v2 Dashboards - new format', () => {
23242325
displayType: 'line',
23252326
connectionId,
23262327
sqlTemplate,
2328+
sourceId,
23272329
compareToPreviousPeriod: true,
23282330
fillNulls: true,
23292331
alignDateRangeToGranularity: true,
@@ -2342,6 +2344,7 @@ describe('External API v2 Dashboards - new format', () => {
23422344
displayType: 'stacked_bar',
23432345
connectionId,
23442346
sqlTemplate,
2347+
sourceId,
23452348
fillNulls: false,
23462349
alignDateRangeToGranularity: false,
23472350
numberFormat: { output: 'byte', decimalBytes: true },
@@ -2359,6 +2362,7 @@ describe('External API v2 Dashboards - new format', () => {
23592362
displayType: 'table',
23602363
connectionId,
23612364
sqlTemplate,
2365+
sourceId,
23622366
numberFormat: { output: 'percent', mantissa: 1 },
23632367
},
23642368
};
@@ -2374,6 +2378,7 @@ describe('External API v2 Dashboards - new format', () => {
23742378
displayType: 'number',
23752379
connectionId,
23762380
sqlTemplate,
2381+
sourceId,
23772382
numberFormat: { output: 'currency', currencySymbol: '$' },
23782383
},
23792384
};
@@ -2389,6 +2394,7 @@ describe('External API v2 Dashboards - new format', () => {
23892394
displayType: 'pie',
23902395
connectionId,
23912396
sqlTemplate,
2397+
sourceId,
23922398
},
23932399
};
23942400

@@ -2457,6 +2463,43 @@ describe('External API v2 Dashboards - new format', () => {
24572463
});
24582464
});
24592465

2466+
it('should return 400 when source connection does not match tile connection', async () => {
2467+
const otherConnection = await Connection.create({
2468+
team: team._id,
2469+
name: 'Other Connection',
2470+
host: config.CLICKHOUSE_HOST,
2471+
username: config.CLICKHOUSE_USER,
2472+
password: config.CLICKHOUSE_PASSWORD,
2473+
});
2474+
2475+
const response = await authRequest('post', BASE_URL)
2476+
.send({
2477+
name: 'Dashboard with Mismatched Source Connection',
2478+
tiles: [
2479+
{
2480+
name: 'Raw SQL Tile',
2481+
x: 0,
2482+
y: 0,
2483+
w: 6,
2484+
h: 3,
2485+
config: {
2486+
configType: 'sql',
2487+
displayType: 'table',
2488+
connectionId: otherConnection._id.toString(),
2489+
sourceId: traceSource._id.toString(),
2490+
sqlTemplate: 'SELECT count() FROM otel_logs',
2491+
},
2492+
},
2493+
],
2494+
tags: [],
2495+
})
2496+
.expect(400);
2497+
2498+
expect(response.body).toEqual({
2499+
message: `The following source IDs do not match the specified connections: ${traceSource._id.toString()}`,
2500+
});
2501+
});
2502+
24602503
it('should create a dashboard with filters', async () => {
24612504
const dashboardPayload = {
24622505
name: 'Dashboard with Filters',
@@ -3100,6 +3143,7 @@ describe('External API v2 Dashboards - new format', () => {
31003143

31013144
it('can round-trip all raw SQL chart config types', async () => {
31023145
const connectionId = connection._id.toString();
3146+
const sourceId = traceSource._id.toString();
31033147
const sqlTemplate = 'SELECT count() FROM otel_logs WHERE {timeFilter}';
31043148

31053149
const lineRawSql: ExternalDashboardTileWithId = {
@@ -3114,6 +3158,7 @@ describe('External API v2 Dashboards - new format', () => {
31143158
displayType: 'line',
31153159
connectionId,
31163160
sqlTemplate,
3161+
sourceId,
31173162
compareToPreviousPeriod: true,
31183163
fillNulls: true,
31193164
alignDateRangeToGranularity: true,
@@ -3133,6 +3178,7 @@ describe('External API v2 Dashboards - new format', () => {
31333178
displayType: 'stacked_bar',
31343179
connectionId,
31353180
sqlTemplate,
3181+
sourceId,
31363182
fillNulls: false,
31373183
alignDateRangeToGranularity: false,
31383184
numberFormat: { output: 'byte', decimalBytes: true },
@@ -3151,6 +3197,7 @@ describe('External API v2 Dashboards - new format', () => {
31513197
displayType: 'table',
31523198
connectionId,
31533199
sqlTemplate,
3200+
sourceId,
31543201
numberFormat: { output: 'percent', mantissa: 1 },
31553202
},
31563203
};
@@ -3167,6 +3214,7 @@ describe('External API v2 Dashboards - new format', () => {
31673214
displayType: 'number',
31683215
connectionId,
31693216
sqlTemplate,
3217+
sourceId,
31703218
numberFormat: { output: 'currency', currencySymbol: '$' },
31713219
},
31723220
};
@@ -3183,6 +3231,7 @@ describe('External API v2 Dashboards - new format', () => {
31833231
displayType: 'pie',
31843232
connectionId,
31853233
sqlTemplate,
3234+
sourceId,
31863235
},
31873236
};
31883237

@@ -3271,6 +3320,45 @@ describe('External API v2 Dashboards - new format', () => {
32713320
});
32723321
});
32733322

3323+
it('should return 400 when source connection does not match tile connection', async () => {
3324+
const dashboard = await createTestDashboard();
3325+
const otherConnection = await Connection.create({
3326+
team: team._id,
3327+
name: 'Other Connection',
3328+
host: config.CLICKHOUSE_HOST,
3329+
username: config.CLICKHOUSE_USER,
3330+
password: config.CLICKHOUSE_PASSWORD,
3331+
});
3332+
3333+
const response = await authRequest('put', `${BASE_URL}/${dashboard._id}`)
3334+
.send({
3335+
name: 'Updated Dashboard with Mismatched Source Connection',
3336+
tiles: [
3337+
{
3338+
id: new ObjectId().toString(),
3339+
name: 'Raw SQL Tile',
3340+
x: 0,
3341+
y: 0,
3342+
w: 6,
3343+
h: 3,
3344+
config: {
3345+
configType: 'sql',
3346+
displayType: 'table',
3347+
connectionId: otherConnection._id.toString(),
3348+
sourceId: traceSource._id.toString(),
3349+
sqlTemplate: 'SELECT count() FROM otel_logs',
3350+
},
3351+
},
3352+
],
3353+
tags: [],
3354+
})
3355+
.expect(400);
3356+
3357+
expect(response.body).toEqual({
3358+
message: `The following source IDs do not match the specified connections: ${traceSource._id.toString()}`,
3359+
});
3360+
});
3361+
32743362
it('should delete alert when tile is updated from builder to raw SQL config', async () => {
32753363
const tileId = new ObjectId().toString();
32763364
const dashboard = await createTestDashboard({

‎packages/api/src/routers/external-api/v2/dashboards.ts‎

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ async function getMissingSources(
5353
}
5454
}
5555
} else if (isConfigTile(tile)) {
56-
if ('sourceId' in tile.config) {
56+
if ('sourceId' in tile.config && tile.config.sourceId) {
5757
sourceIds.add(tile.config.sourceId);
5858
}
5959
}
@@ -99,6 +99,30 @@ async function getMissingConnections(
9999
);
100100
}
101101

102+
async function getSourceConnectionMismatches(
103+
team: string | mongoose.Types.ObjectId,
104+
tiles: ExternalDashboardTileWithId[],
105+
): Promise<string[]> {
106+
const existingSources = await getSources(team.toString());
107+
const sourceById = new Map(existingSources.map(s => [s._id.toString(), s]));
108+
109+
const sourcesWithInvalidConnections: string[] = [];
110+
for (const tile of tiles) {
111+
if (
112+
isConfigTile(tile) &&
113+
isRawSqlExternalTileConfig(tile.config) &&
114+
tile.config.sourceId
115+
) {
116+
const source = sourceById.get(tile.config.sourceId);
117+
if (source && source.connection.toString() !== tile.config.connectionId) {
118+
sourcesWithInvalidConnections.push(tile.config.sourceId);
119+
}
120+
}
121+
}
122+
123+
return sourcesWithInvalidConnections;
124+
}
125+
102126
type SavedQueryLanguage = z.infer<typeof whereLanguageSchema>;
103127

104128
function resolveSavedQueryLanguage(params: {
@@ -823,6 +847,10 @@ const updateDashboardBodySchema = buildDashboardBodySchema(
823847
* maxLength: 100000
824848
* description: SQL query template to execute. Supports HyperDX template variables.
825849
* example: "SELECT count() FROM otel_logs WHERE timestamp > now() - INTERVAL 1 HOUR"
850+
* sourceId:
851+
* type: string
852+
* description: Optional ID of the data source associated with this Raw SQL chart. Used for applying dashboard filters.
853+
* example: "65f5e4a3b9e77c001a567890"
826854
* numberFormat:
827855
* $ref: '#/components/schemas/NumberFormat'
828856
* description: Number formatting options for displayed values.
@@ -1669,10 +1697,12 @@ router.post(
16691697
savedFilterValues,
16701698
} = req.body;
16711699

1672-
const [missingSources, missingConnections] = await Promise.all([
1673-
getMissingSources(teamId, tiles, filters),
1674-
getMissingConnections(teamId, tiles),
1675-
]);
1700+
const [missingSources, missingConnections, sourceConnectionMismatches] =
1701+
await Promise.all([
1702+
getMissingSources(teamId, tiles, filters),
1703+
getMissingConnections(teamId, tiles),
1704+
getSourceConnectionMismatches(teamId, tiles),
1705+
]);
16761706
if (missingSources.length > 0) {
16771707
return res.status(400).json({
16781708
message: `Could not find the following source IDs: ${missingSources.join(
@@ -1687,6 +1717,13 @@ router.post(
16871717
)}`,
16881718
});
16891719
}
1720+
if (sourceConnectionMismatches.length > 0) {
1721+
return res.status(400).json({
1722+
message: `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join(
1723+
', ',
1724+
)}`,
1725+
});
1726+
}
16901727

16911728
const internalTiles = tiles.map(tile => {
16921729
const tileId = new ObjectId().toString();
@@ -1902,10 +1939,12 @@ router.put(
19021939
savedFilterValues,
19031940
} = req.body ?? {};
19041941

1905-
const [missingSources, missingConnections] = await Promise.all([
1906-
getMissingSources(teamId, tiles, filters),
1907-
getMissingConnections(teamId, tiles),
1908-
]);
1942+
const [missingSources, missingConnections, sourceConnectionMismatches] =
1943+
await Promise.all([
1944+
getMissingSources(teamId, tiles, filters),
1945+
getMissingConnections(teamId, tiles),
1946+
getSourceConnectionMismatches(teamId, tiles),
1947+
]);
19091948
if (missingSources.length > 0) {
19101949
return res.status(400).json({
19111950
message: `Could not find the following source IDs: ${missingSources.join(
@@ -1920,6 +1959,13 @@ router.put(
19201959
)}`,
19211960
});
19221961
}
1962+
if (sourceConnectionMismatches.length > 0) {
1963+
return res.status(400).json({
1964+
message: `The following source IDs do not match the specified connections: ${sourceConnectionMismatches.join(
1965+
', ',
1966+
)}`,
1967+
});
1968+
}
19231969

19241970
const existingDashboard = await Dashboard.findOne(
19251971
{ _id: dashboardId, team: teamId },

‎packages/api/src/routers/external-api/v2/utils/dashboards.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ const convertToExternalTileChartConfig = (
102102
displayType: DisplayType.Line,
103103
connectionId: config.connection,
104104
sqlTemplate: config.sqlTemplate,
105+
sourceId: config.source,
105106
alignDateRangeToGranularity: config.alignDateRangeToGranularity,
106107
fillNulls: config.fillNulls !== false,
107108
numberFormat: config.numberFormat,
@@ -113,6 +114,7 @@ const convertToExternalTileChartConfig = (
113114
displayType: config.displayType,
114115
connectionId: config.connection,
115116
sqlTemplate: config.sqlTemplate,
117+
sourceId: config.source,
116118
alignDateRangeToGranularity: config.alignDateRangeToGranularity,
117119
fillNulls: config.fillNulls !== false,
118120
numberFormat: config.numberFormat,
@@ -123,6 +125,7 @@ const convertToExternalTileChartConfig = (
123125
displayType: DisplayType.Table,
124126
connectionId: config.connection,
125127
sqlTemplate: config.sqlTemplate,
128+
sourceId: config.source,
126129
numberFormat: config.numberFormat,
127130
};
128131
case DisplayType.Number:
@@ -131,6 +134,7 @@ const convertToExternalTileChartConfig = (
131134
displayType: DisplayType.Number,
132135
connectionId: config.connection,
133136
sqlTemplate: config.sqlTemplate,
137+
sourceId: config.source,
134138
numberFormat: config.numberFormat,
135139
};
136140
case DisplayType.Pie:
@@ -139,6 +143,7 @@ const convertToExternalTileChartConfig = (
139143
displayType: DisplayType.Pie,
140144
connectionId: config.connection,
141145
sqlTemplate: config.sqlTemplate,
146+
sourceId: config.source,
142147
numberFormat: config.numberFormat,
143148
};
144149
case DisplayType.Search:
@@ -342,6 +347,7 @@ export function convertToInternalTileConfig(
342347
name,
343348
connection: externalConfig.connectionId,
344349
sqlTemplate: externalConfig.sqlTemplate,
350+
source: externalConfig.sourceId,
345351
} satisfies RawSqlSavedChartConfig;
346352
break;
347353
case 'table':
@@ -358,6 +364,7 @@ export function convertToInternalTileConfig(
358364
name,
359365
connection: externalConfig.connectionId,
360366
sqlTemplate: externalConfig.sqlTemplate,
367+
source: externalConfig.sourceId,
361368
numberFormat: externalConfig.numberFormat,
362369
} satisfies RawSqlSavedChartConfig;
363370
break;

‎packages/api/src/utils/zod.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ const externalDashboardRawSqlChartConfigBaseSchema = z.object({
211211
configType: z.literal('sql'),
212212
connectionId: objectIdSchema,
213213
sqlTemplate: z.string().max(100000),
214+
sourceId: objectIdSchema.optional(),
214215
numberFormat: NumberFormatSchema.optional(),
215216
});
216217

0 commit comments

Comments
 (0)