perf: read only median-sized parts in text index value lookups

Author: knudtty

.changeset/text-index-values-no-row-cap.md

@@ -1,5 +1,7 @@
 ---
 '@hyperdx/common-utils': patch
+'@hyperdx/api': patch
+'@hyperdx/app': patch
 ---
 
 fix: show values for every key in text index value lookups
@@ -9,3 +11,9 @@ indexes on plain columns) stopped at `max_rows_to_read` with
 `read_overflow_mode = 'break'`. The index is read in sorted token order, so once
 a high-cardinality key used up the budget, every key that sorts after it came
 back with no values and no error. These lookups no longer apply a row cap.
+
+To keep them bounded, they now read a limited set of parts like Map key
+discovery: the 20 parts closest to the median size by default, or as set by the
+source's "Map key index parts" (`mapKeyIndexMaxParts`) and "Map key index part
+size" (`mapKeyIndexPartSelection`) settings, which now cover values as well as
+keys.

packages/api/openapi.json

@@ -5206,7 +5206,7 @@
           "mapKeyIndexMaxParts": {
             "type": "integer",
             "minimum": 1,
-            "description": "How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.",
+            "description": "How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.",
             "nullable": true,
             "example": 20
           },
@@ -5217,7 +5217,7 @@
               "smallest",
               "largest"
             ],
-            "description": "Which data parts, by row count, Map attribute key discovery reads from a text index. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.",
+            "description": "Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.",
             "nullable": true,
             "example": "median"
           },
@@ -5478,7 +5478,7 @@
           "mapKeyIndexMaxParts": {
             "type": "integer",
             "minimum": 1,
-            "description": "How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.",
+            "description": "How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.",
             "nullable": true,
             "example": 20
           },
@@ -5489,7 +5489,7 @@
               "smallest",
               "largest"
             ],
-            "description": "Which data parts, by row count, Map attribute key discovery reads from a text index. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.",
+            "description": "Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.",
             "nullable": true,
             "example": "median"
           },

packages/api/src/mcp/tools/sources/describeSource.ts

@@ -228,6 +228,12 @@ async function describeSourceSchema(
     'metadataMaterializedViews' in source
       ? source.metadataMaterializedViews
       : undefined;
+  const mapKeyIndexMaxParts =
+    'mapKeyIndexMaxParts' in source ? source.mapKeyIndexMaxParts : undefined;
+  const mapKeyIndexPartSelection =
+    'mapKeyIndexPartSelection' in source
+      ? source.mapKeyIndexPartSelection
+      : undefined;
 
   const now = new Date();
   const dateRange: [Date, Date] = [
@@ -306,14 +312,8 @@ async function describeSourceSchema(
             maxKeys: 50,
             connectionId,
             metadataMVs,
-            mapKeyIndexMaxParts:
-              'mapKeyIndexMaxParts' in source
-                ? source.mapKeyIndexMaxParts
-                : undefined,
-            mapKeyIndexPartSelection:
-              'mapKeyIndexPartSelection' in source
-                ? source.mapKeyIndexPartSelection
-                : undefined,
+            mapKeyIndexMaxParts,
+            mapKeyIndexPartSelection,
             dateRange,
             timestampValueExpression,
             signal,
@@ -360,6 +360,8 @@ async function describeSourceSchema(
         maxValuesPerKey: MAX_LC_VALUES,
         connectionId,
         metadataMVs,
+        mapKeyIndexMaxParts,
+        mapKeyIndexPartSelection,
         dateRange,
         timestampValueExpression,
         signal,
@@ -407,6 +409,8 @@ async function describeSourceSchema(
         maxValuesPerKey: MAX_MAP_KEY_VALUES,
         connectionId,
         metadataMVs,
+        mapKeyIndexMaxParts,
+        mapKeyIndexPartSelection,
         dateRange,
         timestampValueExpression,
         signal,

packages/api/src/mcp/tools/sources/schemas.ts

@@ -454,17 +454,17 @@ const mcpSaveSourceSchema = z.object({
     .min(1)
     .optional()
     .describe(
-      'Log and trace sources. How many data parts Map attribute key discovery ' +
-        'reads from a text index, chosen by mapKeyIndexPartSelection. Raise it ' +
-        'if attribute keys are missing from filters; lower it if field loading ' +
-        'is slow. Omit for the default of 20.',
+      'Log and trace sources. How many data parts are read from text indexes ' +
+        'to discover attribute keys and filter values, chosen by ' +
+        'mapKeyIndexPartSelection. Raise it if keys or values are missing from ' +
+        'filters; lower it if they load slowly. Omit for the default of 20.',
     ),
   mapKeyIndexPartSelection: z
     .enum(['median', 'smallest', 'largest'])
     .optional()
     .describe(
-      'Log and trace sources. Which data parts, by row count, Map attribute ' +
-        'key discovery reads from a text index: "median" (used when unset) ' +
+      'Log and trace sources. Which data parts, by row count, are read from ' +
+        'text indexes for attribute keys and filter values: "median" (used when unset) ' +
         'balances coverage and cost, "smallest" reads the newest unmerged ' +
         'parts, "largest" reads the most merged data.',
     ),

packages/api/src/routers/external-api/v2/sources.ts

@@ -415,13 +415,13 @@ function formatExternalSource(source: SourceDocument) {
  *         mapKeyIndexMaxParts:
  *           type: integer
  *           minimum: 1
- *           description: How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.
+ *           description: How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.
  *           nullable: true
  *           example: 20
  *         mapKeyIndexPartSelection:
  *           type: string
  *           enum: [median, smallest, largest]
- *           description: Which data parts, by row count, Map attribute key discovery reads from a text index. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data.
+ *           description: Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data.
  *           nullable: true
  *           example: median
  *         highlightedTraceAttributeExpressions:
@@ -628,13 +628,13 @@ function formatExternalSource(source: SourceDocument) {
  *         mapKeyIndexMaxParts:
  *           type: integer
  *           minimum: 1
- *           description: How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.
+ *           description: How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.
  *           nullable: true
  *           example: 20
  *         mapKeyIndexPartSelection:
  *           type: string
  *           enum: [median, smallest, largest]
- *           description: Which data parts, by row count, Map attribute key discovery reads from a text index. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data.
+ *           description: Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data.
  *           nullable: true
  *           example: median
  *         highlightedTraceAttributeExpressions:

packages/app/src/components/DBSearchPageFilters/hooks.ts

@@ -255,6 +255,8 @@ function useFacets({
           tableName: tableConnection.tableName,
           connectionId: tableConnection.connectionId,
           metadataMVs: tableConnection.metadataMVs,
+          mapKeyIndexMaxParts: tableConnection.mapKeyIndexMaxParts,
+          mapKeyIndexPartSelection: tableConnection.mapKeyIndexPartSelection,
           keyExpressions: [sqlKey],
           maxValuesPerKey: SINGLE_KEY_VALUES_LIMIT,
           dateRange,

packages/app/src/components/Sources/SourceForm/MapKeyIndexMaxPartsFormRow.tsx

@@ -16,7 +16,7 @@ export function MapKeyIndexMaxPartsFormRow({
   return (
     <FormRow
       label="Map key index parts"
-      helpText={`How many data parts are read from a Map key text index to discover attribute keys; "Map key index part size" picks which ones. Raise it if keys are missing from filters; lower it if fields load slowly. Defaults to ${DEFAULT_MAP_KEY_INDEX_MAX_PARTS}.`}
+      helpText={`How many data parts are read from text indexes to discover attribute keys and filter values; "Map key index part size" picks which ones. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to ${DEFAULT_MAP_KEY_INDEX_MAX_PARTS}.`}
     >
       <Controller
         control={control}
@@ -46,7 +46,7 @@ export function MapKeyIndexPartSelectionFormRow({
   return (
     <FormRow
       label="Map key index part size"
-      helpText='Which data parts, by row count, are read from a Map key text index. "Median" balances coverage and cost; "Smallest" reads the newest unmerged parts, which is cheap and shows new keys first; "Largest" reads the most merged data, which covers the most keys but reads the biggest indexes.'
+      helpText='Which data parts, by row count, are read from text indexes for attribute keys and filter values. "Median" balances coverage and cost; "Smallest" reads the newest unmerged parts, which is cheap and shows new keys first; "Largest" reads the most merged data, which covers the most keys but reads the biggest indexes.'
     >
       <SelectControlled
         control={control}

packages/app/src/hooks/useMetadata.tsx

@@ -577,12 +577,22 @@ export function useMultipleGetKeyValues(
         const source = chartConfig.source
           ? sources?.find(s => s.id === chartConfig.source)
           : undefined;
+        const isLogOrTrace =
+          !!source && (isLogSource(source) || isTraceSource(source));
         const metadataMVs =
           usableMVsOverride ??
-          (source && (isLogSource(source) || isTraceSource(source))
-            ? source.metadataMaterializedViews
-            : undefined);
-        return { chartConfig, source, metadataMVs };
+          (isLogOrTrace ? source.metadataMaterializedViews : undefined);
+        return {
+          chartConfig,
+          source,
+          metadataMVs,
+          mapKeyIndexMaxParts: isLogOrTrace
+            ? source.mapKeyIndexMaxParts
+            : undefined,
+          mapKeyIndexPartSelection: isLogOrTrace
+            ? source.mapKeyIndexPartSelection
+            : undefined,
+        };
       });
 
       if (mode === 'exact') {
@@ -606,18 +616,26 @@ export function useMultipleGetKeyValues(
       }
 
       // mode === 'all'
-      const streams = lookups.map(({ chartConfig, metadataMVs }) =>
-        metadata.streamAllKeyValues({
-          databaseName: chartConfig.from.databaseName,
-          tableName: chartConfig.from.tableName,
-          keyExpressions: limitedKeys,
-          maxValuesPerKey: limit,
-          connectionId: chartConfig.connection,
+      const streams = lookups.map(
+        ({
+          chartConfig,
           metadataMVs,
-          dateRange: chartConfig.dateRange,
-          timestampValueExpression: chartConfig.timestampValueExpression,
-          signal,
-        }),
+          mapKeyIndexMaxParts,
+          mapKeyIndexPartSelection,
+        }) =>
+          metadata.streamAllKeyValues({
+            databaseName: chartConfig.from.databaseName,
+            tableName: chartConfig.from.tableName,
+            keyExpressions: limitedKeys,
+            maxValuesPerKey: limit,
+            connectionId: chartConfig.connection,
+            metadataMVs,
+            mapKeyIndexMaxParts,
+            mapKeyIndexPartSelection,
+            dateRange: chartConfig.dateRange,
+            timestampValueExpression: chartConfig.timestampValueExpression,
+            signal,
+          }),
       );
       for await (const keyValues of mergeStreams(streams)) {
         yield keyValues.map(({ key, value }) => ({

packages/common-utils/src/__tests__/metadata.test.ts

@@ -1429,6 +1429,62 @@ describe('Metadata', () => {
       },
     );
 
+    it.each([
+      ['map KV', "LogAttributes['requestId']"],
+      ['native', 'TraceId'],
+    ])(
+      'reads only the parts nearest the median size from the %s text index',
+      async (_, keyExpression) => {
+        setupDefaultLogsSchema();
+
+        await metadata.getAllKeyValues({
+          ...baseArgs,
+          keyExpressions: [keyExpression],
+        });
+        await metadata.getAllKeyValues({
+          ...baseArgs,
+          keyExpressions: [keyExpression],
+          mapKeyIndexMaxParts: 5,
+        });
+
+        const partsLimits = (mockClickhouseClient.query as jest.Mock).mock.calls
+          .filter(([opts]) => opts.query.includes('mergeTreeTextIndex('))
+          .map(([opts]) => {
+            const match = opts.query.match(
+              /ORDER BY abs\(rn - \(cnt \+ 1\) \/ 2\), name\s+LIMIT \{(\w+):Int32\}/,
+            );
+            expect(match).not.toBeNull();
+            return opts.query_params[match[1]];
+          });
+        expect(partsLimits).toEqual([20, 5]);
+      },
+    );
+
+    it.each([
+      ['map KV', "LogAttributes['requestId']"],
+      ['native', 'TraceId'],
+    ])(
+      'orders the %s text index parts by the configured part selection',
+      async (_, keyExpression) => {
+        setupDefaultLogsSchema();
+
+        await metadata.getAllKeyValues({
+          ...baseArgs,
+          keyExpressions: [keyExpression],
+          mapKeyIndexPartSelection: MapKeyIndexPartSelection.Largest,
+        });
+
+        const textIndexCall = (
+          mockClickhouseClient.query as jest.Mock
+        ).mock.calls.find(([opts]) =>
+          opts.query.includes('mergeTreeTextIndex('),
+        );
+        expect(textIndexCall[0].query).toMatch(
+          /\)\s+ORDER BY rn DESC\s+LIMIT \{\w+:Int32\}/,
+        );
+      },
+    );
+
     it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => {
       setupDefaultLogsSchema();