feat: fetch key limit now applies per map (#3392)

## Summary

The fetch key limit now applies per map. Previously, the behavior was a bit random. If you find 100 LogAttributes, often times the ResourceAttributes wouldn't display in the filters at all, and values would not be fetched for any ResourceAttribute columns. Now, for a fetch key limit of n, there will be n keys attempting to fetch values instead of 100 total across all maps.

### References

- Linear Issue: Closes HDX-5752

Author: knudtty

.changeset/filter-keys-limit-per-map.md

@@ -0,0 +1,7 @@
+---
+'@hyperdx/app': patch
+---
+
+fix: the search page filter key limit (Filter Keys Fetch Limit, default 100)
+now applies to each Map column, so keys from `LogAttributes` no longer crowd out
+`ResourceAttributes` and other Map columns

packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx

@@ -319,7 +319,7 @@ export default function TeamQueryConfigSection() {
           <ClickhouseSettingForm
             settingKey="filterKeysFetchLimit"
             label="Filter Keys Fetch Limit"
-            tooltip="The number of filter keys to fetch when clicking 'More filters' on the search page"
+            tooltip="The number of filter keys to fetch per map column (e.g. LogAttributes, ResourceAttributes) on the search page"
             type="number"
             defaultValue={DEFAULT_FILTER_KEYS_FETCH_LIMIT}
             placeholder={`default = ${DEFAULT_FILTER_KEYS_FETCH_LIMIT}`}

packages/app/src/hooks/__tests__/useMetadata.test.tsx

@@ -8,6 +8,7 @@ import {
   Metadata,
   MetadataCache,
 } from '@hyperdx/common-utils/dist/core/metadata';
+import { FilterState } from '@hyperdx/common-utils/dist/filters';
 import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
 import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 import { renderHook, waitFor } from '@testing-library/react';
@@ -508,6 +509,64 @@ describe('useGetKeyValues', () => {
     },
   );
 
+  it('applies the filter key limit to each Map column separately', async () => {
+    queryClient.setQueryData(['me'], { team: { filterKeysFetchLimit: 2 } });
+    const getKeyValuesWithMVs = jest
+      .spyOn(mockMetadata, 'getKeyValuesWithMVs')
+      .mockResolvedValue([]);
+    const conditionFor = (key: string): FilterState => ({
+      [key]: { included: new Set(['v']), excluded: new Set() },
+    });
+    const allKeys = [
+      "LogAttributes['a']",
+      "LogAttributes['b']",
+      "LogAttributes['c']",
+      "`Resource-Attributes`['x']",
+      'ServiceName',
+      'SeverityText',
+      'SpanName',
+    ];
+    const keptKeys = [
+      "LogAttributes['a']",
+      "LogAttributes['b']",
+      "`Resource-Attributes`['x']",
+      'ServiceName',
+      'SeverityText',
+    ];
+
+    const { result } = renderHook(
+      () =>
+        useGetKeyValues({
+          chartConfig: createMockChartConfig(),
+          keys: allKeys,
+          keyConditions: allKeys.map(conditionFor),
+        }),
+      { wrapper },
+    );
+
+    await waitFor(() => expect(result.current.isSuccess).toBe(true));
+    expect(getKeyValuesWithMVs).toHaveBeenCalledWith(
+      expect.objectContaining({
+        keys: keptKeys,
+        keyConditions: keptKeys.map(conditionFor),
+      }),
+    );
+  });
+
+  it('passes no keyConditions when none are given', async () => {
+    const getKeyValuesWithMVs = jest
+      .spyOn(mockMetadata, 'getKeyValuesWithMVs')
+      .mockResolvedValue([]);
+
+    const { result } = renderHook(
+      () => useGetKeyValues({ chartConfig: createMockChartConfig(), keys }),
+      { wrapper },
+    );
+
+    await waitFor(() => expect(result.current.isSuccess).toBe(true));
+    expect(getKeyValuesWithMVs.mock.calls[0][0].keyConditions).toBeUndefined();
+  });
+
   it("retries a failed 'exact' mode lookup per the client's defaults", async () => {
     queryClient.setDefaultOptions({ queries: { retry: 1, retryDelay: 0 } });
     mockLookups.exact(new Error('transient'), [

packages/app/src/hooks/useMetadata.tsx

@@ -9,6 +9,7 @@ import {
 import {
   Field,
   MetricNames,
+  parseKeyPath,
   TableConnection,
   TableMetadata,
 } from '@hyperdx/common-utils/dist/core/metadata';
@@ -521,6 +522,30 @@ export function useMultipleGetKeyValues(
       if (keys.length === 0) return;
       const usableMVsOverride =
         chartConfigsArr.length === 1 ? metadataMVsOverride : undefined;
+      // The limit applies per Map column, so one wide Map (e.g. LogAttributes)
+      // can't crowd out the others. Top-level columns share a single limit.
+      const { limitedKeys, limitedKeyConditions } = keys.reduce<{
+        countByColumn: Map<string, number>;
+        limitedKeys: string[];
+        limitedKeyConditions?: (FilterState | undefined)[];
+      }>(
+        (acc, key, i) => {
+          const path = parseKeyPath(key);
+          const column = path.length > 1 ? path[0] : '';
+          const count = acc.countByColumn.get(column) ?? 0;
+          acc.countByColumn.set(column, count + 1);
+          if (count < maxKeys) {
+            acc.limitedKeys.push(key);
+            acc.limitedKeyConditions?.push(keyConditions?.[i]);
+          }
+          return acc;
+        },
+        {
+          countByColumn: new Map(),
+          limitedKeys: [],
+          limitedKeyConditions: keyConditions ? [] : undefined,
+        },
+      );
 
       const lookups = chartConfigsArr.map(chartConfig => {
         const source = chartConfig.source
@@ -539,8 +564,8 @@ export function useMultipleGetKeyValues(
           lookups.map(({ chartConfig, source }) =>
             metadata.getKeyValuesWithMVs({
               chartConfig,
-              keys: keys.slice(0, maxKeys),
-              keyConditions: keyConditions?.slice(0, maxKeys),
+              keys: limitedKeys,
+              keyConditions: limitedKeyConditions,
               limit,
               disableRowLimit,
               source,
@@ -559,7 +584,7 @@ export function useMultipleGetKeyValues(
         metadata.streamAllKeyValues({
           databaseName: chartConfig.from.databaseName,
           tableName: chartConfig.from.tableName,
-          keyExpressions: keys.slice(0, maxKeys),
+          keyExpressions: limitedKeys,
           maxValuesPerKey: limit,
           connectionId: chartConfig.connection,
           metadataMVs,