feat: add ability to connect to external prometheus datastore (#2518)

Author: knudtty

.changeset/connection-prometheus-endpoint-ui.md

@@ -0,0 +1,9 @@
+---
+'@hyperdx/app': minor
+'@hyperdx/api': minor
+'@hyperdx/common-utils': minor
+---
+
+Add UI support for configuring an external Prometheus-compatible endpoint on a
+connection. Modify Connections model to now have a boolean
+`isPrometheusEndpoint` field and use host for storing the host.

packages/api/openapi.json

@@ -620,11 +620,10 @@
             "nullable": true,
             "example": "hyperdx_"
           },
-          "prometheusEndpoint": {
-            "type": "string",
-            "description": "Optional Prometheus-compatible API endpoint. When set, PromQL queries are proxied to this endpoint.",
-            "nullable": true,
-            "example": "http://prometheus:9090"
+          "isPrometheusEndpoint": {
+            "type": "boolean",
+            "description": "Optional. When true, `host` is treated as a Prometheus-compatible API endpoint (e.g. Prometheus or Thanos) and PromQL queries are proxied to it. When false or omitted, `host` is a ClickHouse HTTP endpoint.",
+            "example": false
           },
           "createdAt": {
             "type": "string",
@@ -675,10 +674,10 @@
             "nullable": true,
             "example": "hyperdx_"
           },
-          "prometheusEndpoint": {
-            "type": "string",
-            "description": "Optional Prometheus-compatible API endpoint. When set, PromQL queries are proxied to this endpoint.",
-            "example": "http://prometheus:9090"
+          "isPrometheusEndpoint": {
+            "type": "boolean",
+            "description": "Optional. When true, `host` is treated as a Prometheus-compatible API endpoint (e.g. Prometheus or Thanos) and PromQL queries are proxied to it. When false or omitted, `host` is a ClickHouse HTTP endpoint.",
+            "example": false
           }
         }
       },
@@ -717,11 +716,10 @@
             "nullable": true,
             "example": "hyperdx_"
           },
-          "prometheusEndpoint": {
-            "type": "string",
-            "description": "Optional Prometheus-compatible API endpoint. Set to null to clear the existing value. If omitted, the existing value is kept.",
-            "nullable": true,
-            "example": "http://prometheus:9090"
+          "isPrometheusEndpoint": {
+            "type": "boolean",
+            "description": "Optional. When true, `host` is treated as a Prometheus-compatible API endpoint. When false or omitted, `host` is a ClickHouse HTTP endpoint. Omit to keep the existing value unchanged.",
+            "example": false
           }
         }
       },

packages/api/src/models/connection.ts

@@ -12,10 +12,11 @@ export interface IConnection {
   username: string;
   team: ObjectId;
   hyperdxSettingPrefix?: string;
-  /** Optional Prometheus-compatible API endpoint (e.g. http://prometheus:9090).
-   *  When set, PromQL queries are proxied to this endpoint instead of using
-   *  ClickHouse's prometheusQuery() function. */
-  prometheusEndpoint?: string;
+  /** When true, `host` is treated as a Prometheus-compatible API endpoint
+   *  (e.g. Prometheus or Thanos) and PromQL queries are proxied directly to
+   *  it. When false/unset, `host` is a ClickHouse HTTP endpoint and PromQL
+   *  queries use ClickHouse's prometheusQuery() function. */
+  isPrometheusEndpoint?: boolean;
 }
 
 export type ConnectionDocument = mongoose.HydratedDocument<IConnection>;
@@ -37,7 +38,7 @@ export default mongoose.model<IConnection>(
         select: false,
       },
       hyperdxSettingPrefix: String,
-      prometheusEndpoint: String,
+      isPrometheusEndpoint: Boolean,
     },
     {
       timestamps: true,

packages/api/src/routers/api/__tests__/connections.test.ts

@@ -43,67 +43,79 @@ describe('connections router', () => {
     expect(res.body[0].name).toBe('My Team Connection');
   });
 
-  it('persists prometheusEndpoint through POST /connections', async () => {
+  it('persists isPrometheusEndpoint through POST /connections', async () => {
     const { agent } = await getLoggedInAgent(server);
 
     const res = await agent
       .post('/connections')
       .send({
         name: 'Prom-enabled',
-        host: config.CLICKHOUSE_HOST,
-        username: 'default',
+        host: 'http://thanos:10902',
+        username: '',
         password: '',
-        prometheusEndpoint: 'http://prom.example.com',
+        isPrometheusEndpoint: true,
       })
       .expect(200);
 
-    const stored = await Connection.findById(res.body.id).select(
-      '+prometheusEndpoint',
-    );
-    expect(stored?.prometheusEndpoint).toBe('http://prom.example.com');
+    const stored = await Connection.findById(res.body.id);
+    expect(stored?.host).toBe('http://thanos:10902');
+    expect(stored?.isPrometheusEndpoint).toBe(true);
   });
 
-  it('rejects invalid prometheusEndpoint URLs with 400', async () => {
-    const { agent } = await getLoggedInAgent(server);
+  it('persists isPrometheusEndpoint through PUT /connections/:id', async () => {
+    const { agent, team } = await getLoggedInAgent(server);
+
+    const created = await Connection.create({
+      team: team._id,
+      name: 'No-prom',
+      host: config.CLICKHOUSE_HOST,
+      username: 'default',
+      password: '',
+    });
 
     await agent
-      .post('/connections')
+      .put(`/connections/${created._id.toString()}`)
       .send({
-        name: 'Bad-URL',
-        host: config.CLICKHOUSE_HOST,
-        username: 'default',
+        id: created._id.toString(),
+        name: 'No-prom',
+        host: 'http://thanos:10902',
+        username: '',
         password: '',
-        prometheusEndpoint: 'not-a-url',
+        isPrometheusEndpoint: true,
       })
-      .expect(400);
+      .expect(200);
+
+    const stored = await Connection.findById(created._id);
+    expect(stored?.host).toBe('http://thanos:10902');
+    expect(stored?.isPrometheusEndpoint).toBe(true);
   });
 
-  it('persists prometheusEndpoint through PUT /connections/:id', async () => {
+  it('toggles isPrometheusEndpoint back to false through PUT', async () => {
     const { agent, team } = await getLoggedInAgent(server);
 
     const created = await Connection.create({
       team: team._id,
-      name: 'No-prom',
-      host: config.CLICKHOUSE_HOST,
-      username: 'default',
+      name: 'Prom',
+      host: 'http://thanos:10902',
+      username: '',
       password: '',
+      isPrometheusEndpoint: true,
     });
 
     await agent
       .put(`/connections/${created._id.toString()}`)
       .send({
         id: created._id.toString(),
-        name: 'No-prom',
+        name: 'Prom',
         host: config.CLICKHOUSE_HOST,
         username: 'default',
         password: '',
-        prometheusEndpoint: 'http://prom-new.example.com',
+        isPrometheusEndpoint: false,
       })
       .expect(200);
 
-    const stored = await Connection.findById(created._id).select(
-      '+prometheusEndpoint',
-    );
-    expect(stored?.prometheusEndpoint).toBe('http://prom-new.example.com');
+    const stored = await Connection.findById(created._id);
+    expect(stored?.host).toBe(config.CLICKHOUSE_HOST);
+    expect(stored?.isPrometheusEndpoint).toBe(false);
   });
 });

packages/api/src/routers/api/__tests__/prometheus.integration.test.ts

@@ -6,6 +6,26 @@ import Connection from '@/models/connection';
 
 const mockFetch = global.fetch as jest.Mock;
 
+// The proxy now streams the upstream response straight through (no
+// `await resp.json()`), so test mocks must expose the fields the pipeline
+// actually reads: `status`, `headers.get()`, and a web `ReadableStream` body.
+function fakeUpstreamResponse(payload: unknown, status = 200) {
+  const body = new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(new TextEncoder().encode(JSON.stringify(payload)));
+      controller.close();
+    },
+  });
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    headers: new Headers({ 'content-type': 'application/json' }),
+    body,
+    text: jest.fn().mockResolvedValue(JSON.stringify(payload)),
+    json: jest.fn().mockResolvedValue(payload),
+  };
+}
+
 describe('prometheus router', () => {
   const server = getServer();
 
@@ -16,11 +36,7 @@ describe('prometheus router', () => {
   afterEach(async () => {
     await server.clearDBs();
     mockFetch.mockReset();
-    mockFetch.mockResolvedValue({
-      ok: true,
-      text: jest.fn().mockResolvedValue(''),
-      json: jest.fn().mockResolvedValue({}),
-    } as any);
+    mockFetch.mockResolvedValue(fakeUpstreamResponse({}) as any);
   });
 
   afterAll(async () => {
@@ -31,10 +47,10 @@ describe('prometheus router', () => {
     return Connection.create({
       team: teamId,
       name: 'Prom',
-      host: 'http://ch-server:8123',
-      username: 'default',
+      host: 'http://prom.example.com',
+      username: '',
       password: '',
-      prometheusEndpoint: 'http://prom.example.com',
+      isPrometheusEndpoint: true,
     });
   };
 
@@ -95,19 +111,17 @@ describe('prometheus router', () => {
       });
     });
 
-    it('proxies to upstream Prometheus when connection has prometheusEndpoint', async () => {
+    it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => {
       const { agent, team } = await getLoggedInAgent(server);
       const conn = await seedPrometheusConnection(team._id);
 
       const promResponse = {
         status: 'success',
         data: { resultType: 'matrix', result: [] },
       };
-      mockFetch.mockResolvedValueOnce({
-        ok: true,
-        text: jest.fn().mockResolvedValue(''),
-        json: jest.fn().mockResolvedValue(promResponse),
-      } as any);
+      mockFetch.mockResolvedValueOnce(
+        fakeUpstreamResponse(promResponse) as any,
+      );
 
       const res = await agent
         .get('/v1/prometheus/query_range')
@@ -129,7 +143,7 @@ describe('prometheus router', () => {
       expect(calledUrl).not.toContain('connectionId');
     });
 
-    it('does NOT proxy to Prometheus when connection has no prometheusEndpoint', async () => {
+    it('does NOT proxy to Prometheus when connection is not isPrometheusEndpoint', async () => {
       const { agent, team } = await getLoggedInAgent(server);
       const conn = await seedClickHouseConnection(team._id);
 
@@ -197,19 +211,17 @@ describe('prometheus router', () => {
         .expect(404);
     });
 
-    it('proxies to upstream Prometheus when connection has prometheusEndpoint', async () => {
+    it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => {
       const { agent, team } = await getLoggedInAgent(server);
       const conn = await seedPrometheusConnection(team._id);
 
       const promResponse = {
         status: 'success',
         data: { resultType: 'vector', result: [] },
       };
-      mockFetch.mockResolvedValueOnce({
-        ok: true,
-        text: jest.fn().mockResolvedValue(''),
-        json: jest.fn().mockResolvedValue(promResponse),
-      } as any);
+      mockFetch.mockResolvedValueOnce(
+        fakeUpstreamResponse(promResponse) as any,
+      );
 
       const res = await agent
         .get('/v1/prometheus/query')
@@ -238,16 +250,14 @@ describe('prometheus router', () => {
         .expect(404);
     });
 
-    it('proxies to upstream Prometheus when connection has prometheusEndpoint', async () => {
+    it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => {
       const { agent, team } = await getLoggedInAgent(server);
       const conn = await seedPrometheusConnection(team._id);
 
       const promResponse = { status: 'success', data: ['up', 'requests'] };
-      mockFetch.mockResolvedValueOnce({
-        ok: true,
-        text: jest.fn().mockResolvedValue(''),
-        json: jest.fn().mockResolvedValue(promResponse),
-      } as any);
+      mockFetch.mockResolvedValueOnce(
+        fakeUpstreamResponse(promResponse) as any,
+      );
 
       const res = await agent
         .get('/v1/prometheus/label/__name__/values')

packages/api/src/routers/api/connections.ts

@@ -67,7 +67,6 @@ router.put(
         return;
       }
 
-      // Build the base connection update
       const shouldUnsetPrefix =
         req.body.hyperdxSettingPrefix === null ||
         req.body.hyperdxSettingPrefix === '';
@@ -82,7 +81,6 @@ router.put(
           : {
               password: connection.password,
             }),
-        // Only include hyperdxSettingPrefix if it's a valid string
         ...(!shouldUnsetPrefix && hyperdxSettingPrefix
           ? { hyperdxSettingPrefix }
           : {}),