fix(webapp,redis-worker): stop logging raw metadata, alert payloads, and job items

Debug logging around run metadata writes previously included the full,
unfiltered metadata object on every flush. It now logs key/operation counts
and byte sizes instead, and skips the log entirely when there is nothing
buffered. The webapp logger's redaction list also now covers `metadata` and
`seedMetadata` as a backstop.

Alert webhook delivery failures no longer log the outgoing request body or
the (useless, non-serializable) HMAC signature; they log the response
status, the webhook URL host, and the relevant ids instead.

The redis-worker's failure, retry, and dead-letter logs no longer include
the raw job item. The item is still retrievable by id when needed, and the
default worker logger now filters the `item` key as a backstop.

Author: carderne

apps/webapp/app/services/logger.server.ts

@@ -50,7 +50,7 @@ function flattenArgs(args: Array<Record<string, unknown> | undefined>) {
 export const logger = new Logger(
   "webapp",
   (process.env.APP_LOG_LEVEL ?? "info") as LogLevel,
-  ["examples", "output", "connectionString", "payload"],
+  ["examples", "output", "connectionString", "payload", "metadata", "seedMetadata"],
   sensitiveDataReplacer,
   () => {
     const fields = currentFieldsStore.getStore();

apps/webapp/app/services/metadata/updateMetadata.server.ts

@@ -91,9 +91,14 @@ export class UpdateMetadataService {
         this._bufferedOperations.clear();
 
         yield* Effect.sync(() => {
-          if (this.flushLoggingEnabled) {
+          if (this.flushLoggingEnabled && currentOperations.size > 0) {
+            const operationCount = Array.from(currentOperations.values()).reduce(
+              (sum, ops) => sum + ops.length,
+              0
+            );
             this.logger.debug(`[UpdateMetadataService] Flushing operations`, {
-              operations: Object.fromEntries(currentOperations),
+              runCount: currentOperations.size,
+              operationCount,
             });
           }
         });
@@ -520,9 +525,9 @@ export class UpdateMetadataService {
 
       if (this.flushLoggingEnabled) {
         this.logger.debug(`[updateRunMetadataWithOperations] Updated metadata for run`, {
-          metadata: applyResults.newMetadata,
-          operations: operations,
           runId,
+          metadataKeyCount: Object.keys(applyResults.newMetadata).length,
+          operationCount: operations.length,
         });
       }
 
@@ -567,8 +572,8 @@ export class UpdateMetadataService {
     ) {
       if (this.flushLoggingEnabled) {
         this.logger.debug(`[updateRunMetadataDirectly] Updating metadata directly for run`, {
-          metadata: metadataPacket.data,
           runId,
+          metadataSizeBytes: metadataPacket.data?.length ?? 0,
         });
       }
 
@@ -607,7 +612,7 @@ export class UpdateMetadataService {
     if (this.flushLoggingEnabled) {
       this.logger.debug(`[ingestRunOperations] Ingesting operations for run`, {
         runId,
-        bufferedOperations,
+        operationCount: bufferedOperations.length,
       });
     }
 

apps/webapp/app/v3/services/alerts/deliverAlert.server.ts

@@ -455,7 +455,7 @@ export class DeliverAlertService extends BaseService {
                 error,
               };
 
-              await this.#deliverWebhook(payload, webhookProperties.data);
+              await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id, runId: alert.taskRun.friendlyId });
               break;
             }
             case "v2": {
@@ -516,7 +516,7 @@ export class DeliverAlertService extends BaseService {
                 },
               };
 
-              await this.#deliverWebhook(payload, webhookProperties.data);
+              await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id, runId: alert.taskRun.friendlyId });
 
               break;
             }
@@ -577,7 +577,7 @@ export class DeliverAlertService extends BaseService {
                 vercel: this.#buildWebhookVercelObject(deploymentMeta.vercelDeploymentUrl),
               };
 
-              await this.#deliverWebhook(payload, webhookProperties.data);
+              await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id });
               break;
             }
             case "v2": {
@@ -616,7 +616,7 @@ export class DeliverAlertService extends BaseService {
                 },
               };
 
-              await this.#deliverWebhook(payload, webhookProperties.data);
+              await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id });
 
               break;
             }
@@ -671,7 +671,7 @@ export class DeliverAlertService extends BaseService {
                 vercel: this.#buildWebhookVercelObject(deploymentMeta.vercelDeploymentUrl),
               };
 
-              await this.#deliverWebhook(payload, webhookProperties.data);
+              await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id });
               break;
             }
             case "v2": {
@@ -716,7 +716,7 @@ export class DeliverAlertService extends BaseService {
                 },
               };
 
-              await this.#deliverWebhook(payload, webhookProperties.data);
+              await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id });
 
               break;
             }
@@ -1017,7 +1017,11 @@ export class DeliverAlertService extends BaseService {
     }
   }
 
-  async #deliverWebhook<T>(payload: T, webhook: ProjectAlertWebhookProperties) {
+  async #deliverWebhook<T>(
+    payload: T,
+    webhook: ProjectAlertWebhookProperties,
+    context: { webhookId: string; runId?: string }
+  ) {
     const rawPayload = JSON.stringify(payload);
     const hashPayload = Buffer.from(rawPayload, "utf-8");
 
@@ -1046,12 +1050,14 @@ export class DeliverAlertService extends BaseService {
     });
 
     if (!response.ok) {
+      // Never log the request/response body here: it is customer-controlled alert
+      // content and may include stack traces or other application data.
       logger.info("[DeliverAlert] Failed to send alert webhook", {
         status: response.status,
         statusText: response.statusText,
-        url: webhook.url,
-        body: payload,
-        signature,
+        urlHost: safeUrlHost(webhook.url),
+        webhookId: context.webhookId,
+        runId: context.runId,
       });
 
       throw new Error(`Failed to send alert webhook to ${webhook.url}`);
@@ -1435,3 +1441,11 @@ function isWebAPIHTTPError(error: unknown): error is WebAPIHTTPError {
 function isWebAPIRateLimitedError(error: unknown): error is WebAPIRateLimitedError {
   return (error as WebAPIRateLimitedError).code === ErrorCode.RateLimitedError;
 }
+
+function safeUrlHost(url: string): string {
+  try {
+    return new URL(url).host;
+  } catch {
+    return "unknown";
+  }
+}

packages/redis-worker/src/worker.ts

@@ -140,7 +140,7 @@ class Worker<TCatalog extends WorkerCatalog> {
   > = new Map();
 
   constructor(private options: WorkerOptions<TCatalog>) {
-    this.logger = options.logger ?? new Logger("Worker", "debug");
+    this.logger = options.logger ?? new Logger("Worker", "debug", ["item"]);
     this.tracer = options.tracer ?? trace.getTracer(options.name);
     this.meter = options.meter ?? metrics.getMeter(options.name);
 
@@ -608,7 +608,8 @@ class Worker<TCatalog extends WorkerCatalog> {
                 this.logger.error("Unhandled error in processItem:", {
                   error: err,
                   workerId,
-                  item,
+                  id: queueItem.id,
+                  job: queueItem.job,
                 });
               }
             );
@@ -933,11 +934,12 @@ class Worker<TCatalog extends WorkerCatalog> {
       const errorLogLevel =
         error && typeof error === "object" && "logLevel" in error ? error.logLevel : undefined;
 
+      // Never include the raw item/payload here: it is job data that may be
+      // customer-controlled. It is retrievable via `getJob(id)` if needed for triage.
       const logAttributes = {
         name: this.options.name,
         id,
         job,
-        item,
         visibilityTimeoutMs,
         error,
         errorMessage,
@@ -994,7 +996,6 @@ class Worker<TCatalog extends WorkerCatalog> {
           name: this.options.name,
           id,
           job,
-          item,
           retryDate,
           retryDelay,
           visibilityTimeoutMs,
@@ -1015,7 +1016,6 @@ class Worker<TCatalog extends WorkerCatalog> {
             name: this.options.name,
             id,
             job,
-            item,
             visibilityTimeoutMs,
             error: requeueError,
           }