feat: add Zendesk producer connector

change/@useairfoil-producer-zendesk-2c6d6c00-afc7-4357-b3e4-5047cba0b4f0.json

@@ -0,0 +1,7 @@
+{
+  "type": "patch",
+  "comment": "add the Zendesk producer connector",
+  "packageName": "@useairfoil/producer-zendesk",
+  "email": "jadejajaipal5@gmail.com",
+  "dependentChangeType": "patch"
+}

connectors/producer-zendesk/.env.example

@@ -0,0 +1,23 @@
+ZENDESK_SUBDOMAIN=acme
+ZENDESK_CLIENT_ID=airfoil
+ZENDESK_CLIENT_SECRET=XX
+AIRFOIL_HTTP_PORT=8080
+
+WINGS_URI=http://localhost:7777
+AIRFOIL_CATALOG=default-catalog
+AIRFOIL_TABLE_BINDINGS={"tickets":{"namespace":["default"],"name":"zendesk_tickets"},"ticket_comments":{"namespace":["default"],"name":"zendesk_ticket_comments"},"users":{"namespace":["default"],"name":"zendesk_users"},"organizations":{"namespace":["default"],"name":"zendesk_organizations"},"groups":{"namespace":["default"],"name":"zendesk_groups"},"brands":{"namespace":["default"],"name":"zendesk_brands"},"ticket_fields":{"namespace":["default"],"name":"zendesk_ticket_fields"},"ticket_forms":{"namespace":["default"],"name":"zendesk_ticket_forms"},"custom_statuses":{"namespace":["default"],"name":"zendesk_custom_statuses"},"survey_responses":{"namespace":["default"],"name":"zendesk_survey_responses"},"csat_surveys":{"namespace":["default"],"name":"zendesk_csat_surveys"},"ticket_field_history":{"namespace":["default"],"name":"zendesk_ticket_field_history"},"ticket_metric_events":{"namespace":["default"],"name":"zendesk_ticket_metric_events"},"satisfaction_ratings":{"namespace":["default"],"name":"zendesk_satisfaction_ratings"},"group_memberships":{"namespace":["default"],"name":"zendesk_group_memberships"},"organization_memberships":{"namespace":["default"],"name":"zendesk_organization_memberships"},"user_fields":{"namespace":["default"],"name":"zendesk_user_fields"},"organization_fields":{"namespace":["default"],"name":"zendesk_organization_fields"},"custom_roles":{"namespace":["default"],"name":"zendesk_custom_roles"},"sla_policies":{"namespace":["default"],"name":"zendesk_sla_policies"},"schedules":{"namespace":["default"],"name":"zendesk_schedules"},"macros":{"namespace":["default"],"name":"zendesk_macros"},"articles":{"namespace":["default"],"name":"zendesk_articles"},"sections":{"namespace":["default"],"name":"zendesk_sections"},"categories":{"namespace":["default"],"name":"zendesk_categories"},"community_posts":{"namespace":["default"],"name":"zendesk_community_posts"},"community_topics":{"namespace":["default"],"name":"zendesk_community_topics"},"talk_lines":{"namespace":["default"],"name":"zendesk_talk_lines"}}
+
+# Production `start` also requires a mounted connector JSON file and PostgreSQL state.
+# Sandboxes do not use these values.
+# AIRFOIL_CONFIG_PATH=/var/run/airfoil/config/config.json
+# AIRFOIL_CONNECTOR_INSTANCE_ID=team-acme-zendesk-primary
+# POSTGRES_CONNECTION_STRING=postgresql://user:password@localhost:5432/airfoil
+
+# Optional OpenTelemetry trace and metric export. Logs remain local.
+# The endpoint is the OTLP base URL; Connector Kit appends /v1/traces and /v1/metrics.
+OTEL_ENABLED=false
+OTEL_SERVICE_NAME=producer-zendesk
+# OTEL_SERVICE_VERSION=0.1.0
+# OTEL_RESOURCE_ATTRIBUTES=service.instance.id=team-acme-zendesk-primary,airfoil.connector.revision=1,airfoil.team.id=team-acme
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+# OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer <token>,X-Axiom-Dataset=<dataset>

connectors/producer-zendesk/AGENTS.md

@@ -0,0 +1,55 @@
+# Zendesk producer
+
+## Current scope
+
+- Resources: tickets and their comments, field history, and metric events;
+  users and organizations with their memberships and fields; CSAT; Help
+  Center; Talk lines; and the lists in `src/resources/lists.ts`
+- Auth: one OAuth client with the client credentials grant, for one account
+- Changes: every 5 minutes for the exports, every hour for the rest
+- Webhooks: none
+
+## Zendesk rules
+
+- Every request goes through `src/client`, so VCR records it.
+- Incremental exports allow 10 calls a minute, shared with every other tool on
+  the account. The users export has its own 20. Their limiters ignore headers,
+  or they would pick up the account limit. Calls are spaced evenly, because a
+  full bucket bursts past the limit.
+- A `401` drops the cached token and tries once more.
+- The token asks for `read`. Schedules and Talk have no narrower scope.
+- Tokens ask for `expires_in`. Without it, older clients make tokens that
+  never expire and pile up in the account.
+- Use the cursor exports where they exist (tickets, users). Organizations,
+  ticket events, metric events, and Help Center articles only have time-based
+  ones. The articles export has no `end_of_stream`.
+- Zendesk holds back the last minute. The first changes run starts a minute
+  before the cutoff.
+- Lists use `page[size]` cursor paging. Custom statuses, SLA policies, roles,
+  schedules, and Talk lines have none. The survey APIs allow at most 50 a page.
+
+Ticket versions use `generated_timestamp`: `updated_at` misses system updates.
+Comments use the ticket event time. Survey responses use the latest answer's
+`updated_at`. Other resources use `updated_at`, except schedules and Talk
+lines, which use the read time: holidays don't change `updated_at`, and lines
+have none.
+
+Deleted tickets have `status: "deleted"` and can be restored, so ticket rows
+send `_deleted: false`. Users (`active: false`) and organizations
+(`deleted_at`) can't be restored, so their other rows leave it out. The small
+lists drop deleted items, so they have no `_deleted`. Memberships keep the IDs
+they saw in their cursor and send deletes for the ones that are gone.
+
+A comment event shows the comment as it is now. After a comment or attachment
+redaction, read the ticket's comments again. Skip `404` for deleted tickets.
+Privacy changes have no child events and are not caught.
+
+Survey answers can change for 28 days. The list filters by when the survey was
+sent, so read from 29 days before the saved cursor, not the current time.
+Skip responses with no answers.
+
+A column can't go back to null. Before adding a column Zendesk often clears,
+note it in the README.
+
+Record VCR against a test account only. The VCR test builds the connector once,
+because a second token request would replay the redacted token.

connectors/producer-zendesk/Dockerfile

@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1
+
+FROM node:24-bookworm-slim AS build
+
+ENV PNPM_HOME=/pnpm
+ENV PATH=$PNPM_HOME:$PATH
+
+RUN corepack enable && corepack prepare pnpm@10.33.2 --activate
+
+WORKDIR /workspace
+
+COPY pnpm-lock.yaml pnpm-workspace.yaml package.json nx.json ./
+RUN pnpm fetch --frozen-lockfile
+
+COPY . .
+RUN pnpm install --offline --frozen-lockfile
+RUN pnpm nx run @useairfoil/producer-zendesk:build
+RUN pnpm --config.inject-workspace-packages=true \
+  --filter @useairfoil/producer-zendesk deploy --prod /connector
+
+FROM node:24-bookworm-slim AS runtime
+
+ENV NODE_ENV=production
+
+WORKDIR /app
+
+COPY --from=build --chown=node:node /connector ./
+
+USER node
+
+EXPOSE 8080
+
+ENTRYPOINT ["node", "dist/main.mjs"]
+CMD ["start"]

connectors/producer-zendesk/README.md

@@ -0,0 +1,70 @@
+# Zendesk producer
+
+Syncs tickets, comments, field history, metric events, users, organizations,
+satisfaction ratings and surveys, Help Center articles and community posts,
+Talk lines, and the lists that name their IDs from one Zendesk account. It
+backfills each resource and polls for changes. There are no webhooks.
+
+## Config
+
+| Variable                | Required | Default |
+| ----------------------- | -------- | ------- |
+| `ZENDESK_SUBDOMAIN`     | yes      | none    |
+| `ZENDESK_CLIENT_ID`     | yes      | none    |
+| `ZENDESK_CLIENT_SECRET` | yes      | none    |
+
+## Setup
+
+As an admin, add an OAuth client in Admin Center under
+Apps and integrations > APIs > OAuth clients:
+
+- Client kind: Confidential.
+- Scopes: leave empty, or allow `read`.
+- No redirect URL is needed.
+
+Use the Identifier as `ZENDESK_CLIENT_ID` and the secret as
+`ZENDESK_CLIENT_SECRET`. Use `acme` for `acme.zendesk.com`; the full address
+also works.
+
+The token uses the permissions of the admin who made the client.
+
+## Data
+
+- `tickets.metrics` holds reply, wait, and resolution times. `custom_fields`
+  maps field IDs to values, and `ticket_fields` names them.
+- `ticket_comments` includes internal notes (`public` is false). It has the
+  text, not attachment files.
+- Deleted tickets, users, and organizations get `_deleted`. Restored tickets
+  come back.
+- Groups, brands, ticket fields, forms, and statuses keep deleted items, so
+  older tickets still have their names. So do the other lists, articles, and
+  Talk lines. Removed group and organization memberships get `_deleted`.
+- `ticket_field_history` has one row per changed field. Custom fields use
+  their ID.
+- `satisfaction_ratings` holds legacy CSAT ratings. A new score is a new row.
+- `survey_responses` holds CSAT answers. `csat_surveys` names the questions
+  and options. Both need the new CSAT, on Support Professional or Suite Growth
+  and up. Unanswered surveys are skipped.
+- Tables hold names, emails, and ticket text. Treat them as confidential.
+
+## Limits
+
+- Incremental exports share a limit of 10 calls a minute per account, or 30
+  with the High Volume add-on. Large comment backfills can be slow.
+- `ticket_comments` and `ticket_field_history` each read the ticket events
+  export, so both count toward that limit.
+- Exports are polled every 5 minutes. Zendesk leaves out the latest minute.
+- A column can't go back to null. A cleared assignee keeps its old value, and
+  so does `organization_id` when its organization is deleted.
+- A comment made private later stays public in `ticket_comments`. Zendesk's
+  event for it has no details. A redacted comment or attachment is read again.
+
+## Local development
+
+```bash
+cd connectors/producer-zendesk
+pnpm sandbox
+```
+
+`pnpm start` runs the hosted connector. It needs the Wings and PostgreSQL
+variables from the [Connector Kit README](../../packages/connector-kit/README.md).

connectors/producer-zendesk/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@useairfoil/producer-zendesk",
+  "version": "0.0.1",
+  "repository": "useairfoil/typescript-sdk",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "type": "module",
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
+    },
+    "./manifest": {
+      "types": "./dist/manifest.d.mts",
+      "import": "./dist/manifest.mjs",
+      "default": "./dist/manifest.mjs"
+    }
+  },
+  "scripts": {
+    "build": "tsdown",
+    "docker:build": "docker build --load --file Dockerfile --tag airfoil/producer-zendesk:local ../..",
+    "start": "tsx --env-file=.env src/main.ts start",
+    "sandbox": "tsx --env-file=.env src/main.ts sandbox",
+    "test": "dotenvx run --ignore=MISSING_ENV_FILE --quiet -- vitest",
+    "test:ci": "dotenvx run --ignore=MISSING_ENV_FILE --quiet -- vitest run",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@effect/platform-node": "catalog:",
+    "@effect/sql-pg": "catalog:",
+    "@useairfoil/connector-kit": "workspace:*",
+    "effect": "catalog:"
+  },
+  "devDependencies": {
+    "@dotenvx/dotenvx": "^1.62.0",
+    "@effect/vitest": "catalog:",
+    "@types/node": "catalog:",
+    "@useairfoil/effect-vcr": "workspace:*",
+    "tsdown": "catalog:",
+    "tsx": "^4.21.0",
+    "vitest": "catalog:"
+  }
+}

connectors/producer-zendesk/src/client/auth.ts

@@ -0,0 +1,104 @@
+import { Auth, ConnectorError, Telemetry } from "@useairfoil/connector-kit";
+import { Context, Duration, Effect, Layer, Redacted, Schedule, Schema } from "effect";
+import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
+
+import type { ZendeskConfig } from "../manifest";
+
+import { baseUrl } from "./constants";
+
+type ZendeskAuthConfig = Pick<ZendeskConfig, "subdomain" | "clientId" | "clientSecret">;
+
+export type ZendeskAuthService = Auth.TokenCache<ConnectorError>;
+
+export class ZendeskAuth extends Context.Service<ZendeskAuth, ZendeskAuthService>()(
+  "@useairfoil/producer-zendesk/ZendeskAuth",
+) {}
+
+// Schedules and Talk have no narrower scope.
+export const scopes = "read";
+
+// Older OAuth clients issue non-expiring tokens without expires_in.
+const tokenLifetime = Duration.minutes(30);
+
+const TokenResponseSchema = Schema.Struct({
+  access_token: Schema.String,
+  expires_in: Schema.Number,
+});
+
+const TokenErrorSchema = Schema.Struct({ error: Schema.optional(Schema.String) });
+
+const tokenError = (response: HttpClientResponse.HttpClientResponse, subdomain: string) =>
+  HttpClientResponse.schemaBodyJson(TokenErrorSchema)(response).pipe(
+    Effect.map((body) => body.error),
+    Effect.orElseSucceed(() => undefined),
+    Effect.flatMap((error) =>
+      Effect.fail(
+        new ConnectorError({
+          message:
+            response.status === 404
+              ? `No Zendesk account at ${baseUrl(subdomain)}`
+              : error === "invalid_client"
+                ? "Zendesk rejected the OAuth client ID or secret"
+                : error === "invalid_scope"
+                  ? `The Zendesk OAuth client must allow these scopes: ${scopes}`
+                  : `Zendesk returned ${response.status} for the OAuth token`,
+        }),
+      ),
+    ),
+  );
+
+export const make = Effect.fnUntraced(function* (config: ZendeskAuthConfig) {
+  const client = (yield* HttpClient.HttpClient).pipe(
+    HttpClient.retryTransient({
+      schedule: Schedule.exponential("250 millis").pipe(Schedule.jittered),
+      times: 3,
+    }),
+  );
+  const request = HttpClientRequest.post(`${baseUrl(config.subdomain)}/oauth/tokens`).pipe(
+    HttpClientRequest.acceptJson,
+    HttpClientRequest.bodyJsonUnsafe({
+      grant_type: "client_credentials",
+      client_id: config.clientId,
+      client_secret: Redacted.value(config.clientSecret),
+      scope: scopes,
+      expires_in: Duration.toSeconds(tokenLifetime),
+    }),
+  );
+
+  return yield* Auth.makeTokenCache({
+    acquire: Effect.gen(function* () {
+      const response = yield* client
+        .execute(request)
+        .pipe(
+          Effect.mapError(
+            (cause) => new ConnectorError({ message: "Zendesk OAuth token request failed", cause }),
+          ),
+        );
+      const body = yield* HttpClientResponse.matchStatus(response, {
+        "2xx": (ok) =>
+          HttpClientResponse.schemaBodyJson(TokenResponseSchema)(ok).pipe(
+            Effect.mapError(
+              (cause) =>
+                new ConnectorError({ message: "Invalid Zendesk OAuth token response", cause }),
+            ),
+          ),
+        orElse: (response) => tokenError(response, config.subdomain),
+      });
+      return {
+        value: Redacted.make(body.access_token),
+        expiresIn: Duration.seconds(body.expires_in),
+      };
+    }).pipe(
+      Effect.tapError((error) => Telemetry.annotateError("api_auth", error)),
+      Effect.withSpan(Telemetry.SpanName.apiFetch, {
+        kind: "client",
+        attributes: { [Telemetry.Attr.apiPath]: "/oauth/tokens" },
+      }),
+    ),
+  });
+});
+
+export const layer = (
+  config: ZendeskAuthConfig,
+): Layer.Layer<ZendeskAuth, never, HttpClient.HttpClient> =>
+  Layer.effect(ZendeskAuth)(make(config));