diff --git a/.env.example b/.env.example index 10c44bb8..63a99366 100644 --- a/.env.example +++ b/.env.example @@ -149,12 +149,17 @@ GOOGLE_CLIENT_SECRET="" # ── Optional: operations ───────────────────────────────────────────────────── # Shared cache. Without it each instance caches in its own memory, which is -# correct — just not shared. +# correct — just not shared. Recommended in production once website tracking is +# on: the compiled tracking config, the collector's rate limit and the hourly +# cap on contacts created from forms are all counted here, and per-instance +# counters let a multi-instance deployment exceed both. # REDIS_URL="redis://localhost:6379" # CACHE_TTL_MS="60000" # Bearer token guarding POST /internal/sync/google, the Gmail/Calendar cron -# route. The route refuses to run without it. At least 16 characters. +# route, and POST /internal/tracking/retention, the nightly sweep that deletes +# tracked page views older than 90 days. Both refuse to run without it. At +# least 16 characters. # CRON_SECRET="" # Log every SQL statement Prisma runs, at debug level, without bound diff --git a/.github/.release-please-manifest.json b/.github/.release-please-manifest.json index 8b4d9c87..b7046446 100644 --- a/.github/.release-please-manifest.json +++ b/.github/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.5.0" + ".": "1.5.1" } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ba911a7d..59023368 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,16 +45,34 @@ jobs: run: | set -euo pipefail - stuck=$(gh pr list --repo "$GITHUB_REPOSITORY" --base main --state merged --label "autorelease: pending" --limit 1 --json number,title --jq '.[0] | select(.) | "#\(.number) \(.title)"') + candidates=$(gh pr list --repo "$GITHUB_REPOSITORY" --base main --state merged --label "autorelease: pending" --limit 20 --json number --jq '.[].number') + + stuck="" + + for pr in $candidates; do + state=$(gh api "repos/$GITHUB_REPOSITORY/issues/$pr/labels" --jq 'map(.name) | if index("autorelease: tagged") then "tagged" elif index("autorelease: pending") then "pending" else "cleared" end') + + if [ "$state" != "pending" ]; then + continue + fi + + title=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$pr" --jq '.title') + stuck="${stuck}#${pr} ${title}"$'\n' + done if [ -z "$stuck" ]; then exit 0 fi { - echo "**$stuck merged without a tag.** release-please aborts on it, so nothing releases and no release" - echo "pull request opens until it is cleared: create the tag and GitHub Release at that pull request's" - echo "merge commit, then swap its \`autorelease: pending\` label for \`autorelease: tagged\`." + echo "**A merged release pull request was never tagged.** release-please aborts on it, so nothing" + echo "releases and no release pull request opens until it is cleared: create the tag and GitHub" + echo "Release at that pull request's merge commit, then swap its \`autorelease: pending\` label for" + echo "\`autorelease: tagged\`." + echo + echo '```' + echo "$stuck" + echo '```' } >> "$GITHUB_STEP_SUMMARY" exit 1 diff --git a/AGENTS.md b/AGENTS.md index d361a7d0..7083f64c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ context until you read them, and the rules in them are not optional. | UI in `apps/app` or `packages/ui` | `docs/design.md` (below) | | Deal amounts, totals, charts, exchange rates | `docs/currency.md` | | The record sheet's Agent tab | `docs/agent-panel.md` | +| The tracking script, the collector, form submissions | `docs/tracking.md` | | Running it locally, Google Cloud, DB commands, secrets | `docs/setup.md` | | Anything that sends a telemetry event, or a new property on one | `docs/telemetry.md` | | `.github/workflows`, versions, changelog, how a change reaches `release` | `CONTRIBUTING.md` | diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ab95d52..164cfc34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.5.1](https://github.com/trycompai/crm/compare/v1.5.0...v1.5.1) (2026-08-08) + + +### Fixes + +* **api:** warn when the deployed schema does not match schema.prisma ([#88](https://github.com/trycompai/crm/issues/88)) ([f445c68](https://github.com/trycompai/crm/commit/f445c68a815ad1635498591daa494d18d9508ccf)) + ## [1.5.0](https://github.com/trycompai/crm/compare/v1.4.0...v1.5.0) (2026-08-08) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9a0ed91b..e3f0b623 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,6 +52,16 @@ integration tests. When you need to push past it — a WIP branch, a docker-less you are deliberately pushing to ask about — `git push --no-verify` skips it, and `CRM_SKIP_HOOKS=1` skips it for a whole shell. +**`test` runs one package at a time (`turbo run test --concurrency=1`), and that is not an +oversight.** `apps/api`, `apps/agent`, `packages/auth` and `packages/telemetry` all have real +integration tests and they all point at the *same* database, so running them at once lets one +package's fixtures land inside another package's assertions. The specs mutate global singletons — +the workspace `organization` row, `AppSetting`'s reporting currency, the exchange-rate table — and +none of that is namespaced per package. Left parallel it failed roughly three runs in four, on a +different test each time, which reads as "flaky tests" and trains everyone to hit re-run. Serial +costs about ten seconds. **Do not raise the concurrency without giving each package its own +database.** + A few things that trip people up: - **The tRPC router type is generated, and committed.** If the app can't see a procedure you just @@ -171,7 +181,7 @@ release workflow, which cannot wait on a run in another workflow. A jam used to be silent — the workflow reported success while doing nothing, and the symptom was merges landing with no release PR behind them. **The workflow now fails when a merged release PR carries `autorelease: pending`**, which is the state every jam ends in, so the Actions tab tells you -within a minute of the merge that shipped it. Read the log rather than the status anyway. Two +within a minute of the merge that shipped it. Read the log rather than the status anyway. Three failures have actually happened here: - **`There are untagged, merged release PRs outstanding - aborting`.** A release PR was merged but @@ -186,6 +196,15 @@ failures have actually happened here: ever tagged automatically — `v1.0.0` through `v1.3.0` were all cut by hand. **A second package will bring the Merge plugin back**, and with it this bug: give the packages components in the tag, or check that a merged release PR still gets `autorelease: tagged`. +- **The guard failing on the release it just cut.** Every release run failed — `v1.4.0`, `v1.5.0` + and `v1.5.1` were all tagged correctly, and the run that tagged each one then reported it as + stuck. `gh pr list --label` reads the **search index, which is eventually consistent**, and + release-please swaps `autorelease: pending` for `autorelease: tagged` about a second before the + guard runs; the index still held the old label. A guard that cries wolf on every release is worse + than no guard, because the one real jam is indistinguishable from the noise. The search is now + only a prefilter: each candidate's labels are re-read through `gh api .../issues/N/labels`, which + is strongly consistent, and only a pull request that is still genuinely `pending` fails the run. + **Any label check written against `gh pr list --search`/`--label` needs the same treatment.** - **`commit could not be parsed`, in bulk.** Non-conventional subjects reached `main`. They are not errors, they are silently missing changelog lines. The squash-only merge policy and the `conventional commit` check exist to stop this; if you see it again, one of the two has been diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 723ff44f..a6bd22dc 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -28,6 +28,7 @@ import { SettingsModule } from "./settings/settings.module"; import { SsoModule } from "./sso/sso.module"; import { SyncModule } from "./sync/sync.module"; import { TelemetryModule } from "./telemetry/telemetry.module"; +import { TrackingModule } from "./tracking/tracking.module"; import { TrpcModule } from "./trpc/trpc.module"; import { UsersModule } from "./users/users.module"; import { WorkspaceModule } from "./workspace/workspace.module"; @@ -67,6 +68,7 @@ import { WorkspaceModule } from "./workspace/workspace.module"; SsoModule, BackfillModule, TelemetryModule, + TrackingModule, ], }) export class AppModule {} diff --git a/apps/api/src/generated/server.ts b/apps/api/src/generated/server.ts index 695f1d6b..8cc4b950 100644 --- a/apps/api/src/generated/server.ts +++ b/apps/api/src/generated/server.ts @@ -26,6 +26,7 @@ import { setAutoCreateInput, suppressDomainInput, threadInput, calendarEventInpu import { setOutlookAutoCreateInput } from "../microsoft/microsoft.contracts"; import { setAgentModelInput, setResearchKeyInput } from "../settings/settings.contracts"; import { ssoProviderListInput, registerSsoProviderInput, deleteSsoProviderInput } from "../sso/sso.contracts"; +import { trackingFlagInput, cookieLifetimeInput, addDomainInput, removeDomainInput, verifyInput, companyActivityInput, contactActivityInput } from "../tracking/tracking.contracts"; import { memberListInput, updateWorkspaceInput, setMemberRoleInput } from "../workspace/workspace.contracts"; import type { ActivitiesRouter } from "../activities/activities.router"; import type { AgentsRouter } from "../agent/agents.router"; @@ -41,6 +42,7 @@ import type { MicrosoftRouter } from "../microsoft/microsoft.router"; import type { SearchRouter } from "../search/search.router"; import type { SettingsRouter } from "../settings/settings.router"; import type { SsoRouter } from "../sso/sso.router"; +import type { TrackingRouter } from "../tracking/tracking.router"; import type { UsersRouter } from "../users/users.router"; import type { WorkspaceRouter } from "../workspace/workspace.router"; @@ -383,6 +385,35 @@ const appRouter = t.router({ .input(deleteSsoProviderInput) .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>) }), + tracking: t.router({ + settings: publicProcedure + .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>), + setFlag: publicProcedure + .input(trackingFlagInput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>), + setCookieLifetime: publicProcedure + .input(cookieLifetimeInput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>), + addDomain: publicProcedure + .input(addDomainInput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>), + removeDomain: publicProcedure + .input(removeDomainInput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>), + rotateSiteId: publicProcedure + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>), + verify: publicProcedure + .input(verifyInput) + .mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>), + sources: publicProcedure + .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>), + companyActivity: publicProcedure + .input(companyActivityInput) + .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>), + contactActivity: publicProcedure + .input(contactActivityInput) + .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>) + }), users: t.router({ me: publicProcedure .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited>), diff --git a/apps/api/src/telemetry/rollup.service.ts b/apps/api/src/telemetry/rollup.service.ts index 91df3f36..928cd990 100644 --- a/apps/api/src/telemetry/rollup.service.ts +++ b/apps/api/src/telemetry/rollup.service.ts @@ -9,6 +9,7 @@ import { } from "@crm/db"; import { RETIRED_OUTCOME } from "@crm/db/agent-tasks"; import { readAgentModel } from "@crm/db/settings"; +import { CONTACT_CAP_REASON } from "@crm/db/tracking"; import { WORKSPACE_ID } from "@crm/db/workspace"; import { bucket, @@ -495,6 +496,30 @@ export class RollupService { }), ]); + const [ + trackingSite, + trackingDomains, + trackingViews, + trackingForms, + trackingContacts, + trackingCapped, + trackingPaused, + ] = await Promise.all([ + this.db.appSetting.count({ where: { trackingSiteId: { not: null } } }), + this.db.trackedDomain.count(), + this.db.trackedEvent.count({ + where: { type: "page_view", occurredAt: { gte: since } }, + }), + this.db.formSubmission.count({ where: { createdAt: { gte: since } } }), + this.db.contact.count({ + where: { createdAt: { gte: since }, source: RecordSource.TRACKING }, + }), + this.db.formSubmission.count({ + where: { createdAt: { gte: since }, skipReason: CONTACT_CAP_REASON }, + }), + this.db.appSetting.count({ where: { trackingPaused: true } }), + ]); + const configured = syncs.reduce((sum, row) => sum + row._count._all, 0); return { @@ -528,6 +553,14 @@ export class RollupService { Object.values(ActivityType), ), + cap_tracking: trackingSite > 0, + tracking_domains: bucket(trackingDomains), + tracking_page_views: trackingViews, + tracking_forms: trackingForms, + tracking_contacts_created: trackingContacts, + tracking_capped: trackingCapped, + tracking_paused: trackingPaused > 0, + mailbox_sync_configured: configured > 0, mailbox_sync_status: merge( syncs.map((row) => ({ key: row.status, count: row._count._all })), diff --git a/apps/api/src/tracking/tracking-config.service.ts b/apps/api/src/tracking/tracking-config.service.ts new file mode 100644 index 00000000..b6db8efb --- /dev/null +++ b/apps/api/src/tracking/tracking-config.service.ts @@ -0,0 +1,133 @@ +import type { Db } from "@crm/db"; +import { SETTINGS_ID } from "@crm/db/settings"; +import { + configHash, + mintSiteId, + readTrackingConfig, + type TrackingConfig, +} from "@crm/db/tracking"; +import { CACHE_MANAGER } from "@nestjs/cache-manager"; +import { Inject, Injectable, Logger } from "@nestjs/common"; +import type { Cache } from "cache-manager"; +import { InjectDatabase } from "../database/database.constants"; + +const CONFIG_TTL_MS = 5 * 60_000; + +const CONFIG_KEY = "tracking:config"; + +export interface CompiledConfig { + config: TrackingConfig; + hash: string; +} + +@Injectable() +export class TrackingConfigService { + private readonly logger = new Logger(TrackingConfigService.name); + + private generation = 0; + + constructor( + @InjectDatabase() private readonly db: Db, + @Inject(CACHE_MANAGER) private readonly cache: Cache, + ) {} + + async compiled(): Promise { + const cached = await this.cache.get(CONFIG_KEY); + if (cached) return cached; + + const read = this.generation; + const config = await readTrackingConfig(this.db); + if (!config) return null; + + const compiled = { config, hash: configHash(config) }; + + if (read === this.generation && (await this.current(compiled.hash))) { + await this.cache.set(CONFIG_KEY, compiled, CONFIG_TTL_MS); + } + + return compiled; + } + + private async current(hash: string): Promise { + const row = await this.db.appSetting.findUnique({ + where: { id: SETTINGS_ID }, + select: { trackingConfigHash: true }, + }); + + return row?.trackingConfigHash === hash; + } + + async forSite(siteId: string): Promise { + const compiled = await this.compiled(); + return compiled?.config.siteId === siteId ? compiled : null; + } + + async invalidate(): Promise { + this.generation += 1; + const written = this.generation; + + await this.cache.del(CONFIG_KEY); + + const config = await readTrackingConfig(this.db); + + if (!config) { + await this.db.appSetting.updateMany({ + where: { id: SETTINGS_ID }, + data: { trackingConfigHash: null }, + }); + + return; + } + + const hash = configHash(config); + + await this.db.appSetting.update({ + where: { id: SETTINGS_ID }, + data: { trackingConfigHash: hash }, + }); + + if (written !== this.generation) return; + if (!(await this.current(hash))) return; + + await this.cache.set(CONFIG_KEY, { config, hash }, CONFIG_TTL_MS); + } + + async ensureSiteId(): Promise { + const existing = await this.db.appSetting.findUnique({ + where: { id: SETTINGS_ID }, + select: { trackingSiteId: true }, + }); + + if (existing?.trackingSiteId) return existing.trackingSiteId; + + const trackingSiteId = mintSiteId(); + + await this.db.appSetting.upsert({ + where: { id: SETTINGS_ID }, + create: { id: SETTINGS_ID, trackingSiteId }, + update: { trackingSiteId }, + }); + + await this.invalidate(); + + this.logger.log({ message: "Tracking site id minted" }); + + return trackingSiteId; + } + + async rotateSiteId(): Promise { + const trackingSiteId = mintSiteId(); + + await this.db.appSetting.upsert({ + where: { id: SETTINGS_ID }, + create: { id: SETTINGS_ID, trackingSiteId }, + update: { trackingSiteId }, + }); + + await this.invalidate(); + + this.logger.warn({ message: "Tracking site id rotated" }); + + return trackingSiteId; + } +} diff --git a/apps/api/src/tracking/tracking-counter.service.ts b/apps/api/src/tracking/tracking-counter.service.ts new file mode 100644 index 00000000..95006936 --- /dev/null +++ b/apps/api/src/tracking/tracking-counter.service.ts @@ -0,0 +1,59 @@ +import type { Db } from "@crm/db"; +import { windowExpiry } from "@crm/db/tracking"; +import { Injectable, Logger } from "@nestjs/common"; +import { InjectDatabase } from "../database/database.constants"; + +@Injectable() +export class TrackingCounterService { + private readonly logger = new Logger(TrackingCounterService.name); + + constructor(@InjectDatabase() private readonly db: Db) {} + + async take(key: string, limit: number, amount = 1): Promise { + if (amount <= 0) return true; + if (amount > limit) return false; + + try { + const charged = await this.db.$queryRaw<{ value: number }[]>` + INSERT INTO "trackingCounter" ("key", "value", "expiresAt") + VALUES (${key}, ${amount}, ${windowExpiry(key)}) + ON CONFLICT ("key") DO UPDATE + SET "value" = "trackingCounter"."value" + ${amount} + WHERE "trackingCounter"."value" + ${amount} <= ${limit} + RETURNING "value"; + `; + + return charged.length > 0; + } catch (error) { + this.logger.error( + { message: "Tracking counter could not be read — refusing the write" }, + error instanceof Error ? error.stack : String(error), + ); + + return false; + } + } + + async release(key: string, amount = 1): Promise { + try { + await this.db.$executeRaw` + UPDATE "trackingCounter" + SET "value" = GREATEST("value" - ${amount}, 0) + WHERE "key" = ${key}; + `; + } catch (error) { + this.logger.error( + { message: "Tracking counter could not be released" }, + error instanceof Error ? error.stack : String(error), + ); + } + } + + async sweep(): Promise { + const removed = await this.db.trackingCounter.deleteMany({ + where: { expiresAt: { lt: new Date() } }, + }); + + return removed.count; + } +} diff --git a/apps/api/src/tracking/tracking-filing.service.ts b/apps/api/src/tracking/tracking-filing.service.ts new file mode 100644 index 00000000..ceca0b12 --- /dev/null +++ b/apps/api/src/tracking/tracking-filing.service.ts @@ -0,0 +1,261 @@ +import { workspaceDomains } from "@crm/auth"; +import { ActivityType, type Db, Prisma, RecordSource } from "@crm/db"; +import type { Touch } from "@crm/db/attribution"; +import { + CONTACT_CAP_REASON, + CONTACTS_PER_HOUR, + contactWindowKey, +} from "@crm/db/tracking"; +import { Injectable, Logger } from "@nestjs/common"; +import { AgentTriggerService } from "../agent/agent-trigger.service"; +import { CompanyDirectoryService } from "../companies/company-directory.service"; +import { isMachineDomain } from "../companies/domain"; +import { ActivityStampService } from "../crm/activity-stamp.service"; +import { normalizeEmail } from "../crm/values"; +import { InjectDatabase } from "../database/database.constants"; +import { + isAutomatedAddress, + isMachineAddress, + splitName, +} from "../mailbox/participants"; +import { TrackingCounterService } from "./tracking-counter.service"; + +function columns( + touch: Touch | undefined, + prefix: "first" | "last", +): Record { + if (!touch) return {}; + + return { + [`${prefix}Source`]: touch.source, + [`${prefix}Medium`]: touch.medium, + [`${prefix}Campaign`]: touch.campaign, + [`${prefix}Term`]: touch.term, + [`${prefix}Content`]: touch.content, + [`${prefix}Referrer`]: touch.referrer, + [`${prefix}Landing`]: touch.landing, + [`${prefix}TouchAt`]: touch.at, + }; +} + +export type FilingOutcome = + | { filed: true; contactId: string } + | { filed: false; reason: string }; + +@Injectable() +export class TrackingFilingService { + private readonly logger = new Logger(TrackingFilingService.name); + + constructor( + @InjectDatabase() private readonly db: Db, + private readonly counters: TrackingCounterService, + private readonly companies: CompanyDirectoryService, + private readonly agent: AgentTriggerService, + private readonly stamp: ActivityStampService, + ) {} + + async file(submission: { + id: string; + email: string | null; + host: string; + visitorId: string | null; + name: string | null; + firstTouch?: Touch; + lastTouch?: Touch; + }): Promise { + const email = normalizeEmail(submission.email ?? ""); + if (!email) return this.skip(submission.id, "No email address"); + + if (isMachineAddress(email) || isAutomatedAddress(email)) { + return this.skip(submission.id, "Not an address a human reads"); + } + + const domain = email.split("@")[1] ?? null; + if (!domain) return this.skip(submission.id, "No usable domain"); + + if (isMachineDomain(domain)) { + return this.skip(submission.id, "Not a domain a human reads"); + } + + if (workspaceDomains().includes(domain)) { + return this.skip(submission.id, "One of our own addresses"); + } + + const suppressed = await this.suppressed(email, domain); + if (suppressed) return this.skip(submission.id, suppressed); + + const existing = await this.db.contact.findUnique({ + where: { email }, + select: { id: true }, + }); + + if (existing) { + await this.attach(submission.id, existing.id, submission); + return { filed: true, contactId: existing.id }; + } + + const window = contactWindowKey(); + + if (!(await this.counters.take(window, CONTACTS_PER_HOUR))) { + return this.skip(submission.id, CONTACT_CAP_REASON); + } + + const companyId = await this.companies.companyForEmail(email); + const { firstName, lastName } = splitName(submission.name, email); + + let contact: { id: string }; + try { + contact = await this.db.contact.create({ + data: { + firstName, + lastName, + email, + companyId, + source: RecordSource.TRACKING, + lastActivityAt: new Date(), + }, + select: { id: true }, + }); + } catch (error) { + await this.counters.release(window); + + const raced = await this.raced(error, email); + if (!raced) throw error; + + await this.attach(submission.id, raced.id, submission); + + return { filed: true, contactId: raced.id }; + } + + await this.attach(submission.id, contact.id, submission); + + await this.agent.contactCreated( + contact.id, + `Submitted a form on ${submission.host}`, + ); + + this.logger.log({ + message: "Contact filed from a form submission", + contactId: contact.id, + host: submission.host, + }); + + return { filed: true, contactId: contact.id }; + } + + private async raced( + error: unknown, + email: string, + ): Promise<{ id: string } | null> { + if ( + !(error instanceof Prisma.PrismaClientKnownRequestError) || + error.code !== "P2002" + ) { + return null; + } + + return this.db.contact.findUnique({ + where: { email }, + select: { id: true }, + }); + } + + private async attach( + submissionId: string, + contactId: string, + context: { + visitorId: string | null; + firstTouch?: Touch; + lastTouch?: Touch; + }, + ): Promise { + const { visitorId } = context; + const claimed = await this.db.formSubmission.updateMany({ + where: { id: submissionId, filedAt: null }, + data: { contactId, filedAt: new Date(), skipReason: null }, + }); + + if (claimed.count === 0) return; + + const submission = await this.db.formSubmission.findUniqueOrThrow({ + where: { id: submissionId }, + select: { host: true, path: true }, + }); + + const author = await this.author(contactId); + + if (author) { + const activity = await this.db.activity.create({ + data: { + type: ActivityType.NOTE, + subject: `Submitted a form on ${submission.host}`, + body: `${submission.host}${submission.path}`, + contactId, + occurredAt: new Date(), + createdById: author, + meta: { automated: true, source: "tracking" }, + }, + select: { createdAt: true }, + }); + + await this.stamp.touch({ contactId }, activity.createdAt); + } + + if (!visitorId) return; + + const first = columns(context.firstTouch, "first"); + const last = columns(context.lastTouch, "last"); + + await this.db.trackedVisitor.upsert({ + where: { id: visitorId }, + create: { id: visitorId, contactId, ...first, ...last }, + update: { contactId, ...last }, + }); + } + + private async author(contactId: string): Promise { + const contact = await this.db.contact.findUnique({ + where: { id: contactId }, + select: { ownerId: true }, + }); + + if (contact?.ownerId) return contact.ownerId; + + const anyUser = await this.db.user.findFirst({ select: { id: true } }); + + return anyUser?.id ?? null; + } + + private async skip( + submissionId: string, + reason: string, + ): Promise { + await this.db.formSubmission.update({ + where: { id: submissionId }, + data: { skipReason: reason }, + }); + + return { filed: false, reason }; + } + + private async suppressed( + email: string, + domain: string, + ): Promise { + const [contact, host] = await Promise.all([ + this.db.suppressedContact.findFirst({ + where: { email: { equals: email, mode: "insensitive" } }, + select: { email: true }, + }), + this.db.suppressedDomain.findUnique({ + where: { domain }, + select: { domain: true }, + }), + ]); + + if (contact) return "This address was deleted by a rep"; + if (host) return "This domain is suppressed"; + + return null; + } +} diff --git a/apps/api/src/tracking/tracking-ingest.service.ts b/apps/api/src/tracking/tracking-ingest.service.ts new file mode 100644 index 00000000..bffd39ea --- /dev/null +++ b/apps/api/src/tracking/tracking-ingest.service.ts @@ -0,0 +1,349 @@ +import type { Db } from "@crm/db"; +import { classifyTouch, type RawTouch, type Touch } from "@crm/db/attribution"; +import { + dedupeKey, + EVENTS_PER_MINUTE, + hostAllowed, + MAX_EVENTS_PER_BATCH, + matchedHost, + normalizePath, + originAllowed, + rateWindowKey, + stripQuery, + type TrackingConfig, +} from "@crm/db/tracking"; +import { Injectable, Logger } from "@nestjs/common"; +import { normalizeEmail } from "../crm/values"; +import { InjectDatabase } from "../database/database.constants"; +import { TrackingConfigService } from "./tracking-config.service"; +import { TrackingCounterService } from "./tracking-counter.service"; +import { TrackingFilingService } from "./tracking-filing.service"; + +const MAX_LABEL = 80; + +const MAX_PATH = 512; + +const MAX_HOST = 253; + +const KEPT = new Set(["page_view", "click", "form_submit"]); + +const BOT = /bot|crawler|spider|crawling|headlesschrome|lighthouse|preview/i; + +const SENSITIVE = /pass|secret|token|card|cvv|cvc|ssn|iban|routing/i; + +const CARD = /^[0-9 -]{12,25}$/; + +const ADDRESS = /^[^\s@]+@[^\s@.]+\.[^\s@]+$/; + +export interface IncomingEvent { + type: string; + host: string; + path: string; + referrer?: string; + label?: string; + at?: number; + fields?: Record; + touch?: RawTouch; + firstTouch?: RawTouch; +} + +export interface IncomingBatch { + siteId: string; + visitorId: string; + events: IncomingEvent[]; +} + +interface AcceptedEvent { + event: IncomingEvent; + host: string; +} + +@Injectable() +export class TrackingIngestService { + private readonly logger = new Logger(TrackingIngestService.name); + + constructor( + @InjectDatabase() private readonly db: Db, + private readonly config: TrackingConfigService, + private readonly counters: TrackingCounterService, + private readonly filing: TrackingFilingService, + ) {} + + async accept( + batch: IncomingBatch, + request: { origin: string | null; userAgent: string | null }, + ): Promise { + if (request.userAgent && BOT.test(request.userAgent)) return; + + const compiled = await this.config.forSite(batch.siteId); + if (!compiled) return; + + if (!originAllowed(request.origin, compiled.config)) return; + + const visitorId = sanitizeId(batch.visitorId); + if (!visitorId) return; + + const events = batch.events.slice(0, MAX_EVENTS_PER_BATCH); + if (scripted(events)) return; + + const accepted = events.flatMap((event) => { + if (!KEPT.has(event.type)) return []; + + const host = event.host?.toLowerCase().trim().slice(0, MAX_HOST); + + return host && hostAllowed(host, compiled.config) + ? [{ event, host }] + : []; + }); + + if (accepted.length === 0) return; + if (!(await this.withinRate(accepted.length))) return; + + const pageViews = accepted.filter( + ({ event }) => event.type === "page_view" || event.type === "click", + ); + const forms = accepted.filter(({ event }) => event.type === "form_submit"); + + if (pageViews.length > 0) { + await this.events(visitorId, pageViews, compiled.config); + } + + for (const form of forms) { + await this.submission(visitorId, form); + } + } + + private async events( + visitorId: string, + accepted: AcceptedEvent[], + config: TrackingConfig, + ): Promise { + const rows = accepted.map(({ event, host }) => { + const touch = + event.type === "page_view" && event.touch + ? classifyTouch(arriving(event.touch)) + : null; + + const referrer = stripQuery(event.referrer); + + return { + visitorId, + type: event.type, + host, + path: trim(normalizePath(event.path), MAX_PATH), + referrer: referrer ? trim(referrer, MAX_PATH) : null, + label: event.label ? trim(event.label, MAX_LABEL) : null, + source: touch?.source ?? null, + medium: touch?.medium ?? null, + campaign: touch?.campaign ?? null, + occurredAt: occurredAt(event.at), + }; + }); + + await this.db.trackedEvent.createMany({ data: rows }); + + await this.countViews(rows, config); + } + + private async countViews( + rows: { host: string; type: string }[], + config: TrackingConfig, + ): Promise { + const tallies = new Map(); + + for (const row of rows) { + const entry = matchedHost(row.host, config); + if (!entry) continue; + + const views = tallies.get(entry.host) ?? 0; + tallies.set(entry.host, views + (row.type === "page_view" ? 1 : 0)); + } + + const lastSeenAt = new Date(); + + await Promise.all( + [...tallies].map(([host, views]) => + this.db.trackedDomain.updateMany({ + where: { host }, + data: { pageViews: { increment: views }, lastSeenAt }, + }), + ), + ); + } + + private async submission( + visitorId: string, + { event, host }: AcceptedEvent, + ): Promise { + const fields = clean(event.fields ?? {}); + const email = emailFrom(fields); + const path = trim(normalizePath(event.path), MAX_PATH); + const at = occurredAt(event.at); + + const lastTouch = classifyTouch(arriving(event.touch ?? {}), at); + const firstTouch = event.firstTouch + ? classifyTouch(arriving(event.firstTouch), at) + : lastTouch; + + const key = dedupeKey({ host, path, email, at }); + + const created = await this.db.formSubmission.createMany({ + data: [ + { + visitorId, + host, + path, + email, + fields, + firstTouch: stored(firstTouch), + lastTouch: stored(lastTouch), + dedupeKey: key, + }, + ], + skipDuplicates: true, + }); + + const submission = await this.db.formSubmission.findUnique({ + where: { dedupeKey: key }, + select: { id: true, filedAt: true, skipReason: true }, + }); + + if (!submission) return; + if (created.count === 0 && !unfiled(submission)) return; + + const outcome = await this.filing.file({ + id: submission.id, + email, + host, + visitorId, + name: nameFrom(fields), + firstTouch, + lastTouch, + }); + + if (!outcome.filed) { + this.logger.log({ + message: "Form submission stored but not filed", + host, + reason: outcome.reason, + }); + } + } + + private async withinRate(events: number): Promise { + return this.counters.take(rateWindowKey(), EVENTS_PER_MINUTE, events); + } +} + +function unfiled(submission: { + filedAt: Date | null; + skipReason: string | null; +}): boolean { + return submission.filedAt === null && submission.skipReason === null; +} + +function arriving(touch: RawTouch): RawTouch { + return { + ...touch, + referrer: stripQuery(touch.referrer) ?? undefined, + landing: stripQuery(touch.landing) ?? undefined, + }; +} + +function stored(touch: Touch): Record { + return { + source: touch.source, + medium: touch.medium, + campaign: touch.campaign, + term: touch.term, + content: touch.content, + referrer: touch.referrer, + landing: touch.landing, + at: touch.at.toISOString(), + }; +} + +function scripted(events: IncomingEvent[]): boolean { + if (events.length < 3) return false; + + const stamps = events.flatMap((event) => + typeof event.at === "number" && Number.isFinite(event.at) ? [event.at] : [], + ); + + if (stamps.length !== events.length) return false; + + return new Set(stamps).size === 1; +} + +function occurredAt(at: number | undefined): Date { + const now = Date.now(); + if (typeof at !== "number" || !Number.isFinite(at)) return new Date(now); + + const bounded = Math.min(Math.max(at, now - 86_400_000), now); + + return new Date(bounded); +} + +function trim(value: string, max: number): string { + return value.length > max ? value.slice(0, max) : value; +} + +function sanitizeId(value: string | undefined): string | null { + if (typeof value !== "string") return null; + + const trimmed = value.trim(); + + return /^[a-zA-Z0-9_-]{8,64}$/.test(trimmed) ? trimmed : null; +} + +function clean(fields: Record): Record { + const kept: Record = {}; + + for (const [key, value] of Object.entries(fields).slice(0, 40)) { + if (typeof value !== "string") continue; + if (SENSITIVE.test(key)) continue; + if (CARD.test(value.trim())) continue; + + kept[trim(key, 64)] = trim(value, 512); + } + + return kept; +} + +function emailFrom(fields: Record): string | null { + for (const [key, value] of Object.entries(fields)) { + if (!/mail/i.test(key)) continue; + const email = address(value); + if (email) return email; + } + + for (const value of Object.values(fields)) { + const email = address(value); + if (email) return email; + } + + return null; +} + +function address(value: string): string | null { + const email = normalizeEmail(value); + + return email && ADDRESS.test(email) ? email : null; +} + +function nameFrom(fields: Record): string | null { + const first = pick(fields, /^(first[\s_-]?name|fname|given)/i); + const last = pick(fields, /^(last[\s_-]?name|lname|surname|family)/i); + + if (first) return last ? `${first} ${last}` : first; + + return pick(fields, /^(full[\s_-]?name|name)$/i) ?? pick(fields, /name/i); +} + +function pick(fields: Record, pattern: RegExp): string | null { + for (const [key, value] of Object.entries(fields)) { + if (pattern.test(key) && value.trim()) return value.trim(); + } + + return null; +} diff --git a/apps/api/src/tracking/tracking-rollup.service.ts b/apps/api/src/tracking/tracking-rollup.service.ts new file mode 100644 index 00000000..c30e2413 --- /dev/null +++ b/apps/api/src/tracking/tracking-rollup.service.ts @@ -0,0 +1,28 @@ +import type { Db } from "@crm/db"; +import { Injectable } from "@nestjs/common"; +import { InjectDatabase } from "../database/database.constants"; + +@Injectable() +export class TrackingRollupService { + constructor(@InjectDatabase() private readonly db: Db) {} + + async run(before: Date): Promise { + const rolled = await this.db.$executeRaw` + INSERT INTO "trackedPageDaily" ("day", "host", "path", "views", "visitors") + SELECT + date_trunc('day', "occurredAt") AS "day", + "host", + "path", + count(*)::int AS "views", + count(DISTINCT "visitorId")::int AS "visitors" + FROM "trackedEvent" + WHERE "occurredAt" < ${before} AND "type" = 'page_view' + GROUP BY 1, 2, 3 + ON CONFLICT ("day", "host", "path") DO UPDATE + SET "views" = GREATEST("trackedPageDaily"."views", EXCLUDED."views"), + "visitors" = GREATEST("trackedPageDaily"."visitors", EXCLUDED."visitors"); + `; + + return rolled; + } +} diff --git a/apps/api/src/tracking/tracking.contracts.ts b/apps/api/src/tracking/tracking.contracts.ts new file mode 100644 index 00000000..40b4b91e --- /dev/null +++ b/apps/api/src/tracking/tracking.contracts.ts @@ -0,0 +1,38 @@ +import { z } from "zod"; + +export const trackingFlagInput = z.object({ + flag: z.enum([ + "crossDomain", + "limitToDomains", + "cookieSubdomains", + "secureCookies", + "honourDnt", + "paused", + ]), + enabled: z.boolean(), +}); + +export const cookieLifetimeInput = z.object({ + days: z.number().int().min(0).max(400), +}); + +export const addDomainInput = z.object({ + host: z.string().min(1).max(253), + scope: z.enum(["SITE_AND_SUBDOMAINS", "EXACT_HOST"]).default("EXACT_HOST"), +}); + +export const removeDomainInput = z.object({ + id: z.string().min(1), +}); + +export const verifyInput = z.object({ + url: z.string().min(1).max(2048), +}); + +export const companyActivityInput = z.object({ + companyId: z.string().min(1), +}); + +export const contactActivityInput = z.object({ + contactId: z.string().min(1), +}); diff --git a/apps/api/src/tracking/tracking.controller.ts b/apps/api/src/tracking/tracking.controller.ts new file mode 100644 index 00000000..70a3db88 --- /dev/null +++ b/apps/api/src/tracking/tracking.controller.ts @@ -0,0 +1,251 @@ +import type { IncomingMessage } from "node:http"; +import type { Db } from "@crm/db"; +import { + EVENT_RETENTION_DAYS, + isSiteId, + MAX_BODY_BYTES, +} from "@crm/db/tracking"; +import { + Controller, + ForbiddenException, + Get, + Headers, + HttpCode, + Logger, + Param, + Post, + Req, + ServiceUnavailableException, +} from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { AllowAnonymous } from "@thallesp/nestjs-better-auth"; +import type { EnvironmentVariables } from "../config/env.validation"; +import { InjectDatabase } from "../database/database.constants"; +import { TrackingConfigService } from "./tracking-config.service"; +import { TrackingCounterService } from "./tracking-counter.service"; +import { + type IncomingBatch, + TrackingIngestService, +} from "./tracking-ingest.service"; +import { TrackingRollupService } from "./tracking-rollup.service"; + +const SWEEP_BATCH = 10_000; + +const MAX_SWEEP_PASSES = 50; + +@Controller("api/t") +export class TrackingController { + private readonly logger = new Logger(TrackingController.name); + + constructor( + private readonly config: TrackingConfigService, + private readonly ingest: TrackingIngestService, + ) {} + + @Get("config/:siteId") + @AllowAnonymous() + async publicConfig(@Param("siteId") siteId: string) { + if (!isSiteId(siteId)) return { config: null }; + + const compiled = await this.config.forSite(siteId); + + return compiled + ? { config: compiled.config, hash: compiled.hash } + : { config: null }; + } + + @Post("e") + @AllowAnonymous() + @HttpCode(204) + async collect( + @Req() request: IncomingMessage, + @Headers("origin") origin?: string, + @Headers("user-agent") userAgent?: string, + ): Promise { + const raw = await read(request, MAX_BODY_BYTES); + if (!raw) return; + + let batch: IncomingBatch; + try { + batch = JSON.parse(raw) as IncomingBatch; + } catch { + return; + } + + if (!isSiteId(batch?.siteId) || !Array.isArray(batch?.events)) return; + + try { + await this.ingest.accept(batch, { + origin: origin ?? null, + userAgent: userAgent ?? null, + }); + } catch (error) { + this.logger.error( + { message: "Tracking event was not stored" }, + error instanceof Error ? error.stack : String(error), + ); + } + } +} + +@Controller("internal/tracking") +export class TrackingRetentionController { + private readonly logger = new Logger(TrackingRetentionController.name); + private readonly secret: string | undefined; + + constructor( + @InjectDatabase() private readonly db: Db, + private readonly rollups: TrackingRollupService, + private readonly counters: TrackingCounterService, + config: ConfigService, + ) { + this.secret = config.get("CRON_SECRET", { infer: true }); + } + + @Get("retention") + @AllowAnonymous() + async viaGet(@Headers("authorization") authorization?: string) { + return this.run(authorization); + } + + @Post("retention") + @AllowAnonymous() + async viaPost(@Headers("authorization") authorization?: string) { + return this.run(authorization); + } + + private async run(authorization?: string) { + if (!this.secret) { + this.logger.error({ + message: "CRON_SECRET is not set — refusing to run tracking retention.", + }); + throw new ServiceUnavailableException("Retention is not configured."); + } + + if (!timingSafeEquals(authorization ?? "", `Bearer ${this.secret}`)) { + throw new ForbiddenException(); + } + + const before = startOfDay( + new Date(Date.now() - EVENT_RETENTION_DAYS * 24 * 60 * 60_000), + ); + + const rolled = await this.rollups.run(before); + const { removed, complete } = await this.sweepEvents(before); + const visitors = await this.sweepVisitors(before); + const counters = await this.counters.sweep(); + + if (!complete) { + this.logger.warn({ + message: + "Tracking retention hit its pass limit — events older than the window remain", + removed, + retentionDays: EVENT_RETENTION_DAYS, + }); + } + + this.logger.log({ + message: "Tracking retention swept", + rolled, + removed, + complete, + visitors, + counters, + retentionDays: EVENT_RETENTION_DAYS, + }); + + return { rolled, removed, complete, visitors, counters }; + } + + private async sweepEvents( + before: Date, + ): Promise<{ removed: number; complete: boolean }> { + let removed = 0; + + for (let pass = 0; pass < MAX_SWEEP_PASSES; pass += 1) { + const deleted = await this.db.$executeRaw` + DELETE FROM "trackedEvent" + WHERE "id" IN ( + SELECT "id" FROM "trackedEvent" + WHERE "occurredAt" < ${before} + LIMIT ${SWEEP_BATCH} + ); + `; + + removed += deleted; + if (deleted < SWEEP_BATCH) return { removed, complete: true }; + } + + return { removed, complete: false }; + } + + private async sweepVisitors(before: Date): Promise { + const orphaned = await this.db.$executeRaw` + DELETE FROM "trackedVisitor" + WHERE "contactId" IS NULL + AND "lastSeen" < ${before} + AND NOT EXISTS ( + SELECT 1 FROM "trackedEvent" + WHERE "trackedEvent"."visitorId" = "trackedVisitor"."id" + ); + `; + + return orphaned; + } +} + +async function read( + request: IncomingMessage, + limit: number, +): Promise { + const existing = (request as { body?: unknown }).body; + if (typeof existing === "string") { + return existing.length > limit ? null : existing; + } + if (existing && typeof existing === "object") { + return JSON.stringify(existing); + } + + return new Promise((resolve) => { + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + + const finish = (value: string | null) => { + if (settled) return; + settled = true; + resolve(value); + }; + + request.on("data", (chunk: Buffer) => { + size += chunk.length; + if (size > limit) { + request.destroy(); + finish(null); + return; + } + chunks.push(chunk); + }); + + request.on("end", () => finish(Buffer.concat(chunks).toString("utf8"))); + request.on("error", () => finish(null)); + }); +} + +function startOfDay(at: Date): Date { + const day = new Date(at); + day.setUTCHours(0, 0, 0, 0); + + return day; +} + +function timingSafeEquals(a: string, b: string): boolean { + if (a.length !== b.length) return false; + + let mismatch = 0; + for (let index = 0; index < a.length; index += 1) { + mismatch |= a.charCodeAt(index) ^ b.charCodeAt(index); + } + + return mismatch === 0; +} diff --git a/apps/api/src/tracking/tracking.module.ts b/apps/api/src/tracking/tracking.module.ts new file mode 100644 index 00000000..d462dab5 --- /dev/null +++ b/apps/api/src/tracking/tracking.module.ts @@ -0,0 +1,31 @@ +import { Module } from "@nestjs/common"; +import { AgentModule } from "../agent/agent.module"; +import { CompaniesModule } from "../companies/companies.module"; +import { TrpcModule } from "../trpc/trpc.module"; +import { + TrackingController, + TrackingRetentionController, +} from "./tracking.controller"; +import { TrackingRouter } from "./tracking.router"; +import { TrackingService } from "./tracking.service"; +import { TrackingConfigService } from "./tracking-config.service"; +import { TrackingCounterService } from "./tracking-counter.service"; +import { TrackingFilingService } from "./tracking-filing.service"; +import { TrackingIngestService } from "./tracking-ingest.service"; +import { TrackingRollupService } from "./tracking-rollup.service"; + +@Module({ + imports: [TrpcModule, AgentModule, CompaniesModule], + controllers: [TrackingController, TrackingRetentionController], + providers: [ + TrackingConfigService, + TrackingCounterService, + TrackingFilingService, + TrackingIngestService, + TrackingRollupService, + TrackingService, + TrackingRouter, + ], + exports: [TrackingConfigService], +}) +export class TrackingModule {} diff --git a/apps/api/src/tracking/tracking.router.ts b/apps/api/src/tracking/tracking.router.ts new file mode 100644 index 00000000..7b1b3853 --- /dev/null +++ b/apps/api/src/tracking/tracking.router.ts @@ -0,0 +1,95 @@ +import { Inject } from "@nestjs/common"; +import { + Ctx, + Input, + Mutation, + Query, + Router, + UseMiddlewares, +} from "nestjs-trpc"; +import type { z } from "zod"; +import type { AuthedTrpcContext } from "../trpc/context.types"; +import { AuthMiddleware } from "../trpc/middlewares/auth.middleware"; +import { + addDomainInput, + companyActivityInput, + contactActivityInput, + cookieLifetimeInput, + removeDomainInput, + trackingFlagInput, + verifyInput, +} from "./tracking.contracts"; +import { TrackingService } from "./tracking.service"; + +@Router({ alias: "tracking" }) +@UseMiddlewares(AuthMiddleware) +export class TrackingRouter { + constructor( + @Inject(TrackingService) private readonly tracking: TrackingService, + ) {} + + @Query() + async settings(@Ctx() ctx: AuthedTrpcContext) { + return this.tracking.settings(ctx.user.id); + } + + @Mutation({ input: trackingFlagInput }) + async setFlag( + @Ctx() ctx: AuthedTrpcContext, + @Input() input: z.infer, + ) { + return this.tracking.setFlag(ctx.user.id, input.flag, input.enabled); + } + + @Mutation({ input: cookieLifetimeInput }) + async setCookieLifetime( + @Ctx() ctx: AuthedTrpcContext, + @Input() input: z.infer, + ) { + return this.tracking.setCookieDays(ctx.user.id, input.days); + } + + @Mutation({ input: addDomainInput }) + async addDomain( + @Ctx() ctx: AuthedTrpcContext, + @Input() input: z.infer, + ) { + return this.tracking.addDomain(ctx.user.id, input); + } + + @Mutation({ input: removeDomainInput }) + async removeDomain( + @Ctx() ctx: AuthedTrpcContext, + @Input() input: z.infer, + ) { + return this.tracking.removeDomain(ctx.user.id, input.id); + } + + @Mutation() + async rotateSiteId(@Ctx() ctx: AuthedTrpcContext) { + return this.tracking.rotateSiteId(ctx.user.id); + } + + @Mutation({ input: verifyInput }) + async verify( + @Ctx() ctx: AuthedTrpcContext, + @Input() input: z.infer, + ) { + return this.tracking.verify(ctx.user.id, input.url); + } + + @Query() + async sources(@Ctx() ctx: AuthedTrpcContext) { + return this.tracking.sources(ctx.user.id); + } + + @Query({ input: companyActivityInput }) + async companyActivity(@Input() input: z.infer) { + return this.tracking.activityForCompany(input.companyId); + } + + @Query({ input: contactActivityInput }) + async contactActivity(@Input() input: z.infer) { + return this.tracking.activityForContact(input.contactId); + } +} diff --git a/apps/api/src/tracking/tracking.service.ts b/apps/api/src/tracking/tracking.service.ts new file mode 100644 index 00000000..9babc087 --- /dev/null +++ b/apps/api/src/tracking/tracking.service.ts @@ -0,0 +1,563 @@ +import { + appUrl, + canManageTracking, + isWorkspaceRole, + WORKSPACE_ID, + type WorkspaceRole, +} from "@crm/auth"; +import { type Db, DomainScope, Prisma } from "@crm/db"; +import { describeTouch } from "@crm/db/attribution"; +import { safeFetch } from "@crm/db/safe-fetch"; +import { SETTINGS_ID } from "@crm/db/settings"; +import { + COOKIE_LIFETIMES, + hostAllowed, + loaderUrl, + normalizeHost, + trackingReady, + trackingSnippet, + VERIFY_WINDOW_MS, +} from "@crm/db/tracking"; +import { + BadRequestException, + ForbiddenException, + Injectable, + NotFoundException, +} from "@nestjs/common"; +import { InjectDatabase } from "../database/database.constants"; +import { TrackingConfigService } from "./tracking-config.service"; + +export interface TrackedDomainRow { + id: string; + host: string; + scope: DomainScope; + pageViews: number; + lastSeenAt: string | null; +} + +export interface TrackingSettings { + siteId: string | null; + ready: boolean; + scriptUrl: string; + snippet: string | null; + crossDomain: boolean; + limitToDomains: boolean; + cookieSubdomains: boolean; + secureCookies: boolean; + honourDnt: boolean; + cookieDays: number; + paused: boolean; + cookieLifetimes: { days: number; label: string }[]; + domains: TrackedDomainRow[]; + receivingSince: string | null; + pageViews: number; + submissions: number; + canManage: boolean; +} + +export interface VisitedPage { + host: string; + path: string; + views: number; + lastSeenAt: string; +} + +export interface TouchSummary { + label: string; + source: string; + medium: string | null; + campaign: string | null; + landing: string | null; + referrer: string | null; + at: string | null; +} + +export interface WebsiteActivity { + identified: boolean; + visitors: number; + views: number; + lastSeenAt: string | null; + pages: VisitedPage[]; + firstTouch: TouchSummary | null; + lastTouch: TouchSummary | null; +} + +export interface SourceRow { + source: string; + medium: string | null; + views: number; + contacts: number; +} + +export type VerifyResult = + | { + status: "found"; + host: string; + responseMs: number; + allowed: boolean; + pageView: boolean; + } + | { status: "missing"; host: string; responseMs: number } + | { status: "unreachable"; host: string; detail: string }; + +@Injectable() +export class TrackingService { + constructor( + @InjectDatabase() private readonly db: Db, + private readonly config: TrackingConfigService, + ) {} + + async settings(userId: string): Promise { + const [row, domains, latest, pageViews, submissions] = await Promise.all([ + this.db.appSetting.findUnique({ + where: { id: SETTINGS_ID }, + select: { + trackingSiteId: true, + trackingCrossDomain: true, + trackingLimitToDomains: true, + trackingCookieSubdomains: true, + trackingSecureCookies: true, + trackingHonourDnt: true, + trackingCookieDays: true, + trackingPaused: true, + }, + }), + this.db.trackedDomain.findMany({ orderBy: { createdAt: "asc" } }), + this.db.trackedEvent.findFirst({ + orderBy: { occurredAt: "desc" }, + select: { occurredAt: true }, + }), + this.db.trackedEvent.count({ where: { type: "page_view" } }), + this.db.formSubmission.count(), + ]); + + const ready = trackingReady( + row?.trackingLimitToDomains ?? true, + domains.length, + ); + + const siteId = ready + ? await this.config.ensureSiteId() + : (row?.trackingSiteId ?? null); + + return { + siteId, + ready, + scriptUrl: scriptUrl(), + snippet: siteId ? snippet(siteId) : null, + crossDomain: row?.trackingCrossDomain ?? true, + limitToDomains: row?.trackingLimitToDomains ?? true, + cookieSubdomains: row?.trackingCookieSubdomains ?? false, + secureCookies: row?.trackingSecureCookies ?? true, + honourDnt: row?.trackingHonourDnt ?? true, + cookieDays: row?.trackingCookieDays ?? 395, + paused: row?.trackingPaused ?? false, + cookieLifetimes: [...COOKIE_LIFETIMES], + domains: domains.map((domain) => ({ + id: domain.id, + host: domain.host, + scope: domain.scope, + pageViews: domain.pageViews, + lastSeenAt: domain.lastSeenAt?.toISOString() ?? null, + })), + receivingSince: latest?.occurredAt.toISOString() ?? null, + pageViews, + submissions, + canManage: canManageTracking(await this.roleOf(userId)), + }; + } + + async setFlag( + userId: string, + flag: + | "crossDomain" + | "limitToDomains" + | "cookieSubdomains" + | "secureCookies" + | "honourDnt" + | "paused", + enabled: boolean, + ): Promise { + await this.assertCanManage(userId); + + const column = { + crossDomain: "trackingCrossDomain", + limitToDomains: "trackingLimitToDomains", + cookieSubdomains: "trackingCookieSubdomains", + secureCookies: "trackingSecureCookies", + honourDnt: "trackingHonourDnt", + paused: "trackingPaused", + }[flag]; + + await this.db.appSetting.upsert({ + where: { id: SETTINGS_ID }, + create: { id: SETTINGS_ID, [column]: enabled }, + update: { [column]: enabled }, + }); + + await this.config.invalidate(); + } + + async setCookieDays(userId: string, days: number): Promise { + await this.assertCanManage(userId); + + if (!COOKIE_LIFETIMES.some((entry) => entry.days === days)) { + throw new BadRequestException("That is not a cookie lifetime we offer."); + } + + await this.db.appSetting.upsert({ + where: { id: SETTINGS_ID }, + create: { id: SETTINGS_ID, trackingCookieDays: days }, + update: { trackingCookieDays: days }, + }); + + await this.config.invalidate(); + } + + async addDomain( + userId: string, + input: { host: string; scope: DomainScope }, + ): Promise { + await this.assertCanManage(userId); + + const host = normalizeHost(input.host); + if (!host) { + throw new BadRequestException( + "That is not a domain. Try something like acme.com.", + ); + } + + try { + const domain = await this.db.trackedDomain.create({ + data: { host, scope: input.scope }, + }); + + await this.config.invalidate(); + + return { + id: domain.id, + host: domain.host, + scope: domain.scope, + pageViews: domain.pageViews, + lastSeenAt: null, + }; + } catch (error) { + if ( + error instanceof Prisma.PrismaClientKnownRequestError && + error.code === "P2002" + ) { + throw new BadRequestException(`${host} is already on the list.`); + } + + throw error; + } + } + + async removeDomain(userId: string, id: string): Promise { + await this.assertCanManage(userId); + + try { + await this.db.trackedDomain.delete({ where: { id } }); + } catch (error) { + if ( + error instanceof Prisma.PrismaClientKnownRequestError && + error.code === "P2025" + ) { + throw new NotFoundException("That domain is already gone."); + } + + throw error; + } + + await this.config.invalidate(); + } + + async rotateSiteId(userId: string): Promise<{ siteId: string }> { + await this.assertCanManage(userId); + + return { siteId: await this.config.rotateSiteId() }; + } + + async verify(userId: string, url: string): Promise { + await this.assertCanManage(userId); + + const domains = await this.db.trackedDomain.count(); + const row = await this.db.appSetting.findUnique({ + where: { id: SETTINGS_ID }, + select: { trackingLimitToDomains: true }, + }); + + if (!trackingReady(row?.trackingLimitToDomains ?? true, domains)) { + throw new BadRequestException( + "Add the domain your website runs on first — there is no script to find yet.", + ); + } + + const target = absolute(url); + if (!target) { + throw new BadRequestException( + "That is not a URL. Try something like acme.com/pricing.", + ); + } + + const compiled = await this.config.compiled(); + const started = Date.now(); + const fetched = await safeFetch(target.toString(), { timeoutMs: 8_000 }); + const responseMs = Date.now() - started; + const host = (fetched?.url ?? target).hostname.toLowerCase(); + + if (!fetched?.response.ok) { + return { + status: "unreachable", + host, + detail: fetched + ? `The page answered ${fetched.response.status}.` + : "We could not reach that page.", + }; + } + + const body = (await fetched.response.text()).slice(0, 512_000); + const siteId = compiled?.config.siteId; + + if (!siteId || !mentions(body, siteId)) { + return { status: "missing", host, responseMs }; + } + + const since = new Date(Date.now() - VERIFY_WINDOW_MS); + const seen = await this.db.trackedEvent.findFirst({ + where: { host, occurredAt: { gte: since } }, + select: { id: true }, + }); + + return { + status: "found", + host, + responseMs, + allowed: compiled ? hostAllowed(host, compiled.config) : false, + pageView: seen !== null, + }; + } + + async activityForCompany(companyId: string): Promise { + const visitors = await this.db.trackedVisitor.findMany({ + where: { contact: { companyId } }, + select: { id: true }, + }); + + return this.activityFor(visitors.map((visitor) => visitor.id)); + } + + async activityForContact(contactId: string): Promise { + const visitors = await this.db.trackedVisitor.findMany({ + where: { contactId }, + select: { id: true }, + }); + + return this.activityFor(visitors.map((visitor) => visitor.id)); + } + + private async activityFor(visitorIds: string[]): Promise { + if (visitorIds.length === 0) { + return { + identified: false, + visitors: 0, + views: 0, + lastSeenAt: null, + pages: [], + firstTouch: null, + lastTouch: null, + }; + } + + const [first, last] = await Promise.all([ + this.db.trackedVisitor.findFirst({ + where: { id: { in: visitorIds }, firstSource: { not: null } }, + orderBy: { firstTouchAt: "asc" }, + }), + this.db.trackedVisitor.findFirst({ + where: { id: { in: visitorIds }, lastSource: { not: null } }, + orderBy: { lastTouchAt: "desc" }, + }), + ]); + + const [grouped, latest, views] = await Promise.all([ + this.db.trackedEvent.groupBy({ + by: ["host", "path"], + where: { visitorId: { in: visitorIds }, type: "page_view" }, + _count: { _all: true }, + _max: { occurredAt: true }, + orderBy: { _count: { path: "desc" } }, + take: 10, + }), + this.db.trackedEvent.findFirst({ + where: { visitorId: { in: visitorIds } }, + orderBy: { occurredAt: "desc" }, + select: { occurredAt: true }, + }), + this.db.trackedEvent.count({ + where: { visitorId: { in: visitorIds }, type: "page_view" }, + }), + ]); + + return { + identified: true, + visitors: visitorIds.length, + views, + lastSeenAt: latest?.occurredAt.toISOString() ?? null, + pages: grouped.flatMap((row) => + row._max.occurredAt + ? [ + { + host: row.host, + path: row.path, + views: row._count._all, + lastSeenAt: row._max.occurredAt.toISOString(), + }, + ] + : [], + ), + firstTouch: first + ? summarise({ + source: first.firstSource, + medium: first.firstMedium, + campaign: first.firstCampaign, + landing: first.firstLanding, + referrer: first.firstReferrer, + at: first.firstTouchAt, + }) + : null, + lastTouch: last + ? summarise({ + source: last.lastSource, + medium: last.lastMedium, + campaign: last.lastCampaign, + landing: last.lastLanding, + referrer: last.lastReferrer, + at: last.lastTouchAt, + }) + : null, + }; + } + + async sources(userId: string): Promise { + await this.assertCanManage(userId); + + const [views, contacts] = await Promise.all([ + this.db.trackedEvent.groupBy({ + by: ["source", "medium"], + where: { type: "page_view", source: { not: null } }, + _count: { _all: true }, + }), + this.db.$queryRaw< + { firstSource: string; firstMedium: string | null; contacts: bigint }[] + >` + SELECT "firstSource", "firstMedium", count(DISTINCT "contactId") AS contacts + FROM "trackedVisitor" + WHERE "contactId" IS NOT NULL AND "firstSource" IS NOT NULL + GROUP BY 1, 2; + `, + ]); + + const rows = new Map(); + + for (const row of views) { + if (!row.source) continue; + const key = `${row.source}|${row.medium ?? ""}`; + rows.set(key, { + source: row.source, + medium: row.medium, + views: row._count._all, + contacts: 0, + }); + } + + for (const row of contacts) { + if (!row.firstSource) continue; + const key = `${row.firstSource}|${row.firstMedium ?? ""}`; + const existing = rows.get(key); + const count = Number(row.contacts); + + if (existing) { + existing.contacts = count; + continue; + } + + rows.set(key, { + source: row.firstSource, + medium: row.firstMedium, + views: 0, + contacts: count, + }); + } + + return [...rows.values()] + .sort((a, b) => b.contacts - a.contacts || b.views - a.views) + .slice(0, 20); + } + + private async assertCanManage(userId: string): Promise { + if (!canManageTracking(await this.roleOf(userId))) { + throw new ForbiddenException( + "Only an owner or an admin can change tracking.", + ); + } + } + + private async roleOf(userId: string): Promise { + const member = await this.db.member.findFirst({ + where: { organizationId: WORKSPACE_ID, userId }, + select: { role: true }, + }); + + return member && isWorkspaceRole(member.role) ? member.role : null; + } +} + +function summarise(touch: { + source: string | null; + medium: string | null; + campaign: string | null; + landing: string | null; + referrer: string | null; + at: Date | null; +}): TouchSummary | null { + if (!touch.source) return null; + + return { + label: describeTouch(touch), + source: touch.source, + medium: touch.medium, + campaign: touch.campaign, + landing: touch.landing, + referrer: touch.referrer, + at: touch.at?.toISOString() ?? null, + }; +} + +function scriptUrl(): string { + return loaderUrl(appUrl); +} + +function snippet(siteId: string): string { + return trackingSnippet(appUrl, siteId); +} + +function absolute(input: string): URL | null { + const trimmed = input.trim(); + if (!trimmed) return null; + + const withScheme = /^https?:\/\//i.test(trimmed) + ? trimmed + : `https://${trimmed}`; + + try { + const url = new URL(withScheme); + return url.hostname.includes(".") ? url : null; + } catch { + return null; + } +} + +function mentions(body: string, siteId: string): boolean { + return body.includes(siteId) && /\/t\/crm\.js/.test(body); +} diff --git a/apps/api/test/tracking-filing.integration.spec.ts b/apps/api/test/tracking-filing.integration.spec.ts new file mode 100644 index 00000000..a154d7a3 --- /dev/null +++ b/apps/api/test/tracking-filing.integration.spec.ts @@ -0,0 +1,318 @@ +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, +} from "bun:test"; +import { db } from "@crm/db"; +import { + CONTACT_CAP_REASON, + CONTACTS_PER_HOUR, + contactWindowKey, +} from "@crm/db/tracking"; +import type { AgentTriggerService } from "../src/agent/agent-trigger.service"; +import { CompanyDirectoryService } from "../src/companies/company-directory.service"; +import { ActivityStampService } from "../src/crm/activity-stamp.service"; +import { TrackingCounterService } from "../src/tracking/tracking-counter.service"; +import { TrackingFilingService } from "../src/tracking/tracking-filing.service"; + +const suffix = process.env.TEST_RUN_ID ?? "filing-spec"; +const domain = `visitors-${suffix}.test`; +const host = `www.${domain}`; + +const queued: string[] = []; + +const agent = { + contactCreated: async (id: string) => { + queued.push(id); + }, + companyCreated: async () => undefined, + companyRequested: async () => undefined, +} as unknown as AgentTriggerService; + +const stamp = new ActivityStampService(db); +const counters = new TrackingCounterService(db); +const directory = new CompanyDirectoryService(db, agent); +const filing = new TrackingFilingService(db, counters, directory, agent, stamp); + +let userId: string; + +async function submit(email: string | null, name: string | null = "Dana Reed") { + const row = await db.formSubmission.create({ + data: { + host, + path: "/pricing", + email, + fields: name ? { name, email: email ?? "" } : {}, + dedupeKey: `${suffix}-${Math.random().toString(36).slice(2)}`, + }, + select: { id: true }, + }); + + const outcome = await filing.file({ + id: row.id, + email, + host, + visitorId: `visitor-${Math.random().toString(36).slice(2, 12)}`, + name, + }); + + const stored = await db.formSubmission.findUnique({ + where: { id: row.id }, + select: { contactId: true, filedAt: true, skipReason: true }, + }); + + return { outcome, stored }; +} + +async function clean() { + await db.activity.deleteMany({ where: { body: { contains: host } } }); + await db.formSubmission.deleteMany({ where: { host } }); + await db.trackedVisitor.deleteMany({ + where: { id: { startsWith: "visitor-" } }, + }); + await db.contact.deleteMany({ where: { email: { endsWith: `@${domain}` } } }); + await db.contact.deleteMany({ where: { email: `free-${suffix}@gmail.com` } }); + await db.company.deleteMany({ where: { domain } }); + await db.suppressedContact.deleteMany({ + where: { email: { endsWith: `@${domain}` } }, + }); + await db.suppressedDomain.deleteMany({ where: { domain } }); + await db.trackingCounter.deleteMany({ where: {} }); +} + +beforeAll(async () => { + await clean(); + + const user = await db.user.findFirst({ select: { id: true } }); + userId = + user?.id ?? + ( + await db.user.create({ + data: { + id: `user-${suffix}`, + name: "Test Rep", + email: `rep-${suffix}@example.test`, + }, + select: { id: true }, + }) + ).id; +}); + +beforeEach(async () => { + queued.length = 0; + await db.trackingCounter.deleteMany({ where: {} }); +}); + +afterAll(async () => { + await clean(); + if (userId.startsWith(`user-${suffix}`)) { + await db.user.deleteMany({ where: { id: userId } }); + } +}); + +describe("filing a form submission", () => { + it("refuses an address no human reads", async () => { + const { outcome, stored } = await submit(`noreply@${domain}`); + + expect(outcome.filed).toBe(false); + expect(stored?.contactId).toBeNull(); + expect(stored?.skipReason).toBeTruthy(); + }); + + it("refuses a submission with no address at all", async () => { + const { outcome, stored } = await submit(null); + + expect(outcome.filed).toBe(false); + expect(stored?.skipReason).toBe("No email address"); + }); + + it("refuses an address a rep has deleted", async () => { + const email = `deleted@${domain}`; + await db.suppressedContact.create({ data: { email } }); + + const { outcome, stored } = await submit(email); + + expect(outcome.filed).toBe(false); + expect(stored?.skipReason).toContain("deleted"); + }); + + it("refuses a suppressed domain", async () => { + await db.suppressedDomain.create({ data: { domain } }); + + const { outcome, stored } = await submit(`someone@${domain}`); + + expect(outcome.filed).toBe(false); + expect(stored?.skipReason).toContain("suppressed"); + + await db.suppressedDomain.deleteMany({ where: { domain } }); + }); + + it("files a free-mail address as a contact with no company", async () => { + const { outcome, stored } = await submit(`free-${suffix}@gmail.com`); + + expect(outcome.filed).toBe(true); + expect(stored?.filedAt).not.toBeNull(); + + const contact = await db.contact.findUnique({ + where: { email: `free-${suffix}@gmail.com` }, + select: { companyId: true, source: true }, + }); + + expect(contact?.companyId).toBeNull(); + expect(contact?.source).toBe("TRACKING"); + }); + + it("files a work address and queues the agent once", async () => { + const email = `dana@${domain}`; + const { outcome, stored } = await submit(email); + + expect(outcome.filed).toBe(true); + expect(stored?.contactId).toBeTruthy(); + expect(queued).toHaveLength(1); + + const contact = await db.contact.findUnique({ + where: { email }, + select: { companyId: true, firstName: true }, + }); + + expect(contact?.companyId).toBeTruthy(); + expect(contact?.firstName).toBe("Dana"); + }); + + it("writes one note when the same submission is filed twice at once", async () => { + const email = `raced@${domain}`; + const row = await db.formSubmission.create({ + data: { + host, + path: "/pricing", + email, + fields: { email }, + dedupeKey: `${suffix}-raced`, + }, + select: { id: true }, + }); + + const file = () => + filing.file({ + id: row.id, + email, + host, + visitorId: `visitor-raced-${suffix}`.slice(0, 64), + name: "Dana Reed", + }); + + const [first, second] = await Promise.all([file(), file()]); + + expect(first.filed).toBe(true); + expect(second.filed).toBe(true); + expect(await db.contact.count({ where: { email } })).toBe(1); + + const notes = await db.activity.count({ + where: { body: { contains: host }, contact: { email } }, + }); + + expect(notes).toBe(1); + }); + + it("gives the cap slot back to the delivery that lost the create", async () => { + const email = `refunded@${domain}`; + const rows = await Promise.all( + [1, 2].map((n) => + db.formSubmission.create({ + data: { + host, + path: "/pricing", + email, + fields: { email }, + dedupeKey: `${suffix}-refunded-${n}`, + }, + select: { id: true }, + }), + ), + ); + + await Promise.all( + rows.map((row) => + filing.file({ + id: row.id, + email, + host, + visitorId: null, + name: "Dana Reed", + }), + ), + ); + + expect(await db.contact.count({ where: { email } })).toBe(1); + + const counter = await db.trackingCounter.findUnique({ + where: { key: contactWindowKey() }, + select: { value: true }, + }); + + expect(counter?.value).toBe(1); + }); + + it("attaches a second submission to the contact already there", async () => { + const email = `repeat@${domain}`; + + const first = await submit(email); + queued.length = 0; + const second = await submit(email); + + expect(first.outcome.filed).toBe(true); + expect(second.outcome.filed).toBe(true); + expect(second.stored?.contactId).toBe(first.stored?.contactId ?? ""); + expect(queued).toHaveLength(0); + + const count = await db.contact.count({ where: { email } }); + expect(count).toBe(1); + }); + + it("stores but does not file once the hourly cap is reached", async () => { + await db.trackingCounter.upsert({ + where: { key: contactWindowKey() }, + create: { + key: contactWindowKey(), + value: CONTACTS_PER_HOUR, + expiresAt: new Date(Date.now() + 3_600_000), + }, + update: { value: CONTACTS_PER_HOUR }, + }); + + const { outcome, stored } = await submit(`capped@${domain}`); + + expect(outcome.filed).toBe(false); + expect(stored?.contactId).toBeNull(); + expect(stored?.skipReason).toBe(CONTACT_CAP_REASON); + }); +}); + +describe("the hourly counter", () => { + it("counts up and refuses past the limit", async () => { + const key = `contacts:test-${suffix}`; + + expect(await counters.take(key, 2)).toBe(true); + expect(await counters.take(key, 2)).toBe(true); + expect(await counters.take(key, 2)).toBe(false); + expect(await counters.take(key, 2)).toBe(false); + + await db.trackingCounter.deleteMany({ where: { key } }); + }); + + it("keeps a fixed window rather than sliding on every write", async () => { + const key = contactWindowKey(); + + await counters.take(key, 10); + const first = await db.trackingCounter.findUnique({ where: { key } }); + + await counters.take(key, 10); + const second = await db.trackingCounter.findUnique({ where: { key } }); + + expect(second?.expiresAt.getTime()).toBe(first?.expiresAt.getTime() ?? 0); + expect(second?.value).toBe(2); + }); +}); diff --git a/apps/api/test/tracking-ingest.integration.spec.ts b/apps/api/test/tracking-ingest.integration.spec.ts new file mode 100644 index 00000000..51062979 --- /dev/null +++ b/apps/api/test/tracking-ingest.integration.spec.ts @@ -0,0 +1,338 @@ +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, +} from "bun:test"; +import { db } from "@crm/db"; +import { EVENTS_PER_MINUTE, type TrackingConfig } from "@crm/db/tracking"; +import type { TrackingConfigService } from "../src/tracking/tracking-config.service"; +import { TrackingCounterService } from "../src/tracking/tracking-counter.service"; +import type { TrackingFilingService } from "../src/tracking/tracking-filing.service"; +import { + type IncomingEvent, + TrackingIngestService, +} from "../src/tracking/tracking-ingest.service"; + +const suffix = process.env.TEST_RUN_ID ?? "ingest-spec"; +const parent = `sites-${suffix}.test`; +const child = `docs.${parent}`; +const exact = `shop-${suffix}.test`; + +const SITE_ID = "cmp_1234abcd"; + +const config: TrackingConfig = { + siteId: SITE_ID, + crossDomain: true, + limitToDomains: true, + cookieSubdomains: false, + secureCookies: true, + honourDnt: true, + cookieDays: 395, + hosts: [ + { host: parent, scope: "SITE_AND_SUBDOMAINS" }, + { host: exact, scope: "EXACT_HOST" }, + ], +}; + +const configService = { + forSite: async (siteId: string) => + siteId === SITE_ID ? { config, hash: "0123456789ab" } : null, +} as unknown as TrackingConfigService; + +const filed: string[] = []; + +const filing = { + file: async ({ id }: { id: string }) => { + const claimed = await db.formSubmission.updateMany({ + where: { id, filedAt: null }, + data: { filedAt: new Date() }, + }); + + if (claimed.count === 0) { + return { filed: false as const, reason: "another delivery filed it" }; + } + + filed.push(id); + + return { filed: true as const, contactId: `contact-${id}` }; + }, +} as unknown as TrackingFilingService; + +const counters = new TrackingCounterService(db); +const ingest = new TrackingIngestService(db, configService, counters, filing); + +const REQUEST = { + origin: `https://${child}`, + userAgent: "Mozilla/5.0", +}; + +let visitor = 0; + +function visitorId(): string { + visitor += 1; + + return `ingest${suffix.replace(/[^a-zA-Z0-9]/g, "")}${visitor}`; +} + +async function accept(events: IncomingEvent[], id = visitorId()) { + await ingest.accept({ siteId: SITE_ID, visitorId: id, events }, REQUEST); + + return id; +} + +let tick = 0; + +function view(host: string, path = "/pricing"): IncomingEvent { + tick += 1; + + return { type: "page_view", host, path, at: Date.now() - tick }; +} + +async function clean() { + for (const host of [parent, child, exact]) { + await db.formSubmission.deleteMany({ where: { host } }); + await db.trackedEvent.deleteMany({ where: { host } }); + } + + await db.trackedDomain.deleteMany({ + where: { host: { in: [parent, exact] } }, + }); + await db.trackingCounter.deleteMany({ where: {} }); +} + +beforeAll(clean); + +beforeEach(async () => { + filed.length = 0; + await clean(); + await db.trackedDomain.createMany({ + data: [ + { host: parent, scope: "SITE_AND_SUBDOMAINS" }, + { host: exact, scope: "EXACT_HOST" }, + ], + }); +}); + +afterAll(clean); + +describe("counting page views against a domain", () => { + it("credits a subdomain's views to the parent that covers it", async () => { + await accept([view(child), view(child), view(parent)]); + + const row = await db.trackedDomain.findUnique({ + where: { host: parent }, + select: { pageViews: true }, + }); + + expect(row?.pageViews).toBe(3); + }); + + it("gives each domain only the views that were its own", async () => { + await accept([view(child), view(exact), view(exact)]); + + const rows = await db.trackedDomain.findMany({ + where: { host: { in: [parent, exact] } }, + select: { host: true, pageViews: true }, + orderBy: { host: "asc" }, + }); + + expect(rows.find((row) => row.host === parent)?.pageViews).toBe(1); + expect(rows.find((row) => row.host === exact)?.pageViews).toBe(2); + }); + + it("does not count a click as a page view", async () => { + await accept([ + view(parent), + { type: "click", host: parent, path: "/pricing", at: Date.now() }, + ]); + + const row = await db.trackedDomain.findUnique({ + where: { host: parent }, + select: { pageViews: true }, + }); + + expect(row?.pageViews).toBe(1); + }); +}); + +describe("the rate limit", () => { + it("charges every event in the batch, not one per request", async () => { + await accept([view(parent), view(parent), view(parent)]); + + const counter = await db.trackingCounter.findFirst({ + where: { key: { startsWith: "rate:" } }, + select: { value: true }, + }); + + expect(counter?.value).toBe(3); + }); + + it("stops writing once the window is spent", async () => { + await db.trackingCounter.deleteMany({ where: {} }); + + for (const key of spendableWindows()) { + await counters.take(key, EVENTS_PER_MINUTE, EVENTS_PER_MINUTE); + } + + await accept([view(parent)]); + + expect(await db.trackedEvent.count({ where: { host: parent } })).toBe(0); + }); + + it("does not spend the window on a batch it refuses", async () => { + await db.trackingCounter.deleteMany({ where: {} }); + + for (const key of spendableWindows()) { + await counters.take(key, EVENTS_PER_MINUTE, EVENTS_PER_MINUTE - 1); + } + + await accept([view(parent), view(parent)]); + expect(await db.trackedEvent.count({ where: { host: parent } })).toBe(0); + + await accept([view(parent)]); + expect(await db.trackedEvent.count({ where: { host: parent } })).toBe(1); + }); +}); + +describe("what a stored event keeps", () => { + it("never keeps the query string of a referrer", async () => { + await accept([ + { + type: "page_view", + host: parent, + path: "/pricing?plan=team", + referrer: "https://elsewhere.test/reset?token=secret#x", + at: Date.now(), + touch: { referrer: "https://elsewhere.test/reset?token=secret" }, + }, + ]); + + const row = await db.trackedEvent.findFirst({ + where: { host: parent }, + select: { path: true, referrer: true }, + }); + + expect(row?.path).toBe("/pricing"); + expect(row?.referrer).toBe("https://elsewhere.test/reset"); + }); + + it("never keeps it inside a submission's stored touch either", async () => { + await accept([ + { + type: "form_submit", + host: parent, + path: "/contact", + at: Date.now(), + fields: { email: `dana-${suffix}@acme.test` }, + touch: { referrer: "https://elsewhere.test/x?token=secret" }, + firstTouch: { referrer: "https://elsewhere.test/y?token=secret" }, + }, + ]); + + const row = await db.formSubmission.findFirst({ + where: { host: parent }, + select: { firstTouch: true, lastTouch: true }, + }); + + expect(JSON.stringify(row)).not.toContain("token=secret"); + }); +}); + +describe("a batch delivered twice", () => { + it("files the second delivery when the first left the row unfiled", async () => { + const at = Date.now(); + const submission: IncomingEvent = { + type: "form_submit", + host: parent, + path: "/contact", + at, + fields: { email: `twice-${suffix}@acme.test` }, + }; + + const id = visitorId(); + await accept([submission], id); + expect(filed).toHaveLength(1); + + await db.formSubmission.updateMany({ + where: { host: parent }, + data: { filedAt: null, contactId: null, skipReason: null }, + }); + + await accept([submission], id); + + expect(filed).toHaveLength(2); + expect(await db.formSubmission.count({ where: { host: parent } })).toBe(1); + }); + + it("files once when both deliveries arrive together", async () => { + const at = Date.now(); + const submission: IncomingEvent = { + type: "form_submit", + host: parent, + path: "/contact", + at, + fields: { email: `together-${suffix}@acme.test` }, + }; + + const id = visitorId(); + await Promise.all([accept([submission], id), accept([submission], id)]); + + expect(await db.formSubmission.count({ where: { host: parent } })).toBe(1); + expect(filed).toHaveLength(1); + }); + + it("leaves a submission alone once it has been filed", async () => { + const at = Date.now(); + const submission: IncomingEvent = { + type: "form_submit", + host: parent, + path: "/contact", + at, + fields: { email: `once-${suffix}@acme.test` }, + }; + + const id = visitorId(); + await accept([submission], id); + await db.formSubmission.updateMany({ + where: { host: parent }, + data: { filedAt: new Date() }, + }); + + await accept([submission], id); + + expect(filed).toHaveLength(1); + }); +}); + +describe("a batch that looks scripted", () => { + it("drops three events that share one timestamp", async () => { + const at = Date.now(); + + await accept([ + { type: "page_view", host: parent, path: "/a", at }, + { type: "page_view", host: parent, path: "/b", at }, + { type: "page_view", host: parent, path: "/c", at }, + ]); + + expect(await db.trackedEvent.count({ where: { host: parent } })).toBe(0); + }); + + it("keeps a batch whose events carry no timestamp at all", async () => { + await accept([ + { type: "page_view", host: parent, path: "/a" }, + { type: "page_view", host: parent, path: "/b" }, + { type: "page_view", host: parent, path: "/c" }, + ]); + + expect(await db.trackedEvent.count({ where: { host: parent } })).toBe(3); + }); +}); + +function spendableWindows(): string[] { + const bucket = Math.floor(Date.now() / 60_000); + + return [`rate:${bucket}`, `rate:${bucket + 1}`]; +} diff --git a/apps/api/vercel.json b/apps/api/vercel.json index b2cab4c1..28973399 100644 --- a/apps/api/vercel.json +++ b/apps/api/vercel.json @@ -12,6 +12,10 @@ { "path": "/internal/telemetry/rollup", "schedule": "0 7 * * *" + }, + { + "path": "/internal/tracking/retention", + "schedule": "0 4 * * *" } ] } diff --git a/apps/app/app/(app)/[slug]/settings/settings-sidebar.tsx b/apps/app/app/(app)/[slug]/settings/settings-sidebar.tsx index 3c561d1b..34a647f4 100644 --- a/apps/app/app/(app)/[slug]/settings/settings-sidebar.tsx +++ b/apps/app/app/(app)/[slug]/settings/settings-sidebar.tsx @@ -16,6 +16,7 @@ const ROOT = "/settings"; const ITEMS: SettingsNavItem[] = [ { title: "General", href: ROOT }, + { title: "Tracking & Analytics", href: `${ROOT}/tracking` }, { title: "Connections", href: `${ROOT}/connections` }, { title: "Currencies", href: `${ROOT}/currencies` }, { title: "Members", href: `${ROOT}/members` }, diff --git a/apps/app/app/(app)/[slug]/settings/tracking/allowed-domains.tsx b/apps/app/app/(app)/[slug]/settings/tracking/allowed-domains.tsx new file mode 100644 index 00000000..e69cc930 --- /dev/null +++ b/apps/app/app/(app)/[slug]/settings/tracking/allowed-domains.tsx @@ -0,0 +1,217 @@ +"use client"; + +import Add from "@carbon/icons-react/es/Add"; +import { Button } from "@crm/ui/components/button"; +import { + Card, + CardAction, + CardDescription, + CardHeader, + CardTitle, +} from "@crm/ui/components/card"; +import { CardTableEmpty } from "@crm/ui/components/card-table"; +import { Field, FieldLabel } from "@crm/ui/components/field"; +import { Icon } from "@crm/ui/components/icon"; +import { Input } from "@crm/ui/components/input"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@crm/ui/components/popover"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@crm/ui/components/select"; +import { + SimpleTable, + type SimpleTableColumn, + SimpleTableRow, +} from "@crm/ui/components/simple-table"; +import { Spinner } from "@crm/ui/components/spinner"; +import { TableCell } from "@crm/ui/components/table"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { useId, useState } from "react"; +import { toast } from "sonner"; +import { LocalRelativeTime } from "@/components/local-date-time"; +import { useCrmCache } from "@/lib/trpc/cache"; +import { useTRPC } from "@/lib/trpc/client"; + +const CELL = "px-3 py-2.5 align-middle"; + +const COLUMNS: SimpleTableColumn[] = [ + { id: "domain", header: "Domain" }, + { id: "scope", header: "Scope", width: "w-40" }, + { id: "pageViews", header: "Page views", width: "w-28", align: "right" }, + { id: "lastSeen", header: "Last seen", width: "w-28", align: "right" }, + { id: "actions", srLabel: "Actions", width: "w-24" }, +]; + +const SCOPES = { + SITE_AND_SUBDOMAINS: "Site + subdomains", + EXACT_HOST: "Exact host", +} as const; + +export function AllowedDomains() { + const trpc = useTRPC(); + const cache = useCrmCache(); + + const tracking = useQuery(trpc.tracking.settings.queryOptions()); + + const remove = useMutation( + trpc.tracking.removeDomain.mutationOptions({ + onSuccess: async () => { + await cache.tracking(); + toast.success("Domain removed."); + }, + onError: (error) => toast.error(error.message), + }), + ); + + if (!tracking.data) return null; + + const { domains, canManage } = tracking.data; + + return ( + + + Allowed domains + + The script records page views on these hosts only. + + + + + + + + {domains.length === 0 ? ( + + Add the domain your website runs on to get your tracking script. + + ) : ( + + {domains.map((domain) => ( + + + {domain.host} + + + {SCOPES[domain.scope]} + + + {domain.pageViews.toLocaleString()} + + + {domain.lastSeenAt ? ( + + ) : ( + "—" + )} + + + {canManage ? ( + + ) : null} + + + ))} + + )} + + ); +} + +function AddDomain({ disabled }: { disabled: boolean }) { + const trpc = useTRPC(); + const cache = useCrmCache(); + + const hostId = useId(); + const scopeId = useId(); + + const [open, setOpen] = useState(false); + const [host, setHost] = useState(""); + const [scope, setScope] = useState( + "SITE_AND_SUBDOMAINS", + ); + + const add = useMutation( + trpc.tracking.addDomain.mutationOptions({ + onSuccess: async () => { + await cache.tracking(); + setOpen(false); + setHost(""); + toast.success("Domain added."); + }, + onError: (error) => toast.error(error.message), + }), + ); + + return ( + + + + + + +
{ + event.preventDefault(); + add.mutate({ host: host.trim(), scope }); + }} + > + + Domain + setHost(event.target.value)} + placeholder="acme.com" + autoComplete="off" + autoCapitalize="off" + autoCorrect="off" + spellCheck={false} + /> + + + + Scope + + + + +
+ + + ); +} diff --git a/apps/app/app/(app)/[slug]/settings/tracking/page.tsx b/apps/app/app/(app)/[slug]/settings/tracking/page.tsx new file mode 100644 index 00000000..85abe837 --- /dev/null +++ b/apps/app/app/(app)/[slug]/settings/tracking/page.tsx @@ -0,0 +1,64 @@ +import type { Metadata } from "next"; +import { Suspense } from "react"; +import { + PageShell, + PageShellContent, + PageShellDescription, + PageShellHeader, + PageShellHeading, + PageShellLoading, + PageShellTitle, +} from "@/components/page-shell"; +import { requireSession } from "@/lib/session"; +import { HydrateClient } from "@/lib/trpc/hydrate"; +import { getServerQueryClient, getServerTrpc } from "@/lib/trpc/server"; +import { TrackingSections } from "./tracking-sections"; + +export const metadata: Metadata = { + title: "Tracking & Analytics", +}; + +export default function TrackingSettingsPage() { + return ( + + + + Tracking & Analytics + + Track website visitors and automatically add contacts when a form is + submitted. + + + + + + }> + + + + + ); +} + +async function Tracking() { + await requireSession(); + + const trpc = getServerTrpc(); + const queryClient = getServerQueryClient(); + + const settings = await queryClient.fetchQuery( + trpc.tracking.settings.queryOptions(), + ); + + if (settings.canManage) { + await queryClient.prefetchQuery(trpc.tracking.sources.queryOptions()); + } + + return ( + +
+ +
+ + ); +} diff --git a/apps/app/app/(app)/[slug]/settings/tracking/tracking-cookies.tsx b/apps/app/app/(app)/[slug]/settings/tracking/tracking-cookies.tsx new file mode 100644 index 00000000..3b6b054e --- /dev/null +++ b/apps/app/app/(app)/[slug]/settings/tracking/tracking-cookies.tsx @@ -0,0 +1,137 @@ +"use client"; + +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@crm/ui/components/card"; +import { Field, FieldDescription, FieldLabel } from "@crm/ui/components/field"; +import { Label } from "@crm/ui/components/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@crm/ui/components/select"; +import { Switch } from "@crm/ui/components/switch"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { useId } from "react"; +import { toast } from "sonner"; +import { useCrmCache } from "@/lib/trpc/cache"; +import { useTRPC } from "@/lib/trpc/client"; + +const TOGGLES = [ + { + flag: "cookieSubdomains", + label: "Limit cookies to subdomains", + hint: "Set the cookie on the exact host that served the page, never on the parent domain", + }, + { + flag: "secureCookies", + label: "Use secure cookies only", + hint: "Send the cookie over HTTPS and drop it on plain HTTP", + }, + { + flag: "honourDnt", + label: "Honour Do Not Track", + hint: "Record nothing at all when the browser asks not to be tracked", + }, +] as const; + +export function TrackingCookies() { + const trpc = useTRPC(); + const cache = useCrmCache(); + const lifetimeId = useId(); + + const tracking = useQuery(trpc.tracking.settings.queryOptions()); + + const setFlag = useMutation( + trpc.tracking.setFlag.mutationOptions({ + onSuccess: () => cache.tracking({ settle: "record" }), + onError: (error) => toast.error(error.message), + }), + ); + + const setLifetime = useMutation( + trpc.tracking.setCookieLifetime.mutationOptions({ + onSuccess: async () => { + await cache.tracking(); + toast.success("Cookie lifetime saved."); + }, + onError: (error) => toast.error(error.message), + }), + ); + + if (!tracking.data) return null; + + const { canManage, cookieDays, cookieLifetimes } = tracking.data; + const busy = !canManage || setFlag.isPending || setLifetime.isPending; + + return ( + + + Cookies + + How a returning visitor is recognised. + + + + + {TOGGLES.map((toggle) => ( +
+ + + + setFlag.mutate({ flag: toggle.flag, enabled }) + } + /> +
+ ))} + + + Cookie lifetime + + + After this a returning visitor counts as somebody new. Shorten it if + your policy asks you to. + + + + + ); +} diff --git a/apps/app/app/(app)/[slug]/settings/tracking/tracking-rules.tsx b/apps/app/app/(app)/[slug]/settings/tracking/tracking-rules.tsx new file mode 100644 index 00000000..5fb85c70 --- /dev/null +++ b/apps/app/app/(app)/[slug]/settings/tracking/tracking-rules.tsx @@ -0,0 +1,85 @@ +"use client"; + +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@crm/ui/components/card"; +import { Label } from "@crm/ui/components/label"; +import { Switch } from "@crm/ui/components/switch"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { toast } from "sonner"; +import { useCrmCache } from "@/lib/trpc/cache"; +import { useTRPC } from "@/lib/trpc/client"; + +const RULES = [ + { + flag: "crossDomain", + label: "Automatic cross-domain linking", + hint: "Carry the visitor between the domains below, so one journey is not counted as two people", + }, + { + flag: "limitToDomains", + label: "Limit tracking to the domains below", + hint: "On any other domain the script loads and then does nothing", + }, +] as const; + +export function TrackingRules() { + const trpc = useTRPC(); + const cache = useCrmCache(); + + const tracking = useQuery(trpc.tracking.settings.queryOptions()); + + const setFlag = useMutation( + trpc.tracking.setFlag.mutationOptions({ + onSuccess: () => cache.tracking({ settle: "record" }), + onError: (error) => toast.error(error.message), + }), + ); + + if (!tracking.data) return null; + + const { canManage } = tracking.data; + + return ( + + + Tracking rules + + Where the script may run, and how it follows a visitor. + + + + + {RULES.map((rule) => ( +
+ + + + setFlag.mutate({ flag: rule.flag, enabled }) + } + /> +
+ ))} + + + ); +} diff --git a/apps/app/app/(app)/[slug]/settings/tracking/tracking-script.tsx b/apps/app/app/(app)/[slug]/settings/tracking/tracking-script.tsx new file mode 100644 index 00000000..054a36a4 --- /dev/null +++ b/apps/app/app/(app)/[slug]/settings/tracking/tracking-script.tsx @@ -0,0 +1,221 @@ +"use client"; + +import Copy from "@carbon/icons-react/es/Copy"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@crm/ui/components/accordion"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "@crm/ui/components/alert-dialog"; +import { Button } from "@crm/ui/components/button"; +import { + Card, + CardAction, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@crm/ui/components/card"; +import { Icon } from "@crm/ui/components/icon"; +import { Label } from "@crm/ui/components/label"; +import { StatusIndicator } from "@crm/ui/components/status-indicator"; +import { Switch } from "@crm/ui/components/switch"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { toast } from "sonner"; +import { useCrmCache } from "@/lib/trpc/cache"; +import { useTRPC } from "@/lib/trpc/client"; + +export function TrackingScript() { + const trpc = useTRPC(); + const cache = useCrmCache(); + const tracking = useQuery(trpc.tracking.settings.queryOptions()); + + const setFlag = useMutation( + trpc.tracking.setFlag.mutationOptions({ + onSuccess: async (_result, input) => { + await cache.tracking(); + toast.success( + input.enabled + ? "Tracking paused. The script stops recording within five minutes." + : "Tracking resumed.", + ); + }, + onError: (error) => toast.error(error.message), + }), + ); + + const rotate = useMutation( + trpc.tracking.rotateSiteId.mutationOptions({ + onSuccess: async () => { + await cache.tracking(); + toast.success("Site ID rotated. Paste the new tag on your website."); + }, + onError: (error) => toast.error(error.message), + }), + ); + + if (!tracking.data) return null; + + const { siteId, snippet, scriptUrl, receivingSince, paused, canManage } = + tracking.data; + + const copy = () => { + const clipboard = navigator.clipboard; + + if (!snippet || !clipboard) { + toast.error("Could not copy the script. Select it instead."); + return; + } + + clipboard + .writeText(snippet) + .then(() => toast.success("Script copied.")) + .catch(() => toast.error("Could not copy the script.")); + }; + + return ( + + + +
+ Tracking script + +
+ + + One tag, 4 KB, in the head of every page you measure. + + + + + + + + + + + Paste it into your HTML + + 
+								{"
+								{"\n  src="}
+								{`"${scriptUrl}"`}
+								{"\n  data-site="}
+								{`"${siteId}"`}
+								{"\n  async\n  defer\n"}
+								{">"}
+
+

+ Site ID{" "} + {siteId} · + Rotating it stops every copy of the old script at once. +

+ + + + + + Add it through Google Tag Manager + + +
    +
  1. In Tag Manager, add a new Custom HTML tag.
    2. +
  2. Paste the snippet above as the tag's HTML.
    3. +
  3. + Trigger it on All Pages, then publish the container. Keep{" "} + {scriptUrl}{" "} + off any consent-blocked category you do not need. +
    4. +
+ + + + +
+ + + + setFlag.mutate({ flag: "paused", enabled }) + } + /> +
+ + +
+ + + + + + + + Rotate the site ID? + + Every copy of the old script stops recording at once, + including any you have forgotten about. You will need to + paste the new tag on every page that carries the old one. + Nothing already collected is lost. + + + + + Cancel + rotate.mutate()} + > + Rotate + + + + +
+ + + + ); +} diff --git a/apps/app/app/(app)/[slug]/settings/tracking/tracking-sections.tsx b/apps/app/app/(app)/[slug]/settings/tracking/tracking-sections.tsx new file mode 100644 index 00000000..191865a0 --- /dev/null +++ b/apps/app/app/(app)/[slug]/settings/tracking/tracking-sections.tsx @@ -0,0 +1,35 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { useTRPC } from "@/lib/trpc/client"; +import { AllowedDomains } from "./allowed-domains"; +import { TrackingCookies } from "./tracking-cookies"; +import { TrackingRules } from "./tracking-rules"; +import { TrackingScript } from "./tracking-script"; +import { TrafficSources } from "./traffic-sources"; +import { VerifyInstallation } from "./verify-installation"; + +export function TrackingSections() { + const trpc = useTRPC(); + const tracking = useQuery(trpc.tracking.settings.queryOptions()); + + if (!tracking.data?.ready) { + return ( + <> + + + + ); + } + + return ( + <> + + + {tracking.data.canManage ? : null} + + + + + ); +} diff --git a/apps/app/app/(app)/[slug]/settings/tracking/traffic-sources.tsx b/apps/app/app/(app)/[slug]/settings/tracking/traffic-sources.tsx new file mode 100644 index 00000000..164caef7 --- /dev/null +++ b/apps/app/app/(app)/[slug]/settings/tracking/traffic-sources.tsx @@ -0,0 +1,69 @@ +"use client"; + +import { + Card, + CardDescription, + CardHeader, + CardTitle, +} from "@crm/ui/components/card"; +import { CardTableEmpty } from "@crm/ui/components/card-table"; +import { + SimpleTable, + type SimpleTableColumn, + SimpleTableRow, +} from "@crm/ui/components/simple-table"; +import { TableCell } from "@crm/ui/components/table"; +import { useQuery } from "@tanstack/react-query"; +import { useTRPC } from "@/lib/trpc/client"; + +const CELL = "px-3 py-2.5 align-middle"; + +const COLUMNS: SimpleTableColumn[] = [ + { id: "source", header: "Source" }, + { id: "medium", header: "Medium", width: "w-32" }, + { id: "views", header: "Page views", width: "w-28", align: "right" }, + { id: "contacts", header: "Contacts", width: "w-24", align: "right" }, +]; + +export function TrafficSources() { + const trpc = useTRPC(); + const sources = useQuery(trpc.tracking.sources.queryOptions()); + + if (!sources.data) return null; + + return ( + + + Traffic sources + + Where your visitors come from. Only people who have submitted a form + are attributed to a record. + + + + {sources.data.length === 0 ? ( + + No sources yet. They appear once the script records its first page + view. + + ) : ( + + {sources.data.map((row) => ( + + {row.source} + + {row.medium ?? "—"} + + + {row.views.toLocaleString()} + + + {row.contacts.toLocaleString()} + + + ))} + + )} + + ); +} diff --git a/apps/app/app/(app)/[slug]/settings/tracking/verify-installation.tsx b/apps/app/app/(app)/[slug]/settings/tracking/verify-installation.tsx new file mode 100644 index 00000000..bcc19cdd --- /dev/null +++ b/apps/app/app/(app)/[slug]/settings/tracking/verify-installation.tsx @@ -0,0 +1,180 @@ +"use client"; + +import CheckmarkFilled from "@carbon/icons-react/es/CheckmarkFilled"; +import Warning from "@carbon/icons-react/es/Warning"; +import { Alert, AlertDescription, AlertTitle } from "@crm/ui/components/alert"; +import { Button } from "@crm/ui/components/button"; +import { + Card, + CardAction, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@crm/ui/components/card"; +import { Field, FieldDescription, FieldLabel } from "@crm/ui/components/field"; +import { Icon } from "@crm/ui/components/icon"; +import { + InputGroup, + InputGroupAddon, + InputGroupInput, + InputGroupText, +} from "@crm/ui/components/input-group"; +import { Spinner } from "@crm/ui/components/spinner"; +import { StatusIndicator } from "@crm/ui/components/status-indicator"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { useId, useState } from "react"; +import { toast } from "sonner"; +import { useTRPC } from "@/lib/trpc/client"; +import type { RouterOutputs } from "@/lib/trpc/types"; + +type Result = RouterOutputs["tracking"]["verify"]; + +export function VerifyInstallation() { + const trpc = useTRPC(); + const urlId = useId(); + + const [url, setUrl] = useState(""); + const [result, setResult] = useState(null); + + const tracking = useQuery(trpc.tracking.settings.queryOptions()); + + const verify = useMutation( + trpc.tracking.verify.mutationOptions({ + onSuccess: (outcome) => setResult(outcome), + onError: (error) => toast.error(error.message), + }), + ); + + if (!tracking.data) return null; + + const { canManage, siteId } = tracking.data; + + return ( + + + +
+ Verify installation + {result ? : null} +
+ + + We load one page and look for the script. + + + + + + + + +
{ + event.preventDefault(); + setResult(null); + verify.mutate({ url: url.trim() }); + }} + > + + Page to check + + + https:// + + { + setUrl(event.target.value); + setResult(null); + }} + placeholder="acme.com/pricing" + autoComplete="off" + autoCapitalize="off" + autoCorrect="off" + spellCheck={false} + inputMode="url" + disabled={!canManage || verify.isPending} + /> + + + The page has to be public. A page behind a login always fails this + check. + + +
+ + {result && siteId ? : null} + + + ); +} + +function Indicator({ result }: { result: Result }) { + if (result.status === "found" && result.pageView) { + return ( + + ); + } + + return ( + + ); +} + +function Outcome({ result, siteId }: { result: Result; siteId: string }) { + if (result.status === "unreachable") { + return ( + + + Could not open {result.host} + + {result.detail} We only follow public pages, and we never follow a + redirect to a private address. + + + ); + } + + if (result.status === "missing") { + return ( + + + No script on {result.host} + + The page answered in {result.responseMs} ms, but the tag was not in + the HTML. Check that it sits in the head, above anything that rewrites + the page. + + + ); + } + + return ( + + + Script found on {result.host} + + It answered in {result.responseMs} ms. Site ID {siteId} matched, and + this domain is {result.allowed ? "on" : "not on"} the allow list. + {result.pageView + ? " A page view arrived in the last five minutes." + : " No page view has arrived yet — open the page in a browser to send one."} + + + ); +} diff --git a/apps/app/app/t/[site]/route.ts b/apps/app/app/t/[site]/route.ts new file mode 100644 index 00000000..082350e7 --- /dev/null +++ b/apps/app/app/t/[site]/route.ts @@ -0,0 +1,56 @@ +import { CONFIG_MAX_AGE_SECONDS, isSiteId } from "@crm/db/tracking"; +import { API_URL } from "@/lib/env"; +import { trackerSource } from "@/lib/tracking/tracker"; + +const EMPTY = "/* no tracking site is configured */\n"; + +export async function GET( + request: Request, + { params }: { params: Promise<{ site: string }> }, +): Promise { + const { site } = await params; + const siteId = site.replace(/\.js$/, ""); + + if (!isSiteId(siteId)) return empty(); + + let payload: { config: unknown; hash?: string } | null = null; + + try { + const upstream = await fetch(`${API_URL}/api/t/config/${siteId}`, { + headers: { accept: "application/json" }, + }); + + if (upstream.ok) { + payload = (await upstream.json()) as { config: unknown; hash?: string }; + } + } catch { + return empty(); + } + + if (!payload?.config) return empty(); + + const origin = new URL(request.url).origin; + const source = trackerSource( + payload.config as Parameters[0], + `${origin}/api/t/e`, + ); + + return new Response(source, { + headers: { + "content-type": "application/javascript; charset=utf-8", + "cache-control": `public, max-age=${CONFIG_MAX_AGE_SECONDS}, s-maxage=${CONFIG_MAX_AGE_SECONDS}`, + "x-content-type-options": "nosniff", + ...(payload.hash ? { etag: `"${payload.hash}"` } : {}), + }, + }); +} + +function empty(): Response { + return new Response(EMPTY, { + headers: { + "content-type": "application/javascript; charset=utf-8", + "cache-control": "public, max-age=60, s-maxage=60", + "x-content-type-options": "nosniff", + }, + }); +} diff --git a/apps/app/app/t/crm.js/route.ts b/apps/app/app/t/crm.js/route.ts new file mode 100644 index 00000000..2c7d5fd4 --- /dev/null +++ b/apps/app/app/t/crm.js/route.ts @@ -0,0 +1,15 @@ +import { + BUNDLE_MAX_AGE_SECONDS, + LOADER_MAX_AGE_SECONDS, +} from "@crm/db/tracking"; +import { LOADER_SOURCE } from "@/lib/tracking/loader"; + +export function GET(): Response { + return new Response(LOADER_SOURCE, { + headers: { + "content-type": "application/javascript; charset=utf-8", + "cache-control": `public, max-age=${LOADER_MAX_AGE_SECONDS}, s-maxage=${BUNDLE_MAX_AGE_SECONDS}, immutable`, + "x-content-type-options": "nosniff", + }, + }); +} diff --git a/apps/app/components/crm/record-sheet/company-sheet.tsx b/apps/app/components/crm/record-sheet/company-sheet.tsx index 92ca756f..bddb2368 100644 --- a/apps/app/components/crm/record-sheet/company-sheet.tsx +++ b/apps/app/components/crm/record-sheet/company-sheet.tsx @@ -37,6 +37,7 @@ import { OwnerCell } from "@/components/crm/owner-cell"; import { CompanySocials } from "@/components/crm/social-links"; import { DealStageMenu } from "@/components/crm/stage-change"; import { Timeline } from "@/components/crm/timeline/timeline"; +import { WebsiteActivity } from "@/components/crm/website-activity"; import { DetailSheetBody, DetailSheetEmpty, @@ -172,15 +173,7 @@ export function CompanySheet({ companyId }: { companyId: string }) { { value: "overview", label: "Overview", - content: ( - { - setAdding("contact"); - setTab("contacts"); - }} - /> - ), + content: , }, { value: "contacts", @@ -303,13 +296,7 @@ export function CompanySheet({ companyId }: { companyId: string }) { ); } -function CompanyOverview({ - company, - onAddContact, -}: { - company: Company; - onAddContact: () => void; -}) { +function CompanyOverview({ company }: { company: Company }) { const trpc = useTRPC(); const cache = useCrmCache(); @@ -341,14 +328,7 @@ function CompanyOverview({ ) : null} - - undefined} - /> - + diff --git a/apps/app/components/crm/record-sheet/contact-sheet.tsx b/apps/app/components/crm/record-sheet/contact-sheet.tsx index 6ff1f005..a72353f2 100644 --- a/apps/app/components/crm/record-sheet/contact-sheet.tsx +++ b/apps/app/components/crm/record-sheet/contact-sheet.tsx @@ -38,6 +38,7 @@ import { OwnerCell } from "@/components/crm/owner-cell"; import { ContactSocials } from "@/components/crm/social-links"; import { DealStageMenu } from "@/components/crm/stage-change"; import { Timeline } from "@/components/crm/timeline/timeline"; +import { WebsiteActivity } from "@/components/crm/website-activity"; import { DetailSheetBody, DetailSheetEmpty, @@ -429,6 +430,8 @@ function ContactOverview({ contact }: { contact: Contact }) { ) : null} + + ); } diff --git a/apps/app/components/crm/website-activity.tsx b/apps/app/components/crm/website-activity.tsx new file mode 100644 index 00000000..96659509 --- /dev/null +++ b/apps/app/components/crm/website-activity.tsx @@ -0,0 +1,113 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { + DetailSheetProperties, + DetailSheetProperty, + DetailSheetSection, +} from "@/components/detail-sheet"; +import { LocalRelativeTime } from "@/components/local-date-time"; +import { useTRPC } from "@/lib/trpc/client"; + +type Touch = { + source: string; + medium: string | null; + campaign: string | null; + at: string | null; +}; + +export function WebsiteActivity({ + companyId, + contactId, +}: { + companyId?: string; + contactId?: string; +}) { + const trpc = useTRPC(); + + const company = useQuery({ + ...trpc.tracking.companyActivity.queryOptions({ + companyId: companyId ?? "", + }), + enabled: Boolean(companyId), + }); + + const contact = useQuery({ + ...trpc.tracking.contactActivity.queryOptions({ + contactId: contactId ?? "", + }), + enabled: Boolean(contactId) && !companyId, + }); + + const activity = companyId ? company.data : contact.data; + + if (!activity?.identified) return null; + if (activity.pages.length === 0 && !activity.firstTouch) return null; + + const topPage = activity.pages[0]; + const first = activity.firstTouch; + const last = activity.lastTouch; + const channelChanged = + first != null && last != null && channel(last) !== channel(first); + const campaign = last?.campaign ?? first?.campaign ?? null; + + return ( + + + + + {activity.views.toLocaleString()} + + {activity.lastSeenAt ? ( + + {" · last seen "} + + + ) : null} + + + {first ? ( + + {channel(first)} + + ) : null} + + {topPage ? ( + + + + {topPage.path} + + + {"· "} + {topPage.views} views + + + + ) : null} + + {channelChanged ? ( + + {channel(last)} + + ) : null} + + {first?.at ? ( + + + + ) : null} + + {campaign ? ( + {campaign} + ) : null} + + + ); +} + +function channel(touch: Touch | null): string { + if (!touch) return "Unknown"; + if (!touch.medium || touch.medium === "direct") return touch.source; + return `${touch.source} · ${touch.medium}`; +} diff --git a/apps/app/lib/tracking/loader.ts b/apps/app/lib/tracking/loader.ts new file mode 100644 index 00000000..ef2ea13a --- /dev/null +++ b/apps/app/lib/tracking/loader.ts @@ -0,0 +1,17 @@ +export const LOADER_SOURCE = `(function(){ +var s=document.currentScript||document.querySelector("script[data-site][src*='/t/crm.js']"); +if(!s||!s.src)return; +var site=s.getAttribute("data-site"); +if(!site||!/^cmp_[0-9a-f]{8}$/.test(site))return; +var id="crm-tracker-"+site; +if(document.getElementById(id))return; +var origin; +try{origin=new URL(s.src).origin}catch(e){return} +var t=document.createElement("script"); +t.id=id; +t.async=!0; +t.defer=!0; +t.src=origin+"/t/"+site+".js"; +(document.head||document.documentElement).appendChild(t); +})(); +`; diff --git a/apps/app/lib/tracking/tracker.ts b/apps/app/lib/tracking/tracker.ts new file mode 100644 index 00000000..e3955043 --- /dev/null +++ b/apps/app/lib/tracking/tracker.ts @@ -0,0 +1,163 @@ +import { COOKIE_FIRST_TOUCH } from "@crm/db/attribution"; +import type { TrackingConfig } from "@crm/db/tracking"; +import { + COOKIE_NAME, + LINKER_MAX_AGE_SECONDS, + MAX_BODY_BYTES, + MAX_EVENTS_PER_BATCH, +} from "@crm/db/tracking"; + +export function trackerSource( + config: TrackingConfig, + endpoint: string, +): string { + return `(function(){ +var C=${JSON.stringify(config)},E=${JSON.stringify(endpoint)},N=${JSON.stringify(COOKIE_NAME)},FS=${JSON.stringify(COOKIE_FIRST_TOUCH)}; +var B=${MAX_BODY_BYTES},M=${MAX_EVENTS_PER_BATCH}; +var d=document,w=window,loc=w.location; +if(w.__crmT)return;w.__crmT=1; +function host(){return loc.hostname.toLowerCase()} +function known(h){ + for(var i=0;i0)p+="; max-age="+C.cookieDays*86400; + if(C.secureCookies&&loc.protocol==="https:")p+="; secure"; + var dom=cookieDomain(); + if(dom)p+="; domain="+dom; + d.cookie=p} +function mint(){ + if(w.crypto&&w.crypto.randomUUID)return w.crypto.randomUUID().replace(/-/g,""); + return Math.random().toString(36).slice(2)+Date.now().toString(36)} +var existing=read(),linked=null; +if(C.crossDomain){ + var m=loc.hash.match(/_crm=([A-Za-z0-9_-]{8,64})\\.(\\d{10})/); + if(m){ + var age=Math.floor(Date.now()/1000)-parseInt(m[2],10); + if(age>=0&&age<=${LINKER_MAX_AGE_SECONDS}&&!existing)linked=m[1]; + try{history.replaceState(null,"",loc.href.replace(/[#&]_crm=[A-Za-z0-9_.-]+/,""))}catch(e){}}} +var vid=linked||existing||mint(); +write(vid); +function param(q,k){var m=q.match(new RegExp("[?&]"+k+"=([^&]*)"));if(!m)return undefined;var v=decode(m[1].replace(/\\+/g," "));return v?v.slice(0,120):undefined} +function touch(){ + var q=loc.search,t={landing:loc.pathname.slice(0,120),at:Date.now()}; + var s=param(q,"utm_source"),m=param(q,"utm_medium"); + if(s)t.source=s; + if(m)t.medium=m; + var c=param(q,"utm_campaign");if(c)t.campaign=c; + var tm=param(q,"utm_term");if(tm)t.term=tm; + var ct=param(q,"utm_content");if(ct)t.content=ct; + var gc=param(q,"gclid");if(gc&&!s){t.source="Google";t.medium="cpc"} + var r=d.referrer; + if(r){try{if(new URL(r).hostname.toLowerCase()!==host())t.referrer=r.slice(0,300)}catch(e){}} + return t} +function readFirst(){ + var m=d.cookie.match(new RegExp("(?:^|; )"+FS+"=([^;]*)")); + if(!m)return null; + var v=decode(m[1]); + if(!v)return null; + try{return JSON.parse(v)}catch(e){return null}} +function writeFirst(t){ + var p=FS+"="+encodeURIComponent(JSON.stringify(t))+"; path=/; samesite=lax"; + if(C.cookieDays>0)p+="; max-age="+C.cookieDays*86400; + if(C.secureCookies&&loc.protocol==="https:")p+="; secure"; + var dom=cookieDomain(); + if(dom)p+="; domain="+dom; + d.cookie=p} +var last=touch(); +var first=readFirst(); +if(!first){first=last;writeFirst(first)} +var q=[],timer=null; +function pack(evts){return JSON.stringify({siteId:C.siteId,visitorId:vid,events:evts})} +function bytes(s){try{return new Blob([s]).size}catch(e){return s.length*3}} +function shrink(e,body){ + var keys=e.fields?Object.keys(e.fields):[]; + while(keys.length>1&&bytes(body)>B){delete e.fields[keys.pop()];body=pack([e])} + return body} +function post(body){ + try{ + if(navigator.sendBeacon&&navigator.sendBeacon(E,new Blob([body],{type:"text/plain"})))return; + fetch(E,{method:"POST",body:body,keepalive:!0,mode:"no-cors",headers:{"content-type":"text/plain"}}) + }catch(e){}} +function flush(){ + clearTimeout(timer); + while(q.length){ + var take=q.splice(0,M),body=pack(take); + while(bytes(body)>B&&take.length>1){q.unshift(take.pop());body=pack(take)} + if(bytes(body)>B)body=shrink(take[0],body); + if(bytes(body)<=B)post(body)}} +function send(e){ + e.host=host();e.at=Date.now(); + q.push(e); + if(q.length>=M){flush();return} + clearTimeout(timer);timer=setTimeout(flush,2000)} +function view(){send({type:"page_view",path:loc.pathname,referrer:d.referrer||undefined,touch:last})} +function label(el){ + var t=(el.getAttribute("aria-label")||el.textContent||"").replace(/\\s+/g," ").trim(); + return t?t.slice(0,80):undefined} +function onClick(ev){ + var el=ev.target; + while(el&&el!==d.body){ + if(el.hasAttribute&&el.hasAttribute("data-crm-track")){send({type:"click",path:loc.pathname,label:el.getAttribute("data-crm-track")||label(el)});return} + if(el.tagName==="A"||el.tagName==="BUTTON"){ + var href=el.getAttribute&&el.getAttribute("href")||""; + var out=/^https?:/i.test(href)&&href.indexOf(loc.origin)!==0; + if(out||el.tagName==="BUTTON")send({type:"click",path:loc.pathname,label:label(el)}); + return} + el=el.parentElement}} +function onSubmit(ev){ + var f=ev.target; + if(!f||f.tagName!=="FORM")return; + var fields={},n=0; + for(var i=0;i; workspace(options?: Options): Promise; sso(options?: Options): Promise; + tracking(options?: Options): Promise; everything(): Promise; }; @@ -277,6 +278,13 @@ export function useCrmCache(): CrmCache { options, ), + tracking: (options) => + run( + [trpc.tracking.settings.queryKey()], + [trpc.tracking.sources.queryKey()], + options, + ), + everything: () => queryClient.invalidateQueries(), }; } diff --git a/apps/app/lib/trpc/query-client.ts b/apps/app/lib/trpc/query-client.ts index af6f9eeb..e38f3e89 100644 --- a/apps/app/lib/trpc/query-client.ts +++ b/apps/app/lib/trpc/query-client.ts @@ -1,17 +1,9 @@ -import { - defaultShouldDehydrateQuery, - QueryClient, -} from "@tanstack/react-query"; +import { QueryClient } from "@tanstack/react-query"; export function makeQueryClient(): QueryClient { return new QueryClient({ defaultOptions: { queries: { staleTime: 30_000, retry: retryQuery }, - dehydrate: { - shouldDehydrateQuery: (query) => - defaultShouldDehydrateQuery(query) || - query.state.status === "pending", - }, }, }); } diff --git a/apps/app/proxy.ts b/apps/app/proxy.ts index 4d955154..6040b706 100644 --- a/apps/app/proxy.ts +++ b/apps/app/proxy.ts @@ -16,6 +16,8 @@ const SIGN_IN_PATH = "/sign-in"; const UNGATED = ["/grant-access", "/eve"]; +const ANONYMOUS = ["/t"]; + const SECTIONS = ["/companies", "/contacts", "/deals", "/settings"]; export async function proxy(request: NextRequest) { @@ -23,6 +25,8 @@ export async function proxy(request: NextRequest) { if (pathname === SIGN_IN_PATH) return NextResponse.next(); + if (isAnonymous(pathname)) return NextResponse.next(); + if ( getSessionCookie(request, { cookiePrefix: AUTH_COOKIE_PREFIX }) === null ) { @@ -78,6 +82,10 @@ function isUngated(pathname: string): boolean { return UNGATED.some((prefix) => isUnder(pathname, prefix)); } +function isAnonymous(pathname: string): boolean { + return ANONYMOUS.some((prefix) => isUnder(pathname, prefix)); +} + function isSetup(pathname: string): boolean { return pathname === ONBOARDING_PATH || pathname === RESEARCH_PATH; } diff --git a/apps/app/test/tracking-bundle.spec.ts b/apps/app/test/tracking-bundle.spec.ts new file mode 100644 index 00000000..eeb679c4 --- /dev/null +++ b/apps/app/test/tracking-bundle.spec.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test"; +import { brotliCompressSync } from "node:zlib"; +import type { TrackingConfig } from "@crm/db/tracking"; +import { LOADER_SOURCE } from "@/lib/tracking/loader"; +import { trackerSource } from "@/lib/tracking/tracker"; + +const LOADER_BUDGET = 1024; + +const TRACKER_BUDGET = 4096; + +const CONFIG: TrackingConfig = { + siteId: "cmp_8f3ad91c", + crossDomain: true, + limitToDomains: true, + cookieSubdomains: false, + secureCookies: true, + honourDnt: true, + cookieDays: 395, + hosts: [ + { host: "trycomp.ai", scope: "SITE_AND_SUBDOMAINS" }, + { host: "www.trycomp.ai", scope: "EXACT_HOST" }, + { host: "docs.trycomp.ai", scope: "EXACT_HOST" }, + { host: "app.trycomp.ai", scope: "EXACT_HOST" }, + ], +}; + +function brotli(source: string): number { + return brotliCompressSync(Buffer.from(source, "utf8")).length; +} + +describe("the tracking bundle stays inside its budget", () => { + test("the loader is under a kilobyte", () => { + expect(brotli(LOADER_SOURCE)).toBeLessThanOrEqual(LOADER_BUDGET); + }); + + test("the tracker is under the four kilobytes the settings page promises", () => { + const source = trackerSource(CONFIG, "https://crm.example.com/api/t/e"); + + expect(brotli(source)).toBeLessThanOrEqual(TRACKER_BUDGET); + }); + + test("the loader only ever injects a site it was given", () => { + expect(LOADER_SOURCE).toContain("cmp_[0-9a-f]{8}"); + expect(LOADER_SOURCE).toContain("document.currentScript"); + }); + + test("the tracker bakes the config in rather than fetching it", () => { + const source = trackerSource(CONFIG, "https://crm.example.com/api/t/e"); + + expect(source).toContain("cmp_8f3ad91c"); + expect(source).toContain("docs.trycomp.ai"); + expect(source).not.toContain("/api/t/config"); + }); +}); diff --git a/docs/telemetry.md b/docs/telemetry.md index bb3aa7e2..b89a9cdf 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -98,10 +98,11 @@ whose event never arrived, which cannot be recovered. `cap_rapidapi`, `cap_perplexity`, `cap_context_dev`, `cap_blob`, `cap_github`, `cap_redis`, `cap_agent_bridge`, `cap_cron_secret`, `cap_ai_gateway`, `cap_google_oauth`, `cap_sso_provider`, -`is_marketing`. +`cap_tracking`, `is_marketing`. Each is only whether the key is set. `cap_context_dev` is whether an `AppSetting` row holds one, -`cap_sso_provider` whether an `ssoProvider` row exists. No key, value or last-four is sent. +`cap_sso_provider` whether an `ssoProvider` row exists, `cap_tracking` whether a tracking site id +has been minted — never the id itself. No key, value or last-four is sent. #### The agent @@ -159,6 +160,17 @@ sent. | `suppressed_domains`, `suppressed_contacts` | Counts. Never the domains or addresses | | `workspace_profile_written` | Whether a `WorkspaceProfile` row exists | +#### Website tracking + +| Property | What it is | +| --- | --- | +| `tracking_domains` | How many domains are on the allow list, in bands. Never a host | +| `tracking_page_views` | Page views recorded in the window. A count, never a path | +| `tracking_forms` | Form submissions stored in the window. A count, never a field | +| `tracking_contacts_created` | Contacts created **by** tracking in the window — `Contact.source = TRACKING`, not submissions that attached to somebody already filed | +| `tracking_capped` | Submissions the hourly contact cap refused. Matched against `CONTACT_CAP_REASON` (`@crm/db/tracking`) exactly, so the wording is a constant two files share rather than prose one of them may reword | +| `tracking_paused` | Whether collection is paused | + ### The setup funnel Eight events, each sent once per install, ever. Each carries the timestamp of the thing it diff --git a/docs/tracking.md b/docs/tracking.md new file mode 100644 index 00000000..ab5e2faa --- /dev/null +++ b/docs/tracking.md @@ -0,0 +1,212 @@ +# Website tracking + +A first-party script on the customer's own marketing site, a collector in the API, +and one rule about what it is for: **a form submission becomes a contact.** Page +views exist to give that contact a story, not to be a web-analytics product. + +Everything here is one install's own website. There is no second tenant, no shared +pixel, and no vendor: the script is served from the same origin as the app, the +cookie is first-party, and the only thing that ever leaves the browser is a POST to +`/api/t/e` on the install's own API. + +## Two scripts, and why + +| | | +| --- | --- | +| `apps/app/lib/tracking/loader.ts` → `/t/crm.js` | The tag a rep pastes. Reads `data-site`, checks the shape, injects the second script. Immutable and cached for a year at the edge | +| `apps/app/lib/tracking/tracker.ts` → `/t/.js` | The tracker itself, with the config **baked into the source** rather than fetched. Cached for five minutes | + +The split is the whole cache design. The tag never changes, so it is `immutable` +and free forever; the config does change, so the file that carries it is the one +with the short life. Baking the config in also means a page view costs one request, +not a request and then a config fetch before anything can be recorded. + +- **Five minutes is a promise.** Pause tracking and every browser stops within + `CONFIG_MAX_AGE_SECONDS`. That is why `/t/[site]` carries **no + `stale-while-revalidate`** — a revalidation window is exactly a licence to keep + executing the old config after the pause, and 24 hours of it once cost this + guarantee entirely. +- **Both routes are anonymous**, listed in `proxy.ts` as `ANONYMOUS = ["/t"]`. A + stranger's browser on the customer's marketing site has no session and must not + be redirected to `/sign-in`. +- **The tracker has a size budget**, asserted in `apps/app/test/tracking-bundle.spec.ts`: + 1 KB brotli for the loader, 4 KB for the tracker, because the settings page + promises a number. A change that busts it fails the test rather than the promise. + +### Writing tracker source + +It is a string of ES5 in a template literal, minified by hand, and it runs on +somebody else's page. That imposes rules nothing else in this repo has: + +- **Never throw.** An uncaught error on line 40 means no listeners are installed and + the page records nothing at all. Every `decodeURIComponent`, `JSON.parse`, `new + URL` and `history` call is already wrapped; a malformed UTM parameter must read as + *absent*, never as *fatal*. +- **Constants come from `@crm/db/tracking`**, interpolated in — `MAX_BODY_BYTES`, + `MAX_EVENTS_PER_BATCH`, `LINKER_MAX_AGE_SECONDS`, `COOKIE_NAME`. A literal at the + call site is a client and a server that disagree, and the disagreement is silent. +- **The client stays inside the body limit the collector enforces.** `flush()` + measures the packed batch and splits it, then trims a single oversized form's + fields, rather than posting a body the collector will destroy — a rejected POST is + not retried, so the whole batch would simply be lost. + +## The collector + +`POST /api/t/e`, anonymous, 204, in `TrackingController`. It answers nothing: a +tracker that could read a response is a tracker whose failures a stranger can probe. + +The gauntlet, in order, in `TrackingIngestService.accept`: + +1. **User agent** — the `BOT` pattern. +2. **Site id** — must be a live `cmp_` id, and `forSite` refuses a rotated one. +3. **Origin** — `originAllowed`. A missing `Origin` header is refused **even with + the allow list off**, because the header is the only thing tying the POST to a + browser on a page. +4. **Visitor id** — 8–64 of `[a-zA-Z0-9_-]`, or the batch is dropped. +5. **`scripted()`** — three or more events sharing one timestamp is a replay. It + only fires when **every** event carries a numeric `at`; missing timestamps are + no signal, and treating them as proof discarded honest traffic. +6. **Host** — each event's own host against the allow list. The batch is filtered, + not refused, because one bad host in twenty is a bug in somebody's SPA. +7. **Rate** — `EVENTS_PER_MINUTE` charged **per accepted event, atomically**, + through `TrackingCounter`. + +> **The rate limit counts events, not requests, and it counts them in the +> database.** A per-request counter with a full batch behind it admits twenty times +> the number on the constant, and a read-then-write in `cache-manager` admits +> however many requests are in flight. `TrackingCounterService.take(key, limit, +> amount)` is one statement — an `INSERT … ON CONFLICT DO UPDATE` whose `WHERE` +> carries the limit — so the check and the charge cannot be separated, and **a +> refused batch spends nothing**. Increment-then-compare looks equivalent and is +> not: the refused batch still burns its own size out of the window, so one +> oversized batch locks out the honest small ones behind it for the rest of the +> minute. It fails **closed**: a counter it cannot reach refuses the write. + +`MAX_BODY_BYTES` is enforced by destroying the request as it streams, before any +parse. Keep it in step with what the tracker can produce — a form is forty fields +of five hundred characters, and a limit under that silently drops the submission +this whole feature exists to catch. + +### What is never stored + +- **No query strings.** `normalizePath` strips `?` and `#` from every path, and + `stripQuery` does the same to every referrer, on the event row *and* inside the + `firstTouch` / `lastTouch` blobs. A password-reset link in a referrer is a + password-reset link in the database. UTM parameters are captured as their own + fields, so nothing is lost by it. +- **No sensitive fields.** `clean()` drops any field whose *name* matches + `SENSITIVE`, anything shaped like a card number, and every `password`, `hidden` + and `file` input — the last three never leave the browser at all. +- **No IP address**, anywhere, ever. + +## Attribution + +`packages/db/src/attribution.ts`, and it is pure: no database, no config, one +function. `classifyTouch` turns UTM parameters and a referrer into a `Touch`. + +- **An explicit `utm_source` beats the referrer**, because the marketer said so. +- **The referrer host is matched on DNS labels, never on a substring**, and the + provider's labels must sit at the registrable name — one tail label, or two when + the first is a second-level suffix **and the last is a two-letter country code** + (`co.uk`, `com.au`, `co.jp`). So `images.google.de` and `scholar.google.co.jp` are + Google, while `notgoogle.com`, `google.evil.com`, `google.com.phish.example` and + `google.com.example` are all plain referrals. A substring match hands a phisher a + trusted source name in the rep's report, and so does `com.`: the country + code is what stops `SECOND_LEVEL` from waving through every `com.` lookalike. + **It is still a heuristic, not the public suffix list** — a provider under an + unusual two-label suffix reads as a referral, which is the safe way to be wrong. +- **Webmail is checked before search.** `mail.google.com` contains `google.`, so + order alone decides whether a campaign's own click-throughs read as *email* or as + *organic search* — and reading your newsletter as Google organic is how a + marketing report lies. +- **`Direct` is a source, not a null.** Everything lands in one of the seven + `MEDIUMS`; an unrecognised medium is `other`. + +## Filing: from a submission to a contact + +`TrackingFilingService.file` is the only path from a form to a `Contact`, and it +reuses the mailbox pipeline's judgement rather than inventing a second one — +`isMachineAddress`, `isAutomatedAddress`, `isMachineDomain`, `workspaceDomains`, +`SuppressedContact`, `SuppressedDomain`, `companyForEmail`, `splitName`. **A rule +that holds for the inbox holds here.** A second copy is how *deleted contacts stay +deleted* comes to be true of email and not of forms. + +- **Every submission is stored; filing is separate and may decline.** `skipReason` + says why, and the row stays for a rep to look at. +- **A refusal is a `skipReason`, never a lost row**, and `CONTACT_CAP_REASON` is a + shared constant because `rollup.service.ts` matches on it. Telemetry that + substring-matches prose breaks the first time somebody improves the wording. +- **`CONTACTS_PER_HOUR` bounds the blast radius** of a scripted form. It is charged + only when a *new* contact would be created — an existing contact costs nothing, + because attaching to somebody already in the CRM cannot flood it — and **a create + that loses the race hands its slot back**, so duplicate delivery of one form + cannot eat the hour's quota for a contact that was only ever made once. +- **The email race is handled, not hoped away.** Two submissions for one address + land at once, one `create` loses on the unique index, and the loser attaches to + the contact that won. Left as a raw `P2002` it becomes a stored submission that is + never filed and never retried. +- **`attach` claims the row before it writes anything.** It is an + `updateMany` on `filedAt: null`, and a caller that loses the claim returns without + writing — otherwise two concurrent deliveries of one submission each leave the + contact a `Submitted a form on …` note. +- **Duplicate delivery retries an unfiled row.** `dedupeKey` collapses a resend + inside one minute, but if the first attempt died before it filed — no `filedAt`, + no `skipReason` — the resend is the retry. +- **The agent is told, never asked.** `agent.contactCreated` writes an `AgentTask`. + No enrichment, no scoring and no identity matching happens here; see `docs/api.md`. + +## Retention + +`POST /internal/tracking/retention`, nightly at 04:00 via `apps/api/vercel.json`, +`CRON_SECRET` or nothing. + +- **The cutoff is a whole UTC day**, `EVENT_RETENTION_DAYS` back and then truncated. + A mid-day cutoff splits one calendar day across two nightly runs, and + `trackedPageDaily` keeps the larger half of a day it saw twice — so the boundary + day is quietly under-reported forever. Roll whole days or do not roll. +- **Roll before you delete.** `TrackingRollupService` aggregates `page_view` rows + only; a click is not a page view, and counting visitors over both produces a row + reading `views = 0, visitors = 3`. +- **Deleting is batched and bounded** — `SWEEP_BATCH` × `MAX_SWEEP_PASSES`. A sweep + that hits the ceiling **says so**, in the log and in the response, because + silently leaving events behind reads exactly like having deleted them. +- **A visitor outlives their events only if a contact points at them.** An + anonymous visitor with nothing left is removed. + +## Settings, and who may change them + +Settings → Tracking & Analytics. `canManageTracking` (`@crm/auth`) gates every +mutation **in the service**, and the same flag disables the control — the button and +the 403 cannot disagree, as everywhere else. + +`tracking.sources` is manager-only too, so the page must not prefetch it for +everybody: a member's render would fire a request that can only be refused. + +- **Verify reports on the page it actually fetched.** `safeFetch` follows + redirects, so the host in the result comes from `fetched.url`, not from what the + rep typed — otherwise `acme.com` reports the allow-list status of a page that + lives on `www.acme.com`. +- **Rotating the site id is the kill switch for a stolen snippet.** The old id stops + resolving at `forSite` within the cache TTL. +- **The compiled config is cached for five minutes and invalidated on every write.** + Two things guard the cache, because one is not enough. `invalidate()` bumps a + **per-process generation** before it deletes, which stops this replica's own + in-flight read putting the pre-pause config back. That counter means nothing to a + second replica, so nothing is written to the cache unless its hash still equals + `AppSetting.trackingConfigHash` — **the shared version, in the database, that + every replica can see**. A read that raced a change computes the old hash, finds + it no longer current, and declines to cache it. Pausing sets the column to + `null`, so while tracking is paused **no config can validate and none is cached + at all**. + +## Where it lives + +| | | +| --- | --- | +| `packages/db/src/tracking.ts` | Every constant and every pure helper both sides share. **The single source of truth** — a literal copied out of here is a future divergence | +| `packages/db/src/attribution.ts` | `classifyTouch` and the source tables. No imports, no database | +| `apps/app/lib/tracking/` | The loader and the tracker source | +| `apps/app/app/t/` | The two public routes | +| `apps/api/src/tracking/` | Collector, config cache, ingest, filing, counters, rollup, retention, tRPC router | +| `apps/app/app/(app)/[slug]/settings/tracking/` | The settings page | +| `apps/app/components/crm/website-activity.tsx` | The record-sheet section, which renders its own heading so it can render nothing at all | diff --git a/package.json b/package.json index b8026d65..b1fcf24f 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "crm", "private": true, "license": "MIT", - "version": "1.5.0", + "version": "1.5.1", "scripts": { "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true", "build": "turbo run build", @@ -10,7 +10,7 @@ "lint": "turbo run lint", "format": "biome format --write .", "check-types": "turbo run check-types", - "test": "turbo run test", + "test": "turbo run test --concurrency=1", "auth:generate": "turbo run auth:generate", "db:deploy": "turbo run db:deploy", "db:generate": "turbo run db:generate", diff --git a/packages/auth/src/index.ts b/packages/auth/src/index.ts index f66f443f..838b3002 100644 --- a/packages/auth/src/index.ts +++ b/packages/auth/src/index.ts @@ -1,9 +1,10 @@ export { type Auth, auth, type Session, type SessionUser } from "./auth"; export { AUTH_COOKIE_PREFIX } from "./cookies"; -export { isGoogleConfigured, isMicrosoftConfigured } from "./env"; +export { appUrl, isGoogleConfigured, isMicrosoftConfigured } from "./env"; export { canChangeRole, canManageCurrency, + canManageTracking, canRenameWorkspace, DEFAULT_WORKSPACE_NAME, ensureWorkspaceMembership, diff --git a/packages/auth/src/organization.ts b/packages/auth/src/organization.ts index 5ed81a75..100bad85 100644 --- a/packages/auth/src/organization.ts +++ b/packages/auth/src/organization.ts @@ -29,6 +29,10 @@ export function canManageCurrency(role: WorkspaceRole | null): boolean { return isWorkspaceAdmin(role); } +export function canManageTracking(role: WorkspaceRole | null): boolean { + return isWorkspaceAdmin(role); +} + export async function ensureWorkspaceMembership( userId: string, ): Promise { diff --git a/packages/db/package.json b/packages/db/package.json index e4d45c34..036f1903 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -6,6 +6,7 @@ "exports": { ".": "./src/index.ts", "./agent-tasks": "./src/agent-tasks.ts", + "./attribution": "./src/attribution.ts", "./blob": "./src/blob.ts", "./client": "./src/client.ts", "./currency": "./src/currency.ts", @@ -19,6 +20,7 @@ "./idempotency": "./src/idempotency.ts", "./safe-fetch": "./src/safe-fetch.ts", "./settings": "./src/settings.ts", + "./tracking": "./src/tracking.ts", "./workspace": "./src/workspace.ts" }, "scripts": { diff --git a/packages/db/prisma/migrations/20260810120000_website_tracking/migration.sql b/packages/db/prisma/migrations/20260810120000_website_tracking/migration.sql new file mode 100644 index 00000000..6f3c3344 --- /dev/null +++ b/packages/db/prisma/migrations/20260810120000_website_tracking/migration.sql @@ -0,0 +1,93 @@ +-- AlterEnum +ALTER TYPE "RecordSource" ADD VALUE 'TRACKING'; + +-- CreateEnum +CREATE TYPE "DomainScope" AS ENUM ('SITE_AND_SUBDOMAINS', 'EXACT_HOST'); + +-- AlterTable +ALTER TABLE "appSetting" ADD COLUMN "trackingSiteId" TEXT; +ALTER TABLE "appSetting" ADD COLUMN "trackingCrossDomain" BOOLEAN NOT NULL DEFAULT true; +ALTER TABLE "appSetting" ADD COLUMN "trackingLimitToDomains" BOOLEAN NOT NULL DEFAULT true; +ALTER TABLE "appSetting" ADD COLUMN "trackingCookieSubdomains" BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE "appSetting" ADD COLUMN "trackingSecureCookies" BOOLEAN NOT NULL DEFAULT true; +ALTER TABLE "appSetting" ADD COLUMN "trackingHonourDnt" BOOLEAN NOT NULL DEFAULT true; +ALTER TABLE "appSetting" ADD COLUMN "trackingCookieDays" INTEGER NOT NULL DEFAULT 395; +ALTER TABLE "appSetting" ADD COLUMN "trackingConfigHash" TEXT; + +-- CreateIndex +CREATE UNIQUE INDEX "appSetting_trackingSiteId_key" ON "appSetting"("trackingSiteId"); + +-- CreateTable +CREATE TABLE "trackedDomain" ( + "id" TEXT NOT NULL, + "host" TEXT NOT NULL, + "scope" "DomainScope" NOT NULL DEFAULT 'EXACT_HOST', + "pageViews" INTEGER NOT NULL DEFAULT 0, + "lastSeenAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "trackedDomain_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "trackedDomain_host_key" ON "trackedDomain"("host"); + +-- CreateTable +CREATE TABLE "trackedVisitor" ( + "id" TEXT NOT NULL, + "contactId" TEXT, + "firstSeen" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "lastSeen" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "trackedVisitor_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "trackedVisitor_contactId_idx" ON "trackedVisitor"("contactId"); + +-- AddForeignKey +ALTER TABLE "trackedVisitor" ADD CONSTRAINT "trackedVisitor_contactId_fkey" FOREIGN KEY ("contactId") REFERENCES "contact"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- CreateTable +CREATE TABLE "trackedEvent" ( + "id" TEXT NOT NULL, + "visitorId" TEXT NOT NULL, + "type" TEXT NOT NULL, + "host" TEXT NOT NULL, + "path" TEXT NOT NULL, + "referrer" TEXT, + "label" TEXT, + "occurredAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "trackedEvent_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "trackedEvent_visitorId_occurredAt_idx" ON "trackedEvent"("visitorId", "occurredAt"); +CREATE INDEX "trackedEvent_occurredAt_idx" ON "trackedEvent"("occurredAt"); +CREATE INDEX "trackedEvent_host_occurredAt_idx" ON "trackedEvent"("host", "occurredAt"); + +-- CreateTable +CREATE TABLE "formSubmission" ( + "id" TEXT NOT NULL, + "visitorId" TEXT, + "contactId" TEXT, + "host" TEXT NOT NULL, + "path" TEXT NOT NULL, + "email" TEXT, + "fields" JSONB NOT NULL, + "dedupeKey" TEXT NOT NULL, + "filedAt" TIMESTAMP(3), + "skipReason" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "formSubmission_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "formSubmission_dedupeKey_key" ON "formSubmission"("dedupeKey"); +CREATE INDEX "formSubmission_contactId_idx" ON "formSubmission"("contactId"); +CREATE INDEX "formSubmission_createdAt_idx" ON "formSubmission"("createdAt"); + +-- AddForeignKey +ALTER TABLE "formSubmission" ADD CONSTRAINT "formSubmission_contactId_fkey" FOREIGN KEY ("contactId") REFERENCES "contact"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20260810130000_tracking_hardening/migration.sql b/packages/db/prisma/migrations/20260810130000_tracking_hardening/migration.sql new file mode 100644 index 00000000..d419e634 --- /dev/null +++ b/packages/db/prisma/migrations/20260810130000_tracking_hardening/migration.sql @@ -0,0 +1,28 @@ +-- AlterTable +ALTER TABLE "appSetting" ADD COLUMN "trackingPaused" BOOLEAN NOT NULL DEFAULT false; + +-- CreateTable +CREATE TABLE "trackingCounter" ( + "key" TEXT NOT NULL, + "value" INTEGER NOT NULL DEFAULT 0, + "expiresAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "trackingCounter_pkey" PRIMARY KEY ("key") +); + +-- CreateIndex +CREATE INDEX "trackingCounter_expiresAt_idx" ON "trackingCounter"("expiresAt"); + +-- CreateTable +CREATE TABLE "trackedPageDaily" ( + "day" TIMESTAMP(3) NOT NULL, + "host" TEXT NOT NULL, + "path" TEXT NOT NULL, + "views" INTEGER NOT NULL DEFAULT 0, + "visitors" INTEGER NOT NULL DEFAULT 0, + + CONSTRAINT "trackedPageDaily_pkey" PRIMARY KEY ("day","host","path") +); + +-- CreateIndex +CREATE INDEX "trackedPageDaily_host_day_idx" ON "trackedPageDaily"("host", "day"); diff --git a/packages/db/prisma/migrations/20260810140000_source_attribution/migration.sql b/packages/db/prisma/migrations/20260810140000_source_attribution/migration.sql new file mode 100644 index 00000000..52e9433c --- /dev/null +++ b/packages/db/prisma/migrations/20260810140000_source_attribution/migration.sql @@ -0,0 +1,32 @@ +-- AlterTable +ALTER TABLE "trackedEvent" ADD COLUMN "source" TEXT; +ALTER TABLE "trackedEvent" ADD COLUMN "medium" TEXT; +ALTER TABLE "trackedEvent" ADD COLUMN "campaign" TEXT; + +-- CreateIndex +CREATE INDEX "trackedEvent_source_occurredAt_idx" ON "trackedEvent"("source", "occurredAt"); + +-- AlterTable +ALTER TABLE "formSubmission" ADD COLUMN "firstTouch" JSONB; +ALTER TABLE "formSubmission" ADD COLUMN "lastTouch" JSONB; + +-- AlterTable +ALTER TABLE "trackedVisitor" ADD COLUMN "firstSource" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "firstMedium" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "firstCampaign" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "firstTerm" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "firstContent" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "firstReferrer" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "firstLanding" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "firstTouchAt" TIMESTAMP(3); +ALTER TABLE "trackedVisitor" ADD COLUMN "lastSource" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "lastMedium" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "lastCampaign" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "lastTerm" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "lastContent" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "lastReferrer" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "lastLanding" TEXT; +ALTER TABLE "trackedVisitor" ADD COLUMN "lastTouchAt" TIMESTAMP(3); + +-- CreateIndex +CREATE INDEX "trackedVisitor_firstSource_idx" ON "trackedVisitor"("firstSource"); diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index b2705a0a..8807af31 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -143,6 +143,7 @@ enum RecordSource { IMPORT EMAIL CALENDAR + TRACKING } enum AgentConversationKind { @@ -325,6 +326,8 @@ model Contact { calendarEvents CalendarEvent[] eventAttendance CalendarAttendee[] fieldValues FieldValue[] + visitors TrackedVisitor[] @relation("VisitorContact") + submissions FormSubmission[] @relation("SubmissionContact") createdAt DateTime @default(now()) updatedAt DateTime @updatedAt @@ -1205,11 +1208,142 @@ model AppSetting { ratesRefreshedAt DateTime? + trackingSiteId String? @unique + trackingCrossDomain Boolean @default(true) + trackingLimitToDomains Boolean @default(true) + trackingCookieSubdomains Boolean @default(false) + trackingSecureCookies Boolean @default(true) + trackingHonourDnt Boolean @default(true) + trackingCookieDays Int @default(395) + trackingConfigHash String? + trackingPaused Boolean @default(false) + updatedAt DateTime @updatedAt @@map("appSetting") } +enum DomainScope { + SITE_AND_SUBDOMAINS + EXACT_HOST +} + +model TrackedDomain { + id String @id @default(cuid()) + host String @unique + scope DomainScope @default(EXACT_HOST) + + pageViews Int @default(0) + lastSeenAt DateTime? + + createdAt DateTime @default(now()) + + @@map("trackedDomain") +} + +model TrackedVisitor { + id String @id + + contactId String? + contact Contact? @relation("VisitorContact", fields: [contactId], references: [id], onDelete: SetNull) + + firstSource String? + firstMedium String? + firstCampaign String? + firstTerm String? + firstContent String? + firstReferrer String? + firstLanding String? + firstTouchAt DateTime? + + lastSource String? + lastMedium String? + lastCampaign String? + lastTerm String? + lastContent String? + lastReferrer String? + lastLanding String? + lastTouchAt DateTime? + + firstSeen DateTime @default(now()) + lastSeen DateTime @updatedAt + + @@index([contactId]) + @@index([firstSource]) + @@map("trackedVisitor") +} + +model TrackedEvent { + id String @id @default(cuid()) + + visitorId String + type String + host String + path String + referrer String? + label String? + + source String? + medium String? + campaign String? + + occurredAt DateTime + + @@index([visitorId, occurredAt]) + @@index([occurredAt]) + @@index([host, occurredAt]) + @@index([source, occurredAt]) + @@map("trackedEvent") +} + +model TrackingCounter { + key String @id + value Int @default(0) + expiresAt DateTime + + @@index([expiresAt]) + @@map("trackingCounter") +} + +model TrackedPageDaily { + day DateTime + host String + path String + views Int @default(0) + visitors Int @default(0) + + @@id([day, host, path]) + @@index([host, day]) + @@map("trackedPageDaily") +} + +model FormSubmission { + id String @id @default(cuid()) + + visitorId String? + contactId String? + contact Contact? @relation("SubmissionContact", fields: [contactId], references: [id], onDelete: SetNull) + + host String + path String + email String? + + fields Json + + firstTouch Json? + lastTouch Json? + + dedupeKey String @unique + filedAt DateTime? + skipReason String? + + createdAt DateTime @default(now()) + + @@index([contactId]) + @@index([createdAt]) + @@map("formSubmission") +} + model Install { id String @id diff --git a/packages/db/src/attribution.ts b/packages/db/src/attribution.ts new file mode 100644 index 00000000..ba923005 --- /dev/null +++ b/packages/db/src/attribution.ts @@ -0,0 +1,244 @@ +export const COOKIE_FIRST_TOUCH = "_crm_fs"; + +export const MEDIUMS = [ + "organic", + "social", + "referral", + "email", + "cpc", + "direct", + "other", +] as const; + +export type Medium = (typeof MEDIUMS)[number]; + +export interface RawTouch { + source?: string; + medium?: string; + campaign?: string; + term?: string; + content?: string; + referrer?: string; + landing?: string; + at?: number; +} + +export interface Touch { + source: string; + medium: Medium; + campaign: string | null; + term: string | null; + content: string | null; + referrer: string | null; + landing: string | null; + at: Date; +} + +const SEARCH: Record = { + "google.": "Google", + "bing.": "Bing", + "duckduckgo.": "DuckDuckGo", + "yahoo.": "Yahoo", + "baidu.": "Baidu", + "yandex.": "Yandex", + "ecosia.": "Ecosia", + "startpage.": "Startpage", + "search.brave.": "Brave Search", +}; + +const SOCIAL: Record = { + "linkedin.": "LinkedIn", + "lnkd.in": "LinkedIn", + "twitter.": "X", + "x.com": "X", + "t.co": "X", + "facebook.": "Facebook", + "fb.com": "Facebook", + "instagram.": "Instagram", + "youtube.": "YouTube", + "youtu.be": "YouTube", + "reddit.": "Reddit", + "tiktok.": "TikTok", + "news.ycombinator.": "Hacker News", + "producthunt.": "Product Hunt", + "github.": "GitHub", + "slack.": "Slack", +}; + +const MAIL: Record = { + "mail.google.": "Gmail", + "outlook.": "Outlook", + "mail.yahoo.": "Yahoo Mail", +}; + +const SECOND_LEVEL = new Set([ + "co", + "com", + "net", + "org", + "gov", + "edu", + "ac", + "or", + "ne", +]); + +const PAID_MEDIUMS = new Set([ + "cpc", + "ppc", + "paid", + "paidsearch", + "paid_search", +]); + +const MAX = 120; + +export function classifyTouch(raw: RawTouch, at: Date = new Date()): Touch { + const when = raw.at ? new Date(raw.at) : at; + const campaign = clean(raw.campaign); + const term = clean(raw.term); + const content = clean(raw.content); + const referrer = clean(raw.referrer); + const landing = clean(raw.landing); + + const utmSource = clean(raw.source); + const utmMedium = clean(raw.medium)?.toLowerCase(); + + if (utmSource) { + return { + source: utmSource, + medium: mediumFrom(utmMedium), + campaign, + term, + content, + referrer, + landing, + at: valid(when) ? when : at, + }; + } + + const host = hostOf(referrer); + + if (!host) { + return { + source: "Direct", + medium: "direct", + campaign, + term, + content, + referrer: null, + landing, + at: valid(when) ? when : at, + }; + } + + const mail = match(host, MAIL); + const search = mail ? null : match(host, SEARCH); + const social = mail || search ? null : match(host, SOCIAL); + + const medium: Medium = mail + ? "email" + : search + ? "organic" + : social + ? "social" + : "referral"; + + return { + source: mail ?? search ?? social ?? host, + medium: utmMedium ? mediumFrom(utmMedium) : medium, + campaign, + term, + content, + referrer, + landing, + at: valid(when) ? when : at, + }; +} + +export function isSameTouch(a: Touch, b: Touch): boolean { + return ( + a.source === b.source && + a.medium === b.medium && + (a.campaign ?? "") === (b.campaign ?? "") + ); +} + +export function describeTouch(touch: { + source: string | null; + medium: string | null; + campaign: string | null; +}): string { + if (!touch.source) return "Unknown"; + + const medium = + touch.medium && touch.medium !== "direct" ? touch.medium : null; + const parts = [touch.source, medium, touch.campaign].filter(Boolean); + + return parts.join(" · "); +} + +function mediumFrom(value: string | null | undefined): Medium { + if (!value) return "other"; + if (PAID_MEDIUMS.has(value)) return "cpc"; + if ((MEDIUMS as readonly string[]).includes(value)) return value as Medium; + if (value.includes("mail")) return "email"; + if (value.includes("social")) return "social"; + if (value.includes("organic")) return "organic"; + + return "other"; +} + +function match(host: string, table: Record): string | null { + const labels = host.split("."); + + for (const [needle, name] of Object.entries(table)) { + if (matches(labels, needle)) return name; + } + + return null; +} + +function matches(labels: string[], needle: string): boolean { + const open = needle.endsWith("."); + const wanted = (open ? needle.slice(0, -1) : needle).split("."); + + for (let start = 0; start + wanted.length <= labels.length; start += 1) { + if (wanted.some((label, index) => labels[start + index] !== label)) + continue; + + const tail = labels.slice(start + wanted.length); + + if (open ? isSuffix(tail) : tail.length === 0) return true; + } + + return false; +} + +function isSuffix(tail: string[]): boolean { + if (tail.length === 1) return true; + if (tail.length !== 2) return false; + + return SECOND_LEVEL.has(tail[0] ?? "") && /^[a-z]{2}$/.test(tail[1] ?? ""); +} + +function hostOf(referrer: string | null): string | null { + if (!referrer) return null; + + try { + return new URL(referrer).hostname.toLowerCase().replace(/^www\./, ""); + } catch { + return null; + } +} + +function clean(value: string | null | undefined): string | null { + const trimmed = value?.trim(); + if (!trimmed) return null; + + return trimmed.length > MAX ? trimmed.slice(0, MAX) : trimmed; +} + +function valid(date: Date): boolean { + return !Number.isNaN(date.getTime()); +} diff --git a/packages/db/src/tracking.ts b/packages/db/src/tracking.ts new file mode 100644 index 00000000..8b9cd2bc --- /dev/null +++ b/packages/db/src/tracking.ts @@ -0,0 +1,256 @@ +import { createHash, randomBytes } from "node:crypto"; +import type { Db } from "./client"; +import { SETTINGS_ID } from "./settings"; + +export const SITE_ID_PREFIX = "cmp_"; + +export const COOKIE_NAME = "_crm_v"; + +export const COOKIE_LIFETIMES = [ + { days: 395, label: "13 months" }, + { days: 180, label: "6 months" }, + { days: 0, label: "Session only" }, +] as const; + +export const MAX_EVENTS_PER_BATCH = 20; + +export const MAX_BODY_BYTES = 32_768; + +export const EVENTS_PER_MINUTE = 600; + +export const CONTACTS_PER_HOUR = 50; + +export const CONTACT_CAP_REASON = "Hourly contact cap reached — not filed"; + +export const LOADER_MAX_AGE_SECONDS = 600; + +export const CONFIG_MAX_AGE_SECONDS = 300; + +export const BUNDLE_MAX_AGE_SECONDS = 31_536_000; + +export const EVENT_RETENTION_DAYS = 90; + +export const VERIFY_WINDOW_MS = 5 * 60_000; + +export type DomainScopeValue = "SITE_AND_SUBDOMAINS" | "EXACT_HOST"; + +export interface TrackedHost { + host: string; + scope: DomainScopeValue; +} + +export interface TrackingConfig { + siteId: string; + crossDomain: boolean; + limitToDomains: boolean; + cookieSubdomains: boolean; + secureCookies: boolean; + honourDnt: boolean; + cookieDays: number; + hosts: TrackedHost[]; +} + +export const LINKER_MAX_AGE_SECONDS = 120; + +export function rateWindowKey(at: Date = new Date()): string { + return `rate:${Math.floor(at.getTime() / 60_000)}`; +} + +export function contactWindowKey(at: Date = new Date()): string { + return `contacts:${Math.floor(at.getTime() / 3_600_000)}`; +} + +export function windowExpiry(key: string, at: Date = new Date()): Date { + const span = key.startsWith("rate:") ? 60_000 : 3_600_000; + const bucket = Number(key.split(":")[1]); + + if (!Number.isFinite(bucket)) return new Date(at.getTime() + span * 2); + + return new Date((bucket + 1) * span + span); +} + +export function trackingReady( + limitToDomains: boolean, + domainCount: number, +): boolean { + return !limitToDomains || domainCount > 0; +} + +export function mintSiteId(): string { + return `${SITE_ID_PREFIX}${randomBytes(4).toString("hex")}`; +} + +export function isSiteId(value: string | null | undefined): value is string { + return typeof value === "string" && /^cmp_[0-9a-f]{8}$/.test(value); +} + +export function loaderUrl(appUrl: string): string { + return `${appUrl.replace(/\/+$/, "")}/t/crm.js`; +} + +export function trackingSnippet(appUrl: string, siteId: string): string { + return ``; +} + +export function configHash(config: TrackingConfig): string { + const canonical = JSON.stringify({ + siteId: config.siteId, + crossDomain: config.crossDomain, + limitToDomains: config.limitToDomains, + cookieSubdomains: config.cookieSubdomains, + secureCookies: config.secureCookies, + honourDnt: config.honourDnt, + cookieDays: config.cookieDays, + hosts: [...config.hosts] + .map((entry) => `${entry.host}:${entry.scope}`) + .sort(), + }); + + return createHash("sha256").update(canonical).digest("hex").slice(0, 12); +} + +export function isConfigHash( + value: string | null | undefined, +): value is string { + return typeof value === "string" && /^[0-9a-f]{12}$/.test(value); +} + +export function normalizeHost(input: string | null | undefined): string | null { + const trimmed = input?.trim().toLowerCase(); + if (!trimmed) return null; + + const withScheme = /^[a-z][a-z0-9+.-]*:\/\//.test(trimmed) + ? trimmed + : `https://${trimmed}`; + + let host: string; + try { + host = new URL(withScheme).hostname; + } catch { + return null; + } + + const bare = host.replace(/\.$/, ""); + if (!bare.includes(".")) return null; + if (!/^[a-z0-9.-]+$/.test(bare)) return null; + if (bare.startsWith(".") || bare.includes("..")) return null; + + return bare; +} + +export function normalizePath(input: string | null | undefined): string { + const raw = input?.trim(); + if (!raw) return "/"; + + const path = raw.split(/[?#]/)[0] ?? "/"; + if (!path.startsWith("/")) return "/"; + + const trimmed = path.length > 1 ? path.replace(/\/+$/, "") : path; + + return trimmed === "" ? "/" : trimmed; +} + +export function stripQuery(input: string | null | undefined): string | null { + const raw = input?.trim(); + if (!raw) return null; + + const cut = raw.split(/[?#]/)[0] ?? ""; + + return cut === "" ? null : cut; +} + +export function matchedHost( + host: string, + config: TrackingConfig, +): TrackedHost | null { + const candidate = host.toLowerCase(); + + const exact = config.hosts.find((entry) => candidate === entry.host); + if (exact) return exact; + + return ( + config.hosts.find( + (entry) => + entry.scope === "SITE_AND_SUBDOMAINS" && + candidate.endsWith(`.${entry.host}`), + ) ?? null + ); +} + +export function hostAllowed(host: string, config: TrackingConfig): boolean { + if (!config.limitToDomains) return true; + + return matchedHost(host, config) !== null; +} + +export function originAllowed( + origin: string | null | undefined, + config: TrackingConfig, +): boolean { + if (!origin) return false; + + let host: string; + try { + host = new URL(origin).hostname.toLowerCase(); + } catch { + return false; + } + + if (!config.limitToDomains) return true; + + return hostAllowed(host, config); +} + +export function dedupeKey(parts: { + host: string; + path: string; + email: string | null; + at: Date; +}): string { + const minute = Math.floor(parts.at.getTime() / 60_000); + + return createHash("sha256") + .update(`${parts.host}|${parts.path}|${parts.email ?? ""}|${minute}`) + .digest("hex"); +} + +export async function readTrackingConfig( + db: Db, +): Promise { + const [settings, domains] = await Promise.all([ + db.appSetting.findUnique({ + where: { id: SETTINGS_ID }, + select: { + trackingSiteId: true, + trackingCrossDomain: true, + trackingLimitToDomains: true, + trackingCookieSubdomains: true, + trackingSecureCookies: true, + trackingHonourDnt: true, + trackingCookieDays: true, + trackingPaused: true, + }, + }), + db.trackedDomain.findMany({ + select: { host: true, scope: true }, + orderBy: { host: "asc" }, + }), + ]); + + if (!isSiteId(settings?.trackingSiteId)) return null; + if (settings.trackingPaused) return null; + + return { + siteId: settings.trackingSiteId, + crossDomain: settings.trackingCrossDomain, + limitToDomains: settings.trackingLimitToDomains, + cookieSubdomains: settings.trackingCookieSubdomains, + secureCookies: settings.trackingSecureCookies, + honourDnt: settings.trackingHonourDnt, + cookieDays: settings.trackingCookieDays, + hosts: domains.map((entry) => ({ + host: entry.host, + scope: entry.scope as DomainScopeValue, + })), + }; +} diff --git a/packages/db/test/attribution.spec.ts b/packages/db/test/attribution.spec.ts new file mode 100644 index 00000000..7e218099 --- /dev/null +++ b/packages/db/test/attribution.spec.ts @@ -0,0 +1,174 @@ +import { describe, expect, test } from "bun:test"; +import { classifyTouch, describeTouch } from "../src/attribution"; + +const AT = new Date("2026-08-10T12:00:00.000Z"); + +describe("a touch with utm parameters", () => { + test("takes the campaign the marketer named, verbatim", () => { + const touch = classifyTouch( + { + source: "newsletter", + medium: "email", + campaign: "august-launch", + term: "crm", + content: "banner", + landing: "/pricing", + }, + AT, + ); + + expect(touch.source).toBe("newsletter"); + expect(touch.medium).toBe("email"); + expect(touch.campaign).toBe("august-launch"); + expect(touch.term).toBe("crm"); + expect(touch.content).toBe("banner"); + expect(touch.landing).toBe("/pricing"); + }); + + test("beats the referrer, because the marketer was explicit", () => { + const touch = classifyTouch( + { source: "partner-blog", referrer: "https://www.google.com/" }, + AT, + ); + + expect(touch.source).toBe("partner-blog"); + }); + + test("folds the paid aliases onto one medium", () => { + for (const medium of ["cpc", "ppc", "paid", "paidsearch", "paid_search"]) { + expect(classifyTouch({ source: "google", medium }, AT).medium).toBe( + "cpc", + ); + } + }); + + test("keeps an unknown medium out of the vocabulary", () => { + expect( + classifyTouch({ source: "x", medium: "carrier-pigeon" }, AT).medium, + ).toBe("other"); + }); +}); + +describe("a touch with only a referrer", () => { + test("names the search engine and calls it organic", () => { + const touch = classifyTouch( + { referrer: "https://www.google.co.uk/search?q=crm" }, + AT, + ); + + expect(touch.source).toBe("Google"); + expect(touch.medium).toBe("organic"); + }); + + test("names the social network and calls it social", () => { + expect(classifyTouch({ referrer: "https://t.co/abc" }, AT).source).toBe( + "X", + ); + expect(classifyTouch({ referrer: "https://lnkd.in/abc" }, AT).medium).toBe( + "social", + ); + }); + + test("calls a webmail host email, not the search engine it shares a name with", () => { + const gmail = classifyTouch({ referrer: "https://mail.google.com/" }, AT); + + expect(gmail.source).toBe("Gmail"); + expect(gmail.medium).toBe("email"); + + const yahoo = classifyTouch({ referrer: "https://mail.yahoo.com/" }, AT); + + expect(yahoo.source).toBe("Yahoo Mail"); + expect(yahoo.medium).toBe("email"); + }); + + test("never trusts a host that merely contains a provider's name", () => { + for (const referrer of [ + "https://notgoogle.com/", + "https://evilfacebook.com/", + "https://google.com.phish.example/", + ]) { + expect(classifyTouch({ referrer }, AT).medium).toBe("referral"); + } + }); + + test("never trusts a provider's name used as somebody else's subdomain", () => { + for (const referrer of [ + "https://google.evil.com/", + "https://mail.google.evil.com/", + "https://facebook.attacker.net/", + "https://outlook.phish.example/", + "https://google.com.example/", + "https://google.co.example/", + ]) { + expect(classifyTouch({ referrer }, AT).medium).toBe("referral"); + } + }); + + test("still recognises a provider on a subdomain or a country domain", () => { + for (const [referrer, source] of [ + ["https://images.google.de/", "Google"], + ["https://scholar.google.co.jp/", "Google"], + ["https://m.facebook.com/", "Facebook"], + ["https://uk.search.yahoo.com/", "Yahoo"], + ["https://news.ycombinator.com/", "Hacker News"], + ["https://gist.github.com/", "GitHub"], + ["https://t.co/abc", "X"], + ["https://youtu.be/abc", "YouTube"], + ] as const) { + expect(classifyTouch({ referrer }, AT).source).toBe(source); + } + }); + + test("falls back to the bare host for anything else", () => { + const touch = classifyTouch({ referrer: "https://www.acme.com/blog" }, AT); + + expect(touch.source).toBe("acme.com"); + expect(touch.medium).toBe("referral"); + }); + + test("is direct when there is no referrer at all", () => { + const touch = classifyTouch({ landing: "/" }, AT); + + expect(touch.source).toBe("Direct"); + expect(touch.medium).toBe("direct"); + expect(touch.referrer).toBeNull(); + }); + + test("is direct when the referrer is not a URL", () => { + expect(classifyTouch({ referrer: "android-app" }, AT).medium).toBe( + "direct", + ); + }); +}); + +describe("reading a touch back", () => { + test("reads as source, medium and campaign", () => { + expect( + describeTouch({ + source: "Google", + medium: "organic", + campaign: null, + }), + ).toBe("Google · organic"); + + expect( + describeTouch({ + source: "newsletter", + medium: "email", + campaign: "august-launch", + }), + ).toBe("newsletter · email · august-launch"); + }); + + test("does not repeat itself for direct traffic", () => { + expect( + describeTouch({ source: "Direct", medium: "direct", campaign: null }), + ).toBe("Direct"); + }); + + test("says so when there is nothing to say", () => { + expect(describeTouch({ source: null, medium: null, campaign: null })).toBe( + "Unknown", + ); + }); +}); diff --git a/packages/db/test/tracking.spec.ts b/packages/db/test/tracking.spec.ts new file mode 100644 index 00000000..e6806551 --- /dev/null +++ b/packages/db/test/tracking.spec.ts @@ -0,0 +1,241 @@ +import { describe, expect, test } from "bun:test"; +import { + configHash, + dedupeKey, + hostAllowed, + isSiteId, + loaderUrl, + matchedHost, + mintSiteId, + normalizeHost, + normalizePath, + originAllowed, + stripQuery, + type TrackingConfig, + trackingReady, + trackingSnippet, +} from "../src/tracking"; + +const CONFIG: TrackingConfig = { + siteId: "cmp_8f3ad91c", + crossDomain: true, + limitToDomains: true, + cookieSubdomains: false, + secureCookies: true, + honourDnt: true, + cookieDays: 395, + hosts: [ + { host: "trycomp.ai", scope: "SITE_AND_SUBDOMAINS" }, + { host: "shop.example.com", scope: "EXACT_HOST" }, + ], +}; + +describe("a site id", () => { + test("is minted in the shape the loader checks for", () => { + expect(isSiteId(mintSiteId())).toBe(true); + }); + + test("rejects anything else", () => { + for (const value of ["cmp_", "cmp_ZZZZZZZZ", "8f3ad91c", "", null]) { + expect(isSiteId(value)).toBe(false); + } + }); +}); + +describe("the snippet a rep copies", () => { + const value = trackingSnippet("http://localhost:3000", "cmp_6e9356c9"); + + test("is one line, so no paste target can eat the newlines", () => { + expect(value).not.toInclude("\n"); + }); + + test("keeps a real space between every attribute", () => { + expect(value).not.toInclude("scriptsrc"); + expect(value).not.toInclude("asyncdefer"); + expect(value).toBe( + '', + ); + }); + + test("survives a paste target that trims every line", () => { + const trimmed = value + .split("\n") + .map((line) => line.trim()) + .join(""); + + expect(trimmed).toBe(value); + }); + + test("does not double the slash on a trailing-slash APP_URL", () => { + expect(loaderUrl("https://crm.example.com/")).toBe( + "https://crm.example.com/t/crm.js", + ); + }); +}); + +describe("whether there is anything to install yet", () => { + test("is not ready with the limit on and no domains", () => { + expect(trackingReady(true, 0)).toBe(false); + }); + + test("is ready as soon as one domain exists", () => { + expect(trackingReady(true, 1)).toBe(true); + }); + + test("is ready with the limit off, because every host is allowed", () => { + expect(trackingReady(false, 0)).toBe(true); + }); +}); + +describe("a domain", () => { + test("is reduced to its host", () => { + expect(normalizeHost("https://Acme.com/pricing?a=1")).toBe("acme.com"); + expect(normalizeHost(" acme.com ")).toBe("acme.com"); + expect(normalizeHost("acme.com.")).toBe("acme.com"); + }); + + test("is refused when it is not one", () => { + for (const value of ["localhost", "acme", "", " ", "..", "a b.com"]) { + expect(normalizeHost(value)).toBeNull(); + } + }); +}); + +describe("a page path", () => { + test("drops the query and the fragment so one page is one row", () => { + expect(normalizePath("/?utm_source=google&utm_medium=cpc")).toBe("/"); + expect(normalizePath("/pricing?plan=team")).toBe("/pricing"); + expect(normalizePath("/docs/api#install")).toBe("/docs/api"); + }); + + test("drops a trailing slash so /pricing and /pricing/ are one page", () => { + expect(normalizePath("/pricing/")).toBe("/pricing"); + expect(normalizePath("/")).toBe("/"); + }); + + test("falls back to the root when it is not a path", () => { + for (const value of ["", " ", null, undefined, "pricing", "//"]) { + expect(normalizePath(value)).toBe("/"); + } + }); +}); + +describe("the allow-list", () => { + test("covers subdomains only where the scope says so", () => { + expect(hostAllowed("trycomp.ai", CONFIG)).toBe(true); + expect(hostAllowed("docs.trycomp.ai", CONFIG)).toBe(true); + expect(hostAllowed("shop.example.com", CONFIG)).toBe(true); + expect(hostAllowed("www.shop.example.com", CONFIG)).toBe(false); + }); + + test("never matches a host that merely ends the same way", () => { + expect(hostAllowed("nottrycomp.ai", CONFIG)).toBe(false); + expect(hostAllowed("trycomp.ai.evil.com", CONFIG)).toBe(false); + }); + + test("lets everything through when the limit is off", () => { + const open = { ...CONFIG, limitToDomains: false }; + + expect(hostAllowed("anything.example", open)).toBe(true); + }); +}); + +describe("the domain an event belongs to", () => { + test("is the parent when the scope covers subdomains", () => { + expect(matchedHost("docs.trycomp.ai", CONFIG)?.host).toBe("trycomp.ai"); + expect(matchedHost("trycomp.ai", CONFIG)?.host).toBe("trycomp.ai"); + }); + + test("is the exact row when that is the whole of the scope", () => { + expect(matchedHost("shop.example.com", CONFIG)?.host).toBe( + "shop.example.com", + ); + expect(matchedHost("www.shop.example.com", CONFIG)).toBeNull(); + }); + + test("is nothing for a host nobody configured", () => { + expect(matchedHost("elsewhere.example", CONFIG)).toBeNull(); + }); + + test("prefers the host's own row to the parent that also covers it", () => { + const both: TrackingConfig = { + ...CONFIG, + hosts: [ + { host: "trycomp.ai", scope: "SITE_AND_SUBDOMAINS" }, + { host: "docs.trycomp.ai", scope: "EXACT_HOST" }, + ], + }; + + expect(matchedHost("docs.trycomp.ai", both)?.host).toBe("docs.trycomp.ai"); + expect(matchedHost("blog.trycomp.ai", both)?.host).toBe("trycomp.ai"); + }); +}); + +describe("a stored referrer", () => { + test("keeps the page but never the query a secret could sit in", () => { + expect(stripQuery("https://acme.com/reset?token=abc#x")).toBe( + "https://acme.com/reset", + ); + expect(stripQuery("https://acme.com/blog")).toBe("https://acme.com/blog"); + }); + + test("is nothing when there was nothing but a query", () => { + for (const value of ["", " ", null, undefined, "?token=abc"]) { + expect(stripQuery(value)).toBeNull(); + } + }); +}); + +describe("the origin check", () => { + test("accepts an allowed origin and refuses the rest", () => { + expect(originAllowed("https://docs.trycomp.ai", CONFIG)).toBe(true); + expect(originAllowed("https://evil.example", CONFIG)).toBe(false); + }); + + test("refuses a missing or unparseable origin", () => { + expect(originAllowed(null, CONFIG)).toBe(false); + expect(originAllowed("not-a-url", CONFIG)).toBe(false); + }); + + test("refuses a missing origin even with the limit off", () => { + const open = { ...CONFIG, limitToDomains: false }; + + expect(originAllowed(null, open)).toBe(false); + }); +}); + +describe("the config hash", () => { + test("does not move when only the order of the domains does", () => { + const reordered = { ...CONFIG, hosts: [...CONFIG.hosts].reverse() }; + + expect(configHash(reordered)).toBe(configHash(CONFIG)); + }); + + test("moves when a setting does", () => { + expect(configHash({ ...CONFIG, secureCookies: false })).not.toBe( + configHash(CONFIG), + ); + }); +}); + +describe("the submission dedupe key", () => { + test("collapses the same form inside one minute", () => { + const at = new Date("2026-08-10T12:00:10.000Z"); + const later = new Date("2026-08-10T12:00:50.000Z"); + const parts = { host: "trycomp.ai", path: "/pricing", email: "a@b.com" }; + + expect(dedupeKey({ ...parts, at })).toBe( + dedupeKey({ ...parts, at: later }), + ); + }); + + test("separates a different address on the same page", () => { + const at = new Date("2026-08-10T12:00:10.000Z"); + + expect( + dedupeKey({ host: "trycomp.ai", path: "/p", email: "a@b.com", at }), + ).not.toBe( + dedupeKey({ host: "trycomp.ai", path: "/p", email: "c@d.com", at }), + ); + }); +}); diff --git a/packages/telemetry/src/allowlist.ts b/packages/telemetry/src/allowlist.ts index 007ae6a7..1c6d7492 100644 --- a/packages/telemetry/src/allowlist.ts +++ b/packages/telemetry/src/allowlist.ts @@ -20,6 +20,7 @@ export const ALLOWED_PROPERTIES = [ "cap_ai_gateway", "cap_google_oauth", "cap_sso_provider", + "cap_tracking", "is_marketing", "agent_model_id", "agent_model_context_window", @@ -66,6 +67,12 @@ export const ALLOWED_PROPERTIES = [ "enrichment_by_status", "suppressed_domains", "suppressed_contacts", + "tracking_domains", + "tracking_page_views", + "tracking_forms", + "tracking_contacts_created", + "tracking_capped", + "tracking_paused", "workspace_profile_written", "error_class", diff --git a/packages/ui/src/styles/globals.css b/packages/ui/src/styles/globals.css index 173dc527..3b312007 100644 --- a/packages/ui/src/styles/globals.css +++ b/packages/ui/src/styles/globals.css @@ -46,6 +46,9 @@ --severity-low: var(--muted-foreground); --severity-info: oklch(0.71 0.01 286); --severity-unknown: oklch(0.75 0 0); + --code-foreground: #2b2b2b; + --code-accent: #b0402c; + --code-string: var(--primary); --sidebar: #fafafa; --sidebar-foreground: #171717; --sidebar-primary: #006b4f; @@ -115,6 +118,9 @@ --severity-low: var(--muted-foreground); --severity-info: oklch(0.71 0.01 286); --severity-unknown: oklch(0.5 0 0); + --code-foreground: #e3e3e3; + --code-accent: #e5735b; + --code-string: #40be96; --sidebar: #171717; --sidebar-foreground: #f5f5f5; --sidebar-primary: #006b4f; @@ -182,6 +188,9 @@ --color-severity-low: var(--severity-low); --color-severity-info: var(--severity-info); --color-severity-unknown: var(--severity-unknown); + --color-code-foreground: var(--code-foreground); + --color-code-accent: var(--code-accent); + --color-code-string: var(--code-string); --color-sidebar: var(--sidebar); --color-sidebar-foreground: var(--sidebar-foreground); --color-sidebar-primary: var(--sidebar-primary);