From fb19a0d69c3b8ab929b39d0b0b34d6ee113721b2 Mon Sep 17 00:00:00 2001 From: jmgasper Date: Tue, 20 Oct 2026 05:58:11 +1100 Subject: [PATCH] PM-6619: Sort form reports newest first What was broken Form reports and CSV exports listed the oldest submissions first, making recent Let's talk requests difficult to find. Root cause The all-revision report explicitly ordered submission timestamps and UUIDs ascending. What was changed Order all-revision report and export rows by descending submission timestamp and UUID, preserving cursor pagination, date filters and all export columns. Document the ordering and companion website attribution behavior. Any added/updated tests Update PostgreSQL integration coverage for descending pagination across revisions, CSV ordering and stable UUID ties across 1,002 exported rows. Verify authenticated Let's talk submissions retain member ID and UTM answers in reports and Kafka, while anonymous member IDs remain null. Validation: pnpm lint, pnpm build, pnpm test (69 passed), and pnpm test:integration against disposable PostgreSQL (24 passed), after nvm use. --- docs/api.md | 3 ++- docs/lets-talk.md | 8 +++++-- src/forms/forms.service.ts | 4 ++-- test/forms.integration.spec.ts | 44 ++++++++++++++++++++++++++++++---- 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/docs/api.md b/docs/api.md index aa53c99..fbf1568 100644 --- a/docs/api.md +++ b/docs/api.md @@ -153,7 +153,8 @@ filters. A cursor must belong to the selected form/revision and date range. The all-revision report unions field keys in first-seen order. `labels` contains the latest published label for each key; absent answers are null. Existing metadata columns (`submission_id`, `submitted_at`, `member_id`, `source_page`, `form_version`) -remain present. Rows sort by timestamp then UUID; `total` counts matching rows +remain present. All-revision rows and CSV exports sort newest first by timestamp, +then descending UUID for stable ties across cursor pages; `total` counts matching rows before pagination. Decimal values remain strings and multi-select values remain arrays. A field reused with a different type retains each revision's original value. diff --git a/docs/lets-talk.md b/docs/lets-talk.md index d98960a..afb77aa 100644 --- a/docs/lets-talk.md +++ b/docs/lets-talk.md @@ -16,8 +16,12 @@ Option values are normalized to API-safe keys: | `Landing Page` (hidden lead source) | `landing_page` | The website supplies the existing hidden campaign defaults, including -`source__c=/lets-talk`; empty UTM fields remain empty. These are validated answers, -not trusted identity. `sourcePage` is the current pathname. The site submits +`source__c=/lets-talk`; the website maps URL `utm_source`, `utm_medium`, and +`utm_campaign` parameters to the matching `utm_*__c` answers, leaving absent +parameters empty. These are validated answers, not trusted identity. The website +sends the shared `tcjwt` session as a bearer token when available; the API derives +`member_id` from verified claims, while anonymous submissions retain a null identity. +`sourcePage` is the current pathname. The site submits `kafka: true` so each accepted attempt publishes `form.submitted` through the existing Bus API integration. Consumers must deduplicate by `submissionId`. diff --git a/src/forms/forms.service.ts b/src/forms/forms.service.ts index a65dca0..e6eb7a0 100644 --- a/src/forms/forms.service.ts +++ b/src/forms/forms.service.ts @@ -121,7 +121,7 @@ export class FormsService { } /** - * Reads a form across all published revisions for the reports portal. + * Reads a form across all published revisions, newest first with descending UUID ties, for the reports portal and CSV. * @param key Form key. @param page Inclusive dates and cursor pagination. @param snapshot Export upper timestamp bound. * @returns Stable union columns, typed rows, total matching count, and next cursor. * @throws NotFoundException for unknown forms; BadRequestException for invalid dates/cursors; database errors on failure. @@ -156,7 +156,7 @@ export class FormsService { const [submissions, total] = await Promise.all([ this.db.submission.findMany({ where, - orderBy: [{ createdAt: 'asc' }, { id: 'asc' }], + orderBy: [{ createdAt: 'desc' }, { id: 'desc' }], take: page.limit + 1, ...(page.after ? { cursor: { id: page.after }, skip: 1 } : {}), include: { diff --git a/test/forms.integration.spec.ts b/test/forms.integration.spec.ts index eef20c4..8409c29 100644 --- a/test/forms.integration.spec.ts +++ b/test/forms.integration.spec.ts @@ -144,6 +144,34 @@ describe('forms API with real PostgreSQL', () => { expect(report.body.data[0]).toMatchObject({ source_page: '/lets-talk', company: 'Test Company', + member_id: null, + }); + + const attributedAnswers = { + ...body.answers, + utm_source__c: 'newsletter', + utm_medium__c: 'email', + utm_campaign__c: 'Fall & Winter', + }; + const memberReceipt = await request(app.getHttpServer()) + .post(`/v6/forms/${key}/submissions`) + .auth(member, { type: 'bearer' }) + .set('Idempotency-Key', randomUUID()) + .send({ ...body, answers: attributedAnswers }) + .expect(201); + const memberReport = await request(app.getHttpServer()) + .get(`/v6/forms/${key}/submissions`) + .auth(reporter, { type: 'bearer' }) + .expect(200); + expect(memberReport.body.total).toBe(2); + expect(memberReport.body.data[0]).toMatchObject({ + submission_id: memberReceipt.body.id, + member_id: '12345', + ...attributedAnswers, + }); + expect(postEvent.mock.calls[1][0].payload).toMatchObject({ + memberId: '12345', + answers: attributedAnswers, }); }); @@ -184,7 +212,7 @@ describe('forms API with real PostgreSQL', () => { .expect(403); }); - it('combines revisions, includes whole UTC end dates, and validates cursors and calendar dates', async () => { + it('paginates newest first across revisions, includes whole UTC end dates, and validates cursors and dates', async () => { const optionalDefinition = { ...definition(), fields: definition().fields.map((field) => ({ @@ -253,8 +281,9 @@ describe('forms API with real PostgreSQL', () => { .expect(200); expect(page.body.total).toBe(2); expect(page.body.data[0]).toMatchObject({ - form_version: 1, - new_field: null, + submission_id: second.id, + form_version: 2, + new_field: '=SUM(1,2)', updates: false, age: 0, }); @@ -269,8 +298,9 @@ describe('forms API with real PostgreSQL', () => { .auth(reporter, { type: 'bearer' }) .expect(200); expect(last.body.data[0]).toMatchObject({ - form_version: 2, - new_field: '=SUM(1,2)', + submission_id: first.id, + form_version: 1, + new_field: null, }); expect(last.body.nextCursor).toBeNull(); const empty = await request(app.getHttpServer()) @@ -301,6 +331,7 @@ describe('forms API with real PostgreSQL', () => { expect(csv.text).toContain('"\'=SUM(1,2)"'); expect(csv.text).toContain(first.id); expect(csv.text).toContain(second.id); + expect(csv.text.indexOf(second.id)).toBeLessThan(csv.text.indexOf(first.id)); await request(app.getHttpServer()) .get(`${path}/export`) .query({ startDate: '2020-02-30' }) @@ -344,6 +375,9 @@ describe('forms API with real PostgreSQL', () => { expect(new Set(lines.slice(1).map((line) => line.split(',')[0])).size).toBe( 1002, ); + expect(lines.slice(1).map((line) => line.split(',')[0])).toEqual( + [...ids].sort().reverse().map((id) => `"${id}"`), + ); expect(csv.headers['content-disposition']).toContain(`${key}.csv`); expect(csv.headers['cache-control']).toBe('no-store'); const empty = await request(app.getHttpServer())