diff --git a/backend/src/db/newsletterIssues.ts b/backend/src/db/newsletterIssues.ts index 4d5f60ca..e4711d9f 100644 --- a/backend/src/db/newsletterIssues.ts +++ b/backend/src/db/newsletterIssues.ts @@ -48,6 +48,15 @@ export interface NewsletterSlotStats { recordedAt: string; } +/** Where the sponsor's emails for one booking go and which Gmail thread they continue. */ +export interface SponsorEmailRoute { + to: string; + cc: string[]; + sender: string; + /** Gmail thread id of the `email-sponsor` send. */ + threadRef: string; +} + export interface NewsletterSlot { /** The booked sponsor; an open slot has none. Booking adds the slot's Tasks to the Card. */ sponsor?: string; @@ -60,6 +69,8 @@ export interface NewsletterSlot { documentUrl?: string; /** Performance metrics for this slot's placement and sponsored social posts. */ stats?: NewsletterSlotStats; + /** Set when `email-sponsor` is sent; `newsletter report --send` replies in this thread. */ + route?: SponsorEmailRoute; } /** A link the issue keeps before (or without) a Card: a draft, a doc, a campaign. */ diff --git a/backend/src/newsletter/reportSend.ts b/backend/src/newsletter/reportSend.ts new file mode 100644 index 00000000..30c5e9e3 --- /dev/null +++ b/backend/src/newsletter/reportSend.ts @@ -0,0 +1,80 @@ +/** + * `dataops newsletter report --send`: reply to the sponsor in the Gmail thread + * of the booking's `email-sponsor` send, from Orin's mailbox, then close the + * slot's `send-performance-to-sponsor` Task with the Gmail message id. The + * route comes only from the booking; a missing route stops the send. + */ +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +import type { NewsletterIssue, NewsletterSlotPosition } from '../db/newsletterIssues'; +import { getTaskConsistent, listTasksByCardConsistent, updateTask } from '../db/tasks'; +import { deliverRenderedEmail, SPONSOR_DOC_EMAIL_FROM } from '../sponsorDocs/emailDelivery'; +import { buildSponsorStatsReport } from './statsReport'; +import { requireSponsorRoute, ROUTE_SEND_TASK_REFS, SponsorRouteError } from './sponsorRoute'; + +export class ReportSendError extends Error { + constructor(readonly statusCode: number, message: string, readonly code: string, readonly detail: Record = {}) { + super(message); + this.name = 'ReportSendError'; + } +} + +export async function sendIssueReport( + client: DynamoDBDocumentClient, + issue: NewsletterIssue, + options: { slot: NewsletterSlotPosition; contactName: string; actorId: string }, +) { + const { slot: position, contactName, actorId } = options; + const where = `The ${position} slot of ${issue.id}`; + const slot = issue.slots?.[position]; + if (!slot?.sponsor) throw new ReportSendError(422, `${where} has no booked sponsor`, 'slot_not_booked'); + if (!contactName.trim()) throw new ReportSendError(400, 'contactName is required to send the report', 'contact_required'); + const report = buildSponsorStatsReport(issue, position, { contactName }); + if (!report.recordedAt) throw new ReportSendError(422, `${where} has no recorded stats; record them before sending the report`, 'stats_missing'); + + let route; + try { + route = requireSponsorRoute(slot.route, where); + } catch (error) { + if (error instanceof SponsorRouteError) throw new ReportSendError(422, error.message, 'sponsor_route_missing'); + throw error; + } + if (route.sender !== SPONSOR_DOC_EMAIL_FROM) { + throw new ReportSendError(422, `${where} route sender is ${route.sender}; reports are sent from ${SPONSOR_DOC_EMAIL_FROM}`, 'sponsor_route_sender'); + } + if (!issue.cardId) throw new ReportSendError(422, `${issue.id} has no Card, so there is no send task to close`, 'no_card'); + const ref = ROUTE_SEND_TASK_REFS[position][0]; + const task = (await listTasksByCardConsistent(client, issue.cardId)).find((item) => item.templateTaskRef === ref); + if (!task) throw new ReportSendError(422, `Card ${issue.cardId} has no ${ref} task`, 'send_task_missing'); + if (task.status === 'done') throw new ReportSendError(409, `Task ${task.id} (${ref}) is already done; the report was not sent again`, 'already_sent'); + + const delivery = await deliverRenderedEmail(client, { + draft: { subject: 'Newsletter performance', body: report.message }, + recipientEmail: route.to, + taskId: task.id, + cc: route.cc, + threadId: route.threadRef, + }); + if (delivery.state !== 'sent' || !delivery.messageId) { + console.error(JSON.stringify({ level: 'ERROR', message: 'sponsor report not sent', issueId: issue.id, slot: position, reason: delivery.reason, detail: delivery.message })); + throw new ReportSendError(502, delivery.message || `The report was not sent (${delivery.reason || delivery.state})`, delivery.reason || 'not_sent', { emailDelivery: delivery }); + } + + const current = await getTaskConsistent(client, task.id); + if (!current) throw new ReportSendError(500, `Task ${task.id} vanished after the report was sent (Gmail message ${delivery.messageId})`, 'task_missing', { emailDelivery: delivery }); + const now = new Date().toISOString(); + const comment = `Sponsor report sent in thread ${route.threadRef} to ${route.to} (Gmail message ID: ${delivery.messageId})`; + try { + await updateTask(client, current.id, { + currentTask: current, + expectedVersion: current.version, + actorId, + triggerKind: 'sponsor-report-sent', + patch: { status: 'done', completedAt: now, completedBy: actorId, comment, externalStatus: `Gmail message ${delivery.messageId}` }, + }); + } catch (error) { + console.error(JSON.stringify({ level: 'ERROR', message: 'sponsor report sent but task not closed', taskId: task.id, gmailMessageId: delivery.messageId, reason: (error as Error).message })); + throw new ReportSendError(500, `The report was sent (Gmail message ${delivery.messageId}) but Task ${task.id} was not closed: ${(error as Error).message}`, 'task_not_closed', { emailDelivery: delivery }); + } + return { emailDelivery: delivery, taskId: task.id, route }; +} diff --git a/backend/src/newsletter/sponsorRoute.ts b/backend/src/newsletter/sponsorRoute.ts new file mode 100644 index 00000000..965cf79d --- /dev/null +++ b/backend/src/newsletter/sponsorRoute.ts @@ -0,0 +1,132 @@ +/** + * The sponsor email route of a newsletter slot booking: who the sponsor emails + * go to and which Gmail thread they continue. Sending `email-sponsor` records + * it on the slot and copies it onto the slot's later send Tasks; a missing + * route is an error, never a guess. + */ +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +import { + listNewsletterIssuesByCardId, + updateNewsletterIssue, + type NewsletterSlotPosition, + type SponsorEmailRoute, +} from '../db/newsletterIssues'; +import { getTaskConsistent, listTasksByCardConsistent, TaskVersionConflictError, updateTask } from '../db/tasks'; +import type { SponsorDocEmailDelivery } from '../sponsorDocs/emailDelivery'; +import type { Task } from '../types'; + +/** The Task whose send fixes a slot's route. */ +export const ROUTE_SOURCE_TASK_REFS: Record = { + 'email-sponsor': 'primary', + 'secondary-email-sponsor': 'secondary', +}; + +/** The send Tasks of a slot that reuse its route. */ +export const ROUTE_SEND_TASK_REFS: Record = { + primary: ['send-performance-to-sponsor', 'send-email-sponsor-publication-live'], + secondary: ['secondary-send-performance-to-sponsor', 'secondary-send-email-sponsor-publication-live'], +}; + +export class SponsorRouteError extends Error { + constructor(message: string) { + super(message); + this.name = 'SponsorRouteError'; + } +} + +const EMAIL = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + +export function sponsorRouteFromDelivery(delivery: SponsorDocEmailDelivery): SponsorEmailRoute { + if (delivery.state !== 'sent') throw new SponsorRouteError('The email was not sent; there is no route to record'); + if (!delivery.to || !EMAIL.test(delivery.to)) throw new SponsorRouteError('The sent email has no recipient address'); + if (!delivery.threadId) throw new SponsorRouteError('Gmail returned no thread id for the sent email'); + return { to: delivery.to, cc: delivery.cc || [], sender: delivery.from, threadRef: delivery.threadId }; +} + +/** The route as stored, or a loud error naming what is missing. */ +export function requireSponsorRoute(value: unknown, where: string): SponsorEmailRoute { + const route = value as Partial | null | undefined; + if (!route || typeof route !== 'object') throw new SponsorRouteError(`${where} has no sponsor email route (to, cc, sender, threadRef); send email-sponsor first or backfill it`); + const missing = ['to', 'sender', 'threadRef'].filter((name) => typeof route[name as 'to'] !== 'string' || !route[name as 'to']); + if (missing.length || !Array.isArray(route.cc)) { + throw new SponsorRouteError(`${where} has an incomplete sponsor email route (missing ${[...missing, ...(Array.isArray(route.cc) ? [] : ['cc'])].join(', ')})`); + } + return route as SponsorEmailRoute; +} + +async function setTaskRoute(client: DynamoDBDocumentClient, task: Task, route: SponsorEmailRoute): Promise { + let current: Task | null = task; + for (let attempt = 0; attempt < 3 && current; attempt += 1) { + if (JSON.stringify(current.sponsorRoute) === JSON.stringify(route)) return; + try { + await updateTask(client, current.id, { + currentTask: current, + expectedVersion: current.version, + patch: { sponsorRoute: route }, + actorId: 'system:sponsor-route', + triggerKind: 'sponsor-route-set', + }); + return; + } catch (error) { + if (!(error instanceof TaskVersionConflictError) || attempt === 2) throw error; + current = await getTaskConsistent(client, task.id); + } + } +} + +/** Set the slot's route and copy it onto the slot's send Tasks. Returns the Task ids it wrote. */ +export async function applySponsorRoute( + client: DynamoDBDocumentClient, + cardId: string, + position: NewsletterSlotPosition, + route: SponsorEmailRoute, +): Promise { + const issues = await listNewsletterIssuesByCardId(client, cardId); + if (issues.length !== 1) throw new SponsorRouteError(`Card ${cardId} is linked to ${issues.length} newsletter issues; expected one`); + const issue = issues[0]; + const slot = issue.slots?.[position]; + if (!slot?.sponsor) throw new SponsorRouteError(`The ${position} slot of ${issue.id} has no booked sponsor`); + if (JSON.stringify(slot.route) !== JSON.stringify(route)) { + await updateNewsletterIssue(client, issue, { slots: { ...issue.slots, [position]: { ...slot, route } } }); + } + const refs = new Set(ROUTE_SEND_TASK_REFS[position]); + const written: string[] = []; + for (const task of await listTasksByCardConsistent(client, cardId)) { + if (!task.templateTaskRef || !refs.has(task.templateTaskRef)) continue; + await setTaskRoute(client, task, route); + written.push(task.id); + } + return written; +} + +/** + * After `email-sponsor` (or its secondary twin) sent, record the route it + * established. Other Tasks are ignored. Throws SponsorRouteError when the send + * cannot be recorded as a route, after logging a server-side ERROR. + */ +export async function recordSponsorRouteAfterSend( + client: DynamoDBDocumentClient, + taskId: string, + delivery: SponsorDocEmailDelivery, +): Promise { + if (delivery.state !== 'sent') return; + const task = await getTaskConsistent(client, taskId); + const position = task?.templateTaskRef ? ROUTE_SOURCE_TASK_REFS[task.templateTaskRef] : undefined; + if (!task || !position) return; + try { + if (!task.cardId) throw new SponsorRouteError(`Task ${taskId} has no Card`); + await applySponsorRoute(client, task.cardId, position, sponsorRouteFromDelivery(delivery)); + } catch (error) { + console.error(JSON.stringify({ + level: 'ERROR', + message: 'sponsor email route not recorded after email-sponsor send', + taskId, + gmailMessageId: delivery.messageId, + reason: (error as Error).message, + })); + throw new SponsorRouteError( + `The sponsor email was sent (Gmail message ${delivery.messageId}) but its route was not recorded: ${(error as Error).message}`, + ); + } +} diff --git a/backend/src/newsletter/statsReport.ts b/backend/src/newsletter/statsReport.ts index f4897734..60cab3cc 100644 --- a/backend/src/newsletter/statsReport.ts +++ b/backend/src/newsletter/statsReport.ts @@ -1,5 +1,6 @@ import type { NewsletterIssue, + SponsorEmailRoute, NewsletterIssueKind, NewsletterSlotPosition, } from '../db/newsletterIssues'; @@ -16,6 +17,8 @@ export interface SponsorStatsReport { sponsor: string | null; organizationId?: string; recordedAt: string | null; + /** Where the sponsor's emails go (To/Cc/sender/Gmail thread); null until `email-sponsor` was sent. */ + route: SponsorEmailRoute | null; newsletter: { sent?: number; views?: number; @@ -137,6 +140,7 @@ export function buildSponsorStatsReport( sponsor: slot.sponsor || null, ...(slot.organizationId ? { organizationId: slot.organizationId } : {}), recordedAt, + route: slot.route || null, newsletter, linkedin, x, diff --git a/backend/src/routes/emailTemplates.ts b/backend/src/routes/emailTemplates.ts index 52c6d740..4dfa8f93 100644 --- a/backend/src/routes/emailTemplates.ts +++ b/backend/src/routes/emailTemplates.ts @@ -16,6 +16,7 @@ import { type KnowledgeEmailTemplate, type RenderedKnowledgeEmail, } from '../sponsorDocs/knowledgeEmailTemplate'; +import { recordSponsorRouteAfterSend, SponsorRouteError } from '../newsletter/sponsorRoute'; import { noStoreJson, parseJsonObjectEventOrEmpty } from '../http'; import type { LambdaEvent, LambdaResponse } from '../types'; @@ -145,7 +146,15 @@ export async function handleEmailTemplateRoutes( templateRevision: template.revision, ...(emailDelivery.state === 'sent' ? { taskId } : {}), }; - if (delivery.state === 'sent') await recordEmailOnTask(client, taskId, template.id, delivery); + if (delivery.state === 'sent') { + await recordEmailOnTask(client, taskId, template.id, delivery); + try { + await recordSponsorRouteAfterSend(client, taskId, delivery); + } catch (error) { + if (error instanceof SponsorRouteError) return noStoreJson(500, { error: error.message, code: 'sponsor_route_not_recorded', emailDelivery: delivery }); + throw error; + } + } return noStoreJson(200, { emailDelivery: delivery, sender: SPONSOR_DOC_EMAIL_FROM, diff --git a/backend/src/routes/newsletterIssues.ts b/backend/src/routes/newsletterIssues.ts index 2b13180f..8e278d7e 100644 --- a/backend/src/routes/newsletterIssues.ts +++ b/backend/src/routes/newsletterIssues.ts @@ -84,6 +84,9 @@ import { MailchimpMetricsError, pullIssueMailchimpMetrics, } from '../newsletter/mailchimpMetrics'; +import { GmailMessageRouteError, resolveGmailMessageRoute } from '../sponsorDocs/gmailMessageRoute'; +import { applySponsorRoute, SponsorRouteError } from '../newsletter/sponsorRoute'; +import { ReportSendError, sendIssueReport } from '../newsletter/reportSend'; import { json, parseJsonObjectEvent } from '../http'; const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; @@ -978,6 +981,47 @@ async function issueReportRoute(id: string, event: LambdaEvent, client: DynamoDB }); } +async function issueReportSendRoute(id: string, body: Body, event: LambdaEvent, client: DynamoDBDocumentClient) { + const resolved = await resolveInteractiveActor(client, event, 'work-write'); + if (!resolved.ok) return resolved.response; + const item = await resolveNewsletterIssue(client, id); + if (!item) return json(404, { error: 'Not found' }); + const slot = body.slot; + if (slot !== 'primary' && slot !== 'secondary') return json(400, { error: 'slot is required: primary or secondary' }); + if (!(SLOT_POSITIONS[item.kind] as readonly string[]).includes(slot)) return json(400, { error: `A ${item.kind} issue has no ${slot} slot` }); + try { + const sent = await sendIssueReport(client, item, { + slot, + contactName: typeof body.contactName === 'string' ? body.contactName : '', + actorId: resolved.actor.id, + }); + return json(200, sent); + } catch (error) { + if (error instanceof ReportSendError) return json(error.statusCode, { error: error.message, code: error.code, ...error.detail }); + throw error; + } +} + +/** Adopt an `email-sponsor` message Orin already sent as the booking's route. */ +async function slotSponsorRouteRoute(id: string, position: string, body: Body, event: LambdaEvent, client: DynamoDBDocumentClient) { + const resolved = await resolveInteractiveActor(client, event, 'work-write'); + if (!resolved.ok) return resolved.response; + const item = await resolveNewsletterIssue(client, id); + if (!item) return json(404, { error: 'Not found' }); + if (position !== 'primary' && position !== 'secondary') return json(400, { error: 'The slot is primary or secondary' }); + if (!item.cardId) return json(422, { error: `${item.id} has no Card` }); + const messageId = typeof body.messageId === 'string' ? body.messageId.trim() : ''; + if (!/^[A-Za-z0-9]{6,64}$/.test(messageId)) return json(400, { error: 'messageId is required: the Gmail message id of the email-sponsor send' }); + try { + const route = await resolveGmailMessageRoute(messageId); + const taskIds = await applySponsorRoute(client, item.cardId, position, route); + return json(200, { issueId: item.id, slot: position, route, taskIds }); + } catch (error) { + if (error instanceof GmailMessageRouteError || error instanceof SponsorRouteError) return json(422, { error: error.message, code: 'sponsor_route_not_set' }); + throw error; + } +} + async function mailchimpMetricsRoute(id: string, body: Body, event: LambdaEvent, client: DynamoDBDocumentClient) { const existing = await resolveNewsletterIssue(client, id); if (!existing) return json(404, { error: 'Not found' }); @@ -1101,6 +1145,11 @@ export async function handleNewsletterIssueRoutes( if (method !== 'GET') return json(405, { error: 'Method not allowed' }); return issueReportRoute(decodeURIComponent(issueReport[1]), event, client); } + const issueReportSend = path.match(/^\/api\/newsletter-issues\/([^/]+)\/report\/send$/); + if (issueReportSend) { + if (method !== 'POST') return json(405, { error: 'Method not allowed' }); + return issueReportSendRoute(decodeURIComponent(issueReportSend[1]), parse(event) || {}, event, client); + } const mailchimpMetrics = path.match(/^\/api\/newsletter-issues\/([^/]+)\/mailchimp-metrics$/); if (mailchimpMetrics) { if (method !== 'POST') return json(405, { error: 'Method not allowed' }); @@ -1133,6 +1182,11 @@ export async function handleNewsletterIssueRoutes( if (method !== 'GET') return json(405, { error: 'Method not allowed' }); return json(200, { items: await sponsorOptions(client) }); } + const slotRoute_ = path.match(/^\/api\/newsletter-issues\/([^/]+)\/slots\/([^/]+)\/route$/); + if (slotRoute_) { + if (method !== 'PUT') return json(405, { error: 'Method not allowed' }); + return slotSponsorRouteRoute(decodeURIComponent(slotRoute_[1]), slotRoute_[2], parse(event) || {}, event, client); + } const slot = path.match(/^\/api\/newsletter-issues\/([^/]+)\/slots\/([^/]+)$/); if (slot) { if (method !== 'PUT' && method !== 'DELETE') return json(405, { error: 'Method not allowed' }); diff --git a/backend/src/routes/sponsorDocs.ts b/backend/src/routes/sponsorDocs.ts index a1878a1a..19f1a1aa 100644 --- a/backend/src/routes/sponsorDocs.ts +++ b/backend/src/routes/sponsorDocs.ts @@ -59,6 +59,7 @@ import { speakerTemplateView, type SponsorDocTemplateKey, } from '../sponsorDocs/templates'; +import { recordSponsorRouteAfterSend, SponsorRouteError } from '../newsletter/sponsorRoute'; import { listNextRecordingSlots } from '../podcast/resolveRecordingSlot'; import type { Card, LambdaEvent, LambdaResponse } from '../types'; @@ -83,11 +84,23 @@ function json(statusCode: number, body: unknown, extraHeaders?: Record { + const recorded = await recordSponsorEmailNote(client, taskId, cardId, delivery); + if (taskId && recorded.state === 'sent') await recordSponsorRouteAfterSend(client, taskId, recorded); + return recorded; +} + +async function recordSponsorEmailNote( + client: DynamoDBDocumentClient, + taskId: string | null, + cardId: string, + delivery: SponsorDocEmailDelivery, ): Promise { if (!taskId || delivery.state !== 'sent' || !delivery.messageId) return delivery; const noteText = `Sponsor document email sent (Gmail message ID: ${delivery.messageId})`; @@ -1088,6 +1101,9 @@ export async function handleSponsorDocRoutes( if (err instanceof SponsorEmailTaskOwnerError) { return json(err.statusCode, { error: err.message, code: 'invalid_email_task' }); } + if (err instanceof SponsorRouteError) { + return json(500, { error: err.message, code: 'sponsor_route_not_recorded' }); + } recordRequestError(err); return json(500, { error: 'Internal server error' }); } diff --git a/backend/src/sponsorDocs/emailDelivery.ts b/backend/src/sponsorDocs/emailDelivery.ts index e7a5096a..dbac4ec4 100644 --- a/backend/src/sponsorDocs/emailDelivery.ts +++ b/backend/src/sponsorDocs/emailDelivery.ts @@ -111,6 +111,8 @@ export function buildRawMimeEmail(input: { cc?: string[]; subject: string; body: string; + /** Continue an existing Gmail thread: the Message-ID of its last message. */ + inReplyTo?: string; }): string { const headers: string[] = [ `From: Orin <${sanitizeHeaderValue(input.from)}>`, @@ -124,6 +126,9 @@ export function buildRawMimeEmail(input: { } headers.push( `Subject: ${sanitizeHeaderValue(input.subject)}`, + ...(input.inReplyTo + ? [`In-Reply-To: ${sanitizeHeaderValue(input.inReplyTo)}`, `References: ${sanitizeHeaderValue(input.inReplyTo)}`] + : []), 'MIME-Version: 1.0', 'Content-Type: text/plain; charset="UTF-8"', 'Content-Transfer-Encoding: 8bit', @@ -229,6 +234,34 @@ function summarizeDapierGmailTokenProblem(status: number, connectionId: string, return `The Dapier Gmail token broker refused the request (HTTP ${status}).`; } +/** The thread's subject (as a reply) and the Message-ID of its last message; null when Gmail cannot read it. */ +async function readGmailThreadHeaders( + runFetch: typeof fetch, + authHeaders: Record, + threadId: string, +): Promise<{ subject: string; messageId: string } | null> { + let res: Response; + try { + res = await runFetch( + `${GMAIL_API_BASE}/threads/${encodeURIComponent(threadId)}?format=metadata&metadataHeaders=Message-ID&metadataHeaders=Subject`, + { method: 'GET', headers: authHeaders, signal: AbortSignal.timeout(10000) }, + ); + } catch { + return null; + } + if (!res.ok) return null; + const thread = (await res.json().catch(() => null)) as + | { messages?: Array<{ payload?: { headers?: Array<{ name?: string; value?: string }> } }> } + | null; + const messages = thread?.messages || []; + const header = (index: number, name: string) => messages[index]?.payload?.headers + ?.find((item) => item.name?.toLowerCase() === name)?.value?.trim() || ''; + const first = header(0, 'subject'); + const messageId = header(messages.length - 1, 'message-id'); + if (!first || !messageId) return null; + return { subject: /^re:/i.test(first) ? first : `Re: ${first}`, messageId }; +} + export async function deliverRenderedEmail( client: DynamoDBDocumentClient, options: { @@ -241,6 +274,8 @@ export async function deliverRenderedEmail( connectionId?: string | null; /** Domain-owned recipients; sponsor content requests keep their default CC. */ cc?: string[]; + /** Reply in this Gmail thread instead of starting a new one; the subject is taken from the thread. */ + threadId?: string; }, ): Promise { const resolvedRecipient = await resolveSponsorDocRecipient(client, { @@ -330,14 +365,6 @@ export async function deliverRenderedEmail( }; } - const raw = buildRawMimeEmail({ - from: SPONSOR_DOC_EMAIL_FROM, - to, - cc, - subject, - body: options.draft.body, - }); - let preferredDeferred: SponsorDocEmailDelivery | null = null; for (const connectionId of candidateConnectionIds) { @@ -483,12 +510,40 @@ export async function deliverRenderedEmail( continue; } + let replySubject = subject; + let inReplyTo: string | undefined; + if (options.threadId) { + const thread = await readGmailThreadHeaders(runFetch, authHeaders, options.threadId); + if (!thread) { + return { + state: 'deferred', + from: SPONSOR_DOC_EMAIL_FROM, + to, + ...(cc.length > 0 ? { cc } : {}), + subject, + connectionId, + reason: 'thread_not_found', + message: `Gmail thread ${options.threadId} could not be read from ${SPONSOR_DOC_EMAIL_FROM}; no email was sent.`, + }; + } + replySubject = thread.subject; + inReplyTo = thread.messageId; + } + const raw = buildRawMimeEmail({ + from: SPONSOR_DOC_EMAIL_FROM, + to, + cc, + subject: replySubject, + body: options.draft.body, + inReplyTo, + }); + let sendRes: Response; try { sendRes = await runFetch(`${GMAIL_API_BASE}/messages/send`, { method: 'POST', headers: authHeaders, - body: JSON.stringify({ raw }), + body: JSON.stringify({ raw, ...(options.threadId ? { threadId: options.threadId } : {}) }), signal: AbortSignal.timeout(15000), }); } catch { diff --git a/backend/src/sponsorDocs/gmailMessageRoute.ts b/backend/src/sponsorDocs/gmailMessageRoute.ts new file mode 100644 index 00000000..02d3a182 --- /dev/null +++ b/backend/src/sponsorDocs/gmailMessageRoute.ts @@ -0,0 +1,59 @@ +/** + * Read the route (To, Cc, sender, thread) of a message Orin already sent, so a + * booking whose `email-sponsor` was sent before routes were recorded can adopt + * it. Uses the same Dapier-held Orin Gmail connection as the sends. + */ +import type { SponsorEmailRoute } from '../db/newsletterIssues'; +import { CREDENTIAL_OPERATOR_HINT, DAPIER_AGENT, DAPIER_BASE_URL, readDapierCredential } from '../calendar/dapierBroker'; +import { DEFAULT_ORIN_GMAIL_CONNECTION_ID, SPONSOR_DOC_EMAIL_FROM } from './emailDelivery'; + +const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me'; +const ADDRESS = /[^\s<>,;"']+@[^\s<>,;"']+\.[^\s<>,;"']+/g; + +export class GmailMessageRouteError extends Error { + constructor(message: string) { + super(message); + this.name = 'GmailMessageRouteError'; + } +} + +const addresses = (value: string): string[] => (value.match(ADDRESS) || []).map((item) => item.toLowerCase()); + +export async function resolveGmailMessageRoute( + messageId: string, + deps: { fetchImpl?: typeof fetch; credential?: string } = {}, +): Promise { + const runFetch = deps.fetchImpl || fetch; + let credential = deps.credential || process.env.DAPIER_API_TOKEN; + if (!credential) { + const resolved = await readDapierCredential(undefined); + credential = resolved.credential; + if (!credential) throw new GmailMessageRouteError(resolved.problem || `No Dapier broker credential. ${CREDENTIAL_OPERATOR_HINT}`); + } + const connectionId = (process.env.DATAOPS_ORIN_GMAIL_CONNECTION_ID || DEFAULT_ORIN_GMAIL_CONNECTION_ID).trim(); + const brokerUrl = (process.env.DAPIER_BASE_URL || DAPIER_BASE_URL).replace(/\/$/, ''); + const tokenRes = await runFetch(`${brokerUrl}/api/agent/token`, { + method: 'POST', + headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ connection_id: connectionId, agent: DAPIER_AGENT }), + signal: AbortSignal.timeout(10000), + }).catch(() => null); + if (!tokenRes?.ok) throw new GmailMessageRouteError(`The Dapier broker refused the Gmail token for connection "${connectionId}" (HTTP ${tokenRes?.status ?? 'unreachable'}).`); + const token = (await tokenRes.json().catch(() => ({}))) as { access_token?: string; provider_account_id?: string }; + if (!token.access_token) throw new GmailMessageRouteError(`Dapier connection "${connectionId}" returned no access token.`); + if (String(token.provider_account_id || '').toLowerCase() !== SPONSOR_DOC_EMAIL_FROM) { + throw new GmailMessageRouteError(`Dapier connection "${connectionId}" is not verified as ${SPONSOR_DOC_EMAIL_FROM}.`); + } + const res = await runFetch( + `${GMAIL_API_BASE}/messages/${encodeURIComponent(messageId)}?format=metadata&metadataHeaders=From&metadataHeaders=To&metadataHeaders=Cc`, + { headers: { Authorization: `Bearer ${token.access_token}` }, signal: AbortSignal.timeout(10000) }, + ).catch(() => null); + if (!res?.ok) throw new GmailMessageRouteError(`Gmail message ${messageId} could not be read from ${SPONSOR_DOC_EMAIL_FROM} (HTTP ${res?.status ?? 'unreachable'}).`); + const message = (await res.json().catch(() => ({}))) as { threadId?: string; payload?: { headers?: Array<{ name?: string; value?: string }> } }; + const header = (name: string) => message.payload?.headers?.find((item) => item.name?.toLowerCase() === name)?.value || ''; + const [sender] = addresses(header('from')); + const [to] = addresses(header('to')); + if (sender !== SPONSOR_DOC_EMAIL_FROM) throw new GmailMessageRouteError(`Gmail message ${messageId} was sent from ${sender || 'an unknown sender'}, not ${SPONSOR_DOC_EMAIL_FROM}.`); + if (!to || !message.threadId) throw new GmailMessageRouteError(`Gmail message ${messageId} has no recipient or thread id.`); + return { to, cc: addresses(header('cc')), sender, threadRef: message.threadId }; +} diff --git a/backend/src/types.ts b/backend/src/types.ts index 1eb3f1d5..68cc7e6e 100644 --- a/backend/src/types.ts +++ b/backend/src/types.ts @@ -280,6 +280,8 @@ export interface Task { delivery?: import('./sponsorDocs/emailDelivery').SponsorDocEmailDelivery; attempts?: Array<{ claimedAt: string; delivery: import('./sponsorDocs/emailDelivery').SponsorDocEmailDelivery }>; }; + /** Sponsor email route copied from the newsletter slot booking; set on the sponsor send Tasks. */ + sponsorRoute?: import('./db/newsletterIssues').SponsorEmailRoute; waitingFor?: string; followUpAt?: string; followUpChannel?: string; diff --git a/backend/tests/sponsor-route.test.ts b/backend/tests/sponsor-route.test.ts new file mode 100644 index 00000000..e4315a31 --- /dev/null +++ b/backend/tests/sponsor-route.test.ts @@ -0,0 +1,140 @@ +/** + * A newsletter slot booking carries the sponsor email route (To, Cc, sender, + * Gmail thread). Sending `email-sponsor` records it and copies it onto the + * slot's send Tasks; `report/send` replies in that thread and closes the Task; + * a missing route is an error. All sponsors and addresses are synthetic. + */ +import { after, before, describe, it } from "node:test"; +import assert from "node:assert"; +import type { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; +import { handler } from "../src/handler"; +import { createTables, startLocal, stopLocal } from "../scripts/local-dynamodb"; +import { getClient } from "../src/db/client"; +import { createTemplate } from "../src/db/templates"; +import { getTask, listTasksByCard } from "../src/db/tasks"; +import { getNewsletterIssue, updateNewsletterIssue } from "../src/db/newsletterIssues"; +import { recordSponsorRouteAfterSend, SponsorRouteError } from "../src/newsletter/sponsorRoute"; +import { setSponsorDocEmailDepsForTests, type SponsorDocEmailDelivery } from "../src/sponsorDocs/emailDelivery"; + +const invoke = (method: string, pathName: string, body?: unknown) => + handler({ httpMethod: method, path: pathName, headers: {}, queryStringParameters: {}, body: body === undefined ? null : JSON.stringify(body) }, {}); +const data = (response: any) => JSON.parse(response.body); +const addDays = (days: number) => new Date(Date.now() + days * 86400000).toISOString().slice(0, 10); + +const sent = (overrides: Partial = {}): SponsorDocEmailDelivery => ({ + state: "sent", from: "orin@datatalks.club", to: "contact@sponsor.example", cc: ["valeriia@datatalks.club"], + subject: "Sponsorship document", messageId: "msg-initial", threadId: "thread-initial", ...overrides, +}); + +describe("sponsor email route", () => { + let client: DynamoDBDocumentClient; + before(async () => { + client = await getClient(await startLocal()); + await createTables(client); + const slotTask = (refId: string) => ({ refId, description: refId, offsetDays: 1, optionalGroup: "primary-slot" }); + await createTemplate(client, { + name: "Newsletter", type: "newsletter", cardTitle: "{date} · #{issue} · {sponsors}", + taskDefinitions: [ + { refId: "write-issue", description: "Write the issue", offsetDays: -3 }, + slotTask("email-sponsor"), slotTask("send-performance-to-sponsor"), slotTask("send-email-sponsor-publication-live"), + ], + }); + }); + after(() => { + setSponsorDocEmailDepsForTests({}); + return stopLocal(); + }); + + const bookedIssue = async (sponsor = "Synthetic Sponsor") => { + const created = await invoke("POST", "/api/newsletter-issues", { + kind: "weekly", publicationDate: addDays(5), issueNumber: 900 + Math.floor(Math.random() * 1000), + slots: { primary: { sponsor }, secondary: {} }, + }); + assert.equal(created.statusCode, 201, created.body); + const issue = data(created); + assert.ok(issue.cardId); + const tasks = await listTasksByCard(client, issue.cardId); + const byRef = (ref: string) => tasks.find((task) => task.templateTaskRef === ref)!; + return { issue, byRef }; + }; + + it("sending email-sponsor records the route on the slot and copies it onto the send Tasks", async () => { + const { issue, byRef } = await bookedIssue(); + await recordSponsorRouteAfterSend(client, byRef("email-sponsor").id, sent()); + const route = { to: "contact@sponsor.example", cc: ["valeriia@datatalks.club"], sender: "orin@datatalks.club", threadRef: "thread-initial" }; + assert.deepEqual((await getNewsletterIssue(client, issue.id))!.slots.primary!.route, route); + for (const ref of ["send-performance-to-sponsor", "send-email-sponsor-publication-live"]) { + assert.deepEqual((await getTask(client, byRef(ref).id))!.sponsorRoute, route, ref); + } + assert.equal((await getTask(client, byRef("email-sponsor").id))!.sponsorRoute, undefined); + const report = data(await invoke("GET", `/api/newsletter-issues/${issue.id}/report`)); + assert.deepEqual(report.report.route, route); + }); + + it("ignores other Tasks and fails loudly when the send has no thread", async () => { + const { issue, byRef } = await bookedIssue(); + await recordSponsorRouteAfterSend(client, byRef("send-performance-to-sponsor").id, sent()); + assert.equal((await getNewsletterIssue(client, issue.id))!.slots.primary!.route, undefined); + await assert.rejects( + recordSponsorRouteAfterSend(client, byRef("email-sponsor").id, sent({ threadId: undefined })), + (error: Error) => error instanceof SponsorRouteError && /sent \(Gmail message msg-initial\) but its route was not recorded/.test(error.message), + ); + assert.equal((await getNewsletterIssue(client, issue.id))!.slots.primary!.route, undefined); + }); + + it("report --send replies in the recorded thread, closes the Task with the Gmail message id, and refuses without a route", async () => { + const { issue, byRef } = await bookedIssue(); + const withStats = async () => { + const current = (await getNewsletterIssue(client, issue.id))!; + return updateNewsletterIssue(client, current, { + slots: { ...current.slots, primary: { ...current.slots.primary, stats: { clicks: 10, recordedAt: new Date().toISOString() } } }, + }); + }; + await withStats(); + + const missing = await invoke("POST", `/api/newsletter-issues/${issue.id}/report/send`, { slot: "primary", contactName: "Sam" }); + assert.equal(missing.statusCode, 422, missing.body); + assert.equal(data(missing).code, "sponsor_route_missing"); + + await recordSponsorRouteAfterSend(client, byRef("email-sponsor").id, sent()); + const noContact = await invoke("POST", `/api/newsletter-issues/${issue.id}/report/send`, { slot: "primary" }); + assert.equal(noContact.statusCode, 400, noContact.body); + + const sends: Array> = []; + setSponsorDocEmailDepsForTests({ + brokerUrl: "https://dapier.example.test", + credential: "dp_test", + fetchImpl: (async (input: string | URL | Request, init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + const json = (body: unknown) => new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } }); + if (url === "https://dapier.example.test/api/agent/token") return json({ access_token: "ya29.test", provider_account_id: "orin@datatalks.club" }); + if (url.startsWith("https://gmail.googleapis.com/gmail/v1/users/me/threads/thread-initial")) { + return json({ messages: [{ payload: { headers: [{ name: "Subject", value: "Sponsorship document" }, { name: "Message-ID", value: "" }] } }, { payload: { headers: [{ name: "Message-ID", value: "" }] } }] }); + } + if (url === "https://gmail.googleapis.com/gmail/v1/users/me/messages/send") { + sends.push(JSON.parse(String(init?.body))); + return json({ id: "msg-report", threadId: "thread-initial" }); + } + throw new Error(`Unexpected fetch URL: ${url}`); + }) as typeof fetch, + }); + + const ok = await invoke("POST", `/api/newsletter-issues/${issue.id}/report/send`, { slot: "primary", contactName: "Sam" }); + assert.equal(ok.statusCode, 200, ok.body); + assert.equal(sends.length, 1); + assert.equal(sends[0].threadId, "thread-initial"); + const raw = Buffer.from(String(sends[0].raw), "base64url").toString("utf8"); + assert.match(raw, /^To: contact@sponsor\.example$/m); + assert.match(raw, /^Cc: valeriia@datatalks\.club$/m); + assert.match(raw, /^Subject: Re: Sponsorship document$/m); + assert.match(raw, /^In-Reply-To: $/m); + assert.match(raw, /Hi Sam/); + const task = (await getTask(client, byRef("send-performance-to-sponsor").id))!; + assert.equal(task.status, "done"); + assert.match(task.comment || "", /Gmail message ID: msg-report/); + + const again = await invoke("POST", `/api/newsletter-issues/${issue.id}/report/send`, { slot: "primary", contactName: "Sam" }); + assert.equal(again.statusCode, 409, again.body); + assert.equal(sends.length, 1); + }); +}); diff --git a/cli/src/commands/newsletter.mjs b/cli/src/commands/newsletter.mjs index e5c9f7d7..3fce2a0b 100644 --- a/cli/src/commands/newsletter.mjs +++ b/cli/src/commands/newsletter.mjs @@ -73,9 +73,17 @@ Usage Show, set, or pull from Mailchimp the issue metrics and per-slot (primary / secondary) link + LinkedIn + X metrics. With no metric flags it shows the current values. - dataops newsletter report [primary|secondary] [--contact NAME] + dataops newsletter report [primary|secondary] [--contact NAME] [--send] dataops newsletter report --sponsor [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--contact NAME] - Sponsor performance report (template.newsletter.newsletter-performance) + Sponsor performance report (template.newsletter.newsletter-performance). + Prints the booking's To, Cc, sender and Gmail thread. --send + (needs a slot and --contact) replies in that thread from + orin@, then closes send-performance-to-sponsor with the Gmail + message id. A booking with no route is an error: nothing is guessed. + dataops newsletter route --message-id GMAIL_ID + Record the booking's route from an email-sponsor message Orin + already sent (To, Cc, sender, thread), and copy it onto the + slot's send Tasks. Sending email-sponsor does this itself. dataops newsletter sync [--apply] Bring issues in line with a CSV export of the retired schedule sheet's Newsletter tab, every row back to the @@ -546,6 +554,24 @@ export async function metrics(args, io) { return updated; } +function printRoute(io, route) { + if (!route) { + io.print("Route: none recorded (send email-sponsor, or `dataops newsletter route`)"); + return; + } + io.print(`To: ${route.to}`); + io.print(`Cc: ${route.cc.length ? route.cc.join(", ") : "-"}`); + io.print(`From: ${route.sender}`); + io.print(`threadRef: ${route.threadRef}`); +} + +function printReport(io, item) { + io.print(`${item.label} · ${item.publicationDate} · ${item.slot} (${item.sponsor || "open"})`); + printRoute(io, item.route); + io.print(""); + io.print(item.message); +} + export async function report(args, io) { const usage = "dataops newsletter report ( [primary|secondary] | --sponsor ) [--contact ]"; const sponsor = optionalString(args, "sponsor"); @@ -572,8 +598,7 @@ export async function report(args, io) { } for (const [index, item] of result.items.entries()) { if (index > 0) io.print("\n---\n"); - io.print(`${item.label} · ${item.publicationDate} · ${item.slot} (${item.sponsor || "open"})`); - io.print(item.message); + printReport(io, item); } return result; } @@ -585,12 +610,32 @@ export async function report(args, io) { ...(slot ? { slot } : {}), ...(contactName ? { contactName } : {}), }; + if (args.send) { + if (!slot) throw new UsageError(`--send needs a slot. ${usage}`); + if (!contactName) throw new UsageError("--send needs --contact NAME: the greeting is never sent as ."); + const sent = await api.post(`/api/newsletter-issues/${enc(id)}/report/send`, { slot, contactName }); + if (args.json) return sent; + const delivery = sent.emailDelivery; + io.print(`Replied in thread ${sent.route.threadRef} from ${delivery.from} to ${delivery.to}${delivery.cc?.length ? ` (cc ${delivery.cc.join(", ")})` : ""} (Gmail message ${delivery.messageId}).`); + io.print(`Closed task ${sent.taskId}.`); + return sent; + } const result = await api.get(`/api/newsletter-issues/${enc(id)}/report`, { query }); if (args.json) return result; for (const [index, item] of (result.reports || []).entries()) { if (index > 0) io.print("\n---\n"); - io.print(`${item.label} · ${item.publicationDate} · ${item.slot} (${item.sponsor || "open"})`); - io.print(item.message); + printReport(io, item); + } + return result; +} + +export async function route(args, io) { + const { id, position } = slotArgs(args, "dataops newsletter route --message-id GMAIL_ID"); + const messageId = optionalString(args, "message-id") ?? optionalString(args, "messageId"); + if (!messageId) throw new UsageError("--message-id is required: the Gmail message id of the email-sponsor send."); + const result = await apiFor(args).put(`/api/newsletter-issues/${enc(id)}/slots/${position}/route`, { messageId }); + if (!args.json) { + io.print(`Recorded the ${position} route on ${id}: To ${result.route.to}, Cc ${result.route.cc.join(", ") || "-"}, thread ${result.route.threadRef}; copied onto ${result.taskIds.length} task(s).`); } return result; } @@ -973,6 +1018,7 @@ export const commands = { metrics, "pull-mailchimp": pullMailchimp, report, + route, delete: remove, sponsors, book, diff --git a/cli/test/newsletter-report-route.test.mjs b/cli/test/newsletter-report-route.test.mjs new file mode 100644 index 00000000..470e2816 --- /dev/null +++ b/cli/test/newsletter-report-route.test.mjs @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import { describe, test } from "node:test"; + +import { captureIo, fakePortal, isolateConfig } from "./helpers.mjs"; + +isolateConfig(); +const { run } = await import("../src/cli.mjs"); + +const route = { to: "contact@sponsor.example", cc: ["valeriia@datatalks.club"], sender: "orin@datatalks.club", threadRef: "thread-1" }; +const report = (overrides = {}) => ({ + issueId: "issue-1", label: "Weekly #1", publicationDate: "2026-11-02", slot: "primary", sponsor: "Synthetic", message: "Hi ", route, ...overrides, +}); + +describe("newsletter report routing", () => { + test("report prints To, Cc, sender and threadRef", async () => { + fakePortal({ "GET /api/newsletter-issues/issue-1/report": () => ({ reports: [report()] }) }); + const io = captureIo(); + assert.equal(await run(["newsletter", "report", "issue-1"], io), 0); + assert.match(io.text(), /To: contact@sponsor\.example/); + assert.match(io.text(), /Cc: valeriia@datatalks\.club/); + assert.match(io.text(), /From: orin@datatalks\.club/); + assert.match(io.text(), /threadRef: thread-1/); + }); + + test("report says plainly when no route is recorded", async () => { + fakePortal({ "GET /api/newsletter-issues/issue-1/report": () => ({ reports: [report({ route: null })] }) }); + const io = captureIo(); + assert.equal(await run(["newsletter", "report", "issue-1"], io), 0); + assert.match(io.text(), /Route: none recorded/); + }); + + test("--send needs a slot and a contact, then posts and prints the Gmail message id", async () => { + const calls = fakePortal({ + "POST /api/newsletter-issues/issue-1/report/send": () => ({ + taskId: "task-1", route, emailDelivery: { from: route.sender, to: route.to, cc: route.cc, messageId: "gmail-9" }, + }), + }); + const bare = captureIo(); + assert.notEqual(await run(["newsletter", "report", "issue-1", "--send", "--contact", "Sam"], bare), 0); + assert.match(bare.errors.join("\n"), /needs a slot/); + const noContact = captureIo(); + assert.notEqual(await run(["newsletter", "report", "issue-1", "primary", "--send"], noContact), 0); + assert.match(noContact.errors.join("\n"), /--contact/); + assert.equal(calls.length, 0); + + const io = captureIo(); + assert.equal(await run(["newsletter", "report", "issue-1", "primary", "--send", "--contact", "Sam"], io), 0); + assert.deepEqual(calls[0].body, { slot: "primary", contactName: "Sam" }); + assert.match(io.text(), /Replied in thread thread-1 .*Gmail message gmail-9/); + assert.match(io.text(), /Closed task task-1/); + }); + + test("route adopts an email-sponsor message", async () => { + const calls = fakePortal({ + "PUT /api/newsletter-issues/issue-1/slots/primary/route": () => ({ route, taskIds: ["a", "b"] }), + }); + const io = captureIo(); + assert.equal(await run(["newsletter", "route", "issue-1", "primary", "--message-id", "gmail-1"], io), 0); + assert.deepEqual(calls[0].body, { messageId: "gmail-1" }); + assert.match(io.text(), /thread thread-1; copied onto 2 task/); + }); +});