diff --git a/CLAUDE.md b/CLAUDE.md index 0175e09..873bcc5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -143,9 +143,15 @@ face value: unique, which `SubscriberQuery.spec.js` asserts by entry count so a future collision fails loudly instead of dropping a column. - **The server chooses the column order**, not the caller. Parse by header name, never by position. -- **Two columns cannot be exported and are dropped in silence:** `tag_ids` and `group_membership`. - Asking for all 48 returns 46 with no error, so the tool diffs what it asked for against the header - and reports `missing_columns`. This is the same silent-drop hazard as `columnView`. +- **One column cannot be exported and is dropped in silence:** `tag_ids`. Asking for all 48 returns + **47** with no error, so the tool diffs what it asked for against the header and reports + `missing_columns`. This is the same silent-drop hazard as `columnView`. This file said *two* + columns until 2026-09-03 and named `group_membership` as the second; measured that day, all 48 + came back as 47 and `group_membership` carried the value `None`. Whether Substack changed or the + original reading was wrong cannot be told apart after the fact — which is why the tool diffs the + header instead of carrying a list of undeliverable columns, and why the count above is the part + to distrust first. The suite's CSV fixture omitted that header, so the error had a test agreeing + with it: the same shape as the pending-poll mock above. - **The download url is relative to the publication host and cookie-authenticated** (403 without it, not pre-signed) and answers **CSV, not JSON**. `handleResponse` unconditionally `JSON.parse`s, so `readBody` was split out of it and `requestUrl({parse: 'text'})` is the raw path. `requestUrl` also diff --git a/src/server.js b/src/server.js index 0694f47..a7bd970 100644 --- a/src/server.js +++ b/src/server.js @@ -93,8 +93,8 @@ export const tools = { "seen, post views, unique posts seen, comments, shares, links clicked, days active and " + "activity rating. Takes the same filters as list_subscribers and covers the whole matching " + "set — there is no paging. Substack generates the file asynchronously, so this waits for it " + - "and returns the parsed records. Two columns cannot be exported and are reported in " + - "`missing_columns` rather than failing: tag_ids and group_membership.", + "and returns the parsed records. One column cannot be exported and is reported in " + + "`missing_columns` rather than failing: tag_ids.", schema: exportSubscribersSchema, handler: exportSubscribersHandler, }, diff --git a/src/tools/export_subscribers.js b/src/tools/export_subscribers.js index e189961..98f4e36 100644 --- a/src/tools/export_subscribers.js +++ b/src/tools/export_subscribers.js @@ -165,8 +165,11 @@ export const exportSubscribersHandler = async (args, {sleep = sleepSeconds} = {} const csv = await substack_api.downloadExport(url); const {columns: returned, unmapped, subscribers} = recordsFromCsv(csv); - // Unsupported columns are dropped by the API with no error at all — `group_membership` and - // `tag_ids` never come back. Reporting the difference is the only way the caller learns it. + // An unsupported column is dropped by the API with no error at all — `tag_ids` never comes back. + // Reporting the difference is the only way the caller learns it. The check stays a diff rather + // than a hardcoded list of one: which columns the export refuses is not ours to know statically. + // `group_membership` was recorded here as undeliverable and measurably is not — whether Substack + // changed or the original reading was wrong cannot be told apart now, which is the argument. const missing_columns = columns.filter((column) => !returned.includes(column)); if (missing_columns.length > 0) { diff --git a/src/tools/export_subscribers.spec.js b/src/tools/export_subscribers.spec.js index 3bee63b..71a5f5c 100644 --- a/src/tools/export_subscribers.spec.js +++ b/src/tools/export_subscribers.spec.js @@ -194,16 +194,18 @@ describe('exportSubscribersHandler — parsing the CSV back', () => { assert.deepEqual(result.columns, [ 'user_email_address', 'user_name', 'subscription_created_at', 'num_email_opens_last_30d', 'num_web_post_views', 'total_revenue_generated', - 'activity_rating', 'country', + 'activity_rating', 'country', 'group_membership', ]); }); - // Verified against the live API: asking for all 48 returns 46. `group_membership` and `tag_ids` - // are dropped with no error at all, so a caller told only "success" would believe it had them. + // Measured against the live API 2026-09-03: asking for all 48 returns 47, and the only column + // that never comes back is `tag_ids`. It is dropped with no error at all, so a caller told only + // "success" would believe it had the tags. `group_membership` exports fine — this test and the + // CSV fixture both used to claim otherwise. test('names the requested columns that never came back', async () => { - const result = await run({columns: ['user_email_address', 'user_name', 'tag_ids', 'group_membership']}); + const result = await run({columns: ['user_email_address', 'tag_ids', 'group_membership']}); - assert.deepEqual(result.missing_columns, ['tag_ids', 'group_membership']); + assert.deepEqual(result.missing_columns, ['tag_ids']); }); test('missing_columns is empty when everything asked for arrived', async () => { @@ -389,7 +391,8 @@ describe('exportSubscribersHandler — logging', () => { const done = find(lines, 'export_subscribers.done'); assert.equal(done.count, 2); - assert.equal(done.columns, 8); + // The nine headers EXPORT_CSV carries, not the number requested. + assert.equal(done.columns, 9); }); test('records each poll that found the export unfinished', async () => { diff --git a/test/helpers/msw-server.js b/test/helpers/msw-server.js index 661f467..c75e6cf 100644 --- a/test/helpers/msw-server.js +++ b/test/helpers/msw-server.js @@ -42,11 +42,16 @@ export const EXPORT_FILE_URL = `${TEST_ENV.SUBSTACK_PUBLICATION_URL}${EXPORT_FIL * A CSV shaped exactly like a real export: the header carries human LABELS rather than column keys, * the server's own column order (not the requested one), a quoted currency value instead of a * number, and a name containing a comma — the case a `split(',')` gets wrong. + * + * `Group membership` is in the header because the live API returns it: measured 2026-09-03, all 48 + * columns requested came back as 47 with only `tag_ids` missing, and the value is the word "None" + * rather than an empty cell. This fixture omitted the header, which is how the repo came to claim + * the column was undeliverable and had a passing test to prove it. */ export const EXPORT_CSV = [ - 'Email,Name,Start date,Emails opened (30d),Post views,Revenue,Activity,Country', - 'one@example.com,One,2026-07-29T22:07:50.299Z,2,1,"€0.00",5,BR', - 'two@example.com,"Two, Junior",2026-06-01T10:00:00.000Z,0,7,"€50.00",3,IT', + 'Email,Name,Start date,Emails opened (30d),Post views,Revenue,Activity,Country,Group membership', + 'one@example.com,One,2026-07-29T22:07:50.299Z,2,1,"€0.00",5,BR,None', + 'two@example.com,"Two, Junior",2026-06-01T10:00:00.000Z,0,7,"€50.00",3,IT,None', ].join('\n'); export const DRAFT_RESPONSE = {