From be05c49b968f9799b96d92cbcc0d9b0080d55311 Mon Sep 17 00:00:00 2001 From: leemour Date: Wed, 18 Nov 2026 14:36:39 +0100 Subject: [PATCH 1/3] feat(crm): a manifest source reads Attio's deal stages, typed own fields and a live people list A pipeline call may answer the stages themselves (stages.each ".", grouped by stages.pipeline or one pipeline), and hooks.pipelines..outcome decides what the data cannot say. custom.typed reads the account's own fields from typed entries with their history; fields.keep takes { path, is } as data. entities.person|organization.search gives the source a live list for people list and organizations list. extend gets the service's call. The Attio pilot needs all four to work as bitrix24-cli does: deals in the copy, contacts and companies read live. Co-Authored-By: Claude Opus 5.5 --- changes/feat-manifest-attio-deals.md | 16 ++ docs/dev/CRM.md | 15 +- src/crm/manifest/manifest.ts | 59 ++++- src/crm/manifest/map.ts | 45 +++- src/crm/manifest/paths.ts | 12 +- src/crm/manifest/source.ts | 93 ++++++- src/crm/manifest/typed-values.test.ts | 352 ++++++++++++++++++++++++++ src/crm/testing/manifest-fake.ts | 75 +++++- 8 files changed, 630 insertions(+), 37 deletions(-) create mode 100644 changes/feat-manifest-attio-deals.md create mode 100644 src/crm/manifest/typed-values.test.ts diff --git a/changes/feat-manifest-attio-deals.md b/changes/feat-manifest-attio-deals.md new file mode 100644 index 000000000..fe0f6f9a3 --- /dev/null +++ b/changes/feat-manifest-attio-deals.md @@ -0,0 +1,16 @@ +### Added + +- **A CRM source from a manifest reads a pipeline answered as a flat list of stages** (`stages.each: "."`, grouped by + `stages.pipeline` or one pipeline), and a hook `pipelines..outcome` decides a stage's outcome the data cannot + say. Attio's deal stages and Pipedrive's `/stages` answer this way. +- **`custom.typed` reads the account's own fields from typed entries with their history**, as Attio keeps them: the + current entry's value by its type (`TYPED_VALUE_PATHS`, overridable), several current entries as a list. + `fields.keep` takes `{ path, is }`, so a config file can say which fields are the account's own. +- **`entities.person.search` and `entities.organization.search` give a manifest source a live `list`**, for `people + list` and `organizations list`: words through the search call's `"{query}"`, none through `list`. A condition it + cannot send is refused, not dropped. +- `hooks.extend` gets the service's `call` as its second argument. + +### Fixed + +- `crmManifestFake` matches a call whatever page size and offset the source asked for: a live list sets its own. diff --git a/docs/dev/CRM.md b/docs/dev/CRM.md index 280c6b577..1474b4571 100644 --- a/docs/dev/CRM.md +++ b/docs/dev/CRM.md @@ -242,8 +242,17 @@ hooks })` builds the `CrmSource` (design and evidence: cli-private `plans/2026-1 changes mid-round moves nothing under the next page (a service's own page token is an offset into a moving list). Past `cap` rows at one time the round answers `reset`. - What a manifest cannot say is a hook: `after` per kind for one record (async, with the metered `call`), - `extend(base)` for whole methods (`self`, `activities`, a live `list`). Bitrix24's `mine` scope is that kind of - policy, and stays code. + `extend(base, { call })` for whole methods (`self`, `activities`), `pipelines..outcome(stage, pipeline, + found)` for a stage's outcome the data cannot say (Attio names a lost stage only by its title). Bitrix24's `mine` + scope is that kind of policy, and stays code. +- A pipeline call that answers the stages themselves (Attio's statuses, Pipedrive's `/stages`) sets `stages.each` to + `.`: the stages group by `stages.pipeline`, or are one pipeline, and the pipeline's map reads its first stage. +- `custom.typed` reads values kept as a list of typed entries with their history (Attio's `values.`): the + entry's type (`by`) picks the path of its value from `TYPED_VALUE_PATHS` or `typed.paths`, an entry with a time at + `current` is history, and several current entries are a list. `fields.keep` takes `{ path, is }` as data. +- `entities..search` names a call with `"{query}"`: the source then has a live `list` for + `people list` and `organizations list`, words through `search` and none through `list`, paged by the operation's + offset. A condition it cannot send is refused (`query.unsupported-field`), never dropped. - `crmManifestProblems(manifest, api, definitions)` checks every mapped path against the schema each operation answers; a CLI runs it in a test, so a wrong path fails before any sync. - `hooks.enrich` reads more for a whole page in one batch before the map; `optional` makes a refused kind a gap; @@ -254,7 +263,7 @@ hooks })` builds the `CrmSource` (design and evidence: cli-private `plans/2026-1 sends the one operation, refuses a token the answer does not call `active`, and keeps the id and name the answer holds. `hostsFetch(hosts, environment.fetch)` is the fetch for it and the source: only the CLI's hosts, over a test's fake when there is one. A CLI made from the config holds no check of its own. -- Not yet: deletions named in the answer, a live `list`. +- Not yet: deletions named in the answer. - `src/crm/manifest/manifest.test.ts` writes HubSpot's contacts, companies and deals as a manifest and runs `crmSourceContract`, faults included, against a strict fake of HubSpot's API. diff --git a/src/crm/manifest/manifest.ts b/src/crm/manifest/manifest.ts index 8d6763088..ca63ed072 100644 --- a/src/crm/manifest/manifest.ts +++ b/src/crm/manifest/manifest.ts @@ -58,6 +58,11 @@ export interface CrmManifestEntity { ids?: CrmOperationCall /** One row by `{id}`, for a live read of a contact or company; a 403 or 404 is no row. */ get?: CrmOperationCall + /** + * Contacts or companies whose name holds `{query}`, for a live `people list` and `organizations list`; a list with no + * words reads `list`. Both page by the operation's own `pagination`. Absent, the source has no live list. + */ + search?: CrmOperationCall /** * The service names the fields a read returns (HubSpot `properties=`, Bitrix24 `select`): `{select}` becomes the * names under `from` that `map` reads, the custom field codes, and `also`. @@ -70,7 +75,7 @@ export interface CrmManifestEntity { } map: { readonly [F in keyof CrmRecords[K]]?: CrmRule } & { amount?: CrmRule; currency?: CrmRule } /** The account's own fields' values, by the codes `fields` lists, under this object of the row. */ - custom?: { from: string; split?: string } + custom?: { from: string; split?: string; typed?: CrmTypedValues } /** * A kind the plan or the sign-in may not have (HubSpot's leads): the service refusing it is a gap with this text and * scope, and the other kinds still sync. @@ -78,11 +83,28 @@ export interface CrmManifestEntity { optional?: { message: string; scope?: string } } +/** + * Values kept as a list of typed entries, each naming its type (Attio's `values.`): the entry's type picks where + * its value is, and a replaced entry, one with a time at `current`, is skipped. + */ +export interface CrmTypedValues { + /** Where an entry names its type: `attribute_type`. */ + by: string + /** A time an entry was replaced at; an entry with one is history, not the value. */ + current?: string + /** Where the value is for each type, over `TYPED_VALUE_PATHS`. */ + paths?: Readonly> +} + export interface CrmManifestPipelines { call: CrmOperationCall + /** Read from the pipeline's row, or from its first stage when the call answers stages (`stages.each` is `.`). */ map: { externalId: CrmRule; name: CrmRule } stages: { + /** The stages in a pipeline's row; `.` when the call answers the stages themselves, one row each. */ each: string + /** With `each` `.`: the stage's pipeline, by which the stages are grouped; absent, they are all one pipeline. */ + pipeline?: string map: { externalId: CrmRule; name: CrmRule; position?: CrmRule } /** Which stages are won or lost by one field; any other is open. A rule a field cannot say is a function. */ outcome: @@ -105,7 +127,7 @@ export interface CrmManifestFields { | ((raw: CrmRaw) => { id: string; label: string }[] | undefined) } /** Which listed fields are the account's own, not the service's built-in ones the map already reads. */ - keep?: (raw: CrmRaw) => boolean + keep?: { path: string; is: unknown } | ((raw: CrmRaw) => boolean) } /** A CRM account described as data: what to call for each kind and where each of our fields is in the answer. */ @@ -143,8 +165,14 @@ export interface CrmManifestHooks { enrich?: { readonly [K in CrmKind]?: (rows: CrmRaw[], service: { call: ServiceApiCall }) => Promise } + /** The outcome of a stage the data cannot say (Attio names a lost stage only by its title); `found` is the data's. */ + pipelines?: { + readonly [K in DealKind]?: { + outcome?: (stage: CrmRaw, pipeline: CrmRaw, found: StageOutcome) => StageOutcome + } + } /** Adds or replaces source methods (`self`, `activities`, a live `list`); `base` is what the manifest built. */ - extend?: (base: CrmSource) => Partial + extend?: (base: CrmSource, service: { call: ServiceApiCall }) => Partial } const path = v.pipe(v.string(), v.regex(/^[^.\s]+(\.[^.\s]+)*$/, "a dotted path such as properties.name")) @@ -189,6 +217,7 @@ const entity = v.strictObject({ ), ids: v.optional(call), get: v.optional(call), + search: v.optional(call), select: v.optional( v.strictObject({ from: path, @@ -197,7 +226,15 @@ const entity = v.strictObject({ }), ), map: v.record(v.string(), rule), - custom: v.optional(v.strictObject({ from: path, split: v.optional(v.string()) })), + custom: v.optional( + v.strictObject({ + from: path, + split: v.optional(v.string()), + typed: v.optional( + v.strictObject({ by: path, current: v.optional(path), paths: v.optional(v.record(v.string(), path)) }), + ), + }), + ), optional: v.optional(v.strictObject({ message: v.pipe(v.string(), v.minLength(1)), scope: v.optional(v.string()) })), }) @@ -205,7 +242,8 @@ const pipelines = v.strictObject({ call, map: v.strictObject({ externalId: rule, name: rule }), stages: v.strictObject({ - each: path, + each: v.union([v.literal("."), path]), + pipeline: v.optional(path), map: v.strictObject({ externalId: rule, name: rule, position: v.optional(rule) }), outcome: v.union([ v.strictObject({ path, won: v.optional(v.array(v.string())), lost: v.optional(v.array(v.string())) }), @@ -225,7 +263,7 @@ const fields = v.strictObject({ filterable: v.optional(rule), options: v.optional(v.union([v.strictObject({ each: path, id: path, label: path }), v.function()])), }), - keep: v.optional(v.function()), + keep: v.optional(v.union([v.strictObject({ path, is: v.unknown() }), v.function()])), }) const kind = v.picklist(CRM_KINDS) @@ -308,6 +346,13 @@ export const parseCrmManifest = (input: unknown, api: ServiceApiManifest): CrmMa if (!hasPlaceholder([one.get.path, one.get.query, one.get.body], "{id}")) throw invalid(`entities.${kind}.get`, 'no "{id}" in its path, query or body') } + if (one.search) { + if (kind !== "person" && kind !== "organization") + throw invalid(`entities.${kind}.search`, "only contacts and companies are read live") + checkCall(`entities.${kind}.search`, one.search, true) + if (!hasPlaceholder([one.search.path, one.search.query, one.search.body], "{query}")) + throw invalid(`entities.${kind}.search`, 'no "{query}" in its path, query or body') + } if (one.map.externalId === undefined) throw invalid(`entities.${kind}.map.externalId`, "every kind maps its id") const fields = [...Object.keys(FIELDS[kind]), "amount", "currency"] for (const field of Object.keys(one.map)) @@ -323,6 +368,8 @@ export const parseCrmManifest = (input: unknown, api: ServiceApiManifest): CrmMa const one = manifest.pipelines?.[kind] if (!one) throw invalid(`pipelines.${kind}`, `${kind} is in kinds, and every deal or lead stands in a pipeline`) checkCall(`pipelines.${kind}.call`, one.call, true) + if (one.stages.pipeline !== undefined && one.stages.each !== ".") + throw invalid(`pipelines.${kind}.stages.pipeline`, 'groups stages only when the call answers them: each "."') } for (const [kind, one] of Object.entries(manifest.fields ?? {})) if (one) checkCall(`fields.${kind}.call`, one.call, true) diff --git a/src/crm/manifest/map.ts b/src/crm/manifest/map.ts index 433f1a937..6f41ea5b3 100644 --- a/src/crm/manifest/map.ts +++ b/src/crm/manifest/map.ts @@ -1,7 +1,7 @@ import type { CrmField, CrmKind, CrmRecords } from "../../domain/crm.js" import { DEAL_STATUSES } from "../../domain/deals.js" import { amountOf, type CrmRaw, crmValuesOf, epoch, fullName, id, isRaw, number, text } from "../values.js" -import type { CrmRule } from "./manifest.js" +import type { CrmRule, CrmTypedValues } from "./manifest.js" /** The service sent nothing at this path: the field is left out, so the stored copy keeps what it held. */ export const MISSING = Symbol("missing") @@ -211,13 +211,50 @@ export const mapFields = ( return out } +/** Where a typed entry keeps its value, by its type, as Attio names them; a manifest's `typed.paths` adds or replaces. */ +export const TYPED_VALUE_PATHS: Readonly> = { + text: "value", + number: "value", + checkbox: "value", + date: "value", + timestamp: "value", + rating: "value", + currency: "currency_value", + status: "status.title", + select: "option.title", + "record-reference": "target_record_id", + "actor-reference": "referenced_actor_id", + domain: "domain", + "email-address": "email_address", + "phone-number": "phone_number", + "personal-name": "full_name", + interaction: "interacted_at", +} + +/** The current entries' values: one alone is the value, several a list, none no value. */ +const typedOf = (entries: unknown, typed: CrmTypedValues): unknown => { + const values = listOf(entries).flatMap((entry) => { + if (!isRaw(entry)) return [] + if (typed.current !== undefined && at(entry, typed.current) != null) return [] + const type = text(at(entry, typed.by)) + const path = type === null ? undefined : (typed.paths?.[type] ?? TYPED_VALUE_PATHS[type]) + const value = path === undefined ? null : at(entry, path) + return value === null || value === undefined || value === "" ? [] : [value] + }) + return values.length > 1 ? values : (values[0] ?? null) +} + /** The account's own fields' values under `from`, labelled by their definitions. */ -const customOf = (raw: CrmRaw, custom: { from: string; split?: string }, defined: readonly CrmField[]) => { +const customOf = ( + raw: CrmRaw, + custom: { from: string; split?: string; typed?: CrmTypedValues }, + defined: readonly CrmField[], +) => { const under = at(raw, custom.from) if (!isRaw(under)) return undefined const values: CrmRaw = {} for (const field of defined) { - const value = under[field.code] + const value = custom.typed ? typedOf(under[field.code], custom.typed) : under[field.code] if (value === null || value === undefined || value === "") continue values[field.code] = field.multiple && custom.split && typeof value === "string" ? value.split(custom.split) : value } @@ -226,7 +263,7 @@ const customOf = (raw: CrmRaw, custom: { from: string; split?: string }, defined export interface RecordRules { map: Readonly> - custom?: { from: string; split?: string } + custom?: { from: string; split?: string; typed?: CrmTypedValues } } /** One row as our record of the kind, or null for a row without an id. */ diff --git a/src/crm/manifest/paths.ts b/src/crm/manifest/paths.ts index 1c6319b87..d5404737a 100644 --- a/src/crm/manifest/paths.ts +++ b/src/crm/manifest/paths.ts @@ -143,20 +143,24 @@ export const crmManifestProblems = ( ...(entity.map as Record), ...(entity.custom ? { "(custom)": entity.custom.from } : {}), } - for (const call of [entity.list, entity.changes, entity.get]) if (call) check(`entities.${kind}.map`, call, rules) + for (const call of [entity.list, entity.changes, entity.get, entity.search]) + if (call) check(`entities.${kind}.map`, call, rules) } for (const [kind, pipelines] of Object.entries(manifest.pipelines ?? {})) { if (!pipelines) continue - check(`pipelines.${kind}.map`, pipelines.call, { ...pipelines.map, "(stages)": pipelines.stages.each }) - const { outcome } = pipelines.stages + const { outcome, each, pipeline } = pipelines.stages + // A call that answers stages has one row per stage: the pipeline's map and the stage's read the same row. + const flat = each === "." + check(`pipelines.${kind}.map`, pipelines.call, flat ? pipelines.map : { ...pipelines.map, "(stages)": each }) check( `pipelines.${kind}.stages.map`, pipelines.call, { ...pipelines.stages.map, ...(typeof outcome === "object" && "path" in outcome ? { outcome: outcome.path } : {}), + ...(pipeline ? { "(pipeline)": pipeline } : {}), }, - pipelines.stages.each, + flat ? undefined : each, ) } for (const [kind, fields] of Object.entries(manifest.fields ?? {})) { diff --git a/src/crm/manifest/source.ts b/src/crm/manifest/source.ts index 6d27e538c..52c5bba0a 100644 --- a/src/crm/manifest/source.ts +++ b/src/crm/manifest/source.ts @@ -5,8 +5,9 @@ import type { ServiceApiCall } from "../../cli/service-api-call.js" import { nextPage, parameterValue, type ServiceApiManifest, withValue } from "../../cli/service-api-manifest.js" import type { CrmField, CrmKind, CrmPipeline, CrmRecords, CrmUser } from "../../domain/crm.js" import type { StageOutcome } from "../../domain/deals.js" -import type { CrmCallContext, CrmChanges, CrmIdPage, CrmLiveKind, CrmSource } from "../source.js" +import type { CrmCallContext, CrmChanges, CrmIdPage, CrmLiveKind, CrmLiveQuery, CrmSource } from "../source.js" import { type CrmRaw, id, isRaw, number, rawsOf, text } from "../values.js" +import { crmWindowOf, walk } from "../walk.js" import { type CrmAfterHook, type CrmChangesCall, @@ -29,6 +30,8 @@ export interface CrmManifestSourceOptions { } const OVERLAP_MS = 10 * 60_000 +/** The most rows a live list asks for in one request. */ +const MOST_LIVE = 100 /** How far one read of every page goes: a service that never stops paging must not hold a sync forever. */ const MOST_PAGES = 500 @@ -36,15 +39,17 @@ interface Slots { since?: string select?: readonly string[] id?: string + query?: string } const DROP = Symbol("drop") -/** Fills `"{since}"`, `"{select}"` and `"{id}"`; an empty select drops the parameter it stands for. */ +/** Fills `"{since}"`, `"{select}"`, `"{id}"` and `"{query}"`; an empty select drops the parameter it stands for. */ const fill = (value: unknown, slots: Slots): unknown => { if (value === "{since}") return slots.since ?? DROP if (value === "{select}") return slots.select?.length ? [...slots.select] : DROP if (value === "{id}") return slots.id ?? DROP + if (value === "{query}") return slots.query ?? DROP if (Array.isArray(value)) return value.map((one) => fill(one, slots)).filter((one) => one !== DROP) if (isRaw(value)) return Object.fromEntries( @@ -318,6 +323,47 @@ export const crmSourceFromManifest = (input: unknown, options: CrmManifestSource return next === undefined ? { items, hasMore: false } : { items, hasMore: true, nextCursor: next } } + const live = (["person", "organization"] as const).filter((kind) => manifest.entities[kind]?.search) + + /** + * A live page of contacts or companies, newest first as the service sorts them: words read `search`, none `list`. + * Only the words narrow it; a condition the manifest cannot send is refused rather than left out of the answer. + */ + async function readLive(kind: K, query: CrmLiveQuery, context?: CrmCallContext) { + const entity = entityOf(kind) + if (!entity.search) + throw new CliError("validation_error", `${manifest.provider} reads no ${kind} live`, { + reason: "query.unsupported-field", + }) + if (query.conditions.length) + throw new CliError( + "validation_error", + `${manifest.provider} narrows a live list only by words in the name, not by ${query.conditions.map(({ field }) => field).join(", ")}`, + { reason: "query.unsupported-field" }, + ) + const words = query.text?.trim() || undefined + const call = words ? entity.search : entity.list + const send = metered(context) + const defined = entity.custom ? await fieldsOf(kind, send, context) : [] + const slots: Slots = { select: selectOf(kind, defined, context), ...(words ? { query: words } : {}) } + const pagination = operations.get(call.operation)?.pagination + const largest = pagination?.style === "offset" ? MOST_LIVE : query.limit + 1 + const page = await walk(crmWindowOf(query), largest, async (offset, size) => { + let request = inputOf(call, slots) + if (pagination?.style === "offset") { + request = withValue(request, pagination.offset, offset) + if (pagination.limit) request = withValue(request, pagination.limit, size) + } + const answer = await send(call.operation, request) + const items = call.items ?? (pagination && "items" in pagination ? pagination.items : undefined) + const rows = rawsOf(items === undefined ? answer : at(answer, items)) + const all = pagination?.style === "offset" ? rows : rows.slice(offset, offset + size) + // A short page is not the last: a service may cap a page below the size asked for; only an empty one ends it. + return { rows: all, more: pagination?.style === "offset" ? rows.length > 0 : rows.length > offset + size } + }) + return { ...page, items: await mapped(kind, page.items, defined, send) } + } + const base: CrmSource = { provider: manifest.provider, traits: { kinds: manifest.kinds, removedInChanges: false }, @@ -325,9 +371,10 @@ export const crmSourceFromManifest = (input: unknown, options: CrmManifestSource async pipelines(kind, context) { const spec = manifest.pipelines?.[kind] if (!spec) return [] + const outcome = hooks.pipelines?.[kind]?.outcome try { const rows = await readAll(spec.call, metered(context)) - return rows.flatMap((raw) => toPipeline(spec, raw) ?? []) + return groupsOf(spec, rows).flatMap(({ raw, stages }) => toPipeline(spec, raw, stages, outcome) ?? []) } catch (error) { if (optionalRefused(kind, error, context)) return [] throw error @@ -357,6 +404,8 @@ export const crmSourceFromManifest = (input: unknown, options: CrmManifestSource throw error } }, + ...(live.length ? { list: readLive } : {}), + async get(kind: K, externalId: string, context?: CrmCallContext) { const entity = entityOf(kind) if (!entity.get) return null @@ -398,24 +447,47 @@ export const crmSourceFromManifest = (input: unknown, options: CrmManifestSource } : {}), } - return { ...base, ...hooks.extend?.(base) } + return { ...base, ...hooks.extend?.(base, { call: options.call }) } } +type OutcomeHook = (stage: CrmRaw, pipeline: CrmRaw, found: StageOutcome) => StageOutcome + const outcomeOf = (spec: CrmManifestPipelines["stages"]["outcome"], stage: CrmRaw, pipeline: CrmRaw): StageOutcome => { if (typeof spec === "function") return spec(stage, pipeline) if ("value" in spec) return spec.value - const found = text(at(stage, spec.path)) ?? "" + const value = at(stage, spec.path) + const found = typeof value === "boolean" ? String(value) : (text(value) ?? "") if (spec.won?.includes(found)) return "won" return spec.lost?.includes(found) ? "lost" : "open" } -const toPipeline = (spec: CrmManifestPipelines, raw: CrmRaw): CrmPipeline | null => { +/** + * Each pipeline's row and its stages. A call that answers stages (`each` `.`) is grouped by `stages.pipeline`, or is + * one pipeline; its first stage is the row the pipeline's own map reads. + */ +const groupsOf = (spec: CrmManifestPipelines, rows: readonly CrmRaw[]) => { + if (spec.stages.each !== ".") return rows.map((raw) => ({ raw, stages: rawsOf(at(raw, spec.stages.each)) })) + const by = spec.stages.pipeline + const groups = new Map() + for (const row of rows) { + const key = by === undefined ? "" : (id(at(row, by)) ?? "") + groups.set(key, [...(groups.get(key) ?? []), row]) + } + return [...groups.values()].map((stages) => ({ raw: stages[0] as CrmRaw, stages })) +} + +const toPipeline = ( + spec: CrmManifestPipelines, + raw: CrmRaw, + stages: readonly CrmRaw[], + hook?: OutcomeHook, +): CrmPipeline | null => { const { externalId, name } = mapFields(spec.map, { externalId: "id", name: "text" }, raw) if (typeof externalId !== "string") return null return { externalId, name: typeof name === "string" ? name : externalId, - stages: rawsOf(at(raw, spec.stages.each)).flatMap((stage, place) => { + stages: stages.flatMap((stage, place) => { const found = mapFields(spec.stages.map, { externalId: "id", name: "text", position: "text" }, stage) if (typeof found.externalId !== "string") return [] return [ @@ -423,7 +495,9 @@ const toPipeline = (spec: CrmManifestPipelines, raw: CrmRaw): CrmPipeline | null externalId: found.externalId, name: typeof found.name === "string" ? found.name : found.externalId, position: number(found.position) ?? place, - outcome: outcomeOf(spec.stages.outcome, stage, raw), + outcome: hook + ? hook(stage, raw, outcomeOf(spec.stages.outcome, stage, raw)) + : outcomeOf(spec.stages.outcome, stage, raw), }, ] }), @@ -431,7 +505,8 @@ const toPipeline = (spec: CrmManifestPipelines, raw: CrmRaw): CrmPipeline | null } const toField = (spec: CrmManifestFields, raw: CrmRaw): CrmField | null => { - if (spec.keep && !spec.keep(raw)) return null + if (spec.keep && !(typeof spec.keep === "function" ? spec.keep(raw) : at(raw, spec.keep.path) === spec.keep.is)) + return null const { options: optionsRule, ...rules } = spec.map const found = mapFields( rules, diff --git a/src/crm/manifest/typed-values.test.ts b/src/crm/manifest/typed-values.test.ts new file mode 100644 index 000000000..b95b67ad3 --- /dev/null +++ b/src/crm/manifest/typed-values.test.ts @@ -0,0 +1,352 @@ +import { isCliError } from "@wirecat/cli-base" +import type { ManifestOperation, SchemaNode } from "@wirecat/cli-base/codegen" +import { describe, expect, it } from "vitest" +import type { ServiceApiManifest } from "../../cli/service-api-manifest.js" +import { crmManifestContract } from "../testing/manifest-fake.js" +import type { CrmRaw } from "../values.js" +import { type CrmManifest, type CrmManifestHooks, parseCrmManifest } from "./manifest.js" +import { crmManifestProblems } from "./paths.js" +import { crmSourceFromManifest } from "./source.js" + +// A trimmed copy of Attio's API: records whose attributes are lists of typed entries with their history, a deal +// pipeline answered as a flat list of stages, attribute definitions, and a records query that pages by offset in +// its body and filters by a name. + +const str: SchemaNode = { type: "string" } +const int: SchemaNode = { type: "integer" } +const bool: SchemaNode = { type: "boolean" } +const obj = (properties: Record, required: string[] = []): SchemaNode => ({ + type: "object", + properties, + required, +}) +const list = (items: SchemaNode): SchemaNode => ({ type: "array", items }) +const ref = (name: string): SchemaNode => ({ type: "ref", ref: name }) + +const definitions: Record = { + Entry: obj({ + attribute_type: str, + active_until: { ...str, nullable: true }, + value: str, + full_name: str, + email_address: str, + target_record_id: str, + currency_value: { type: "number" }, + currency_code: str, + status: obj({ id: obj({ status_id: str }), title: str }), + option: obj({ title: str }), + }), + Row: obj( + { + id: obj({ record_id: str }, ["record_id"]), + created_at: str, + values: { type: "object", properties: {}, required: [], additionalProperties: list(ref("Entry")) }, + }, + ["id", "values"], + ), + Records: obj({ data: list(ref("Row")) }, ["data"]), + One: obj({ data: ref("Row") }, ["data"]), + Query: obj({ + filter: { type: "object", properties: {}, required: [], additionalProperties: { type: "unknown" } }, + limit: int, + offset: int, + }), + Statuses: obj( + { data: list(obj({ id: obj({ status_id: str }), title: str, celebration_enabled: bool, is_archived: bool })) }, + ["data"], + ), + Attributes: obj({ data: list(obj({ api_slug: str, title: str, type: str, is_system_attribute: bool })) }, ["data"]), +} + +const path = (name: string) => ({ name, in: "path", required: true, schema: str }) as const + +const op = (id: string, method: "GET" | "POST", url: string, extra: Partial): ManifestOperation => ({ + id, + command: id, + binding: { kind: "http", method, path: url }, + effect: "read", + tags: [], + parameters: [], + ...extra, +}) + +const api: ServiceApiManifest = { + operations: [ + op("query", "POST", "/v2/objects/{object}/records/query", { + parameters: [path("object")], + request: { required: false, confidence: "contract", schema: "Query" }, + response: { confidence: "contract", schema: "Records" }, + pagination: { + style: "offset", + offset: { in: "body", name: "offset" }, + limit: { in: "body", name: "limit" }, + items: "data", + }, + }), + op("get", "GET", "/v2/objects/{object}/records/{record_id}", { + parameters: [path("object"), path("record_id")], + response: { confidence: "contract", schema: "One" }, + }), + op("statuses", "GET", "/v2/objects/{object}/attributes/{attribute}/statuses", { + parameters: [path("object"), path("attribute")], + response: { confidence: "contract", schema: "Statuses" }, + }), + op("attributes", "GET", "/v2/objects/{object}/attributes", { + parameters: [path("object")], + response: { confidence: "contract", schema: "Attributes" }, + }), + ], + servers: [{ url: "https://api.attio.test" }], + securitySchemes: { token: { type: "http", scheme: "bearer" } }, + security: [{ token: [] }], +} + +const records = (object: string, filter: unknown) => ({ + list: { operation: "query", path: { object }, body: { limit: 2 }, items: "data" }, + search: { operation: "query", path: { object }, body: { limit: 2, filter }, items: "data" }, + get: { operation: "get", path: { object, record_id: "{id}" }, items: "data" }, + custom: { from: "values", typed: { by: "attribute_type", current: "active_until" } }, +}) + +const attributes = (object: string) => ({ + call: { operation: "attributes", path: { object }, items: "data" }, + map: { code: "api_slug", label: "title", type: "type" }, + keep: { path: "is_system_attribute", is: false }, +}) + +const MANIFEST: CrmManifest = { + provider: "attio", + kinds: ["organization", "person", "deal"], + entities: { + organization: { + ...records("companies", { name: { $contains: "{query}" } }), + map: { externalId: "id.record_id", name: "values.name.0.value", createdAt: "created_at" }, + }, + person: { + ...records("people", { name: { full_name: { $contains: "{query}" } } }), + map: { + externalId: "id.record_id", + name: "values.name.0.full_name", + emails: { each: "values.email_addresses", value: "email_address" }, + createdAt: "created_at", + }, + }, + deal: { + list: { operation: "query", path: { object: "deals" }, body: { limit: 2 }, items: "data" }, + custom: { from: "values", typed: { by: "attribute_type", current: "active_until" } }, + map: { + externalId: "id.record_id", + title: "values.name.0.value", + pipeline: { value: "deals" }, + stage: "values.stage.0.status.id.status_id", + amount: "values.value.0.currency_value", + currency: "values.value.0.currency_code", + createdAt: "created_at", + }, + }, + }, + pipelines: { + deal: { + call: { operation: "statuses", path: { object: "deals", attribute: "stage" }, items: "data" }, + map: { externalId: { value: "deals" }, name: { value: "Deals" } }, + stages: { + each: ".", + map: { externalId: "id.status_id", name: "title" }, + outcome: { path: "celebration_enabled", won: ["true"] }, + }, + }, + }, + fields: { organization: attributes("companies"), person: attributes("people"), deal: attributes("deals") }, +} + +const hooks: CrmManifestHooks = { + pipelines: { deal: { outcome: (stage, _, found) => (stage.title === "Lost" ? "lost" : found) } }, +} + +describe("crmManifestContract over Attio's shapes", () => { + for (const { name, run } of crmManifestContract(MANIFEST, { api, definitions, hooks })) it(name, run) +}) + +const current = { active_until: null } +const replaced = { active_until: "2026-01-01T00:00:00Z" } + +/** A source over a stub that answers each operation from `answers` and records what was asked. */ +const stubbed = (answers: Record unknown>, manifest: CrmManifest = MANIFEST) => { + const asked: { operation: string; body: CrmRaw }[] = [] + const source = crmSourceFromManifest(manifest, { + api, + hooks, + call: async (operation, request) => { + const body = (request.body ? JSON.parse(request.body) : {}) as CrmRaw + asked.push({ operation, body }) + const answer = answers[operation] + if (!answer) throw new Error(`no answer for ${operation}`) + return answer(body) + }, + }) + return { source, asked } +} + +describe("Attio's shapes", () => { + it("passes the path checks: stages read from each row, a search read like the list", () => { + expect(crmManifestProblems(MANIFEST, api, definitions)).toEqual([]) + }) + + it("reads one pipeline from a flat list of stages, won by a flag and lost by the hook", async () => { + const { source } = stubbed({ + statuses: () => ({ + data: [ + { id: { status_id: "s1" }, title: "Lead", celebration_enabled: false }, + { id: { status_id: "s2" }, title: "Won", celebration_enabled: true }, + { id: { status_id: "s3" }, title: "Lost", celebration_enabled: false }, + ], + }), + }) + expect(await source.pipelines("deal")).toEqual([ + { + externalId: "deals", + name: "Deals", + stages: [ + { externalId: "s1", name: "Lead", position: 0, outcome: "open" }, + { externalId: "s2", name: "Won", position: 1, outcome: "won" }, + { externalId: "s3", name: "Lost", position: 2, outcome: "lost" }, + ], + }, + ]) + }) + + it("groups a flat list of stages by the pipeline each names", async () => { + const manifest: CrmManifest = { + ...MANIFEST, + pipelines: { + deal: { + call: { operation: "statuses", path: { object: "deals", attribute: "stage" }, items: "data" }, + map: { externalId: "pipeline", name: "pipeline" }, + stages: { + each: ".", + pipeline: "pipeline", + map: { externalId: "id", name: "title" }, + outcome: { value: "open" }, + }, + }, + }, + } + const { source } = stubbed( + { + statuses: () => ({ + data: [ + { id: "a1", title: "New", pipeline: "A" }, + { id: "b1", title: "New", pipeline: "B" }, + { id: "a2", title: "Talks", pipeline: "A" }, + ], + }), + }, + manifest, + ) + const found = await source.pipelines("deal") + expect(found.map(({ externalId, stages }) => [externalId, stages.map((one) => one.externalId)])).toEqual([ + ["A", ["a1", "a2"]], + ["B", ["b1"]], + ]) + }) + + it("keeps the current entry of each own attribute, a list when several are current, and skips built-in ones", async () => { + const deal = { + id: { record_id: "d1" }, + values: { + name: [{ ...current, attribute_type: "text", value: "Kitchen" }], + priority: [ + { ...replaced, attribute_type: "select", option: { title: "Low" } }, + { ...current, attribute_type: "select", option: { title: "High" } }, + ], + tags: [ + { ...current, attribute_type: "select", option: { title: "A" } }, + { ...current, attribute_type: "select", option: { title: "B" } }, + ], + budget: [{ ...current, attribute_type: "currency", currency_value: 900 }], + }, + } + const { source } = stubbed({ + attributes: () => ({ + data: [ + { api_slug: "name", title: "Name", type: "text", is_system_attribute: true }, + { api_slug: "priority", title: "Priority", type: "select", is_system_attribute: false }, + { api_slug: "tags", title: "Tags", type: "select", is_system_attribute: false }, + { api_slug: "budget", title: "Budget", type: "currency", is_system_attribute: false }, + ], + }), + query: () => ({ data: [deal] }), + }) + const page = await source.changes("deal", null) + const [item] = "items" in page ? page.items : [] + expect(item?.metadata).toEqual({ + priority: { label: "Priority", value: "High" }, + tags: { label: "Tags", value: ["A", "B"] }, + budget: { label: "Budget", value: 900 }, + }) + }) + + it("lists people live: words through search, none through the list, and refuses a filter it cannot send", async () => { + const ann = { id: { record_id: "p1" }, values: { name: [{ ...current, full_name: "Ann Lee" }] } } + const { source, asked } = stubbed({ + attributes: () => ({ data: [] }), + query: (body) => ({ data: body.offset ? [] : [ann] }), + }) + const found = await source.list?.("person", { conditions: [], text: "ann", limit: 1 }) + expect(found?.items.map(({ name }) => name)).toEqual(["Ann Lee"]) + await source.list?.("person", { conditions: [], limit: 5, page: 2 }) + expect(asked.filter(({ operation }) => operation === "query").map(({ body }) => body)).toEqual([ + { limit: 2, filter: { name: { full_name: { $contains: "ann" } } }, offset: 0 }, + { limit: 1, filter: { name: { full_name: { $contains: "ann" } } }, offset: 1 }, + { limit: 6, offset: 5 }, + ]) + const refused = await source + .list?.("person", { conditions: [{ field: "created_at", op: ">", value: 1 }], limit: 1 }) + .catch((error: unknown) => error) + expect(isCliError(refused) && refused.code).toBe("validation_error") + }) + + it("refuses a search without {query}, one on a deal, and a stage group without a flat list", () => { + const person = MANIFEST.entities.person as NonNullable + const refused = (manifest: unknown) => { + try { + parseCrmManifest(manifest, api) + return "" + } catch (error) { + return error instanceof Error ? error.message : String(error) + } + } + expect( + refused({ + ...MANIFEST, + entities: { ...MANIFEST.entities, person: { ...person, search: { ...person.search, body: { limit: 2 } } } }, + }), + ).toContain('entities.person.search: no "{query}"') + const deal = MANIFEST.entities.deal as NonNullable + expect( + refused({ ...MANIFEST, entities: { ...MANIFEST.entities, deal: { ...deal, search: person.search } } }), + ).toContain("only contacts and companies are read live") + const pipeline = MANIFEST.pipelines?.deal as NonNullable["deal"]> + expect( + refused({ + ...MANIFEST, + pipelines: { deal: { ...pipeline, stages: { ...pipeline.stages, each: "data", pipeline: "pipeline" } } }, + }), + ).toContain("groups stages only when the call answers them") + }) + + it("hands extend the service call, so a hook can read what the manifest does not", async () => { + const manifest = parseCrmManifest(MANIFEST, api) + const source = crmSourceFromManifest(manifest, { + api, + call: async (operation) => ({ operation }), + hooks: { + extend: (_, { call }) => ({ + self: async () => + ((await call("get", { path: {}, query: {}, headers: {}, fields: {} })) as CrmRaw).operation as string, + }), + }, + }) + expect(await source.self?.()).toBe("get") + }) +}) diff --git a/src/crm/testing/manifest-fake.ts b/src/crm/testing/manifest-fake.ts index 4cc2414f5..95393b076 100644 --- a/src/crm/testing/manifest-fake.ts +++ b/src/crm/testing/manifest-fake.ts @@ -3,6 +3,7 @@ import { type FakeApi, type FakeCall, fakeApi, + type Pagination, type SchemaNode, sampleOf, } from "@wirecat/cli-base/codegen" @@ -98,6 +99,15 @@ const SINCE: Record number> = { datetime: (value) => Date.parse(`${value.replace(" ", "T")}Z`), } +const pageParameter = (pagination: Pagination) => + pagination.style === "cursor" + ? pagination.cursor + : pagination.style === "offset" + ? pagination.offset + : pagination.style === "page" + ? pagination.page + : undefined + /** Every fixed value of a call's template by its dotted path; a placeholder is not fixed. */ const fixedOf = (value: unknown, trail: readonly string[] = []): [string, string | number | boolean][] => { if (Array.isArray(value) || isRaw(value)) @@ -114,6 +124,8 @@ interface Route { rows: () => CrmRaw[] kind?: CrmKind get?: Place + /** A live search: only the rows whose text holds the words at this place. */ + query?: Place since?: { place: Place; format: CrmChangesCall["since"]; at: (row: CrmRaw) => number | null } } @@ -295,7 +307,8 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti const rule = manifest.entities[kind]?.map.pipeline const id = typeof rule === "object" && "value" in rule ? String(rule.value) : String(9001 + index) const node = rowNodeOf(spec.call) - const stageNode = itemOf(nodeAt(node, spec.stages.each), definitions) + const flat = spec.stages.each === "." + const stageNode = flat ? node : itemOf(nodeAt(node, spec.stages.each), definitions) const outcome = spec.stages.outcome const stages = Array.from({ length: ROWS }, (_, n) => { const stage = rowOf(stageNode, spec.stages.map, { name: "text" }, n, { @@ -305,12 +318,18 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti const won = typeof outcome === "object" && "path" in outcome ? [outcome.won?.[0], outcome.lost?.[0]][n - 1] : undefined if (won !== undefined && typeof outcome === "object" && "path" in outcome) - put(stage, outcome.path, won, stageNode) + put(stage, outcome.path, won === "true" ? true : won === "false" ? false : won, stageNode) + if (flat && spec.stages.pipeline) put(stage, spec.stages.pipeline, id, stageNode) return stage }) + const ids = stages.map((_, n) => `${id}0${n + 1}`) + if (flat) { + pipelines.set(kind, { rows: stages, id, stages: ids }) + continue + } const row = rowOf(node, spec.map, { name: "text" }, 0, { externalId: id }) putList(row, spec.stages.each, stages, node) - pipelines.set(kind, { rows: [row], id, stages: stages.map((_, n) => `${id}0${n + 1}`) }) + pipelines.set(kind, { rows: [row], id, stages: ids }) } const rows: Partial> = {} @@ -369,6 +388,8 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti } for (const call of [entity.list, entity.ids]) if (call) routes.push({ call, kind, rows: all }) if (entity.get) routes.push({ call: entity.get, kind, rows: all, get: placeOf(entity.get, "{id}") as Place }) + if (entity.search) + routes.push({ call: entity.search, kind, rows: all, query: placeOf(entity.search, "{query}") as Place }) } for (const [kind, one] of pipelines) { const spec = manifest.pipelines?.[kind] @@ -378,9 +399,12 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti if (!spec) continue const node = rowNodeOf(spec.call) const { options: _, ...map } = spec.map - const fields = Array.from({ length: PAGE }, (_, n) => - rowOf(node, map, { code: "text", label: "text" }, n, { code: `${kind}_field_${n + 1}` }), - ) + const keep = typeof spec.keep === "object" ? spec.keep : undefined + const fields = Array.from({ length: PAGE }, (_, n) => { + const field = rowOf(node, map, { code: "text", label: "text" }, n, { code: `${kind}_field_${n + 1}` }) + if (keep && typeof keep.is !== "object") putList(field, keep.path, keep.is, node) + return field + }) routes.push({ call: spec.call, rows: () => fields }) } if (manifest.users) routes.push({ call: manifest.users.call, rows: () => users }) @@ -392,17 +416,38 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti return { status, body: { message } } } + /** A page's size and place are the engine's to set: a live list asks for its own, whatever the template says. */ + const paging = (operation: string, where: "query" | "body", key: string) => { + const pagination = operations.get(operation)?.pagination + if (!pagination) return false + const parameters = [pageParameter(pagination), "limit" in pagination ? pagination.limit : undefined] + return parameters.some((one) => one?.in === where && one.name === key) + } + const matches = (route: Route, call: FakeCall) => route.call.operation === call.operation && fixedOf(route.call.path).every(([key, value]) => call.path[key] === String(value)) && - fixedOf(route.call.query).every(([key, value]) => readPlace(call, { in: "query", path: key }) === String(value)) && - fixedOf(route.call.body).every(([key, value]) => at(call.body, key) === value) + fixedOf(route.call.query).every( + ([key, value]) => + paging(route.call.operation, "query", key) || readPlace(call, { in: "query", path: key }) === String(value), + ) && + fixedOf(route.call.body).every( + ([key, value]) => paging(route.call.operation, "body", key) || at(call.body, key) === value, + ) - /** The route whose template the call matches most closely; a list wins a tie with a change read. */ + /** + * The route whose template the call matches most closely; a list wins a tie with a change read, and a search a tie + * with a list only when its words were sent. + */ const routeOf = (call: FakeCall) => routes - .filter((route) => matches(route, call)) - .sort((a, b) => fixedCount(b.call) - fixedCount(a.call) || Number(!!a.since) - Number(!!b.since))[0] + .filter((route) => matches(route, call) && (!route.query || readPlace(call, route.query) !== undefined)) + .sort( + (a, b) => + fixedCount(b.call) - fixedCount(a.call) || + Number(!!b.query) - Number(!!a.query) || + Number(!!a.since) - Number(!!b.since), + )[0] const page = (route: Route, call: FakeCall, listed: CrmRaw[]): FakeAnswer => { const operation = operations.get(route.call.operation) @@ -485,6 +530,14 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti .sort((a, b) => (timeAt(a) ?? 0) - (timeAt(b) ?? 0)) return page(changes, call, listed) } + if (route.query) { + const words = String(readPlace(call, route.query)).toLowerCase() + return page( + route, + call, + route.rows().filter((row) => JSON.stringify(row).toLowerCase().includes(words)), + ) + } return page(route, call, route.rows()) } From 66e0d4096225f2ea0cb106332578be3ce2a28dcc Mon Sep 17 00:00:00 2001 From: leemour Date: Wed, 18 Nov 2026 15:02:15 +0100 Subject: [PATCH 2/3] =?UTF-8?q?fix(crm):=20the=20review=20=E2=80=94=20a=20?= =?UTF-8?q?live=20list=20pages=20by=20offset=20only,=20a=20multiple=20fiel?= =?UTF-8?q?d=20is=20always=20a=20list,=20the=20fake=20routes=20a=20search?= =?UTF-8?q?=20by=20its=20words?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A search paged by cursor read one service page and sliced it, so people list showed a short page and no more; the manifest now refuses it. A multi-select with one current entry came back as a value, not a list. The fake counted page parameters when it chose between a list and a search on the same operation, and wrote a stage's flag only as a boolean. A short page still does not end a live list: a service may cap its pages. Co-Authored-By: Claude Opus 5.5 --- changes/feat-manifest-attio-deals.md | 6 ++--- src/crm/manifest/manifest.ts | 8 ++++++ src/crm/manifest/map.ts | 7 ++++- src/crm/manifest/source.ts | 1 + src/crm/manifest/typed-values.test.ts | 38 ++++++++++++++++++++++----- src/crm/testing/manifest-fake.ts | 15 ++++++++--- 6 files changed, 60 insertions(+), 15 deletions(-) diff --git a/changes/feat-manifest-attio-deals.md b/changes/feat-manifest-attio-deals.md index fe0f6f9a3..c9dd405b2 100644 --- a/changes/feat-manifest-attio-deals.md +++ b/changes/feat-manifest-attio-deals.md @@ -4,13 +4,13 @@ `stages.pipeline` or one pipeline), and a hook `pipelines..outcome` decides a stage's outcome the data cannot say. Attio's deal stages and Pipedrive's `/stages` answer this way. - **`custom.typed` reads the account's own fields from typed entries with their history**, as Attio keeps them: the - current entry's value by its type (`TYPED_VALUE_PATHS`, overridable), several current entries as a list. + current entry's value by its type (`TYPED_VALUE_PATHS`, overridable), a multiple field or several current entries as a list. `fields.keep` takes `{ path, is }`, so a config file can say which fields are the account's own. - **`entities.person.search` and `entities.organization.search` give a manifest source a live `list`**, for `people list` and `organizations list`: words through the search call's `"{query}"`, none through `list`. A condition it - cannot send is refused, not dropped. + cannot send is refused, not dropped; a live list pages by offset, so a cursor-paged search is refused. - `hooks.extend` gets the service's `call` as its second argument. ### Fixed -- `crmManifestFake` matches a call whatever page size and offset the source asked for: a live list sets its own. +- `crmManifestFake` matches a call whatever page size and offset the source asked for, a live list setting its own, and writes a stage's won or lost flag as a boolean or a word, as its schema takes it. diff --git a/src/crm/manifest/manifest.ts b/src/crm/manifest/manifest.ts index ca63ed072..678198d5a 100644 --- a/src/crm/manifest/manifest.ts +++ b/src/crm/manifest/manifest.ts @@ -350,6 +350,14 @@ export const parseCrmManifest = (input: unknown, api: ServiceApiManifest): CrmMa if (kind !== "person" && kind !== "organization") throw invalid(`entities.${kind}.search`, "only contacts and companies are read live") checkCall(`entities.${kind}.search`, one.search, true) + for (const [name, live] of [ + ["search", one.search], + ["list", one.list], + ] as const) { + const style = operations.get(live.operation)?.pagination?.style + if (style !== undefined && style !== "offset") + throw invalid(`entities.${kind}.${name}`, `a live list pages by offset or not at all, not by ${style}`) + } if (!hasPlaceholder([one.search.path, one.search.query, one.search.body], "{query}")) throw invalid(`entities.${kind}.search`, 'no "{query}" in its path, query or body') } diff --git a/src/crm/manifest/map.ts b/src/crm/manifest/map.ts index 6f41ea5b3..5dab700d3 100644 --- a/src/crm/manifest/map.ts +++ b/src/crm/manifest/map.ts @@ -254,7 +254,12 @@ const customOf = ( if (!isRaw(under)) return undefined const values: CrmRaw = {} for (const field of defined) { - const value = custom.typed ? typedOf(under[field.code], custom.typed) : under[field.code] + const typed = custom.typed ? typedOf(under[field.code], custom.typed) : undefined + const value = !custom.typed + ? under[field.code] + : field.multiple && typed !== null && !Array.isArray(typed) + ? [typed] + : typed if (value === null || value === undefined || value === "") continue values[field.code] = field.multiple && custom.split && typeof value === "string" ? value.split(custom.split) : value } diff --git a/src/crm/manifest/source.ts b/src/crm/manifest/source.ts index 52c5bba0a..61f59c5ac 100644 --- a/src/crm/manifest/source.ts +++ b/src/crm/manifest/source.ts @@ -359,6 +359,7 @@ export const crmSourceFromManifest = (input: unknown, options: CrmManifestSource const rows = rawsOf(items === undefined ? answer : at(answer, items)) const all = pagination?.style === "offset" ? rows : rows.slice(offset, offset + size) // A short page is not the last: a service may cap a page below the size asked for; only an empty one ends it. + // That costs one request past the last row, which losing the rows past a cap would not repay. return { rows: all, more: pagination?.style === "offset" ? rows.length > 0 : rows.length > offset + size } }) return { ...page, items: await mapped(kind, page.items, defined, send) } diff --git a/src/crm/manifest/typed-values.test.ts b/src/crm/manifest/typed-values.test.ts index b95b67ad3..a5e8e0453 100644 --- a/src/crm/manifest/typed-values.test.ts +++ b/src/crm/manifest/typed-values.test.ts @@ -55,7 +55,10 @@ const definitions: Record = { { data: list(obj({ id: obj({ status_id: str }), title: str, celebration_enabled: bool, is_archived: bool })) }, ["data"], ), - Attributes: obj({ data: list(obj({ api_slug: str, title: str, type: str, is_system_attribute: bool })) }, ["data"]), + Attributes: obj( + { data: list(obj({ api_slug: str, title: str, type: str, is_system_attribute: bool, is_multiselect: bool })) }, + ["data"], + ), } const path = (name: string) => ({ name, in: "path", required: true, schema: str }) as const @@ -83,6 +86,12 @@ const api: ServiceApiManifest = { items: "data", }, }), + op("scroll", "POST", "/v2/objects/{object}/records/scroll", { + parameters: [path("object")], + request: { required: false, confidence: "contract", schema: "Query" }, + response: { confidence: "contract", schema: "Records" }, + pagination: { style: "cursor", cursor: { in: "body", name: "after" }, next: "next", items: "data" }, + }), op("get", "GET", "/v2/objects/{object}/records/{record_id}", { parameters: [path("object"), path("record_id")], response: { confidence: "contract", schema: "One" }, @@ -110,7 +119,7 @@ const records = (object: string, filter: unknown) => ({ const attributes = (object: string) => ({ call: { operation: "attributes", path: { object }, items: "data" }, - map: { code: "api_slug", label: "title", type: "type" }, + map: { code: "api_slug", label: "title", type: "type", multiple: "is_multiselect" }, keep: { path: "is_system_attribute", is: false }, }) @@ -250,7 +259,7 @@ describe("Attio's shapes", () => { ]) }) - it("keeps the current entry of each own attribute, a list when several are current, and skips built-in ones", async () => { + it("keeps the current entries of each own attribute, a list for a multi-select or several, and skips built-in ones", async () => { const deal = { id: { record_id: "d1" }, values: { @@ -261,7 +270,11 @@ describe("Attio's shapes", () => { ], tags: [ { ...current, attribute_type: "select", option: { title: "A" } }, - { ...current, attribute_type: "select", option: { title: "B" } }, + { ...replaced, attribute_type: "select", option: { title: "B" } }, + ], + regions: [ + { ...current, attribute_type: "select", option: { title: "EU" } }, + { ...current, attribute_type: "select", option: { title: "US" } }, ], budget: [{ ...current, attribute_type: "currency", currency_value: 900 }], }, @@ -271,7 +284,8 @@ describe("Attio's shapes", () => { data: [ { api_slug: "name", title: "Name", type: "text", is_system_attribute: true }, { api_slug: "priority", title: "Priority", type: "select", is_system_attribute: false }, - { api_slug: "tags", title: "Tags", type: "select", is_system_attribute: false }, + { api_slug: "tags", title: "Tags", type: "select", is_system_attribute: false, is_multiselect: true }, + { api_slug: "regions", title: "Regions", type: "select", is_system_attribute: false }, { api_slug: "budget", title: "Budget", type: "currency", is_system_attribute: false }, ], }), @@ -281,7 +295,8 @@ describe("Attio's shapes", () => { const [item] = "items" in page ? page.items : [] expect(item?.metadata).toEqual({ priority: { label: "Priority", value: "High" }, - tags: { label: "Tags", value: ["A", "B"] }, + tags: { label: "Tags", value: ["A"] }, + regions: { label: "Regions", value: ["EU", "US"] }, budget: { label: "Budget", value: 900 }, }) }) @@ -306,7 +321,7 @@ describe("Attio's shapes", () => { expect(isCliError(refused) && refused.code).toBe("validation_error") }) - it("refuses a search without {query}, one on a deal, and a stage group without a flat list", () => { + it("refuses a search without {query}, one on a deal, one that pages by cursor, and a stage group without a flat list", () => { const person = MANIFEST.entities.person as NonNullable const refused = (manifest: unknown) => { try { @@ -333,6 +348,15 @@ describe("Attio's shapes", () => { pipelines: { deal: { ...pipeline, stages: { ...pipeline.stages, each: "data", pipeline: "pipeline" } } }, }), ).toContain("groups stages only when the call answers them") + expect( + refused({ + ...MANIFEST, + entities: { + ...MANIFEST.entities, + person: { ...person, search: { ...person.search, operation: "scroll" } }, + }, + }), + ).toContain("a live list pages by offset or not at all, not by cursor") }) it("hands extend the service call, so a hook can read what the manifest does not", async () => { diff --git a/src/crm/testing/manifest-fake.ts b/src/crm/testing/manifest-fake.ts index 95393b076..af27c8a84 100644 --- a/src/crm/testing/manifest-fake.ts +++ b/src/crm/testing/manifest-fake.ts @@ -116,8 +116,10 @@ const fixedOf = (value: unknown, trail: readonly string[] = []): [string, string return [[trail.join("."), value as string | number | boolean]] } -const fixedCount = (call: CrmOperationCall) => - fixedOf(call.path).length + fixedOf(call.query).length + fixedOf(call.body).length +const fixedCount = (call: CrmOperationCall, paging: (where: "query" | "body", key: string) => boolean) => + fixedOf(call.path).length + + fixedOf(call.query).filter(([key]) => !paging("query", key)).length + + fixedOf(call.body).filter(([key]) => !paging("body", key)).length interface Route { call: CrmOperationCall @@ -318,7 +320,11 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti const won = typeof outcome === "object" && "path" in outcome ? [outcome.won?.[0], outcome.lost?.[0]][n - 1] : undefined if (won !== undefined && typeof outcome === "object" && "path" in outcome) - put(stage, outcome.path, won === "true" ? true : won === "false" ? false : won, stageNode) + for (const value of won === "true" || won === "false" ? [won === "true", won] : [won]) { + // A flag the schema keeps as a boolean takes one; a text field the word. + put(stage, outcome.path, value, stageNode) + if (at(stage, outcome.path) === value) break + } if (flat && spec.stages.pipeline) put(stage, spec.stages.pipeline, id, stageNode) return stage }) @@ -444,7 +450,8 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti .filter((route) => matches(route, call) && (!route.query || readPlace(call, route.query) !== undefined)) .sort( (a, b) => - fixedCount(b.call) - fixedCount(a.call) || + fixedCount(b.call, (where, key) => paging(b.call.operation, where, key)) - + fixedCount(a.call, (where, key) => paging(a.call.operation, where, key)) || Number(!!b.query) - Number(!!a.query) || Number(!!a.since) - Number(!!b.since), )[0] From 710f59f8a5ee0f54911f7784a3ffba8bfc1940cd Mon Sep 17 00:00:00 2001 From: leemour Date: Wed, 18 Nov 2026 15:13:44 +0100 Subject: [PATCH 3/3] fix(crm): the manifest fake samples the union branch that takes the value it writes Attio's attribute value is text in one branch of its union and a flag in another; the fake sampled the first branch and wrote a stage's won flag as text, which its strict check then refused. A branch now fits when the path exists and takes the value, and a flag whose place the walk cannot name stays a flag. Co-Authored-By: Claude Opus 5.5 --- src/crm/testing/manifest-fake.ts | 61 ++++++++++++++++---------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/crm/testing/manifest-fake.ts b/src/crm/testing/manifest-fake.ts index af27c8a84..c3ccfe98c 100644 --- a/src/crm/testing/manifest-fake.ts +++ b/src/crm/testing/manifest-fake.ts @@ -171,7 +171,8 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti const typed = (node: SchemaNode | undefined, value: Value): unknown => { const found = scalarOf(node) const type = found?.type ?? "string" - if (typeof value === "boolean") return type === "boolean" ? value : undefined + // A place whose schema the walk cannot name keeps a flag a flag: the strict fake still checks it. + if (typeof value === "boolean") return type === "boolean" || found === undefined ? value : undefined if (typeof value === "object") return type === "integer" || type === "number" ? value.ms : new Date(value.ms).toISOString() if (type === "integer" || type === "number") return Number.isFinite(Number(value)) ? Number(value) : undefined @@ -183,23 +184,28 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti * The schema with each union on the way to `keys` cut to the branch that has the rest of the path: a sample of the * first branch of Attio's seventeen value types would carry another type's fields, and the strict fake refuses it. */ - const narrowed = (node: SchemaNode, keys: readonly string[]): SchemaNode => { + const narrowed = (node: SchemaNode, keys: readonly string[], value?: Value): SchemaNode => { if (keys.length === 0) return node const found = resolve(node, definitions) const [key, ...rest] = keys as [string, ...string[]] - const fits = (one: SchemaNode) => "node" in walk(one, keys.join("."), definitions) + // A branch fits when it has the path, and, with a value to write, takes it there: Attio's `value` is text in one + // branch and a flag in another. + const fits = (one: SchemaNode) => { + const leaf = walk(one, keys.join("."), definitions) + return "node" in leaf && (value === undefined || typed(leaf.node, value) !== undefined) + } switch (found.type) { case "union": { - const branch = found.of.find(fits) - return branch ? narrowed(branch, keys) : node + const branch = found.of.find(fits) ?? found.of.find((one) => "node" in walk(one, keys.join("."), definitions)) + return branch ? narrowed(branch, keys, value) : node } case "allOf": - return { ...found, of: found.of.map((one) => (fits(one) ? narrowed(one, keys) : one)) } + return { ...found, of: found.of.map((one) => (fits(one) ? narrowed(one, keys, value) : one)) } case "array": - return /^\d+$/.test(key) ? { ...found, items: narrowed(found.items, rest) } : node + return /^\d+$/.test(key) ? { ...found, items: narrowed(found.items, rest, value) } : node case "object": { const inner = found.properties[key] ?? found.additionalProperties - return inner ? { ...found, properties: { ...found.properties, [key]: narrowed(inner, rest) } } : node + return inner ? { ...found, properties: { ...found.properties, [key]: narrowed(inner, rest, value) } } : node } default: return node @@ -262,34 +268,29 @@ export const crmManifestFake = (input: CrmManifest, options: CrmManifestFakeOpti n: number, given: Readonly>, ): CrmRaw => { - // Sampled already cut to the branches the rules write into, so a list the first sample holds has the right type. - const paths = Object.values(rules).flatMap((rule) => pathOf(rule) ?? []) - const made = sample(node && paths.reduce((cut, path) => narrowed(cut, path.split(".")), node)) - const row = isRaw(made) ? made : {} - for (const [field, rule] of Object.entries(rules)) { - const path = pathOf(rule) - if (path === undefined) continue + const sampledValue = (field: string, rule: CrmRule | undefined): Value | undefined => { const mapped = typeof rule === "object" && "map" in rule ? Object.keys(rule.map)[0] : undefined const reverse = typeof rule === "object" && "map" in rule && field in given ? Object.keys(rule.map).find((key) => String(rule.map[key]) === String(given[field])) : undefined - const value = - reverse !== undefined - ? reverse - : field in given - ? given[field] - : mapped !== undefined - ? mapped - : types[field] === "text" - ? `${field} ${n + 1}` - : types[field] === "epoch" - ? { ms: SEEDED + (n + 1) * 60_000 } - : types[field] === "day" - ? `2026-11-0${n + 1}` - : undefined - if (value !== undefined) put(row, path, value, node) + if (reverse !== undefined) return reverse + if (field in given) return given[field] + if (mapped !== undefined) return mapped + if (types[field] === "text") return `${field} ${n + 1}` + if (types[field] === "epoch") return { ms: SEEDED + (n + 1) * 60_000 } + return types[field] === "day" ? `2026-11-0${n + 1}` : undefined } + const writes = Object.entries(rules).flatMap(([field, rule]) => { + const path = pathOf(rule) + return path === undefined ? [] : [{ path, value: sampledValue(field, rule) }] + }) + // Sampled from the schema cut to the branches the rules write into, each one that takes its value: Attio's + // `value` is text in one branch and a flag in another, and a list the first sample holds must have the right type. + const cut = node && writes.reduce((one, { path, value }) => narrowed(one, path.split("."), value), node) + const made = sample(cut) + const row = isRaw(made) ? made : {} + for (const { path, value } of writes) if (value !== undefined) put(row, path, value, cut) return row }