diff --git a/changes/feat-tracker-manifest.md b/changes/feat-tracker-manifest.md new file mode 100644 index 000000000..d3892d323 --- /dev/null +++ b/changes/feat-tracker-manifest.md @@ -0,0 +1,7 @@ +### Added + +- **A tracker CLI can be described as data: `trackerSourceFromManifest`.** A `TrackerManifest` names, for each kind, + the list, changes and ids operations of the generated API and maps our fields to paths in a row, in the CRM + manifest's rule language. Over GraphQL each call may send its own selection, and `parseTrackerManifest` refuses a + map path the selection leaves out. `wirecat.config` takes `category: "tracker"` with its `source`, and + `spec.kind: "graphql"`. See [Trackers](../docs/dev/TRACKER.md#a-tracker-described-as-data). diff --git a/docs/dev/TRACKER.md b/docs/dev/TRACKER.md index fb4180a7b..a810b8285 100644 --- a/docs/dev/TRACKER.md +++ b/docs/dev/TRACKER.md @@ -38,11 +38,12 @@ stops keeps its ids and the page it reached. `state: "partial"` and `stopped`; the kinds it finished are skipped by the next run until every kind has had its round, so a budget smaller than the number of kinds still moves on. - **Held passes.** A pass is held (`pass: "held"`, with the reason in `held`) and marks nothing when the list is empty - while rows are stored, when it would mark more than half of ten or more stored rows deleted, or when the sign-in - reported a gap for the kind: a tracker that lost rights lists none of what it can no longer see. + while rows are stored, or when it would mark more than half of ten or more stored rows deleted: a tracker that lost + rights lists none of what it can no longer see. `--rescan` lets those through. A pass after the sign-in reported a + gap for the kind is held even then: what it could not read is not gone. - **Failures.** A kind that fails is named in its result's `error` and the others still sync. A page that says there is more but gives no next page fails the kind. -- **`rescan`.** Lists everything again and every id, dropping a round in progress, and lets a held pass through. A +- **`rescan`.** Lists everything again and every id, dropping a round in progress, and lets a held pass through unless a gap held it. A rescan that stops goes on as one in the next run. A save merges: a field the read left out keeps its stored value, so a partial selection never erases. @@ -66,6 +67,24 @@ to the issue and copies none of its text. The same issue again answers the task `projects`, `teams`, `users` (each with its person), `states` and `labels`. Every text field from the tracker is named in the row's `untrusted`. +## A tracker described as data + +`trackerSourceFromManifest(manifest, { api, call, hooks? })` builds a `TrackerSource` from the generated API manifest +and a `TrackerManifest`, as `crmSourceFromManifest` does for a CRM. Each kind names a `list` call, a `changes` call +whose `variables` hold `"{since}"` (the cursor less `overlapMs`, ten minutes by default), an `ids` call that selects +only the id, and a `map` from our fields to paths in a row, in the CRM manifest's rule language. A selected object +the tracker answers `null` (`project: null`) is none, so the stored link is cleared. + +Over GraphQL a call's `selection` replaces the operation's default one, so a tracker reads `state { id }` or +`labels { nodes { id } }`. `parseTrackerManifest` refuses a map path the selection leaves out, before any request: +a field missing from the request would read as "the tracker cleared it". The variables travel as the request body; +the selection rides on the request as `selection`, which the `api` group's GraphQL executor sends. + +`self: { call, path }` names the signed-in user. `hooks.after` corrects a record; `hooks.extend` adds methods. + +A `wirecat.config.ts` with `category: "tracker"` needs its `source`; `spec.kind: "graphql"` names an introspection +result. + ## The contract kit `trackerSourceContract(make)` checks an adapter: full lists in the domain's shape, every held row listed, a cursor diff --git a/src/crm/manifest/manifest.ts b/src/crm/manifest/manifest.ts index 331ad0d1e..8d6763088 100644 --- a/src/crm/manifest/manifest.ts +++ b/src/crm/manifest/manifest.ts @@ -150,7 +150,7 @@ export interface CrmManifestHooks { const path = v.pipe(v.string(), v.regex(/^[^.\s]+(\.[^.\s]+)*$/, "a dotted path such as properties.name")) const paths = v.pipe(v.array(path), v.minLength(1)) -const rule: v.GenericSchema = v.union( +export const crmRuleSchema: v.GenericSchema = v.union( [ path, v.strictObject({ first: paths }), @@ -166,6 +166,8 @@ const rule: v.GenericSchema = v.union( "a path, one of first, join, all, path, each, value, or a function", ) as v.GenericSchema +const rule = crmRuleSchema + const call = v.strictObject({ operation: v.pipe(v.string(), v.minLength(1)), path: v.optional(v.record(v.string(), v.string())), diff --git a/src/crm/manifest/map.ts b/src/crm/manifest/map.ts index e4ddd4507..5d3985bed 100644 --- a/src/crm/manifest/map.ts +++ b/src/crm/manifest/map.ts @@ -1,6 +1,6 @@ import type { CrmField, CrmKind, CrmRecords } from "../../domain/crm.js" import { DEAL_STATUSES } from "../../domain/deals.js" -import { amountOf, type CrmRaw, crmValuesOf, epoch, fullName, id, isRaw, text } from "../values.js" +import { amountOf, type CrmRaw, crmValuesOf, epoch, fullName, id, isRaw, number, text } from "../values.js" import type { CrmRule } from "./manifest.js" /** The service sent nothing at this path: the field is left out, so the stored copy keeps what it held. */ @@ -105,6 +105,7 @@ const CONVERT = { epoch: epochOf, day: dayOf, bool: yes, + number, texts: (value: unknown) => unique(listOf(value).map(text)), ids: (value: unknown) => linksOf(value).map(({ externalId }) => externalId), status: (value: unknown) => { diff --git a/src/manifest/index.test.ts b/src/manifest/index.test.ts index 4e12889b8..c87237af8 100644 --- a/src/manifest/index.test.ts +++ b/src/manifest/index.test.ts @@ -23,6 +23,13 @@ describe("parseCliConfig", () => { hooks: "./src/hooks.ts", } expect(parseCliConfig(oauth)).toEqual(oauth) + const tracker = { + ...config, + spec: { kind: "graphql" as const, from: "https://api.acme.test/graphql" }, + category: "tracker" as const, + source: { provider: "acme", kinds: ["issue"], entities: {} }, + } + expect(parseCliConfig(tracker)).toEqual(tracker) }) it("names where a config is wrong", () => { @@ -40,6 +47,8 @@ describe("parseCliConfig", () => { "spec.from", ) expect(() => parseCliConfig({ ...config, category: "crm" })).toThrow("a crm CLI needs its source") + expect(() => parseCliConfig({ ...config, category: "tracker" })).toThrow("a tracker CLI needs its source") + expect(() => parseCliConfig({ ...config, source: { provider: "acme" } })).toThrow("only a crm or tracker CLI") for (const from of [ "/etc/spec.json", "//evil.test/x", @@ -55,7 +64,7 @@ describe("parseCliConfig", () => { expect(() => parseCliConfig({ ...config, identity: { ...config.identity, owner: "-owner" } })).toThrow( "identity.owner", ) - expect(() => parseCliConfig({ ...config, source: { provider: "acme" } })).toThrow("only a crm CLI") + expect(() => parseCliConfig({ ...config, source: { provider: "acme" } })).toThrow("only a crm or tracker CLI") expect(() => parseCliConfig({ ...config, extra: true })).toThrow( "wirecat.config: extra: not a field of wirecat.config", ) @@ -64,7 +73,7 @@ describe("parseCliConfig", () => { it("takes each category, a crm CLI with its source, and a token's verify call", () => { for (const category of ["mail", "files", "other"] as const) expect(parseCliConfig({ ...config, category }).category).toBe(category) - expect(() => parseCliConfig({ ...config, category: "tracker" })).toThrow("wirecat.config: category") + expect(() => parseCliConfig({ ...config, category: "erp" })).toThrow("wirecat.config: category") const crm = { ...config, category: "crm" as const, source: { provider: "acme" } } expect(parseCliConfig(crm).source).toEqual({ provider: "acme" }) const verify = { operation: "getV2Self", id: "workspace_id", name: "workspace_name" } diff --git a/src/manifest/index.ts b/src/manifest/index.ts index b53b8a7f9..41468765f 100644 --- a/src/manifest/index.ts +++ b/src/manifest/index.ts @@ -2,6 +2,7 @@ import { CliError } from "@wirecat/cli-base" import * as v from "valibot" import type { CrmManifest } from "../crm/manifest/manifest.js" import type { OAuthProvider } from "../oauth/provider.js" +import type { TrackerManifest } from "../tracker/manifest/manifest.js" /** * The operation that answers whose the token is, and where its answer holds the account's id (and a name to show): @@ -30,7 +31,17 @@ export type CliAuth = rotating?: boolean } -export const CLI_CATEGORIES = ["messenger", "mail", "notes", "files", "meetings", "calendar", "crm", "other"] as const +export const CLI_CATEGORIES = [ + "messenger", + "mail", + "notes", + "files", + "meetings", + "calendar", + "crm", + "tracker", + "other", +] as const export type CliCategory = (typeof CLI_CATEGORIES)[number] /** One file, `wirecat.config.ts`, that `new --from-config` builds a CLI from. */ @@ -48,12 +59,15 @@ export interface CliConfig { owner: string } /** `from`: the https URL the description is published at. */ - spec: { kind: "openapi" | "swagger"; from: string } + spec: { kind: "openapi" | "swagger" | "graphql"; from: string } auth: CliAuth /** cli-template's `--category`; `other` is any API outside them. */ category: CliCategory - /** The CRM source; checked against the generated API by `parseCrmManifest` once the api group exists. */ - source?: CrmManifest + /** + * A crm or tracker CLI's source; checked against the generated API by `parseCrmManifest` or `parseTrackerManifest` + * once the api group exists. + */ + source?: CrmManifest | TrackerManifest /** The module of named hooks, relative to the config: `./src/hooks.ts`. */ hooks?: string } @@ -126,7 +140,7 @@ const schema = v.strictObject({ owner: v.pipe(v.string(), v.regex(/^[A-Za-z0-9](?:-?[A-Za-z0-9]){0,38}$/, "a GitHub owner")), }), spec: v.strictObject({ - kind: v.picklist(["openapi", "swagger"]), + kind: v.picklist(["openapi", "swagger", "graphql"]), from: httpsUrl, }), auth: v.variant("preset", [ @@ -161,7 +175,7 @@ const schema = v.strictObject({ }) /** - * The config checked against its shape; every error names where it is. A CRM CLI needs its source. `source` comes back + * The config checked against its shape; every error names where it is. A crm or tracker CLI needs its source. `source` comes back * unchecked: `parseCrmManifest` checks it against the generated API. */ export const parseCliConfig = (input: unknown): CliConfig => { @@ -184,12 +198,13 @@ export const parseCliConfig = (input: unknown): CliConfig => { // Typed, not cast: a field CliConfig needs that the schema lacks or types otherwise fails to compile (not the reverse). const output: Omit & { source?: Readonly> } = parsed.output const config = output as CliConfig - if (config.category !== "crm" && config.source) - throw new CliError("configuration_error", "wirecat.config: source: only a crm CLI reads a source yet", { + const sourced = config.category === "crm" || config.category === "tracker" + if (!sourced && config.source) + throw new CliError("configuration_error", "wirecat.config: source: only a crm or tracker CLI reads a source yet", { reason: "manifest.invalid", }) - if (config.category === "crm" && !config.source) - throw new CliError("configuration_error", "wirecat.config: source: a crm CLI needs its source", { + if (sourced && !config.source) + throw new CliError("configuration_error", `wirecat.config: source: a ${config.category} CLI needs its source`, { reason: "manifest.invalid", }) return config diff --git a/src/tracker/index.ts b/src/tracker/index.ts index 2025549ce..2d9556c96 100644 --- a/src/tracker/index.ts +++ b/src/tracker/index.ts @@ -13,6 +13,20 @@ export { type TrackerUser, } from "../domain/tracker.js" export { addTrackerCommands, type TrackerCommandOptions, type TrackerRun } from "./cli/index.js" +export { + parseTrackerManifest, + type TrackerAfterHook, + type TrackerChangesCall, + type TrackerManifest, + type TrackerManifestEntity, + type TrackerManifestHooks, + type TrackerOperationCall, +} from "./manifest/manifest.js" +export { TRACKER_FIELDS } from "./manifest/map.js" +export { + type TrackerManifestSourceOptions, + trackerSourceFromManifest, +} from "./manifest/source.js" export type { TrackerCallContext, TrackerChanges, diff --git a/src/tracker/manifest/manifest.ts b/src/tracker/manifest/manifest.ts new file mode 100644 index 000000000..950969f2f --- /dev/null +++ b/src/tracker/manifest/manifest.ts @@ -0,0 +1,214 @@ +import { CliError } from "@wirecat/cli-base" +import { parseSelection, type Selection } from "@wirecat/cli-base/codegen" +import * as v from "valibot" +import type { ServiceApiCall } from "../../cli/service-api-call.js" +import type { ServiceApiManifest } from "../../cli/service-api-manifest.js" +import type { crmValues } from "../../crm/index.js" +import { type CrmRule, crmRuleSchema } from "../../crm/manifest/manifest.js" +import { rulePaths } from "../../crm/manifest/map.js" +import { TRACKER_KINDS, type TrackerKind, type TrackerRecords } from "../../domain/tracker.js" +import { editDistance } from "../../search/correct.js" +import type { TrackerSource } from "../source.js" +import { TRACKER_FIELDS } from "./map.js" + +/** + * One request: an operation of the generated API manifest by its id, with fixed variables. A string `"{since}"` + * anywhere in `variables` is filled in by the engine. `selection` replaces a GraphQL operation's default one: a + * tracker reads deeper than one level (`state { id }`, `labels { nodes { id } }`). + */ +export interface TrackerOperationCall { + operation: string + variables?: Readonly> + selection?: string + /** Where the answer's rows are when the operation has no `pagination`; absent, the answer itself is the list. */ + items?: string +} + +export interface TrackerChangesCall extends TrackerOperationCall { + /** How `{since}` is written: ISO, or epoch ms or seconds as digits. */ + since: "iso" | "ms" | "seconds" +} + +export interface TrackerManifestEntity { + /** Every row of the kind, archived ones too, page by page by the operation's own `pagination`. */ + list: TrackerOperationCall + /** Rows changed since `{since}`, the last cursor less the overlap; absent, every sync lists in full. */ + changes?: TrackerChangesCall + /** Every id, archived ones too, to find deletions; absent, `list`. Give one that selects only the id. */ + ids?: TrackerOperationCall + map: { readonly [F in keyof TrackerRecords[K]]?: CrmRule } + /** A kind the plan or the sign-in may not have: its refusal is a gap with this text, and the others still sync. */ + optional?: { message: string; scope?: string } +} + +/** A tracker account described as data: what to call for each kind and where each of our fields is in the answer. */ +export interface TrackerManifest { + /** The registry name: `linear`. */ + provider: string + /** In `TRACKER_KINDS` order: teams and states before the issues on them. */ + kinds: readonly TrackerKind[] + /** Re-read before the last cursor: rows saved while the round ran, and a clock running ahead. Default 10 min. */ + overlapMs?: number + entities: { readonly [K in TrackerKind]?: TrackerManifestEntity } + /** The sign-in's own user id: `viewer { id }`. */ + self?: { call: TrackerOperationCall; path: string } +} + +/** Corrects one record after the map; `call` reads more from the service, spent from the run's budget. */ +export type TrackerAfterHook = ( + record: TrackerRecords[K], + raw: crmValues.CrmRaw, + service: { call: ServiceApiCall }, +) => TrackerRecords[K] | Promise + +/** Code a tracker needs beyond what the manifest says. */ +export interface TrackerManifestHooks { + after?: { readonly [K in TrackerKind]?: TrackerAfterHook } + /** Adds or replaces source methods; `base` is what the manifest built. */ + extend?: (base: TrackerSource) => Partial +} + +const path = v.pipe(v.string(), v.regex(/^[^.\s]+(\.[^.\s]+)*$/, "a dotted path such as state.id")) + +const call = v.strictObject({ + operation: v.pipe(v.string(), v.minLength(1)), + variables: v.optional(v.record(v.string(), v.unknown())), + selection: v.optional(v.pipe(v.string(), v.minLength(1))), + items: v.optional(path), +}) + +const kind = v.picklist(TRACKER_KINDS) + +const schema = v.strictObject({ + provider: v.pipe(v.string(), v.regex(/^[a-z][a-z0-9-]*$/, "a registry name such as linear")), + kinds: v.pipe(v.array(kind), v.minLength(1)), + overlapMs: v.optional(v.pipe(v.number(), v.integer(), v.minValue(0))), + entities: v.record( + kind, + v.optional( + v.strictObject({ + list: call, + changes: v.optional(v.strictObject({ ...call.entries, since: v.picklist(["iso", "ms", "seconds"]) })), + ids: v.optional(call), + map: v.record(v.string(), crmRuleSchema), + optional: v.optional( + v.strictObject({ message: v.pipe(v.string(), v.minLength(1)), scope: v.optional(v.string()) }), + ), + }), + ), + ), + self: v.optional(v.strictObject({ call, path })), +}) + +const invalid = (where: string, message: string) => + new CliError("configuration_error", `source manifest: ${where}: ${message}`, { reason: "manifest.invalid" }) + +const nearest = (wanted: string, known: readonly string[]) => + known + .map((one) => ({ one, edits: editDistance(wanted, one, 2) })) + .filter(({ edits }) => edits <= 2) + .sort((a, b) => a.edits - b.edits)[0]?.one + +const hasSince = (value: unknown): boolean => + value === "{since}" || + (Array.isArray(value) && value.some(hasSince)) || + (typeof value === "object" && value !== null && Object.values(value).some(hasSince)) + +/** The selection a call sends, as a tree: its own, else the operation's default; none for an operation over HTTP. */ +export const selectionOf = (one: TrackerOperationCall, api: ServiceApiManifest): Selection | undefined => { + const binding = api.operations.find((operation) => operation.id === one.operation)?.binding + if (binding?.kind !== "graphql") return undefined + return parseSelection(one.selection ?? binding.selection) +} + +/** Whether `path`, read in a row, is selected: each step a field the selection names, and a leaf where it ends. */ +const selects = (tree: Selection, steps: readonly string[]): boolean => { + const [first, ...rest] = steps + if (first === undefined) return true + if (/^\d+$/.test(first)) return selects(tree, rest) + const found = tree[first] + if (found === undefined) return false + return found === true ? rest.every((step) => /^\d+$/.test(step)) : selects(found, rest) +} + +/** The selection of one row: under the operation's `items` (a connection's `nodes`) or the call's own. */ +const rowSelection = (tree: Selection, items: string | undefined): Selection | undefined => { + let inside: Selection | true | undefined = tree + for (const step of items?.split(".") ?? []) inside = inside === true ? undefined : inside?.[step] + return inside === true ? undefined : inside +} + +/** + * The manifest checked against its shape and the generated API: each operation exists and reads, a list pages or + * says where its rows are, a change call places `{since}`, and over GraphQL every path the map reads is selected, so + * a field is never left out of a request by mistake and read as "the tracker cleared it". Every error names where. + */ +export const parseTrackerManifest = (input: unknown, api: ServiceApiManifest): TrackerManifest => { + const parsed = v.safeParse(schema, input) + if (!parsed.success) { + const [issue] = parsed.issues + throw invalid(v.getDotPath(issue) ?? "(root)", issue.message) + } + const manifest = parsed.output as TrackerManifest + const order = manifest.kinds.map((one) => TRACKER_KINDS.indexOf(one)) + if (order.some((at, index) => index > 0 && at <= (order[index - 1] as number))) + throw invalid("kinds", `list them once each, in this order: ${TRACKER_KINDS.join(", ")}`) + const operations = new Map(api.operations.map((one) => [one.id, one])) + const known = [...operations.keys()] + const checkCall = (where: string, one: TrackerOperationCall, map?: Readonly>) => { + const operation = operations.get(one.operation) + if (!operation) { + const near = nearest(one.operation, known) + throw invalid( + `${where}.operation`, + `no operation ${one.operation} in the API${near ? `; did you mean ${near}?` : ""}`, + ) + } + if (operation.effect !== "read") + throw invalid(`${where}.operation`, `${one.operation} is a ${operation.effect}, and a source only reads`) + if (operation.pagination && operation.pagination.style === "link") + throw invalid(`${where}.operation`, `${one.operation} pages by a link, which a source does not read yet`) + let tree: Selection | undefined + try { + tree = selectionOf(one, api) + } catch (error) { + throw invalid(`${where}.selection`, error instanceof Error ? error.message : String(error)) + } + if (one.selection !== undefined && tree === undefined) + throw invalid(`${where}.selection`, `${one.operation} is not a GraphQL operation, so it takes no selection`) + if (!tree || !map) return + const row = rowSelection(tree, one.items ?? operation.pagination?.items) + if (!row) throw invalid(`${where}.selection`, "selects no rows where the operation's items are") + for (const [field, rule] of Object.entries(map)) + for (const { path: read, inside } of rulePaths(rule)) { + const steps = [...(inside?.split(".") ?? []), ...read.split(".")] + if (!selects(row, steps)) + throw invalid(`${where}.selection`, `map.${field} reads ${steps.join(".")}, which the selection leaves out`) + } + } + for (const kind of manifest.kinds) { + const one = manifest.entities[kind] + if (!one) throw invalid(`entities.${kind}`, `${kind} is in kinds but has no entity`) + if (one.map.externalId === undefined) throw invalid(`entities.${kind}.map.externalId`, "every kind maps its id") + const fields = Object.keys(TRACKER_FIELDS[kind]) + for (const field of Object.keys(one.map)) + if (!fields.includes(field)) { + const near = nearest(field, fields) + throw invalid( + `entities.${kind}.map.${field}`, + `${kind} has no such field${near ? `; did you mean ${near}?` : ""}`, + ) + } + checkCall(`entities.${kind}.list`, one.list, one.map) + if (one.changes) { + checkCall(`entities.${kind}.changes`, one.changes, one.map) + if (!hasSince(one.changes.variables)) throw invalid(`entities.${kind}.changes`, 'no "{since}" in its variables') + } + if (one.ids) checkCall(`entities.${kind}.ids`, one.ids, { externalId: one.map.externalId }) + } + for (const [kind, one] of Object.entries(manifest.entities)) + if (one && !manifest.kinds.includes(kind as TrackerKind)) + throw invalid(`entities.${kind}`, `${kind} has an entity but is not in kinds`) + if (manifest.self) checkCall("self.call", manifest.self.call, { self: manifest.self.path }) + return manifest +} diff --git a/src/tracker/manifest/map.test.ts b/src/tracker/manifest/map.test.ts new file mode 100644 index 000000000..5b6443dcf --- /dev/null +++ b/src/tracker/manifest/map.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest" +import { mapTrackerRecord } from "./map.js" + +describe("mapTrackerRecord", () => { + it("clears a field whose selected object the tracker answered null, whatever the rule, converted by its type", () => { + const record = mapTrackerRecord( + "issue", + { + externalId: "id", + title: "title", + assignee: { first: ["assignee.id", "delegate.id"] }, + project: { path: "project.id" }, + labels: "labels.nodes", + parent: "parent.id", + }, + { id: "i", title: "x", assignee: null, delegate: null, project: null, labels: null }, + ) + expect(record).toEqual({ externalId: "i", title: "x", assignee: null, project: null, labels: [] }) + }) + + it("keeps out a state of a type the domain does not know, and a comment without its issue", () => { + expect(mapTrackerRecord("state", { externalId: "id", name: "name", type: "type" }, { id: "s", type: "x" })).toBe( + null, + ) + expect(mapTrackerRecord("comment", { externalId: "id", body: "body" }, { id: "c", body: "hi" })).toBe(null) + }) +}) diff --git a/src/tracker/manifest/map.ts b/src/tracker/manifest/map.ts new file mode 100644 index 000000000..e7cdbcd00 --- /dev/null +++ b/src/tracker/manifest/map.ts @@ -0,0 +1,89 @@ +import { crmValues } from "../../crm/index.js" +import type { CrmRule } from "../../crm/manifest/manifest.js" +import type { Conversion } from "../../crm/manifest/map.js" +import { convert, mapFields, rulePaths } from "../../crm/manifest/map.js" +import { STATE_TYPES, type TrackerKind, type TrackerRecords } from "../../domain/tracker.js" + +const TIMES = { externalId: "id", createdAt: "epoch", updatedAt: "epoch", archivedAt: "epoch" } as const +// A literal after `key:` reads as a permission key to src/cli/permission-keys.test.ts. +const KEY: Conversion = "text" + +/** Each of our fields and how a value the tracker sent becomes it. */ +export const TRACKER_FIELDS: { readonly [K in TrackerKind]: Readonly> } = { + team: { ...TIMES, name: "text", key: KEY }, + user: { ...TIMES, name: "text", email: "text", active: "bool" }, + state: { ...TIMES, name: "text", type: "text", team: "id", position: "number" }, + label: { ...TIMES, name: "text", team: "id" }, + project: { + ...TIMES, + name: "text", + description: "text", + status: "text", + lead: "id", + teams: "ids", + targetDate: "day", + url: "text", + }, + issue: { + ...TIMES, + title: "text", + key: KEY, + description: "text", + state: "id", + team: "id", + project: "id", + assignee: "id", + creator: "id", + labels: "ids", + parent: "id", + priority: "number", + dueDate: "day", + completedAt: "epoch", + canceledAt: "epoch", + url: "text", + }, + comment: { ...TIMES, issue: "id", body: "text", author: "id", url: "text" }, +} + +/** The fields a record of the kind always has; a missing one gets its empty value, never a guess. */ +const REQUIRED: { readonly [K in TrackerKind]: Readonly> } = { + team: { name: "" }, + user: { name: "" }, + state: { name: "" }, + label: { name: "" }, + project: { name: "" }, + issue: { title: "" }, + comment: { body: "" }, +} + +/** A selected object the tracker answered null (`project: null`): the field is none, not "not sent". */ +const nulled = (raw: crmValues.CrmRaw, path: string): boolean => { + let inner: unknown = raw + for (const step of path.split(".")) { + if (inner === null) return true + if (!crmValues.isRaw(inner)) return false + inner = inner[step] + } + return false +} + +/** One row as our record of the kind, or null for a row without an id, or a state whose type is none of ours. */ +export const mapTrackerRecord = ( + kind: K, + map: Readonly>, + raw: crmValues.CrmRaw, +): TrackerRecords[K] | null => { + const out = mapFields(map, TRACKER_FIELDS[kind], raw) + for (const [field, rule] of Object.entries(map)) { + const type = TRACKER_FIELDS[kind][field] + if (out[field] !== undefined || type === undefined) continue + if (rulePaths(rule).some(({ path, inside }) => inside === undefined && nulled(raw, path))) + out[field] = convert(type, null) + } + if (typeof out.externalId !== "string") return null + for (const [field, empty] of Object.entries(REQUIRED[kind])) if (out[field] == null) out[field] = empty + // "Open work" is read by the type, so a state of an unknown type would hide its issues from every filter. + if (kind === "state" && !(STATE_TYPES as readonly unknown[]).includes(out.type)) return null + if (kind === "comment" && typeof out.issue !== "string") return null + return out as unknown as TrackerRecords[K] +} diff --git a/src/tracker/manifest/source.test.ts b/src/tracker/manifest/source.test.ts new file mode 100644 index 000000000..e1dbd3849 --- /dev/null +++ b/src/tracker/manifest/source.test.ts @@ -0,0 +1,517 @@ +import { + type GraphqlFakeCall, + graphqlFake, + type ManifestOperation, + type SchemaNode, + type Selection, +} from "@wirecat/cli-base/codegen" +import { describe, expect, it } from "vitest" +import { serviceApiCall } from "../../cli/service-api-call.js" +import type { ServiceApiManifest } from "../../cli/service-api-manifest.js" +import type { TrackerKind } from "../../domain/tracker.js" +import { trackerSourceContract } from "../testing/index.js" +import type { TrackerManifest } from "./manifest.js" +import { parseTrackerManifest } from "./manifest.js" +import { trackerSourceFromManifest } from "./source.js" + +const URL = "https://api.example-tracker.test/graphql" +const KEY = "lin_api_fake" + +const text: SchemaNode = { type: "string", nullable: true } +const ref = (name: string, nullable = true): SchemaNode => ({ + type: "ref", + ref: name, + ...(nullable ? { nullable } : {}), +}) +const object = (properties: Record): SchemaNode => ({ type: "object", properties, required: [] }) +const times = { createdAt: text, updatedAt: text, archivedAt: text } +const node = (properties: Record) => object({ id: { type: "string" }, ...times, ...properties }) +const connection = (of: string) => + object({ nodes: { type: "array", items: ref(of, false) }, pageInfo: ref("PageInfo", false) }) + +const definitions: Record = { + PageInfo: object({ hasNextPage: { type: "boolean" }, endCursor: text }), + ListRequest: { + type: "object", + properties: { + first: { type: "integer", nullable: true }, + after: text, + includeArchived: { type: "boolean", nullable: true }, + filter: { + type: "object", + properties: {}, + required: [], + additionalProperties: { type: "unknown" }, + nullable: true, + }, + }, + required: [], + }, + Team: node({ name: text, key: text }), + User: node({ name: text, email: text, active: { type: "boolean", nullable: true } }), + WorkflowState: node({ name: text, type: text, position: { type: "number", nullable: true }, team: ref("Team") }), + IssueLabel: node({ name: text, team: ref("Team") }), + Project: node({ + name: text, + description: text, + state: text, + lead: ref("User"), + teams: ref("TeamConnection", false), + targetDate: text, + url: text, + }), + Issue: node({ + identifier: text, + title: text, + description: text, + state: ref("WorkflowState"), + team: ref("Team"), + project: ref("Project"), + assignee: ref("User"), + creator: ref("User"), + labels: ref("IssueLabelConnection", false), + parent: ref("Issue"), + priority: { type: "number", nullable: true }, + dueDate: text, + completedAt: text, + canceledAt: text, + url: text, + }), + Comment: node({ body: text, issue: ref("Issue"), user: ref("User"), url: text }), +} +for (const name of ["Team", "User", "WorkflowState", "IssueLabel", "Project", "Issue", "Comment"]) + definitions[`${name}Connection`] = connection(name) + +const FIELDS: Record = { + team: ["teams", "Team"], + user: ["users", "User"], + state: ["workflowStates", "WorkflowState"], + label: ["issueLabels", "IssueLabel"], + project: ["projects", "Project"], + issue: ["issues", "Issue"], + comment: ["comments", "Comment"], +} + +const listOperation = (field: string, type: string): ManifestOperation => ({ + id: field, + command: field.toLowerCase(), + binding: { + kind: "graphql", + operation: "query", + field, + variables: { first: "Int", after: "String", filter: `${type}Filter`, includeArchived: "Boolean" }, + selection: "nodes { id } pageInfo { hasNextPage endCursor }", + }, + effect: "read", + tags: ["query"], + parameters: ["first", "after", "filter", "includeArchived"].map((name) => ({ + name, + in: "body" as const, + required: false, + schema: (definitions.ListRequest as { properties: Record }).properties[name] as SchemaNode, + })), + request: { required: false, confidence: "contract", schema: "ListRequest" }, + response: { confidence: "contract", schema: `${type}Connection` }, + pagination: { + style: "cursor", + cursor: { in: "body", name: "after" }, + next: "pageInfo.endCursor", + items: "nodes", + hasMore: "pageInfo.hasNextPage", + limit: { in: "body", name: "first" }, + }, +}) + +const api: ServiceApiManifest = { + servers: [{ url: URL }], + securitySchemes: { key: { type: "apiKey", in: "header", name: "Authorization" } }, + security: [{ key: [] }], + operations: [ + ...Object.values(FIELDS).map(([field, type]) => listOperation(field, type)), + { + id: "viewer", + command: "viewer", + binding: { kind: "graphql", operation: "query", field: "viewer", variables: {}, selection: "id" }, + effect: "read", + tags: ["query"], + parameters: [], + response: { confidence: "contract", schema: "User" }, + }, + ], +} + +const PAGE_INFO = "pageInfo { hasNextPage endCursor }" +const TIMES = "id createdAt updatedAt archivedAt" +const SELECTIONS: Record = { + team: `nodes { ${TIMES} name key } ${PAGE_INFO}`, + user: `nodes { ${TIMES} name email active } ${PAGE_INFO}`, + state: `nodes { ${TIMES} name type position team { id } } ${PAGE_INFO}`, + label: `nodes { ${TIMES} name team { id } } ${PAGE_INFO}`, + project: `nodes { ${TIMES} name description state lead { id } teams { nodes { id } } targetDate url } ${PAGE_INFO}`, + issue: + `nodes { ${TIMES} identifier title description state { id } team { id } project { id } assignee { id } ` + + `creator { id } labels { nodes { id } } parent { id } priority dueDate completedAt canceledAt url } ${PAGE_INFO}`, + comment: `nodes { ${TIMES} body issue { id } user { id } url } ${PAGE_INFO}`, +} +const TIME_MAP = { externalId: "id", createdAt: "createdAt", updatedAt: "updatedAt", archivedAt: "archivedAt" } +const MAPS: { [K in TrackerKind]: Record } = { + team: { ...TIME_MAP, name: "name", key: "key" }, + user: { ...TIME_MAP, name: "name", email: "email", active: "active" }, + state: { ...TIME_MAP, name: "name", type: "type", position: "position", team: "team.id" }, + label: { ...TIME_MAP, name: "name", team: "team.id" }, + project: { + ...TIME_MAP, + name: "name", + description: "description", + status: "state", + lead: "lead.id", + teams: { each: "teams.nodes", value: "id" }, + targetDate: "targetDate", + url: "url", + }, + issue: { + ...TIME_MAP, + key: "identifier", + title: "title", + description: "description", + state: "state.id", + team: "team.id", + project: "project.id", + assignee: "assignee.id", + creator: "creator.id", + labels: { each: "labels.nodes", value: "id" }, + parent: "parent.id", + priority: "priority", + dueDate: "dueDate", + completedAt: "completedAt", + canceledAt: "canceledAt", + url: "url", + }, + comment: { ...TIME_MAP, body: "body", issue: "issue.id", author: "user.id", url: "url" }, +} + +const ALL = { first: 50, includeArchived: true } +const MANIFEST = { + provider: "example-tracker", + kinds: ["team", "user", "state", "label", "project", "issue", "comment"], + entities: Object.fromEntries( + (Object.keys(FIELDS) as TrackerKind[]).map((kind) => { + const [operation] = FIELDS[kind] + return [ + kind, + { + list: { operation, variables: ALL, selection: SELECTIONS[kind] }, + changes: { + operation, + variables: { ...ALL, filter: { updatedAt: { gte: "{since}" } } }, + selection: SELECTIONS[kind], + since: "iso", + }, + ids: { operation, variables: ALL, selection: `nodes { id } ${PAGE_INFO}` }, + map: MAPS[kind], + }, + ] + }), + ), + self: { call: { operation: "viewer" }, path: "id" }, +} as unknown as TrackerManifest + +type Row = Record + +const project = (value: unknown, selection: Selection | true): unknown => { + if (selection === true || value === null || value === undefined) return value ?? null + if (Array.isArray(value)) return value.map((one) => project(one, selection)) + return Object.fromEntries( + Object.entries(selection).map(([field, inner]) => [field, project((value as Row)[field], inner)]), + ) +} + +const at = (ms: number) => new Date(ms).toISOString() +const SEEDED = Date.UTC(2026, 10, 1) +const ref1 = (id: string | null) => (id === null ? null : { id }) + +const seed = (): Record => { + const times = (n: number) => ({ createdAt: at(SEEDED), updatedAt: at(SEEDED + n * 1000), archivedAt: null }) + return { + team: [ + { id: "t1", name: "Engineering", key: "ENG", ...times(1) }, + { id: "t2", name: "Design", key: "DES", ...times(2) }, + { id: "t3", name: "Support", key: "SUP", ...times(3) }, + ], + user: [ + { id: "u1", name: "Ada", email: "ada@example.com", active: true, ...times(1) }, + { id: "u2", name: "Bob", email: "bob@example.com", active: true, ...times(2) }, + { id: "u3", name: "Cy", email: "cy@example.com", active: false, ...times(3) }, + ], + state: [ + { id: "s1", name: "Todo", type: "unstarted", position: 1, team: ref1("t1"), ...times(1) }, + { id: "s2", name: "Doing", type: "started", position: 2, team: ref1("t1"), ...times(2) }, + { id: "s3", name: "Done", type: "completed", position: 3, team: ref1("t1"), ...times(3) }, + ], + label: [ + { id: "l1", name: "Bug", team: ref1("t1"), ...times(1) }, + { id: "l2", name: "Feature", team: null, ...times(2) }, + { id: "l3", name: "Chore", team: ref1("t2"), ...times(3) }, + ], + project: [1, 2, 3].map((n) => ({ + id: `p${n}`, + name: `Project ${n}`, + description: null, + state: "started", + lead: ref1("u1"), + teams: { nodes: [{ id: "t1" }] }, + targetDate: "2026-12-31", + url: `https://tracker.example/p${n}`, + ...times(n), + })), + issue: [1, 2, 3].map((n) => ({ + id: `i${n}`, + identifier: `ENG-${n}`, + title: `Issue ${n}`, + description: null, + state: ref1("s1"), + team: ref1("t1"), + project: n === 3 ? null : ref1("p1"), + assignee: n === 2 ? null : ref1("u1"), + creator: ref1("u2"), + labels: { nodes: n === 1 ? [{ id: "l1" }] : [] }, + parent: null, + priority: n, + dueDate: null, + completedAt: null, + canceledAt: null, + url: `https://tracker.example/ENG-${n}`, + ...times(n), + })), + comment: [1, 2, 3].map((n) => ({ + id: `c${n}`, + body: `Comment ${n}`, + issue: ref1("i1"), + user: ref1("u2"), + url: null, + ...times(n), + })), + } +} + +/** A tracker that pages by two, filters by `updatedAt.gte`, leaves archived rows out unless asked, and stamps changes. */ +const tracker = (answers: Readonly unknown>> = {}) => { + const rows = seed() + let tick = 0 + const clock = () => Date.now() + tick++ * 1000 + const list = (kind: TrackerKind) => (call: GraphqlFakeCall) => { + const { first, after, filter, includeArchived } = call.variables as { + first?: number + after?: string | null + filter?: { updatedAt?: { gte?: string } } + includeArchived?: boolean + } + const since = filter?.updatedAt?.gte === undefined ? null : Date.parse(filter.updatedAt.gte) + const listed = rows[kind] + .filter((row) => includeArchived || row.archivedAt === null) + .filter((row) => since === null || Date.parse(String(row.updatedAt)) >= since) + .sort( + (a, b) => String(a.updatedAt).localeCompare(String(b.updatedAt)) || String(a.id).localeCompare(String(b.id)), + ) + const offset = after ? Number(after.slice(2)) : 0 + const size = Math.min(first ?? 50, 2) + const more = offset + size < listed.length + return { + data: project( + { + nodes: listed.slice(offset, offset + size), + pageInfo: { hasNextPage: more, endCursor: more ? `o:${offset + size}` : null }, + }, + call.selection ?? true, + ), + } + } + const fake = graphqlFake({ + operations: api.operations, + definitions, + url: URL, + auth: { header: "authorization", value: KEY }, + answers: { + ...Object.fromEntries((Object.keys(FIELDS) as TrackerKind[]).map((kind) => [FIELDS[kind][0], list(kind)])), + viewer: () => ({ data: { id: "u1" } }), + ...answers, + }, + }) + const find = (kind: TrackerKind, id: string) => { + const row = rows[kind].find((one) => one.id === id) + if (!row) throw new Error(`no ${kind} ${id}`) + return row + } + const call = serviceApiCall({ manifest: api, service: "Tracker", fetch: fake.fetch, token: () => KEY }) + const sourceAt = (skew = 0) => trackerSourceFromManifest(MANIFEST, { api, call, now: () => clock() + skew }) + return { + rows, + fake, + sourceAt, + touch(kind: TrackerKind, id: string) { + find(kind, id).updatedAt = at(clock()) + }, + archive(kind: TrackerKind, id: string) { + const row = find(kind, id) + row.archivedAt = at(clock()) + row.updatedAt = row.archivedAt + }, + remove(kind: TrackerKind, id: string) { + rows[kind].splice(rows[kind].indexOf(find(kind, id)), 1) + }, + } +} + +describe("trackerSourceFromManifest over a Linear-shaped GraphQL fake", () => { + for (const c of trackerSourceContract(() => { + const one = tracker() + return { + source: one.sourceAt(), + held: Object.fromEntries( + (Object.keys(FIELDS) as TrackerKind[]).map((kind) => [kind, one.rows[kind].map((row) => String(row.id))]), + ), + deadCursor: { kind: "issue", cursor: "not a time" }, + faults: { + skewed: (ms) => one.sourceAt(ms), + change: () => { + one.touch("issue", "i2") + return { kind: "issue", id: "i2" } + }, + archive: () => { + one.archive("issue", "i3") + return { kind: "issue", id: "i3" } + }, + remove: () => { + one.remove("issue", "i3") + return { kind: "issue", id: "i3" } + }, + }, + } + })) + it(c.name, async () => { + await c.run() + }) + + it("maps a row onto the tracker domain, reading a null object as none, and names the signed-in user", async () => { + const one = tracker() + const source = one.sourceAt() + const page = await source.changes("issue", null) + if ("reset" in page) throw new Error("a full list is never a reset") + expect(page.items[0]).toEqual({ + externalId: "i1", + createdAt: SEEDED, + updatedAt: SEEDED + 1000, + archivedAt: null, + key: "ENG-1", + title: "Issue 1", + description: null, + state: "s1", + team: "t1", + project: "p1", + assignee: "u1", + creator: "u2", + labels: ["l1"], + parent: null, + priority: 1, + dueDate: null, + completedAt: null, + canceledAt: null, + url: "https://tracker.example/ENG-1", + }) + expect(page.items[1]).toMatchObject({ externalId: "i2", assignee: null, labels: [] }) + expect(await source.self?.()).toBe("u1") + expect(one.fake.refusals).toEqual([]) + }) +}) + +describe("an optional kind", () => { + const refused = () => ({ errors: [{ message: "Forbidden", extensions: { code: "FORBIDDEN" } }] }) + const optional = { + ...MANIFEST, + entities: { ...MANIFEST.entities, comment: { ...MANIFEST.entities.comment, optional: { message: "no comments" } } }, + } as TrackerManifest + + it("is a gap when the tracker refuses it in errors[], for its changes and its ids, and the others still read", async () => { + const one = tracker({ [FIELDS.comment[0]]: refused }) + const call = serviceApiCall({ manifest: api, service: "Tracker", fetch: one.fake.fetch, token: () => KEY }) + const source = trackerSourceFromManifest(optional, { api, call }) + const gaps: unknown[] = [] + const context = { spend: () => undefined, gap: (gap: unknown) => gaps.push(gap) } + expect(await source.changes("comment", null, undefined, context)).toEqual({ + items: [], + hasMore: false, + cursor: null, + }) + expect(await source.ids("comment", undefined, context)).toEqual({ items: [], hasMore: false }) + expect(gaps).toEqual([ + { kind: "comment", message: "no comments" }, + { kind: "comment", message: "no comments" }, + ]) + const issues = await source.changes("issue", null, undefined, context) + expect("items" in issues && issues.items.length).toBe(2) + }) + + it("is an error when the kind is not optional", async () => { + const one = tracker({ [FIELDS.comment[0]]: refused }) + await expect(one.sourceAt().changes("comment", null)).rejects.toMatchObject({ code: "permission_error" }) + }) +}) + +describe("parseTrackerManifest", () => { + const broken = (change: (manifest: Row) => void) => { + const copy = structuredClone({ ...MANIFEST }) as unknown as Row + change(copy) + return () => parseTrackerManifest(copy, api) + } + const issue = (manifest: Row) => (manifest.entities as Record).issue as Row + + it("refuses a map path the selection leaves out", () => { + expect( + broken((manifest) => { + ;(issue(manifest).list as Row).selection = `nodes { id title } ${PAGE_INFO}` + }), + ).toThrow(/entities\.issue\.list\.selection: map\.createdAt reads createdAt, which the selection leaves out/) + }) + + it("refuses a selection GraphQL could not send", () => { + expect( + broken((manifest) => { + ;(issue(manifest).list as Row).selection = "nodes { id title(first: 2) }" + }), + ).toThrow(/entities\.issue\.list\.selection/) + }) + + it("names a mistyped operation and field, and a change call without {since}", () => { + expect( + broken((manifest) => { + ;(issue(manifest).list as Row).operation = "isues" + }), + ).toThrow(/no operation isues in the API; did you mean issues\?/) + expect( + broken((manifest) => { + ;(issue(manifest).map as Row).titel = "title" + }), + ).toThrow(/issue has no such field; did you mean title\?/) + expect( + broken((manifest) => { + ;(issue(manifest).changes as Row).variables = ALL + }), + ).toThrow(/no "\{since\}"/) + }) + + it("checks the signed-in user's path against the selection", () => { + expect( + broken((manifest) => { + ;(manifest.self as Row).path = "viewer.id" + }), + ).toThrow(/self\.call\.selection: map\.self reads viewer\.id/) + }) + + it("refuses kinds out of order", () => { + expect( + broken((manifest) => { + manifest.kinds = ["issue", "team"] + }), + ).toThrow(/in this order/) + }) +}) diff --git a/src/tracker/manifest/source.ts b/src/tracker/manifest/source.ts new file mode 100644 index 000000000..14907ef9a --- /dev/null +++ b/src/tracker/manifest/source.ts @@ -0,0 +1,216 @@ +import { CliError, isCliError } from "@wirecat/cli-base" +import type { Pagination } from "@wirecat/cli-base/codegen" +import type { ApiCommandInput } from "../../cli/bot/api.js" +import type { ServiceApiCall } from "../../cli/service-api-call.js" +import { nextPage, parameterValue, type ServiceApiManifest, withValue } from "../../cli/service-api-manifest.js" +import { crmValues } from "../../crm/index.js" +import { at, evaluate, MISSING } from "../../crm/manifest/map.js" +import type { TrackerKind, TrackerRecords } from "../../domain/tracker.js" +import type { TrackerCallContext, TrackerChanges, TrackerIdPage, TrackerSource } from "../source.js" +import { + parseTrackerManifest, + type TrackerAfterHook, + type TrackerChangesCall, + type TrackerManifest, + type TrackerManifestHooks, + type TrackerOperationCall, +} from "./manifest.js" +import { mapTrackerRecord } from "./map.js" + +export interface TrackerManifestSourceOptions { + /** The generated API manifest the operations are named in: their bindings, paging and sign-in. */ + api: ServiceApiManifest + /** Sends one operation: `serviceApiCall` over the CLI's fetch and token, or a test's fake. */ + call: ServiceApiCall + now?: () => number + hooks?: TrackerManifestHooks +} + +const OVERLAP_MS = 10 * 60_000 + +const DROP = Symbol("drop") + +const fill = (value: unknown, since: string | undefined): unknown => { + if (value === "{since}") return since ?? DROP + if (Array.isArray(value)) return value.map((one) => fill(one, since)).filter((one) => one !== DROP) + if (crmValues.isRaw(value)) + return Object.fromEntries( + Object.entries(value).flatMap(([key, one]) => { + const filled = fill(one, since) + return filled === DROP ? [] : [[key, filled]] + }), + ) + return value +} + +/** The variables travel as the body: each GraphQL argument is a body parameter of the operation. */ +const inputOf = (call: TrackerOperationCall, since?: string): ApiCommandInput => ({ + path: {}, + query: {}, + headers: {}, + fields: {}, + ...(call.variables ? { body: JSON.stringify(fill(call.variables, since)) } : {}), + ...(call.selection ? { selection: call.selection } : {}), +}) + +const SINCE: Record string> = { + iso: (ms) => new Date(ms).toISOString(), + ms: (ms) => String(ms), + seconds: (ms) => String(Math.floor(ms / 1000)), +} + +const cursorParameter = (pagination: Pagination) => + pagination.style === "cursor" + ? pagination.cursor + : pagination.style === "offset" + ? pagination.offset + : pagination.style === "page" + ? pagination.page + : undefined + +/** `@`: a round spread over runs ends with its first page's time. */ +const pageOf = (page: string | undefined, now: number) => { + const cut = page?.lastIndexOf("@") ?? -1 + const started = Number(page?.slice(cut + 1)) + if (page === undefined || cut < 0 || !Number.isSafeInteger(started)) return { token: undefined, started: now } + return { token: page.slice(0, cut) || undefined, started } +} + +/** + * A `TrackerSource` read from a manifest: each kind listed and read for changes by the operations it names, paged by + * each operation's own `pagination`, each row mapped onto the tracker domain. Deletions are found by a pass over + * every id. What the manifest cannot say comes from `hooks`. + */ +export const trackerSourceFromManifest = (input: unknown, options: TrackerManifestSourceOptions): TrackerSource => { + const manifest: TrackerManifest = parseTrackerManifest(input, options.api) + const now = options.now ?? Date.now + const hooks = options.hooks ?? {} + const overlap = manifest.overlapMs ?? OVERLAP_MS + const operations = new Map(options.api.operations.map((one) => [one.id, one])) + + /** The engine paid for a call's first request; each further one is spent from the run's budget. */ + const metered = (context?: TrackerCallContext): ServiceApiCall => { + let paid = false + return (operation, request) => { + if (paid) context?.spend() + paid = true + return options.call(operation, request) + } + } + + const readPage = async ( + call: TrackerOperationCall, + since: string | undefined, + token: string | undefined, + send: ServiceApiCall, + ) => { + const pagination = operations.get(call.operation)?.pagination + const parameter = pagination && cursorParameter(pagination) + let request: ApiCommandInput = inputOf(call, since) + if (token !== undefined && parameter) + request = { ...withValue(request, parameter, pagination.style === "cursor" ? token : Number(token)) } + const answer = await send(call.operation, request) + if (!pagination || !parameter) { + const listed = call.items === undefined ? answer : at(answer, call.items) + return { rows: crmValues.rawsOf(listed), next: undefined } + } + const step = nextPage(pagination, request, answer) + const rows = crmValues.rawsOf(call.items === undefined ? step.items : at(answer, call.items)) + const value = step.next && "input" in step.next ? parameterValue(step.next.input, parameter) : undefined + return { rows, next: value === undefined || value === null ? undefined : String(value) } + } + + const entityOf = (kind: TrackerKind) => { + const found = manifest.entities[kind] + if (!found) throw new CliError("configuration_error", `the source manifest has no ${kind}`) + return found + } + + /** A refusal of a kind the manifest marks optional is a gap; the other kinds still sync. */ + const optionalRefused = (kind: TrackerKind, error: unknown, context?: TrackerCallContext) => { + const optional = manifest.entities[kind]?.optional + if (!optional || !isCliError(error) || error.code !== "permission_error") return false + context?.gap({ kind, message: optional.message, ...(optional.scope ? { scope: optional.scope } : {}) }) + return true + } + + const mapped = async (kind: K, rows: readonly crmValues.CrmRaw[], send: ServiceApiCall) => { + const entity = entityOf(kind) + const after = hooks.after?.[kind] as TrackerAfterHook | undefined + const records: TrackerRecords[K][] = [] + for (const raw of rows) { + const record = mapTrackerRecord(kind, entity.map, raw) + if (record !== null) records.push(after ? await after(record, raw, { call: send }) : record) + } + return records + } + + const readChanges = async ( + kind: K, + cursor: string | null, + page: string | undefined, + context?: TrackerCallContext, + ): Promise> => { + const entity = entityOf(kind) + const since = cursor === null ? null : Date.parse(cursor) + if (since !== null && Number.isNaN(since)) return { reset: true } + const { token, started } = pageOf(page, now()) + const send = metered(context) + const changes = since !== null ? entity.changes : undefined + const { rows, next } = await readPage( + changes ?? entity.list, + changes && since !== null ? SINCE[changes.since](since - overlap) : undefined, + token, + send, + ) + const items = await mapped(kind, rows, send) + return next === undefined + ? { items, hasMore: false, cursor: new Date(started).toISOString() } + : { items, hasMore: true, nextCursor: `${next}@${started}` } + } + + const readIds = async (kind: TrackerKind, page: string | undefined, context?: TrackerCallContext) => { + const entity = entityOf(kind) + const { rows, next } = await readPage(entity.ids ?? entity.list, undefined, page, metered(context)) + const rule = entity.map.externalId as NonNullable + const items = rows.flatMap((raw) => { + const found = evaluate(rule, raw) + const externalId = found === MISSING ? null : crmValues.id(found) + return externalId === null ? [] : [externalId] + }) + return next === undefined ? { items, hasMore: false } : { items, hasMore: true, nextCursor: next } + } + + const base: TrackerSource = { + provider: manifest.provider, + traits: { kinds: manifest.kinds }, + async changes(kind: K, cursor: string | null, page?: string, context?: TrackerCallContext) { + try { + return await readChanges(kind, cursor, page, context) + } catch (error) { + if (optionalRefused(kind, error, context)) return { items: [], hasMore: false, cursor: null } + throw error + } + }, + async ids(kind, page, context): Promise { + try { + return await readIds(kind, page, context) + } catch (error) { + // Refused part way through a pass, the ids read so far would mark every unread row deleted. + if (page === undefined && optionalRefused(kind, error, context)) return { items: [], hasMore: false } + throw error + } + }, + ...(manifest.self + ? { + async self(context?: TrackerCallContext) { + const self = manifest.self as NonNullable + const found = crmValues.id(at(await metered(context)(self.call.operation, inputOf(self.call)), self.path)) + if (found === null) throw new CliError("invalid_response", `${manifest.provider} named no signed-in user`) + return found + }, + } + : {}), + } + return { ...base, ...hooks.extend?.(base) } +} diff --git a/src/tracker/sync.ts b/src/tracker/sync.ts index 2cf5120e1..a05926fbd 100644 --- a/src/tracker/sync.ts +++ b/src/tracker/sync.ts @@ -210,12 +210,15 @@ const passIds = async (run: Run, kind: TrackerKind, result: TrackerSyncResult, m }) const unread = run.report.gaps.some((gap) => gap.kind === kind) const emptied = pass.ids.length === 0 && unlisted > 0 - if (!mode.full && (unread || emptied || (stored >= GUARDED_FROM && unlisted > stored * MOST_UNLISTED))) { + const many = emptied || (stored >= GUARDED_FROM && unlisted > stored * MOST_UNLISTED) + // What the sign-in could not read is not gone: even a rescan never marks it deleted. + if (unlisted > 0 && (unread || (many && !mode.full))) { result.pass = "held" - result.held = - `${source.provider} listed ${pass.ids.length} ${kind} ids${unread ? " and could not read all of them" : ""}, ` + - `so ${unlisted} of the ${stored} stored would be marked deleted; nothing was. Check the sign-in's rights, ` + - "then run sync --rescan to let it through" + result.held = unread + ? `${source.provider} could not read all ${kind}s, so the ${unlisted} it did not list stay; nothing was ` + + "marked deleted. Check the sign-in's rights, then run sync again" + : `${source.provider} listed ${pass.ids.length} ${kind} ids, so ${unlisted} of the ${stored} stored would be ` + + "marked deleted; nothing was. Check the sign-in's rights, then run sync --rescan to let it through" } else { result.unlisted += await store.tracker.removeUnlisted(account, kind, pass.ids, pass.started, run.now) result.pass = "complete" diff --git a/src/tracker/tracker.test.ts b/src/tracker/tracker.test.ts index b56985253..0ff523c74 100644 --- a/src/tracker/tracker.test.ts +++ b/src/tracker/tracker.test.ts @@ -145,7 +145,11 @@ describe("syncTracker guards", () => { ...lost, changes: (async () => ({ items: [], hasMore: false, cursor: null })) as unknown as typeof source.changes, } - const through = await syncTracker(emptied, store, account, { now: T + 3 * DAYS, rescan: true }) + const refused = await syncTracker(gapped, store, account, { now: T + 3 * DAYS, rescan: true }) + expect(refused.synced.find((one) => one.kind === "comment")?.pass).toBe("held") + expect(await store.tracker.count(account, "comment")).toBe(1) + + const through = await syncTracker(emptied, store, account, { now: T + 4 * DAYS, rescan: true }) expect(through.synced.find((one) => one.kind === "issue")).toMatchObject({ pass: "complete", unlisted: 3 }) })