From 81283d816801d7b2284fa0213ebd4002286944f7 Mon Sep 17 00:00:00 2001 From: M3gA-Mind Date: Thu, 24 Sep 2026 19:34:39 +0530 Subject: [PATCH 01/11] test(e2e): make every spec runnable and type-checked Adds `typecheck:e2e` (tsc -p test/tsconfig.e2e.json --noEmit) and wires it into CI Lite. 205 spec files were never type-checked by any lane; 34 errors were hiding, 7 of them unused `@ts-expect-error` directives suppressing nothing. Restores the 15 WDIO specs that no lane invoked. `e2e-run-all-flows.sh` is a hand-maintained allowlist that had drifted from the spec directory, and `--spec` overrides the wdio.conf glob that would otherwise have caught them. The orphan guard in generate-test-inventory.mjs now fails on any spec no lane runs, so the drift cannot recur silently: verified by adding an unlisted spec and observing a non-zero exit that names it. --- .github/ci-paths-filter.yml | 4 ++ .github/workflows/ci-lite.yml | 12 ++++ app/package.json | 1 + app/scripts/e2e-run-all-flows.sh | 16 +++++ scripts/generate-test-inventory.mjs | 107 ++++++++++++++++++++++++++-- scripts/mock-api-core.d.mts | 60 ++++++++++++++++ 6 files changed, 195 insertions(+), 5 deletions(-) create mode 100644 scripts/mock-api-core.d.mts diff --git a/.github/ci-paths-filter.yml b/.github/ci-paths-filter.yml index 3ed696098d..e0952a1ed8 100644 --- a/.github/ci-paths-filter.yml +++ b/.github/ci-paths-filter.yml @@ -23,6 +23,10 @@ frontend: - 'app/src/**' - 'app/scripts/**' - 'app/test/vitest.config.ts' + # The e2e/Playwright specs themselves: `typecheck:e2e` in the frontend + # job exists to check them, so a PR that touches only a spec has to + # match this filter or the job skips and the check reads as green. + - 'app/test/**' - 'app/tsconfig*.json' - 'app/vite.config.*' - 'app/tailwind.config.*' diff --git a/.github/workflows/ci-lite.yml b/.github/workflows/ci-lite.yml index 46be12daca..99f16612f1 100644 --- a/.github/workflows/ci-lite.yml +++ b/.github/workflows/ci-lite.yml @@ -190,6 +190,18 @@ jobs: env: NODE_ENV: test + # `compile` above covers `app/src`; its tsconfig does not include the + # test trees, so the 205 WDIO + Playwright specs were never type-checked + # by any lane. `test/tsconfig.e2e.json` was already correct and simply had + # no runner (#6566). Running it found 34 errors, 7 of which were + # `@ts-expect-error` directives that had stopped suppressing anything and + # would have silently swallowed the next real error on the line below them. + - name: Type check E2E and Playwright specs + if: needs.changes.outputs.frontend == 'true' + run: pnpm --filter openhuman-app typecheck:e2e + env: + NODE_ENV: test + - name: Check Prettier formatting if: needs.changes.outputs.frontend == 'true' run: pnpm --filter openhuman-app format:check diff --git a/app/package.json b/app/package.json index a2db7a4b9a..eb8fd2c6eb 100644 --- a/app/package.json +++ b/app/package.json @@ -26,6 +26,7 @@ "build:web:e2e": "bash ./scripts/e2e-web-build.sh", "build:web": "cross-env VITE_OPENHUMAN_TARGET=web node scripts/build-parallel.mjs", "compile": "tsc --noEmit", + "typecheck:e2e": "tsc -p test/tsconfig.e2e.json --noEmit", "preview": "vite preview", "tauri": "node scripts/tauri.cjs", "tauri:build:ui": "node scripts/tauri.cjs build -- --bin OpenHuman", diff --git a/app/scripts/e2e-run-all-flows.sh b/app/scripts/e2e-run-all-flows.sh index e80ebf249f..b0b1060acf 100755 --- a/app/scripts/e2e-run-all-flows.sh +++ b/app/scripts/e2e-run-all-flows.sh @@ -265,6 +265,13 @@ if should_run_suite "chat"; then run "test/e2e/specs/chat-live-history-parity.spec.ts" "chat-live-history-parity" "chat" run "test/e2e/specs/chat-tool-error-recovery.spec.ts" "chat-error-recovery" "chat" run "test/e2e/specs/agent-review.spec.ts" "agent-review" "chat" + run "test/e2e/specs/chat-harness-subagent-continue.spec.ts" "chat-subagent-continue" "chat" + run "test/e2e/specs/chat-background-activity-panel.spec.ts" "chat-background-activity" "chat" + run "test/e2e/specs/chat-todos-goals.spec.ts" "chat-todos-goals" "chat" + run "test/e2e/specs/agent-harness-behaviors.spec.ts" "agent-harness-behaviors" "chat" + run "test/e2e/specs/agent-teams-live.spec.ts" "agent-teams-live" "chat" + run "test/e2e/specs/file-drop-guard.spec.ts" "file-drop-guard" "chat" + run "test/e2e/specs/chat-external-link.spec.ts" "chat-external-link" "chat" run "test/e2e/specs/mega-flow.spec.ts" "mega-flow" "chat" _mini_summary "chat" fi @@ -281,6 +288,8 @@ if should_run_suite "skills"; then run "test/e2e/specs/skill-multi-round.spec.ts" "skill-multi-round" "skills" run "test/e2e/specs/skill-oauth.spec.ts" "skill-oauth" "skills" run "test/e2e/specs/skill-socket-reconnect.spec.ts" "skill-socket-reconnect" "skills" + run "test/e2e/specs/skill-activate-invoke-chat.spec.ts" "skill-activate-invoke" "skills" + run "test/e2e/specs/skill-activation-persistence.spec.ts" "skill-activation-persist" "skills" _mini_summary "skills" fi @@ -293,6 +302,8 @@ if should_run_suite "notifications"; then run "test/e2e/specs/notifications.spec.ts" "notifications" "notifications" run "test/e2e/specs/memory-roundtrip.spec.ts" "memory-roundtrip" "notifications" run "test/e2e/specs/coding-session-memory.spec.ts" "coding-session-memory" "notifications" + run "test/e2e/specs/memory-sources-conversation.spec.ts" "memory-sources-conv" "notifications" + run "test/e2e/specs/memory-sync-schedule.spec.ts" "memory-sync-schedule" "notifications" run "test/e2e/specs/cron-jobs-flow.spec.ts" "cron-jobs" "notifications" _mini_summary "notifications" fi @@ -326,6 +337,7 @@ if should_run_suite "providers"; then run "test/e2e/specs/telegram-channel-flow.spec.ts" "telegram-channel" "providers" run "test/e2e/specs/gmail-flow.spec.ts" "gmail" "providers" run "test/e2e/specs/accounts-provider-modal.spec.ts" "accounts-providers" "providers" + run "test/e2e/specs/credential-channels-flow.spec.ts" "credential-channels" "providers" _mini_summary "providers" fi @@ -372,6 +384,7 @@ if should_run_suite "connectors"; then run "test/e2e/specs/connector-gmail-composio.spec.ts" "connector-gmail-composio" "connectors" run "test/e2e/specs/connector-jira.spec.ts" "connector-jira" "connectors" run "test/e2e/specs/connector-session-guard.spec.ts" "connector-session-guard" "connectors" + run "test/e2e/specs/composio-github-tools-tags.spec.ts" "composio-github-tags" "connectors" _mini_summary "connectors" fi @@ -399,6 +412,7 @@ if should_run_suite "settings"; then run "test/e2e/specs/settings-account-preferences.spec.ts" "settings-account" "settings" run "test/e2e/specs/settings-advanced-config.spec.ts" "settings-advanced" "settings" run "test/e2e/specs/settings-feature-preferences.spec.ts" "settings-features" "settings" + run "test/e2e/specs/settings-search.spec.ts" "settings-search" "settings" _mini_summary "settings" fi @@ -418,6 +432,7 @@ if should_run_suite "system"; then # PR #1061 (core is now in-process). Skip by not setting OPENHUMAN_SERVICE_MOCK=1. run "test/e2e/specs/service-connectivity-flow.spec.ts" "service-connectivity" "system" run "test/e2e/specs/core-port-conflict-recovery.spec.ts" "core-port-conflict" "system" + run "test/e2e/specs/ptt-flow.spec.ts" "ptt-flow" "system" if [[ "$(uname -s)" == "Linux" ]]; then run "test/e2e/specs/linux-cef-deb-runtime.spec.ts" "linux-cef-deb-runtime" "system" fi @@ -433,6 +448,7 @@ if should_run_suite "journeys"; then run "test/e2e/specs/user-journey-full-task.spec.ts" "journey-full-task" "journeys" run "test/e2e/specs/user-journey-settings-round-trip.spec.ts" "journey-settings" "journeys" run "test/e2e/specs/chat-conversation-history.spec.ts" "chat-history" "journeys" + run "test/e2e/specs/flows.spec.ts" "flows" "journeys" _mini_summary "journeys" fi diff --git a/scripts/generate-test-inventory.mjs b/scripts/generate-test-inventory.mjs index 7952d284ad..2b5c58c131 100644 --- a/scripts/generate-test-inventory.mjs +++ b/scripts/generate-test-inventory.mjs @@ -7,10 +7,23 @@ // (a) ORPHAN CHECK — every discovered script-level test file // (`scripts/**/*.test.mjs` and the PowerShell install test) is invoked // by >=1 package.json script (directly or via a `node --test `) -// OR referenced by a workflow. Framework-globbed suites (Vitest, WDIO, -// Playwright, cargo test) are discovered by their runners' own config -// globs, not enumerated here, so they are out of scope for the orphan -// check — the orphans the audit found all live under `scripts/`. +// OR referenced by a workflow. Vitest, Playwright and cargo test really +// are discovered by their runners' own config globs, so they stay out of +// scope here. +// +// WDIO IS NOT, and used to be exempted on that false premise. Its config +// glob (`wdio.conf.ts`: test/e2e/specs/**/*.spec.ts) is overridden the +// moment a caller passes spec paths, and the only path CI takes does +// exactly that: e2e-run-all-flows.sh collects a HAND-MAINTAINED list and +// e2e-run-session.sh turns it into `--spec` flags. A spec absent from +// that list is therefore run by nothing, while the config glob makes it +// look covered. Fourteen specs had drifted out this way before check (c) +// below existed. See check (c). +// +// (c) WDIO LANE CHECK — every `app/test/e2e/specs/*.spec.ts` is named by an +// active `run "..."` line in `app/scripts/e2e-run-all-flows.sh`, the only +// orchestrator CI uses. Catches a spec that exists, typechecks and is +// never executed. // // (b) CONTROLLER-DOMAIN CHECK — every controller domain registered in // `crates/openhuman-core/src/core/all.rs` (via `crate::::all_*_controllers`) @@ -44,6 +57,17 @@ const JSON_OUT = argv.has('--json'); // instead of allowlisting it. const ORPHAN_ALLOWLIST = new Set([]); +// WDIO specs permitted to be absent from `e2e-run-all-flows.sh`. An entry is a +// deliberate, reviewable disable WITH a cause — not a parking space for a spec +// someone forgot to wire up. Delete the entry when the spec goes back in. +const WDIO_LANE_ALLOWLIST = new Map([ + [ + 'slack-flow.spec.ts', + 'Crashes the CEF session mid-spec on Linux (#1850-style state issue); its ' + + '`run` line is commented out in e2e-run-all-flows.sh with the same cause.', + ], +]); + // Controller domains permitted to lack any reference under tests/. Each entry // is a Rust integration-coverage gap tracked in plan.md §4/§A.3 — remove the // entry when the domain gains a tests/ reference. @@ -207,6 +231,55 @@ function computeUnreferencedDomains(domains) { return missing; } +// ───────────────────────────────────────────────────────────────────────────── +// (c) WDIO lane check +// ───────────────────────────────────────────────────────────────────────────── + +const WDIO_SPEC_DIR = path.join(ROOT, 'app', 'test', 'e2e', 'specs'); +const WDIO_ORCHESTRATOR = path.join(ROOT, 'app', 'scripts', 'e2e-run-all-flows.sh'); + +function discoverWdioSpecs() { + if (!fs.existsSync(WDIO_SPEC_DIR)) return []; + return fs + .readdirSync(WDIO_SPEC_DIR) + .filter((f) => f.endsWith('.spec.ts')) + .sort(); +} + +/// Spec basenames named by an ACTIVE `run "..."` line. +/// +/// Anchored at line start so a commented-out `# run "..."` does not count — a +/// disabled spec is exactly the case this check exists to surface, and matching +/// the comment would make the guard agree with the bug. +function specsNamedByOrchestrator() { + if (!fs.existsSync(WDIO_ORCHESTRATOR)) return new Set(); + const named = new Set(); + const re = /^[ \t]*run[ \t]+"test\/e2e\/specs\/([^"]+)"/gm; + for (const m of read(WDIO_ORCHESTRATOR).matchAll(re)) named.add(m[1]); + return named; +} + +function computeUnrunWdioSpecs() { + const specs = discoverWdioSpecs(); + const named = specsNamedByOrchestrator(); + // Guard the guard: if the orchestrator parse yields nothing while specs do + // exist, the regex has drifted from the script's format and every spec would + // be reported as unrun. That is a tooling failure, not a coverage finding, + // and must not be reported as one. + if (specs.length > 0 && named.size === 0) { + throw new Error( + `WDIO lane check parsed 0 \`run\` lines from ${path.relative(ROOT, WDIO_ORCHESTRATOR)} ` + + `while ${specs.length} spec files exist. The matcher has drifted from the script's ` + + `format — fix the regex rather than treating this as missing coverage.`, + ); + } + return { + specs, + named, + unrun: specs.filter((f) => !named.has(f) && !WDIO_LANE_ALLOWLIST.has(f)), + }; +} + // ───────────────────────────────────────────────────────────────────────────── // Run // ───────────────────────────────────────────────────────────────────────────── @@ -216,6 +289,7 @@ const orphans = computeOrphans(scriptTests); const domains = discoverControllerDomains(); const unreferencedDomains = computeUnreferencedDomains(domains); +const wdio = computeUnrunWdioSpecs(); const referencedDomainCount = domains.length - unreferencedDomains.length - DOMAIN_ALLOWLIST.size; if (JSON_OUT) { @@ -228,6 +302,10 @@ if (JSON_OUT) { domains, unreferencedDomains, domainAllowlist: [...DOMAIN_ALLOWLIST], + wdioSpecs: wdio.specs, + wdioSpecsNamedByOrchestrator: [...wdio.named].sort(), + wdioSpecsUnrun: wdio.unrun, + wdioLaneAllowlist: [...WDIO_LANE_ALLOWLIST.keys()], }, null, 2, @@ -243,6 +321,10 @@ if (JSON_OUT) { console.log(` referenced in tests/: ${referencedDomainCount}`); console.log(` allowlisted (known gaps): ${DOMAIN_ALLOWLIST.size}`); console.log(` newly unreferenced: ${unreferencedDomains.length}`); + console.log(`WDIO specs on disk: ${wdio.specs.length}`); + console.log(` named by e2e-run-all-flows.sh: ${wdio.named.size}`); + console.log(` allowlisted (deliberate): ${WDIO_LANE_ALLOWLIST.size}`); + console.log(` run by no lane: ${wdio.unrun.length}`); } let failed = false; @@ -254,6 +336,18 @@ if (orphans.length > 0) { console.error(' Wire each into `test:scripts` (or a dedicated script), or allowlist with cause.'); } +if (wdio.unrun.length > 0) { + failed = true; + console.error( + '\n\u2716 WDIO specs that exist but are run by no lane (absent from app/scripts/e2e-run-all-flows.sh):', + ); + for (const file of wdio.unrun) console.error(` - app/test/e2e/specs/${file}`); + console.error( + ' Add a `run "test/e2e/specs/" "