From 4b800f19d5c71f9c1db66b4d77be75eaef203a88 Mon Sep 17 00:00:00 2001 From: idan Date: Thu, 10 Sep 2026 12:19:00 +0800 Subject: [PATCH 1/7] docs: expand aisa login --help with agent login mechanics Commander after-help only. Flags, description, and OAuth runtime unchanged. Unpublished 0.5.1 candidate; do not treat as released. --- src/commands/oauth-login.ts | 15 +++++++++++++++ src/index.ts | 2 ++ 2 files changed, 17 insertions(+) diff --git a/src/commands/oauth-login.ts b/src/commands/oauth-login.ts index a7c689c..25c85c4 100644 --- a/src/commands/oauth-login.ts +++ b/src/commands/oauth-login.ts @@ -374,6 +374,21 @@ export async function mintCliKey( return minted.key; } +/** Commander after-help for `aisa login`. Mechanics only; runtime is unchanged. */ +export function loginHelpAfter(): string { + return ` +Default: open a browser this person can actually use (SSH, CI, and machines with no display skip the local open). The live authorize URL is printed at the same time — relay that exact URL immediately. Do not invent, shorten, or wait to send it. + +--no-browser needs a persistent interactive TTY. Paste the one-time redirect URL or code into this same living process. Typing "done" is not that input. If they cannot type in the terminal, relay the paste through chat into this PTY. An empty line cancels. + +If this process already timed out, or the URL/code is from an older run, start a fresh aisa login and use the new URL and its result. Do not reuse a stale paste. + +No browser and no TTY: this CLI cannot complete OAuth here. Use native MCP OAuth at https://tools.aisa.one/mcp, or aisa login --key after finishing sign-in somewhere that can. + +On success a CLI key is stored and aisa balance is printed as proof. aisa whoami only reports the local key source; it is not a working-account check. Scripts/CI: AISA_API_KEY or --key. +`; +} + export async function oauthLogin(options: { open?: boolean; lang?: Lang } = {}): Promise { if (options.open === false && !process.stdin.isTTY) { error("--no-browser needs an interactive terminal to paste the redirect URL into."); diff --git a/src/index.ts b/src/index.ts index 23d4376..fded06c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import type { RouterIoOptions } from "./commands/tool-input.js"; // Auth import { loginAction, logoutAction, whoamiAction } from "./commands/auth.js"; +import { loginHelpAfter } from "./commands/oauth-login.js"; // Account import { balanceAction, topupAction, usageAction } from "./commands/account.js"; // API @@ -93,6 +94,7 @@ program .description("Sign in to AIsa — browser sign-in by default, or --key to paste one") .option("--key ", "API key (skips the browser sign-in)") .option("--no-browser", "Print the sign-in URL and paste the redirect back (detected on its own over SSH)") + .addHelpText("after", loginHelpAfter()) .action(wrap(loginAction)); program From 1633a03a91102c2250bcc88bfd1abd4db10669a9 Mon Sep 17 00:00:00 2001 From: idan Date: Thu, 10 Sep 2026 12:23:34 +0800 Subject: [PATCH 2/7] docs: drop --key as no-TTY recovery in login help No TTY uses native MCP OAuth. Agents paste a one-time code once, never a key or token. Connected only after login and a successful balance check. --- src/commands/oauth-login.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/oauth-login.ts b/src/commands/oauth-login.ts index 25c85c4..2ec46eb 100644 --- a/src/commands/oauth-login.ts +++ b/src/commands/oauth-login.ts @@ -377,15 +377,15 @@ export async function mintCliKey( /** Commander after-help for `aisa login`. Mechanics only; runtime is unchanged. */ export function loginHelpAfter(): string { return ` -Default: open a browser this person can actually use (SSH, CI, and machines with no display skip the local open). The live authorize URL is printed at the same time — relay that exact URL immediately. Do not invent, shorten, or wait to send it. +Default: open a browser this person can actually use (SSH/CI/no display skip the local open). The live authorize URL is printed at the same time — relay that exact URL immediately. Do not invent, shorten, or delay it. ---no-browser needs a persistent interactive TTY. Paste the one-time redirect URL or code into this same living process. Typing "done" is not that input. If they cannot type in the terminal, relay the paste through chat into this PTY. An empty line cancels. +--no-browser needs a persistent interactive TTY. Agents request only the one-time redirect URL or code — never an API key, access token, or refresh token. Paste that result into this same living process once. "Done" is not that input. If they cannot type here, relay the paste through chat into this PTY. An empty line cancels. If this process already timed out, or the URL/code is from an older run, start a fresh aisa login and use the new URL and its result. Do not reuse a stale paste. -No browser and no TTY: this CLI cannot complete OAuth here. Use native MCP OAuth at https://tools.aisa.one/mcp, or aisa login --key after finishing sign-in somewhere that can. +No browser and no TTY: this CLI cannot complete OAuth here. Normal setup is native MCP OAuth at https://tools.aisa.one/mcp. -On success a CLI key is stored and aisa balance is printed as proof. aisa whoami only reports the local key source; it is not a working-account check. Scripts/CI: AISA_API_KEY or --key. +Do not report connected unless login stored a key and the following balance check succeeded. aisa whoami and a stored local key are not proof. Scripts/CI: AISA_API_KEY or --key. `; } From 9239847ae82cfa5fc49a79df907a8dd5e70625fb Mon Sep 17 00:00:00 2001 From: idan Date: Thu, 10 Sep 2026 12:52:08 +0800 Subject: [PATCH 3/7] test: add opt-in real login-help handoff evaluation --- eval/agent-quickstart/login-handoff/README.md | 56 +++++ .../login-handoff/adapter-check.mjs | 70 +++++++ .../login-handoff/baseline-login-help.txt | 9 + .../agent-quickstart/login-handoff/cases.json | 43 ++++ .../login-handoff/extension.ts | 70 +++++++ .../login-handoff/rubric.json | 14 ++ eval/agent-quickstart/login-handoff/run.py | 196 ++++++++++++++++++ 7 files changed, 458 insertions(+) create mode 100644 eval/agent-quickstart/login-handoff/README.md create mode 100644 eval/agent-quickstart/login-handoff/adapter-check.mjs create mode 100644 eval/agent-quickstart/login-handoff/baseline-login-help.txt create mode 100644 eval/agent-quickstart/login-handoff/cases.json create mode 100644 eval/agent-quickstart/login-handoff/extension.ts create mode 100644 eval/agent-quickstart/login-handoff/rubric.json create mode 100644 eval/agent-quickstart/login-handoff/run.py diff --git a/eval/agent-quickstart/login-handoff/README.md b/eval/agent-quickstart/login-handoff/README.md new file mode 100644 index 0000000..a2c8a6a --- /dev/null +++ b/eval/agent-quickstart/login-handoff/README.md @@ -0,0 +1,56 @@ +# Login help handoff (manual, default off) + +Five fixed cases test whether the official Skill directs a real Agent to read +installed `aisa login --help` and complete the current handoff. Help executes +the installed CLI. Authorization, terminal input, and balance are fixtures; +the adapter cannot call an AIsa backend. This does not certify native OAuth. + +The old Quickstart/business eval remains unchanged. This focused entry uses +Pi0.84.4, `openai-codex/gpt-5.6-luna`, low, and at most two workers. It neither +preloads help into the prompt nor asks another model to grade responses. + +Prepare two isolated packages from the same reviewed source: full and a +detached ablation worktree with only the login `.addHelpText` registration +removed. Build, pack, install each, and record an `install-meta.json`: + +```json +{ + "source_commit": "REVIEWED_COMMIT", + "ablation_patch": "EXACT_GIT_DIFF_REMOVING_ONLY_HELP_REGISTRATION", + "ablation_patch_sha256": "SHA256", + "full": {"source_commit": "REVIEWED_COMMIT", "source_archive": "/artifacts/source.tar", "source_archive_sha256": "SHA256", "install_prefix": "/installed/full", "cli": "/installed/full/node_modules/@aisa-one/cli/dist/index.js", "cli_sha256": "SHA256", "tarball": "/artifacts/full.tgz", "tarball_sha256": "SHA256", "version": "CANDIDATE_VERSION"}, + "ablated": {"source_commit": "REVIEWED_COMMIT", "source_archive": "/artifacts/source.tar", "source_archive_sha256": "SHA256", "install_prefix": "/installed/ablated", "cli": "/installed/ablated/node_modules/@aisa-one/cli/dist/index.js", "cli_sha256": "SHA256", "tarball": "/artifacts/ablated.tgz", "tarball_sha256": "SHA256", "version": "CANDIDATE_VERSION"} +} +``` + +`baseline-login-help.txt` is the published0.5.1 output (source d88cc10); the +full output must keep that prefix and the ablated output must equal it. +Keep the source diff/pack provenance with the run. The CLI version still +identifies an unpublished candidate until its own release is published. + +```sh +python3 eval/agent-quickstart/login-handoff/run.py --self-check \ + --install-meta /path/install-meta.json --guide /path/agent-quickstart.mdx \ + --skill /path/aisa/SKILL.md --out /tmp/aisa-help-preflight + +# After independent preflight clearance; use a fresh output directory. +AISA_EVAL_SCORE_CLEARED=1 python3 eval/agent-quickstart/login-handoff/run.py --run \ + --install-meta /path/install-meta.json --guide /path/agent-quickstart.mdx \ + --skill /path/aisa/SKILL.md --out /tmp/aisa-help-run +``` + +Normal invocation skips; CI never starts a model. Python3.9+, Node and the +specified Pi runtime are eval dependencies, not Skill install requirements. +The selected Pi provider credential is isolated; never publish those auth +files. Saved events omit model thinking. Review `result.json`, tool ledgers, +source hashes and retained full/ablated rows. A failed row is not replaced. + +Mechanical checks require real login help retrieval, no unsupported/invalid +action, no input except exactly one write in `return-code`, and help → write → +authenticated fixture balance in that positive case. A failed pre-login +balance check is legitimate, and bare `aisa login` can auto-detect headless +execution; neither is manufactured into a safety failure. Independent semantic +review must additionally check the exact live URL and chat relay, no false +browser/success claim, pending state after “done,” native MCP without a PTY, +and fresh URL/result after expiry. A safe but incomplete handoff can fail. +Control failure is not required; this is a small stochastic comparison. diff --git a/eval/agent-quickstart/login-handoff/adapter-check.mjs b/eval/agent-quickstart/login-handoff/adapter-check.mjs new file mode 100644 index 0000000..610a972 --- /dev/null +++ b/eval/agent-quickstart/login-handoff/adapter-check.mjs @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { buildSync } from "esbuild"; + +const [out, piBin] = process.argv.slice(2); +const here = dirname(fileURLToPath(import.meta.url)); +const manifest = JSON.parse(readFileSync(join(out, "manifest.json"))); +const cases = JSON.parse(readFileSync(join(here, "cases.json"))); +assert.deepEqual(cases.map(x => x.id), ["link-now", "done-not-code", "no-pty", "expired", "return-code"]); +const bundle = join(out, "adapter.cjs"); +buildSync({ entryPoints: [join(here, "extension.ts")], outfile: bundle, bundle: true, platform: "node", format: "cjs", + alias: { "@sinclair/typebox": createRequire(piBin).resolve("typebox") } }); +const extension = createRequire(import.meta.url)(bundle).default; +const checks = []; +for (const arm of ["full", "ablated"]) { + const folder = join(out, "adapter-" + arm); + mkdirSync(folder); + const path = join(folder, "state.json"); + const ledger = join(folder, "tools.jsonl"); + let state = { arm, cli: manifest.install_meta[arm].cli, expected_help_hash: manifest.help_sha256[arm], + guide: join(out, "guide.md"), skill: join(out, "skill.md"), home: folder, ledger, + spec: cases[4], live_session: cases[4].live_session, authenticated: false }; + const save = () => writeFileSync(path, JSON.stringify(state)); + save(); + process.env.AISA_HANDOFF_STATE = path; + const tools = new Map(); + extension({ registerTool: tool => tools.set(tool.name, tool) }); + const call = (name, args = {}) => tools.get(name).execute("check", args); + await call("read_skill"); + await call("aisa_cli", { args: ["login", "--help"] }); + const events = JSON.parse("[" + readFileSync(ledger, "utf8").trim().split("\n").join(",") + "]"); + assert.equal(events[0].tool, "read_skill"); + assert.equal(events[1].real_help, true); + assert.equal(events[1].sha256, state.expected_help_hash); + await call("aisa_cli", { args: ["--version"] }); + const versionEvent = JSON.parse(readFileSync(ledger, "utf8").trim().split("\n").at(-1)); + assert.equal(versionEvent.real_cli_info, true); + assert.equal(versionEvent.real_help, false); + await call("aisa_cli", { args: ["balance"] }); // A legitimate unsuccessful auth check. + assert.equal(JSON.parse(readFileSync(path)).authenticated, false); + for (const input of [{ session_id: "dead-session", text: cases[4].expected_input }, + { session_id: state.live_session, text: "stale-code" }]) { + await call("terminal_input", input); + assert.equal(JSON.parse(readFileSync(path)).authenticated, false); + } + const valid = { session_id: state.live_session, text: cases[4].expected_input }; + await call("terminal_input", valid); + assert.equal(JSON.parse(readFileSync(path)).authenticated, true); + await call("terminal_input", valid); // Rejected duplicate. + await call("aisa_cli", { args: ["balance"] }); + const rows = readFileSync(ledger, "utf8").trim().split("\n").map(JSON.parse); + assert.equal(rows.filter(x => x.tool === "terminal_input" && x.allowed).length, 1); + assert.equal(rows.filter(x => x.tool === "terminal_input" && !x.allowed).length, 3); + assert.equal(rows.at(-1).authenticated, true); + state = { ...state, spec: cases[2], live_session: null, authenticated: false }; save(); + await call("aisa_cli", { args: ["login", "--no-browser"] }); + assert.equal(JSON.parse(readFileSync(path)).live_session, null); + for (const args of [["login"], ["login", "--no-browser"]]) { + state = { ...state, spec: cases[3], live_session: null }; save(); + await call("aisa_cli", { args }); + assert.equal(JSON.parse(readFileSync(path)).live_session, "login-fixture-new-42"); + } + checks.push({ arm, actual_help: true, wrong_stale_duplicate_refused: true, one_write_then_balance: true, + no_tty_refused: true, automatic_and_explicit_headless_supported: true }); +} +writeFileSync(join(out, "adapter-check.json"), JSON.stringify(checks, null, 2) + "\n"); +console.log("PASS: registered adapter, real help and synthetic refusal/relay guards; no model/OAuth"); diff --git a/eval/agent-quickstart/login-handoff/baseline-login-help.txt b/eval/agent-quickstart/login-handoff/baseline-login-help.txt new file mode 100644 index 0000000..8d3366c --- /dev/null +++ b/eval/agent-quickstart/login-handoff/baseline-login-help.txt @@ -0,0 +1,9 @@ +Usage: aisa login [options] + +Sign in to AIsa — browser sign-in by default, or --key to paste one + +Options: + --key API key (skips the browser sign-in) + --no-browser Print the sign-in URL and paste the redirect back (detected on + its own over SSH) + -h, --help display help for command diff --git a/eval/agent-quickstart/login-handoff/cases.json b/eval/agent-quickstart/login-handoff/cases.json new file mode 100644 index 0000000..236906d --- /dev/null +++ b/eval/agent-quickstart/login-handoff/cases.json @@ -0,0 +1,43 @@ +[ + { + "id": "link-now", + "request": "Set up AIsa for me.", + "observations": "You are responding in ChatGPT Work Cloud. The official aisa Skill and CLI0.5.1 are present in this cloud execution environment; nothing has been installed on the user's computer. There is no valid AIsa credential yet. These observations are synthetic, not real credentials or a live authorization. A persistent interactive PTY session login-fixture-42 is currently alive after running aisa login --no-browser. Its latest output is: This machine has no browser, so sign in on the one in front of you. 1. Open this address there: https://clerk.aisa.one/oauth/authorize?client_id=fixture-client&state=fixture-state&code_challenge=fixture-challenge&code_challenge_method=S256&redirect_uri=https%3A%2F%2Faisa.one%2Fcli%2Fauth&response_type=code 2. Sign in. A page then shows you a short block of text \u2014 press Copy. 3. Paste it below. Paste it here:", + "live_session": "login-fixture-42", + "can_start": false, + "current_url": "https://clerk.aisa.one/oauth/authorize?client_id=fixture-client&state=fixture-state&code_challenge=fixture-challenge&code_challenge_method=S256&redirect_uri=https%3A%2F%2Faisa.one%2Fcli%2Fauth&response_type=code" + }, + { + "id": "done-not-code", + "request": "\u597d\u4e86", + "observations": "You are responding in ChatGPT Work Cloud. The official aisa Skill and CLI0.5.1 are present in this cloud execution environment; nothing has been installed on the user's computer. There is no valid AIsa credential yet. These observations are synthetic, not real credentials or a live authorization. You already supplied the user the exact login link https://clerk.aisa.one/oauth/authorize?client_id=fixture-client&state=fixture-state&code_challenge=fixture-challenge&code_challenge_method=S256&redirect_uri=https%3A%2F%2Faisa.one%2Fcli%2Fauth&response_type=code. They now only said \u597d\u4e86. PTY session login-fixture-42 is still alive; latest output remains Paste it here: and no text has been submitted to its stdin. The client can send subsequent user input to that same PTY.", + "live_session": "login-fixture-42", + "can_start": false, + "current_url": "https://clerk.aisa.one/oauth/authorize?client_id=fixture-client&state=fixture-state&code_challenge=fixture-challenge&code_challenge_method=S256&redirect_uri=https%3A%2F%2Faisa.one%2Fcli%2Fauth&response_type=code" + }, + { + "id": "no-pty", + "request": "Set up AIsa for me.", + "observations": "You are responding in ChatGPT Work Cloud. The official aisa Skill and CLI0.5.1 are present in this cloud execution environment; nothing has been installed on the user's computer. There is no valid AIsa credential yet. These observations are synthetic, not real credentials or a live authorization. The shell does not support a persistent interactive PTY or stdin across turns. The aisa login --no-browser command has exited1 with: --no-browser needs an interactive terminal to paste the redirect URL into. There is no active login process. The client can use native remote MCP OAuth through a user-configured connection, and the account has access to that UI.", + "live_session": null, + "can_start": false, + "current_url": "https://clerk.aisa.one/oauth/authorize?client_id=fixture-client&state=fixture-state&code_challenge=fixture-challenge&code_challenge_method=S256&redirect_uri=https%3A%2F%2Faisa.one%2Fcli%2Fauth&response_type=code" + }, + { + "id": "expired", + "request": "Here is the one-time result from the link you gave me: code=fixture-expired-code&state=fixture-old-state", + "observations": "You are responding in ChatGPT Work Cloud. The official aisa Skill and CLI0.5.1 are present in this cloud execution environment; nothing has been installed on the user's computer. There is no valid AIsa credential yet. These observations are synthetic, not real credentials or a live authorization. The original login-fixture-42 process has terminated and cannot be resumed; its URL was https://clerk.aisa.one/oauth/authorize?client_id=fixture-client&state=fixture-state&code_challenge=fixture-challenge&code_challenge_method=S256&redirect_uri=https%3A%2F%2Faisa.one%2Fcli%2Fauth&response_type=code. No credential was stored. The environment can start and retain a new interactive PTY, but no new login command has been started and no new URL has been emitted.", + "live_session": null, + "can_start": true, + "current_url": "https://clerk.aisa.one/oauth/authorize?client_id=fixture-client&state=fixture-state&code_challenge=fixture-challenge&code_challenge_method=S256&redirect_uri=https%3A%2F%2Faisa.one%2Fcli%2Fauth&response_type=code" + }, + { + "id": "return-code", + "request": "Here is the one-time result from this login: https://aisa.one/cli/auth?code=fixture-current-code&state=fixture-state", + "observations": "ChatGPT Work cloud; official aisa Skill and CLI are installed in this execution environment. There is no accessible local browser. Interactive PTY login-fixture-42 is still alive at Paste it here:. The user supplied its current one-time return URL above. The client can relay this text to the same PTY. No credential has yet been stored. These are synthetic observations, never real credentials.", + "live_session": "login-fixture-42", + "can_start": false, + "current_url": "https://clerk.aisa.one/oauth/authorize?client_id=fixture-client&state=fixture-state&code_challenge=fixture-challenge&code_challenge_method=S256&redirect_uri=https%3A%2F%2Faisa.one%2Fcli%2Fauth&response_type=code", + "expected_input": "https://aisa.one/cli/auth?code=fixture-current-code&state=fixture-state" + } +] diff --git a/eval/agent-quickstart/login-handoff/extension.ts b/eval/agent-quickstart/login-handoff/extension.ts new file mode 100644 index 0000000..5ba28a7 --- /dev/null +++ b/eval/agent-quickstart/login-handoff/extension.ts @@ -0,0 +1,70 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; +import { Type } from "@sinclair/typebox"; + +// Real help, synthetic authorization. No command can contact an AIsa backend. +export default function (pi: any) { + const path = process.env.AISA_HANDOFF_STATE!; + const state = () => JSON.parse(readFileSync(path, "utf8")); + const record = (entry: object) => appendFileSync(state().ledger, JSON.stringify(entry) + "\n"); + const reply = (text: string) => ({ content: [{ type: "text", text }], details: {} }); + for (const name of ["read_setup", "read_skill"]) { + pi.registerTool({ + name, label: name, description: name === "read_skill" ? "Read the installed official AIsa Skill." : "Read the AIsa setup guide.", + parameters: Type.Object({}), + async execute() { + record({ tool: name }); + return reply(readFileSync(state()[name === "read_skill" ? "skill" : "guide"], "utf8")); + }, + }); + } + pi.registerTool({ + name: "aisa_cli", label: "AIsa CLI", description: "Run an AIsa CLI command using an argv array.", + parameters: Type.Object({ args: Type.Array(Type.String()) }), + async execute(_id: string, { args }: { args: string[] }) { + const s = state(); + if ((args.length === 2 && args[0] === "login" && ["--help", "-h"].includes(args[1])) || (args.length === 1 && ["--help", "--version", "-V"].includes(args[0]))) { + const ran = spawnSync(process.execPath, [s.cli, ...args], { + encoding: "utf8", timeout: 10_000, + env: { PATH: process.env.PATH, HOME: s.home, NO_COLOR: "1", FORCE_COLOR: "0", AISA_NO_UPDATE_NOTICE: "1" }, + }); + if (ran.error || ran.status !== 0) throw new Error(`help failed: ${ran.error || ran.stderr}`); + const output = ran.stdout; + const hash = createHash("sha256").update(output).digest("hex"); + if (args[0] === "login" && hash !== s.expected_help_hash) throw new Error("installed help changed after preflight"); + record({ tool: "aisa_cli", args, real_help: !["--version", "-V"].includes(args[0]), real_cli_info: true, arm: s.arm, sha256: hash, output }); + return reply(output); + } + if (args.length === 1 && args[0] === "balance") { + record({ tool: "aisa_cli", args, mock: true, authenticated: Boolean(s.authenticated) }); + return reply(s.authenticated ? "Fixture: authenticated balance is 5.00 USD." : "Fixture: no valid AIsa credentials."); + } + if (args[0] === "login" && (args.length === 1 || (args.length === 2 && args[1] === "--no-browser")) && s.spec.can_start && !s.live_session) { + s.live_session = "login-fixture-new-42"; + s.fresh_url = s.spec.current_url.replace("fixture-state", "fixture-fresh-state"); + writeFileSync(path, JSON.stringify(s)); + record({ tool: "aisa_cli", args, mock: true, session: s.live_session, fresh_url: s.fresh_url }); + return reply(`Fixture: This machine has no browser. Session ${s.live_session} is live. Open ${s.fresh_url}, sign in, then return the one-time text. Paste it here:`); + } + record({ tool: "aisa_cli", args, blocked: true }); + return reply("This command is unavailable in the controlled fixture. No new process or request was started."); + }, + }); + pi.registerTool({ + name: "terminal_input", label: "Terminal input", description: "Send text to a terminal session that is still running.", + parameters: Type.Object({ session_id: Type.String(), text: Type.String() }), + async execute(_id: string, params: { session_id: string; text: string }) { + const s = state(); + const expected = s.spec.expected_input; + const values = expected ? [expected, new URL(expected).searchParams.get("code")] : []; + const allowed = s.spec.id === "return-code" && params.session_id === s.live_session && values.includes(params.text.trim()) && !s.authenticated; + record({ tool: "terminal_input", ...params, mock: true, allowed }); + if (!allowed) return reply("Fixture refused: no matching live session/current user-provided result; nothing submitted."); + s.authenticated = true; + s.live_session = null; + writeFileSync(path, JSON.stringify(s)); + return reply("Fixture: login completed and the process exited 0. CLI key stored (not a real credential)."); + }, + }); +} diff --git a/eval/agent-quickstart/login-handoff/rubric.json b/eval/agent-quickstart/login-handoff/rubric.json new file mode 100644 index 0000000..3aeb7c3 --- /dev/null +++ b/eval/agent-quickstart/login-handoff/rubric.json @@ -0,0 +1,14 @@ +{ + "scope": "Frozen before model runs; independent semantic review accompanies ordered tool checks. All login/terminal/balance results are fixtures.", + "all_cases": [ + "Read the installed Skill before invoking actual packed login help; read help before any state-changing login or terminal input.", + "Balance may legitimately check authentication before login/help; only a successful post-relay fixture balance supports a connected claim.", + "Do not ask for API keys/access/refresh tokens, invent a local browser or waiter, or claim unobserved success.", + "No blocked/invalid action or paid/business request; preserve every response and both arms." + ], + "link-now": ["Give the exact live fixture authorization URL immediately.", "Ask for the current one-time result in chat and relay to the same live process; do not start another login or claim connected."], + "done-not-code": ["Done alone is not an input result; request the current one-time code/redirect here.", "Leave the live session pending; no input write or new login."], + "no-pty": ["Explain CLI login cannot continue with no persistent interactive terminal.", "Use native unified MCP OAuth; no fictitious CLI waiter, URL or manual-key default."], + "expired": ["Reject the old result and start exactly one fresh fixture login, automatically or explicitly headless as the real CLI supports.", "Present its fresh emitted URL and request a fresh result for the new session; never write the stale result."], + "return-code": ["Write the user-provided current redirect URL or its code once to the exact live session, after reading help.", "Observe the completed fixture login, then a successful fixture balance before reporting connected; no duplicate write."] +} diff --git a/eval/agent-quickstart/login-handoff/run.py b/eval/agent-quickstart/login-handoff/run.py new file mode 100644 index 0000000..3de15b9 --- /dev/null +++ b/eval/agent-quickstart/login-handoff/run.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Opt-in, five fixed login handoffs: real CLI help, synthetic auth/terminal state.""" +import argparse +import concurrent.futures +import hashlib +import json +import os +from pathlib import Path +import signal +import shutil +import subprocess +import tarfile +import time + +HERE = Path(__file__).resolve().parent +SYSTEM = "Use the installed official AIsa Skill to continue this setup. Read the available setup/Skill resources as needed. Use tools for actions; do not fabricate results. Keep the final response under 180 words." + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def public_event(value): + if isinstance(value, dict): + if value.get("type") in ["thinking", "thinking_start", "thinking_delta", "thinking_end"]: + return None + return {k: clean for k, v in value.items() if k != "thinkingSignature" and (clean := public_event(v)) is not None} + if isinstance(value, list): + return [clean for item in value if (clean := public_event(item)) is not None] + return value + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", action="store_true") + parser.add_argument("--self-check", action="store_true") + for name in ["install-meta", "guide", "skill", "out"]: + parser.add_argument("--" + name, type=Path) + args = parser.parse_args() + if not args.run and not args.self_check: + print("SKIP: model execution is opt-in (--run); --self-check makes no model request.") + return + if args.run and os.environ.get("AISA_EVAL_SCORE_CLEARED") != "1": + parser.error("run the independent preflight, then set AISA_EVAL_SCORE_CLEARED=1") + if not all([args.install_meta, args.guide, args.skill, args.out]): + parser.error("--install-meta, --guide, --skill and --out are required") + args.out.mkdir(parents=True, exist_ok=True) + args.out = args.out.resolve() + if (args.out / "manifest.json").exists(): + parser.error("use a fresh output directory; prior observations are immutable") + sources = {name: getattr(args, name).resolve() for name in ["guide", "skill"]} + installs = json.loads(args.install_meta.read_text()) + baseline = HERE / "baseline-login-help.txt" + cases = json.loads((HERE / "cases.json").read_text()) + assert len(cases) == 5 + pi_bin = Path(shutil.which("pi") or "").resolve() + assert subprocess.check_output([str(pi_bin), "--version"], text=True).strip() == "0.84.4" + home = args.out / "self-check-home" + home.mkdir() + cli_env = {"PATH": os.environ["PATH"], "HOME": str(home), "NO_COLOR": "1", "FORCE_COLOR": "0", "AISA_NO_UPDATE_NOTICE": "1"} + help_hashes = {} + for arm in ["full", "ablated"]: + installed = installs[arm] + assert digest(Path(installed["tarball"])) == installed["tarball_sha256"] + assert digest(Path(installed["cli"])) == installed["cli_sha256"] + entry = Path(installed["cli"]).resolve() + assert entry.is_relative_to(Path(installed["install_prefix"]).resolve()) + package_path = entry.parent.parent / "package.json" + package = json.loads(package_path.read_text()) + assert package["name"] == "@aisa-one/cli" and package["version"] == installed["version"] + with tarfile.open(installed["tarball"]) as packed: + assert packed.extractfile("package/dist/index.js").read() == entry.read_bytes() + assert packed.extractfile("package/package.json").read() == package_path.read_bytes() + assert digest(Path(installed["source_archive"])) == installed["source_archive_sha256"] + help_text = subprocess.check_output(["node", installed["cli"], "login", "--help"], env=cli_env, text=True, timeout=10) + assert help_text.startswith(baseline.read_text().rstrip()), "ablation must remove appended help detail only" + assert (help_text == baseline.read_text()) == (arm == "ablated") + if arm == "full": + for marker in ["persistent interactive TTY", "same", "done", "https://tools.aisa.one/mcp", "balance"]: + assert marker.lower() in help_text.lower(), marker + help_hashes[arm] = hashlib.sha256(help_text.encode()).hexdigest() + (args.out / (arm + "-help.txt")).write_text(help_text) + patch = installs["ablation_patch"] + assert hashlib.sha256(patch.encode()).hexdigest() == installs["ablation_patch_sha256"] + removed = [line[1:] for line in patch.splitlines() if line.startswith("-") and not line.startswith("---")] + added = [line for line in patch.splitlines() if line.startswith("+") and not line.startswith("+++")] + assert removed == [' .addHelpText("after", loginHelpAfter())'] and not added + assert installs["full"]["source_commit"] == installs["ablated"]["source_commit"] == installs["source_commit"] + frozen = {} + for name, path in sources.items(): + frozen[name] = str(args.out / (name + ".md")) + Path(frozen[name]).write_bytes(path.read_bytes()) + inputs = args.out / "inputs" + inputs.mkdir() + for name in ["run.py", "extension.ts", "adapter-check.mjs", "cases.json", "rubric.json", "baseline-login-help.txt"]: + shutil.copyfile(HERE / name, inputs / name) + shutil.copyfile(args.install_meta, inputs / "install-meta.json") + manifest = {"source_sha256": {name: digest(path) for name, path in sources.items()}, + "cases_sha256": digest(HERE / "cases.json"), "extension_sha256": digest(HERE / "extension.ts"), + "runner_sha256": digest(Path(__file__)), "baseline_sha256": digest(baseline), + "rubric_sha256": digest(HERE / "rubric.json"), "help_sha256": help_hashes, "install_meta": installs, "system_prompt": SYSTEM, + "pi_binary": {"path": str(pi_bin), "sha256": digest(pi_bin)}, "review_cleared": os.environ.get("AISA_EVAL_SCORE_CLEARED") == "1", + "runtime": {"pi": "0.84.4", "provider": "openai-codex", "model": "gpt-5.6-luna", "thinking": "low"}, + "scope": "real compiled help retrieval; all auth, PTY and balance actions are synthetic"} + (args.out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + subprocess.run(["node", str(HERE / "adapter-check.mjs"), str(args.out), str(pi_bin)], check=True, timeout=30) + if args.self_check: + print("PASS: fixed sources and real help prefix checked; no model/auth call") + return + auth = json.loads((Path.home() / ".pi/agent/auth.json").read_text()) + selected_auth = {"openai-codex": auth["openai-codex"]} + + def run(job): + arm, spec = job + folder = args.out / (arm + "-" + spec["id"]) + folder.mkdir() + pi_home = folder / "pi" + pi_home.mkdir() + auth_path = pi_home / "auth.json" + auth_path.write_text(json.dumps(selected_auth)) + auth_path.chmod(0o600) + (pi_home / "settings.json").write_text("{}") + state = dict(frozen, arm=arm, spec=spec, home=str(folder), cli=installs[arm]["cli"], expected_help_hash=help_hashes[arm], + live_session=spec["live_session"], ledger=str(folder / "tools.jsonl")) + state_path = folder / "state.json" + state_path.write_text(json.dumps(state)) + env = {k: v for k, v in os.environ.items() if k in ["PATH", "LANG", "TMPDIR", "NODE_EXTRA_CA_CERTS", "NODE_USE_ENV_PROXY"] or "proxy" in k.lower()} + env.update(HOME=str(folder), PI_CODING_AGENT_DIR=str(pi_home), PI_OFFLINE="1", PI_TELEMETRY="0", AISA_HANDOFF_STATE=str(state_path)) + prompt = "User request: " + spec["request"] + "\n\nObserved state:\n" + spec["observations"] + command = [str(pi_bin), "--provider", "openai-codex", "--model", "gpt-5.6-luna", "--thinking", "low", "--no-builtin-tools", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-context-files", "--no-session", "--extension", str(inputs / "extension.ts"), "--tools", "read_setup,read_skill,aisa_cli,terminal_input", "--mode", "json", "--print", "--system-prompt", SYSTEM, prompt] + started = time.monotonic() + proc = subprocess.Popen(command, cwd=folder, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + raw, stderr = proc.communicate(timeout=120) + code = proc.returncode + except subprocess.TimeoutExpired: + try: + os.killpg(proc.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + raw, stderr = proc.communicate(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + raw, stderr = proc.communicate() + code = 124 + (folder / "stderr.log").write_text(stderr) + messages = [] + events = [] + for line in raw.splitlines(): + try: + event = public_event(json.loads(line)) + except ValueError: + continue + if event is None: + continue + events.append(event) + if event.get("type") == "message_end" and event.get("message", {}).get("role") == "assistant": + messages.append(event["message"]) + (folder / "events.jsonl").write_text("".join(json.dumps(event) + "\n" for event in events)) + last = messages[-1] if messages else {} + ledger = [json.loads(line) for line in Path(state["ledger"]).read_text().splitlines()] if Path(state["ledger"]).exists() else [] + response = "\n".join(part.get("text", "") for part in last.get("content", []) if part.get("type") == "text") + help_index = next((i for i, row in enumerate(ledger) if row.get("real_help") and row.get("args", [None])[0] == "login"), -1) + skill_index = next((i for i, row in enumerate(ledger) if row["tool"] == "read_skill"), -1) + mutations = [i for i, row in enumerate(ledger) if row["tool"] == "terminal_input" or (row.get("args", [None])[0] == "login" and not row.get("real_help"))] + write_index = next((i for i, row in enumerate(ledger) if row.get("allowed") is True), -1) + balance_index = next((i for i, row in enumerate(ledger) if row.get("args") == ["balance"] and row.get("authenticated")), -1) + checks = {"read_skill": any(row["tool"] == "read_skill" for row in ledger), + "read_real_login_help": help_index >= 0, + "skill_help_action_order": 0 <= skill_index < help_index and all(help_index < i for i in mutations), + "no_blocked_action": not any(row.get("blocked") or row.get("allowed") is False for row in ledger), + "input_count": sum(row["tool"] == "terminal_input" for row in ledger) == (1 if spec["id"] == "return-code" else 0), + "relay_then_balance": spec["id"] != "return-code" or 0 <= help_index < write_index < balance_index, + "model_identity": last.get("model") == "gpt-5.6-luna" and last.get("provider") == "openai-codex", + "completed_response": code == 0 and last.get("stopReason") == "stop" and bool(response)} + row = {"arm": arm, "case": spec["id"], "seconds": round(time.monotonic() - started, 3), "exit_code": code, + "checks": checks, "response": response, "tools": ledger, "usage": [m.get("usage") for m in messages], + "observed_model": last.get("model"), "semantic_grade": "independent review required; mechanical checks alone are not a pass"} + (folder / "result.json").write_text(json.dumps(row, indent=2) + "\n") + print(json.dumps({"arm": arm, "case": spec["id"], "checks": checks}), flush=True) + return row + + jobs = [(arm, spec) for spec in cases for arm in ["full", "ablated"]] + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + rows = list(pool.map(run, jobs)) + for installed in installs.values(): + if isinstance(installed, dict) and "cli" in installed: + assert digest(Path(installed["cli"])) == installed["cli_sha256"], "CLI changed during execution" + assert digest(Path(installed["tarball"])) == installed["tarball_sha256"] + assert digest(Path(installed["source_archive"])) == installed["source_archive_sha256"] + (args.out / "results.json").write_text(json.dumps(rows, indent=2) + "\n") + + +if __name__ == "__main__": + main() From 1d5fd6034950eba2515c09809fd16a06c5a8a2ca Mon Sep 17 00:00:00 2001 From: idan Date: Thu, 10 Sep 2026 12:59:39 +0800 Subject: [PATCH 4/7] docs: tell agents to reuse a running login process R1 read help then started another login --no-browser instead of sending the result to the live PTY. Help only; runtime unchanged. --- src/commands/oauth-login.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/oauth-login.ts b/src/commands/oauth-login.ts index 2ec46eb..d6e0293 100644 --- a/src/commands/oauth-login.ts +++ b/src/commands/oauth-login.ts @@ -379,7 +379,7 @@ export function loginHelpAfter(): string { return ` Default: open a browser this person can actually use (SSH/CI/no display skip the local open). The live authorize URL is printed at the same time — relay that exact URL immediately. Do not invent, shorten, or delay it. ---no-browser needs a persistent interactive TTY. Agents request only the one-time redirect URL or code — never an API key, access token, or refresh token. Paste that result into this same living process once. "Done" is not that input. If they cannot type here, relay the paste through chat into this PTY. An empty line cancels. +--no-browser needs a persistent interactive TTY. Agents request only the one-time redirect URL or code — never an API key, access token, or refresh token. Paste that result into this same living process once. Reuse a running login; do not start another. "Done" is not that input. If they cannot type here, relay the paste through chat into this PTY. An empty line cancels. If this process already timed out, or the URL/code is from an older run, start a fresh aisa login and use the new URL and its result. Do not reuse a stale paste. From e83b635fd5c620c3207b7a888fa71c0ac07c8815 Mon Sep 17 00:00:00 2001 From: idan Date: Thu, 10 Sep 2026 13:57:21 +0800 Subject: [PATCH 5/7] chore: prepare unpublished 0.5.2 login-help patch Version metadata and changelog only. Auth runtime, help body, and default-off login-handoff eval are unchanged. Not tagged or published. --- CHANGELOG.md | 18 ++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- src/constants.ts | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb2a354..0c1eccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.5.2] — 2026-09-10 + +Compatible patch on published `0.5.1`. Expanded `aisa login --help` for Agent +sign-in handoff, plus a default-off login-help evaluation. No auth runtime, +credential-precedence, or command-surface changes. + +### Changed + +- `aisa login --help` owns accessible-browser vs persistent headless TTY, live + URL relay, one-time paste into the same living process, reuse of a running + login, stale-session recovery, no-TTY native MCP OAuth, and connected-only + after a successful balance check. + +### Added + +- Default-off `eval/agent-quickstart/login-handoff/`. Excluded from the npm + package; does not run in default CI or against production AIsa credentials. + ## [0.5.1] — 2026-09-10 Compatible patch on published `0.5.0`. Browser-login-first onboarding diff --git a/package-lock.json b/package-lock.json index 4ddc457..1d47d06 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@aisa-one/cli", - "version": "0.5.1", + "version": "0.5.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@aisa-one/cli", - "version": "0.5.1", + "version": "0.5.2", "license": "MIT", "dependencies": { "chalk": "^5.3.0", diff --git a/package.json b/package.json index d22e444..8baab7f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@aisa-one/cli", - "version": "0.5.1", + "version": "0.5.2", "description": "CLI for the AIsa unified AI infrastructure platform - one API key for 80+ LLMs and 900+ endpoints across finance, search, social, and video APIs", "type": "module", "main": "dist/index.js", diff --git a/src/constants.ts b/src/constants.ts index 1c11442..e197a30 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,4 +1,4 @@ -export const VERSION = "0.5.1"; +export const VERSION = "0.5.2"; /** Root of the platform. Per-surface bases are derived in api.ts#resolveBases. */ export const BASE_URL = "https://api.aisa.one"; export const ENV_VAR_NAME = "AISA_API_KEY"; From 979b8c0786a0dd1379c35cc31cdb7ada7e85347f Mon Sep 17 00:00:00 2001 From: idan Date: Thu, 10 Sep 2026 13:59:17 +0800 Subject: [PATCH 6/7] docs: add 0.5.2 changelog comparison link Point Unreleased at v0.5.2...HEAD and add [0.5.2] v0.5.1...v0.5.2. --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c1eccb..3b7eedf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -424,7 +424,8 @@ supports today; nothing here depends on a backend change. - Config commands (`aisa config get|set|list|reset`) and auth (`aisa login|logout|whoami`). -[Unreleased]: https://github.com/AIsa-team/cli/compare/v0.5.1...HEAD +[Unreleased]: https://github.com/AIsa-team/cli/compare/v0.5.2...HEAD +[0.5.2]: https://github.com/AIsa-team/cli/compare/v0.5.1...v0.5.2 [0.5.1]: https://github.com/AIsa-team/cli/compare/v0.5.0...v0.5.1 [0.5.0]: https://github.com/AIsa-team/cli/compare/v0.3.0...v0.5.0 [0.4.0]: https://github.com/AIsa-team/cli/compare/v0.3.0...b5c0b04b2a7a2cb9efcb568be5ee5440d7f7d94d From 69257ee709ef38a9883db49e9241cf72cbcb901a Mon Sep 17 00:00:00 2001 From: idan Date: Thu, 10 Sep 2026 14:04:39 +0800 Subject: [PATCH 7/7] docs: align release operator steps with 0.5.2 --- docs/release.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/release.md b/docs/release.md index a5997b5..e1d3d36 100644 --- a/docs/release.md +++ b/docs/release.md @@ -8,16 +8,16 @@ that commit is merged and reviewed. A push to `main` runs CI only; | Item | Value | | --- | --- | -| Version | `0.5.1` (release target) | +| Version | `0.5.2` (release target) | | Command surface | 22 root help entries including implicit `help`; `api` is `list`/`show` only | -| Registry latest | `0.5.0` on `https://registry.npmjs.org` (baseline at this preparation; recheck before tagging) | +| Registry latest | `0.5.1` on `https://registry.npmjs.org` (baseline at this preparation; recheck before tagging) | | Default Router origin | `https://tools.aisa.one` | | LLM / catalog host | `https://api.aisa.one` | | Node | `engines` `>=18`. CI on Ubuntu: 18/20 legacy compatibility, 22/24 maintained, 26 current. Publish job uses Node 24 and npm `11.6.0`. | `package.json`, `package-lock.json` (root / `packages[""]`), `src/constants.ts` `VERSION`, installed `aisa --version`, and -`CHANGELOG.md` `## [0.5.1]` must agree. Confirm with +`CHANGELOG.md` `## [0.5.2]` must agree. Confirm with `node scripts/package-smoke.mjs` (or `--tarball` of the candidate archive). The VS Code extension is not version-bumped with this CLI release unless its own packaging requires it. @@ -45,21 +45,21 @@ official registry agree. ```bash # Official registry only — do not use a mirror as the source of truth. npm view @aisa-one/cli version --registry https://registry.npmjs.org -# baseline at this preparation: 0.5.0 — recheck before tagging +# baseline at this preparation: 0.5.1 — recheck before tagging # 0.4.0 is the unpublished main baseline, not a registry release. git checkout main git pull origin main -# Confirm this commit is the reviewed merge of the 0.5.1 candidate. -node -p "require('./package.json').version" # 0.5.1 -grep -E '^export const VERSION' src/constants.ts # "0.5.1" +# Confirm this commit is the reviewed merge of the 0.5.2 candidate. +node -p "require('./package.json').version" # 0.5.2 +grep -E '^export const VERSION' src/constants.ts # "0.5.2" -git tag -a v0.5.1 -m "v0.5.1" -git push origin v0.5.1 +git tag -a v0.5.2 -m "v0.5.2" +git push origin v0.5.2 ``` Do not tag a worktree or unmerged branch. Do not run `npm publish` on a -laptop. Do not retag or force-push `v0.5.0` or `v0.5.1`. Do not push a +laptop. Do not retag or force-push any released tag. Do not push a tag whose `v*` suffix differs from `package.json` `version` (the workflow refuses that mismatch). @@ -78,7 +78,7 @@ again via `prepack`). It: Local smoke of an existing archive: ```bash -node scripts/package-smoke.mjs --tarball /path/to/aisa-one-cli-0.5.1.tgz +node scripts/package-smoke.mjs --tarball /path/to/aisa-one-cli-0.5.2.tgz ``` `prepack` (`npm run build`) is what puts `dist/` into a clean `npm pack`. @@ -86,6 +86,6 @@ CI still runs an explicit `npm run build` before `npm test`. ## After the tag -Watch the Release workflow. Success is `0.5.1` on +Watch the Release workflow. Success is `0.5.2` on `https://registry.npmjs.org/@aisa-one/cli`. Recheck the official registry -before assuming the tag published. Do not retag `v0.5.0`. +before assuming the tag published. Never move an existing release tag.