diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..bed30b6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,61 @@ +name: Bug report +description: Something doesn't work as it should. +labels: [bug] +body: + - type: markdown + attributes: + value: | + Thanks for reporting. **Security problems go to SECURITY.md, not here.** + Don't paste tokens, cookies, API keys or private messages. Settings → Diagnostics exports a report with those removed. + - type: textarea + id: what + attributes: + label: What happened + description: What you did, what you expected, and what happened instead. + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to reproduce + placeholder: | + 1. Connect … + 2. Send … + 3. See … + validations: + required: true + - type: dropdown + id: area + attributes: + label: Area + options: + - Inbox and chat + - Approvals + - Ask an agent / drafts + - Notifications + - Connector: Grok Bot + - Connector: Muse + - Connector: ChatGPT Dots + - Connector: Cue + - Agents on the server (Concierge, A2A, ACP, endpoint) + - Local dev stack (dev:up, Synapse, Postgres) + - Mobile + - Other + validations: + required: true + - type: input + id: version + attributes: + label: Commit or version + description: "`git rev-parse --short HEAD`" + - type: input + id: macos + attributes: + label: macOS version and Mac + placeholder: macOS 26.1, Apple M3 + - type: textarea + id: logs + attributes: + label: Logs or diagnostics + description: The relevant part of `.local/logs/.log`, or the exported diagnostics. Check it has no private content first. + render: text diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..3f0ffb8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Report a security vulnerability + url: https://github.com/DolphinMiner/agentstudio/security/advisories/new + about: Privately, so it can be fixed before it's public. See SECURITY.md. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..d5cd14e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,24 @@ +name: Feature request +description: Suggest an idea or a change. +labels: [enhancement] +body: + - type: textarea + id: problem + attributes: + label: The problem + description: What are you trying to do, and what gets in the way today? + validations: + required: true + - type: textarea + id: idea + attributes: + label: What you'd like + description: How it could work. Sketches welcome. + - type: textarea + id: alternatives + attributes: + label: Alternatives you've considered + - type: markdown + attributes: + value: | + New connectors for third-party services must keep to personal use with your own account (see README). Check docs/roadmap.md first: it may already be planned. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..f780a54 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,13 @@ +## What and why + + + +## How I tested it + + + +## Checklist + +- [ ] Docs updated where behavior changed (`docs/`, README, `docs/clients.md` test ids) +- [ ] No tokens, cookies, API keys or real messages in code, fixtures or logs +- [ ] Connector changes keep to personal use with the user's own account (see CONTRIBUTING.md) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..41a74c7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,50 @@ +name: CI + +on: + push: + branches: [main, "feat/**"] + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + go: + name: Go (vet, test) + runs-on: ubuntu-latest + defaults: + run: + working-directory: server + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: server/go.mod + cache-dependency-path: server/go.sum + # v0.1 has no end-to-end encryption, so mautrix is built without libolm (-tags nocrypto). + # The integration tests need the local stack and skip themselves here (AGENTSTUDIO_IT unset). + - run: go vet -tags nocrypto ./... + - run: go test -tags nocrypto ./... + + typescript: + name: TypeScript (typecheck, unit tests) + runs-on: ubuntu-latest + env: + # Unit tests don't start Electron; skip downloading it. + ELECTRON_SKIP_BINARY_DOWNLOAD: "1" + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm --filter @agentstudio/core build + - run: pnpm -r --if-present typecheck + - run: pnpm run test:dev + - run: pnpm -r --if-present test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..509a259 --- /dev/null +++ b/.gitignore @@ -0,0 +1,52 @@ +# Dependencies +node_modules/ + +# Local dev stack: databases, Synapse, logs, generated configs and secrets +.local/ + +# Build output +dist/ +out/ +build/ +*.tsbuildinfo +server/bin/ +/server/agentstudio-* +apps/mobile/web-build/ + +# Expo / React Native +.expo/ +apps/mobile/ios/ +apps/mobile/android/ + +# Tests +coverage/ +test-results/ +playwright-report/ +blob-report/ +playwright/.cache/ +e2e-smoke/artifacts/ +apps/desktop/e2e-smoke/artifacts/ + +# Environment and secrets +.env +.env.* +!.env.example +*.pem +*.p12 +*.key + +# Logs +*.log +npm-debug.log* +pnpm-debug.log* + +# Editors and OS +.DS_Store +.idea/ +.vscode/* +!.vscode/extensions.json +*.swp + +# Claude Code (per-user) +.claude/settings.local.json +.claude/worktrees/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b45cda5 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,50 @@ +# AgentStudio: working agreement for humans and coding agents + +AgentStudio is an open-source inbox for your chats and your AI agents (Apache-2.0). Read these first; they are the contracts: + +- `docs/architecture.md`: scope, processes, ports, layout +- `docs/api.md`: REST API +- `docs/agent-protocol.md`: how agents connect +- `docs/matrix-events.md`: Matrix conventions, custom events, demo network +- `docs/clients.md`: screens and the `data-testid` contract used by e2e +- `docs/connectors.md`: the Mac app's connectors for Grok Bot, Muse, ChatGPT Dots and Cue (personal use only: see README) +- `docs/roadmap.md`: what's done and what's next +- `CONTRIBUTING.md`, `SECURITY.md`: how to contribute, and how to report a vulnerability + +When code and these docs disagree, fix whichever one is wrong in the same change. Never leave them out of sync. + +## Commands + +| Command | What | +|---|---| +| `pnpm install` | install JS workspace deps | +| `pnpm dev:up` | build Go binaries and start the whole local stack (idempotent) | +| `pnpm dev:down` | stop everything | +| `pnpm dev:status` | show process health | +| `pnpm dev:reset` | stop and wipe `.local/` | +| `pnpm dev:desktop` | run the Electron app (electron-vite dev mode) against the local stack | +| `pnpm dev:mobile` | run Expo (web) against the local stack on http://localhost:18300 | +| `pnpm test` | unit tests: stack manager (`node --test dev/stack.test.mjs`), Go (`cd server && go test -tags nocrypto ./...`) and TS (vitest) | +| `pnpm test:e2e` | e2e (Playwright projects `desktop` and `mobile-web`) against the local stack; starts it if needed and builds both apps. See `e2e/README.md` | + +Runtime state, logs and generated secrets live in `.local/`, which is git-ignored: + +- logs: `.local/logs/.log` +- generated env: `.local/env` + +## Conventions + +- **Go:** + - module `github.com/DolphinMiner/agentstudio/server` + - Go 1.26, `log/slog` for logs, `pgx/v5` for Postgres + - bridges are built on `maunium.net/go/mautrix` bridgev2 + - always build/test with `-tags nocrypto` (no E2EE in v0.1, so mautrix is built without libolm; plain `go build ./...` fails on `olm/olm.h`) + - never log message bodies or tokens +- **TypeScript:** + - strict mode, ESM, pnpm workspaces + - `@agentstudio/core` must not import DOM, Node or React Native APIs + - React 19 function components +- **Tests:** select elements only by `data-testid` / `testID` from `docs/clients.md`. Use no sleeps when a condition can be awaited. +- **No Docker.** The local stack runs native processes: Homebrew Postgres 18 in a private cluster under `.local/pg`, and Synapse in a uv venv. +- **Keep diffs focused.** Do not reformat files you did not change. +- **Connectors for third-party services** reach the user's own agents with the user's own account, on the user's Mac. Don't add anything that works at scale, for other people's accounts, or around a service's limits. Never commit real transcripts, tokens or cookies; scrub fixtures to their structure. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..fe2cfc3 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,13 @@ +# Code of Conduct + +AgentStudio follows the [Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). + +In short: be welcoming, be kind, and assume good intent. Harassment, insults, and personal or political attacks aren't accepted, in any of the project's spaces (issues, pull requests, discussions, and anywhere you represent the project). + +## Reporting + +If someone's behavior concerns you, report it privately to the maintainers: open a private security advisory (**Security → Report a vulnerability**) and say it's a conduct report, or contact a maintainer directly. Reports are handled confidentially. + +## Enforcement + +Maintainers decide on the response, following the Contributor Covenant's enforcement guidelines: from a private correction to a temporary or permanent ban from the project's spaces. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..6d7655f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,52 @@ +# Contributing to AgentStudio + +Thanks for helping. AgentStudio is alpha, so the code moves fast; small, focused changes are the easiest to review. + +## Before you start + +- **Bugs:** open an issue with the bug template. For anything security-related, follow [SECURITY.md](SECURITY.md) instead. +- **Features and bigger changes:** open an issue (or a discussion) first, so we can agree on the approach before you write the code. The plan lives in [docs/roadmap.md](docs/roadmap.md). +- **Read the contracts.** [AGENTS.md](AGENTS.md) lists the docs that define how things work (architecture, API, agent protocol, Matrix events, the client test ids, connectors). When code and docs disagree, fix the wrong one in the same change. + +## Set up + +Requirements: macOS, Go 1.26+, Node 22+, pnpm, Homebrew PostgreSQL 18 and uv. + +```bash +pnpm install +pnpm dev:up # Postgres, Synapse, the API, bridges and reference agents +pnpm dev:desktop # the Mac app in dev mode +``` + +More in [README.md](README.md). + +## Make the change + +- **Go:** build and test with `-tags nocrypto`. Use `log/slog`, and never log message bodies or tokens. +- **TypeScript:** strict mode, ESM. `@agentstudio/core` must not import DOM, Node or React Native APIs. +- **UI:** follow [docs/design.md](docs/design.md). Icons come from Lucide (desktop) and Ionicons (mobile); service images from `packages/brand`. Don't draw icons by hand. +- **Tests:** select elements only by the `data-testid` values in [docs/clients.md](docs/clients.md), and wait for conditions instead of sleeping. +- **Keep diffs focused.** Don't reformat code you didn't change. + +### Connectors for other services + +Changes to the Grok Bot, Muse, Dots or Cue connectors must keep to their purpose: reaching **your own** agents, with **your own** account, on **your own** Mac. We won't accept changes that make them work at scale, for other people's accounts, or that get around a service's limits or protections. See the note in [README.md](README.md#disclaimer). + +When a service changes, start the app with `AGENTSTUDIO_CONNECTOR_TRACE=1` (or `AGENTSTUDIO_CONNECTOR_TRACE_NETWORK=1` for Muse and Dots) and compare with the fakes in `e2e/fakes/agents.ts`. Never commit a real transcript or token; scrub fixtures down to their structure, like `apps/desktop/test/fixtures/grok-entries.json`. + +## Test + +```bash +pnpm test # unit tests: stack manager, Go, TypeScript +pnpm test:e2e # Playwright against the local stack (desktop and mobile web) +``` + +Run the e2e specs your change touches, for example `cd e2e && npx playwright test specs/local-agents.spec.ts --project desktop`. CI runs the unit tests and type checks on every pull request. + +## Send it + +- Open a pull request against the development branch with the template filled in: what changed, why, and how you tested it. +- Update the docs in the same pull request when behavior changes. +- By contributing, you agree that your contribution is licensed under the [Apache License 2.0](LICENSE). + +Everyone taking part follows the [Code of Conduct](CODE_OF_CONDUCT.md). diff --git a/LICENSE b/LICENSE index c7e2b02..261eeb9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,201 @@ -MIT License - -Copyright (c) 2026 White - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICES.md b/NOTICES.md new file mode 100644 index 0000000..c85d5b6 --- /dev/null +++ b/NOTICES.md @@ -0,0 +1,29 @@ +# Third-party notices + +## beeper-muse + +The Muse page driver in `apps/desktop/src/main/connectors/drivers.ts` adapts selectors and busy-state logic from beeper-muse (https://github.com/nishu-builder/beeper-muse). + +``` +MIT License + +Copyright (c) 2026 nishu-builder + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` diff --git a/README.md b/README.md index f7d6ed8..8b69623 100644 --- a/README.md +++ b/README.md @@ -1 +1,135 @@ -# agentstudio \ No newline at end of file +

+ +

AgentStudio

+ +**English** · [简体中文](README.zh-CN.md) + +

+ AgentStudio: your people and your agents, in one inbox. The Mac app with a dinner booking from Muse waiting for approval. +

+ +> [!WARNING] +> **FOR PERSONAL USE ONLY.** AgentStudio is a personal tool for learning and research: run it on your own Mac, with your own accounts, to reach your own agents. **NOT FOR COMMERCIAL USE**, and not for running a service for other people. Use it at your own risk. Read the [Disclaimer](#disclaimer) first. + +> **Alpha.** macOS only, and you run it from source with a local development stack. Expect rough edges and breaking changes. + +An open-source inbox for your chats and your AI agents. Agents show up as contacts next to the people you talk to. When an agent needs a decision, it sends you an approval card. You can hand any conversation to an agent and get reply drafts back. + +Status: early development (v0.1). macOS first: the desktop app (Electron) comes first, then mobile (Expo). Nothing here is hosted yet; everything runs on your Mac. + +

+ + A 75-second tour: chat with a Grok bot and approve its report, ask Muse about the weather and approve a dinner booking, have ChatGPT Dots draft a reply to Leo, and connect Grok Bot, Cue, Muse and ChatGPT + +
+ A 75-second tour. Watch it in 4K (MP4). +

+ +## What it does + +- **One inbox** for chats and agents. People are circles, agents are squircles, and each one carries a small mark for the network or service it lives on. +- **Approvals.** An agent asks, you decide in one tap, from the chat or from the Approvals screen. +- **Ask an agent.** Forward a chat with its context to an agent; its reply drafts come back, ready to edit and send. +- **Agents you already use**, through your own sign-in on the Mac: Grok Bot (every bot of your account, with its own name and avatar), Muse, ChatGPT Dots and Cue (sign in with Manus, or use a Manus API key). Experimental and for personal use only: see [below](#disclaimer). +- **Agents that speak an open protocol**: the [AgentStudio Agent Protocol](docs/agent-protocol.md), A2A, and ACP. Hermes Agent and OpenClaw are recognized and shown with their own icons. + +

+ Agents ask before they act: a Grok bot asks whether to send the weekly report, with Send it and Don't send buttons +

+

+ Ask an agent about any message: pick an agent and the lines it may read, and get reply drafts back +

+

+ Bring the agents you already use: the Connect screen with Grok Bot, Muse, ChatGPT Dots and Cue, and Grok Bot's two connected bots +

+ +## How it is built + +| Part | Where | What | +|---|---|---| +| Desktop app | `apps/desktop` | Electron + React. Also hosts the connectors for Grok Bot, Muse, Dots and Cue. | +| Mobile app | `apps/mobile` | Expo / React Native. Tested on the web target until native builds land. | +| Client core | `packages/core` | Matrix client, inbox, approvals, tasks; shared by both apps. | +| Brand assets | `packages/brand` | Official images of the services AgentStudio connects to, with sources. | +| Server | `server` (Go) | `agentstudio-api`, the `agentstudio-agents` and `demo` bridges (mautrix bridgev2), protocol adapters, the reference agent. | +| Homeserver | Synapse | Matrix underneath every chat, federation off. | +| Dev stack | `dev/stack.mjs` | Starts everything locally as native processes, no Docker. | +| End-to-end tests | `e2e` | Playwright against the Electron app and the mobile web build. | + +More in [docs/architecture.md](docs/architecture.md). Other docs: [API](docs/api.md), [clients](docs/clients.md), [design](docs/design.md), [Matrix events](docs/matrix-events.md). + +## Quick start (macOS) + +Requirements: Go 1.26+, Node 22+, pnpm, Homebrew PostgreSQL 18, uv. + +```bash +pnpm install +pnpm dev:up # Postgres, Synapse, API, bridges, reference agents (first run: about 30 s) +pnpm dev:desktop # Electron app (dev mode) against the local stack +pnpm dev:mobile # mobile app on the Expo web target: http://localhost:18300 +``` + +There's no account to create: the stack runs in local mode, so the app opens straight to your inbox. +(To try email sign-in instead, pick "Sign in with email instead" after signing out. In dev mode the code isn't mailed; it is shown at +`http://127.0.0.1:18090/v1/dev/outbox?email=` and in `.local/logs/api.log`.) +Then, on the Connect screen, add the **Demo** network (simulated contacts) and the **Concierge** agent, or sign in to one of the agents you already use. + +Managing the stack: + +```bash +pnpm dev:status # what's running +pnpm dev:restart # restart one service, e.g. api, agents +pnpm dev:logs # follow a service's log +pnpm dev:down # stop everything +pnpm dev:reset # stop and delete all local data in .local/ +``` + +All runtime state (databases, Synapse, logs) lives in `.local/`, which is git-ignored. + +## Tests + +```bash +pnpm test # unit tests: stack manager, Go, TypeScript +pnpm test:e2e # Playwright: Electron app + mobile web, against the local stack +``` + +If the Electron binary is missing after `pnpm install` (pnpm can skip its postinstall), run +`node node_modules/electron/install.js` in `apps/desktop`. Install Playwright's Chromium with +`pnpm --filter @agentstudio/e2e exec playwright install chromium`. + +## Connecting agents + +| Agent | How | +|---|---| +| Concierge (built-in reference agent) | Connect → Agents on your server → Concierge | +| Any AgentStudio Agent Protocol endpoint | Connect → AgentStudio agent endpoint, then its URL and token | +| A2A agents (for example Hermes Agent) | Connect → A2A agent, then the agent's URL | +| ACP agents allowed by the server (`AGENTSTUDIO_ACP_AGENTS`) | Connect → ACP agent | +| Grok Bot, Muse, ChatGPT Dots, Cue (Mac app only, experimental) | Connect → Agents you already use. Your sign-in (or Cue's API key) stays on the Mac, and these agents answer while AgentStudio is open. | + +Read [the note on third-party connectors](#disclaimer) before connecting Grok Bot, Muse, Dots or Cue. + +To debug a connector, start the app with `AGENTSTUDIO_CONNECTOR_TRACE=1` (and `AGENTSTUDIO_CONNECTOR_SHOW=1` to see the hidden Muse / Dots windows). See [docs/connectors.md](docs/connectors.md). + +## Disclaimer + +**⚖️ FOR PERSONAL LEARNING AND RESEARCH USE ONLY.** Read this before you use AgentStudio, and in particular its connectors for Grok Bot, Muse, ChatGPT Dots and Cue. + +- **Personal use only.** Use AgentStudio on your own Mac, with your own accounts, to reach your own agents. Don't use it commercially, to run a service for others, to automate at scale, or with accounts that aren't yours. +- **Local only.** Everything runs on your Mac. Your sign-ins stay there, encrypted with the system keychain; AgentStudio uploads nothing to any server of ours (there is none). +- **Not affiliated.** AgentStudio isn't affiliated with or endorsed by xAI / SpaceX AI, Cursor (Anysphere), Meta, OpenAI, Manus, Nous Research or OpenClaw. Their names and logos are trademarks of their owners and appear only to say which service a connector reaches. +- **Unofficial connectors.** Except for Cue's messaging (Manus's documented API), the connectors work the way each service's own app or website does. That may be against a service's terms, and it can stop working at any time. +- **Your responsibility.** You are responsible for following each service's terms and the laws where you live. If you aren't sure, don't connect it. +- **Use at your own risk.** AgentStudio comes "as is", without warranty of any kind (see the Apache-2.0 license). The authors aren't liable for any loss, account restriction or other consequence of using it. + +Details per service: [docs/connectors.md](docs/connectors.md). + +## Contributing and security + +- [CONTRIBUTING.md](CONTRIBUTING.md): how to set up, test and send a change. +- [SECURITY.md](SECURITY.md): how to report a vulnerability privately. Please don't open a public issue for one. +- [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). + +## License + +Apache-2.0. Third-party notices are in [NOTICES.md](NOTICES.md); brand images belong to their owners (see [packages/brand/README.md](packages/brand/README.md)). diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 0000000..bafcc16 --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,135 @@ +

+ +

AgentStudio

+ +[English](README.md) · **简体中文** + +

+ AgentStudio:把你的联系人和 agent 放进同一个收件箱。图为 Mac 版,Muse 订好的晚餐正等你批准。 +

+ +> [!WARNING] +> **仅供个人学习和研究使用。** AgentStudio 是一个个人工具:请在你自己的 Mac 上、用你自己的账号,连接你自己的 agent。**请勿用于商业用途**,也不要用来为他人提供服务。使用风险自负,使用前请先阅读[免责声明](#免责声明)。 + +> **Alpha 阶段。** 目前只支持 macOS,需要从源码运行,并在本机启动开发环境。可能会有不少粗糙的地方,也可能出现不兼容的改动。 + +一个开源的收件箱,用来统一管理你的聊天和 AI agent。agent 和你的联系人排在一起,像联系人一样出现。agent 需要你拍板时,会给你发一张审批卡片。你也可以把任意一段对话交给 agent,让它帮你起草回复。 + +当前状态:早期开发阶段(v0.1)。macOS 优先:先做桌面端(Electron),之后再做手机端(Expo)。目前没有任何托管服务,所有东西都运行在你自己的 Mac 上。 + +

+ + 75 秒演示:和 Grok bot 聊天并批准它的周报,问 Muse 天气并批准订餐,让 ChatGPT Dots 起草给 Leo 的回复,然后连接 Grok Bot、Cue、Muse 和 ChatGPT + +
+ 75 秒演示。观看 4K 高清版(MP4)。 +

+ +## 能做什么 + +- **一个收件箱**,装下聊天和 agent。人显示为圆形头像,agent 显示为圆角方形头像,右下角的小标记表示它来自哪个网络或服务。 +- **审批。** agent 提出请求,你在聊天里或审批页点一下就能决定。 +- **交给 agent。** 把一段聊天连同上下文交给 agent,它会返回几条回复草稿,可以修改后直接发送。 +- **接入你已经在用的 agent**,用你自己的账号在 Mac 上登录:Grok Bot(账号下的每个 bot 都会接入,名字和头像各自保留)、Muse、ChatGPT Dots 和 Cue(用 Manus 账号登录,或者使用 Manus API key)。这部分是实验性功能,仅供个人使用,请看[下文](#免责声明)。 +- **接入支持开放协议的 agent**:[AgentStudio Agent Protocol](docs/agent-protocol.md)、A2A 和 ACP。Hermes Agent 和 OpenClaw 会被自动识别,显示它们自己的图标。 + +

+ agent 先问再做:一个 Grok bot 询问是否发送周报,下面有 Send it 和 Don't send 两个按钮 +

+

+ 对任意消息询问 agent:选一个 agent 和它可以读的消息,拿回回复草稿 +

+

+ 接入你已经在用的 agent:Connect 页上有 Grok Bot、Muse、ChatGPT Dots 和 Cue,Grok Bot 下已连接了两个 bot +

+ +## 项目结构 + +| 部分 | 位置 | 说明 | +|---|---|---| +| 桌面端 | `apps/desktop` | Electron + React。Grok Bot、Muse、Dots、Cue 的连接器也运行在这里。 | +| 手机端 | `apps/mobile` | Expo / React Native。原生版本完成之前,先在网页版上测试。 | +| 客户端核心 | `packages/core` | Matrix 客户端、收件箱、审批、任务,两个端共用。 | +| 品牌素材 | `packages/brand` | AgentStudio 所连接服务的官方图片,附来源。 | +| 服务端 | `server`(Go) | `agentstudio-api`、`agentstudio-agents` 和 `demo` 两个桥接(mautrix bridgev2)、协议适配器、示例 agent。 | +| 消息服务器 | Synapse | 每个聊天底层都是 Matrix,联邦功能关闭。 | +| 开发环境 | `dev/stack.mjs` | 在本机用原生进程启动全部服务,不需要 Docker。 | +| 端到端测试 | `e2e` | 用 Playwright 测试 Electron 应用和手机网页版。 | + +更多内容见 [docs/architecture.md](docs/architecture.md)。其他文档:[API](docs/api.md)、[客户端](docs/clients.md)、[设计](docs/design.md)、[Matrix 事件](docs/matrix-events.md)。(文档目前为英文。) + +## 快速开始(macOS) + +需要安装:Go 1.26+、Node 22+、pnpm、Homebrew 版 PostgreSQL 18、uv。 + +```bash +pnpm install +pnpm dev:up # 启动 Postgres、Synapse、API、桥接和示例 agent(第一次约 30 秒) +pnpm dev:desktop # 以开发模式启动 Electron 应用,连接本地服务 +pnpm dev:mobile # 在 Expo 网页版上运行手机端:http://localhost:18300 +``` + +不需要注册账号:开发环境默认是本机模式,打开应用就直接进入收件箱。 +(想试邮箱登录的话,退出登录后点"Sign in with email instead"。开发模式下验证码不会真的发邮件,可以在 +`http://127.0.0.1:18090/v1/dev/outbox?email=<你的邮箱>` 查看,`.local/logs/api.log` 里也有。) +然后在 Connect 页添加 **Demo** 网络(模拟的联系人)和 **Concierge** agent,或者登录一个你已经在用的 agent。 + +管理本地服务: + +```bash +pnpm dev:status # 查看运行状态 +pnpm dev:restart # 重启某个服务,例如 api、agents +pnpm dev:logs # 查看某个服务的日志 +pnpm dev:down # 停止全部服务 +pnpm dev:reset # 停止服务并删除 .local/ 下的全部本地数据 +``` + +所有运行时数据(数据库、Synapse、日志)都在 `.local/` 目录下,这个目录不会提交到 git。 + +## 测试 + +```bash +pnpm test # 单元测试:开发环境脚本、Go、TypeScript +pnpm test:e2e # Playwright:Electron 应用和手机网页版,连接本地服务 +``` + +如果 `pnpm install` 之后找不到 Electron 程序(pnpm 有时会跳过它的安装脚本),请在 `apps/desktop` 下运行 +`node node_modules/electron/install.js`。Playwright 用的 Chromium 用 +`pnpm --filter @agentstudio/e2e exec playwright install chromium` 安装。 + +## 接入 agent + +| agent | 方式 | +|---|---| +| Concierge(内置示例 agent) | Connect → Agents on your server → Concierge | +| 任何实现了 AgentStudio Agent Protocol 的服务 | Connect → AgentStudio agent endpoint,填入它的地址和 token | +| A2A agent(例如 Hermes Agent) | Connect → A2A agent,填入 agent 的地址 | +| 服务端允许的 ACP agent(`AGENTSTUDIO_ACP_AGENTS`) | Connect → ACP agent | +| Grok Bot、Muse、ChatGPT Dots、Cue(仅 Mac 版,实验性) | Connect → Agents you already use。你的登录凭据(或 Cue 的 API key)只保存在这台 Mac 上;AgentStudio 开着时这些 agent 才会回复。 | + +连接 Grok Bot、Muse、Dots 或 Cue 之前,请先阅读[关于第三方连接器的说明](#免责声明)。 + +调试连接器时,可以用 `AGENTSTUDIO_CONNECTOR_TRACE=1` 启动应用(再加上 `AGENTSTUDIO_CONNECTOR_SHOW=1` 可以显示 Muse / Dots 的后台窗口)。详见 [docs/connectors.md](docs/connectors.md)。 + +## 免责声明 + +**⚖️ 本项目仅供个人学习和研究使用。** 使用 AgentStudio 之前,尤其是使用 Grok Bot、Muse、ChatGPT Dots 和 Cue 的连接器之前,请先阅读以下内容。 + +- **仅限个人使用:** 请在你自己的 Mac 上、用你自己的账号连接你自己的 agent。请勿用于商业用途,不要用来为他人提供服务、批量自动化,或使用不属于你的账号。 +- **只在本机运行:** 所有数据和登录凭据都留在你的 Mac 上,由系统钥匙串加密保存。AgentStudio 不会把任何内容上传到我们的服务器(我们也没有服务器)。 +- **与相关服务无关联:** 本项目与 xAI / SpaceX AI、Cursor(Anysphere)、Meta、OpenAI、Manus、Nous Research、OpenClaw 均无关联,也未获得其认可。相关名称和图标归各自所有者,仅用于标明连接的是哪个服务。 +- **非官方接入:** 除 Cue 的收发消息使用 Manus 公开的官方 API 外,其余连接器都模仿对方 App 或网页的工作方式,可能违反对方的服务条款,也可能随时失效。 +- **责任自负:** 你需要自行遵守各服务的条款和所在地的法律法规。如不确定,请不要连接。 +- **风险自担:** 本项目按"原样"提供,不提供任何担保(见 Apache-2.0 许可证)。作者不对使用本项目造成的任何损失、账号受限或其他后果负责。 + +各服务的具体情况见 [docs/connectors.md](docs/connectors.md)(英文)。 + +## 参与贡献和安全问题 + +- [CONTRIBUTING.md](CONTRIBUTING.md):如何搭建环境、运行测试、提交改动。 +- [SECURITY.md](SECURITY.md):如何私下报告安全漏洞。请不要为安全漏洞开公开的 issue。 +- [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md):行为准则。 + +## 许可证 + +Apache-2.0。第三方声明见 [NOTICES.md](NOTICES.md);品牌图片归各自所有者(见 [packages/brand/README.md](packages/brand/README.md))。 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..2347946 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,30 @@ +# Security policy + +AgentStudio holds sensitive things: your chats, your agents' approvals, and the sign-ins you give its connectors for other services. Please report security problems privately. + +## Reporting a vulnerability + +- **Use GitHub's private vulnerability reporting:** on the repository page, open **Security → Report a vulnerability**. Only the maintainers see the report. +- **Don't open a public issue, discussion or pull request** for a vulnerability. +- Include what you found, how to reproduce it, which version or commit you tested, and the impact you expect. + +We aim to acknowledge a report within 3 working days and to agree on a fix and disclosure date with you. We'll credit you in the release notes unless you'd rather not be named. + +## Scope + +In scope, for example: + +- The connectors' handling of credentials: the encrypted store (`connectors.bin`), the connector host on `127.0.0.1:18401` and its per-instance tokens, the Cue sign-in link catcher. +- `agentstudio-api`: sign-in, sessions, the Matrix token it mints per session, the `/_matrix` proxy, approvals and tasks. +- The agents bridge and the AgentStudio Agent Protocol (callbacks, bearer tokens). +- The Electron app: IPC between the renderer and the main process, external-link handling, the content security policy. + +Out of scope: + +- Problems in the services AgentStudio connects to (Grok Bot, Muse, ChatGPT, Cue, Matrix homeservers). Report those to their owners. +- The local development stack's default settings (dev-mode email codes shown in `/v1/dev/outbox`, plain HTTP on loopback). It isn't meant to face a network. +- Attacks that need someone who already controls your Mac account. + +## Supported versions + +AgentStudio is alpha. Fixes land on the main development branch; there are no maintained release branches yet. diff --git a/apps/desktop/.gitignore b/apps/desktop/.gitignore new file mode 100644 index 0000000..c363b71 --- /dev/null +++ b/apps/desktop/.gitignore @@ -0,0 +1,2 @@ +out/ +e2e-smoke/artifacts/ diff --git a/apps/desktop/README.md b/apps/desktop/README.md new file mode 100644 index 0000000..dbf821d --- /dev/null +++ b/apps/desktop/README.md @@ -0,0 +1,21 @@ +# @agentstudio/desktop + +Electron (macOS) app: electron 45 + electron-vite 6 + React 19. Screens and test ids follow `docs/clients.md`. + +| Command | What | +|---|---| +| `pnpm dev` (or `pnpm dev:desktop` at the root) | Vite dev server + Electron against the local stack | +| `pnpm build` / `pnpm start` | build to `out/`, then run the built app | +| `pnpm typecheck`, `pnpm test` | tsc and vitest (pure helpers) | +| `pnpm e2e:smoke` | build, then Playwright `_electron` smoke test against the running stack (`pnpm dev:up` first) | + +Environment: + +- `AGENTSTUDIO_SERVER_URL`: server base URL (beats the saved setting; default `http://127.0.0.1:18090`). +- `AGENTSTUDIO_USER_DATA_DIR`: profile directory (settings and the safeStorage-encrypted `session.bin`). Use a fresh one per e2e run. +- `AGENTSTUDIO_MOCK_KEYCHAIN=1`: tests only; safeStorage uses Chromium's mock keychain instead of the login keychain. +- `AGENTSTUDIO_TEST_NOTIFICATIONS=1`: tests only; notifications and the Dock badge are recorded in the main process's `globalThis.__agentStudioNotifications` instead of shown (`docs/clients.md` § Notifications). + +Security: `contextIsolation`, `sandbox`, no Node integration; the renderer only sees `window.agentStudioDesktop` (see `src/shared/desktop-api.ts`). The renderer is served from `file://` through a `protocol.handle("file")` interceptor that only serves `out/renderer` and adds a CSP header whose `connect-src` is the server origin (plus its `ws:`/`wss:` origin). Changing the server URL reloads the window so the CSP follows. + +Electron's postinstall (binary download) is allowed via `allowBuilds` in `pnpm-workspace.yaml`. If `node_modules/electron/dist` is missing, run `node node_modules/electron/install.js` in this directory. diff --git a/apps/desktop/e2e-smoke/playwright.config.ts b/apps/desktop/e2e-smoke/playwright.config.ts new file mode 100644 index 0000000..c662b77 --- /dev/null +++ b/apps/desktop/e2e-smoke/playwright.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from "@playwright/test"; + +// Smoke tests for the BUILT desktop app (run `pnpm build` first, or use +// `pnpm e2e:smoke`, which builds). Needs the local stack: `pnpm dev:up`. +export default defineConfig({ + testDir: ".", + testMatch: /.*\.spec\.ts/, + timeout: 120_000, + expect: { timeout: 20_000 }, + workers: 1, + reporter: [["list"]], + outputDir: "./artifacts/test-results", +}); diff --git a/apps/desktop/e2e-smoke/smoke.spec.ts b/apps/desktop/e2e-smoke/smoke.spec.ts new file mode 100644 index 0000000..c0453da --- /dev/null +++ b/apps/desktop/e2e-smoke/smoke.spec.ts @@ -0,0 +1,220 @@ +// Desktop smoke test: launches the built Electron app with a fresh profile and +// walks the main flows against the running local stack. + +import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { _electron as electron, expect, test, type ElectronApplication, type Page } from "@playwright/test"; + +const SERVER = process.env.AGENTSTUDIO_SERVER_URL ?? "http://127.0.0.1:18090"; +// Launch the package directory so Electron reads package.json (name, version, main). +const APP_DIR = join(__dirname, ".."); +const ARTIFACTS = join(__dirname, "artifacts"); +const run = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`; +const EMAIL = `desktop-smoke-${run}@example.com`; + +let profileDir: string; +let app: ElectronApplication; +let page: Page; + +async function launch(): Promise { + const env: Record = {}; + for (const [k, v] of Object.entries(process.env)) if (v !== undefined) env[k] = v; + delete env.ELECTRON_RUN_AS_NODE; + app = await electron.launch({ + args: [APP_DIR], + env: { ...env, AGENTSTUDIO_USER_DATA_DIR: profileDir, AGENTSTUDIO_SERVER_URL: SERVER, AGENTSTUDIO_MOCK_KEYCHAIN: "1" }, + }); + page = await app.firstWindow(); + page.on("pageerror", (e) => console.log("[renderer error]", e.message)); +} + +async function signInCode(email: string): Promise { + for (let i = 0; i < 50; i++) { + const res = await fetch(`${SERVER}/v1/dev/outbox?email=${encodeURIComponent(email)}`); + if (res.ok) { + const body = (await res.json()) as { messages: { code: string }[] }; + if (body.messages[0]?.code) return body.messages[0].code; + } + await new Promise((r) => setTimeout(r, 200)); + } + throw new Error("no sign-in code in the dev outbox"); +} + +function row(name: string) { + return page.locator(`[data-testid="chat-row"][data-room-name="${name}"]`); +} + +async function shot(name: string) { + mkdirSync(ARTIFACTS, { recursive: true }); + await page.screenshot({ path: join(ARTIFACTS, `${name}.png`) }); +} + +test.describe.configure({ mode: "serial" }); + +test.beforeAll(async () => { + const health = await fetch(`${SERVER}/v1/health`).then((r) => r.json() as Promise<{ ok: boolean }>); + expect(health.ok, "local stack must be running (pnpm dev:up)").toBe(true); + profileDir = mkdtempSync(join(tmpdir(), "agentstudio-desktop-smoke-")); + await launch(); +}); + +test.afterAll(async () => { + await app?.close(); + if (profileDir) rmSync(profileDir, { recursive: true, force: true }); +}); + +test("window is locked down", async () => { + const info = await page.evaluate(() => ({ + hasRequire: typeof (globalThis as { require?: unknown }).require !== "undefined", + hasProcess: typeof (globalThis as { process?: unknown }).process !== "undefined", + bridge: Object.keys((globalThis as unknown as { agentStudioDesktop: object }).agentStudioDesktop).sort(), + server: (globalThis as unknown as { agentStudioDesktop: { getServerUrl(): string } }).agentStudioDesktop.getServerUrl(), + })); + expect(info.hasRequire).toBe(false); + expect(info.hasProcess).toBe(false); + expect(info.bridge).toEqual( + ["connectors", "getServerUrl", "notifications", "openExternal", "platform", "serverUrlFromEnv", "session", "setServerUrl", "version"].sort(), + ); + expect(info.server).toBe(SERVER); + // The CSP only allows the configured server origin: other origins raise a violation. + const violated = await page.evaluate( + () => + new Promise((resolve) => { + document.addEventListener("securitypolicyviolation", (e) => resolve(e.violatedDirective), { once: true }); + fetch("http://127.0.0.1:18008/_matrix/client/versions").catch(() => undefined); + setTimeout(() => resolve("none"), 3000); + }), + ); + expect(violated).toBe("connect-src"); + const allowed = await page.evaluate((s) => fetch(`${s}/v1/health`).then((r) => r.ok), SERVER); + expect(allowed).toBe(true); +}); + +test("sign in with an emailed code", async () => { + await page.getByTestId("signin-email").fill(EMAIL); + await page.getByTestId("signin-send").click(); + await expect(page.getByTestId("signin-code")).toBeVisible(); + // A wrong code shows an error first. + await page.getByTestId("signin-code").fill("000000"); + await page.getByTestId("signin-verify").click(); + await expect(page.getByTestId("signin-error")).toBeVisible(); + await page.getByTestId("signin-code").fill(await signInCode(EMAIL)); + await page.getByTestId("signin-verify").click(); + await expect(page.getByTestId("nav-inbox")).toBeVisible(); + await expect(page.getByTestId("inbox-empty")).toBeVisible(); + await shot("01-inbox-empty"); +}); + +test("connect Demo and see the three demo chats", async () => { + await page.getByTestId("nav-connect").click(); + await page.getByTestId("network-demo").click(); + await page.getByTestId("flow-demo").click(); + await expect(page.getByTestId("login-step")).toHaveAttribute("data-step-type", "user_input"); + await expect(page.getByTestId("login-field-display_name")).toHaveValue("Demo User"); + await page.getByTestId("login-submit").click(); + await expect(page.getByTestId("login-complete")).toBeVisible(); + await expect(page.locator('[data-testid="connection-row"][data-network="demo"]')).toBeVisible(); + await shot("02-connect"); + + await page.getByTestId("nav-inbox").click(); + for (const name of ["Mom", "Leo Park", "Book Club"]) await expect(row(name)).toBeVisible({ timeout: 30_000 }); + await expect(row("Leo Park").getByTestId("chat-row-network")).toHaveText("Demo"); +}); + +test("open Leo Park, send a message, get the echo", async () => { + await row("Leo Park").click(); + await expect(page.getByTestId("chat-title")).toHaveText("Leo Park"); + await expect(page.getByTestId("chat-network")).toHaveText("Demo"); + await expect(page.getByTestId("message-body").filter({ hasText: "Are we still on for 7:15?" })).toBeVisible(); + const text = `hello from smoke ${run}`; + await page.getByTestId("composer-input").fill(text); + await page.getByTestId("composer-send").click(); + await expect(page.locator('[data-testid="message"][data-own="true"]').getByTestId("message-body").last()).toHaveText(text); + const echo = page.locator('[data-testid="message"][data-own="false"][data-sender="Leo Park"]').getByTestId("message-body").filter({ hasText: `Echo: ${text}` }); + await expect(echo).toBeVisible(); + await shot("03-chat-echo"); +}); + +test("ask Concierge about a message, use a draft, task done", async () => { + await page.getByTestId("nav-connect").click(); + await page.getByTestId("network-agentstudio-agents").click(); + await page.getByTestId("flow-concierge").click(); + const step = page.getByTestId("login-step"); + await expect(step).toBeVisible(); + if ((await step.getAttribute("data-step-type")) === "user_input") await page.getByTestId("login-submit").click(); + await expect(page.getByTestId("login-complete")).toBeVisible(); + await page.getByTestId("nav-inbox").click(); + await expect(row("Concierge")).toBeVisible({ timeout: 30_000 }); + await expect(row("Concierge").getByTestId("chat-row-network")).toHaveText("Agent"); + + await row("Leo Park").click(); + const anchor = page.locator('[data-testid="message"]').filter({ hasText: "Are we still on for 7:15?" }); + await anchor.hover(); + await anchor.getByTestId("message-ask").click(); + await expect(page.getByTestId("ask-agent-Concierge")).toBeVisible(); + await expect(page.getByTestId("ask-request")).toHaveValue("Draft a reply"); + await expect(page.getByTestId("ask-line").last()).toContainText("Are we still on for 7:15?"); + await shot("04-ask-modal"); + await page.getByTestId("ask-send").click(); + + await expect(page.getByTestId("chat-title")).toHaveText("Concierge"); + const drafts = page.getByTestId("drafts-card").last(); + await expect(drafts).toBeVisible({ timeout: 30_000 }); + await expect(drafts.getByTestId("draft-item").first()).toContainText("Leo, sounds good, see you then!"); + await shot("05-drafts"); + await drafts.getByTestId("draft-use").first().click(); + + await expect(page.getByTestId("chat-title")).toHaveText("Leo Park"); + await expect(page.getByTestId("composer-input")).toHaveValue("Leo, sounds good, see you then!"); + await page.getByTestId("composer-send").click(); + await expect( + page.locator('[data-testid="message"][data-sender="Leo Park"]').filter({ hasText: "Echo: Leo, sounds good, see you then!" }), + ).toBeVisible(); + + await page.getByTestId("nav-tasks").click(); + await expect(page.getByTestId("tasks-board")).toBeVisible(); + await expect(page.getByTestId("tasks-col-done").locator('[data-testid="task-card"][data-state="done"]')).toHaveCount(1); + await shot("06-tasks"); +}); + +test("approval card: flight request, approve, badge clears", async () => { + await page.getByTestId("nav-inbox").click(); + await row("Concierge").click(); + await page.getByTestId("composer-input").fill("Find me a flight to Toronto"); + await page.getByTestId("composer-send").click(); + const card = page.getByTestId("approval-card").last(); + await expect(card.getByTestId("approval-title")).toHaveText("Book the 6pm flight to Toronto for $412?"); + await expect(card.getByTestId("approval-detail")).toHaveCount(3); + await expect(card.getByTestId("approval-expiry")).toHaveText(/min left/); + await expect(page.getByTestId("nav-approvals-badge")).toHaveText("1"); + await page.getByTestId("nav-approvals").click(); + await expect(page.getByTestId("approvals-list").getByTestId("approval-card")).toHaveCount(1); + await shot("07-approvals"); + await page.getByTestId("approval-choice-book").click(); + await expect(page.getByTestId("approval-state")).toHaveText("Approved: Book"); + await expect(page.getByTestId("nav-approvals-badge")).toHaveCount(0); + await expect(row("Concierge").getByTestId("chat-row-needs-you")).toHaveCount(0); + await page.getByTestId("nav-inbox").click(); + await row("Concierge").click(); + await expect(page.getByTestId("message-body").filter({ hasText: "Booked. Seat 14A, confirmation PCH412." })).toBeVisible(); + await expect(page.getByTestId("approval-card").last().getByTestId("approval-state")).toHaveText("Approved: Book"); + await shot("08-approved-in-chat"); +}); + +test("settings show the account; session survives a restart; sign out", async () => { + await page.getByTestId("nav-settings").click(); + await expect(page.getByTestId("settings-email")).toHaveText(EMAIL); + await expect(page.getByTestId("settings-server")).toHaveValue(SERVER); + await expect(page.getByTestId("settings-version")).toHaveText("0.1.0"); + await shot("09-settings"); + + await app.close(); + await launch(); + await expect(page.getByTestId("nav-inbox")).toBeVisible(); + await expect(row("Leo Park")).toBeVisible({ timeout: 30_000 }); + + await page.getByTestId("nav-settings").click(); + await page.getByTestId("settings-signout").click(); + await expect(page.getByTestId("signin-email")).toBeVisible(); +}); diff --git a/apps/desktop/e2e-smoke/tsconfig.json b/apps/desktop/e2e-smoke/tsconfig.json new file mode 100644 index 0000000..1b244de --- /dev/null +++ b/apps/desktop/e2e-smoke/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../tsconfig.node.json", + "compilerOptions": { "lib": ["ES2023", "DOM"] }, + "include": ["."] +} diff --git a/apps/desktop/electron.vite.config.ts b/apps/desktop/electron.vite.config.ts new file mode 100644 index 0000000..f58ee3d --- /dev/null +++ b/apps/desktop/electron.vite.config.ts @@ -0,0 +1,26 @@ +import { resolve } from "node:path"; +import react from "@vitejs/plugin-react"; +import { defineConfig } from "electron-vite"; +import pkg from "./package.json"; + +// Main and preload are emitted as CommonJS (the package is not "type": "module"), +// which the sandboxed preload requires. The renderer is bundled locally and +// loaded with loadFile in production; it never loads remote code. +export default defineConfig({ + main: { + // Also correct when Electron is started on the bundle file directly. + define: { __APP_VERSION__: JSON.stringify(pkg.version) }, + // Brand images are imported with ?asset and copied into out/; don't leave them as a runtime require. + build: { externalizeDeps: { exclude: ["@agentstudio/brand"] } }, + }, + preload: {}, + renderer: { + root: resolve(__dirname, "src/renderer"), + plugins: [react()], + // Dev server stays on loopback like the rest of the local stack. + server: { host: "127.0.0.1" }, + build: { + rollupOptions: { input: resolve(__dirname, "src/renderer/index.html") }, + }, + }, +}); diff --git a/apps/desktop/package.json b/apps/desktop/package.json new file mode 100644 index 0000000..8fcf8e3 --- /dev/null +++ b/apps/desktop/package.json @@ -0,0 +1,39 @@ +{ + "name": "@agentstudio/desktop", + "version": "0.1.0", + "private": true, + "license": "Apache-2.0", + "description": "AgentStudio desktop app (Electron, macOS)", + "main": "out/main/index.js", + "scripts": { + "dev": "electron-vite dev", + "build": "electron-vite build", + "start": "electron-vite preview", + "typecheck": "tsc -p tsconfig.node.json --noEmit && tsc -p tsconfig.web.json --noEmit && tsc -p e2e-smoke/tsconfig.json --noEmit", + "test": "vitest run", + "e2e:smoke": "electron-vite build && playwright test -c e2e-smoke/playwright.config.ts" + }, + "dependencies": { + "@fontsource-variable/bricolage-grotesque": "5.3.0", + "@agentstudio/brand": "workspace:*", + "@agentstudio/core": "workspace:*", + "lucide-react": "^1.66.0", + "matrix-js-sdk": "45.0.0", + "qrcode": "1.5.4", + "react": "19.3.1", + "react-dom": "19.3.1" + }, + "devDependencies": { + "@playwright/test": "1.64.1", + "@types/node": "26.6.6", + "@types/qrcode": "1.5.6", + "@types/react": "19.3.0", + "@types/react-dom": "19.3.0", + "@vitejs/plugin-react": "6.2.1", + "electron": "45.2.1", + "electron-vite": "6.0.0", + "typescript": "7.0.2", + "vite": "8.3.5", + "vitest": "5.0.5" + } +} diff --git a/apps/desktop/resources/icon.icns b/apps/desktop/resources/icon.icns new file mode 100644 index 0000000..e796cf6 Binary files /dev/null and b/apps/desktop/resources/icon.icns differ diff --git a/apps/desktop/resources/icon.png b/apps/desktop/resources/icon.png new file mode 100644 index 0000000..65b19e7 Binary files /dev/null and b/apps/desktop/resources/icon.png differ diff --git a/apps/desktop/src/main/connectors/choices.ts b/apps/desktop/src/main/connectors/choices.ts new file mode 100644 index 0000000..f7ca53f --- /dev/null +++ b/apps/desktop/src/main/connectors/choices.ts @@ -0,0 +1,19 @@ +// Mapping an agent page's buttons to approval choices. Pure, so it is unit-tested. + +import type { Choice } from "./protocol"; + +const APPROVE = /^(approve|allow|yes|book|confirm|accept|continue|send|buy|pay|proceed|ok|go ahead|run)\b/i; +const DENY = /^(deny|reject|no|don'?t|do not|decline|cancel|skip|not now|stop|dismiss)\b/i; +/** Choice key for "open the service so the user can answer there". */ +export const OPEN = "__agentstudio_open"; + +/** Choices for an agent message's buttons, or null when they don't read as a yes/no decision. */ +export function buttonChoices(labels: string[]): Choice[] | null { + const choices: Choice[] = []; + for (const [i, label] of labels.entries()) { + const category = APPROVE.test(label) ? "approve" : DENY.test(label) ? "deny" : null; + if (category) choices.push({ key: `b${i}`, label, category }); + } + if (!choices.some((c) => c.category === "approve") || !choices.some((c) => c.category === "deny")) return null; + return choices.slice(0, 4); +} diff --git a/apps/desktop/src/main/connectors/cue.ts b/apps/desktop/src/main/connectors/cue.ts new file mode 100644 index 0000000..cc80788 --- /dev/null +++ b/apps/desktop/src/main/connectors/cue.ts @@ -0,0 +1,476 @@ +// Cue (cue.im, by Manus): personal agents, each with its own email, phone, +// wallet and computer. Cue has no web app, but Manus publishes an API that +// covers Cue's agents (https://open.manus.ai/docs/v2/agents-overview), so this +// connector uses it with an API key the user creates in Manus settings. Each +// agent has a main task, its conversation; AgentStudio reads it with +// task.listMessages and talks with task.sendMessage / task.confirmAction. + +import { createHash } from "node:crypto"; +import { askPrompt, parseDrafts, truncate, HISTORY_ON_CONNECT, MAX_TEXT, type AgentEvent, type AgentInfo, type Attachment, type Choice, type InboundMessage } from "./protocol"; +import { UnknownApprovalError, type AvatarImage, type Connector, type ConnectorState, type Emitter, type Trace } from "./types"; + +export const CUE_DEFAULTS = { + apiUrl: "https://api.manus.ai", + /** Where the user creates an API key. */ + settingsUrl: "https://manus.im/app#settings/developers", + /** Sign-in pages, and Manus's own API (used once, to create the key). */ + webUrl: "https://manus.im", + rpcUrl: "https://api.manus.im", +}; + +export type CueConfig = { + apiUrl: string; + pollActiveMs: number; + pollIdleMs: number; + /** task.listMessages allows 100 requests a minute per user; stay under it across all agents. */ + listBudgetPerMinute: number; + trace?: Trace; +}; + +export type CueAgentRef = { id: string; taskId: string; name: string; about: string }; +export type CueData = { apiKey: string; agent: CueAgentRef; since?: number }; + +export class CueAuthError extends Error {} + +/** + * Manus sign-in hands its session back on a cue:// or cuedev:// link. cue:// + * belongs to the Cue app; cuedev:// (Cue's dev builds) is free, so AgentStudio + * takes that one while a sign-in runs. + */ +export const CUE_SCHEME = "cuedev"; + +/** The token in a cuedev://oauth/desktop-callback link, if it is the answer to this sign-in. */ +export function callbackToken(raw: string, nonce: string): { token: string } | { error: true } | null { + let url: URL; + try { + url = new URL(raw); + } catch { + return null; + } + if (url.protocol !== `${CUE_SCHEME}:` || url.hostname !== "oauth" || url.pathname !== "/desktop-callback") return null; + if (url.searchParams.get("nonce") !== nonce) return null; + if (url.searchParams.has("error")) return { error: true }; + const token = url.searchParams.get("token"); + return token ? { token } : null; +} + + + +// ---- the API ------------------------------------------------------------------------- + +type Envelope = { ok?: boolean; error?: { code?: string; message?: string } } & Record; + +export type CueEvent = { + id: string; + type: string; + timestamp: number; + user_message?: { content?: string; attachments?: CueAttachment[] }; + assistant_message?: { + content?: string; + attachments?: CueAttachment[]; + question_expectation?: { options?: string[]; selection_mode?: string }; + }; + error_message?: { content?: string; error_type?: string }; + status_update?: { + agent_status?: "running" | "stopped" | "waiting" | "error"; + brief?: string; + status_detail?: { + waiting_for_event_id?: string; + waiting_for_event_type?: string; + waiting_description?: string; + confirm_input_schema?: { properties?: Record }; + }; + }; +}; +type CueAttachment = { type?: string; filename?: string; url?: string; content_type?: string }; + +/** A minimal sliding-window limiter, shared by every agent of one account. */ +class Budget { + private stamps: number[] = []; + constructor(private readonly perMinute: number) {} + async take(signal?: AbortSignal) { + for (;;) { + const now = Date.now(); + this.stamps = this.stamps.filter((t) => now - t < 60_000); + if (this.stamps.length < this.perMinute) { + this.stamps.push(now); + return; + } + await new Promise((r) => setTimeout(r, 60_000 - (now - this.stamps[0]!) + 50)); + if (signal?.aborted) return; + } + } +} + +export class CueClient { + private readonly listBudget: Budget; + constructor( + private readonly cfg: CueConfig, + readonly apiKey: string, + ) { + this.listBudget = new Budget(cfg.listBudgetPerMinute); + } + + private async call(method: "GET" | "POST", path: string, params: Record): Promise { + let url = `${this.cfg.apiUrl}/v2/${path}`; + const init: RequestInit = { + method, + headers: { "x-manus-api-key": this.apiKey, ...(method === "POST" ? { "content-type": "application/json" } : {}) }, + signal: AbortSignal.timeout(20_000), + }; + if (method === "GET") { + const q = new URLSearchParams(Object.entries(params).map(([k, v]) => [k, String(v)])); + url += `?${q}`; + } else { + init.body = JSON.stringify(params); + } + const res = await fetch(url, init); + const body = (await res.json().catch(() => ({}))) as Envelope; + this.cfg.trace?.("cue.api", { path, status: res.status, ok: body.ok, code: body.error?.code }); + if (res.status === 401 || res.status === 403 || body.error?.code === "unauthenticated" || body.error?.code === "permission_denied") { + throw new CueAuthError("The Manus API key was rejected; connect Cue again with a new key"); + } + if (!res.ok || body.ok === false) throw new Error(body.error?.message || `Manus API ${path}: HTTP ${res.status}`); + return body; + } + + /** The account the key belongs to (also checks the key). */ + async me(): Promise<{ name: string; email: string }> { + const r = await this.call("GET", "user.me", {}); + return { name: String(r.name ?? ""), email: String(r.email ?? "") }; + } + + /** Every agent of the account: the default IM agent ("Cue") and the ones created in the Cue app. */ + async listAgents(): Promise { + const r = await this.call("GET", "agent.list", {}); + const list = ((r.data as Record[] | undefined) ?? []).map(agentRef); + const im = await this.call("GET", "agent.detail", { agent_id: "agent-default" }) + .then((d) => agentRef((d.agent ?? {}) as Record)) + .catch(() => null); + if (im?.taskId && !list.some((a) => a.taskId === im.taskId)) list.unshift({ ...im, name: im.name || "Cue" }); + return list.filter((a) => a.taskId); + } + + /** The newest events of a task, oldest first. */ + async events(taskId: string, limit = 30): Promise { + await this.listBudget.take(); + const r = await this.call("GET", "task.listMessages", { task_id: taskId, order: "desc", limit }); + return ((r.messages as CueEvent[] | undefined) ?? []).slice().sort((a, b) => a.timestamp - b.timestamp); + } + + async send(taskId: string, text: string) { + await this.call("POST", "task.sendMessage", { task_id: taskId, message: { content: text } }); + } + + async confirm(taskId: string, eventId: string, input: Record) { + const r = await this.call("POST", "task.confirmAction", { task_id: taskId, event_id: eventId, input }); + if (r.confirmed === false) throw new Error("Cue didn't accept the answer"); + } +} + +function agentRef(a: Record): CueAgentRef { + return { id: String(a.id ?? ""), taskId: String(a.task_id ?? ""), name: String(a.nickname ?? ""), about: String(a.about ?? "") }; +} + +// ---- turning events into AgentStudio events (pure, unit-tested) ------------------------------- + +const DENY_WORDS = /^(no|don'?t|do not|cancel|skip|reject|deny|decline|not now|stop)\b/i; +/** Choice key meaning "leave the question for the Cue app". */ +export const CUE_DISMISS = "__agentstudio_dismiss"; + +/** Approval choices for an agent question's options. */ +export function questionChoices(options: string[]): Choice[] { + const choices: Choice[] = options.slice(0, 6).map((o, i) => ({ key: `o${i}`, label: truncate(o, 80), category: DENY_WORDS.test(o) ? "deny" : "approve" })); + if (!choices.some((c) => c.category === "deny")) choices.push({ key: CUE_DISMISS, label: "Answer in Cue", category: "deny" }); + return choices; +} + +/** What to send for an action confirmation, keyed by choice; null when AgentStudio can't answer it. */ +export function confirmationChoices( + detail: NonNullable["status_detail"]>, +): { choices: Choice[]; inputs: Record> } | null { + const props = detail.confirm_input_schema?.properties ?? {}; + if (props.accept?.type === "boolean" || !detail.confirm_input_schema) { + return { + choices: [ + { key: "accept", label: "Approve", category: "approve" }, + { key: "reject", label: "Deny", category: "deny" }, + ], + inputs: { accept: { accept: true }, reject: { accept: false } }, + }; + } + for (const field of ["action", "choice"]) { + const values = (props[field]?.enum ?? []).filter((v): v is string => typeof v === "string"); + if (values.length > 0 && values.length <= 4) { + const choices: Choice[] = values.map((v) => ({ key: v, label: v.replace(/_/g, " ").replace(/^./, (c) => c.toUpperCase()), category: DENY_WORDS.test(v) ? "deny" : "approve" })); + if (!choices.some((c) => c.category === "approve")) choices[0]!.category = "approve"; + if (!choices.some((c) => c.category === "deny")) choices[choices.length - 1]!.category = "deny"; + return { choices, inputs: Object.fromEntries(values.map((v) => [v, { [field]: v }])) }; + } + } + return null; +} + +/** A message's files, for the bridge to bring over (signed URLs; images show inline). */ +export function attachmentsOf(list: CueAttachment[] | undefined): Attachment[] { + return (list ?? []) + .filter((a) => a.url && /^https?:\/\//.test(a.url)) + .slice(0, 10) + .map((a) => ({ url: a.url!, name: a.filename || a.type || "file", ...(a.content_type ? { mime_type: a.content_type } : {}) })); +} + +/** The chat message for a Cue event (an answer, or what the user said in the Cue app), or null. */ +export function messageOf(e: CueEvent): Extract | null { + if (e.type === "assistant_message") { + const m = e.assistant_message ?? {}; + const text = (m.content ?? "").trim(); + const attachments = attachmentsOf(m.attachments); + if (!text && attachments.length === 0) return null; + return { type: "message", text: truncate(text, MAX_TEXT), ...(attachments.length ? { attachments } : {}) }; + } + if (e.type === "user_message") { + const m = e.user_message ?? {}; + const text = (m.content ?? "").trim(); + const attachments = attachmentsOf(m.attachments); + if (!text && attachments.length === 0) return null; + return { type: "message", text: truncate(text, MAX_TEXT), from_user: true, via: "Cue", ...(attachments.length ? { attachments } : {}) }; + } + return null; +} + +/** The last messages of a conversation, as history (oldest first). */ +export function historyOf(events: CueEvent[], max = HISTORY_ON_CONNECT): Extract[] { + const out: Extract[] = []; + for (const e of events) { + const m = messageOf(e); + if (m) out.push({ ...m, history: true, ts: new Date(e.timestamp).toISOString() }); + } + return out.slice(-max); +} + +// ---- the connector ------------------------------------------------------------------------ + +type Pending = + | { kind: "question"; options: string[] } + | { kind: "confirm"; eventId: string; inputs: Record> }; + +export class CueConnector implements Connector { + private emitter: Emitter | null = null; + private running = false; + private status: ConnectorState = { status: "starting" }; + private timer: NodeJS.Timeout | null = null; + private activeUntil = 0; + private seen = new Set(); + private ownTexts = new Set(); + private pending = new Map(); + private pendingAsk: { taskId: string; texts: string[] } | null = null; + + constructor( + private readonly cfg: CueConfig, + private data: CueData, + private readonly client: CueClient, + private readonly onChange: () => void, + /** The Cue app icon: the Manus API doesn't expose agents' avatars. */ + private readonly fallbackAvatar: () => Promise, + ) {} + + info(): AgentInfo { + return { + name: this.data.agent.name || "Cue", + description: this.data.agent.about || "Your Cue agent, through your own Manus account.", + capabilities: ["chat", "ask", "approvals"], + protocol: "agentstudio-agent/0", + }; + } + + state(): ConnectorState { + return this.status; + } + + serialize(): unknown { + return this.data; + } + + async avatar(): Promise { + return this.fallbackAvatar(); + } + + async start(emitter: Emitter): Promise { + this.emitter = emitter; + this.running = true; + try { + await this.refreshProfile(); + // Earlier events are history, not new messages. On the first connect, + // the last few come over as history so the chat isn't empty. + const events = await this.client.events(this.data.agent.taskId, 50); + for (const e of events) this.seen.add(e.id); + if (this.data.since === undefined) { + for (const m of historyOf(events)) emitter.emit(m); + this.data.since = Math.max(0, ...events.map((e) => e.timestamp)); + this.onChange(); + } + this.status = { status: "ready" }; + } catch (err) { + this.status = { status: err instanceof CueAuthError ? "needs_login" : "error", message: String((err as Error).message) }; + } + this.schedule(0); + } + + async stop(): Promise { + this.running = false; + if (this.timer) clearTimeout(this.timer); + } + + async send(msg: InboundMessage): Promise { + const text = msg.ask ? askPrompt(msg.ask) : msg.text; + if (msg.ask) this.pendingAsk = { taskId: msg.ask.task_id, texts: [] }; + this.ownTexts.add(text.trim()); + await this.client.send(this.data.agent.taskId, text); + // A plain message answers (or skips) whatever Cue was waiting for. + for (const id of this.pending.keys()) this.emitter?.emit({ type: "approval_cancel", external_id: id }); + this.pending.clear(); + this.emitter?.emit({ type: "status", state: "working", text: "Working" }); + this.wake(); + } + + async decide(externalId: string, choice: string): Promise { + const p = this.pending.get(externalId); + if (!p) throw new UnknownApprovalError(); + this.pending.delete(externalId); + if (p.kind === "question") { + if (choice === CUE_DISMISS) return; + const option = p.options[Number(choice.slice(1))]; + if (option === undefined) throw new Error("unknown choice"); + this.ownTexts.add(option.trim()); + await this.client.send(this.data.agent.taskId, option); + } else { + const input = p.inputs[choice]; + if (!input) throw new Error("unknown choice"); + await this.client.confirm(this.data.agent.taskId, p.eventId, input); + } + this.wake(); + } + + /** Picks up a new name or description set in Cue. */ + private async refreshProfile() { + const list = await this.client.listAgents().catch(() => null); + const fresh = list?.find((a) => a.taskId === this.data.agent.taskId); + if (!fresh) return; + const cur = this.data.agent; + if (fresh.name === cur.name && fresh.about === cur.about) return; + this.data.agent = { ...cur, ...fresh, name: fresh.name || cur.name }; + this.onChange(); + this.emitter?.profileChanged(); + } + + private wake() { + this.activeUntil = Date.now() + 10 * 60_000; + this.schedule(this.cfg.pollActiveMs); + } + + private schedule(ms: number) { + if (!this.running) return; + if (this.timer) clearTimeout(this.timer); + this.timer = setTimeout(() => void this.tick(), ms); + } + + private async tick() { + if (!this.running) return; + try { + this.process(await this.client.events(this.data.agent.taskId)); + this.status = { status: "ready" }; + } catch (err) { + this.status = { status: err instanceof CueAuthError ? "needs_login" : "error", message: String((err as Error).message) }; + this.cfg.trace?.("cue.poll_error", { message: this.status.message }); + } + this.schedule(Date.now() < this.activeUntil ? this.cfg.pollActiveMs : this.cfg.pollIdleMs); + } + + private process(events: CueEvent[]) { + const emit = (e: Parameters[0]) => this.emitter?.emit(e); + for (const e of events) { + if (this.seen.has(e.id)) continue; + this.seen.add(e.id); + if (e.timestamp <= (this.data.since ?? 0)) continue; + this.cfg.trace?.("cue.event", { type: e.type, status: e.status_update?.agent_status, waiting: e.status_update?.status_detail?.waiting_for_event_type }); + switch (e.type) { + case "user_message": { + const text = (e.user_message?.content ?? "").trim(); + if (this.ownTexts.delete(text)) break; + // Said in the Cue app: show it, and whatever AgentStudio was asking is moot. + const m = messageOf(e); + if (m) emit(m); + for (const id of this.pending.keys()) emit({ type: "approval_cancel", external_id: id }); + this.pending.clear(); + break; + } + case "assistant_message": { + const m = e.assistant_message ?? {}; + const options = m.question_expectation?.options ?? []; + if (options.length > 0) { + const id = `q:${e.id}`; + this.pending.set(id, { kind: "question", options }); + emit({ type: "approval_request", external_id: id, title: truncate((m.content ?? "").trim() || "Cue has a question", 300), choices: questionChoices(options) }); + break; + } + const msg = messageOf(e); + if (!msg) break; + if (this.pendingAsk && msg.text) this.pendingAsk.texts.push(msg.text); + else emit(msg); + break; + } + case "error_message": + emit({ type: "message", text: `⚠️ ${e.error_message?.content || "Cue ran into an error."}` }); + emit({ type: "status", state: "idle" }); + break; + case "status_update": { + const s = e.status_update ?? {}; + if (s.agent_status === "running") { + emit({ type: "status", state: "working", text: truncate(s.brief || "Working", 80) }); + this.activeUntil = Math.max(this.activeUntil, Date.now() + 10 * 60_000); + break; + } + if (s.agent_status === "waiting") { + const d = s.status_detail ?? {}; + const type = d.waiting_for_event_type ?? ""; + if (type !== "messageAskUser" && type !== "cascadeAskUser" && d.waiting_for_event_id) { + const c = confirmationChoices(d); + const id = `c:${d.waiting_for_event_id}`; + if (c && !this.pending.has(id)) { + this.pending.set(id, { kind: "confirm", eventId: d.waiting_for_event_id, inputs: c.inputs }); + emit({ type: "approval_request", external_id: id, title: truncate(d.waiting_description || "Cue needs your confirmation", 300), choices: c.choices }); + } else if (!c) { + emit({ type: "message", text: `Cue is waiting for you: ${d.waiting_description || type}. Answer it in the Cue app.` }); + } + } + } + if (s.agent_status === "stopped" || s.agent_status === "error" || s.agent_status === "waiting") { + this.flushAsk(); + emit({ type: "status", state: "idle" }); + } + break; + } + } + } + // Remember how far we got (server time), so a restart doesn't replay. + const latest = Math.max(this.data.since ?? 0, ...events.map((e) => e.timestamp)); + if (latest !== this.data.since) { + this.data.since = latest; + this.onChange(); + } + // Keep the memory of seen events bounded. + if (this.seen.size > 2000) this.seen = new Set([...this.seen].slice(-1000)); + } + + private flushAsk() { + const ask = this.pendingAsk; + if (!ask || ask.texts.length === 0) return; + this.pendingAsk = null; + this.emitter?.emit({ type: "drafts", task_id: ask.taskId, drafts: parseDrafts(ask.texts[ask.texts.length - 1]!).map((text) => ({ text })) }); + } +} + +/** A stable key for one API key, to share a client between its agents without keeping the key in a map key. */ +export function cueAccountKey(apiKey: string): string { + return createHash("sha256").update(apiKey).digest("hex").slice(0, 16); +} diff --git a/apps/desktop/src/main/connectors/cueweb.ts b/apps/desktop/src/main/connectors/cueweb.ts new file mode 100644 index 0000000..fb4828e --- /dev/null +++ b/apps/desktop/src/main/connectors/cueweb.ts @@ -0,0 +1,176 @@ +// Signing in to Cue with the user's Manus account, the way the Cue desktop app +// does: manus.im/login runs in the user's browser (Google and Apple refuse +// sign-in inside embedded windows), and the page hands a session token back on +// a cuedev:// link, which a small helper app catches for AgentStudio (see +// ensureLinkCatcher). The Cue app doesn't need to be installed. With the session AgentStudio creates a Manus +// API key named "AgentStudio", which the connector then uses with the documented +// API. Only that one call uses Manus's internal API. + +import { randomBytes } from "node:crypto"; +import { execFile } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { CUE_SCHEME as SCHEME, callbackToken } from "./cue"; + +export type CueWebConfig = { + /** manus.im: the sign-in pages. */ + webUrl: string; + /** api.manus.im: Manus's own Connect RPC API (the API-key call). */ + rpcUrl: string; +}; + +/** Sign-ins waiting for their callback link, by nonce. */ +const waiting = new Map void>(); + +/** + * Hands a cuedev:// link to the sign-in waiting for it (from the link catcher, + * a test window, or app "open-url"). Returns true when it was one. + */ +export function handleCueLink(url: string): boolean { + if (!url.startsWith(`${SCHEME}:`)) return false; + for (const [nonce, deliver] of waiting) { + if (callbackToken(url, nonce)) { + deliver(url); + return true; + } + } + return true; +} + +const LS_REGISTER = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister"; +const CATCHER_ID = "org.agentstudio.link-catcher"; +const CATCHER_VERSION = "3"; // 3: bundle id org.agentstudio.link-catcher + +/** + * A tiny AppleScript app that claims cuedev:// and writes each link it gets to + * a file AgentStudio reads. AgentStudio can't claim the scheme itself: unpackaged + * Electron shares one bundle id between every copy on the Mac, so macOS may + * hand the link to another one. Built once (osacompile) into userData. + */ +export async function ensureLinkCatcher(dir: string): Promise<{ inbox: string }> { + const appPath = join(dir, "AgentStudio Link Catcher.app"); + const inbox = join(dir, "inbox.txt"); + const stamp = join(dir, "version"); + const current = existsSync(stamp) ? readFileSync(stamp, "utf8") : ""; + if (current !== CATCHER_VERSION || !existsSync(appPath)) { + rmSync(appPath, { recursive: true, force: true }); + // Earlier catchers under another name share the bundle id and write elsewhere: remove them. + for (const old of existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith(" Link Catcher.app")) : []) { + await run(LS_REGISTER, ["-u", join(dir, old)]).catch(() => {}); + rmSync(join(dir, old), { recursive: true, force: true }); + } + mkdirSync(dir, { recursive: true }); + const script = join(dir, "catcher.applescript"); + const quoted = (s: string) => `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; + writeFileSync( + script, + `on open location theURL\n\tdo shell script "printf %s " & quoted form of theURL & " > " & quoted form of ${quoted(inbox)}\nend open location\n`, + ); + await run("osacompile", ["-o", appPath, script]); + const plist = join(appPath, "Contents", "Info.plist"); + const pb = (...cmds: string[]) => run("/usr/libexec/PlistBuddy", [...cmds.flatMap((c) => ["-c", c]), plist]); + await pb("Delete :CFBundleIdentifier").catch(() => {}); + await pb( + `Add :CFBundleIdentifier string ${CATCHER_ID}`, + "Add :LSUIElement bool true", + "Add :CFBundleURLTypes array", + "Add :CFBundleURLTypes:0 dict", + "Add :CFBundleURLTypes:0:CFBundleURLName string AgentStudio sign-in", + "Add :CFBundleURLTypes:0:CFBundleURLSchemes array", + `Add :CFBundleURLTypes:0:CFBundleURLSchemes:0 string ${SCHEME}`, + ); + await run("codesign", ["-f", "-s", "-", appPath]); + writeFileSync(stamp, CATCHER_VERSION); + } + await run(LS_REGISTER, ["-f", appPath]); + // Make it the handler, even if something else claimed the scheme before. + await run("osascript", [ + "-l", + "JavaScript", + "-e", + `ObjC.import("CoreServices"); $.LSSetDefaultHandlerForURLScheme($("${SCHEME}"), $("${CATCHER_ID}"))`, + ]); + return { inbox }; +} + +function run(cmd: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + execFile(cmd, args, { timeout: 30_000 }, (err, _out, stderr) => (err ? reject(new Error(`${cmd}: ${stderr || err.message}`)) : resolve())); + }); +} + +/** Reads links the catcher writes, until stopped. */ +function watchInbox(inbox: string): () => void { + rmSync(inbox, { force: true }); + const t = setInterval(() => { + if (!existsSync(inbox)) return; + const url = readFileSync(inbox, "utf8").trim(); + rmSync(inbox, { force: true }); + if (url) handleCueLink(url); + }, 400); + return () => clearInterval(t); +} + +/** The Manus sign-in page for a nonce. */ +export function signInUrl(cfg: CueWebConfig, nonce: string): string { + const login = new URL(`${cfg.webUrl}/login`); + const back = new URL(`${cfg.webUrl}/login-success`); + back.searchParams.set("desktopAgents", "1"); + back.searchParams.set("agentsScheme", SCHEME); + back.searchParams.set("nonce", nonce); + for (const [k, v] of [["from", "desktop"], ["type", "signIn"], ["desktopAgents", "1"], ["nonce", nonce], ["redirectUrl", back.href]]) { + login.searchParams.set(k!, v!); + } + return login.href; +} + +/** + * Opens Manus sign-in (with openExternal: the browser, or a test window) and + * resolves with the session token once the callback link arrives. + */ +export async function cueWebSignIn( + cfg: CueWebConfig, + openExternal: (url: string) => Promise, + signal: AbortSignal, + /** Where to keep the link catcher; null when sign-in runs in an AgentStudio window (tests). */ + catcherDir: string | null, +): Promise { + const nonce = randomBytes(32).toString("hex"); + const stop = catcherDir ? watchInbox((await ensureLinkCatcher(catcherDir)).inbox) : () => {}; + try { + return await new Promise((resolve, reject) => { + const onAbort = () => reject(new Error("cancelled")); + signal.addEventListener("abort", onAbort, { once: true }); + waiting.set(nonce, (url) => { + const r = callbackToken(url, nonce); + if (r && "token" in r) resolve(r.token); + else reject(new Error("Signing in to Manus wasn't completed.")); + }); + openExternal(signInUrl(cfg, nonce)).catch(reject); + // The browser may ask "Open AgentStudio?"; the user has the whole sign-in window to answer. + setTimeout(() => reject(new Error("Signing in timed out.")), 15 * 60_000); + }); + } finally { + waiting.delete(nonce); + stop(); + } +} + +/** Creates a Manus API key with a signed-in session. */ +export async function createCueApiKey(cfg: CueWebConfig, sessionToken: string): Promise { + const res = await fetch(`${cfg.rpcUrl}/openapi.v1.OpenapiService/CreateAPIKey`, { + method: "POST", + headers: { + "content-type": "application/json", + "connect-protocol-version": "1", + authorization: `Bearer ${sessionToken}`, + "x-client-type": "web", + }, + body: JSON.stringify({ name: `AgentStudio (${new Date().toISOString().slice(0, 10)})` }), + signal: AbortSignal.timeout(20_000), + }); + const body = (await res.json().catch(() => ({}))) as { apiKeyData?: { apiKey?: string }; message?: string }; + const key = body.apiKeyData?.apiKey; + if (!res.ok || !key) throw new Error(body.message ? `Manus: ${body.message}` : `Couldn't create a Manus API key (HTTP ${res.status})`); + return key; +} diff --git a/apps/desktop/src/main/connectors/drivers.ts b/apps/desktop/src/main/connectors/drivers.ts new file mode 100644 index 0000000..066fdbc --- /dev/null +++ b/apps/desktop/src/main/connectors/drivers.ts @@ -0,0 +1,202 @@ +// Page drivers: JavaScript that runs inside a service's own web page (in an +// isolated world) to read the conversation and to send a message. They are +// strings, not functions, so the bundler can't rewrite them. +// +// Every driver defines the same three entry points on globalThis.__agentstudio: +// snapshot() -> { signedIn, ready, busy, messages: [{ id, role, text, actions: [label] }], problem } +// (problem: a connection error the page shows, or null) +// submit(text) -> { ok, error? } +// click(messageId, label) -> { ok, error? } +// +// The Muse selectors come from beeper-muse (https://github.com/nishu-builder/beeper-muse, +// MIT, Copyright (c) 2026 nishu-builder), which validated them against live Muse. + +const COMMON = String.raw` +const visible = (el) => { const r = el.getBoundingClientRect(); return r.width > 0 && r.height > 0; }; +// A connection problem the page itself shows ("Connection error", "Reconnecting…"). +const PROBLEM = /^(connection error|connection lost|reconnecting|you'?re offline|offline|network error|unable to connect|can'?t connect|something went wrong)\b/i; +const pageProblem = () => { + const w = document.createTreeWalker(document.body || document.documentElement, NodeFilter.SHOW_TEXT); + for (let n = w.nextNode(), i = 0; n && i < 4000; n = w.nextNode(), i++) { + const t = (n.textContent || '').trim(); + if (t.length > 0 && t.length < 40 && PROBLEM.test(t) && n.parentElement && visible(n.parentElement) && !n.parentElement.closest('[data-message-item],[data-message-author-role],[data-user-message-actions],[data-assistant-message-actions],.prose')) return t; + } + return null; +}; +const SKIP = 'button,svg,script,style,[aria-hidden="true"],[role="toolbar"],[role="status"],[data-message-actions],.sr-only,textarea,input'; +const BLOCK = /^(P|DIV|LI|UL|OL|H[1-6]|PRE|BLOCKQUOTE|TABLE|TR|SECTION|ARTICLE|HR)$/; +// Text with paragraph and list breaks kept. innerText can't be used on a detached +// clone (no layout), so walk the live tree and skip chrome. +const clean = (el) => { + const out = []; + const walk = (n, depth) => { + if (n.nodeType === 3) { + const ws = n.parentElement ? getComputedStyle(n.parentElement).whiteSpace : 'normal'; + out.push(/pre/.test(ws) ? n.textContent : n.textContent.replace(/[ \t\r\n]+/g, ' ')); + return; + } + if (n.nodeType !== 1 || n.matches(SKIP)) return; + if (n.tagName === 'BR') { out.push('\n'); return; } + const block = BLOCK.test(n.tagName); + if (block) out.push('\n'); + if (n.tagName === 'LI') out.push(n.parentElement && n.parentElement.tagName === 'OL' ? (([...n.parentElement.children].indexOf(n) + 1) + '. ') : '• '); + if (n.tagName === 'PRE') { out.push(n.textContent); out.push('\n'); return; } + for (const c of n.childNodes) walk(c, depth + 1); + if (block) out.push('\n'); + }; + walk(el, 0); + return out.join('').split('\n').map((l) => l.trim()).join('\n').replace(/\n{3,}/g, '\n\n').trim(); +}; +// Chat chrome, not decisions. +const CHROME = /^(more options|more|react|reply|copy|copy response|share|like|dislike|good response|bad response|edit|retry|regenerate|try again|read aloud|stop|thumbs up|thumbs down|open generated image.*)$/i; +const buttonsIn = (el) => [...el.querySelectorAll('button,[role="button"]')] + .filter(visible) + .filter((b) => !b.closest('[data-message-actions],[role="toolbar"]')) + .map((b) => (b.getAttribute('aria-label') || b.innerText || '').trim()) + .filter((t) => t && t.length <= 60 && !CHROME.test(t)); +const setNative = (field, value) => { + const proto = field instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype; + Object.getOwnPropertyDescriptor(proto, 'value').set.call(field, value); + field.dispatchEvent(new Event('input', { bubbles: true })); +}; + +// The persona's avatar: an labelled with its name (outside the message log), +// or a frame of an animated avatar