From 02a68b30a53da7396d81c4f8fd585dcf436ebe8c Mon Sep 17 00:00:00 2001 From: Owain Lewis Date: Wed, 29 Jul 2026 14:57:19 +0100 Subject: [PATCH] docs: add root architecture reference --- ARCHITECTURE.md | 826 +++++++++++++++++++++++++++++++++++++ README.md | 1 + docs/architecture.md | 422 ------------------- docs/contributing.md | 4 +- docs/core-system/design.md | 4 +- docs/getting-started.md | 2 +- docs/index.md | 4 +- mkdocs.yml | 2 +- tests/docs.rs | 2 +- 9 files changed, 838 insertions(+), 429 deletions(-) create mode 100644 ARCHITECTURE.md delete mode 100644 docs/architecture.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..a63c75c --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,826 @@ +# Push Architecture + +## 1. Executive Summary + +Push is a local Rust gateway that turns Claude Code, Codex, or Pi into a +personal assistant available through iMessage, Telegram, and Slack. It also +runs validated Markdown jobs on a schedule. Push owns messaging, routing, +durable history, scheduling, recovery, and delivery. The selected agent owns +reasoning, tools, skills, permissions, MCP servers, and authentication. + +The central boundary is: + +```text +message channel -> Push gateway -> agent CLI -> Push gateway -> message channel +``` + +Push is one binary and one local process. It opens no inbound server port. +iMessage reads the local Messages database, Telegram uses outbound HTTPS long +polling, and Slack uses an outbound Socket Mode WebSocket. + +The main architectural rule is that agent runtimes are disposable adapters. +Push does not implement an agent loop, tool runner, plugin system, or model +client. It gives the selected agent a request, receives a final answer, stores +that answer, and delivers it. + +### System Architecture + +```text +┌──────────────────────────────────────────────────────────────────────────┐ +│ INPUTS │ +│ │ +│ iMessage chat.db Telegram Bot API Slack Socket Mode │ +│ │ │ │ │ +│ └──────────── poll / receive outbound connections ───────────────┘│ +└────────────────────────────────────┬─────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────────────────┐ +│ push process │ +│ │ +│ ┌──────────────┐ ┌───────────────┐ ┌───────────────────────────────┐ │ +│ │ Channel │ │ GatewayGroup │ │ Scheduler │ │ +│ │ contract │─▶│ per-channel │ │ validated jobs + delivery │ │ +│ └──────────────┘ │ loops │ └───────────────────────────────┘ │ +│ └───────┬───────┘ │ +│ ▼ │ +│ ┌───────────────┐ │ +│ │ per-thread │ │ +│ │ worker queues │ │ +│ └───────┬───────┘ │ +│ ▼ │ +│ ┌──────────────┐ ┌───────────────┐ ┌───────────────────────────────┐ │ +│ │ state.json │ │ push.db │ │ audit.jsonl │ │ +│ │ cursors and │ │ conversations │ │ redacted operational events │ │ +│ │ sessions │ │ jobs/delivery │ │ │ │ +│ └──────────────┘ └───────────────┘ └───────────────────────────────┘ │ +└────────────────────────────────────┬─────────────────────────────────────┘ + │ subprocess + ▼ +┌──────────────────────────────────────────────────────────────────────────┐ +│ AGENT RUNTIMES │ +│ │ +│ claude -p codex exec pi --print │ +└────────────────────────────────────┬─────────────────────────────────────┘ + │ final response + ▼ + originating channel only +``` + +### Module Dependency Hierarchy + +Push is a single crate. Module boundaries, not separate packages, keep +responsibilities isolated. + +```text +main + ├── config + doctor + assistant init + ├── gateway + │ ├── channel + │ │ ├── imessage + │ │ ├── telegram + │ │ └── slack + │ ├── agent + │ │ ├── claude + │ │ ├── codex + │ │ └── pi + │ ├── store + │ ├── history + │ ├── audit + │ ├── rehydration + │ ├── voice + │ └── approval answer handling + └── jobs + ├── catalog + validation + ├── scheduler + ├── ledger in push.db + ├── agent runner + └── evaluator +``` + +The gateway coordinates the modules. Channel implementations do not call agent +backends directly, and backend adapters do not know about messaging providers. + +--- + +## 2. Runtime Contracts + +Push divides ownership among three systems: + +| Owner | Responsibilities | +| --- | --- | +| Push runtime | channels, allowlists, routing, scheduling, canonical history, cursors, session mappings, audit, retries, and delivery | +| Assistant repository | `SOUL.md`, shared instructions, context, evals, jobs, and optional project skills | +| Agent runtime | reasoning, models, tools, global skills, MCP, authentication, sandbox, and interactive permissions | + +### Channel Contract + +Every built-in channel implements the closed `ChannelContract` in +[`src/channel.rs`](src/channel.rs). The `Channel` enum provides static +dispatch. This is an internal compile-time boundary, not a dynamic plugin API. + +A channel owns: + +- polling after an opaque monotonic cursor; +- first-start cursor discovery so old messages are skipped; +- sender, chat, group, content, and loopback acceptance rules; +- a stable channel-qualified thread key; +- the exact reply target; +- routing aliases for parent and legacy thread identities; +- outbound chunking, formatting, and provider limits; +- typing indicators; +- voice download and upload; +- delivery timeouts and retries; +- worker shutdown grace. + +The shared gateway never branches on provider names for polling, routing, +worker ordering, reply recovery, or shutdown. + +### Agent Contract + +The gateway calls backends through the boundary in +[`src/agent.rs`](src/agent.rs): + +```rust +Request { + session_id, + is_new, + work_dir, + instructions, + prompt, +} + +RunOutput { + reply, + session_id, +} +``` + +`RunError` separates timeouts, missing sessions, and other failures so the +worker can recover or produce the correct stored fallback response. + +### Durable State Contract + +Push uses separate stores because they have different update and query needs: + +| Store | Purpose | +| --- | --- | +| `state.json` | small atomically replaced map of channel cursors and backend session IDs | +| `push.db` | transactional conversation, delivery, approval compatibility, and job-run history | +| `.slack-inbox.db` | Slack Socket Mode inbox committed before envelope acknowledgement | +| `audit.jsonl` | append-only operational audit events | +| assistant Git repository | user-owned identity, context, evals, jobs, and project resources | + +Runtime databases, secrets, locks, sessions, and logs must stay outside the +assistant repository. + +--- + +## 3. Process Lifecycle + +The entry point is [`src/main.rs`](src/main.rs). + +### Step 1: Parse the Command + +The binary supports the long-running gateway plus `init`, `doctor`, `reload`, +and job management commands. Commands that need configuration resolve +`--config`, defaulting to `~/.push/config.toml`. + +### Step 2: Load and Validate Configuration + +[`src/config.rs`](src/config.rs) parses TOML, flattens supported provider +sections, expands paths, migrates compatible legacy layout, and rejects: + +- unsupported channels or backends; +- missing allowlists and credentials; +- removed Push-owned agent permission settings; +- unsafe overlaps between the assistant repository and runtime state; +- unsafe job work directories; +- provider tokens stored inside the assistant repository; +- invalid routes, durations, and primary delivery targets. + +`push doctor` adds environment checks for agent binaries, database access, +channel credentials, local paths, job validity, and optional voice support. +The gateway runs the required preflight checks before starting. + +### Step 3: Build Shared State + +`GatewayGroup::new` opens one shared `Store`, one shared `History`, one runner +per required backend, and one serialized audit-log lock. It then creates one +`Gateway` for each enabled channel. + +Each gateway owns its own: + +- channel adapter; +- polling loop; +- acknowledgement tracker; +- per-thread queue map; +- worker handles. + +### Step 4: Start Channel and Scheduler Tasks + +`GatewayGroup::run` starts: + +1. one task per enabled channel; +2. one scheduler task; +3. one coordinator waiting for shutdown or channel termination. + +The scheduler starts in full mode only when `primary_delivery` resolves to an +enabled, allowlisted target. Otherwise it disables new cron enqueues while +continuing recovery of previously queued runs and persisted delivery work. + +### Step 5: Establish Initial Cursors + +On the first run for a channel, Push asks the adapter for its latest cursor and +persists it. Existing backlog is skipped. Later starts resume from the stored +cursor. + +### Step 6: Poll Until Shutdown + +Each gateway polls on its configured interval. Missed timer ticks are skipped, +so a slow poll does not create a burst of catch-up polls. A failed provider +poll is logged and retried on the next interval. + +### Step 7: Drain + +SIGINT or SIGTERM broadcasts shutdown. Pending poll futures are dropped, queue +senders are closed, accepted per-thread work drains for the channel's grace +period, and remaining workers are aborted. The scheduler also gets a bounded +grace period before releasing delivery claims and recovering interrupted work. + +--- + +## 4. Message Pipeline + +The main pipeline lives in [`src/gateway/mod.rs`](src/gateway/mod.rs) and +[`src/gateway/worker.rs`](src/gateway/worker.rs). + +```text + 1. POLL read messages after durable channel cursor + 2. DEDUPLICATE skip old or already in-flight channel rows + 3. AUTHORIZE apply provider allowlist and message rules + 4. ANSWER RESOLUTION consume a matching durable numbered answer, if any + 5. HISTORY INSERT persist accepted inbound message in push.db + 6. ROUTE select backend by exact thread, parent, channel, default + 7. ENQUEUE place message on its per-thread bounded queue + 8. PREPARE load SOUL.md, resolved paths, session, optional voice + 9. REHYDRATE add bounded canonical history for a fresh session +10. RUN BACKEND invoke the selected agent CLI with a timeout +11. HISTORY INSERT persist generated outbound response +12. SESSION SAVE persist backend-owned session ID when required +13. DELIVER send durable chunks through the originating channel +14. ACK CURSOR advance across the contiguous completed row prefix +``` + +### Authorization Before Work + +Channel `accept` runs before transcription, routing, backend dispatch, or +delivery. Rejected, group, looped-back, empty, unsupported, or non-allowlisted +messages are audited and completed without reaching an agent. + +### Persist Before Dispatch + +Accepted inbound messages are inserted in `push.db` before local commands or +backend execution. `channel_event_id` is unique, so a retried provider event +resolves to the same canonical inbound row. + +### Routing Order + +Backend selection has this precedence: + +1. exact thread or topic route; +2. parent Telegram private-chat route for a topic; +3. channel route; +4. root `agent`. + +Session storage uses the final channel-qualified thread key. A backend change +for the same thread starts a new session instead of resuming another runtime's +session. + +### Per-Thread Queue + +Each thread gets an MPSC queue with depth 32. One worker drains it in order. +Different threads and channels may run concurrently. If a queue is full, Push +stores and sends a gateway-authored retry message rather than silently dropping +the input. If a worker closes unexpectedly, retained jobs are moved to a +replacement worker. + +### Agent Preparation + +For every turn, the worker: + +1. canonicalizes `assistant_root`; +2. loads `SOUL.md`; +3. appends gateway-owned absolute assistant paths in memory; +4. validates the optional `context/` boundary; +5. resolves or creates the backend session mapping. + +Push does not inject every context file. The selected backend decides what to +inspect in the assistant repository. + +### Session Rehydration + +Normal resumed turns contain only the new user request. Fresh sessions include +at most 20 recent messages from the exact channel-qualified conversation. +Each historical message is capped at 4 KiB and the history block at 16 KiB. +Roles and content are JSON-delimited so historical text remains prompt content, +not system instructions. + +If a backend reports that a resumed session is missing, Push rotates the stored +session and retries once as a fresh, rehydrated session. + +### Persist Before Delivery + +Backend, command, timeout, interruption, and failure responses are inserted as +outbound messages before channel delivery. Generation state and delivery state +are separate. Worker-managed replies retry the stored outbound record, and +restart recovery resends it without rerunning the agent. + +The queue-full fallback is a deliberate exception. It records the reply, makes +one delivery attempt, and completes the input row even if that attempt fails. +This prevents an overloaded thread from blocking its channel cursor. + +Each successful outbound chunk advances a durable chunk checkpoint. A crash +between provider acceptance and checkpoint persistence may create an +at-least-once duplicate, because provider sends and local SQLite commits cannot +form one atomic transaction. + +### Cursor Advancement + +Poll rows can finish out of order because separate thread workers run in +parallel. `AckState` tracks in-flight and completed row IDs. Push advances the +channel cursor only across a contiguous completed prefix, so unfinished older +work is never skipped. + +--- + +## 5. Channel System + +### iMessage + +Files: [`src/imessage/`](src/imessage/) + +- Reads the macOS Messages `chat.db` database. +- Accepts one-to-one self chats and explicitly allowed senders. +- Uses local row IDs as the monotonic cursor. +- Sends through `osascript`. +- Keeps legacy unprefixed route and session aliases for migration. +- Supports text only. + +### Telegram + +File: [`src/telegram.rs`](src/telegram.rs) + +- Uses outbound Bot API long polling. +- Authorizes stable numeric user and private-chat IDs. +- Preserves private-chat topic IDs in thread keys and reply targets. +- Splits rich Markdown within Telegram's 4,096-character limit. +- Refreshes typing activity during long runs. +- Downloads voice notes only after allowlist acceptance. +- Sends text and optional generated voice replies. + +### Slack + +File: [`src/slack.rs`](src/slack.rs) + +- Connects with an outbound authenticated Socket Mode WebSocket. +- Verifies the workspace with `auth.test`. +- Authorizes stable member IDs. +- Accepts app direct messages and replies in the exact originating Slack + message thread. +- Commits accepted envelopes to a dedicated SQLite inbox before acknowledging + them to Slack. +- Continues receiving while the gateway processes earlier messages. +- Deduplicates stable Slack `event_id` values. + +### Adding a Channel + +A new channel requires: + +1. one `ChannelContract` implementation; +2. one `Channel` and `ChannelKind` variant; +3. configuration validation and doctor checks; +4. provider-specific contract and integration tests; +5. documentation for identity, allowlists, limits, and failure behavior. + +It should not require provider-name branches in the shared gateway or worker. + +--- + +## 6. Agent Adapters + +### Claude Code + +File: [`src/claude.rs`](src/claude.rs) + +| Operation | Invocation | +| --- | --- | +| New chat | `claude -p --session-id ` | +| Resumed chat | `claude -p --resume ` | +| Instructions | `--append-system-prompt ` | +| Unattended job | `--permission-mode bypassPermissions` | +| Evaluator | safe mode, no tools, no MCP, no Chrome, no session persistence | + +Push chooses Claude's initial session ID and marks it started before execution +so a partial create failure is not retried as another create. + +### Codex + +File: [`src/codex.rs`](src/codex.rs) + +| Operation | Invocation | +| --- | --- | +| New chat | `codex exec --json` | +| Resumed chat | `codex exec resume --json` | +| Instructions | `-c developer_instructions=` | +| Unattended job | full access with approval prompts disabled | +| Evaluator | read-only, ephemeral, tools and project instructions disabled | + +Codex owns its thread ID. Push reads `thread.started.thread_id` from JSONL and +stores it after the first successful turn. + +### Pi + +File: [`src/pi.rs`](src/pi.rs) + +| Operation | Invocation | +| --- | --- | +| New chat | `pi --print --mode json` | +| Resumed chat | `pi --print --mode json --session ` | +| Instructions | `--append-system-prompt ` | +| Unattended job | `--approve` only when the project resources are trusted | +| Evaluator | no approval, tools, extensions, skills, templates, context, or session | + +Pi owns its session ID and reports it in the JSON event stream. Prompts are +written to stdin so large requests do not depend on command-line argument +limits. + +### Adding a Backend + +A new backend adapter must preserve the `Request` and `RunOutput` contract, +support fresh and resumed chat sessions, provide a missing-session +classification, terminate when its future is dropped, and define explicit +configured, unattended, and evaluator modes. + +--- + +## 7. Durable Data Model + +### `state.json` + +[`src/store.rs`](src/store.rs) owns a small JSON document: + +```json +{ + "last_row_id": 123, + "cursors": { + "imessage": 123, + "telegram": 456, + "slack": 789 + }, + "sessions": { + "imessage:self:you@icloud.com": { + "uuid": "backend-session-id", + "started": true, + "backend": "codex" + } + } +} +``` + +`last_row_id` and the field name `uuid` remain for backward compatibility. +Writes use a new owner-only temporary file followed by atomic rename. In-memory +state is rolled back if persistence fails. + +### `push.db` + +[`src/history.rs`](src/history.rs) owns the schema and forward migrations. +SQLite foreign keys are enabled and the connection uses a five-second busy +timeout. + +| Table | Responsibility | +| --- | --- | +| `conversations` | unique channel and thread identity | +| `messages` | canonical inbound and outbound content, generation state, delivery state, chunk checkpoint | +| `approval_questions` | retained durable question and answer state | +| `job_runs` | immutable job claims, bounded results, evaluation, scheduling, and delivery | +| `gateway_control_actions` | idempotent control actions such as `/stop` targets | + +Important constraints: + +- channel event IDs deduplicate inbound messages; +- each inbound message has at most one outbound response; +- one active job run exists per job; +- scheduled occurrence identity is unique; +- control actions are idempotent by inbound message. + +The approval tables and inbound answer path remain durable, but current +production job creation writes runbooks directly and does not create approval +questions. + +### Slack Inbox + +The Slack receiver stores accepted events in `.slack-inbox.db` +before acknowledging Socket Mode envelopes. Its local row ID becomes the +gateway cursor. Ignored envelopes keep redacted rejection metadata rather than +message content. + +### Audit Log + +[`src/audit.rs`](src/audit.rs) appends JSON Lines under a process-wide lock. +Events include message metadata, routing, backend starts and failures, answer +outcomes, delivery results, and completion. Content is omitted by default. +`audit_log_content = true` opts into message and reply text. + +--- + +## 8. Job System + +Jobs are regular UTF-8 Markdown files directly under +`/jobs`. [`src/jobs.rs`](src/jobs.rs) contains validation, +execution, evaluation, scheduling, and durable delivery. + +### Runbook Format + +```markdown ++++ +version = 1 +timeout = "5m" +backend = "codex" +evals = ["task-completion"] + +[[triggers]] +id = "weekday-morning" +kind = "cron" +schedule = "0 8 * * 1-5" +timezone = "Europe/London" +enabled = true ++++ + +Run the morning review and return the three most important actions. +``` + +Validation rejects unknown frontmatter fields, unsafe work directories, +symlinks, subdirectories, invalid names, invalid UTF-8, oversized evals, bad +timeouts, invalid cron expressions, duplicate triggers, and unsupported +backends. + +### Manual Run Pipeline + +```text +1. LOAD validate the requested runbook +2. LOCK acquire a non-blocking per-job OS file lock +3. REREAD validate the exact file again after locking +4. CLAIM insert the running row in an immediate SQLite transaction +5. EXECUTE start one fresh unattended backend session +6. STORE RESULT persist bounded output before evaluation +7. EVALUATE optionally run a fresh restricted evaluator +8. FINISH persist terminal execution and evaluation state +9. PRINT return the result to the invoking CLI +``` + +Manual runs never reuse chat sessions or proactively deliver to a channel. + +### Scheduled Run Pipeline + +The gateway ticks the scheduler once per second: + +```text +1. RECOVER inspect stale runs and delivery claims +2. CATALOG reload and validate installed jobs +3. PLAN calculate next cron occurrence in its IANA timezone +4. ENQUEUE record one due occurrence in push.db +5. CLAIM take work up to jobs_max_workers +6. EXECUTE run a fresh unattended backend session +7. EVALUATE optionally run the restricted evaluator +8. STORE commit result, error, and evaluation state +9. CLAIM SEND take due delivery work across gateway processes +10. DELIVER send stored chunks and checkpoint progress +``` + +Push does not catch up occurrences missed while offline. A clock jump queues at +most one occurrence, daylight-saving gaps are skipped, and repeated local +times run once at their first instant. + +### Execution and Delivery Semantics + +- A failed or timed-out job is not rerun because the agent may already have + completed an external side effect. +- Job results are bounded to 64 KiB. +- One OS advisory lock and one SQLite active-run constraint prevent overlap. +- `jobs_max_workers` bounds scheduled execution. +- Delivery uses up to four workers and five persisted attempts. +- Delivery backoff progresses through 30 seconds, 2 minutes, 10 minutes, and + 30 minutes. +- Delivery retries use the stored result and never rerun the backend. +- Partial delivery resumes from the first uncheckpointed chunk. +- Interrupted execution is marked terminal only after the released OS lock + proves the prior executor is gone. + +### Evaluators + +Assigned evals come from `/evals`. After successful execution, +Push starts a fresh restricted session with the original runbook, candidate +response, and evaluation criteria. The evaluator must finish with +`VERDICT: PASS` or `VERDICT: FAIL`. + +Evaluation state is separate from execution state. A failed, malformed, or +timed-out evaluator does not rewrite the job result. + +--- + +## 9. Concurrency and Shutdown + +### Concurrency Model + +| Scope | Model | +| --- | --- | +| Process | one Tokio multi-thread runtime | +| Channels | one independent gateway task per enabled provider | +| Conversations | one worker task per channel-qualified thread | +| Thread messages | serialized through a bounded queue | +| Different threads | concurrent | +| Scheduled jobs | bounded by `jobs_max_workers` | +| Scheduled delivery | at most four workers | +| State access | short shared mutex sections | +| Audit writes | one shared process lock | + +This prevents two messages in one conversation from racing the same backend +session while allowing unrelated conversations and providers to progress. + +### `/stop` + +`/stop` records an idempotent control action, targets the current row in that +conversation, and signals its worker. Dropping the backend future kills the +subprocess. Already queued messages remain and continue after the interrupted +turn stores its response. + +### Provider Isolation + +One provider may fail, disconnect, or rate-limit without cancelling the other +channel loops. The process exits only when shutdown is requested or every +enabled channel task has stopped. + +### Shutdown Guarantees + +- pending channel operations must be cancellation-safe; +- accepted work drains within the provider's worker grace; +- workers remaining after the deadline are aborted; +- scheduler execution and delivery workers get a separate 30-second grace; +- delivery claims owned by the process are released during cleanup; +- persisted inbound, outbound, and delivery state remains recoverable. + +--- + +## 10. Security Model + +An accepted sender can cause an agent to use local tools and credentials. The +channel allowlist is therefore an operator boundary, not a spam filter. + +### Trust Boundaries + +| Boundary | Enforcement | +| --- | --- | +| iMessage sender | exact normalized `self_handles` and `allow_from` values | +| Telegram sender/chat | stable numeric user and private-chat IDs | +| Slack sender/workspace | stable member IDs plus authenticated workspace verification | +| Chat agent permissions | selected agent's own configuration | +| Unattended jobs | explicit backend mode with no interactive prompt dependency | +| Evaluators | tools, external integrations, project instructions, and persistence disabled | +| Assistant files | canonical path and symlink checks | +| Runtime state | path separation and owner-only permissions | + +### Permission Rule + +Push does not override sandbox, approval, or tool configuration for chats. +Claude and Codex jobs bypass interactive permissions so unattended work can +finish. Pi has no native filesystem sandbox or interactive permission prompt. +Every enabled job should be treated as code execution by the Push service user. + +### Network Rule + +Push accepts no inbound network connection: + +- iMessage reads a local SQLite database; +- Telegram uses outbound HTTPS; +- Slack uses an outbound authenticated WebSocket; +- agent CLIs and their tools own any further network access. + +### Filesystem Rule + +Configuration validation keeps runtime state and secrets outside the +Git-versioned assistant repository. Jobs and evals reject symlinks and unsafe +paths. State, database, audit, inbox, and lock files are restricted to the +service user where supported. + +--- + +## 11. Failure and Recovery + +| Failure | Recovery behavior | +| --- | --- | +| Channel poll fails | log and retry on the next poll | +| Cursor save fails | restore in-memory cursor and retry later | +| Canonical inbound write fails | stop processing the batch so the message is retried | +| Thread queue fills | store a retry response, attempt delivery once, then complete the row | +| Worker closes | replace it and recover retained jobs | +| Backend times out | kill it, store a timeout response, do not retry | +| Backend session is missing | rotate once and retry with bounded history | +| Backend fails | store a gateway-authored error response | +| Worker-managed reply send fails | retry the stored outbound without rerunning | +| Process crashes after send | possible at-least-once duplicate on recovery | +| Scheduled execution stops | mark interrupted after lock-based liveness proof | +| Scheduled delivery stops | release or expire the claim and resume stored chunks | +| Invalid job appears | disable that job and continue messaging and other jobs | + +The key crash boundary is always persistence before an irreversible next step: +accepted input before execution, generated output before delivery, and job +result before proactive delivery. + +--- + +## 12. Source Map and Change Rules + +### Module Reference + +| Module | Responsibility | +| --- | --- | +| [`src/main.rs`](src/main.rs) | CLI parsing and process entry | +| [`src/config.rs`](src/config.rs) | configuration parsing, migration, validation, and routing | +| [`src/gateway/`](src/gateway/) | channel coordination, polling, queues, workers, acknowledgement, delivery | +| [`src/channel.rs`](src/channel.rs) | provider-neutral channel contract and static dispatch | +| [`src/imessage/`](src/imessage/) | Messages database polling and AppleScript sending | +| [`src/telegram.rs`](src/telegram.rs) | Telegram polling, authorization, formatting, voice transfer | +| [`src/slack.rs`](src/slack.rs) | Slack Socket Mode receiver, inbox, identity, and delivery | +| [`src/agent.rs`](src/agent.rs) | provider-neutral backend request and result contract | +| [`src/claude.rs`](src/claude.rs) | Claude Code CLI adapter | +| [`src/codex.rs`](src/codex.rs) | Codex CLI adapter | +| [`src/pi.rs`](src/pi.rs) | Pi CLI adapter | +| [`src/store.rs`](src/store.rs) | atomic cursor and session state | +| [`src/history.rs`](src/history.rs) | SQLite schema, conversation history, delivery, and migrations | +| [`src/jobs.rs`](src/jobs.rs) | runbook validation, execution, evaluation, scheduler, and ledger | +| [`src/audit.rs`](src/audit.rs) | redacted JSONL audit log | +| [`src/rehydration.rs`](src/rehydration.rs) | bounded fresh-session transcript | +| [`src/voice.rs`](src/voice.rs) | provider-neutral transcription and speech | +| [`src/assistant.rs`](src/assistant.rs) | assistant repository initialization | +| [`src/doctor.rs`](src/doctor.rs) | environment and deployment checks | + +### Change Rules + +When changing the system: + +1. Keep provider behavior behind `ChannelContract`. +2. Keep agent-specific CLI behavior behind `Runner`. +3. Persist accepted input before execution. +4. Persist generated output before delivery. +5. Do not retry agent execution when external side effects may have occurred. +6. Preserve per-thread ordering and cross-thread concurrency. +7. Keep runtime state outside the assistant repository. +8. Add forward-only SQLite migrations and recovery tests. +9. Document exact delivery, retry, and shutdown semantics. +10. Avoid adding a gateway plugin or tool system unless a concrete capability + cannot belong to the selected agent. + +--- + +## 13. Build and Verification + +Push targets stable Rust on macOS and Linux. iMessage is macOS-only. + +The required local checks are: + +```sh +cargo fmt --all --check +cargo clippy --locked --all-targets -- -D warnings +cargo build --locked +cargo test --locked +python3 -m mkdocs build --strict +``` + +CI runs the documentation build plus formatting, linting, build, installer, +release-tooling, and test checks on Linux and macOS. Security audit runs +separately when Rust dependencies change and on a weekly schedule. + +Tests are colocated in module `tests` blocks, with integration checks under +[`tests/`](tests/). Channel and backend adapters use contract-style fakes to +exercise command construction, session behavior, cancellation, retries, and +failure classification without depending on live providers. + +--- + +## 14. Deliberate Non-Goals + +Push is intentionally not: + +- an agent runtime; +- a model API client; +- an inbound web service; +- a dynamic channel plugin host; +- an MCP server manager; +- a secrets manager; +- a multi-tenant assistant platform; +- a distributed queue or replicated database. + +Likely future work includes additional built-in channels, additional agent +adapters, and audited memory write-back. Those additions should preserve the +gateway, channel, and agent boundaries above. diff --git a/README.md b/README.md index dd34a2a..d0d59f8 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,7 @@ Push is early software. Please read the [security policy](SECURITY.md) before reporting a vulnerability. Bug reports, ideas, and pull requests are welcome. - [Contributing](CONTRIBUTING.md) +- [Architecture](ARCHITECTURE.md) - [Code of conduct](CODE_OF_CONDUCT.md) - [Security policy](SECURITY.md) - [MIT license](LICENSE) diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index 2e0f638..0000000 --- a/docs/architecture.md +++ /dev/null @@ -1,422 +0,0 @@ -# Push Architecture - -Push is one local Rust process. It receives from configured iMessage, Telegram, -and Slack channels, filters messages, loads the configured assistant repository, -runs a configured agent backend, and sends the -final reply. - -The important boundary is not iMessage or Claude. The important boundary is: - -```text -message gateway -> agent backend -> message gateway -``` - -Ownership is split deliberately: - -```text -Push runtime = channels, scheduling, history, security, delivery -Assistant repository = SOUL.md, context, jobs, optional project skills -Agent runtime = reasoning, tool and skill execution, permissions, - global skills, MCP, authentication -``` - -## Principles - -### 1. Gateway First - -Push is a messaging gateway for a personal assistant. It should stay small and -own the durable pieces: - -- channels -- allowlists -- routing -- assistant repository location and runtime instruction composition -- canonical conversation history -- validated runbook jobs and their durable run ledger -- cursor and backend-session state -- delivery - -### 2. Runtime Disposable - -Agent runtimes are replaceable. Claude Code, Codex, and Pi are the current -adapters. More can be added without changing the messaging core. - -The gateway should not build: - -- its own agent loop -- its own plugin system -- its own MCP layer -- its own coding workflow -- its own tool runner - -Those belong to the selected backend. - -### 3. Outbound connections only - -Push polls channel adapters and shells out to local agent commands. It opens no -server port and accepts no inbound network connection. Telegram uses outbound -HTTPS long polling and Slack uses an outbound Socket Mode WebSocket. - -The trust boundary is the messaging account plus the configured channel -allowlist. - -## System Overview - -```mermaid -flowchart LR - user([You]) -->|iMessage| db[(chat.db)] - user -->|private chat| tg[Telegram Bot API] - user -->|app DM| slack[Slack] - db -->|poll| push - tg -->|long poll| push - slack -->|Socket Mode| push - subgraph push[Push gateway] - poller[Channel poller] --> gateway[Gateway loop] - gateway --> worker[Per-thread worker] - store[(state.json)] <--> gateway - history[(push.db)] <--> gateway - assistant[/assistant repo: SOUL.md, context, jobs, project skills/] --> worker - worker --> adapter[Agent adapter] - end - adapter -->|claude -p| claude[Claude Code] - adapter -->|codex exec| codex[Codex] - adapter -->|pi --print| pi[Pi] - claude --> adapter - codex --> adapter - pi --> adapter - adapter --> sender[Sender] - sender -->|osascript| db - sender -->|sendMessage| tg - sender -->|chat.postMessage| slack - db -->|reply| user - tg -->|reply| user - slack -->|reply| user -``` - -## Channel Boundary - -The gateway depends on one closed, compile-time channel contract. iMessage, -Telegram, and Slack implement it, and the `Channel` enum provides static dispatch. This is -an internal Rust boundary, not a dynamic plugin system or a configuration -extension point. - -Each channel implementation owns these semantics: - -- **Inbound polling:** read events after an opaque monotonic cursor and report - the latest cursor used to skip backlog on first start. A pending poll must be - cancellation-safe because shutdown drops its future. -- **Authorization:** reject unsupported, group, empty, looped-back, or - non-allowlisted input according to provider rules before backend dispatch. -- **Thread identity:** return a stable channel-qualified thread key, the exact - reply target, approval sender/chat identity, and ordered route-key groups. Telegram - topics inherit a parent-chat route; iMessage retains its legacy unprefixed - route aliases. Slack keeps one workspace-scoped DM identity while targeting - the exact originating Slack message thread. -- **Outbound delivery:** validate proactive targets, plan durable chunks, and - send one chunk to the exact accepted target. Replies never cross from one - channel loop to another. -- **Typing:** declare an optional refresh interval and send best-effort activity - updates. Typing failures do not fail an assistant turn. -- **Rich messages:** choose plain or rich delivery per chunk. iMessage keeps one - plain marked reply. Telegram and Slack own their formatting, size limits, - splitting, and provider-specific addressing. -- **Retry:** expose the timeout and bounded retry cadence used for stored chat - replies. A send call is one attempt. Generated output is persisted before - delivery and retried without rerunning the backend. Scheduled delivery keeps - its separate durable attempt ledger and resumes at the first unsent chunk. -- **Shutdown:** stop polling when its future is cancelled, drop queue senders, - drain accepted per-thread work for the contract's grace period, then abort any - remaining workers. Transport futures must release their resources when - dropped. - -Adding another built-in channel therefore requires a concrete contract -implementation and one enum variant. It does not require channel-name branches -in the shared polling, routing, worker, retry, or shutdown loops. - -## Message Lifecycle - -```mermaid -sequenceDiagram - participant U as User - participant C as iMessage or Telegram - participant P as Poller - participant G as Gateway - participant W as Worker - participant H as push.db - participant A as Agent backend - participant S as Sender - - U->>C: send message - P->>C: poll after channel cursor - C-->>P: messages - P->>G: Message values - G->>G: apply channel allowlist and filters - G->>H: persist accepted inbound message - alt slash command - G->>S: handle command locally - else assistant turn - G->>W: enqueue job by thread - W->>W: load SOUL.md and append resolved assistant paths - W->>W: resolve routed backend session - opt fresh backend session - W->>H: read bounded recent conversation - end - W->>A: run new message or rehydrated prompt - A-->>W: reply and optional backend session id - W->>H: persist generated outbound message - W->>S: send reply - W->>G: ack completed row - end - S->>C: send through channel adapter - C->>U: delivered reply -``` - -## Backend Boundary - -The gateway calls an agent through this internal shape: - -```rust -Request { - session_id, - is_new, - work_dir, - instructions, - prompt, -} - -RunOutput { - reply, - session_id, -} -``` - -That keeps the gateway independent of backend-specific mechanics. - -Normal resumed turns contain only the new request. Fresh sessions use at most -20 prior messages from the exact channel-qualified conversation. Push caps each -historical message at 4 KiB and the history block at 16 KiB, then JSON-delimits -roles and content before appending the current user message. This transcript is -prompt content; `SOUL.md` remains separate instruction context. A recognized -missing-session error rotates the stored backend session and retries once with -rehydration. Audit metadata records the rehydrated message count. - -### Claude Code Adapter - -Claude Code lets Push choose the session id. - -- New conversation: `claude -p --session-id ` -- Existing conversation: `claude -p --resume ` -- Instructions: `--append-system-prompt ` -- Work dir: `assistant_root` - -### Codex Adapter - -Codex creates its own thread id. - -- New conversation: `codex exec --json ...` -- Existing conversation: `codex exec resume ...` -- Instructions: `-c developer_instructions=` -- Work dir: `assistant_root`, including resumed runs - -The adapter reads Codex JSONL events to capture `thread.started.thread_id` and -stores that id for future turns. - -### Pi Adapter - -Pi creates its own session id and reports it in the first JSON event. - -- New conversation: `pi --print --mode json ...` -- Existing conversation: `pi --print --mode json --session ...` -- Instructions: `--append-system-prompt ` -- Work dir: `assistant_root` - -The adapter reads the session header and final assistant `message_end` event. -Push passes no tool override to Pi. Pi has no native filesystem sandbox or -interactive permission prompts, so its own configuration is the boundary. - -## State Model - -`state.json` stores channel-specific cursors and channel-qualified sessions: - -```json -{ - "last_row_id": 123, - "cursors": { - "imessage": 123, - "telegram": 456, - "slack": 789 - }, - "sessions": { - "imessage:self:you@icloud.com": { - "uuid": "backend-session-id", - "started": true, - "backend": "codex" - } - } -} -``` - -`last_row_id` remains for compatibility with old iMessage state files. The -field named `uuid` also remains for compatibility, but it -now means "backend session id". - -If the configured backend changes for a thread, Push starts a fresh backend -session instead of trying to resume the old runtime's session. - -Slack's dedicated receiver commits accepted Socket Mode events to -`.slack-inbox.db` before acknowledging their envelopes. It -continues receiving while the gateway processes earlier events. The inbox -assigns monotonic cursor rows and deduplicates Slack's stable `event_id`, so -accepted input survives a process restart without relying on an in-memory -WebSocket buffer. Ignored envelopes retain redacted rejection metadata for the -audit path without retaining message content. - -With advanced `channels = ["imessage", "telegram", "slack"]` configuration, one -coordinator starts an independent polling loop, acknowledgement tracker, and -thread queue map for each enabled provider. The loops share one locked state -store, canonical history database, backend runner set, and serialized audit log. -One provider can fail or rate-limit without cancelling the other. A shared -shutdown signal cancels pending polls and lets every channel drain its workers. -Replies remain bound to the originating loop and exact target. - -## Voice Boundary - -Voice has two independent adapters. A channel adapter exposes opaque inbound -voice metadata, downloads bytes only after the normal allowlist accepts the -message, and uploads a generated audio clip. The provider-neutral voice layer -transcribes and synthesizes in-memory audio. Its output is plain text or a -generic audio clip, so it has no Telegram identifiers or Bot API behavior. -Download and transcription run in the accepted message's per-thread worker, -so slow audio cannot pause polling or work in another conversation. - -The first provider uses `voice.openai_api_key`, with `OPENAI_API_KEY` as a -higher-priority environment override, for both `gpt-4o-transcribe` and -`gpt-4o-mini-tts`. Speech uses the configured `voice.name`, which defaults to -`cedar`. The first channel implementation is Telegram. A future -channel needs only voice download and upload support. It does not need to change -gateway routing, agent requests, or the OpenAI client. - -Voice is an optional enhancement. Missing credentials stop voice processing -for that message with a text explanation, while ordinary text traffic remains -available. Speech synthesis and voice delivery happen after durable text -delivery, so a voice-side failure cannot discard an agent answer. - -Optional `primary_delivery` selects one enabled, allowlisted channel target for -proactive output. Resolution is lazy: an absent or invalid primary returns a -scoped error to the proactive caller without affecting reply polling. - -When primary delivery resolves, the gateway also runs the cron scheduler. It -evaluates validated five-field triggers against their IANA timezone, enqueues at -most one occurrence after an in-process clock jump, and initializes future-only -occurrences after restart so downtime is never caught up. A configured worker -limit bounds fresh-session job execution. Scheduled and manual processes share -the per-job advisory lock and SQLite active-run uniqueness boundary. - -Scheduled output or bounded failure detail is committed before notification. -Delivery has its own persisted state and is retried up to five times from that -stored result. Restart recovery resumes queued work and pending delivery, but -never reruns a backend run that had already started. A running row is marked -interrupted only after the released advisory lock proves its executor is gone. - -`push.db` stores channel-qualified conversations and their inbound and outbound -messages. Accepted inbound messages are inserted before gateway commands or -backend dispatch. Generated backend, command, and error replies are inserted -before delivery, with generation and delivery state tracked separately. A -unique channel event ID makes inbound retries idempotent, and a unique link from -each inbound message to its outbound response preserves the generation/delivery -crash boundary. SQLite history does not replace `state.json` cursors or backend -session IDs in this phase. - -The same database stores immutable job-run claims and bounded terminal results. -Markdown runbooks live under `/jobs`; their write boundary is -set by the selected agent's configuration. A manual CLI start rereads and -validates the exact file, acquires a non-blocking per-job lock, -then records and claims the run in one immediate SQLite transaction before -spawning a fresh backend session. The CLI holds the lock through result -persistence. Only a process that acquired the released lock may fail a stale -manual claim, so a live local executor is never reclaimed from ledger state -alone. Manual runs do not reuse chat history or backend session ids. - -The same database stores `ask_user` questions before delivery. Each question -has a UUID correlation id, two to nine bounded choices, an expiry, delivery -state, and an exact channel, sender, chat, and thread/topic binding. Inbound -numbered replies pass the normal allowlist first, then resolve transactionally -to one normalized value. The workflow consumes an answer at most once. Pending -questions survive restart; cancellation and expiry are terminal, and rejected -or duplicate answer attempts are audited without reaching a backend session or -the rehydrated conversation transcript. Workflows can poll the durable question -state and consume an answered value once; crossing the expiry first records an -expired terminal state, so later cancellation cannot overwrite the timeout. - -Agent-created jobs live directly under `/jobs`. The gateway -includes that absolute path in its in-memory instructions and tells the agent -to run `push job validate` after a change. Job creation has no separate draft -or approval step. The selected agent's filesystem permissions control whether -it can change the assistant repository. - -`audit_log_path` stores a local JSONL event stream for production debugging. -Audit events record message metadata, routing decisions, approval outcomes, -backend run starts and failures, reply delivery metadata, and row completion. -Message and reply text are redacted by default; `audit_log_content` opts into -content logging. - -## Assistant Repository - -Push supports one assistant and stores one canonical `assistant_root`. It -derives `SOUL.md`, `context/`, and `jobs/` from that root. `push init [path]` -creates the conventional structure, initializes Git when needed, and persists -the root through the selected `--config` file. There are no assistant IDs, -registries, active selections, or multi-assistant commands. - -For every conversation and scheduled or manual job run, Push reads `SOUL.md` -and appends a gateway-owned footer in memory containing the resolved absolute -assistant, context, and jobs paths. The footer directs the backend to begin -with `context/README.md` when useful, protect `SOUL.md` and evals unless asked, -write requested jobs directly under `jobs/`, and validate them before reporting -success. Push does not write the footer to the repository or inject all context -files into each prompt. The selected backend and its configuration decide what -to inspect. - -Sessions, databases, audit logs, delivery state, locks, config secrets, -and other runtime state stay outside the Git-versioned assistant repository. -Project-scoped skills and their helper scripts may live in the assistant -repository. The backend still owns skill discovery and execution, global -skills, permissions, MCP, and authentication. - -## Concurrency - -One worker task exists per conversation thread. Messages in the same thread run -in order. Different threads can run in parallel. - -`/stop` cancels the active backend subprocess for one conversation without -discarding messages already queued behind it. A closed worker queue is replaced -and the message that discovered it is retried once. - -This prevents two messages in the same conversation from racing against the same -backend session. - -## Security Posture - -An allowed inbound message can cause an agent to run tools. The sender filter is -the trust boundary. iMessage uses `imessage.self_handles` and -`imessage.allow_from`; Telegram uses stable numeric `telegram.allow_user_ids` -and `telegram.allow_chat_ids`; Slack uses stable `slack.allow_user_ids` member -IDs and verifies the authenticated workspace. - -Push preserves sandbox, approval, permission-mode, and tool-list settings for -chats. Codex and Claude jobs bypass interactive permissions so unattended work -can complete. Every job must use a work directory that does not overlap -Push-owned state or configuration. - -## Roadmap - -The following items are possible future work, not shipped behavior or release -commitments: - -1. More agent adapters. -2. More channels. -3. Memory write-back with audit and review. -4. Per-task backend routing. - -Avoid adding a gateway plugin system until there is a specific capability that -cannot live in the selected backend. diff --git a/docs/contributing.md b/docs/contributing.md index 4b74fc1..01388da 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -37,7 +37,9 @@ cargo test --locked | Durable user questions | `src/approval.rs`, `src/history.rs` | | Production diagnostics | `src/doctor.rs`, `src/audit.rs` | -Read [architecture](architecture.md) before changing state, crash recovery, +Read the +[architecture](https://github.com/owainlewis/push/blob/main/ARCHITECTURE.md) +before changing state, crash recovery, channel cursors, session ownership, scheduling, or delivery semantics. ## Documentation workflow diff --git a/docs/core-system/design.md b/docs/core-system/design.md index 7f24f5d..252b36b 100644 --- a/docs/core-system/design.md +++ b/docs/core-system/design.md @@ -5,7 +5,9 @@ !!! warning "Historical record" This file preserves the approved design and rollout sequence. It is not a - current roadmap. See [Architecture](../architecture.md) for shipped behavior. + current roadmap. See the + [architecture](https://github.com/owainlewis/push/blob/main/ARCHITECTURE.md) + for shipped behavior. **Author:** Owain Lewis **Date:** 2026-07-11 diff --git a/docs/getting-started.md b/docs/getting-started.md index 3dcfce9..4033d43 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -181,4 +181,4 @@ a Telegram- or Slack-only Linux host. - [Configure both channels and per-thread routes](configuration.md) - [Create a manual or scheduled job](jobs.md) - [Inspect every CLI command](reference/cli.md) -- [Understand durable state and recovery](architecture.md) +- [Understand durable state and recovery](https://github.com/owainlewis/push/blob/main/ARCHITECTURE.md) diff --git a/docs/index.md b/docs/index.md index eb0334a..b489172 100644 --- a/docs/index.md +++ b/docs/index.md @@ -103,7 +103,7 @@ tools, skills, and authentication. Chats preserve configured agent permissions; Codex and Claude jobs bypass interactive permissions so scheduled work can finish without an operator. -[See the full architecture](architecture.md){ .push-inline-link } +[See the full architecture](https://github.com/owainlewis/push/blob/main/ARCHITECTURE.md){ .push-inline-link } @@ -179,7 +179,7 @@ finish without an operator. | choose backend permissions safely | [Permissions and security](security.md) | | keep Push online after logout or reboot | [Run as a service](services.md) | | inspect commands and outputs | [CLI reference](reference/cli.md) | -| understand or extend the code | [Architecture](architecture.md) and [contributing](contributing.md) | +| understand or extend the code | [Architecture](https://github.com/owainlewis/push/blob/main/ARCHITECTURE.md) and [contributing](contributing.md) | !!! note "Canonical source" diff --git a/mkdocs.yml b/mkdocs.yml index 1ddec55..6bf30bd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -58,7 +58,7 @@ nav: - Run as a service: services.md - Reference: - CLI commands: reference/cli.md - - Architecture: architecture.md + - Architecture: https://github.com/owainlewis/push/blob/main/ARCHITECTURE.md - Develop: - Contributing: contributing.md - Product strategy: strategy.md diff --git a/tests/docs.rs b/tests/docs.rs index 28cadec..deb465a 100644 --- a/tests/docs.rs +++ b/tests/docs.rs @@ -6,7 +6,7 @@ use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd}; #[test] fn local_documentation_links_resolve() { let root = Path::new(env!("CARGO_MANIFEST_DIR")); - let mut markdown = vec![root.join("README.md")]; + let mut markdown = vec![root.join("README.md"), root.join("ARCHITECTURE.md")]; collect_markdown(&root.join("docs"), &mut markdown); let mut broken = Vec::new();