From f42bee14a28ff2a9ad31e22907993e898b42b082 Mon Sep 17 00:00:00 2001 From: Adam Cohen Hillel Date: Mon, 6 Jul 2026 22:10:24 -0700 Subject: [PATCH 1/3] Add iOS agent support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce OpenPhone iOS: a phone-resident launchd daemon (openphone-agentd) that mirrors the Android agent boundary — owning task state, tool execution, autonomy policy, memory, watchers, background jobs, audit, and trajectories on the device, with host-side tooling limited to install/debug/recovery. Layout under ios/: agentd/ daemon + companion tweaks (Theos rootless build) contracts/ iOS command/event/capability contracts and JSON schemas shared/ platform-neutral contracts shared with the Android agent tools/ macOS install/debug/validation tooling Only OpenPhone-owned, distributable source is included. No third-party code, no device-preparation instructions, and no private device material. Connection details are environment-supplied; build outputs are gitignored. Co-Authored-By: Claude Opus 4.7 --- .gitignore | 5 + ios/README.md | 62 + ios/agentd/Makefile | 70 + ios/agentd/OpenPhoneAppIntrospector.plist | 20 + ios/agentd/OpenPhoneVolumeTrigger.plist | 10 + ios/agentd/README.md | 367 + ios/agentd/TOOLS_PARITY.md | 87 + ios/agentd/control | 9 + ios/agentd/entitlements.plist | 22 + ios/agentd/launchd/com.openphone.agentd.plist | 24 + .../LaunchDaemons/com.openphone.agentd.plist | 24 + .../com.openphone.protecteddatahelper.plist | 31 + .../Preferences/OpenPhoneAgentPrefs.plist | 9 + .../usr/local/bin/openphone-helper-wrapper.sh | 13 + .../OpenPhoneAgentPrefsResources/Info.plist | 26 + ios/agentd/src/agentctl.m | 362 + ios/agentd/src/app_introspector.xm | 2310 ++ ios/agentd/src/main.m | 18952 ++++++++++++++++ .../OpenPhoneAgentPrefsRootListController.m | 561 + ios/agentd/src/screencap_helper.m | 360 + ios/agentd/src/volume_trigger.xm | 7002 ++++++ ios/contracts/README.md | 23 + .../protocol/openphone-capabilities.json | 30 + .../protocol/openphone-commands.json | 2124 ++ ios/contracts/protocol/openphone-events.json | 30 + .../protocol/openphone-ios-tools.json | 368 + .../protocol/openphone-runtime.schema.json | 195 + .../schemas/action-registry.schema.json | 88 + .../schemas/action-request.schema.json | 79 + .../schemas/action-result.schema.json | 34 + ios/contracts/schemas/agent-job.schema.json | 60 + ios/contracts/schemas/audit-event.schema.json | 71 + .../schemas/screen-context.schema.json | 273 + .../schemas/trajectory-event.schema.json | 40 + ios/shared/contracts/README.md | 37 + ios/shared/contracts/capabilities.json | 30 + .../contracts/model-decision.v3.schema.json | 56 + ios/shared/contracts/model-tools.json | 34 + ios/tools/mac/agentd/README.md | 213 + ios/tools/mac/agentd/bedrock-model-broker.py | 229 + .../mac/agentd/install-agentd-package.sh | 160 + ios/tools/mac/agentd/install-agentd.sh | 77 + ios/tools/mac/agentd/pull-crashes.sh | 59 + ios/tools/mac/agentd/restart-agentd.sh | 15 + ios/tools/mac/agentd/smoke-agentd-local.sh | 1903 ++ ios/tools/mac/agentd/tail-agentd-log.sh | 17 + ios/tools/mac/agentd/validate-agentd-store.py | 157 + ios/tools/mac/agentd/validate-on-device.sh | 5784 +++++ .../mac/doctor/openphone-agentd-health.sh | 21 + 49 files changed, 42533 insertions(+) create mode 100644 ios/README.md create mode 100644 ios/agentd/Makefile create mode 100644 ios/agentd/OpenPhoneAppIntrospector.plist create mode 100644 ios/agentd/OpenPhoneVolumeTrigger.plist create mode 100644 ios/agentd/README.md create mode 100644 ios/agentd/TOOLS_PARITY.md create mode 100644 ios/agentd/control create mode 100644 ios/agentd/entitlements.plist create mode 100644 ios/agentd/launchd/com.openphone.agentd.plist create mode 100644 ios/agentd/layout/Library/LaunchDaemons/com.openphone.agentd.plist create mode 100644 ios/agentd/layout/Library/LaunchDaemons/com.openphone.protecteddatahelper.plist create mode 100644 ios/agentd/layout/Library/PreferenceLoader/Preferences/OpenPhoneAgentPrefs.plist create mode 100755 ios/agentd/layout/usr/local/bin/openphone-helper-wrapper.sh create mode 100644 ios/agentd/prefs/OpenPhoneAgentPrefsResources/Info.plist create mode 100644 ios/agentd/src/agentctl.m create mode 100644 ios/agentd/src/app_introspector.xm create mode 100644 ios/agentd/src/main.m create mode 100644 ios/agentd/src/prefs/OpenPhoneAgentPrefsRootListController.m create mode 100644 ios/agentd/src/screencap_helper.m create mode 100644 ios/agentd/src/volume_trigger.xm create mode 100644 ios/contracts/README.md create mode 100644 ios/contracts/protocol/openphone-capabilities.json create mode 100644 ios/contracts/protocol/openphone-commands.json create mode 100644 ios/contracts/protocol/openphone-events.json create mode 100644 ios/contracts/protocol/openphone-ios-tools.json create mode 100644 ios/contracts/protocol/openphone-runtime.schema.json create mode 100644 ios/contracts/schemas/action-registry.schema.json create mode 100644 ios/contracts/schemas/action-request.schema.json create mode 100644 ios/contracts/schemas/action-result.schema.json create mode 100644 ios/contracts/schemas/agent-job.schema.json create mode 100644 ios/contracts/schemas/audit-event.schema.json create mode 100644 ios/contracts/schemas/screen-context.schema.json create mode 100644 ios/contracts/schemas/trajectory-event.schema.json create mode 100644 ios/shared/contracts/README.md create mode 100644 ios/shared/contracts/capabilities.json create mode 100644 ios/shared/contracts/model-decision.v3.schema.json create mode 100644 ios/shared/contracts/model-tools.json create mode 100644 ios/tools/mac/agentd/README.md create mode 100755 ios/tools/mac/agentd/bedrock-model-broker.py create mode 100755 ios/tools/mac/agentd/install-agentd-package.sh create mode 100755 ios/tools/mac/agentd/install-agentd.sh create mode 100755 ios/tools/mac/agentd/pull-crashes.sh create mode 100755 ios/tools/mac/agentd/restart-agentd.sh create mode 100755 ios/tools/mac/agentd/smoke-agentd-local.sh create mode 100755 ios/tools/mac/agentd/tail-agentd-log.sh create mode 100755 ios/tools/mac/agentd/validate-agentd-store.py create mode 100755 ios/tools/mac/agentd/validate-on-device.sh create mode 100755 ios/tools/mac/doctor/openphone-agentd-health.sh diff --git a/.gitignore b/.gitignore index 5415516..8a853c7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,8 @@ docs/site/.source/ docs/site/content/docs/ docs/site/node_modules/ docs/site/tsconfig.tsbuildinfo + +# iOS agentd build outputs +ios/agentd/.theos/ +ios/agentd/packages/ +*.deb diff --git a/ios/README.md b/ios/README.md new file mode 100644 index 0000000..0db5cb1 --- /dev/null +++ b/ios/README.md @@ -0,0 +1,62 @@ +# OpenPhone iOS + +OpenPhone iOS brings the OpenPhone agent to iPhone as a phone-resident runtime. +The design mirrors the Android side of this repository: the phone owns the +agent, not a host computer. A launchd daemon on the device owns task state, +tool execution, autonomy policy, memory, watchers, background jobs, audit, and +trajectories. Host-side tools exist only for install, debug, and recovery. + +This is experimental proof-of-concept work, not a supported product. It targets +an owned device that already exposes a rootless runtime prefix at `/var/jb`; +reaching that state is the owner's responsibility and out of scope here. + +## Layout + +```text +agentd/ Phone-resident daemon and companion tweaks (Theos rootless build). +contracts/ iOS command/event/capability contracts and JSON schemas. +shared/ Platform-neutral contracts shared with the Android agent. +tools/ macOS install / debug / validation tooling (not the runtime). +``` + +## agentd + +`openphone-agentd` is the runtime authority. It listens on a phone-local Unix +domain socket and serves the Android-shaped command surface (tasks, screen, +actions, memory, context, commitments, watchers, background jobs, providers, +and the model loop). Companion tweaks provide a SpringBoard hardware trigger and +in-process app UI introspection. See [agentd/README.md](agentd/README.md) for +the command list, build, and on-device debug notes. + +Build the rootless package (requires Theos on the build host): + +```sh +cd ios/agentd +make package +``` + +## Contracts + +`contracts/` holds the iOS command, event, and capability definitions plus the +action/audit/trajectory JSON schemas. `shared/contracts/` is the deduplicated +set both iOS and Android are meant to build against: the model tool registry, +the model decision schema, and the canonical capability/risk list. The daemon +is the source of truth for the tool surface; keep the shared files in sync when +the daemon tool surface changes. + +## Tools + +`tools/mac/` installs, restarts, inspects, and validates the daemon over SSH or +a USB-forwarded tunnel. These are development and recovery tools and never run +the agent on the host. Connection details are supplied through environment +variables (`OPENPHONE_IOS_HOST`, `OPENPHONE_IOS_USER`, and so on); no device +identifiers or credentials are committed. See +[tools/mac/agentd/README.md](tools/mac/agentd/README.md). + +## Runtime dependencies + +At runtime the daemon assumes the device provides `OpenSSH` (for host tooling), +a Substrate-compatible tweak loader (for the SpringBoard tweak), a preference +loader (for the settings pane), the rootless prefix `/var/jb`, and `launchd` +(which runs the daemon from `/var/jb/Library/LaunchDaemons`). This project does +not provide or bundle those components. diff --git a/ios/agentd/Makefile b/ios/agentd/Makefile new file mode 100644 index 0000000..1ef57ea --- /dev/null +++ b/ios/agentd/Makefile @@ -0,0 +1,70 @@ +TARGET := iphone:clang:latest:15.0 +THEOS_PACKAGE_SCHEME = rootless + +ifeq ($(THEOS),) + ifneq ($(wildcard $(HOME)/theos/makefiles/common.mk),) + THEOS := $(HOME)/theos + else ifneq ($(wildcard /opt/theos/makefiles/common.mk),) + THEOS := /opt/theos + else ifneq ($(wildcard /var/theos/makefiles/common.mk),) + THEOS := /var/theos + else + $(error THEOS is not set. Install Theos or run with THEOS=/path/to/theos) + endif +endif + +include $(THEOS)/makefiles/common.mk + +INSTALL_TARGET_PROCESSES = SpringBoard MobileSafari Preferences MobileNotes MobileSMS MobileCal com.apple.WebKit.WebContent + +TOOL_NAME = openphone-agentd openphone-protected-data-helper openphone-agentctl openphone-screencap-helper +TWEAK_NAME = OpenPhoneVolumeTrigger OpenPhoneAppIntrospector +BUNDLE_NAME = OpenPhoneAgentPrefs + +openphone-agentd_FILES = src/main.m +openphone-agentd_CFLAGS = -fobjc-arc -Wall -Wextra +openphone-agentd_FRAMEWORKS = Foundation AVFoundation AudioToolbox Speech +openphone-agentd_LIBRARIES = compression sqlite3 +openphone-agentd_INSTALL_PATH = /usr/local/bin +openphone-agentd_CODESIGN_FLAGS = -Sentitlements.plist + +openphone-protected-data-helper_FILES = src/main.m +openphone-protected-data-helper_CFLAGS = -fobjc-arc -Wall -Wextra +openphone-protected-data-helper_FRAMEWORKS = Foundation AVFoundation AudioToolbox Speech +openphone-protected-data-helper_LIBRARIES = compression sqlite3 +openphone-protected-data-helper_INSTALL_PATH = /usr/local/bin + +openphone-agentctl_FILES = src/agentctl.m +openphone-agentctl_CFLAGS = -fobjc-arc -Wall -Wextra +openphone-agentctl_FRAMEWORKS = Foundation +openphone-agentctl_INSTALL_PATH = /usr/local/bin + +openphone-screencap-helper_FILES = src/screencap_helper.m +openphone-screencap-helper_CFLAGS = -fobjc-arc -Wall -Wextra +openphone-screencap-helper_FRAMEWORKS = Foundation CoreGraphics ImageIO +openphone-screencap-helper_INSTALL_PATH = /usr/local/bin + +OpenPhoneVolumeTrigger_FILES = src/volume_trigger.xm +OpenPhoneVolumeTrigger_CFLAGS = -fobjc-arc -Wall -Wextra +OpenPhoneVolumeTrigger_FRAMEWORKS = Foundation UIKit QuartzCore AudioToolbox +OpenPhoneVolumeTrigger_INSTALL_PATH = /usr/lib/TweakInject + +OpenPhoneAppIntrospector_FILES = src/app_introspector.xm +OpenPhoneAppIntrospector_CFLAGS = -fobjc-arc -Wall -Wextra +OpenPhoneAppIntrospector_FRAMEWORKS = Foundation UIKit +OpenPhoneAppIntrospector_INSTALL_PATH = /usr/lib/TweakInject + +OpenPhoneAgentPrefs_FILES = src/prefs/OpenPhoneAgentPrefsRootListController.m +OpenPhoneAgentPrefs_CFLAGS = -fobjc-arc -Wall -Wextra +OpenPhoneAgentPrefs_FRAMEWORKS = Foundation UIKit +OpenPhoneAgentPrefs_LDFLAGS = -undefined dynamic_lookup +OpenPhoneAgentPrefs_INSTALL_PATH = /Library/PreferenceBundles +OpenPhoneAgentPrefs_RESOURCE_DIRS = prefs/OpenPhoneAgentPrefsResources + +include $(THEOS_MAKE_PATH)/tool.mk +include $(THEOS_MAKE_PATH)/tweak.mk +include $(THEOS_MAKE_PATH)/bundle.mk + +after-stage:: + chmod 4755 $(THEOS_STAGING_DIR)/usr/local/bin/openphone-protected-data-helper + chmod 0755 $(THEOS_STAGING_DIR)/usr/local/bin/openphone-helper-wrapper.sh diff --git a/ios/agentd/OpenPhoneAppIntrospector.plist b/ios/agentd/OpenPhoneAppIntrospector.plist new file mode 100644 index 0000000..168f5a8 --- /dev/null +++ b/ios/agentd/OpenPhoneAppIntrospector.plist @@ -0,0 +1,20 @@ +{ + Filter = { + Bundles = ( + "com.apple.mobilesafari", + "com.apple.WebKit.WebContent", + "com.apple.Preferences", + "com.apple.mobilenotes", + "com.apple.MobileSMS", + "com.apple.mobilecal" + ); + Executables = ( + "MobileSafari", + "com.apple.WebKit.WebContent", + "Preferences", + "MobileNotes", + "MobileSMS", + "MobileCal" + ); + }; +} diff --git a/ios/agentd/OpenPhoneVolumeTrigger.plist b/ios/agentd/OpenPhoneVolumeTrigger.plist new file mode 100644 index 0000000..f0e8a26 --- /dev/null +++ b/ios/agentd/OpenPhoneVolumeTrigger.plist @@ -0,0 +1,10 @@ +{ + Filter = { + Bundles = ( + "com.apple.springboard" + ); + Executables = ( + "SpringBoard" + ); + }; +} diff --git a/ios/agentd/README.md b/ios/agentd/README.md new file mode 100644 index 0000000..32e92fd --- /dev/null +++ b/ios/agentd/README.md @@ -0,0 +1,367 @@ +# openphone-agentd + +`openphone-agentd` is the phone-resident runtime authority for OpenPhone-iOS. + +This first skeleton is intentionally small. It creates the OpenPhone data +directory, listens on a local Unix domain socket, defaults to YOLO mode, and +serves Android-shaped stubs for: + +- `health` +- `start_task` +- `stop_task` +- `list_tasks` +- `get_task` +- `get_audit` +- `get_trajectory` +- `model_status` +- `model_configure` +- `list_apps` +- `get_screen` +- `execute_action` +- `run_task` +- `finish_task` +- `fail_task` +- `memory_save` +- `memory_search` +- `memory_update` +- `memory_delete` +- `memory_merge` +- `context_search` +- `clipboard_read` +- `clipboard_write` +- `contacts_search` +- `calendar_search` +- `calls_search` +- `messages_search` +- `commitment_create` +- `commitment_search` +- `commitment_update_status` +- `watcher_create` +- `watcher_list` +- `watcher_stop` +- `watcher_repair_stuck` +- `watcher_run_due` +- `background_job_create` +- `background_job_list` +- `background_job_stop` +- `background_job_repair_stuck` +- `background_job_run_due` +- `hardware_trigger` + +The current implemented actions are `wait`, `open_app`, `open_url`, `home`, +`wake_and_home`, `tap`, `long_press`, `swipe`, and `type_text`. +`open_app` and `open_url` use `/var/jb/usr/bin/uiopen` when it exists. +`list_apps` scans known app roots and reads `.app/Info.plist` locally on the +phone. It accepts either a numeric limit or a text query. +`home` uses SpringBoardServices plus GraphicsServices menu-button events. +`wake_and_home` calls `SBSUndimScreen` and then sends a menu-button event. +Coordinate input uses IOKit `IOHIDEventSystemClient` from inside the daemon. +`tap_element` resolves `element_id` or `view_id` from the current UI tree, +computes the bounds center, and includes the target summary in the action +result. Tap-style actions first ask the app-process input bridge for app-owned +elements or the SpringBoard input bridge for SpringBoard-owned views, then fall +back to daemon HID if those bridges cannot activate the target. On the iPhone 14 +Pro Max, tap and swipe were verified through raw `execute_action` and +deterministic `run_task`; package `0.1.0-107+debug` also verified a Settings +app-process `tap_element` visible effect through +`OpenPhoneAppIntrospector.AppInput`. The SpringBoard input bridge exposes +`show_passcode`, verified visually with the SpringBoard screenshot bridge. +`unlock_with_passcode` is implemented as an experimental bridge action, but it +returns unavailable unless lock-state verification confirms the device actually +unlocked; current on-device attempts still leave the lock state true. +Text input uses IOHID keyboard events plus the app-process input bridge. Package +`0.1.0-117+debug` verifies direct Notes search text entry through +`OpenPhoneAppIntrospector.AppInput` by checking the app-side text-length delta +and the after UI tree/screenshot. Package `0.1.0-121+debug` verifies a live +Bedrock model task in Notes: the model selected `tap_element` and `type_text`, +the app-process provider returned `user_facing_status=verified`, the requested +text was visible afterward, and the task finished with +`stop_reason=verified_type_text_goal_complete`. Package `0.1.0-122+debug` +extends verified text entry to the Notes body `UITextView`, publishes editable +text values back through the app UI tree, and verifies a live Bedrock-selected +body `type_text` task. Package `0.1.0-126+debug` verifies direct Safari page +text input through the MobileSafari WebKit DOM bridge: Wikipedia search field +`type_text` returned `user_facing_status=verified` with +`verification.source=web_content_dom_state`. HID-only text dispatch remains +unverified unless a later screen/UI diff proves a visible effect. + +`get_screen` reports running app bundle IDs by spawning `/bin/ps` directly and +reading each `.app/Info.plist`. It also reports GraphicsServices display +dimensions/scale plus SpringBoard lock/passcode state. It decodes +SpringBoard's phone-local `RecentAppLayouts.pb.lzfse` file to expose +`recent_apps` and `last_known_foreground_candidate`; this is treated as a +fallback/inferred source, not true foreground when the device is locked. When a +fresh SpringBoard state publication has a foreground app and agrees with the +direct daemon foreground read, `get_screen` reports +`foreground_source=springboard_state` so validation can distinguish true +SpringBoard foreground from recent-layout inference. When `include_screenshot` +is true, the daemon invokes `openphone-screencap-helper` and returns a +phone-local PNG path, byte count, and hash if capture succeeds. +Screenshot bytes are not returned through JSON. On the current iPhone 14 Pro +Max, the helper can load the framebuffer and IOSurface symbols, but direct +framebuffer acquisition is denied by the runtime and returns an explicit +`display_connection_unavailable` result. The daemon falls back to the +SpringBoard screenshot bridge, which renders SpringBoard windows into a +phone-local PNG. App-process UI snapshots are published through a narrow +daemon-owned `app_ui_publish` path: app tweaks connect to `127.0.0.1:27631`, +the daemon stores `/var/mobile/Library/OpenPhone/app-ui/.json`, and +`get_screen` prefers a fresh foreground-matched app tree before falling back to +SpringBoard. This is local-smoke covered and was verified on the iPhone 14 Pro +Max after installing `0.1.0-106+debug`: Safari and Settings both returned +`get_screen.context.ui_tree_source=app_process`. Package `0.1.0-107+debug` +adds `app_input_poll`/`app_input_complete` on the same loopback intake so the +app introspector can activate app-owned controls from inside the target process. +Manual validation tapped Settings `General` and observed the app UI tree change +to the General page. Notes search text entry is also verified on-device with +`user_facing_status=verified` in package `0.1.0-117+debug`, and the live +Bedrock Notes body text-entry loop is verified in package `0.1.0-122+debug`. +Package `0.1.0-123+debug` adds bounded non-UIView accessibility child +enumeration and matching element lookup for app-process trees. Current Safari +evidence shows MobileSafari still publishes zero WebKit accessibility children; +package `0.1.0-125+debug` attempted a separate +`com.apple.WebKit.WebContent` XPC tweak bridge but did not observe a WebContent +tweak load or WebContent app UI publication. Package `0.1.0-126+debug` adds the +working route for now: MobileSafari `_SFWebView`/WebKit DOM evaluation from the +app process, with DOM-backed web elements published under `ui_tree.web_dom`. + +`memory_save`, `memory_search`, `memory_update`, `memory_delete`, +`memory_merge`, and `context_search` use a phone-local SQLite store at +`/var/mobile/Library/OpenPhone/db/openphone.sqlite`. The daemon creates the +`memory` and `context_event` tables, uses FTS5 when available, and falls back to +SQL `LIKE` search when FTS is unavailable. Memory updates and merges keep the +FTS index synchronized; merges update the target memory and delete the source +memory. `clipboard_read` and `clipboard_write` expose bounded phone clipboard +text through daemon commands and model/Realtime tools; both create context and +audit events. Package `0.1.0-132+debug` validates the real on-device provider +through `OpenPhoneVolumeTrigger.SpringBoardClipboard`, a SpringBoard-context +`UIPasteboard` bridge. Local macOS smoke uses a daemon-local fallback file when +SpringBoard is unavailable. `contacts_search` is the first read-only contacts +provider slice: it searches AddressBook SQLite on the phone when available, +falls back to a local fixture in macOS smoke, returns bounded contact summaries +to the caller/model, and stores only query length/hash plus count/provider +metadata in context/audit. Package `0.1.0-133+debug` validates read-only +AddressBook access on the iPhone 14 with a nonexistent query and sanitized +`contacts_searched` context event. +`calendar_search`, `calls_search`, and `messages_search` are read-only phone +data provider slices. They search Calendar, CallHistory, and SMS/iMessage +SQLite stores through a dedicated protected-data helper when available, fall +back to local fixtures in macOS smoke, return bounded summaries/previews to the +caller/model, and store only query length/hash, time range, count, and provider +metadata in context/audit. Package `0.1.0-143+debug` validates the protected +path on the iPhone 14 with nonexistent queries: main daemon calls returned +provider `Calendar.sqlitedb`, `CallHistory.storedata`, and `SMS.sqlite`, each +with count `0`, and indexed sanitized `calendar_searched`, `calls_searched`, +and `messages_searched` context events. The helper is currently started by the +installer from the proven SSH/sudo context; launchd-root, launchd-mobile, and +LaunchAgent helper starts did not receive equivalent protected SQLite access. +`commitment_create`, +`commitment_search`, `commitment_update_status`, `commitment_run_due`, +`watcher_create`, +`watcher_list`, `watcher_stop`, `watcher_run_due`, `watcher_repair_stuck`, +`background_job_create`, +`background_job_list`, `background_job_run_due`, +`background_job_repair_stuck`, and `background_job_stop` use the same SQLite +store. Background jobs now have a first deterministic runner for explicit +`background_job_run_due` and hardware-trigger paths: before claiming due work it +requeues stale scheduler-enabled `running` rows through +`background_job_repair_stuck`, records `payload.stuck_repair` metadata, then +claims due `queued` jobs, runs each through the phone-local `run_task` loop, and +persists the result back onto the job row. Periodic daemon scheduler ticks wait +through a startup grace period and perform repair/materialization only, so daemon +restart recovery can unstick durable rows without launching task bodies from the +maintenance thread. Jobs with `interval_ms` or explicit `recurring=true` now +requeue after successful runs with the next interval time, and recurring +failures requeue with bounded exponential backoff. The scheduler records +`run_policy`, `failure_count`, and `retry_backoff_ms` in both schedule/payload +metadata, and local smoke covers interval requeue, explicit +`recurring=false` terminal interval jobs, and first-failure backoff. +Due commitments now have a local bridge too: active commitments with +`due_at_ms`/`due_at` are claimed as `running`, materialize into audited +`commitment_due` background jobs, and then move to `triggered` with scheduler +evidence containing the generated job id. `commitment_run_due` runs that bridge +directly, and `background_job_run_due` includes it during scheduler ticks. +Timer watchers now have a local bridge: active +`time`/`timer`/`deadline` watchers with a due `next_run_at_ms` are first claimed +as `running`, then materialize into audited scheduler-enabled background jobs. +After successful job creation they move to `fired` or, for recurring watchers, +back to `active` with the next run time. Stale `running` timer watchers are +repaired by `watcher_repair_stuck` and by `watcher_run_due` before new due +watchers are claimed. Notification, message, call, web, and semantic watchers +remain durable-only until their providers exist. Notification, message, and call +providers are still not implemented. + +`model_status` reports phone-local model-loop readiness without returning +provider credential values. `model_configure` writes only non-credential +metadata such as broker/direct-dev mode, endpoint URL, model name, and loop +limits under `/var/mobile/Library/OpenPhone/config/` with protected file +permissions. `run_task` supports `deterministic`, `model`, and `auto` modes. +Model mode now has the daemon-owned observe/parse/execute/verify loop, +`openphone.model_decision.v1` parser, conservative parser repair for +prose/fenced-code wrapped decision JSON, `finish_task`/`fail_task` terminal +tools, registered-tool execution through daemon APIs, cooperative cancellation +for adopted task IDs stopped through `stop_task`, and fixture-provider smoke +coverage. Broker HTTP execution is implemented inside the daemon and covered by +local smoke through a localhost broker fixture. Host-side provider validation +can use `tools/mac/agentd/bedrock-model-broker.py` with SSH reverse forwarding +so the phone calls a local broker endpoint while provider credentials stay off +the phone. Direct Bedrock execution from the phone is verified, including a +live Notes body UI-action task in package `0.1.0-122+debug`. Live model-loop +evidence for Safari DOM fields is still pending after the package +`0.1.0-126+debug` direct tool-path verification. Package `0.1.0-145+debug` +validates the SpringBoard prompt-to-agent handoff through the local prompt +bridge: request `prompt-run-1782975246` showed the `Agent request submitted` +and `Agent loop started` overlays, created +`ios-task-1782975246781-41566` from `source=springboard_prompt`, and completed +through Bedrock with `stop_reason=finish_task`, `steps_used=1`, and +`tool_errors=0`. Package `0.1.0-129+debug` +adds direct OpenAI Realtime WebSocket modes: `openai_realtime` defaults to +`gpt-realtime`, and `openai_realtime2` defaults to `gpt-realtime-2` with low +reasoning effort. The daemon keeps one Realtime session open per task, exposes +the same iPhone tools as function tools, executes function calls locally, sends +function outputs back into the session, and records normal model-loop +trajectory/audit events. Local smoke validates Realtime-2 config/status without +calling OpenAI; live execution requires a real OpenAI credential in +`/var/mobile/Library/OpenPhone/config/model-credential.json`. + +`hardware_trigger` records a phone-local hardware activation, indexes it as a +context event, audits it, and starts an async phone-local model task when the +model provider is ready. Package `0.1.0-90+debug` returned a stable +acknowledgement without synchronously draining a deterministic job while the +device-only runner restart was investigated. The +physical path was confirmed on 2026-07-01 at 12:41 PDT: the SpringBoard hook +detected the combo, the daemon returned `ok`, queued `ios-job-72`, and the +daemon stayed on the same PID. The +daemon also starts an +experimental `IOHIDEventSystemClient` listener +for Volume Up + Volume Down. On the current phone this listener registers but +does not call `IOHIDEventSystemClientActivate`, because activating the +run-loop-scheduled client aborts `openphone-agentd` on this iOS 16.5 device. +If a daemon-side event reaches it later, it now honors the same +`com.openphone.volumetrigger` preferences as the SpringBoard path. The +package also ships `OpenPhoneVolumeTrigger.dylib` under +`/var/jb/usr/lib/TweakInject/` as the active SpringBoard tweak-loader fallback. The +fallback currently hooks `SBVolumeHardwareButtonActions` and related +SpringBoard volume selectors, plus object/raw-HID button probes such as the +narrow `SBCameraHardwareButtonStudyLogger logButtonEvent:` raw `IOHIDEvent` +hook. The raw-HID hook only classifies consumer Volume Up/Down usages and does +not start or activate a HID client. Package `0.1.0-175+debug` also installs a +passive `AVSystemController_SystemVolumeDidChangeNotification` observer inside +SpringBoard; it seeds the current system volume and feeds inferred up/down +deltas into the same combo state machine. The tweak logs individual button +events, retries the daemon Unix socket, uses a 1.2s Volume Up + Volume Down +combo window, opens an OpenPhone prompt by default, plays haptic feedback, and +shows scene-attached SpringBoard prompt/overlay state. Do not use a +SpringBoard-context `IOHIDEventSystemClient` listener on this device; +`IOHIDEventSystemClientActivate` crashed SpringBoard into safe mode. + +## Build + +This directory uses Theos rootless packaging: + +```sh +cd ios/agentd +make package +``` + +The build host must have Theos installed. The Makefile auto-detects +`$HOME/theos`, `/opt/theos`, and `/var/theos`; otherwise run with +`THEOS=/path/to/theos`. + +The package installs: + +```text +/var/jb/usr/local/bin/openphone-agentd +/var/jb/usr/local/bin/openphone-agentctl +/var/jb/usr/local/bin/openphone-screencap-helper +/var/jb/Library/LaunchDaemons/com.openphone.agentd.plist +``` + +## On-Device Debug + +After installing and loading the daemon, query health on the phone: + +```sh +/var/jb/usr/local/bin/openphone-agentctl +``` + +Or send a command: + +```sh +/var/jb/usr/local/bin/openphone-agentctl start_task "open Safari" +/var/jb/usr/local/bin/openphone-agentctl list_tasks 10 +/var/jb/usr/local/bin/openphone-agentctl list_apps 50 +/var/jb/usr/local/bin/openphone-agentctl list_apps safari +/var/jb/usr/local/bin/openphone-agentctl get_task +/var/jb/usr/local/bin/openphone-agentctl get_trajectory 20 +/var/jb/usr/local/bin/openphone-agentctl get_audit 20 +/var/jb/usr/local/bin/openphone-agentctl get_screen screenshot +/var/jb/usr/local/bin/openphone-agentctl memory_save "user prefers concise updates" preference user +/var/jb/usr/local/bin/openphone-agentctl memory_search concise 5 +/var/jb/usr/local/bin/openphone-agentctl memory_update ios-memory-1 "user prefers concise status updates" preference user +/var/jb/usr/local/bin/openphone-agentctl memory_merge ios-memory-1 ios-memory-2 "user prefers concise updates" +/var/jb/usr/local/bin/openphone-agentctl memory_delete ios-memory-3 "obsolete memory" +/var/jb/usr/local/bin/openphone-agentctl context_search concise 5 +/var/jb/usr/local/bin/openphone-agentctl '{"command":"clipboard_write","text":"copy this","reason":"user asked to copy text"}' +/var/jb/usr/local/bin/openphone-agentctl '{"command":"clipboard_read","max_chars":4096,"reason":"user asked about copied text"}' +/var/jb/usr/local/bin/openphone-agentctl '{"command":"contacts_search","query":"Ada","limit":5,"reason":"user asked for a saved contact"}' +/var/jb/usr/local/bin/openphone-agentctl '{"command":"calendar_search","query":"standup","limit":5,"reason":"user asked about schedule context"}' +/var/jb/usr/local/bin/openphone-agentctl '{"command":"calls_search","query":"Ada","limit":5,"reason":"user asked about recent calls"}' +/var/jb/usr/local/bin/openphone-agentctl '{"command":"messages_search","query":"Ada","limit":5,"reason":"user asked about saved messages"}' +/var/jb/usr/local/bin/openphone-agentctl commitment_create "follow up with Adam" +/var/jb/usr/local/bin/openphone-agentctl commitment_search Adam 5 +/var/jb/usr/local/bin/openphone-agentctl commitment_update_status ios-commitment-1 completed +/var/jb/usr/local/bin/openphone-agentctl '{"command":"commitment_run_due","limit":5}' +/var/jb/usr/local/bin/openphone-agentctl watcher_create "watch for smoke condition" time +/var/jb/usr/local/bin/openphone-agentctl watcher_list smoke 5 +/var/jb/usr/local/bin/openphone-agentctl '{"command":"watcher_run_due","limit":5}' +/var/jb/usr/local/bin/openphone-agentctl '{"command":"watcher_repair_stuck","limit":5}' +/var/jb/usr/local/bin/openphone-agentctl watcher_stop ios-watcher-1 +/var/jb/usr/local/bin/openphone-agentctl background_job_create "smoke job" "summarize durable stores" +/var/jb/usr/local/bin/openphone-agentctl background_job_run_due 5 1 15000 +/var/jb/usr/local/bin/openphone-agentctl '{"command":"background_job_repair_stuck","limit":5}' +/var/jb/usr/local/bin/openphone-agentctl background_job_list durable 5 +/var/jb/usr/local/bin/openphone-agentctl background_job_stop ios-job-1 +/var/jb/usr/local/bin/openphone-agentctl hardware_trigger volume_up_down_combo +/var/jb/usr/local/bin/openphone-agentctl model_status +/var/jb/usr/local/bin/openphone-agentctl model_configure broker https://broker.example/v1/decision openphone-broker-model true +/var/jb/usr/local/bin/openphone-agentctl '{"command":"model_configure","mode":"openai_realtime2","enabled":true,"model":"gpt-realtime-2","max_steps":40,"max_duration_ms":600000}' +/var/jb/usr/local/bin/openphone-agentctl '{"command":"execute_action","action":{"type":"wait","duration_ms":1000}}' +/var/jb/usr/local/bin/openphone-agentctl home +/var/jb/usr/local/bin/openphone-agentctl wake_and_home +/var/jb/usr/local/bin/openphone-agentctl show_passcode +/var/jb/usr/local/bin/openphone-agentctl tap 120 300 +/var/jb/usr/local/bin/openphone-agentctl tap_element springboard-3-1 +/var/jb/usr/local/bin/openphone-agentctl long_press 120 300 800 +/var/jb/usr/local/bin/openphone-agentctl swipe 200 800 200 250 350 +/var/jb/usr/local/bin/openphone-agentctl type_text "hello from OpenPhone" +/var/jb/usr/local/bin/openphone-agentctl '{"command":"execute_action","action":{"type":"unlock_with_passcode","passcode":""}}' +/var/jb/usr/local/bin/openphone-agentctl open_app com.apple.mobilesafari +/var/jb/usr/local/bin/openphone-agentctl open_url https://example.com +/var/jb/usr/local/bin/openphone-agentctl run_task "open Safari" +/var/jb/usr/local/bin/openphone-agentctl run_task "tap 120 300" +/var/jb/usr/local/bin/openphone-agentctl run_task "swipe 200 800 200 250 350" +/var/jb/usr/local/bin/openphone-agentctl run_task "type hello from OpenPhone" +/var/jb/usr/local/bin/openphone-agentctl run_task "open Safari" 3 60000 +/var/jb/usr/local/bin/openphone-agentctl run_task "inspect the screen" model 3 60000 +/var/jb/usr/local/bin/openphone-agentctl finish_task ios-task-1 "Task complete" +/var/jb/usr/local/bin/openphone-agentctl fail_task ios-task-1 "Unable to complete" +``` + +Logs and durable data live under: + +```text +/var/mobile/Library/OpenPhone/ +``` + +Important files: + +```text +/var/mobile/Library/OpenPhone/tasks/.json +/var/mobile/Library/OpenPhone/audit/audit-events.jsonl +/var/mobile/Library/OpenPhone/trajectories/.jsonl +/var/mobile/Library/OpenPhone/db/openphone.sqlite +/var/mobile/Library/OpenPhone/openphone-agentd.log +``` + +This daemon is the runtime target. Host-side scripts may install, restart, and +query it, but they should not own agent reasoning or task execution. diff --git a/ios/agentd/TOOLS_PARITY.md b/ios/agentd/TOOLS_PARITY.md new file mode 100644 index 0000000..0f46144 --- /dev/null +++ b/ios/agentd/TOOLS_PARITY.md @@ -0,0 +1,87 @@ +# Tools parity — iOS vs Android + +This is a first-pass audit of the tool surface. Marked ✅ when iOS has a +direct equivalent, ⚠️ when partial, ❌ when missing. + +## Screen observation + +| Capability | iOS | Notes | +|---|---|---| +| `get_screen` (foreground, UI tree, screenshot) | ✅ | `OPGetScreen` | +| Web DOM state (Safari) | ✅ | `web_content_dom_state` via app introspector | +| Recent apps / app switcher | ⚠️ | Metadata via SpringBoard state; no explicit `open_recent` | + +## Input + +| Capability | iOS | Notes | +|---|---|---| +| `tap`, `tap_element` | ✅ | `tap` (coords) + `tap_element` (id) | +| `long_press` | ✅ | present | +| `swipe` | ✅ | present | +| `type_text` | ✅ | Safari DOM + first-party UITextField bridge | +| `home` / `wake_and_home` | ✅ | present | +| `wait` | ✅ | present | + +## First-party data + +| Capability | iOS | Notes | +|---|---|---| +| `contacts_search` | ✅ | present | +| `calendar_search` | ✅ | present | +| `calls_search` | ✅ | present | +| `messages_search` | ✅ | present | +| Send a Message | ⚠️ | Only via UI navigation to Messages + type_text. No direct `messages_send`. | +| Place a Call | ⚠️ | Only via UI. No direct `calls_dial`. | +| Calendar create | ❌ | Add `calendar_create` — Android has direct create/edit. | +| Contacts create/edit | ❌ | Add `contacts_create` / `contacts_update`. | + +## Clipboard / interop + +| Capability | iOS | Notes | +|---|---|---| +| `clipboard_read` / `clipboard_write` | ✅ | present | +| `open_url` | ✅ | present | +| `open_app` | ✅ | present | +| Deep-link handoff (`x-callback-url`, universal links) | ⚠️ | Works via `open_url` but no dedicated `handoff` tool with expected-response schema. | + +## Memory / context + +| Capability | iOS | Notes | +|---|---|---| +| `memory_save`, `memory_search` | ✅ | present | +| `context_search` | ✅ | present | +| Chat history (last N voice turns) | ✅ | Added tonight — `OPRecordVoiceTurn` prepends recent turns to follow-up goals within 10s. | + +## Autonomy / meta + +| Capability | iOS | Notes | +|---|---|---| +| `finish_task`, `fail_task` | ✅ | present | +| `ask_user_confirmation` | ⚠️ | Not a model-emitted tool. iOS gates *any* UI-driving tool via `AutonomyMode=reviewed` and pauses with island Approve/Deny chips. Different mechanism, similar UX. | + +## Watchers / background jobs + +| Capability | iOS | Notes | +|---|---|---| +| Create / list / cancel `watcher` | ✅ | daemon commands exist | +| Recurring / due background jobs | ✅ | daemon commands exist | +| Model can create them mid-loop | ⚠️ | Tools are not in `OPModelToolNames()`. Only accessible via daemon RPC, not directly by the model. Consider adding `watcher_create`, `background_job_create` to the model tool list so it can schedule future work. | + +## Suggested additions (highest-impact) + +1. **`messages_send`** — write to iMessage via SpringBoard/Messages app plus a + composed UI flow. Direct tool preferred over multi-step UI navigation. +2. **`calls_dial`** — start a call to a phone number or contact. +3. **`calendar_create`** and **`contacts_create`** — EventKit / Contacts + frameworks via the daemon. +4. **`watcher_create` / `background_job_create`** — expose scheduler to the model. +5. **`ask_user_confirmation`** — turn the current island Approve/Deny into a + model-callable tool so the model itself decides when to pause. + +## Not applicable to iOS + +- Android accessibility service `dispatchGesture` — iOS uses UIKit hit-testing + bridge and SpringBoard tweak instead. Different implementation, same net + capability. +- `dispatchNotification` / notification listeners — iOS does not expose + notification stream to third parties. diff --git a/ios/agentd/control b/ios/agentd/control new file mode 100644 index 0000000..e52fea6 --- /dev/null +++ b/ios/agentd/control @@ -0,0 +1,9 @@ +Package: com.openphone.agentd +Name: OpenPhone Agent Daemon +Version: 0.1.0 +Architecture: iphoneos-arm64 +Description: Phone-resident OpenPhone agent daemon for owned rootless iOS devices. +Maintainer: OpenPhone +Author: OpenPhone +Section: Utilities +Depends: firmware (>= 15.0) diff --git a/ios/agentd/entitlements.plist b/ios/agentd/entitlements.plist new file mode 100644 index 0000000..94ea2fd --- /dev/null +++ b/ios/agentd/entitlements.plist @@ -0,0 +1,22 @@ + + + + + platform-application + + com.apple.private.security.no-container + + com.apple.private.skip-library-validation + + com.apple.private.memorystatus + + com.apple.developer.speech-recognition + + com.apple.private.tcc.allow + + kTCCServiceMicrophone + kTCCServiceSpeechRecognition + + + diff --git a/ios/agentd/launchd/com.openphone.agentd.plist b/ios/agentd/launchd/com.openphone.agentd.plist new file mode 100644 index 0000000..ae5609f --- /dev/null +++ b/ios/agentd/launchd/com.openphone.agentd.plist @@ -0,0 +1,24 @@ + + + + + Label + com.openphone.agentd + ProgramArguments + + /var/jb/usr/local/bin/openphone-agentd + + UserName + mobile + RunAtLoad + + KeepAlive + + StandardOutPath + /var/mobile/Library/OpenPhone/openphone-agentd.stdout.log + StandardErrorPath + /var/mobile/Library/OpenPhone/openphone-agentd.stderr.log + WorkingDirectory + /var/mobile + + diff --git a/ios/agentd/layout/Library/LaunchDaemons/com.openphone.agentd.plist b/ios/agentd/layout/Library/LaunchDaemons/com.openphone.agentd.plist new file mode 100644 index 0000000..ae5609f --- /dev/null +++ b/ios/agentd/layout/Library/LaunchDaemons/com.openphone.agentd.plist @@ -0,0 +1,24 @@ + + + + + Label + com.openphone.agentd + ProgramArguments + + /var/jb/usr/local/bin/openphone-agentd + + UserName + mobile + RunAtLoad + + KeepAlive + + StandardOutPath + /var/mobile/Library/OpenPhone/openphone-agentd.stdout.log + StandardErrorPath + /var/mobile/Library/OpenPhone/openphone-agentd.stderr.log + WorkingDirectory + /var/mobile + + diff --git a/ios/agentd/layout/Library/LaunchDaemons/com.openphone.protecteddatahelper.plist b/ios/agentd/layout/Library/LaunchDaemons/com.openphone.protecteddatahelper.plist new file mode 100644 index 0000000..0165c0e --- /dev/null +++ b/ios/agentd/layout/Library/LaunchDaemons/com.openphone.protecteddatahelper.plist @@ -0,0 +1,31 @@ + + + + + Label + com.openphone.protecteddatahelper + Program + /var/jb/bin/sh + ProgramArguments + + /var/jb/bin/sh + /var/jb/usr/local/bin/openphone-helper-wrapper.sh + + ProcessType + Interactive + POSIXSpawnType + Interactive + SessionCreate + + RunAtLoad + + KeepAlive + + StandardOutPath + /var/mobile/Library/OpenPhone/openphone-protected-data-helper.stdout.log + StandardErrorPath + /var/mobile/Library/OpenPhone/openphone-protected-data-helper.stderr.log + WorkingDirectory + /var/mobile + + diff --git a/ios/agentd/layout/Library/PreferenceLoader/Preferences/OpenPhoneAgentPrefs.plist b/ios/agentd/layout/Library/PreferenceLoader/Preferences/OpenPhoneAgentPrefs.plist new file mode 100644 index 0000000..dc7063d --- /dev/null +++ b/ios/agentd/layout/Library/PreferenceLoader/Preferences/OpenPhoneAgentPrefs.plist @@ -0,0 +1,9 @@ +{ + entry = { + bundle = OpenPhoneAgentPrefs; + cell = PSLinkCell; + detail = OpenPhoneAgentPrefsRootListController; + isController = 1; + label = "OpenPhone Agent"; + }; +} diff --git a/ios/agentd/layout/usr/local/bin/openphone-helper-wrapper.sh b/ios/agentd/layout/usr/local/bin/openphone-helper-wrapper.sh new file mode 100755 index 0000000..5c21a33 --- /dev/null +++ b/ios/agentd/layout/usr/local/bin/openphone-helper-wrapper.sh @@ -0,0 +1,13 @@ +#!/var/jb/bin/sh +# launchd starts the setuid protected-data helper through this /bin/sh wrapper so +# the helper inherits sh's unsandboxed session context. Launching the daemon +# binary directly from launchd applies the platform sandbox and the helper gets +# "authorization denied" on SMS/CallHistory/Calendar SQLite. See IOS_PLAN.md T2-7. +export OPENPHONE_AGENTD_STORE=/var/mobile/Library/OpenPhone/protected-data-helper +export OPENPHONE_AGENTD_ROLE=protected_data_helper +export OPENPHONE_AGENTD_DISABLE_TASK_REPAIR=1 +export OPENPHONE_AGENTD_DISABLE_VOLUME_TRIGGER=1 +export OPENPHONE_AGENTD_DISABLE_APP_UI_INTAKE=1 +export OPENPHONE_AGENTD_DISABLE_BACKGROUND_SCHEDULER=1 +export HOME=/var/mobile +exec /var/jb/usr/local/bin/openphone-protected-data-helper diff --git a/ios/agentd/prefs/OpenPhoneAgentPrefsResources/Info.plist b/ios/agentd/prefs/OpenPhoneAgentPrefsResources/Info.plist new file mode 100644 index 0000000..0415090 --- /dev/null +++ b/ios/agentd/prefs/OpenPhoneAgentPrefsResources/Info.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + OpenPhone Agent + CFBundleExecutable + OpenPhoneAgentPrefs + CFBundleIdentifier + com.openphone.agentprefs + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + OpenPhoneAgentPrefs + CFBundlePackageType + BNDL + CFBundleShortVersionString + 0.1.0 + CFBundleVersion + 1 + NSPrincipalClass + OpenPhoneAgentPrefsRootListController + + diff --git a/ios/agentd/src/agentctl.m b/ios/agentd/src/agentctl.m new file mode 100644 index 0000000..5c7efa7 --- /dev/null +++ b/ios/agentd/src/agentctl.m @@ -0,0 +1,362 @@ +#import + +#import +#import +#import +#import +#import +#import +#import +#import +#import + +static NSString *const OPDefaultStorePath = @"/var/mobile/Library/OpenPhone"; + +static NSString *OPSocketPath(void) { + const char *override = getenv("OPENPHONE_AGENTD_STORE"); + NSString *storePath = (override && override[0] != '\0') + ? [NSString stringWithUTF8String:override] : OPDefaultStorePath; + return [[storePath stringByAppendingPathComponent:@"run"] + stringByAppendingPathComponent:@"agentd.sock"]; +} + +static BOOL OPAgentCtlWriteAll(int fd, NSData *data) { + const uint8_t *bytes = (const uint8_t *)data.bytes; + NSUInteger remaining = data.length; + while (remaining > 0) { + ssize_t written = write(fd, bytes, remaining); + if (written > 0) { + bytes += written; + remaining -= (NSUInteger)written; + continue; + } + if (written < 0 && errno == EINTR) { + continue; + } + return NO; + } + return YES; +} + +static NSString *OPRequestFromArguments(int argc, char **argv) { + if (argc < 2 || argv[1] == NULL) { + return @"{\"command\":\"health\"}"; + } + NSString *first = [NSString stringWithUTF8String:argv[1]]; + if ([first hasPrefix:@"{"]) { + return first; + } + NSArray *actionCommands = @[ + @"home", + @"wake_and_home", + @"show_passcode", + @"unlock_with_passcode", + @"wait", + @"tap", + @"tap_element", + @"long_press", + @"swipe", + @"type_text", + @"open_app", + @"open_url" + ]; + if ([actionCommands containsObject:first]) { + NSMutableDictionary *action = [@{@"type": first} mutableCopy]; + if ([first isEqualToString:@"open_app"] && argc >= 3 && argv[2] != NULL) { + action[@"target"] = @{@"package": [NSString stringWithUTF8String:argv[2]]}; + } else if ([first isEqualToString:@"open_url"] && argc >= 3 && argv[2] != NULL) { + action[@"url"] = [NSString stringWithUTF8String:argv[2]]; + } else if ([first isEqualToString:@"unlock_with_passcode"] && argc >= 3 && argv[2] != NULL) { + action[@"passcode"] = [NSString stringWithUTF8String:argv[2]]; + } else if ([first isEqualToString:@"wait"] && argc >= 3 && argv[2] != NULL) { + action[@"duration_ms"] = @([[NSString stringWithUTF8String:argv[2]] integerValue]); + } else if (([first isEqualToString:@"tap"] || + [first isEqualToString:@"long_press"]) && argc >= 4) { + action[@"x"] = @([[NSString stringWithUTF8String:argv[2]] doubleValue]); + action[@"y"] = @([[NSString stringWithUTF8String:argv[3]] doubleValue]); + if ([first isEqualToString:@"long_press"] && argc >= 5 && argv[4] != NULL) { + action[@"duration_ms"] = @([[NSString stringWithUTF8String:argv[4]] integerValue]); + } + } else if ([first isEqualToString:@"tap_element"] && argc >= 3 && argv[2] != NULL) { + action[@"element_id"] = [NSString stringWithUTF8String:argv[2]]; + if (argc >= 5 && argv[3] != NULL && argv[4] != NULL) { + action[@"x"] = @([[NSString stringWithUTF8String:argv[3]] doubleValue]); + action[@"y"] = @([[NSString stringWithUTF8String:argv[4]] doubleValue]); + } + } else if ([first isEqualToString:@"swipe"] && argc >= 6) { + action[@"start_x"] = @([[NSString stringWithUTF8String:argv[2]] doubleValue]); + action[@"start_y"] = @([[NSString stringWithUTF8String:argv[3]] doubleValue]); + action[@"end_x"] = @([[NSString stringWithUTF8String:argv[4]] doubleValue]); + action[@"end_y"] = @([[NSString stringWithUTF8String:argv[5]] doubleValue]); + if (argc >= 7 && argv[6] != NULL) { + action[@"duration_ms"] = @([[NSString stringWithUTF8String:argv[6]] integerValue]); + } + } else if ([first isEqualToString:@"type_text"] && argc >= 3 && argv[2] != NULL) { + NSMutableString *text = [NSMutableString string]; + for (int i = 2; i < argc; i++) { + if (argv[i] == NULL) { + continue; + } + if (text.length > 0) { + [text appendString:@" "]; + } + [text appendString:[NSString stringWithUTF8String:argv[i]] ?: @""]; + } + action[@"text"] = text; + } + NSDictionary *request = @{@"command": @"execute_action", @"action": action}; + NSData *data = [NSJSONSerialization dataWithJSONObject:request options:0 error:nil]; + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"{\"command\":\"health\"}"; + } + NSMutableDictionary *request = [@{@"command": first} mutableCopy]; + if (argc >= 3 && argv[2] != NULL) { + NSString *second = [NSString stringWithUTF8String:argv[2]]; + if ([first isEqualToString:@"get_task"] || + [first isEqualToString:@"get_trajectory"] || + [first isEqualToString:@"stop_task"]) { + request[@"task_id"] = second; + } else if ([first isEqualToString:@"finish_task"]) { + request[@"task_id"] = second; + request[@"summary"] = @"agentctl finish_task"; + } else if ([first isEqualToString:@"fail_task"]) { + request[@"task_id"] = second; + request[@"reason"] = @"agentctl fail_task"; + } else if ([first isEqualToString:@"list_tasks"] || + [first isEqualToString:@"get_audit"]) { + request[@"limit"] = @([second integerValue]); + } else if ([first isEqualToString:@"model_configure"]) { + request[@"mode"] = second; + } else if ([first isEqualToString:@"agent_control"]) { + request[@"action"] = second; + request[@"source"] = @"openphone-agentctl"; + if ([second isEqualToString:@"pause"]) { + request[@"reason"] = @"agentctl pause"; + } + } else if ([first isEqualToString:@"list_apps"]) { + NSInteger limit = [second integerValue]; + if (limit > 0 || [second isEqualToString:@"0"]) { + request[@"limit"] = @(limit); + } else { + request[@"query"] = second; + } + } else if ([first isEqualToString:@"get_screen"]) { + NSString *lower = second.lowercaseString; + if ([lower isEqualToString:@"screenshot"] || [lower isEqualToString:@"capture"] + || [lower isEqualToString:@"true"] || [lower isEqualToString:@"1"]) { + request[@"include_screenshot"] = @YES; + } + } else if ([first isEqualToString:@"memory_save"]) { + request[@"text"] = second; + request[@"reason"] = @"agentctl memory_save"; + } else if ([first isEqualToString:@"memory_update"]) { + request[@"memory_id"] = second; + request[@"reason"] = @"agentctl memory_update"; + } else if ([first isEqualToString:@"memory_delete"]) { + request[@"memory_id"] = second; + request[@"reason"] = @"agentctl memory_delete"; + } else if ([first isEqualToString:@"memory_merge"]) { + request[@"target_memory_id"] = second; + request[@"reason"] = @"agentctl memory_merge"; + } else if ([first isEqualToString:@"memory_search"] || + [first isEqualToString:@"context_search"]) { + request[@"query"] = second; + request[@"reason"] = @"agentctl search"; + } else if ([first isEqualToString:@"commitment_create"]) { + request[@"title"] = second; + request[@"reason"] = @"agentctl commitment_create"; + } else if ([first isEqualToString:@"commitment_search"]) { + request[@"query"] = second; + request[@"reason"] = @"agentctl commitment_search"; + } else if ([first isEqualToString:@"commitment_update_status"]) { + request[@"commitment_id"] = second; + request[@"reason"] = @"agentctl commitment_update_status"; + } else if ([first isEqualToString:@"commitment_run_due"]) { + request[@"limit"] = @([second integerValue]); + request[@"reason"] = @"agentctl commitment_run_due"; + } else if ([first isEqualToString:@"watcher_create"]) { + request[@"title"] = second; + request[@"reason"] = @"agentctl watcher_create"; + } else if ([first isEqualToString:@"watcher_list"]) { + request[@"query"] = second; + request[@"reason"] = @"agentctl watcher_list"; + } else if ([first isEqualToString:@"watcher_stop"]) { + if ([second isEqualToString:@"all"]) { + request[@"all"] = @YES; + } else { + request[@"watcher_id"] = second; + } + request[@"reason"] = @"agentctl watcher_stop"; + } else if ([first isEqualToString:@"background_job_create"]) { + request[@"title"] = second; + NSMutableString *prompt = [NSMutableString string]; + for (int i = 3; i < argc; i++) { + if (argv[i] == NULL) { + continue; + } + if (prompt.length > 0) { + [prompt appendString:@" "]; + } + [prompt appendString:[NSString stringWithUTF8String:argv[i]] ?: @""]; + } + if (prompt.length > 0) { + request[@"prompt"] = prompt; + } + request[@"reason"] = @"agentctl background_job_create"; + } else if ([first isEqualToString:@"background_job_list"]) { + request[@"query"] = second; + request[@"reason"] = @"agentctl background_job_list"; + } else if ([first isEqualToString:@"background_job_stop"]) { + request[@"job_id"] = second; + request[@"reason"] = @"agentctl background_job_stop"; + } else if ([first isEqualToString:@"background_job_run_due"]) { + request[@"limit"] = @([second integerValue]); + request[@"reason"] = @"agentctl background_job_run_due"; + } else if ([first isEqualToString:@"hardware_trigger"]) { + request[@"trigger"] = second; + request[@"source"] = @"agentctl"; + request[@"reason"] = @"agentctl hardware_trigger"; + } else { + request[@"goal"] = second; + } + } + if (argc >= 4 && argv[3] != NULL) { + NSString *third = [NSString stringWithUTF8String:argv[3]]; + if ([first isEqualToString:@"stop_task"]) { + request[@"reason"] = third; + } else if ([first isEqualToString:@"finish_task"]) { + request[@"summary"] = third; + } else if ([first isEqualToString:@"fail_task"]) { + request[@"reason"] = third; + } else if ([first isEqualToString:@"model_configure"]) { + request[@"endpoint_url"] = third; + } else if ([first isEqualToString:@"agent_control"]) { + request[@"reason"] = third; + } else if ([first isEqualToString:@"run_task"]) { + NSString *lower = third.lowercaseString; + if ([lower isEqualToString:@"model"] || [lower isEqualToString:@"auto"] || + [lower isEqualToString:@"deterministic"]) { + request[@"mode"] = lower; + } else { + request[@"max_steps"] = @([third integerValue]); + } + } else if ([first isEqualToString:@"memory_save"]) { + request[@"type"] = third; + } else if ([first isEqualToString:@"memory_update"]) { + request[@"text"] = third; + } else if ([first isEqualToString:@"memory_delete"]) { + request[@"reason"] = third; + } else if ([first isEqualToString:@"memory_merge"]) { + request[@"source_memory_id"] = third; + } else if ([first isEqualToString:@"memory_search"] || + [first isEqualToString:@"context_search"]) { + request[@"limit"] = @([third integerValue]); + } else if ([first isEqualToString:@"commitment_create"]) { + if ([third longLongValue] > 0) { + request[@"due_at"] = @([third longLongValue]); + } else { + request[@"description"] = third; + } + } else if ([first isEqualToString:@"commitment_search"]) { + request[@"limit"] = @([third integerValue]); + } else if ([first isEqualToString:@"commitment_update_status"]) { + request[@"status"] = third; + } else if ([first isEqualToString:@"commitment_run_due"]) { + request[@"source"] = third; + } else if ([first isEqualToString:@"watcher_create"]) { + request[@"source"] = third; + } else if ([first isEqualToString:@"watcher_list"]) { + request[@"limit"] = @([third integerValue]); + } else if ([first isEqualToString:@"watcher_stop"]) { + request[@"reason"] = third; + } else if ([first isEqualToString:@"background_job_list"]) { + request[@"limit"] = @([third integerValue]); + } else if ([first isEqualToString:@"background_job_stop"]) { + request[@"reason"] = third; + } else if ([first isEqualToString:@"background_job_run_due"]) { + request[@"max_steps"] = @([third integerValue]); + } else if ([first isEqualToString:@"hardware_trigger"]) { + request[@"goal"] = third; + } else { + request[@"limit"] = @([third integerValue]); + } + } + if (argc >= 5 && argv[4] != NULL && [first isEqualToString:@"run_task"]) { + NSString *fourth = [NSString stringWithUTF8String:argv[4]]; + if (request[@"mode"]) { + request[@"max_steps"] = @([fourth integerValue]); + } else { + request[@"max_duration_ms"] = @([fourth integerValue]); + } + } else if (argc >= 5 && argv[4] != NULL && [first isEqualToString:@"model_configure"]) { + NSString *fourth = [NSString stringWithUTF8String:argv[4]]; + request[@"model"] = fourth; + } else if (argc >= 5 && argv[4] != NULL && [first isEqualToString:@"background_job_run_due"]) { + NSString *fourth = [NSString stringWithUTF8String:argv[4]]; + request[@"max_duration_ms"] = @([fourth integerValue]); + } else if (argc >= 5 && argv[4] != NULL && [first isEqualToString:@"memory_save"]) { + NSString *fourth = [NSString stringWithUTF8String:argv[4]]; + request[@"subject"] = fourth; + } else if (argc >= 5 && argv[4] != NULL && [first isEqualToString:@"memory_update"]) { + NSString *fourth = [NSString stringWithUTF8String:argv[4]]; + request[@"type"] = fourth; + } else if (argc >= 5 && argv[4] != NULL && [first isEqualToString:@"memory_merge"]) { + NSString *fourth = [NSString stringWithUTF8String:argv[4]]; + request[@"text"] = fourth; + } + if (argc >= 6 && argv[5] != NULL && [first isEqualToString:@"memory_update"]) { + NSString *fifth = [NSString stringWithUTF8String:argv[5]]; + request[@"subject"] = fifth; + } else if (argc >= 6 && argv[5] != NULL && [first isEqualToString:@"run_task"] && request[@"mode"]) { + NSString *fifth = [NSString stringWithUTF8String:argv[5]]; + request[@"max_duration_ms"] = @([fifth integerValue]); + } else if (argc >= 6 && argv[5] != NULL && [first isEqualToString:@"model_configure"]) { + NSString *fifth = [NSString stringWithUTF8String:argv[5]]; + request[@"enabled"] = @([fifth boolValue]); + } + NSData *data = [NSJSONSerialization dataWithJSONObject:request options:0 error:nil]; + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"{\"command\":\"health\"}"; +} + +int main(int argc, char **argv) { + @autoreleasepool { + signal(SIGPIPE, SIG_IGN); + NSString *request = OPRequestFromArguments(argc, argv); + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + fprintf(stderr, "socket failed: %s\n", strerror(errno)); + return 1; + } + + struct sockaddr_un address; + memset(&address, 0, sizeof(address)); + address.sun_family = AF_UNIX; + NSString *socketPath = OPSocketPath(); + strlcpy(address.sun_path, socketPath.UTF8String, sizeof(address.sun_path)); + + if (connect(fd, (struct sockaddr *)&address, sizeof(address)) != 0) { + fprintf(stderr, "connect %s failed: %s\n", socketPath.UTF8String, strerror(errno)); + close(fd); + return 1; + } + + NSData *requestData = [[request stringByAppendingString:@"\n"] dataUsingEncoding:NSUTF8StringEncoding]; + if (!OPAgentCtlWriteAll(fd, requestData)) { + fprintf(stderr, "write failed: %s\n", strerror(errno)); + close(fd); + return 1; + } + shutdown(fd, SHUT_WR); + + char buffer[4096]; + while (true) { + ssize_t count = read(fd, buffer, sizeof(buffer)); + if (count > 0) { + fwrite(buffer, 1, (size_t)count, stdout); + continue; + } + break; + } + close(fd); + } + return 0; +} diff --git a/ios/agentd/src/app_introspector.xm b/ios/agentd/src/app_introspector.xm new file mode 100644 index 0000000..c18e6f2 --- /dev/null +++ b/ios/agentd/src/app_introspector.xm @@ -0,0 +1,2310 @@ +#import +#import + +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import + +static NSString *const OPAIStorePath = @"/var/mobile/Library/OpenPhone"; +static NSString *const OPAIAppUIDir = @"/var/mobile/Library/OpenPhone/app-ui"; +static const char *OPAILogPath = "/var/mobile/Library/OpenPhone/openphone-app-introspector.log"; + +static long long OPAINowMs(void) { + return (long long)([[NSDate date] timeIntervalSince1970] * 1000.0); +} + +static void OPAILog(NSString *format, ...) { + va_list args; + va_start(args, format); + NSString *message = [[NSString alloc] initWithFormat:format arguments:args]; + va_end(args); + + NSDateFormatter *formatter = [[NSDateFormatter alloc] init]; + formatter.locale = [NSLocale localeWithLocaleIdentifier:@"en_US_POSIX"]; + formatter.dateFormat = @"yyyy-MM-dd'T'HH:mm:ssZZZZZ"; + NSString *line = [NSString stringWithFormat:@"%@ %@\n", + [formatter stringFromDate:[NSDate date]], + message ?: @""]; + NSData *data = [line dataUsingEncoding:NSUTF8StringEncoding]; + if (data.length == 0) { + return; + } + if (![[NSFileManager defaultManager] fileExistsAtPath:@(OPAILogPath)]) { + [[NSFileManager defaultManager] createFileAtPath:@(OPAILogPath) + contents:nil + attributes:nil]; + } + NSFileHandle *handle = [NSFileHandle fileHandleForWritingAtPath:@(OPAILogPath)]; + @try { + [handle seekToEndOfFile]; + [handle writeData:data]; + } @catch (__unused NSException *exception) { + } + [handle closeFile]; +} + +static NSString *OPAIString(id value) { + if ([value isKindOfClass:[NSString class]]) { + return [(NSString *)value stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + } + return @""; +} + +static double OPAINumberDouble(id value, double fallback) { + return [value respondsToSelector:@selector(doubleValue)] ? [value doubleValue] : fallback; +} + +static long long OPAINumberLongLong(id value, long long fallback) { + return [value respondsToSelector:@selector(longLongValue)] ? [value longLongValue] : fallback; +} + +static NSString *OPAIJSONStringLiteral(NSString *value) { + NSError *error = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:@[value ?: @""] + options:0 + error:&error]; + if (!data || error) { + return @"\"\""; + } + NSString *arrayJSON = [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; + if (arrayJSON.length < 2) { + return @"\"\""; + } + return [arrayJSON substringWithRange:NSMakeRange(1, arrayJSON.length - 2)]; +} + +static NSDictionary *OPAIDictionaryFromJSONString(NSString *json) { + if (json.length == 0) { + return @{}; + } + NSData *data = [json dataUsingEncoding:NSUTF8StringEncoding]; + if (data.length == 0) { + return @{}; + } + id value = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + return [value isKindOfClass:[NSDictionary class]] ? value : @{}; +} + +static NSString *OPAIAppBundleId(void) { + NSString *bundleId = [[NSBundle mainBundle] bundleIdentifier]; + return bundleId.length > 0 ? bundleId : NSProcessInfo.processInfo.processName ?: @"unknown"; +} + +static BOOL OPAIIsWebContentProcess(void) { + NSString *bundleId = OPAIAppBundleId(); + NSString *processName = NSProcessInfo.processInfo.processName ?: @""; + return [bundleId isEqualToString:@"com.apple.WebKit.WebContent"] || + [processName isEqualToString:@"com.apple.WebKit.WebContent"] || + [processName containsString:@"WebKit.WebContent"]; +} + +static NSString *OPAIEffectiveAppBundleId(void) { + if (OPAIIsWebContentProcess()) { + return @"com.apple.mobilesafari"; + } + return OPAIAppBundleId(); +} + +static NSString *OPAIProviderName(void) { + return OPAIIsWebContentProcess() + ? @"OpenPhoneAppIntrospector.WebContentAccessibility" + : @"OpenPhoneAppIntrospector.UIKitAccessibility"; +} + +static NSString *OPAIInputProviderName(void) { + return OPAIIsWebContentProcess() + ? @"OpenPhoneAppIntrospector.WebContentInput" + : @"OpenPhoneAppIntrospector.AppInput"; +} + +static NSString *OPAIElementScope(void) { + return OPAIIsWebContentProcess() ? @"web_content_process" : @"app_process"; +} + +static NSString *OPAIElementRiskHint(void) { + return OPAIIsWebContentProcess() ? @"web_content_process" : @"app_process"; +} + +static NSString *OPAISafeFilenameForBundleId(NSString *bundleId) { + NSMutableString *safe = [NSMutableString string]; + NSCharacterSet *allowed = [NSCharacterSet characterSetWithCharactersInString: + @"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-"]; + for (NSUInteger index = 0; index < bundleId.length; index++) { + unichar ch = [bundleId characterAtIndex:index]; + if ([allowed characterIsMember:ch]) { + [safe appendFormat:@"%C", ch]; + } else { + [safe appendString:@"_"]; + } + } + return safe.length > 0 ? safe : @"unknown"; +} + +static NSArray *OPAIBoundsArray(CGRect rect) { + return @[ + @((double)rect.origin.x), + @((double)rect.origin.y), + @((double)rect.size.width), + @((double)rect.size.height) + ]; +} + +static NSArray *OPAIApplicationWindows(UIApplication *application) { + NSMutableArray *windows = [NSMutableArray array]; + @try { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + for (UIWindow *window in application.windows ?: @[]) { + if (window && ![windows containsObject:window]) { + [windows addObject:window]; + } + } +#pragma clang diagnostic pop + if (@available(iOS 13.0, *)) { + for (UIScene *scene in application.connectedScenes ?: [NSSet set]) { + if (![scene isKindOfClass:[UIWindowScene class]]) { + continue; + } + UIWindowScene *windowScene = (UIWindowScene *)scene; + for (UIWindow *window in windowScene.windows ?: @[]) { + if (window && ![windows containsObject:window]) { + [windows addObject:window]; + } + } + } + } + } @catch (__unused NSException *exception) { + } + return windows; +} + +static NSString *OPAIViewKind(UIView *view) { + if ([view isKindOfClass:[UIButton class]]) { + return @"button"; + } + if ([view isKindOfClass:[UITextField class]]) { + return @"text_field"; + } + if ([view isKindOfClass:[UITextView class]]) { + return @"text_area"; + } + if ([view isKindOfClass:[UISearchBar class]]) { + return @"search"; + } + if ([view isKindOfClass:[UISwitch class]]) { + return @"switch"; + } + if ([view isKindOfClass:[UISegmentedControl class]]) { + return @"segmented_control"; + } + if ([view isKindOfClass:[UILabel class]]) { + return @"text"; + } + if (view.isAccessibilityElement) { + return @"accessibility_element"; + } + return @"view"; +} + +static NSString *OPAIKindForAccessibilityTraits(UIAccessibilityTraits traits) { + if ((traits & UIAccessibilityTraitButton) == UIAccessibilityTraitButton) { + return @"button"; + } + if ((traits & UIAccessibilityTraitLink) == UIAccessibilityTraitLink) { + return @"link"; + } + if ((traits & UIAccessibilityTraitKeyboardKey) == UIAccessibilityTraitKeyboardKey) { + return @"keyboard_key"; + } + if ((traits & UIAccessibilityTraitSearchField) == UIAccessibilityTraitSearchField) { + return @"search_field"; + } + if ((traits & UIAccessibilityTraitStaticText) == UIAccessibilityTraitStaticText) { + return @"text"; + } + return @"accessibility_element"; +} + +static BOOL OPAIViewIsSensitive(UIView *view) { + if ([view isKindOfClass:[UITextField class]]) { + return ((UITextField *)view).secureTextEntry; + } + UIView *textInput = nil; + if ([view isKindOfClass:[UISearchBar class]]) { + @try { + textInput = ((UISearchBar *)view).searchTextField; + } @catch (__unused NSException *exception) { + textInput = nil; + } + } + if ([textInput isKindOfClass:[UITextField class]]) { + return ((UITextField *)textInput).secureTextEntry; + } + return NO; +} + +static NSString *OPAIViewLabel(UIView *view) { + BOOL sensitive = OPAIViewIsSensitive(view); + NSMutableArray *candidates = [NSMutableArray array]; + NSString *accessibilityLabel = OPAIString(view.accessibilityLabel); + if (accessibilityLabel.length > 0) { + [candidates addObject:accessibilityLabel]; + } + if ([view isKindOfClass:[UIButton class]]) { + NSString *title = OPAIString([(UIButton *)view titleForState:UIControlStateNormal]); + if (title.length > 0) { + [candidates addObject:title]; + } + } + if ([view isKindOfClass:[UILabel class]]) { + NSString *text = OPAIString([(UILabel *)view text]); + if (text.length > 0) { + [candidates addObject:text]; + } + } + if ([view isKindOfClass:[UITextField class]]) { + UITextField *field = (UITextField *)view; + NSString *placeholder = OPAIString(field.placeholder); + if (placeholder.length > 0) { + [candidates addObject:placeholder]; + } + NSString *text = sensitive ? @"" : OPAIString(field.text); + if (text.length > 0) { + [candidates addObject:text]; + } + } + if ([view isKindOfClass:[UITextView class]]) { + NSString *text = OPAIString([(UITextView *)view text]); + if (text.length > 0 && text.length <= 300) { + [candidates addObject:text]; + } + } + NSString *accessibilityValue = sensitive ? @"" : OPAIString(view.accessibilityValue); + if (accessibilityValue.length > 0) { + [candidates addObject:accessibilityValue]; + } + for (NSString *candidate in candidates) { + if (candidate.length > 0) { + return candidate.length <= 300 ? candidate : [candidate substringToIndex:300]; + } + } + return @""; +} + +static NSString *OPAIEditableTextValue(UIView *view) { + if (OPAIViewIsSensitive(view)) { + return @""; + } + if ([view isKindOfClass:[UITextField class]]) { + return OPAIString([(UITextField *)view text]); + } + if ([view isKindOfClass:[UITextView class]]) { + return OPAIString([(UITextView *)view text]); + } + return @""; +} + +static NSString *OPAIAccessibilityStringValue(id object, SEL selector) { + if (!object || !selector || ![object respondsToSelector:selector]) { + return @""; + } + id value = nil; + @try { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Warc-performSelector-leaks" + value = [object performSelector:selector]; +#pragma clang diagnostic pop + } @catch (__unused NSException *exception) { + value = nil; + } + return OPAIString(value); +} + +static CGRect OPAIAccessibilityFrameValue(id object) { + if (!object || ![object respondsToSelector:@selector(accessibilityFrame)]) { + return CGRectZero; + } + @try { + return [object accessibilityFrame]; + } @catch (__unused NSException *exception) { + return CGRectZero; + } +} + +static NSArray *OPAIAccessibilityChildrenForView(UIView *view) { + if (!view) { + return @[]; + } + NSMutableArray *children = [NSMutableArray array]; + @try { + NSArray *elements = [view.accessibilityElements isKindOfClass:[NSArray class]] + ? view.accessibilityElements : nil; + for (id element in elements ?: @[]) { + if (element && element != view) { + [children addObject:element]; + if (children.count >= 80) { + return children; + } + } + } + } @catch (__unused NSException *exception) { + } + if (children.count > 0 || + ![view respondsToSelector:@selector(accessibilityElementCount)] || + ![view respondsToSelector:@selector(accessibilityElementAtIndex:)]) { + return children; + } + @try { + NSInteger count = [view accessibilityElementCount]; + if (count <= 0 || count > 80) { + return children; + } + for (NSInteger index = 0; index < count && children.count < 80; index++) { + id element = [view accessibilityElementAtIndex:index]; + if (element && element != view) { + [children addObject:element]; + } + } + } @catch (__unused NSException *exception) { + } + return children; +} + +static BOOL OPAIViewLooksInteractive(UIView *view, NSString *label, NSString *identifier) { + (void)label; + (void)identifier; + return view.isAccessibilityElement || + view.userInteractionEnabled || + [view isKindOfClass:[UIButton class]] || + [view isKindOfClass:[UITextField class]] || + [view isKindOfClass:[UITextView class]] || + [view isKindOfClass:[UISearchBar class]] || + [view isKindOfClass:[UISwitch class]] || + [view isKindOfClass:[UISegmentedControl class]]; +} + +static NSDictionary *OPAIElementSnapshotForView(UIView *view, + NSString *bundleId, + NSInteger windowIndex, + NSUInteger elementIndex, + CGRect bounds, + NSString *label, + BOOL sensitive, + NSString *identifier) { + NSString *safeBundle = OPAISafeFilenameForBundleId(bundleId ?: @"app"); + NSString *elementId = [NSString stringWithFormat:@"app-%@-%ld-%lu", + safeBundle, (long)windowIndex, (unsigned long)elementIndex]; + NSMutableDictionary *element = [@{ + @"id": elementId, + @"kind": OPAIViewKind(view), + @"class": NSStringFromClass([view class]) ?: @"", + @"label": sensitive ? @"" : label ?: @"", + @"bounds": OPAIBoundsArray(bounds), + @"enabled": @(view.userInteractionEnabled), + @"focused": @(view.isFirstResponder), + @"window_id": @(windowIndex), + @"source_bundle_id": bundleId ?: @"", + @"scope": OPAIElementScope(), + @"sensitive": @(sensitive), + @"risk_hint": OPAIElementRiskHint() + } mutableCopy]; + if (identifier.length > 0) { + element[@"view_id"] = identifier; + } + if (!sensitive) { + NSString *textValue = OPAIEditableTextValue(view); + if (textValue.length > 0) { + element[@"value"] = textValue.length <= 300 ? textValue : [textValue substringToIndex:300]; + } + NSString *value = OPAIString(view.accessibilityValue); + if (value.length > 0 && !element[@"value"]) { + element[@"value"] = value.length <= 300 ? value : [value substringToIndex:300]; + } + } + return element; +} + +static NSDictionary *OPAIElementSnapshotForAccessibilityObject(id object, + NSString *bundleId, + NSInteger windowIndex, + NSUInteger elementIndex, + CGRect bounds) { + NSString *safeBundle = OPAISafeFilenameForBundleId(bundleId ?: @"app"); + NSString *elementId = [NSString stringWithFormat:@"app-%@-%ld-%lu", + safeBundle, (long)windowIndex, (unsigned long)elementIndex]; + NSString *label = OPAIAccessibilityStringValue(object, @selector(accessibilityLabel)); + NSString *value = OPAIAccessibilityStringValue(object, @selector(accessibilityValue)); + NSString *identifier = OPAIAccessibilityStringValue(object, @selector(accessibilityIdentifier)); + UIAccessibilityTraits traits = 0; + if ([object respondsToSelector:@selector(accessibilityTraits)]) { + @try { + traits = [object accessibilityTraits]; + } @catch (__unused NSException *exception) { + traits = 0; + } + } + NSMutableDictionary *element = [@{ + @"id": elementId, + @"kind": OPAIKindForAccessibilityTraits(traits), + @"class": NSStringFromClass([object class]) ?: @"", + @"label": label.length <= 300 ? label : [label substringToIndex:300], + @"bounds": OPAIBoundsArray(bounds), + @"enabled": @YES, + @"focused": @NO, + @"window_id": @(windowIndex), + @"source_bundle_id": bundleId ?: @"", + @"scope": OPAIElementScope(), + @"sensitive": @NO, + @"risk_hint": OPAIElementRiskHint(), + @"accessibility_backed": @YES, + @"accessibility_traits": @((unsigned long long)traits) + } mutableCopy]; + if (value.length > 0) { + element[@"value"] = value.length <= 300 ? value : [value substringToIndex:300]; + } + if (identifier.length > 0) { + element[@"view_id"] = identifier; + } + return element; +} + +static NSDictionary *OPAIWindowSnapshot(UIWindow *window, NSInteger windowIndex) { + return @{ + @"id": @(windowIndex), + @"type": @((double)window.windowLevel), + @"focused": @(window.isKeyWindow), + @"active": @(!window.hidden && window.alpha >= 0.02), + @"bounds": OPAIBoundsArray(window.bounds) + }; +} + +static void OPAICollectAccessibilityChildren(UIView *view, + NSString *bundleId, + NSInteger windowIndex, + NSUInteger *elementIndex, + NSMutableArray *interactiveElements, + NSMutableArray *visibleText) { + if (!view || !elementIndex || interactiveElements.count >= 120) { + return; + } + for (id child in OPAIAccessibilityChildrenForView(view)) { + if (interactiveElements.count >= 120) { + return; + } + if ([child isKindOfClass:[UIView class]]) { + continue; + } + NSString *label = OPAIAccessibilityStringValue(child, @selector(accessibilityLabel)); + NSString *value = OPAIAccessibilityStringValue(child, @selector(accessibilityValue)); + if (label.length == 0 && value.length == 0) { + continue; + } + CGRect frame = OPAIAccessibilityFrameValue(child); + if (CGRectIsEmpty(frame) || frame.size.width < 1.0 || frame.size.height < 1.0) { + continue; + } + if (label.length > 0 && visibleText.count < 120 && + ![visibleText containsObject:label]) { + [visibleText addObject:label.length <= 300 ? label : [label substringToIndex:300]]; + } + if (value.length > 0 && visibleText.count < 120 && + ![visibleText containsObject:value]) { + [visibleText addObject:value.length <= 300 ? value : [value substringToIndex:300]]; + } + NSDictionary *element = OPAIElementSnapshotForAccessibilityObject(child, bundleId, + windowIndex, *elementIndex, frame); + (*elementIndex)++; + [interactiveElements addObject:element]; + } +} + +static void OPAICollectViewTree(UIView *view, + UIWindow *window, + NSString *bundleId, + NSInteger windowIndex, + NSInteger depth, + NSUInteger *elementIndex, + NSMutableArray *interactiveElements, + NSMutableArray *visibleText) { + if (!view || depth > 32 || interactiveElements.count >= 120) { + return; + } + if (view.hidden || view.alpha < 0.02) { + return; + } + CGRect bounds = [view convertRect:view.bounds toView:window]; + if (CGRectIsEmpty(bounds) || bounds.size.width < 1.0 || bounds.size.height < 1.0) { + return; + } + + BOOL sensitive = OPAIViewIsSensitive(view); + NSString *label = OPAIViewLabel(view); + if (!sensitive) { + if (label.length > 0 && visibleText.count < 120 && + ![visibleText containsObject:label]) { + [visibleText addObject:label]; + } + NSString *textValue = OPAIEditableTextValue(view); + if (textValue.length > 0 && visibleText.count < 120 && + ![visibleText containsObject:textValue]) { + [visibleText addObject:textValue.length <= 300 ? textValue : [textValue substringToIndex:300]]; + } + } + + NSString *identifier = OPAIString(view.accessibilityIdentifier); + BOOL interactive = OPAIViewLooksInteractive(view, label, identifier); + BOOL editableText = [view isKindOfClass:[UITextField class]] || + [view isKindOfClass:[UITextView class]] || + [view isKindOfClass:[UISearchBar class]]; + if (interactive && (label.length > 0 || identifier.length > 0 || editableText)) { + NSDictionary *element = OPAIElementSnapshotForView(view, bundleId, windowIndex, + *elementIndex, bounds, label, sensitive, identifier); + (*elementIndex)++; + [interactiveElements addObject:element]; + if (interactiveElements.count >= 120) { + return; + } + } + + OPAICollectAccessibilityChildren(view, bundleId, windowIndex, elementIndex, + interactiveElements, visibleText); + if (interactiveElements.count >= 120) { + return; + } + + for (UIView *subview in view.subviews ?: @[]) { + OPAICollectViewTree(subview, window, bundleId, windowIndex, depth + 1, + elementIndex, interactiveElements, visibleText); + if (interactiveElements.count >= 120) { + return; + } + } +} + +static BOOL OPAIViewCanEvaluateJavaScript(UIView *view) { + SEL selector = NSSelectorFromString(@"evaluateJavaScript:completionHandler:"); + return view && [view respondsToSelector:selector]; +} + +static UIView *OPAIFindWebViewInView(UIView *view, NSInteger depth) { + if (!view || depth > 32 || view.hidden || view.alpha < 0.02) { + return nil; + } + NSString *className = NSStringFromClass([view class]) ?: @""; + if (OPAIViewCanEvaluateJavaScript(view) || + [className isEqualToString:@"_SFWebView"] || + [className isEqualToString:@"WKWebView"] || + [className isEqualToString:@"WKContentView"]) { + return view; + } + for (UIView *subview in view.subviews ?: @[]) { + UIView *match = OPAIFindWebViewInView(subview, depth + 1); + if (match) { + return match; + } + } + return nil; +} + +static UIView *OPAIFindFirstWebView(NSArray *windows) { + for (UIWindow *window in windows ?: @[]) { + if (!window || window.hidden || window.alpha < 0.02) { + continue; + } + UIView *match = OPAIFindWebViewInView(window, 0); + if (match) { + return match; + } + } + return nil; +} + +static NSString *OPAIWebDOMQueryFunction(void) { + return + @"function opq(){" + "function clean(v){return (v==null?'':String(v)).replace(/\\s+/g,' ').trim().slice(0,300);}" + "function visible(el){var r=el.getBoundingClientRect();var s=getComputedStyle(el);" + "return r.width>=1&&r.height>=1&&s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0';}" + "function label(el){return clean(el.getAttribute('aria-label')||el.getAttribute('title')||" + "el.getAttribute('placeholder')||el.innerText||el.value||el.textContent||el.tagName);}" + "function css(el){var p=[];for(var n=el;n&&n.nodeType===1&&p.length<5;n=n.parentElement){" + "var part=n.tagName.toLowerCase();if(n.id){part+='#'+n.id;p.unshift(part);break;}" + "var i=1,s=n;while((s=s.previousElementSibling)!=null){if(s.tagName===n.tagName)i++;}" + "p.unshift(part+':nth-of-type('+i+')');}return p.join('>');}" + "var selector='input,textarea,select,button,a,[contenteditable=\"\"],[contenteditable=\"true\"]," + "[role=\"textbox\"],[role=\"button\"],[tabindex]';" + "var nodes=Array.prototype.slice.call(document.querySelectorAll(selector)).filter(visible).slice(0,80);" + "var active=document.activeElement;" + "var elements=nodes.map(function(el,i){var r=el.getBoundingClientRect();var tag=el.tagName.toLowerCase();" + "var type=clean(el.getAttribute('type')).toLowerCase();var sensitive=(tag==='input'&&type==='password');" + "var value=sensitive?'':clean(('value' in el)?el.value:'');" + "return {index:i,tag:tag,type:type,role:clean(el.getAttribute('role')).toLowerCase()," + "label:label(el),value:value,disabled:!!el.disabled,focused:el===active,editable:" + "(tag==='textarea'||(tag==='input'&&!['button','submit','reset','checkbox','radio','file','image','range','color'].includes(type))||el.isContentEditable||el.getAttribute('role')==='textbox')," + "href:clean(el.href),selector:css(el),rect:{x:r.left,y:r.top,width:r.width,height:r.height}};});" + "var texts=[];function add(t){t=clean(t);if(t&&texts.indexOf(t)<0&&texts.length<40)texts.push(t);}" + "add(document.title);var walker=document.createTreeWalker(document.body||document.documentElement," + "NodeFilter.SHOW_TEXT,{acceptNode:function(n){var t=clean(n.nodeValue);if(!t)return NodeFilter.FILTER_REJECT;" + "var p=n.parentElement;if(!p||!visible(p))return NodeFilter.FILTER_REJECT;return NodeFilter.FILTER_ACCEPT;}});" + "var n;while((n=walker.nextNode())&&texts.length<40)add(n.nodeValue);" + "var vv=window.visualViewport;" + "return {status:'ok',url:String(location.href),title:clean(document.title),active_index:nodes.indexOf(active)," + "viewport:{width:window.innerWidth||1,height:window.innerHeight||1,scale:vv?vv.scale:1," + "offset_left:vv?vv.offsetLeft:0,offset_top:vv?vv.offsetTop:0},visible_text:texts,elements:elements};}"; +} + +static NSDictionary *OPAIWebViewEvaluateJSON(UIView *webView, NSString *script, NSTimeInterval timeoutSeconds) { + if (!OPAIViewCanEvaluateJavaScript(webView)) { + return @{@"status": @"unavailable", @"reason": @"webview_evaluate_javascript_unavailable"}; + } + SEL selector = NSSelectorFromString(@"evaluateJavaScript:completionHandler:"); + __block BOOL done = NO; + __block id rawResult = nil; + __block NSError *rawError = nil; + void (^completion)(id, NSError *) = ^(id result, NSError *error) { + rawResult = result; + rawError = error; + done = YES; + }; + @try { + typedef void (*EvalSend)(id, SEL, id, id); + ((EvalSend)objc_msgSend)(webView, selector, script ?: @"", completion); + } @catch (NSException *exception) { + return @{ + @"status": @"unavailable", + @"reason": @"webview_evaluate_exception", + @"exception_name": exception.name ?: @"", + @"exception_reason": exception.reason ?: @"" + }; + } + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:timeoutSeconds > 0.0 ? timeoutSeconds : 0.75]; + while (!done && [deadline timeIntervalSinceNow] > 0.0) { + [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode + beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.02]]; + } + if (!done) { + return @{@"status": @"unavailable", @"reason": @"webview_evaluate_timeout"}; + } + if (rawError) { + return @{ + @"status": @"unavailable", + @"reason": @"webview_evaluate_error", + @"error": rawError.localizedDescription ?: @"" + }; + } + if ([rawResult isKindOfClass:[NSString class]]) { + NSDictionary *parsed = OPAIDictionaryFromJSONString(rawResult); + if (parsed.count > 0) { + return parsed; + } + } + if ([rawResult isKindOfClass:[NSDictionary class]]) { + return rawResult; + } + return @{@"status": @"unavailable", @"reason": @"webview_evaluate_empty_result"}; +} + +static NSString *OPAIWebDOMDiscoveryScript(void) { + return [NSString stringWithFormat:@"(function(){%@ return JSON.stringify(opq());})()", + OPAIWebDOMQueryFunction()]; +} + +static void OPAIAppendUniqueText(NSMutableArray *visibleText, NSString *text) { + NSString *clean = OPAIString(text); + if (clean.length > 0 && visibleText.count < 120 && ![visibleText containsObject:clean]) { + [visibleText addObject:clean.length <= 300 ? clean : [clean substringToIndex:300]]; + } +} + +static NSString *OPAIWebElementKind(NSDictionary *domElement) { + NSString *tag = [domElement[@"tag"] isKindOfClass:[NSString class]] ? domElement[@"tag"] : @""; + NSString *type = [domElement[@"type"] isKindOfClass:[NSString class]] ? domElement[@"type"] : @""; + NSString *role = [domElement[@"role"] isKindOfClass:[NSString class]] ? domElement[@"role"] : @""; + if ([role isEqualToString:@"textbox"] || [tag isEqualToString:@"textarea"] || + ([tag isEqualToString:@"input"] && ![type isEqualToString:@"button"] && + ![type isEqualToString:@"submit"] && ![type isEqualToString:@"reset"])) { + return @"web_text_field"; + } + if ([tag isEqualToString:@"button"] || [role isEqualToString:@"button"] || + ([tag isEqualToString:@"input"] && ([type isEqualToString:@"button"] || + [type isEqualToString:@"submit"] || [type isEqualToString:@"reset"]))) { + return @"web_button"; + } + if ([tag isEqualToString:@"a"]) { + return @"web_link"; + } + if ([tag isEqualToString:@"select"]) { + return @"web_select"; + } + return @"web_element"; +} + +static NSArray *OPAIWebDOMElements(NSDictionary *dom, + CGRect webBounds, + NSString *bundleId) { + NSArray *elements = [dom[@"elements"] isKindOfClass:[NSArray class]] ? dom[@"elements"] : @[]; + NSDictionary *viewport = [dom[@"viewport"] isKindOfClass:[NSDictionary class]] ? dom[@"viewport"] : @{}; + double viewportWidth = MAX(1.0, OPAINumberDouble(viewport[@"width"], webBounds.size.width)); + double viewportHeight = MAX(1.0, OPAINumberDouble(viewport[@"height"], webBounds.size.height)); + NSString *safeBundle = OPAISafeFilenameForBundleId(bundleId ?: @"app"); + NSMutableArray *snapshots = [NSMutableArray array]; + NSUInteger index = 0; + for (id object in elements) { + if (![object isKindOfClass:[NSDictionary class]] || snapshots.count >= 80) { + continue; + } + NSDictionary *domElement = object; + NSDictionary *rect = [domElement[@"rect"] isKindOfClass:[NSDictionary class]] + ? domElement[@"rect"] : @{}; + double x = OPAINumberDouble(rect[@"x"], 0.0); + double y = OPAINumberDouble(rect[@"y"], 0.0); + double width = OPAINumberDouble(rect[@"width"], 0.0); + double height = OPAINumberDouble(rect[@"height"], 0.0); + if (width < 1.0 || height < 1.0) { + continue; + } + CGRect bounds = CGRectMake(webBounds.origin.x + x * webBounds.size.width / viewportWidth, + webBounds.origin.y + y * webBounds.size.height / viewportHeight, + width * webBounds.size.width / viewportWidth, + height * webBounds.size.height / viewportHeight); + NSString *label = OPAIString(domElement[@"label"]); + NSString *value = OPAIString(domElement[@"value"]); + NSString *selector = OPAIString(domElement[@"selector"]); + long long domIndex = OPAINumberLongLong(domElement[@"index"], (long long)index); + NSMutableDictionary *snapshot = [@{ + @"id": [NSString stringWithFormat:@"app-%@-web-%lld", safeBundle, domIndex], + @"kind": OPAIWebElementKind(domElement), + @"class": @"DOMElement", + @"label": label.length <= 300 ? label : [label substringToIndex:300], + @"bounds": OPAIBoundsArray(bounds), + @"enabled": @(![domElement[@"disabled"] boolValue]), + @"focused": @([domElement[@"focused"] boolValue]), + @"window_id": @0, + @"source_bundle_id": bundleId ?: @"", + @"scope": @"web_content_process", + @"input_scope": @"app_process", + @"sensitive": @NO, + @"risk_hint": @"web_content_process", + @"dom_index": @(domIndex), + @"tag": OPAIString(domElement[@"tag"]), + @"input_type": OPAIString(domElement[@"type"]) + } mutableCopy]; + if (value.length > 0) { + snapshot[@"value"] = value.length <= 300 ? value : [value substringToIndex:300]; + } + if (selector.length > 0) { + snapshot[@"view_id"] = selector; + } + [snapshots addObject:snapshot]; + index++; + } + return snapshots; +} + +static NSDictionary *OPAIWebDOMSnapshot(UIView *webView, + CGRect webBounds, + NSString *bundleId, + NSMutableArray *visibleText, + NSMutableArray *interactiveElements) { + if (!webView || ![bundleId isEqualToString:@"com.apple.mobilesafari"]) { + return @{@"status": @"not_applicable"}; + } + NSDictionary *dom = OPAIWebViewEvaluateJSON(webView, OPAIWebDOMDiscoveryScript(), 0.80); + if (![dom[@"status"] isEqualToString:@"ok"]) { + return dom.count > 0 ? dom : @{@"status": @"unavailable", @"reason": @"web_dom_empty"}; + } + for (id text in ([dom[@"visible_text"] isKindOfClass:[NSArray class]] ? dom[@"visible_text"] : @[])) { + if ([text isKindOfClass:[NSString class]]) { + OPAIAppendUniqueText(visibleText, text); + } + } + NSArray *domElements = OPAIWebDOMElements(dom, webBounds, bundleId); + for (NSDictionary *element in domElements) { + if (interactiveElements.count >= 120) { + break; + } + [interactiveElements addObject:element]; + } + return @{ + @"status": @"ok", + @"provider": @"OpenPhoneAppIntrospector.WebKitDOM", + @"scope": @"web_content_process", + @"url": OPAIString(dom[@"url"]), + @"title": OPAIString(dom[@"title"]), + @"element_count": @(domElements.count), + @"text_count": @(([dom[@"visible_text"] isKindOfClass:[NSArray class]] + ? [dom[@"visible_text"] count] : 0)), + @"web_view_class": NSStringFromClass([webView class]) ?: @"", + @"web_view_bounds": OPAIBoundsArray(webBounds) + }; +} + +static NSString *OPAIApplicationStateName(UIApplicationState state) { + switch (state) { + case UIApplicationStateActive: + return @"active"; + case UIApplicationStateInactive: + return @"inactive"; + case UIApplicationStateBackground: + return @"background"; + } + return @"unknown"; +} + +static NSDictionary *OPAIAppSnapshot(void) { + UIApplication *application = nil; + @try { + application = [UIApplication sharedApplication]; + } @catch (__unused NSException *exception) { + application = nil; + } + NSString *bundleId = OPAIEffectiveAppBundleId(); + NSString *processBundleId = OPAIAppBundleId(); + NSString *provider = OPAIProviderName(); + if (!application) { + return @{ + @"schema": @"openphone.app_ui_state.v1", + @"status": @"unavailable", + @"provider": provider, + @"bundle_id": bundleId, + @"process_bundle_id": processBundleId, + @"effective_bundle_id": bundleId, + @"reason": @"application_unavailable", + @"timestamp_ms": @(OPAINowMs()), + @"process_name": NSProcessInfo.processInfo.processName ?: @"", + @"pid": @(getpid()), + @"source": OPAIElementScope() + }; + } + + NSMutableArray *windows = [NSMutableArray array]; + NSMutableArray *interactiveElements = [NSMutableArray array]; + NSMutableArray *visibleText = [NSMutableArray array]; + NSArray *applicationWindows = OPAIApplicationWindows(application); + NSUInteger elementIndex = 0; + NSInteger windowIndex = 0; + for (UIWindow *window in applicationWindows) { + if (!window || window.hidden || window.alpha < 0.02) { + windowIndex++; + continue; + } + [windows addObject:OPAIWindowSnapshot(window, windowIndex)]; + OPAICollectViewTree(window, window, bundleId, windowIndex, 0, + &elementIndex, interactiveElements, visibleText); + windowIndex++; + if (windows.count >= 16 || interactiveElements.count >= 120) { + break; + } + } + + UIView *webView = OPAIFindFirstWebView(applicationWindows); + NSDictionary *webDOM = @{@"status": @"not_applicable"}; + if (webView && [bundleId isEqualToString:@"com.apple.mobilesafari"]) { + UIWindow *webWindow = webView.window ?: applicationWindows.firstObject; + CGRect webBounds = webWindow + ? [webView convertRect:webView.bounds toView:webWindow] + : webView.bounds; + webDOM = OPAIWebDOMSnapshot(webView, webBounds, bundleId, + visibleText, interactiveElements); + } + + NSMutableDictionary *uiTree = [@{ + @"status": @"ok", + @"provider": provider, + @"scope": OPAIElementScope(), + @"bundle_id": bundleId, + @"process_bundle_id": processBundleId, + @"effective_bundle_id": bundleId, + @"window_count": @(windows.count), + @"element_count": @(interactiveElements.count), + @"text_count": @(visibleText.count), + @"windows": windows, + @"interactive_elements": interactiveElements, + @"visible_text": visibleText + } mutableCopy]; + if (webDOM.count > 0 && ![webDOM[@"status"] isEqualToString:@"not_applicable"]) { + uiTree[@"web_dom"] = webDOM; + } + UIApplicationState applicationState = application.applicationState; + return @{ + @"schema": @"openphone.app_ui_state.v1", + @"status": @"ok", + @"provider": provider, + @"bundle_id": bundleId, + @"process_bundle_id": processBundleId, + @"effective_bundle_id": bundleId, + @"process_name": NSProcessInfo.processInfo.processName ?: @"", + @"pid": @(getpid()), + @"timestamp_ms": @(OPAINowMs()), + @"application_state": @((NSInteger)applicationState), + @"application_state_name": OPAIApplicationStateName(applicationState), + @"ui_tree": uiTree, + @"source": OPAIElementScope() + }; +} + +static BOOL OPAIWriteJSONFile(NSString *path, NSDictionary *object) { + if (path.length == 0 || !object) { + return NO; + } + NSError *error = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:object + options:0 + error:&error]; + if (!data || error) { + return NO; + } + BOOL wrote = [data writeToFile:path atomically:YES]; + if (wrote) { + chmod(path.UTF8String, 0644); + } + return wrote; +} + +static BOOL OPAIWriteAll(int fd, NSData *data) { + const uint8_t *bytes = (const uint8_t *)data.bytes; + NSUInteger remaining = data.length; + while (remaining > 0) { + ssize_t written = write(fd, bytes, remaining); + if (written > 0) { + bytes += written; + remaining -= (NSUInteger)written; + continue; + } + if (written < 0 && errno == EINTR) { + continue; + } + return NO; + } + return YES; +} + +static NSDictionary *OPAIErrorResponse(NSString *reason) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneAppIntrospector.DaemonClient", + @"reason": reason ?: @"unknown", + @"timestamp_ms": @(OPAINowMs()), + @"source": @"app_process" + }; +} + +static NSDictionary *OPAIDaemonRequest(NSDictionary *request) { + if (!request) { + return OPAIErrorResponse(@"missing_request"); + } + NSError *error = nil; + NSData *json = [NSJSONSerialization dataWithJSONObject:request options:0 error:&error]; + if (!json || error) { + return OPAIErrorResponse(@"json_encode_failed"); + } + NSMutableData *payload = [json mutableCopy]; + const uint8_t newline = '\n'; + [payload appendBytes:&newline length:1]; + + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { + return OPAIErrorResponse(@"socket_failed"); + } + struct timeval timeout; + timeout.tv_sec = 1; + timeout.tv_usec = 0; + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + + struct sockaddr_in address; + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(27631); + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + BOOL ok = connect(fd, (struct sockaddr *)&address, sizeof(address)) == 0; + if (!ok) { + close(fd); + return OPAIErrorResponse(@"connect_failed"); + } + ok = OPAIWriteAll(fd, payload); + if (ok) { + shutdown(fd, SHUT_WR); + } + NSMutableData *responseData = [NSMutableData data]; + if (ok) { + char buffer[4096]; + while (responseData.length < (128 * 1024)) { + ssize_t count = read(fd, buffer, sizeof(buffer)); + if (count > 0) { + [responseData appendBytes:buffer length:(NSUInteger)count]; + if (memchr(buffer, '\n', (size_t)count) != NULL) { + break; + } + continue; + } + if (count < 0 && errno == EINTR) { + continue; + } + break; + } + } + close(fd); + if (!ok) { + return OPAIErrorResponse(@"write_failed"); + } + if (responseData.length == 0) { + return OPAIErrorResponse(@"empty_response"); + } + id response = [NSJSONSerialization JSONObjectWithData:responseData options:0 error:nil]; + if (![response isKindOfClass:[NSDictionary class]]) { + return OPAIErrorResponse(@"json_decode_failed"); + } + return response; +} + +static BOOL OPAIPublishSnapshotToDaemon(NSDictionary *snapshot) { + if (!snapshot) { + return NO; + } + NSDictionary *response = OPAIDaemonRequest(@{ + @"command": @"app_ui_publish", + @"transport": @"app_process_tcp_loopback", + @"state": snapshot + }); + return [response[@"status"] isEqualToString:@"ok"]; +} + +static BOOL OPAIWebContentSnapshotReadyForDaemon(NSDictionary *snapshot) { + if (!OPAIIsWebContentProcess()) { + return YES; + } + if (![snapshot[@"status"] isEqualToString:@"ok"]) { + return NO; + } + NSDictionary *uiTree = [snapshot[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? snapshot[@"ui_tree"] : @{}; + if (![uiTree[@"status"] isEqualToString:@"ok"]) { + return NO; + } + long long elementCount = [uiTree[@"element_count"] respondsToSelector:@selector(longLongValue)] + ? [uiTree[@"element_count"] longLongValue] : 0; + long long textCount = [uiTree[@"text_count"] respondsToSelector:@selector(longLongValue)] + ? [uiTree[@"text_count"] longLongValue] : 0; + return elementCount > 0 || textCount > 0; +} + +static NSString *OPAIStorageBundleIdForSnapshot(NSDictionary *snapshot) { + if (OPAIIsWebContentProcess() && !OPAIWebContentSnapshotReadyForDaemon(snapshot)) { + return [NSString stringWithFormat:@"%@.%d", OPAIAppBundleId(), getpid()]; + } + NSString *bundleId = [snapshot[@"bundle_id"] isKindOfClass:[NSString class]] + ? snapshot[@"bundle_id"] : OPAIEffectiveAppBundleId(); + return bundleId.length > 0 ? bundleId : OPAIAppBundleId(); +} + +static void OPAIPublishAppState(void) { + @autoreleasepool { + [[NSFileManager defaultManager] createDirectoryAtPath:OPAIStorePath + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + [[NSFileManager defaultManager] createDirectoryAtPath:OPAIAppUIDir + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + @try { + NSDictionary *snapshot = OPAIAppSnapshot(); + if (OPAIWebContentSnapshotReadyForDaemon(snapshot) && + OPAIPublishSnapshotToDaemon(snapshot)) { + return; + } + NSString *bundleId = OPAIStorageBundleIdForSnapshot(snapshot); + NSString *filename = [OPAISafeFilenameForBundleId(bundleId) + stringByAppendingPathExtension:@"json"]; + NSString *path = [OPAIAppUIDir stringByAppendingPathComponent:filename]; + if (!OPAIWriteJSONFile(path, snapshot)) { + OPAILog(@"state write failed bundle=%@ path=%@", bundleId ?: @"", path ?: @""); + } + } @catch (NSException *exception) { + OPAILog(@"state publish exception name=%@ reason=%@", + exception.name ?: @"", exception.reason ?: @""); + } + } +} + +static void OPAIPublishAppStateOnMain(void) { + if ([NSThread isMainThread]) { + OPAIPublishAppState(); + } else { + dispatch_async(dispatch_get_main_queue(), ^{ + OPAIPublishAppState(); + }); + } +} + +static BOOL OPAIDoubleForKey(NSDictionary *dictionary, NSString *key, double *outValue) { + id value = dictionary[key]; + if ([value respondsToSelector:@selector(doubleValue)]) { + if (outValue) { + *outValue = [value doubleValue]; + } + return YES; + } + return NO; +} + +static long long OPAILongLongForKey(NSDictionary *dictionary, + NSString *key, + long long defaultValue, + long long minValue, + long long maxValue) { + id value = dictionary[key]; + long long result = [value respondsToSelector:@selector(longLongValue)] + ? [value longLongValue] : defaultValue; + if (result < minValue) { + result = minValue; + } + if (result > maxValue) { + result = maxValue; + } + return result; +} + +static CGPoint OPAINormalizeInputPoint(double x, double y) { + CGFloat px = (CGFloat)x; + CGFloat py = (CGFloat)y; + UIScreen *screen = [UIScreen mainScreen]; + CGFloat scale = screen.scale > 0.0 ? screen.scale : 1.0; + CGRect bounds = screen.bounds; + if ((px > CGRectGetWidth(bounds) * 1.5 || py > CGRectGetHeight(bounds) * 1.5) && + scale > 1.0) { + px = px / scale; + py = py / scale; + } + return CGPointMake(px, py); +} + +static UIView *OPAIFindElementInView(UIView *view, + UIWindow *window, + NSString *bundleId, + NSInteger windowIndex, + NSInteger depth, + NSUInteger *elementIndex, + NSString *wantedElementId, + NSDictionary **outElement) { + if (!view || depth > 32 || !elementIndex || wantedElementId.length == 0) { + return nil; + } + if (view.hidden || view.alpha < 0.02) { + return nil; + } + CGRect bounds = [view convertRect:view.bounds toView:window]; + if (CGRectIsEmpty(bounds) || bounds.size.width < 1.0 || bounds.size.height < 1.0) { + return nil; + } + BOOL sensitive = OPAIViewIsSensitive(view); + NSString *label = OPAIViewLabel(view); + NSString *identifier = OPAIString(view.accessibilityIdentifier); + BOOL interactive = OPAIViewLooksInteractive(view, label, identifier); + BOOL editableText = [view isKindOfClass:[UITextField class]] || + [view isKindOfClass:[UITextView class]] || + [view isKindOfClass:[UISearchBar class]]; + if (interactive && (label.length > 0 || identifier.length > 0 || editableText)) { + NSDictionary *element = OPAIElementSnapshotForView(view, bundleId, windowIndex, + *elementIndex, bounds, label, sensitive, identifier); + (*elementIndex)++; + if ([element[@"id"] isEqualToString:wantedElementId]) { + if (outElement) { + *outElement = element; + } + return view; + } + } + for (id child in OPAIAccessibilityChildrenForView(view)) { + if ([child isKindOfClass:[UIView class]]) { + continue; + } + NSString *childLabel = OPAIAccessibilityStringValue(child, @selector(accessibilityLabel)); + NSString *childValue = OPAIAccessibilityStringValue(child, @selector(accessibilityValue)); + if (childLabel.length == 0 && childValue.length == 0) { + continue; + } + CGRect frame = OPAIAccessibilityFrameValue(child); + if (CGRectIsEmpty(frame) || frame.size.width < 1.0 || frame.size.height < 1.0) { + continue; + } + NSDictionary *element = OPAIElementSnapshotForAccessibilityObject(child, bundleId, + windowIndex, *elementIndex, frame); + (*elementIndex)++; + if ([element[@"id"] isEqualToString:wantedElementId]) { + if (outElement) { + *outElement = element; + } + return view; + } + if (*elementIndex >= 120) { + return nil; + } + } + for (UIView *subview in view.subviews ?: @[]) { + UIView *match = OPAIFindElementInView(subview, window, bundleId, windowIndex, + depth + 1, elementIndex, wantedElementId, outElement); + if (match) { + return match; + } + if (*elementIndex >= 120) { + return nil; + } + } + return nil; +} + +static UIView *OPAIFindElementById(NSString *elementId, + NSDictionary **outElement, + UIWindow **outWindow) { + UIApplication *application = nil; + @try { + application = [UIApplication sharedApplication]; + } @catch (__unused NSException *exception) { + application = nil; + } + if (!application || elementId.length == 0) { + return nil; + } + NSString *bundleId = OPAIEffectiveAppBundleId(); + NSUInteger elementIndex = 0; + NSInteger windowIndex = 0; + for (UIWindow *window in OPAIApplicationWindows(application)) { + if (!window || window.hidden || window.alpha < 0.02) { + windowIndex++; + continue; + } + UIView *match = OPAIFindElementInView(window, window, bundleId, windowIndex, 0, + &elementIndex, elementId, outElement); + if (match) { + if (outWindow) { + *outWindow = window; + } + return match; + } + windowIndex++; + if (elementIndex >= 120) { + break; + } + } + return nil; +} + +static NSInteger OPAIWindowIndexForWindow(UIWindow *targetWindow, NSArray *windows) { + NSInteger index = 0; + for (UIWindow *window in windows ?: @[]) { + if (window == targetWindow) { + return index; + } + index++; + } + return 0; +} + +static NSDictionary *OPAIHitElementSummary(UIView *view, UIWindow *window) { + if (!view || !window) { + return @{}; + } + UIApplication *application = nil; + @try { + application = [UIApplication sharedApplication]; + } @catch (__unused NSException *exception) { + application = nil; + } + NSArray *windows = application ? OPAIApplicationWindows(application) : @[]; + NSInteger windowIndex = OPAIWindowIndexForWindow(window, windows); + NSString *bundleId = OPAIEffectiveAppBundleId(); + BOOL sensitive = OPAIViewIsSensitive(view); + NSString *label = OPAIViewLabel(view); + NSString *identifier = OPAIString(view.accessibilityIdentifier); + CGRect bounds = [view convertRect:view.bounds toView:window]; + NSMutableDictionary *summary = [OPAIElementSnapshotForView(view, bundleId, windowIndex, + 0, bounds, label, sensitive, identifier) mutableCopy]; + summary[@"id"] = [NSString stringWithFormat:@"app-%@-hit-%p", + OPAISafeFilenameForBundleId(bundleId), view]; + return summary; +} + +static UIView *OPAIHitTestView(CGPoint point, + UIWindow **outWindow, + NSDictionary **outElement) { + UIApplication *application = nil; + @try { + application = [UIApplication sharedApplication]; + } @catch (__unused NSException *exception) { + application = nil; + } + if (!application) { + return nil; + } + NSArray *windows = [OPAIApplicationWindows(application) sortedArrayUsingComparator: + ^NSComparisonResult(UIWindow *a, UIWindow *b) { + if (a.windowLevel > b.windowLevel) { + return NSOrderedAscending; + } + if (a.windowLevel < b.windowLevel) { + return NSOrderedDescending; + } + return NSOrderedSame; + }]; + for (UIWindow *window in windows ?: @[]) { + if (!window || window.hidden || window.alpha < 0.02 || !window.userInteractionEnabled) { + continue; + } + CGPoint localPoint = [window convertPoint:point fromWindow:nil]; + if (![window pointInside:localPoint withEvent:nil]) { + continue; + } + UIView *hit = [window hitTest:localPoint withEvent:nil]; + if (hit) { + if (outWindow) { + *outWindow = window; + } + if (outElement) { + *outElement = OPAIHitElementSummary(hit, window); + } + return hit; + } + } + return nil; +} + +static NSDictionary *OPAIInputResponse(NSString *status, + NSString *reason, + NSString *actionType, + NSDictionary *target, + NSDictionary *extra) { + NSMutableDictionary *response = [@{ + @"status": status ?: @"unavailable", + @"provider": OPAIInputProviderName(), + @"action_type": actionType ?: @"", + @"source": OPAIElementScope(), + @"timestamp_ms": @(OPAINowMs()) + } mutableCopy]; + if (reason.length > 0) { + response[@"reason"] = reason; + } + if (target) { + response[@"target"] = target; + } + for (NSString *key in extra ?: @{}) { + id value = extra[key]; + if (value) { + response[key] = value; + } + } + return response; +} + +static NSArray *OPAIMethodCandidatesForClass(Class cls) { + if (!cls) { + return @[]; + } + NSArray *needles = @[ + @"activ", @"address", @"bar", @"begin", @"capsule", @"edit", + @"focus", @"location", @"navig", @"select", @"text", @"title", @"url" + ]; + unsigned int count = 0; + Method *methods = class_copyMethodList(cls, &count); + if (!methods) { + return @[]; + } + NSMutableArray *candidates = [NSMutableArray array]; + for (unsigned int i = 0; i < count && candidates.count < 40; i++) { + SEL selector = method_getName(methods[i]); + const char *name = selector ? sel_getName(selector) : NULL; + if (!name) { + continue; + } + NSString *selectorName = [NSString stringWithUTF8String:name] ?: @""; + NSString *lower = selectorName.lowercaseString; + for (NSString *needle in needles) { + if ([lower containsString:needle]) { + [candidates addObject:selectorName]; + break; + } + } + } + free(methods); + return candidates; +} + +static NSArray *OPAIDiagnosticsForViewAncestors(UIView *view) { + NSMutableArray *diagnostics = [NSMutableArray array]; + NSInteger depth = 0; + for (UIView *cursor = view; cursor && depth < 8; cursor = cursor.superview, depth++) { + Class cls = [cursor class]; + NSArray *methods = OPAIMethodCandidatesForClass(cls); + if (methods.count > 0) { + [diagnostics addObject:@{ + @"depth": @(depth), + @"class": NSStringFromClass(cls) ?: @"", + @"methods": methods + }]; + } else { + [diagnostics addObject:@{ + @"depth": @(depth), + @"class": NSStringFromClass(cls) ?: @"", + @"methods": @[] + }]; + } + } + return diagnostics; +} + +static UIView *OPAIEditableTextViewForView(UIView *view) { + if (!view) { + return nil; + } + if ([view isKindOfClass:[UITextField class]] || + [view isKindOfClass:[UITextView class]]) { + return view; + } + if ([view isKindOfClass:[UISearchBar class]]) { + @try { + UITextField *field = ((UISearchBar *)view).searchTextField; + if (field) { + return field; + } + } @catch (__unused NSException *exception) { + } + } + if ([OPAIAppBundleId() isEqualToString:@"com.apple.mobilesafari"] && + [NSStringFromClass([view class]) isEqualToString:@"SFCapsuleNavigationBar"]) { + SEL selector = NSSelectorFromString(@"textField"); + if ([view respondsToSelector:selector]) { + id value = nil; + @try { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Warc-performSelector-leaks" + value = [view performSelector:selector]; +#pragma clang diagnostic pop + } @catch (__unused NSException *exception) { + value = nil; + } + if ([value isKindOfClass:[UITextField class]] || + [value isKindOfClass:[UITextView class]]) { + return (UIView *)value; + } + if ([value isKindOfClass:[UIView class]]) { + UIView *valueView = (UIView *)value; + if ([valueView conformsToProtocol:@protocol(UITextInput)] || + [valueView conformsToProtocol:@protocol(UIKeyInput)] || + [valueView canBecomeFirstResponder]) { + return valueView; + } + } + if ([value isKindOfClass:[UIView class]] && value != view) { + UIView *match = OPAIEditableTextViewForView((UIView *)value); + if (match) { + return match; + } + } + } + } + for (UIView *subview in view.subviews ?: @[]) { + UIView *match = OPAIEditableTextViewForView(subview); + if (match) { + return match; + } + } + return nil; +} + +static UIView *OPAIEditableTextViewForViewOrAncestors(UIView *view) { + NSInteger depth = 0; + for (UIView *cursor = view; cursor && depth < 10; cursor = cursor.superview, depth++) { + UIView *editable = OPAIEditableTextViewForView(cursor); + if (editable) { + return editable; + } + } + return nil; +} + +static BOOL OPAIInvokeZeroArgumentSelector(id target, + NSString *selectorName, + NSString **outStatus) { + SEL selector = NSSelectorFromString(selectorName ?: @""); + if (!target || !selector || ![target respondsToSelector:selector]) { + if (outStatus) { + *outStatus = @"missing"; + } + return NO; + } + NSMethodSignature *signature = [target methodSignatureForSelector:selector]; + if (!signature || signature.numberOfArguments != 2) { + if (outStatus) { + *outStatus = @"unsupported_signature"; + } + return NO; + } + @try { + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.target = target; + invocation.selector = selector; + [invocation invoke]; + if (outStatus) { + *outStatus = @"invoked"; + } + return YES; + } @catch (__unused NSException *exception) { + if (outStatus) { + *outStatus = @"exception"; + } + return NO; + } +} + +static BOOL OPAIInvokeBooleanArgumentSelector(id target, + NSString *selectorName, + BOOL value, + NSString **outStatus) { + SEL selector = NSSelectorFromString(selectorName ?: @""); + if (!target || !selector || ![target respondsToSelector:selector]) { + if (outStatus) { + *outStatus = @"missing"; + } + return NO; + } + NSMethodSignature *signature = [target methodSignatureForSelector:selector]; + if (!signature || signature.numberOfArguments != 3) { + if (outStatus) { + *outStatus = @"unsupported_signature"; + } + return NO; + } + @try { + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.target = target; + invocation.selector = selector; + BOOL argument = value; + [invocation setArgument:&argument atIndex:2]; + [invocation invoke]; + if (outStatus) { + *outStatus = @"invoked"; + } + return YES; + } @catch (__unused NSException *exception) { + if (outStatus) { + *outStatus = @"exception"; + } + return NO; + } +} + +static NSDictionary *OPAIActivateSafariAddressView(UIView *view, + NSDictionary *target, + NSString *actionType, + long long durationMs, + NSArray *priorAttempts) { + if (![OPAIAppBundleId() isEqualToString:@"com.apple.mobilesafari"] || !view) { + return nil; + } + NSMutableArray *attempts = [NSMutableArray arrayWithArray:priorAttempts ?: @[]]; + BOOL sawAddressCapsule = NO; + BOOL invokedPrivateSelector = NO; + for (UIView *cursor = view; cursor && attempts.count < 80; cursor = cursor.superview) { + NSString *className = NSStringFromClass([cursor class]) ?: @""; + if ([className isEqualToString:@"SFUnifiedTabBarItemTitleContainerView"]) { + sawAddressCapsule = YES; + NSString *status = nil; + if (OPAIInvokeZeroArgumentSelector(cursor, @"beginTransitioningSearchField", &status)) { + invokedPrivateSelector = YES; + } + [attempts addObject:[NSString stringWithFormat:@"%@.beginTransitioningSearchField:%@", + className, status ?: @"unknown"]]; + } else if ([className isEqualToString:@"SFCapsuleNavigationBar"] || + [className isEqualToString:@"SFCapsuleView"]) { + sawAddressCapsule = YES; + NSString *status = nil; + if (OPAIInvokeBooleanArgumentSelector(cursor, @"setSelected:", YES, &status)) { + invokedPrivateSelector = YES; + } + [attempts addObject:[NSString stringWithFormat:@"%@.setSelected:YES:%@", + className, status ?: @"unknown"]]; + } else if ([className isEqualToString:@"SFCapsuleCollectionView"]) { + sawAddressCapsule = YES; + NSString *status = nil; + if (OPAIInvokeZeroArgumentSelector(cursor, @"_tapToShowBarBottomRegion", &status)) { + invokedPrivateSelector = YES; + } + [attempts addObject:[NSString stringWithFormat:@"%@._tapToShowBarBottomRegion:%@", + className, status ?: @"unknown"]]; + } + } + if (!sawAddressCapsule) { + return nil; + } + if (invokedPrivateSelector) { + [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.10]]; + } + + UIView *editable = OPAIEditableTextViewForViewOrAncestors(view); + if (!editable) { + return OPAIInputResponse(@"unavailable", @"safari_address_textfield_not_found", + actionType, target, @{ + @"activation_method": invokedPrivateSelector + ? @"safari_address_private_focus_probe" : @"safari_address_focus_probe", + @"activated_class": NSStringFromClass([view class]) ?: @"", + @"duration_ms": @(durationMs), + @"attempts": attempts, + @"diagnostics": @{ + @"view_ancestors": OPAIDiagnosticsForViewAncestors(view) + } + }); + } + if (OPAIViewIsSensitive(editable)) { + return OPAIInputResponse(@"unavailable", @"sensitive_text_input", + actionType, target, @{@"attempts": attempts}); + } + if ([editable canBecomeFirstResponder]) { + BOOL becameFirstResponder = [editable becomeFirstResponder]; + if (becameFirstResponder || editable.isFirstResponder) { + return OPAIInputResponse(@"ok", nil, actionType, target, @{ + @"activation_method": @"safari_address_textfield_becomeFirstResponder", + @"activated_class": NSStringFromClass([editable class]) ?: @"", + @"duration_ms": @(durationMs), + @"attempts": attempts + }); + } + [attempts addObject:[NSString stringWithFormat:@"%@.becomeFirstResponder:false", + NSStringFromClass([editable class]) ?: @"UIView"]]; + } + return OPAIInputResponse(@"unavailable", @"safari_address_focus_failed", + actionType, target, @{ + @"activation_method": invokedPrivateSelector + ? @"safari_address_private_focus_probe" : @"safari_address_focus_probe", + @"activated_class": NSStringFromClass([editable class]) ?: @"", + @"duration_ms": @(durationMs), + @"attempts": attempts + }); +} + +static UIView *OPAIFindFirstResponderInView(UIView *view) { + if (!view) { + return nil; + } + if (view.isFirstResponder) { + return view; + } + for (UIView *subview in view.subviews ?: @[]) { + UIView *match = OPAIFindFirstResponderInView(subview); + if (match) { + return match; + } + } + return nil; +} + +static UIView *OPAIFindFirstResponder(void) { + UIApplication *application = nil; + @try { + application = [UIApplication sharedApplication]; + } @catch (__unused NSException *exception) { + application = nil; + } + if (!application) { + return nil; + } + for (UIWindow *window in OPAIApplicationWindows(application)) { + UIView *match = OPAIFindFirstResponderInView(window); + if (match) { + return match; + } + } + return nil; +} + +static NSString *OPAIEditableTextLength(UIView *view) { + NSString *text = nil; + if ([view isKindOfClass:[UITextField class]]) { + text = ((UITextField *)view).text; + } else if ([view isKindOfClass:[UITextView class]]) { + text = ((UITextView *)view).text; + } + return [NSString stringWithFormat:@"%lu", (unsigned long)(text.length)]; +} + +static UITableViewCell *OPAITabularCellForView(UIView *view) { + for (UIView *cursor = view; cursor; cursor = cursor.superview) { + if ([cursor isKindOfClass:[UITableViewCell class]]) { + return (UITableViewCell *)cursor; + } + } + return nil; +} + +static UITableView *OPAITabularSuperviewForCell(UITableViewCell *cell) { + for (UIView *cursor = cell.superview; cursor; cursor = cursor.superview) { + if ([cursor isKindOfClass:[UITableView class]]) { + return (UITableView *)cursor; + } + } + return nil; +} + +static UICollectionViewCell *OPAICollectionCellForView(UIView *view) { + for (UIView *cursor = view; cursor; cursor = cursor.superview) { + if ([cursor isKindOfClass:[UICollectionViewCell class]]) { + return (UICollectionViewCell *)cursor; + } + } + return nil; +} + +static UICollectionView *OPAICollectionSuperviewForCell(UICollectionViewCell *cell) { + for (UIView *cursor = cell.superview; cursor; cursor = cursor.superview) { + if ([cursor isKindOfClass:[UICollectionView class]]) { + return (UICollectionView *)cursor; + } + } + return nil; +} + +static NSDictionary *OPAIActivateTableCell(UIView *view) { + UITableViewCell *cell = OPAITabularCellForView(view); + UITableView *tableView = cell ? OPAITabularSuperviewForCell(cell) : nil; + NSIndexPath *indexPath = (tableView && cell) ? [tableView indexPathForCell:cell] : nil; + if (!tableView || !indexPath) { + return nil; + } + [tableView selectRowAtIndexPath:indexPath animated:NO scrollPosition:UITableViewScrollPositionNone]; + id delegate = tableView.delegate; + SEL selector = @selector(tableView:didSelectRowAtIndexPath:); + if ([delegate respondsToSelector:selector]) { + [delegate tableView:tableView didSelectRowAtIndexPath:indexPath]; + return @{ + @"status": @"ok", + @"activation_method": @"uitableview_delegate_did_select", + @"index_path": [NSString stringWithFormat:@"%ld.%ld", + (long)indexPath.section, (long)indexPath.row] + }; + } + return @{ + @"status": @"unavailable", + @"reason": @"uitableview_delegate_unavailable", + @"activation_method": @"uitableview_select_only", + @"index_path": [NSString stringWithFormat:@"%ld.%ld", + (long)indexPath.section, (long)indexPath.row] + }; +} + +static NSDictionary *OPAIActivateCollectionCell(UIView *view) { + UICollectionViewCell *cell = OPAICollectionCellForView(view); + UICollectionView *collectionView = cell ? OPAICollectionSuperviewForCell(cell) : nil; + NSIndexPath *indexPath = (collectionView && cell) ? [collectionView indexPathForCell:cell] : nil; + if (!collectionView || !indexPath) { + return nil; + } + [collectionView selectItemAtIndexPath:indexPath + animated:NO + scrollPosition:UICollectionViewScrollPositionNone]; + id delegate = collectionView.delegate; + SEL selector = @selector(collectionView:didSelectItemAtIndexPath:); + if ([delegate respondsToSelector:selector]) { + [delegate collectionView:collectionView didSelectItemAtIndexPath:indexPath]; + return @{ + @"status": @"ok", + @"activation_method": @"uicollectionview_delegate_did_select", + @"index_path": [NSString stringWithFormat:@"%ld.%ld", + (long)indexPath.section, (long)indexPath.item] + }; + } + return @{ + @"status": @"unavailable", + @"reason": @"uicollectionview_delegate_unavailable", + @"activation_method": @"uicollectionview_select_only", + @"index_path": [NSString stringWithFormat:@"%ld.%ld", + (long)indexPath.section, (long)indexPath.item] + }; +} + +static NSDictionary *OPAIActivateView(UIView *view, + NSDictionary *target, + NSString *actionType, + long long durationMs) { + if (!view) { + return OPAIInputResponse(@"unavailable", @"target_view_missing", + actionType, target, nil); + } + NSMutableArray *attempts = [NSMutableArray array]; + NSDictionary *safariAddressActivation = OPAIActivateSafariAddressView(view, + target, actionType, durationMs, attempts); + if (safariAddressActivation) { + return safariAddressActivation; + } + NSInteger depth = 0; + for (UIView *cursor = view; cursor && depth < 10; cursor = cursor.superview, depth++) { + if (cursor.hidden || cursor.alpha < 0.02) { + [attempts addObject:@"hidden_or_transparent"]; + continue; + } + @try { + if ([cursor respondsToSelector:@selector(accessibilityActivate)] && + [cursor accessibilityActivate]) { + return OPAIInputResponse(@"ok", nil, actionType, target, @{ + @"activation_method": @"accessibilityActivate", + @"activated_class": NSStringFromClass([cursor class]) ?: @"", + @"duration_ms": @(durationMs), + @"attempts": attempts + }); + } + [attempts addObject:[NSString stringWithFormat:@"%@.accessibilityActivate:false", + NSStringFromClass([cursor class]) ?: @"UIView"]]; + } @catch (NSException *exception) { + [attempts addObject:[NSString stringWithFormat:@"%@.accessibilityActivate:exception", + NSStringFromClass([cursor class]) ?: @"UIView"]]; + } + + UIView *editable = OPAIEditableTextViewForView(cursor); + if (editable) { + if (OPAIViewIsSensitive(editable)) { + return OPAIInputResponse(@"unavailable", @"sensitive_text_input", + actionType, target, @{@"attempts": attempts}); + } + if ([editable canBecomeFirstResponder]) { + [editable becomeFirstResponder]; + return OPAIInputResponse(@"ok", nil, actionType, target, @{ + @"activation_method": @"becomeFirstResponder", + @"activated_class": NSStringFromClass([editable class]) ?: @"", + @"duration_ms": @(durationMs), + @"attempts": attempts + }); + } + [attempts addObject:[NSString stringWithFormat:@"%@.becomeFirstResponder:false", + NSStringFromClass([editable class]) ?: @"UIView"]]; + } + + if ([cursor isKindOfClass:[UISwitch class]]) { + UISwitch *toggle = (UISwitch *)cursor; + [toggle setOn:!toggle.on animated:YES]; + [toggle sendActionsForControlEvents:UIControlEventValueChanged]; + return OPAIInputResponse(@"ok", nil, actionType, target, @{ + @"activation_method": @"uiswitch_toggle", + @"activated_class": NSStringFromClass([cursor class]) ?: @"", + @"duration_ms": @(durationMs), + @"attempts": attempts + }); + } + if ([cursor isKindOfClass:[UIControl class]]) { + UIControl *control = (UIControl *)cursor; + if (!control.enabled) { + return OPAIInputResponse(@"unavailable", @"control_disabled", + actionType, target, @{@"attempts": attempts}); + } + [control sendActionsForControlEvents:UIControlEventTouchUpInside]; + [control sendActionsForControlEvents:UIControlEventPrimaryActionTriggered]; + return OPAIInputResponse(@"unavailable", @"uicontrol_send_actions_unverified", + actionType, target, @{ + @"activation_method": @"uicontrol_send_actions", + @"activated_class": NSStringFromClass([cursor class]) ?: @"", + @"duration_ms": @(durationMs), + @"attempts": attempts, + @"diagnostics": @{ + @"view_ancestors": OPAIDiagnosticsForViewAncestors(view) + } + }); + } + + NSDictionary *tableActivation = OPAIActivateTableCell(cursor); + if ([tableActivation[@"status"] isEqualToString:@"ok"]) { + NSMutableDictionary *extra = [tableActivation mutableCopy]; + extra[@"activated_class"] = NSStringFromClass([cursor class]) ?: @""; + extra[@"duration_ms"] = @(durationMs); + extra[@"attempts"] = attempts; + return OPAIInputResponse(@"ok", nil, actionType, target, extra); + } + + NSDictionary *collectionActivation = OPAIActivateCollectionCell(cursor); + if ([collectionActivation[@"status"] isEqualToString:@"ok"]) { + NSMutableDictionary *extra = [collectionActivation mutableCopy]; + extra[@"activated_class"] = NSStringFromClass([cursor class]) ?: @""; + extra[@"duration_ms"] = @(durationMs); + extra[@"attempts"] = attempts; + return OPAIInputResponse(@"ok", nil, actionType, target, extra); + } + } + return OPAIInputResponse(@"unavailable", @"activation_unhandled", + actionType, target, @{@"attempts": attempts}); +} + +static NSDictionary *OPAITypeTextIntoView(UIView *view, + NSDictionary *target, + NSString *text, + NSString *actionType) { + if (!view) { + return OPAIInputResponse(@"unavailable", @"target_view_missing", + actionType, target, nil); + } + UIView *editable = OPAIEditableTextViewForViewOrAncestors(view); + if (!editable) { + return OPAIInputResponse(@"unavailable", @"editable_text_input_not_found", + actionType, target, @{ + @"target_class": NSStringFromClass([view class]) ?: @"" + }); + } + if (OPAIViewIsSensitive(editable)) { + return OPAIInputResponse(@"unavailable", @"sensitive_text_input", + actionType, target, @{ + @"target_class": NSStringFromClass([editable class]) ?: @"" + }); + } + BOOL becameFirstResponder = NO; + if (![editable isFirstResponder] && [editable canBecomeFirstResponder]) { + becameFirstResponder = [editable becomeFirstResponder]; + } + if (![editable isFirstResponder] && !becameFirstResponder) { + return OPAIInputResponse(@"unavailable", @"text_input_focus_failed", + actionType, target, @{ + @"target_class": NSStringFromClass([editable class]) ?: @"" + }); + } + + NSString *beforeLength = OPAIEditableTextLength(editable); + @try { + if ([editable conformsToProtocol:@protocol(UITextInput)]) { + id input = (id)editable; + UITextRange *range = input.selectedTextRange; + if (range) { + [input replaceRange:range withText:text ?: @""]; + } else if ([editable conformsToProtocol:@protocol(UIKeyInput)]) { + [(id)editable insertText:text ?: @""]; + } else { + return OPAIInputResponse(@"unavailable", @"text_input_selection_unavailable", + actionType, target, @{ + @"target_class": NSStringFromClass([editable class]) ?: @"" + }); + } + } else if ([editable conformsToProtocol:@protocol(UIKeyInput)]) { + [(id)editable insertText:text ?: @""]; + } else { + return OPAIInputResponse(@"unavailable", @"text_input_protocol_unavailable", + actionType, target, @{ + @"target_class": NSStringFromClass([editable class]) ?: @"" + }); + } + } @catch (NSException *exception) { + return OPAIInputResponse(@"unavailable", @"type_text_exception", + actionType, target, @{ + @"exception_name": exception.name ?: @"", + @"exception_reason": exception.reason ?: @"", + @"target_class": NSStringFromClass([editable class]) ?: @"" + }); + } + if ([editable isKindOfClass:[UITextField class]]) { + [(UITextField *)editable sendActionsForControlEvents:UIControlEventEditingChanged]; + } else if ([editable isKindOfClass:[UITextView class]]) { + UITextView *textView = (UITextView *)editable; + [[NSNotificationCenter defaultCenter] postNotificationName:UITextViewTextDidChangeNotification + object:textView]; + id delegate = textView.delegate; + if ([delegate respondsToSelector:@selector(textViewDidChange:)]) { + [delegate textViewDidChange:textView]; + } + } + NSString *afterLength = OPAIEditableTextLength(editable); + return OPAIInputResponse(@"ok", nil, actionType, target, @{ + @"activation_method": @"text_input_insert", + @"target_class": NSStringFromClass([editable class]) ?: @"", + @"became_first_responder": @(becameFirstResponder), + @"text_length": @((text ?: @"").length), + @"before_text_length": @([beforeLength longLongValue]), + @"after_text_length": @([afterLength longLongValue]) + }); +} + +static UIView *OPAISafariWebViewForInput(void) { + if (![OPAIEffectiveAppBundleId() isEqualToString:@"com.apple.mobilesafari"]) { + return nil; + } + UIApplication *application = nil; + @try { + application = [UIApplication sharedApplication]; + } @catch (__unused NSException *exception) { + application = nil; + } + if (!application) { + return nil; + } + return OPAIFindFirstWebView(OPAIApplicationWindows(application)); +} + +static long long OPAIWebDOMIndexFromElementId(NSString *elementId) { + if (elementId.length == 0) { + return -1; + } + NSRange range = [elementId rangeOfString:@"-web-" options:NSBackwardsSearch]; + if (range.location == NSNotFound) { + return -1; + } + NSString *suffix = [elementId substringFromIndex:range.location + range.length]; + return suffix.length > 0 ? [suffix longLongValue] : -1; +} + +static long long OPAIWebDOMIndexForAction(NSDictionary *action, NSDictionary *target) { + long long domIndex = OPAINumberLongLong(target[@"dom_index"], -1); + if (domIndex >= 0) { + return domIndex; + } + NSString *elementId = OPAIString(action[@"element_id"]); + return OPAIWebDOMIndexFromElementId(elementId); +} + +static BOOL OPAIActionTargetsWebDOM(NSDictionary *action, NSDictionary *target) { + NSString *scope = [target[@"scope"] isKindOfClass:[NSString class]] ? target[@"scope"] : @""; + NSString *riskHint = [target[@"risk_hint"] isKindOfClass:[NSString class]] ? target[@"risk_hint"] : @""; + NSString *elementId = OPAIString(action[@"element_id"]); + return [scope isEqualToString:@"web_content_process"] || + [riskHint isEqualToString:@"web_content_process"] || + [elementId containsString:@"-web-"]; +} + +static NSString *OPAIWebDOMTextInputScript(long long domIndex, NSString *text) { + NSString *textJSON = OPAIJSONStringLiteral(text ?: @""); + return [NSString stringWithFormat: + @"(function(){%@" + "function opnodes(){var selector='input,textarea,select,button,a,[contenteditable=\"\"],[contenteditable=\"true\"],[role=\"textbox\"],[role=\"button\"],[tabindex]';" + "return Array.prototype.slice.call(document.querySelectorAll(selector)).filter(function(el){var r=el.getBoundingClientRect();var s=getComputedStyle(el);" + "return r.width>=1&&r.height>=1&&s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0';}).slice(0,80);}" + "var nodes=opnodes();" + "var index=%lld;var text=%@;var el=index>=0?nodes[index]:document.activeElement;" + "if(!el||el===document.body||el===document.documentElement)return JSON.stringify({status:'unavailable',reason:'web_dom_target_not_found'});" + "var tag=el.tagName.toLowerCase();var type=(el.getAttribute('type')||'').toLowerCase();" + "var editable=(tag==='textarea'||(tag==='input'&&!['button','submit','reset','checkbox','radio','file','image','range','color'].includes(type))||el.isContentEditable||el.getAttribute('role')==='textbox');" + "if(!editable)return JSON.stringify({status:'unavailable',reason:'web_dom_element_not_editable',tag:tag,type:type});" + "var before=('value' in el)?String(el.value||''):String(el.textContent||'');" + "el.focus({preventScroll:true});" + "if('value' in el){var start=(typeof el.selectionStart==='number')?el.selectionStart:before.length;" + "var end=(typeof el.selectionEnd==='number')?el.selectionEnd:start;" + "el.value=before.slice(0,start)+text+before.slice(end);" + "var pos=start+text.length;if(typeof el.setSelectionRange==='function')el.setSelectionRange(pos,pos);}" + "else if(el.isContentEditable){document.execCommand('insertText',false,text);}" + "var after=('value' in el)?String(el.value||''):String(el.textContent||'');" + "try{el.dispatchEvent(new InputEvent('input',{bubbles:true,inputType:'insertText',data:text}));}catch(e){el.dispatchEvent(new Event('input',{bubbles:true}));}" + "el.dispatchEvent(new Event('change',{bubbles:true}));" + "return JSON.stringify({status:'ok',activation_method:'webkit_dom_text_input',tag:tag,type:type,focused:document.activeElement===el,before_text_length:before.length,after_text_length:after.length,text_length:text.length});" + "})()", + OPAIWebDOMQueryFunction(), domIndex, textJSON]; +} + +static NSString *OPAIWebDOMActivateScript(long long domIndex) { + return [NSString stringWithFormat: + @"(function(){%@" + "function opnodes(){var selector='input,textarea,select,button,a,[contenteditable=\"\"],[contenteditable=\"true\"],[role=\"textbox\"],[role=\"button\"],[tabindex]';" + "return Array.prototype.slice.call(document.querySelectorAll(selector)).filter(function(el){var r=el.getBoundingClientRect();var s=getComputedStyle(el);" + "return r.width>=1&&r.height>=1&&s.display!=='none'&&s.visibility!=='hidden'&&s.opacity!=='0';}).slice(0,80);}" + "var nodes=opnodes();" + "var index=%lld;var el=index>=0?nodes[index]:null;" + "if(!el)return JSON.stringify({status:'unavailable',reason:'web_dom_target_not_found'});" + "var tag=el.tagName.toLowerCase();var type=(el.getAttribute('type')||'').toLowerCase();" + "if(typeof el.scrollIntoView==='function')el.scrollIntoView({block:'center',inline:'center'});" + "if(typeof el.focus==='function')el.focus({preventScroll:true});" + "if(typeof el.click==='function')el.click();" + "return JSON.stringify({status:'ok',activation_method:'webkit_dom_activate',tag:tag,type:type,focused:document.activeElement===el});" + "})()", + OPAIWebDOMQueryFunction(), domIndex]; +} + +static NSDictionary *OPAIWebDOMInputResponse(NSString *status, + NSString *reason, + NSString *actionType, + NSDictionary *target, + NSDictionary *extra) { + NSMutableDictionary *metadata = [NSMutableDictionary dictionaryWithDictionary:extra ?: @{}]; + metadata[@"provider"] = @"OpenPhoneAppIntrospector.WebContentInput"; + metadata[@"scope"] = @"web_content_process"; + return OPAIInputResponse(status, reason, actionType, target, metadata); +} + +static NSDictionary *OPAIPerformWebDOMInput(NSDictionary *action, + NSDictionary *target, + NSString *actionType, + NSString *text) { + UIView *webView = OPAISafariWebViewForInput(); + if (!webView) { + return OPAIWebDOMInputResponse(@"unavailable", @"webview_not_found", + actionType, target, nil); + } + long long domIndex = OPAIWebDOMIndexForAction(action, target); + if (domIndex < 0 && ![actionType isEqualToString:@"type_text"]) { + return OPAIWebDOMInputResponse(@"unavailable", @"web_dom_index_missing", + actionType, target, nil); + } + NSDictionary *result = nil; + if ([actionType isEqualToString:@"type_text"]) { + result = OPAIWebViewEvaluateJSON(webView, + OPAIWebDOMTextInputScript(domIndex, text ?: @""), 0.80); + } else { + result = OPAIWebViewEvaluateJSON(webView, OPAIWebDOMActivateScript(domIndex), 0.80); + } + if (![result[@"status"] isEqualToString:@"ok"]) { + NSString *reason = [result[@"reason"] isKindOfClass:[NSString class]] + ? result[@"reason"] : @"web_dom_input_failed"; + return OPAIWebDOMInputResponse(@"unavailable", reason, actionType, target, result); + } + NSMutableDictionary *extra = [result mutableCopy]; + extra[@"activation_method"] = result[@"activation_method"] ?: + ([actionType isEqualToString:@"type_text"] ? @"webkit_dom_text_input" : @"webkit_dom_activate"); + extra[@"target_class"] = @"DOMElement"; + extra[@"dom_index"] = @(domIndex); + return OPAIWebDOMInputResponse(@"ok", nil, actionType, target, extra); +} + +static NSDictionary *OPAIPerformInputRequest(NSDictionary *pendingRequest) { + NSDictionary *action = [pendingRequest[@"action"] isKindOfClass:[NSDictionary class]] + ? pendingRequest[@"action"] : @{}; + NSString *actionType = OPAIString(action[@"type"]); + if (![actionType isEqualToString:@"tap"] && + ![actionType isEqualToString:@"tap_element"] && + ![actionType isEqualToString:@"long_press"] && + ![actionType isEqualToString:@"type_text"]) { + return OPAIInputResponse(@"unavailable", @"unsupported_action_type", + actionType, nil, nil); + } + NSString *text = [action[@"text"] isKindOfClass:[NSString class]] + ? action[@"text"] : @""; + if ([actionType isEqualToString:@"type_text"] && text.length == 0) { + return OPAIInputResponse(@"unavailable", @"missing_text", + actionType, nil, nil); + } + long long durationMs = OPAILongLongForKey(action, @"duration_ms", + [actionType isEqualToString:@"long_press"] ? 700 : 80, 50, 5000); + NSDictionary *providedTarget = [action[@"target"] isKindOfClass:[NSDictionary class]] + ? action[@"target"] : nil; + if (OPAIActionTargetsWebDOM(action, providedTarget)) { + return OPAIPerformWebDOMInput(action, providedTarget, actionType, text); + } + NSDictionary *target = nil; + UIWindow *targetWindow = nil; + UIView *targetView = nil; + NSString *elementId = OPAIString(action[@"element_id"]); + if (elementId.length > 0) { + targetView = OPAIFindElementById(elementId, &target, &targetWindow); + } + if (!targetView) { + double x = 0.0; + double y = 0.0; + if (OPAIDoubleForKey(action, @"x", &x) && OPAIDoubleForKey(action, @"y", &y)) { + CGPoint point = OPAINormalizeInputPoint(x, y); + targetView = OPAIHitTestView(point, &targetWindow, &target); + } + } + if (!targetView && [actionType isEqualToString:@"type_text"]) { + targetView = OPAIFindFirstResponder(); + if (targetView) { + target = OPAIHitElementSummary(targetView, targetWindow ?: (UIWindow *)targetView.window); + } + } + if (!targetView) { + return OPAIInputResponse(@"unavailable", @"target_not_found", + actionType, target, nil); + } + (void)targetWindow; + if ([actionType isEqualToString:@"type_text"]) { + return OPAITypeTextIntoView(targetView, target, text, actionType); + } + return OPAIActivateView(targetView, target, actionType, durationMs); +} + +static void OPAIPollAndPerformInput(void) { + @autoreleasepool { + NSString *bundleId = OPAIEffectiveAppBundleId(); + NSDictionary *poll = OPAIDaemonRequest(@{ + @"command": @"app_input_poll", + @"transport": @"app_process_tcp_loopback", + @"bundle_id": bundleId, + @"scope": OPAIElementScope() + }); + if (![poll[@"status"] isEqualToString:@"ok"]) { + return; + } + NSDictionary *pendingRequest = [poll[@"request"] isKindOfClass:[NSDictionary class]] + ? poll[@"request"] : nil; + NSString *requestId = OPAIString(pendingRequest[@"request_id"]); + if (!pendingRequest || requestId.length == 0) { + return; + } + + __block NSDictionary *response = nil; + void (^performBlock)(void) = ^{ + @try { + response = OPAIPerformInputRequest(pendingRequest); + } @catch (NSException *exception) { + response = OPAIInputResponse(@"unavailable", @"perform_exception", + OPAIString([pendingRequest[@"action"] isKindOfClass:[NSDictionary class]] + ? pendingRequest[@"action"][@"type"] : @""), + nil, + @{ + @"exception_name": exception.name ?: @"", + @"exception_reason": exception.reason ?: @"" + }); + } + }; + if ([NSThread isMainThread]) { + performBlock(); + } else { + dispatch_sync(dispatch_get_main_queue(), performBlock); + } + if (!response) { + response = OPAIInputResponse(@"unavailable", @"empty_perform_response", + OPAIString([pendingRequest[@"action"] isKindOfClass:[NSDictionary class]] + ? pendingRequest[@"action"][@"type"] : @""), + nil, nil); + } + + NSDictionary *complete = OPAIDaemonRequest(@{ + @"command": @"app_input_complete", + @"transport": @"app_process_tcp_loopback", + @"bundle_id": bundleId, + @"request_id": requestId, + @"response": response + }); + if (![complete[@"status"] isEqualToString:@"ok"]) { + OPAILog(@"app input complete failed request_id=%@ reason=%@", + requestId, OPAIString(complete[@"reason"])); + } + } +} + +static void *OPAIStateThread(void *unused) { + (void)unused; + usleep(1500000); + long long lastPublishMs = 0; + while (1) { + long long now = OPAINowMs(); + if (now - lastPublishMs >= 2000) { + OPAIPublishAppStateOnMain(); + lastPublishMs = now; + } + OPAIPollAndPerformInput(); + usleep(350000); + } + return NULL; +} + +__attribute__((constructor)) +static void OPAIInit(void) { + NSString *bundleId = OPAIAppBundleId(); + OPAILog(@"OpenPhoneAppIntrospector loaded bundle=%@ pid=%d", + bundleId ?: @"", getpid()); + pthread_t thread; + int rc = pthread_create(&thread, NULL, OPAIStateThread, NULL); + if (rc == 0) { + pthread_detach(thread); + } else { + OPAILog(@"state thread failed rc=%d", rc); + } +} diff --git a/ios/agentd/src/main.m b/ios/agentd/src/main.m new file mode 100644 index 0000000..a3279cd --- /dev/null +++ b/ios/agentd/src/main.m @@ -0,0 +1,18952 @@ +#import + +#import +#import +#import +#import + +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import + +extern char **environ; + +// Private memorystatus syscall — public header (sys/kern_memorystatus.h) is not +// shipped in the SDK, but the syscall works under the rootless runtime prefix. +// Declaring the +// pieces we need locally so we can raise our jetsam priority and avoid being +// sacrificed under memory pressure during long model tasks. +extern int memorystatus_control(uint32_t command, int32_t pid, uint32_t flags, + void *buffer, size_t buffersize); +#ifndef MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES +#define MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES 2 +#endif +#ifndef MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK +#define MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK 5 +#endif +#ifndef MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT +#define MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT 7 +#endif +// xnu jetsam priority bands are small integers (0..JETSAM_PRIORITY_MAX==21): +// 0=idle, 6=phone, 10=foreground app, 12=audio_and_accessory, 19=critical. +// Aim for the AUDIO band since we own the microphone and stream audio to +// STT / Realtime. NOTE: earlier code used 100/120 here, which are out of range +// and made memorystatus_control fail with EINVAL(22). +#ifndef JETSAM_PRIORITY_FOREGROUND +#define JETSAM_PRIORITY_FOREGROUND 10 +#endif +#ifndef JETSAM_PRIORITY_AUDIO_AND_ACCESSORY +#define JETSAM_PRIORITY_AUDIO_AND_ACCESSORY 12 +#endif +#ifndef JETSAM_PRIORITY_MAX +#define JETSAM_PRIORITY_MAX 21 +#endif +// Must match xnu's layout exactly: the kernel rejects the call with EINVAL if +// buffersize != sizeof(memorystatus_priority_properties_t). user_data is +// uint64_t (an earlier uint32_t here made the struct 8 bytes instead of 16 and +// caused persistent EINVAL). +typedef struct { + int32_t priority; + uint64_t user_data; +} memorystatus_priority_properties_t; + +static NSString *const OPAgentVersion = @"0.1.0-dev"; +static NSString *const OPDefaultStorePath = @"/var/mobile/Library/OpenPhone"; +static NSString *const OPVolumeTriggerPreferencesPath = @"/var/mobile/Library/Preferences/com.openphone.volumetrigger.plist"; +static NSString *const OPDefaultHardwareTriggerGoal = @"Use the current phone context to handle this hardware volume trigger as an immediate OpenPhone agent turn. Inspect the visible state, act only when the next useful phone action is clear, otherwise finish with a concise status."; +static NSString *const OPLegacyHardwareTriggerGoal = @"Handle the OpenPhone hardware volume trigger using the current phone context."; +static NSString *const OPOpenAIRealtimeModel = @"gpt-realtime"; +static NSString *const OPOpenAIRealtime2Model = @"gpt-realtime-2"; + +static volatile sig_atomic_t OPRunning = 1; +static int OPServerFd = -1; +static int OPAppUIIntakeFd = -1; +static pthread_mutex_t OPHardwareTriggerMutex = PTHREAD_MUTEX_INITIALIZER; +static long long OPHardwareTriggerLastAcceptedMs = 0; +static pthread_mutex_t OPVoiceTriggerMutex = PTHREAD_MUTEX_INITIALIZER; +static BOOL OPVoiceTriggerRunning = NO; +static volatile int OPVoiceCancelRequested = 0; +static long long OPVoiceTriggerLastStartedMs = 0; +static long long OPVoiceTriggerLastFinishedMs = 0; +static NSString *OPVoiceTriggerLastState = nil; +static NSString *OPVoiceTriggerLastTranscript = nil; +static NSString *OPVoiceTriggerLastError = nil; +static NSString *OPVoiceTriggerLastProvider = nil; +static volatile int OPAppUIIntakeThreadStarted = 0; +static volatile int OPAppUIIntakeReady = 0; +static unsigned long long OPAppUIIntakePublishCount = 0; +static long long OPAppUIIntakeLastPublishMs = 0; +static NSString *OPAppUIIntakeLastBundleId = nil; +static NSString *OPAppUIIntakeStartError = nil; +static unsigned long long OPNotificationIngestCount = 0; +static long long OPNotificationLastIngestMs = 0; +static NSString *OPNotificationLastBundleId = nil; +static NSMutableDictionary *OPAppInputRequests = nil; +static pid_t OPProtectedDataHelperPid = 0; +static long long OPProtectedDataHelperLastSpawnMs = 0; +static NSString *OPProtectedDataHelperLastSpawnError = nil; +// Wall-clock ms at which this daemon process started. App-process UI state +// received before this instant is from a prior process (pre-restart) and must +// not be trusted as a fresh observation after a resume. +static long long OPProcessStartMs = 0; + +static NSDictionary *OPVolumeTriggerStatus(void); +static void OPStartVolumeTriggerListener(void); +static NSDictionary *OPRunTask(NSDictionary *request); +static NSDictionary *OPVoiceTrigger(NSDictionary *request); +static NSDictionary *OPVoiceStatus(NSDictionary *request); +static NSDictionary *OPVoiceTranscribeFile(NSDictionary *request); +static NSDictionary *OPBackgroundJobRunDue(NSDictionary *request); +static void OPStartBackgroundJobScheduler(void); +static NSDictionary *OPSpringBoardPublishedState(void); +static NSDictionary *OPRepairStaleActiveTasks(NSDictionary *request); +static void OPStartAppUIIntakeServer(void); +static NSString *OPStringFromRequest(NSDictionary *request, NSString *key, NSString *defaultValue); +static NSDictionary *OPReadJSONFile(NSString *path); +static BOOL OPTaskCancellationRequested(NSString *taskId, NSString **reasonOut); +static void OPRecordTrajectory(NSString *taskId, NSString *event, NSDictionary *payload); +static BOOL OPModelModeIsOpenAIRealtime(NSString *mode); +static NSDictionary *OPJetsamPrioritySet(NSDictionary *request); +static NSString *OPExplicitURLFromText(NSString *text); +static NSDictionary *OPError(NSString *reason); +static NSData *OPJSONData(id object); +static BOOL OPWriteAll(int fd, NSData *data); +static NSData *OPReadClient(int clientFd); +static NSDictionary *OPContactsProviderStatus(void); +static NSDictionary *OPCalendarProviderStatus(void); +static NSDictionary *OPCallsProviderStatus(void); +static NSDictionary *OPMessagesProviderStatus(void); +static NSDictionary *OPNotificationProviderStatus(void); +static NSDictionary *OPNotificationIngest(NSDictionary *request); +static NSDictionary *OPNotificationList(NSDictionary *request); +static NSDictionary *OPNotificationFireWatchers(NSDictionary *notification); +static NSDictionary *OPModelScreenTraceSummary(NSDictionary *screen); + +static NSString *OPBackgroundJobSchedulerStatus(void) { + return @"implemented_agent_loop"; +} + +static NSString *OPCommitmentSchedulerStatus(void) { + return @"implemented_time_bridge"; +} + +static NSString *OPWatcherSchedulerStatus(void) { + return @"implemented_timer_bridge"; +} + +static NSString *OPStorePath(void) { + const char *override = getenv("OPENPHONE_AGENTD_STORE"); + if (override && override[0] != '\0') { + return [NSString stringWithUTF8String:override]; + } + return OPDefaultStorePath; +} + +static NSString *OPRunPath(void) { + return [OPStorePath() stringByAppendingPathComponent:@"run"]; +} + +static NSString *OPSocketPath(void) { + return [OPRunPath() stringByAppendingPathComponent:@"agentd.sock"]; +} + +static NSString *OPLogPath(void) { + return [OPStorePath() stringByAppendingPathComponent:@"openphone-agentd.log"]; +} + +static NSString *OPTasksPath(void) { + return [OPStorePath() stringByAppendingPathComponent:@"tasks"]; +} + +static NSString *OPConfigPath(void) { + return [OPStorePath() stringByAppendingPathComponent:@"config"]; +} + +static NSString *OPModelConfigPath(void) { + return [OPConfigPath() stringByAppendingPathComponent:@"model.json"]; +} + +static NSString *OPModelCredentialPath(void) { + return [OPConfigPath() stringByAppendingPathComponent:@"model-credential.json"]; +} + +static NSString *OPAgentControlPath(void) { + return [OPConfigPath() stringByAppendingPathComponent:@"agent-control.json"]; +} + +static NSString *OPVoiceCredentialPath(void) { + return [OPConfigPath() stringByAppendingPathComponent:@"voice-credential.json"]; +} + +static NSString *OPVoicePath(void) { + return [OPStorePath() stringByAppendingPathComponent:@"voice"]; +} + +static NSString *OPVoiceMemoryWatermarkPath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"springboard"] + stringByAppendingPathComponent:@"voice-memory-watermark.json"]; +} + +static NSString *OPContactsFixturePath(void) { + return [OPConfigPath() stringByAppendingPathComponent:@"contacts-fixture.json"]; +} + +static NSString *OPCalendarFixturePath(void) { + return [OPConfigPath() stringByAppendingPathComponent:@"calendar-fixture.json"]; +} + +static NSString *OPCallsFixturePath(void) { + return [OPConfigPath() stringByAppendingPathComponent:@"calls-fixture.json"]; +} + +static NSString *OPMessagesFixturePath(void) { + return [OPConfigPath() stringByAppendingPathComponent:@"messages-fixture.json"]; +} + +static NSString *OPAuditPath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"audit"] + stringByAppendingPathComponent:@"audit-events.jsonl"]; +} + +static NSString *OPTrajectoriesPath(void) { + return [OPStorePath() stringByAppendingPathComponent:@"trajectories"]; +} + +static NSString *OPScreenshotsPath(void) { + return [OPStorePath() stringByAppendingPathComponent:@"screenshots"]; +} + +static NSString *OPAppUIPath(void) { + return [OPStorePath() stringByAppendingPathComponent:@"app-ui"]; +} + +static NSString *OPSpringBoardStatePath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"springboard"] + stringByAppendingPathComponent:@"state.json"]; +} + +static NSString *OPSpringBoardTriggerStatusPath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"springboard"] + stringByAppendingPathComponent:@"trigger-status.json"]; +} + +static NSString *OPSpringBoardScreenshotRequestPath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"springboard"] + stringByAppendingPathComponent:@"screenshot-request.json"]; +} + +static NSString *OPSpringBoardScreenshotResponsePath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"springboard"] + stringByAppendingPathComponent:@"screenshot-response.json"]; +} + +static NSString *OPSpringBoardInputRequestPath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"springboard"] + stringByAppendingPathComponent:@"input-request.json"]; +} + +static NSString *OPSpringBoardInputResponsePath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"springboard"] + stringByAppendingPathComponent:@"input-response.json"]; +} + +static NSString *OPSpringBoardClipboardRequestPath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"springboard"] + stringByAppendingPathComponent:@"clipboard-request.json"]; +} + +static NSString *OPSpringBoardClipboardResponsePath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"springboard"] + stringByAppendingPathComponent:@"clipboard-response.json"]; +} + +static NSString *OPNotificationsPath(void) { + return [OPStorePath() stringByAppendingPathComponent:@"notifications"]; +} + +// Recent-notification ring buffer file. SpringBoard's OpenPhoneVolumeTrigger +// tweak hooks NCNotificationDispatcher and posts each banner here via the +// daemon Unix socket; the daemon keeps a bounded, redacted rolling log. +static NSString *OPNotificationLogPath(void) { + return [OPNotificationsPath() stringByAppendingPathComponent:@"recent.json"]; +} + +static void OPHandleSignal(int signum) { + (void)signum; + OPRunning = 0; + if (OPServerFd >= 0) { + close(OPServerFd); + OPServerFd = -1; + } + if (OPAppUIIntakeFd >= 0) { + close(OPAppUIIntakeFd); + OPAppUIIntakeFd = -1; + } +} + +static void OPEnsureDirectories(void) { + NSFileManager *fileManager = [NSFileManager defaultManager]; + NSArray *paths = @[ + OPStorePath(), + OPRunPath(), + OPTasksPath(), + [OPStorePath() stringByAppendingPathComponent:@"audit"], + [OPStorePath() stringByAppendingPathComponent:@"trajectories"], + OPScreenshotsPath(), + OPAppUIPath(), + OPNotificationsPath(), + OPVoicePath(), + [OPStorePath() stringByAppendingPathComponent:@"springboard"], + [OPStorePath() stringByAppendingPathComponent:@"db"], + OPConfigPath() + ]; + for (NSString *path in paths) { + [fileManager createDirectoryAtPath:path + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + } + chmod(OPConfigPath().UTF8String, 0700); +} + +static long long OPNowMs(void) { + return (long long)([[NSDate date] timeIntervalSince1970] * 1000.0); +} + +static BOOL OPPathExists(NSString *path) { + if (path.length == 0) { + return NO; + } + struct stat st; + return stat(path.UTF8String, &st) == 0; +} + +static void OPRestrictSocketToMobile(NSString *path) { + if (path.length == 0) { + return; + } + struct passwd *mobile = getpwnam("mobile"); + uid_t uid = mobile ? mobile->pw_uid : 501; + gid_t gid = mobile ? mobile->pw_gid : 501; + chown(path.UTF8String, uid, gid); + chmod(path.UTF8String, 0600); +} + +static BOOL OPEnvFlagEnabled(const char *name) { + const char *value = getenv(name); + if (!value || value[0] == '\0') { + return NO; + } + return strcmp(value, "0") != 0 && + strcasecmp(value, "false") != 0 && + strcasecmp(value, "no") != 0; +} + +static BOOL OPProtectedDataHelperRole(void) { + const char *role = getenv("OPENPHONE_AGENTD_ROLE"); + return role && strcmp(role, "protected_data_helper") == 0; +} + +static BOOL OPProtectedDataHelperCommandAllowed(NSString *command) { + if ([command isEqualToString:@"health"]) { + return YES; + } + if ([command isEqualToString:@"calendar_search"] || + [command isEqualToString:@"calendar_events_search"] || + [command isEqualToString:@"openphone.calendar.search"] || + [command isEqualToString:@"openphone.calendar.events.search"]) { + return YES; + } + if ([command isEqualToString:@"calls_search"] || + [command isEqualToString:@"call_history_search"] || + [command isEqualToString:@"openphone.calls.search"] || + [command isEqualToString:@"openphone.call_history.search"]) { + return YES; + } + if ([command isEqualToString:@"messages_search"] || + [command isEqualToString:@"message_search"] || + [command isEqualToString:@"sms_search"] || + [command isEqualToString:@"openphone.messages.search"] || + [command isEqualToString:@"openphone.sms.search"]) { + return YES; + } + // The helper runs setuid-root, so it can raise agentd's jetsam band when the + // mobile-uid daemon itself gets EPERM from memorystatus_control. + if ([command isEqualToString:@"jetsam_priority_set"] || + [command isEqualToString:@"openphone.jetsam.priority_set"]) { + return YES; + } + return NO; +} + +static NSString *OPProtectedDataHelperStorePath(void) { + const char *override = getenv("OPENPHONE_PROTECTED_DATA_HELPER_STORE"); + if (override && override[0] != '\0') { + return [NSString stringWithUTF8String:override]; + } + return [OPDefaultStorePath stringByAppendingPathComponent:@"protected-data-helper"]; +} + +static NSString *OPProtectedDataHelperSocketPath(void) { + const char *override = getenv("OPENPHONE_PROTECTED_DATA_HELPER_SOCKET"); + if (override && override[0] != '\0') { + return [NSString stringWithUTF8String:override]; + } + return [[OPProtectedDataHelperStorePath() stringByAppendingPathComponent:@"run"] + stringByAppendingPathComponent:@"agentd.sock"]; +} + +static NSString *OPProtectedDataHelperBinaryPath(void) { + NSArray *candidates = @[ + @"/var/jb/usr/local/bin/openphone-protected-data-helper", + @"/usr/local/bin/openphone-protected-data-helper" + ]; + NSFileManager *fileManager = [NSFileManager defaultManager]; + for (NSString *candidate in candidates) { + if ([fileManager isExecutableFileAtPath:candidate]) { + return candidate; + } + } + return nil; +} + +static void OPProtectedDataHelperEnsureDirectories(void) { + NSString *storePath = OPProtectedDataHelperStorePath(); + NSFileManager *fileManager = [NSFileManager defaultManager]; + NSArray *paths = @[ + storePath, + [storePath stringByAppendingPathComponent:@"run"], + [storePath stringByAppendingPathComponent:@"config"], + [storePath stringByAppendingPathComponent:@"db"], + [storePath stringByAppendingPathComponent:@"springboard"] + ]; + for (NSString *path in paths) { + [fileManager createDirectoryAtPath:path + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + } +} + +static BOOL OPProtectedDataHelperSocketConnectable(void) { + NSString *socketPath = OPProtectedDataHelperSocketPath(); + if (!OPPathExists(socketPath)) { + return NO; + } + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + return NO; + } + struct sockaddr_un address; + memset(&address, 0, sizeof(address)); + address.sun_family = AF_UNIX; + strlcpy(address.sun_path, socketPath.UTF8String, sizeof(address.sun_path)); + BOOL ok = connect(fd, (struct sockaddr *)&address, sizeof(address)) == 0; + close(fd); + return ok; +} + +static BOOL OPProtectedDataHelperEnsureStarted(BOOL forceRestart) { + if (OPProtectedDataHelperRole() || OPEnvFlagEnabled("OPENPHONE_DISABLE_PROTECTED_DATA_HELPER")) { + return NO; + } + if (!OPEnvFlagEnabled("OPENPHONE_AGENTD_ALLOW_PROTECTED_HELPER_SPAWN")) { + return OPProtectedDataHelperSocketConnectable(); + } + if (!forceRestart && OPProtectedDataHelperSocketConnectable()) { + return YES; + } + if (OPProtectedDataHelperPid > 0) { + int status = 0; + pid_t waited = waitpid(OPProtectedDataHelperPid, &status, WNOHANG); + if (waited == 0 && !forceRestart) { + return OPProtectedDataHelperSocketConnectable(); + } + if (waited == OPProtectedDataHelperPid || waited < 0) { + OPProtectedDataHelperPid = 0; + } + } + long long nowMs = OPNowMs(); + if (!forceRestart && OPProtectedDataHelperLastSpawnMs > 0 && + nowMs - OPProtectedDataHelperLastSpawnMs < 1000) { + return OPProtectedDataHelperSocketConnectable(); + } + + OPProtectedDataHelperEnsureDirectories(); + unlink(OPProtectedDataHelperSocketPath().UTF8String); + + NSString *binary = OPProtectedDataHelperBinaryPath(); + if (binary.length == 0) { + OPProtectedDataHelperLastSpawnError = @"protected_data_helper_binary_missing"; + return NO; + } + + NSArray *envStrings = @[ + [NSString stringWithFormat:@"OPENPHONE_AGENTD_STORE=%@", OPProtectedDataHelperStorePath()], + @"OPENPHONE_AGENTD_ROLE=protected_data_helper", + @"OPENPHONE_AGENTD_DISABLE_TASK_REPAIR=1", + @"OPENPHONE_AGENTD_DISABLE_VOLUME_TRIGGER=1", + @"OPENPHONE_AGENTD_DISABLE_APP_UI_INTAKE=1", + @"OPENPHONE_AGENTD_DISABLE_BACKGROUND_SCHEDULER=1", + @"HOME=/var/mobile", + @"PATH=/var/jb/usr/bin:/var/jb/bin:/usr/bin:/bin:/usr/sbin:/sbin" + ]; + + char *argv[] = {(char *)binary.UTF8String, NULL}; + char **envp = calloc(envStrings.count + 1, sizeof(char *)); + if (!envp) { + OPProtectedDataHelperLastSpawnError = @"protected_data_helper_env_alloc_failed"; + return NO; + } + for (NSUInteger i = 0; i < envStrings.count; i++) { + envp[i] = strdup(envStrings[i].UTF8String ?: ""); + if (!envp[i]) { + for (NSUInteger j = 0; j < i; j++) { + free(envp[j]); + } + free(envp); + OPProtectedDataHelperLastSpawnError = @"protected_data_helper_env_strdup_failed"; + return NO; + } + } + envp[envStrings.count] = NULL; + + pid_t pid = 0; + int spawnResult = posix_spawn(&pid, binary.UTF8String, NULL, NULL, argv, envp); + for (NSUInteger i = 0; i < envStrings.count; i++) { + free(envp[i]); + } + free(envp); + OPProtectedDataHelperLastSpawnMs = nowMs; + if (spawnResult != 0) { + OPProtectedDataHelperLastSpawnError = [NSString stringWithFormat:@"protected_data_helper_spawn_failed:%s", + strerror(spawnResult)]; + return NO; + } + OPProtectedDataHelperPid = pid; + OPProtectedDataHelperLastSpawnError = @""; + for (NSUInteger i = 0; i < 20; i++) { + usleep(100000); + if (OPProtectedDataHelperSocketConnectable()) { + return YES; + } + int status = 0; + pid_t waited = waitpid(pid, &status, WNOHANG); + if (waited == pid) { + OPProtectedDataHelperPid = 0; + OPProtectedDataHelperLastSpawnError = [NSString stringWithFormat:@"protected_data_helper_exited:%d", + status]; + return NO; + } + } + return OPProtectedDataHelperSocketConnectable(); +} + +static NSDictionary *OPJSONObjectFromData(NSData *data) { + if (data.length == 0) { + return @{}; + } + id object = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + return [object isKindOfClass:[NSDictionary class]] ? object : @{}; +} + +static NSDictionary *OPProtectedDataHelperRequest(NSDictionary *request) { + if (OPProtectedDataHelperRole() || OPEnvFlagEnabled("OPENPHONE_DISABLE_PROTECTED_DATA_HELPER")) { + return nil; + } + OPProtectedDataHelperEnsureStarted(NO); + NSString *socketPath = OPProtectedDataHelperSocketPath(); + if (!OPPathExists(socketPath)) { + return nil; + } + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + return OPError([NSString stringWithFormat:@"protected_data_helper_socket_failed:%s", strerror(errno)]); + } + struct sockaddr_un address; + memset(&address, 0, sizeof(address)); + address.sun_family = AF_UNIX; + strlcpy(address.sun_path, socketPath.UTF8String, sizeof(address.sun_path)); + BOOL connected = connect(fd, (struct sockaddr *)&address, sizeof(address)) == 0; + if (!connected) { + close(fd); + OPProtectedDataHelperEnsureStarted(YES); + fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + return OPError([NSString stringWithFormat:@"protected_data_helper_socket_failed:%s", strerror(errno)]); + } + memset(&address, 0, sizeof(address)); + address.sun_family = AF_UNIX; + strlcpy(address.sun_path, socketPath.UTF8String, sizeof(address.sun_path)); + connected = connect(fd, (struct sockaddr *)&address, sizeof(address)) == 0; + } + if (!connected) { + NSString *reason = [NSString stringWithFormat:@"protected_data_helper_connect_failed:%s", strerror(errno)]; + close(fd); + return OPError(reason); + } + struct timeval timeout; + timeout.tv_sec = 10; + timeout.tv_usec = 0; + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); + + NSMutableData *payload = [NSMutableData dataWithData:OPJSONData(request ?: @{})]; + const char newline = '\n'; + [payload appendBytes:&newline length:1]; + if (!OPWriteAll(fd, payload)) { + NSString *reason = [NSString stringWithFormat:@"protected_data_helper_write_failed:%s", strerror(errno)]; + close(fd); + return OPError(reason); + } + NSData *responseData = OPReadClient(fd); + close(fd); + NSDictionary *response = OPJSONObjectFromData(responseData); + if (response.count == 0) { + return OPError(@"protected_data_helper_empty_response"); + } + return response; +} + +static void OPLog(NSString *format, ...) { + va_list args; + va_start(args, format); + NSString *message = [[NSString alloc] initWithFormat:format arguments:args]; + va_end(args); + + NSDateFormatter *formatter = [[NSDateFormatter alloc] init]; + formatter.locale = [NSLocale localeWithLocaleIdentifier:@"en_US_POSIX"]; + formatter.dateFormat = @"yyyy-MM-dd'T'HH:mm:ssZZZZZ"; + NSString *line = [NSString stringWithFormat:@"%@ %@\n", + [formatter stringFromDate:[NSDate date]], + message ?: @""]; + + NSData *data = [line dataUsingEncoding:NSUTF8StringEncoding]; + if (data.length > 0) { + if (![[NSFileManager defaultManager] fileExistsAtPath:OPLogPath()]) { + [[NSFileManager defaultManager] createFileAtPath:OPLogPath() contents:nil attributes:nil]; + } + NSFileHandle *handle = [NSFileHandle fileHandleForWritingAtPath:OPLogPath()]; + @try { + [handle seekToEndOfFile]; + [handle writeData:data]; + [handle closeFile]; + } @catch (NSException *exception) { + (void)exception; + } + fwrite(data.bytes, 1, data.length, stderr); + } +} + +static NSArray *OPFullYoloCapabilities(void) { + return @[ + @"screen.read.visible", + @"screen.capture", + @"input.perform", + @"apps.read", + @"apps.launch", + @"tasks.observe", + @"memory.read", + @"memory.write", + @"commitments.read", + @"commitments.write", + @"watchers.read", + @"watchers.write", + @"notifications.read", + @"notifications.act", + @"clipboard.read", + @"clipboard.write", + @"share.content", + @"files.read.scoped", + @"contacts.read", + @"calendar.read", + @"calendar.write", + @"calendar.delete", + @"messages.read", + @"messages.draft", + @"messages.send", + @"calls.read", + @"calls.place", + @"settings.read", + @"settings.write", + @"background.run", + @"network.use", + @"account.access" + ]; +} + +static NSString *OPTaskId(void) { + long long ms = OPNowMs(); + return [NSString stringWithFormat:@"ios-task-%lld-%d", ms, getpid()]; +} + +static NSString *OPSafeFileComponent(NSString *value) { + if (![value isKindOfClass:[NSString class]] || value.length == 0) { + return @"unknown"; + } + NSMutableString *out = [NSMutableString stringWithCapacity:value.length]; + NSCharacterSet *allowed = [NSCharacterSet characterSetWithCharactersInString: + @"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-"]; + for (NSUInteger i = 0; i < value.length; i++) { + unichar c = [value characterAtIndex:i]; + if ([allowed characterIsMember:c]) { + [out appendFormat:@"%C", c]; + } else { + [out appendString:@"_"]; + } + } + return out.length > 0 ? out : @"unknown"; +} + +static NSString *OPTaskPath(NSString *taskId) { + return [OPTasksPath() stringByAppendingPathComponent: + [NSString stringWithFormat:@"%@.json", OPSafeFileComponent(taskId)]]; +} + +static NSString *OPTrajectoryPath(NSString *taskId) { + return [OPTrajectoriesPath() stringByAppendingPathComponent: + [NSString stringWithFormat:@"%@.jsonl", OPSafeFileComponent(taskId)]]; +} + +static NSData *OPJSONData(id object) { + if (!object) { + object = @{}; + } + NSData *data = [NSJSONSerialization dataWithJSONObject:object options:0 error:nil]; + if (!data) { + data = [@"{\"status\":\"error\",\"reason\":\"json_encode_failed\"}" dataUsingEncoding:NSUTF8StringEncoding]; + } + NSMutableData *line = [data mutableCopy]; + const char newline = '\n'; + [line appendBytes:&newline length:1]; + return line; +} + +static BOOL OPWriteAll(int fd, NSData *data) { + const uint8_t *bytes = (const uint8_t *)data.bytes; + NSUInteger remaining = data.length; + while (remaining > 0) { + ssize_t written = write(fd, bytes, remaining); + if (written > 0) { + bytes += written; + remaining -= (NSUInteger)written; + continue; + } + if (written < 0 && errno == EINTR) { + continue; + } + return NO; + } + return YES; +} + +static NSData *OPCanonicalJSONData(id object) { + NSJSONWritingOptions options = NSJSONWritingSortedKeys; + if (@available(iOS 13.0, macOS 10.15, *)) { + options |= NSJSONWritingWithoutEscapingSlashes; + } + NSData *data = [NSJSONSerialization dataWithJSONObject:object ?: @{} + options:options + error:nil]; + if (!data) { + data = [@"{}" dataUsingEncoding:NSUTF8StringEncoding]; + } + return data; +} + +static NSString *OPSHA256Hex(NSData *data) { + unsigned char digest[CC_SHA256_DIGEST_LENGTH]; + CC_SHA256(data.bytes, (CC_LONG)data.length, digest); + NSMutableString *hex = [NSMutableString stringWithCapacity:CC_SHA256_DIGEST_LENGTH * 2]; + for (NSUInteger i = 0; i < CC_SHA256_DIGEST_LENGTH; i++) { + [hex appendFormat:@"%02x", digest[i]]; + } + return hex; +} + +static BOOL OPSensitiveKey(NSString *key) { + NSString *lower = key.lowercaseString ?: @""; + NSArray *markers = @[ + @"password", + @"passcode", + @"token", + @"secret", + @"authorization", + @"cookie", + @"api_key", + @"apikey", + @"private_key", + @"hostkey", + @"host_key" + ]; + for (NSString *marker in markers) { + if ([lower containsString:marker]) { + return YES; + } + } + return NO; +} + +static NSString *OPRedactedKeyName(NSString *key) { + NSData *data = [(key ?: @"") dataUsingEncoding:NSUTF8StringEncoding] ?: [NSData data]; + NSString *hash = OPSHA256Hex(data); + NSString *shortHash = hash.length >= 12 ? [hash substringToIndex:12] : hash; + return [NSString stringWithFormat:@"redacted_field_%@", shortHash ?: @"unknown"]; +} + +static BOOL OPLooksBase64Payload(NSString *value) { + if (value.length < 160) { + return NO; + } + NSCharacterSet *allowed = [NSCharacterSet characterSetWithCharactersInString: + @"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/=\r\n"]; + for (NSUInteger i = 0; i < value.length; i++) { + if (![allowed characterIsMember:[value characterAtIndex:i]]) { + return NO; + } + } + return YES; +} + +static id OPRedactedObject(id object, NSUInteger depth) { + if (!object || object == [NSNull null]) { + return [NSNull null]; + } + if (depth > 8) { + return @""; + } + if ([object isKindOfClass:[NSDictionary class]]) { + NSMutableDictionary *redacted = [NSMutableDictionary dictionary]; + NSDictionary *dictionary = object; + for (id key in dictionary) { + NSString *keyString = [key isKindOfClass:[NSString class]] + ? key : [key description]; + id value = dictionary[key]; + if (OPSensitiveKey(keyString)) { + redacted[OPRedactedKeyName(keyString)] = @""; + } else { + redacted[keyString] = OPRedactedObject(value, depth + 1); + } + } + return redacted; + } + if ([object isKindOfClass:[NSArray class]]) { + NSMutableArray *redacted = [NSMutableArray array]; + NSArray *array = object; + NSUInteger maxItems = MIN(array.count, 200); + for (NSUInteger i = 0; i < maxItems; i++) { + [redacted addObject:OPRedactedObject(array[i], depth + 1)]; + } + if (array.count > maxItems) { + [redacted addObject:@{@"truncated_items": @(array.count - maxItems)}]; + } + return redacted; + } + if ([object isKindOfClass:[NSString class]]) { + NSString *value = object; + if (OPLooksBase64Payload(value)) { + return [NSString stringWithFormat:@"", + (unsigned long)value.length]; + } + if (value.length > 4096) { + return [NSString stringWithFormat:@"", + (unsigned long)value.length]; + } + return value; + } + if ([object isKindOfClass:[NSNumber class]]) { + return object; + } + return [[object description] copy] ?: @""; +} + +static NSDictionary *OPError(NSString *reason) { + return @{ + @"status": @"error", + @"reason": reason ?: @"unknown", + @"source": @"openphone.agentd" + }; +} + +static BOOL OPWriteJSONFile(NSString *path, NSDictionary *object) { + NSData *data = [NSJSONSerialization dataWithJSONObject:object ?: @{} + options:NSJSONWritingPrettyPrinted + error:nil]; + if (!data) { + return NO; + } + return [data writeToFile:path atomically:YES]; +} + +static BOOL OPWriteProtectedJSONFile(NSString *path, NSDictionary *object) { + BOOL ok = OPWriteJSONFile(path, object); + if (ok) { + chmod(path.UTF8String, 0600); + } + return ok; +} + +// Live status the daemon publishes so SpringBoard's island UI can render +// listening/transcribing/thinking/tool/done states in real time. The path is +// atomic-write JSON; a Darwin notification wakes any observer. +static NSString *const OPIslandStatusPath = + @"/var/mobile/Library/OpenPhone/springboard/island-status.json"; +static const char *const OPIslandStatusNotification = + "com.openphone.island.status"; +static pthread_mutex_t OPIslandMutex = PTHREAD_MUTEX_INITIALIZER; +static NSMutableDictionary *OPIslandState = nil; +static unsigned long long OPIslandSequence = 0; + +static long long OPNowMs(void); + +static NSString *OPIslandDir(void) { + return [OPStorePath() stringByAppendingPathComponent:@"springboard"]; +} + +static void OPIslandEnsureState(void) { + if (!OPIslandState) { + OPIslandState = [NSMutableDictionary dictionary]; + OPIslandState[@"mode"] = @"idle"; + OPIslandState[@"title"] = @"OpenPhone"; + OPIslandState[@"subtitle"] = @""; + OPIslandState[@"transcript"] = @""; + OPIslandState[@"tool"] = @""; + OPIslandState[@"tool_arguments_summary"] = @""; + OPIslandState[@"step"] = @0; + OPIslandState[@"max_steps"] = @0; + OPIslandState[@"task_id"] = @""; + OPIslandState[@"goal"] = @""; + OPIslandState[@"accent"] = @"cyan"; + OPIslandState[@"sequence"] = @0; + OPIslandState[@"updated_at_ms"] = @0; + } +} + +static void OPIslandPublishLocked(void) { + OPIslandSequence += 1; + OPIslandState[@"sequence"] = @(OPIslandSequence); + OPIslandState[@"updated_at_ms"] = @(OPNowMs()); + NSString *dir = OPIslandDir(); + [[NSFileManager defaultManager] createDirectoryAtPath:dir + withIntermediateDirectories:YES + attributes:nil error:nil]; + NSDictionary *snapshot = [OPIslandState copy]; + if (OPWriteJSONFile(OPIslandStatusPath, snapshot)) { + chmod(OPIslandStatusPath.UTF8String, 0644); + } + notify_post(OPIslandStatusNotification); +} + +static void OPIslandUpdate(NSDictionary *patch) { + if (patch.count == 0) { + return; + } + pthread_mutex_lock(&OPIslandMutex); + OPIslandEnsureState(); + for (NSString *key in patch) { + id value = patch[key]; + if (value == nil || value == [NSNull null]) { + continue; + } + OPIslandState[key] = value; + } + OPIslandPublishLocked(); + pthread_mutex_unlock(&OPIslandMutex); +} + +static NSString *OPIslandToolLabel(NSString *tool) { + if (![tool isKindOfClass:[NSString class]] || tool.length == 0) { + return @"Thinking"; + } + NSDictionary *labels = @{ + @"get_screen": @"Looking at screen", + @"tap_element": @"Tapping", + @"type_text": @"Typing", + @"scroll": @"Scrolling", + @"swipe": @"Swiping", + @"open_url": @"Opening URL", + @"launch_app": @"Launching app", + @"press_home": @"Going home", + @"web_content_dom_state": @"Reading page", + @"contacts_search": @"Searching contacts", + @"calls_search": @"Searching calls", + @"calendar_search": @"Searching calendar", + @"messages_search": @"Searching messages", + @"memory_search": @"Recalling memory", + @"memory_save": @"Saving memory", + @"finish_task": @"Finishing", + @"fail_task": @"Failing", + @"clipboard_read": @"Reading clipboard", + @"clipboard_write": @"Writing clipboard" + }; + NSString *label = labels[tool]; + if (label) { + return label; + } + // Fallback: prettify the tool name. + NSString *pretty = [tool stringByReplacingOccurrencesOfString:@"_" withString:@" "]; + if (pretty.length == 0) { + return @"Working"; + } + return [[pretty substringToIndex:1].uppercaseString + stringByAppendingString:[pretty substringFromIndex:1]]; +} + +static void OPIslandPublishToolStep(NSString *taskId, NSString *tool, + NSString *status, long long step, long long maxSteps) { + if (![taskId isKindOfClass:[NSString class]]) { + taskId = @""; + } + NSString *mode = [status isEqualToString:@"tool_running"] ? @"action" : @"thinking"; + NSString *accent = [status isEqualToString:@"tool_running"] ? @"cyan" : @"blue"; + NSString *subtitle = OPIslandToolLabel(tool); + OPIslandUpdate(@{ + @"mode": mode, + @"subtitle": subtitle ?: @"", + @"tool": tool ?: @"", + @"step": @(step), + @"max_steps": @(maxSteps), + @"task_id": taskId ?: @"", + @"accent": accent + }); +} + +// Publish a short-form assistant reasoning line to the island so the pill's +// expanded row can render live streaming text like Android does. Called every +// time a model decision arrives with a "thought" field. +__attribute__((unused)) +static void OPIslandPublishThought(NSString *thought) { + // Legacy: kept as no-op. Reasoning is never shown. Use OPIslandPublishAssistantMessage. + (void)thought; +} + +// Show a user-facing message from the model on the island right now. +// Called per step so the pill breathes with real answers instead of dead +// state text. Also persists into the chat history so the answer stays +// visible in the expanded chat panel forever. +static void OPRecordAssistantTurn(NSString *text, NSString *taskId, BOOL succeeded); +static void OPIslandPublishAssistantMessage(NSString *msg, NSString *taskId) { + if (![msg isKindOfClass:[NSString class]]) return; + NSString *trimmed = [msg stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (trimmed.length == 0) return; + if (trimmed.length > 400) { + trimmed = [[trimmed substringToIndex:397] stringByAppendingString:@"…"]; + } + OPIslandUpdate(@{@"reply": trimmed, @"task_id": taskId ?: @""}); + OPRecordAssistantTurn(trimmed, taskId, YES); +} + +// Remember the last few voice turns so a follow-up voice turn within 10s can +// reference "the one I just did". Persisted only in-memory; that's enough for +// Android-parity conversational follow-up. +static NSMutableArray *OPRecentVoiceTurns = nil; +static pthread_mutex_t OPRecentVoiceTurnsMutex = PTHREAD_MUTEX_INITIALIZER; + +static void OPPublishChatHistory(void); // fwd decl + +static void OPRecordVoiceTurn(NSString *transcript, NSString *taskId) { + if (transcript.length == 0) return; + pthread_mutex_lock(&OPRecentVoiceTurnsMutex); + if (!OPRecentVoiceTurns) OPRecentVoiceTurns = [NSMutableArray array]; + [OPRecentVoiceTurns addObject:@{ + @"role": @"user", + @"text": transcript ?: @"", + @"task_id": taskId ?: @"", + @"at_ms": @(OPNowMs()) + }]; + if (OPRecentVoiceTurns.count > 20) { + [OPRecentVoiceTurns removeObjectAtIndex:0]; + } + pthread_mutex_unlock(&OPRecentVoiceTurnsMutex); + OPPublishChatHistory(); +} + +static void OPRecordAssistantTurn(NSString *text, NSString *taskId, BOOL succeeded) { + if (text.length == 0) return; + pthread_mutex_lock(&OPRecentVoiceTurnsMutex); + if (!OPRecentVoiceTurns) OPRecentVoiceTurns = [NSMutableArray array]; + // Skip if the previous assistant turn for the same task has the same + // text (avoids the terminal publish dup'ing what the step already said). + NSDictionary *last = OPRecentVoiceTurns.lastObject; + if ([last isKindOfClass:[NSDictionary class]] && + [last[@"role"] isEqualToString:@"assistant"] && + [last[@"text"] isKindOfClass:[NSString class]] && + [last[@"text"] isEqualToString:text] && + (![last[@"task_id"] isKindOfClass:[NSString class]] || + [last[@"task_id"] isEqualToString:taskId ?: @""])) { + pthread_mutex_unlock(&OPRecentVoiceTurnsMutex); + return; + } + [OPRecentVoiceTurns addObject:@{ + @"role": @"assistant", + @"text": text ?: @"", + @"task_id": taskId ?: @"", + @"at_ms": @(OPNowMs()), + @"status": succeeded ? @"ok" : @"error" + }]; + if (OPRecentVoiceTurns.count > 20) { + [OPRecentVoiceTurns removeObjectAtIndex:0]; + } + pthread_mutex_unlock(&OPRecentVoiceTurnsMutex); + OPPublishChatHistory(); +} + +static void OPPublishChatHistory(void) { + pthread_mutex_lock(&OPRecentVoiceTurnsMutex); + NSArray *snapshot = [OPRecentVoiceTurns ?: @[] copy]; + pthread_mutex_unlock(&OPRecentVoiceTurnsMutex); + NSString *dir = [OPStorePath() stringByAppendingPathComponent:@"springboard"]; + [[NSFileManager defaultManager] createDirectoryAtPath:dir + withIntermediateDirectories:YES + attributes:nil error:nil]; + NSString *path = [dir stringByAppendingPathComponent:@"chat-history.json"]; + OPWriteJSONFile(path, @{@"turns": snapshot}); + chmod(path.UTF8String, 0644); + notify_post("com.openphone.island.chat"); +} + +static NSArray *OPRecentVoiceTurnsSnapshot(long long withinMs) { + long long cutoff = OPNowMs() - withinMs; + pthread_mutex_lock(&OPRecentVoiceTurnsMutex); + NSMutableArray *out = [NSMutableArray array]; + for (NSDictionary *t in OPRecentVoiceTurns ?: @[]) { + long long at = [t[@"at_ms"] longLongValue]; + if (at >= cutoff) [out addObject:t]; + } + pthread_mutex_unlock(&OPRecentVoiceTurnsMutex); + return out; +} + +static void OPIslandPublishTerminal(NSString *taskId, BOOL succeeded, + NSString *summary) { + NSString *mode = succeeded ? @"success" : @"error"; + NSString *accent = succeeded ? @"green" : @"red"; + NSString *fallback = succeeded ? @"Done" : @"Failed"; + NSString *display = summary.length > 0 ? summary : fallback; + OPIslandUpdate(@{ + @"mode": mode, + @"subtitle": display, + @"reply": display, + @"task_id": taskId ?: @"", + @"accent": accent, + @"tool": @"" + }); +} + +// Autonomy modes: yolo (default) executes without asking; reviewed pauses on +// UI-driving tools for an Approve/Deny chip; dry_run refuses all UI mutations. +static BOOL OPAutonomyModeValid(NSString *v) { + return [v isEqualToString:@"reviewed"] || [v isEqualToString:@"dry_run"] || + [v isEqualToString:@"yolo"]; +} + +// Single source of truth is the daemon-owned agent-control config +// (set via the agent_control command / prefs pane). The volume-trigger +// prefs plist AutonomyMode is honored only as a legacy fallback for when +// the tweak wrote it directly and agent-control has not been set. +static NSString *OPAutonomyMode(void) { + NSDictionary *stored = OPReadJSONFile(OPAgentControlPath()); + if ([stored isKindOfClass:[NSDictionary class]] && + [stored[@"autonomy_mode"] isKindOfClass:[NSString class]]) { + NSString *v = [stored[@"autonomy_mode"] lowercaseString]; + if (OPAutonomyModeValid(v)) { + return v; + } + } + NSDictionary *prefs = OPReadJSONFile(OPVolumeTriggerPreferencesPath); + if (![prefs isKindOfClass:[NSDictionary class]]) { + // Try plist parse via NSDictionary API which handles binary plists. + prefs = [NSDictionary dictionaryWithContentsOfFile:OPVolumeTriggerPreferencesPath]; + } + if ([prefs[@"AutonomyMode"] isKindOfClass:[NSString class]]) { + NSString *v = [prefs[@"AutonomyMode"] lowercaseString]; + if (OPAutonomyModeValid(v)) { + return v; + } + } + return @"yolo"; +} + +// Ask the user via the island's Approve/Deny chips. Returns YES if approved, +// NO on deny or timeout. Called from the model loop before executing a tool +// when AutonomyMode=reviewed. Timeout defaults to 30s. +static volatile int OPConfirmationState = 0; // 0=idle 1=waiting 2=approved 3=denied +static NSString *OPConfirmationRequestPath(void) { + return [OPStorePath() stringByAppendingPathComponent:@"springboard/confirmation-response.json"]; +} + +static BOOL OPRequestUserConfirmation(NSString *taskId, NSString *tool, + NSString *summary, double timeoutSeconds) { + // Publish request via island. + OPIslandUpdate(@{ + @"mode": @"needs_review", + @"subtitle": @"Approve to run", + @"tool": tool ?: @"", + @"reply": summary ?: [NSString stringWithFormat:@"Run %@?", tool ?: @"tool"], + @"task_id": taskId ?: @"", + @"accent": @"orange" + }); + // Clear any stale response. + [[NSFileManager defaultManager] removeItemAtPath:OPConfirmationRequestPath() error:nil]; + OPConfirmationState = 1; + long long deadline = OPNowMs() + (long long)(timeoutSeconds * 1000); + while (OPNowMs() < deadline) { + usleep(200000); + if (OPTaskCancellationRequested(taskId, NULL)) { + OPConfirmationState = 3; + return NO; + } + NSDictionary *resp = OPReadJSONFile(OPConfirmationRequestPath()); + if ([resp isKindOfClass:[NSDictionary class]]) { + NSString *decision = [resp[@"decision"] isKindOfClass:[NSString class]] + ? resp[@"decision"] : @""; + [[NSFileManager defaultManager] removeItemAtPath:OPConfirmationRequestPath() error:nil]; + BOOL approved = [decision isEqualToString:@"approve"]; + OPConfirmationState = approved ? 2 : 3; + OPRecordTrajectory(taskId, @"user_confirmation", @{ + @"tool": tool ?: @"", + @"decision": decision ?: @"deny", + @"summary": summary ?: @"" + }); + return approved; + } + } + OPConfirmationState = 3; + OPRecordTrajectory(taskId, @"user_confirmation_timeout", @{ + @"tool": tool ?: @"", + @"summary": summary ?: @"" + }); + return NO; +} + +static void OPIslandReset(NSString *mode, NSString *subtitle, NSString *accent) { + pthread_mutex_lock(&OPIslandMutex); + OPIslandEnsureState(); + OPIslandState[@"mode"] = mode ?: @"idle"; + OPIslandState[@"title"] = @"OpenPhone"; + OPIslandState[@"subtitle"] = subtitle ?: @""; + OPIslandState[@"transcript"] = @""; + OPIslandState[@"tool"] = @""; + OPIslandState[@"tool_arguments_summary"] = @""; + OPIslandState[@"step"] = @0; + OPIslandState[@"max_steps"] = @0; + OPIslandState[@"task_id"] = @""; + OPIslandState[@"goal"] = @""; + OPIslandState[@"accent"] = accent ?: @"cyan"; + OPIslandPublishLocked(); + pthread_mutex_unlock(&OPIslandMutex); +} + +static NSDictionary *OPReadJSONFile(NSString *path) { + NSData *data = [NSData dataWithContentsOfFile:path]; + if (!data) { + return nil; + } + id object = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + return [object isKindOfClass:[NSDictionary class]] ? object : nil; +} + +static NSUInteger OPLimitFromRequest(NSDictionary *request, NSUInteger defaultLimit, NSUInteger maxLimit) { + id value = request[@"limit"]; + long long limit = (long long)defaultLimit; + if ([value isKindOfClass:[NSNumber class]]) { + limit = [value longLongValue]; + } else if ([value isKindOfClass:[NSString class]]) { + limit = [(NSString *)value longLongValue]; + } + if (limit < 0) { + limit = 0; + } + if ((NSUInteger)limit > maxLimit) { + limit = (long long)maxLimit; + } + return (NSUInteger)limit; +} + +static long long OPLongLongFromRequest(NSDictionary *request, NSString *key, + long long defaultValue, long long minValue, long long maxValue) { + id value = request[key]; + long long parsed = defaultValue; + if ([value isKindOfClass:[NSNumber class]]) { + parsed = [value longLongValue]; + } else if ([value isKindOfClass:[NSString class]]) { + parsed = [(NSString *)value longLongValue]; + } + if (parsed < minValue) { + parsed = minValue; + } + if (parsed > maxValue) { + parsed = maxValue; + } + return parsed; +} + +static NSArray *OPReadJSONLines(NSString *path, NSUInteger limit) { + NSData *data = [NSData dataWithContentsOfFile:path]; + if (!data) { + return @[]; + } + NSString *contents = [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; + if (contents.length == 0) { + return @[]; + } + NSMutableArray *events = [NSMutableArray array]; + NSArray *lines = [contents componentsSeparatedByCharactersInSet: + [NSCharacterSet newlineCharacterSet]]; + for (NSString *line in lines) { + if (line.length == 0) { + continue; + } + NSData *lineData = [line dataUsingEncoding:NSUTF8StringEncoding]; + id object = [NSJSONSerialization JSONObjectWithData:lineData options:0 error:nil]; + if ([object isKindOfClass:[NSDictionary class]]) { + [events addObject:object]; + if (limit > 0 && events.count > limit) { + [events removeObjectAtIndex:0]; + } + } + } + return events; +} + +static NSArray *OPRedactedEvents(NSArray *events) { + NSMutableArray *redactedEvents = [NSMutableArray array]; + for (NSDictionary *event in events) { + id redacted = OPRedactedObject(event, 0); + if ([redacted isKindOfClass:[NSDictionary class]]) { + [redactedEvents addObject:redacted]; + } + } + return redactedEvents; +} + +static NSDictionary *OPLastJSONLineDictionary(NSString *path, NSUInteger maxBytes) { + NSFileHandle *handle = [NSFileHandle fileHandleForReadingAtPath:path]; + if (!handle) { + return nil; + } + @try { + unsigned long long fileLength = [handle seekToEndOfFile]; + if (fileLength == 0) { + [handle closeFile]; + return nil; + } + unsigned long long readLength = MIN((unsigned long long)maxBytes, fileLength); + [handle seekToFileOffset:fileLength - readLength]; + NSData *data = [handle readDataToEndOfFile]; + [handle closeFile]; + NSString *contents = [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; + if (contents.length == 0) { + return nil; + } + NSArray *lines = [contents componentsSeparatedByCharactersInSet: + [NSCharacterSet newlineCharacterSet]]; + for (NSInteger i = (NSInteger)lines.count - 1; i >= 0; i--) { + NSString *line = [lines[(NSUInteger)i] stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (line.length == 0) { + continue; + } + NSData *lineData = [line dataUsingEncoding:NSUTF8StringEncoding]; + id object = [NSJSONSerialization JSONObjectWithData:lineData options:0 error:nil]; + if ([object isKindOfClass:[NSDictionary class]]) { + return object; + } + } + } @catch (NSException *exception) { + @try { + [handle closeFile]; + } @catch (NSException *closeException) { + (void)closeException; + } + OPLog(@"read last jsonl failed path=%@ exception=%@", path, exception.name ?: @"NSException"); + } + return nil; +} + +static NSString *OPLastAuditHash(void) { + NSDictionary *last = OPLastJSONLineDictionary(OPAuditPath(), 256 * 1024); + NSString *eventHash = [last[@"event_hash"] isKindOfClass:[NSString class]] + ? last[@"event_hash"] : @""; + return eventHash ?: @""; +} + +static void OPAppendJSONLine(NSString *path, NSDictionary *object) { + NSData *line = OPJSONData(object ?: @{}); + if (![[NSFileManager defaultManager] fileExistsAtPath:path]) { + [[NSFileManager defaultManager] createFileAtPath:path contents:nil attributes:nil]; + } + NSFileHandle *handle = [NSFileHandle fileHandleForWritingAtPath:path]; + @try { + [handle seekToEndOfFile]; + [handle writeData:line]; + [handle closeFile]; + } @catch (NSException *exception) { + OPLog(@"append jsonl failed path=%@ exception=%@", path, exception.name); + } +} + +static void OPRecordAudit(NSString *eventType, NSString *taskId, NSString *capability, + NSString *decision, NSDictionary *input, NSString *detail) { + @try { + NSMutableDictionary *event = [NSMutableDictionary dictionary]; + event[@"schema"] = @"openphone.audit_event.v1"; + event[@"event_type"] = eventType ?: @"unknown"; + event[@"timestamp_ms"] = @(OPNowMs()); + event[@"task_id"] = taskId ?: @""; + event[@"capability"] = capability ?: @""; + event[@"decision"] = decision ?: @""; + event[@"input"] = OPRedactedObject(input ?: @{}, 0); + event[@"detail"] = detail ?: @""; + event[@"source"] = @"openphone.agentd"; + event[@"previous_hash"] = OPLastAuditHash(); + event[@"event_hash"] = OPSHA256Hex(OPCanonicalJSONData(event)); + OPAppendJSONLine(OPAuditPath(), event); + } @catch (NSException *exception) { + OPLog(@"record audit failed event=%@ task_id=%@ exception=%@", + eventType ?: @"unknown", taskId ?: @"", exception.name ?: @"NSException"); + } +} + +static void OPRecordTrajectory(NSString *taskId, NSString *eventName, NSDictionary *payload) { + if (![taskId isKindOfClass:[NSString class]] || taskId.length == 0) { + return; + } + NSDictionary *event = @{ + @"schema": @"openphone.trajectory_event.v1", + @"timestamp_ms": @(OPNowMs()), + @"event": eventName ?: @"unknown", + @"payload": OPRedactedObject(payload ?: @{}, 0), + @"source": @"openphone.agentd" + }; + OPAppendJSONLine(OPTrajectoryPath(taskId), event); +} + +static void OPUpdateTask(NSString *taskId, NSString *status, NSDictionary *fields) { + if (![taskId isKindOfClass:[NSString class]] || taskId.length == 0) { + return; + } + NSMutableDictionary *task = [(OPReadJSONFile(OPTaskPath(taskId)) ?: @{}) mutableCopy]; + task[@"task_id"] = taskId; + task[@"status"] = status ?: task[@"status"] ?: @"unknown"; + task[@"updated_at"] = @(OPNowMs()); + for (NSString *key in fields) { + id value = fields[key]; + if (key && value) { + task[key] = value; + } + } + OPWriteJSONFile(OPTaskPath(taskId), task); +} + +static NSString *OPDatabasePath(void) { + return [[OPStorePath() stringByAppendingPathComponent:@"db"] + stringByAppendingPathComponent:@"openphone.sqlite"]; +} + +static NSString *OPJSONString(id object) { + NSData *data = [NSJSONSerialization dataWithJSONObject:object ?: @{} + options:NSJSONWritingSortedKeys + error:nil]; + if (!data) { + return @"{}"; + } + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"{}"; +} + +static NSDictionary *OPJSONDictionary(NSString *string) { + if (string.length == 0) { + return @{}; + } + NSData *data = [string dataUsingEncoding:NSUTF8StringEncoding]; + if (!data) { + return @{}; + } + id object = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + return [object isKindOfClass:[NSDictionary class]] ? object : @{}; +} + +static NSString *OPSQLiteColumnString(sqlite3_stmt *statement, int column) { + const unsigned char *text = sqlite3_column_text(statement, column); + return text ? [NSString stringWithUTF8String:(const char *)text] : @""; +} + +static BOOL OPSQLiteExec(sqlite3 *db, NSString *sql, NSString **errorOut) { + char *error = NULL; + int rc = sqlite3_exec(db, sql.UTF8String, NULL, NULL, &error); + if (rc != SQLITE_OK) { + if (errorOut) { + *errorOut = error ? [NSString stringWithUTF8String:error] : @"sqlite_exec_failed"; + } + if (error) { + sqlite3_free(error); + } + return NO; + } + return YES; +} + +static BOOL OPSQLiteTableExists(sqlite3 *db, NSString *name) { + sqlite3_stmt *statement = NULL; + BOOL exists = NO; + if (sqlite3_prepare_v2(db, + "SELECT 1 FROM sqlite_master WHERE name = ? LIMIT 1", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_text(statement, 1, name.UTF8String, -1, SQLITE_TRANSIENT); + exists = sqlite3_step(statement) == SQLITE_ROW; + } + sqlite3_finalize(statement); + return exists; +} + +static BOOL OPSQLiteMigrate(sqlite3 *db, NSString **errorOut) { + NSArray *statements = @[ + @"PRAGMA journal_mode=WAL", + @"PRAGMA synchronous=NORMAL", + @"CREATE TABLE IF NOT EXISTS schema_migrations (name TEXT PRIMARY KEY, applied_at_ms INTEGER NOT NULL)", + @"CREATE TABLE IF NOT EXISTS memory (id INTEGER PRIMARY KEY AUTOINCREMENT, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, type TEXT NOT NULL, subject TEXT NOT NULL, text TEXT NOT NULL, confidence REAL NOT NULL, source TEXT NOT NULL, reason TEXT, metadata_json TEXT)", + @"CREATE TABLE IF NOT EXISTS context_event (id INTEGER PRIMARY KEY AUTOINCREMENT, created_at_ms INTEGER NOT NULL, type TEXT NOT NULL, source TEXT NOT NULL, task_id TEXT, title TEXT, body TEXT, metadata_json TEXT)", + @"CREATE TABLE IF NOT EXISTS commitment (id INTEGER PRIMARY KEY AUTOINCREMENT, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, title TEXT NOT NULL, description TEXT, trigger_type TEXT NOT NULL, trigger_spec_json TEXT, due_at_ms INTEGER NOT NULL DEFAULT 0, expires_at_ms INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL, confidence REAL NOT NULL DEFAULT 1.0, evidence_json TEXT, source TEXT NOT NULL, reason TEXT)", + @"CREATE TABLE IF NOT EXISTS watcher (id INTEGER PRIMARY KEY AUTOINCREMENT, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, status TEXT NOT NULL, source TEXT NOT NULL, type TEXT NOT NULL, evaluator TEXT, title TEXT NOT NULL, query TEXT, url TEXT, address TEXT, number TEXT, condition_json TEXT, schedule_json TEXT, delivery_json TEXT, next_run_at_ms INTEGER NOT NULL DEFAULT 0, interval_ms INTEGER NOT NULL DEFAULT 0, recurring INTEGER NOT NULL DEFAULT 0, reason TEXT, metadata_json TEXT)", + @"CREATE TABLE IF NOT EXISTS agent_job (id INTEGER PRIMARY KEY AUTOINCREMENT, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, status TEXT NOT NULL, type TEXT NOT NULL, title TEXT NOT NULL, prompt TEXT NOT NULL, schedule_json TEXT, next_run_at_ms INTEGER NOT NULL DEFAULT 0, interval_ms INTEGER NOT NULL DEFAULT 0, session_target TEXT, delivery_json TEXT, notification_text TEXT, payload_json TEXT, reason TEXT, source TEXT NOT NULL, scheduler_enabled INTEGER NOT NULL DEFAULT 0)", + @"CREATE INDEX IF NOT EXISTS memory_updated_idx ON memory(updated_at_ms DESC)", + @"CREATE INDEX IF NOT EXISTS memory_type_subject_idx ON memory(type, subject)", + @"CREATE INDEX IF NOT EXISTS context_event_created_idx ON context_event(created_at_ms DESC)", + @"CREATE INDEX IF NOT EXISTS context_event_type_idx ON context_event(type)", + @"CREATE INDEX IF NOT EXISTS commitment_status_due_idx ON commitment(status, due_at_ms)", + @"CREATE INDEX IF NOT EXISTS commitment_updated_idx ON commitment(updated_at_ms DESC)", + @"CREATE INDEX IF NOT EXISTS watcher_status_next_run_idx ON watcher(status, next_run_at_ms)", + @"CREATE INDEX IF NOT EXISTS watcher_updated_idx ON watcher(updated_at_ms DESC)", + @"CREATE INDEX IF NOT EXISTS agent_job_status_next_run_idx ON agent_job(status, next_run_at_ms)", + @"CREATE INDEX IF NOT EXISTS agent_job_updated_idx ON agent_job(updated_at_ms DESC)" + ]; + for (NSString *sql in statements) { + if (!OPSQLiteExec(db, sql, errorOut)) { + return NO; + } + } + + NSString *ftsError = nil; + OPSQLiteExec(db, + @"CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts " + "USING fts5(memory_id UNINDEXED, text, subject, type)", + &ftsError); + ftsError = nil; + OPSQLiteExec(db, + @"CREATE VIRTUAL TABLE IF NOT EXISTS context_event_fts " + "USING fts5(event_id UNINDEXED, title, body, type)", + &ftsError); + ftsError = nil; + OPSQLiteExec(db, + @"CREATE VIRTUAL TABLE IF NOT EXISTS commitment_fts " + "USING fts5(commitment_id UNINDEXED, title, description, trigger_type)", + &ftsError); + ftsError = nil; + OPSQLiteExec(db, + @"CREATE VIRTUAL TABLE IF NOT EXISTS watcher_fts " + "USING fts5(watcher_id UNINDEXED, title, query, source, type)", + &ftsError); + ftsError = nil; + OPSQLiteExec(db, + @"CREATE VIRTUAL TABLE IF NOT EXISTS agent_job_fts " + "USING fts5(job_id UNINDEXED, title, prompt, type)", + &ftsError); + ftsError = nil; + OPSQLiteExec(db, + @"ALTER TABLE agent_job ADD COLUMN scheduler_enabled INTEGER NOT NULL DEFAULT 0", + &ftsError); + + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "INSERT OR REPLACE INTO schema_migrations(name, applied_at_ms) VALUES(?, ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_text(statement, 1, "openphone_store_v1", -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 2, OPNowMs()); + sqlite3_step(statement); + } + sqlite3_finalize(statement); + statement = NULL; + if (sqlite3_prepare_v2(db, + "INSERT OR REPLACE INTO schema_migrations(name, applied_at_ms) VALUES(?, ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_text(statement, 1, "openphone_store_v2", -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 2, OPNowMs()); + sqlite3_step(statement); + } + sqlite3_finalize(statement); + statement = NULL; + if (sqlite3_prepare_v2(db, + "INSERT OR REPLACE INTO schema_migrations(name, applied_at_ms) VALUES(?, ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_text(statement, 1, "openphone_store_v3_scheduler_enabled", -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(statement, 2, OPNowMs()); + sqlite3_step(statement); + } + sqlite3_finalize(statement); + return YES; +} + +static BOOL OPSQLiteOpen(sqlite3 **dbOut, NSString **errorOut) { + OPEnsureDirectories(); + sqlite3 *db = NULL; + int rc = sqlite3_open_v2(OPDatabasePath().UTF8String, &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, NULL); + if (rc != SQLITE_OK) { + if (errorOut) { + *errorOut = db ? [NSString stringWithUTF8String:sqlite3_errmsg(db)] : @"sqlite_open_failed"; + } + if (db) { + sqlite3_close(db); + } + return NO; + } + sqlite3_busy_timeout(db, 5000); + if (!OPSQLiteMigrate(db, errorOut)) { + sqlite3_close(db); + return NO; + } + *dbOut = db; + return YES; +} + +static void OPSQLiteBindText(sqlite3_stmt *statement, int index, NSString *value) { + sqlite3_bind_text(statement, index, (value ?: @"").UTF8String, -1, SQLITE_TRANSIENT); +} + +static double OPDoubleFromRequest(NSDictionary *request, NSString *key, + double defaultValue, double minValue, double maxValue) { + id value = request[key]; + double parsed = defaultValue; + if ([value isKindOfClass:[NSNumber class]]) { + parsed = [value doubleValue]; + } else if ([value isKindOfClass:[NSString class]]) { + parsed = [(NSString *)value doubleValue]; + } + if (parsed < minValue) { + parsed = minValue; + } + if (parsed > maxValue) { + parsed = maxValue; + } + return parsed; +} + +static NSString *OPStringFromRequest(NSDictionary *request, NSString *key, NSString *defaultValue) { + id value = request[key]; + if ([value isKindOfClass:[NSString class]]) { + return value; + } + if ([value isKindOfClass:[NSNumber class]]) { + return [value stringValue]; + } + return defaultValue ?: @""; +} + +static id OPJSONObjectFromRequest(NSDictionary *request, NSString *key, id defaultValue) { + id value = request[key]; + if ([value isKindOfClass:[NSDictionary class]] || [value isKindOfClass:[NSArray class]]) { + return OPRedactedObject(value, 0); + } + if ([value isKindOfClass:[NSString class]]) { + NSDictionary *dictionary = OPJSONDictionary(value); + if (dictionary.count > 0) { + return OPRedactedObject(dictionary, 0); + } + } + return defaultValue ?: @{}; +} + +static long long OPRecordIdFromValue(id value) { + if ([value isKindOfClass:[NSNumber class]]) { + return [(NSNumber *)value longLongValue]; + } + if (![value isKindOfClass:[NSString class]]) { + return 0; + } + NSString *string = [(NSString *)value stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (string.length == 0) { + return 0; + } + long long direct = [string longLongValue]; + if (direct > 0) { + return direct; + } + NSInteger end = (NSInteger)string.length - 1; + while (end >= 0) { + unichar c = [string characterAtIndex:(NSUInteger)end]; + if (c < '0' || c > '9') { + break; + } + end--; + } + if (end == (NSInteger)string.length - 1) { + return 0; + } + NSString *suffix = [string substringFromIndex:(NSUInteger)(end + 1)]; + return [suffix longLongValue]; +} + +static long long OPRecordIdFromRequest(NSDictionary *request, NSArray *keys) { + for (NSString *key in keys) { + long long value = OPRecordIdFromValue(request[key]); + if (value > 0) { + return value; + } + } + return 0; +} + +static NSString *OPFTSQuery(NSString *query) { + if (query.length == 0) { + return @""; + } + NSCharacterSet *split = [[NSCharacterSet alphanumericCharacterSet] invertedSet]; + NSArray *rawTokens = [query componentsSeparatedByCharactersInSet:split]; + NSMutableArray *tokens = [NSMutableArray array]; + for (NSString *raw in rawTokens) { + NSString *token = [raw stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (token.length == 0) { + continue; + } + NSString *escaped = [token stringByReplacingOccurrencesOfString:@"\"" withString:@"\"\""]; + [tokens addObject:[NSString stringWithFormat:@"\"%@\"", escaped]]; + } + return [tokens componentsJoinedByString:@" OR "]; +} + +static NSDictionary *OPMemoryFromStatement(sqlite3_stmt *statement) { + NSString *metadataJSON = OPSQLiteColumnString(statement, 9); + long long rowId = sqlite3_column_int64(statement, 0); + return @{ + @"id": @(rowId), + @"memory_id": [NSString stringWithFormat:@"ios-memory-%lld", rowId], + @"created_at_ms": @(sqlite3_column_int64(statement, 1)), + @"updated_at_ms": @(sqlite3_column_int64(statement, 2)), + @"type": OPSQLiteColumnString(statement, 3), + @"subject": OPSQLiteColumnString(statement, 4), + @"text": OPSQLiteColumnString(statement, 5), + @"confidence": @(sqlite3_column_double(statement, 6)), + @"source": OPSQLiteColumnString(statement, 7), + @"reason": OPSQLiteColumnString(statement, 8), + @"metadata": OPJSONDictionary(metadataJSON) + }; +} + +static NSDictionary *OPMemoryReadById(sqlite3 *db, long long memoryId) { + if (memoryId <= 0) { + return nil; + } + NSDictionary *memory = nil; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "SELECT id, created_at_ms, updated_at_ms, type, subject, text, confidence, source, reason, metadata_json " + "FROM memory WHERE id = ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, memoryId); + if (sqlite3_step(statement) == SQLITE_ROW) { + memory = OPMemoryFromStatement(statement); + } + } + sqlite3_finalize(statement); + return memory; +} + +static void OPMemoryDeleteFTS(sqlite3 *db, long long memoryId) { + if (memoryId <= 0 || !OPSQLiteTableExists(db, @"memory_fts")) { + return; + } + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, "DELETE FROM memory_fts WHERE memory_id = ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, memoryId); + sqlite3_step(statement); + } + sqlite3_finalize(statement); +} + +static void OPMemoryIndexFTS(sqlite3 *db, NSDictionary *memory) { + if (![memory isKindOfClass:[NSDictionary class]] || !OPSQLiteTableExists(db, @"memory_fts")) { + return; + } + long long memoryId = [memory[@"id"] respondsToSelector:@selector(longLongValue)] + ? [memory[@"id"] longLongValue] : 0; + if (memoryId <= 0) { + return; + } + OPMemoryDeleteFTS(db, memoryId); + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "INSERT INTO memory_fts(memory_id, text, subject, type) VALUES(?, ?, ?, ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, memoryId); + OPSQLiteBindText(statement, 2, memory[@"text"]); + OPSQLiteBindText(statement, 3, memory[@"subject"]); + OPSQLiteBindText(statement, 4, memory[@"type"]); + sqlite3_step(statement); + } + sqlite3_finalize(statement); +} + +static long long OPRecordContextEvent(NSString *type, NSString *source, NSString *taskId, + NSString *title, NSString *body, NSDictionary *metadata) { + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + OPLog(@"context db open failed: %@", error ?: @"unknown"); + return 0; + } + long long eventId = 0; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "INSERT INTO context_event(created_at_ms, type, source, task_id, title, body, metadata_json) " + "VALUES(?, ?, ?, ?, ?, ?, ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, OPNowMs()); + OPSQLiteBindText(statement, 2, type ?: @"event"); + OPSQLiteBindText(statement, 3, source ?: @"openphone.agentd"); + OPSQLiteBindText(statement, 4, taskId ?: @""); + OPSQLiteBindText(statement, 5, title ?: @""); + OPSQLiteBindText(statement, 6, body ?: @""); + OPSQLiteBindText(statement, 7, OPJSONString(metadata ?: @{})); + if (sqlite3_step(statement) == SQLITE_DONE) { + eventId = sqlite3_last_insert_rowid(db); + } + } + sqlite3_finalize(statement); + + if (eventId > 0 && OPSQLiteTableExists(db, @"context_event_fts")) { + sqlite3_stmt *fts = NULL; + if (sqlite3_prepare_v2(db, + "INSERT INTO context_event_fts(event_id, title, body, type) VALUES(?, ?, ?, ?)", + -1, &fts, NULL) == SQLITE_OK) { + sqlite3_bind_int64(fts, 1, eventId); + OPSQLiteBindText(fts, 2, title ?: @""); + OPSQLiteBindText(fts, 3, body ?: @""); + OPSQLiteBindText(fts, 4, type ?: @"event"); + sqlite3_step(fts); + } + sqlite3_finalize(fts); + } + sqlite3_close(db); + return eventId; +} + +static NSString *OPExecutablePath(NSArray *candidates) { + NSFileManager *fileManager = [NSFileManager defaultManager]; + for (NSString *candidate in candidates) { + if ([fileManager isExecutableFileAtPath:candidate]) { + return candidate; + } + } + return nil; +} + +static NSDictionary *OPSpawn(NSArray *arguments) { + if (arguments.count == 0) { + return @{@"ok": @NO, @"reason": @"missing_command"}; + } + char **argv = calloc(arguments.count + 1, sizeof(char *)); + if (!argv) { + return @{@"ok": @NO, @"reason": @"calloc_failed"}; + } + for (NSUInteger i = 0; i < arguments.count; i++) { + NSString *argument = [arguments[i] isKindOfClass:[NSString class]] ? arguments[i] : @""; + argv[i] = strdup(argument.UTF8String ?: ""); + if (!argv[i]) { + for (NSUInteger j = 0; j < i; j++) { + free(argv[j]); + } + free(argv); + return @{@"ok": @NO, @"reason": @"strdup_failed"}; + } + } + argv[arguments.count] = NULL; + + pid_t pid = 0; + int spawnResult = posix_spawn(&pid, argv[0], NULL, NULL, argv, environ); + for (NSUInteger i = 0; i < arguments.count; i++) { + free(argv[i]); + } + free(argv); + if (spawnResult != 0) { + return @{ + @"ok": @NO, + @"reason": @"spawn_failed", + @"errno": @(spawnResult), + @"detail": [NSString stringWithUTF8String:strerror(spawnResult)] + }; + } + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + return @{@"ok": @NO, @"reason": @"waitpid_failed", @"errno": @(errno)}; + } + BOOL exited = WIFEXITED(status); + int exitCode = exited ? WEXITSTATUS(status) : -1; + return @{ + @"ok": @(exited && exitCode == 0), + @"pid": @(pid), + @"exit_code": @(exitCode), + @"raw_status": @(status) + }; +} + +static NSString *OPSpawnCapture(NSArray *arguments, NSUInteger maxBytes) { + if (arguments.count == 0) { + return nil; + } + int pipeFds[2]; + if (pipe(pipeFds) != 0) { + return nil; + } + char **argv = calloc(arguments.count + 1, sizeof(char *)); + if (!argv) { + close(pipeFds[0]); + close(pipeFds[1]); + return nil; + } + for (NSUInteger i = 0; i < arguments.count; i++) { + NSString *argument = [arguments[i] isKindOfClass:[NSString class]] ? arguments[i] : @""; + argv[i] = strdup(argument.UTF8String ?: ""); + if (!argv[i]) { + for (NSUInteger j = 0; j < i; j++) { + free(argv[j]); + } + free(argv); + close(pipeFds[0]); + close(pipeFds[1]); + return nil; + } + } + argv[arguments.count] = NULL; + + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_adddup2(&actions, pipeFds[1], STDOUT_FILENO); + posix_spawn_file_actions_addclose(&actions, pipeFds[0]); + posix_spawn_file_actions_addclose(&actions, pipeFds[1]); + + pid_t pid = 0; + int spawnResult = posix_spawn(&pid, argv[0], &actions, NULL, argv, environ); + posix_spawn_file_actions_destroy(&actions); + for (NSUInteger i = 0; i < arguments.count; i++) { + free(argv[i]); + } + free(argv); + close(pipeFds[1]); + + if (spawnResult != 0) { + close(pipeFds[0]); + return nil; + } + + NSMutableData *data = [NSMutableData data]; + char buffer[4096]; + while (data.length < maxBytes) { + ssize_t count = read(pipeFds[0], buffer, sizeof(buffer)); + if (count > 0) { + NSUInteger remaining = maxBytes - data.length; + [data appendBytes:buffer length:MIN((NSUInteger)count, remaining)]; + continue; + } + break; + } + close(pipeFds[0]); + int status = 0; + waitpid(pid, &status, 0); + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + return nil; + } + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; +} + +static NSString *OPForegroundBundleIdentifier(void) { + typedef CFStringRef (*OPCopyFrontmostApplicationDisplayIdentifierFunc)(void); + typedef mach_port_t (*OPSpringBoardServerPortFunc)(void); + typedef void (*OPFrontmostApplicationDisplayIdentifierFunc)(mach_port_t port, char *result); + static OPCopyFrontmostApplicationDisplayIdentifierFunc frontmostFunc = NULL; + static OPSpringBoardServerPortFunc springBoardPortFunc = NULL; + static OPFrontmostApplicationDisplayIdentifierFunc frontmostCStringFunc = NULL; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + NSArray *frameworks = @[ + @"/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices", + @"/var/jb/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices" + ]; + for (NSString *framework in frameworks) { + void *handle = dlopen(framework.UTF8String, RTLD_LAZY); + if (!handle) { + continue; + } + frontmostFunc = (OPCopyFrontmostApplicationDisplayIdentifierFunc)dlsym( + handle, "SBSCopyFrontmostApplicationDisplayIdentifier"); + springBoardPortFunc = (OPSpringBoardServerPortFunc)dlsym( + handle, "SBSSpringBoardServerPort"); + frontmostCStringFunc = (OPFrontmostApplicationDisplayIdentifierFunc)dlsym( + handle, "SBFrontmostApplicationDisplayIdentifier"); + if (frontmostFunc || (springBoardPortFunc && frontmostCStringFunc)) { + break; + } + } + }); + if (frontmostFunc) { + CFStringRef copied = frontmostFunc(); + if (copied) { + NSString *identifier = CFBridgingRelease(copied); + if ([identifier isKindOfClass:[NSString class]] && identifier.length > 0) { + return identifier; + } + } + } + if (springBoardPortFunc && frontmostCStringFunc) { + char result[512]; + memset(result, 0, sizeof(result)); + frontmostCStringFunc(springBoardPortFunc(), result); + if (result[0] != '\0') { + return [NSString stringWithUTF8String:result]; + } + } + return nil; +} + +static NSArray *OPRunningApplications(void) { + NSString *output = OPSpawnCapture(@[@"/bin/ps", @"ax", @"-o", @"pid=", @"-o", @"comm="], + 128 * 1024); + if (output.length == 0) { + return @[]; + } + NSMutableArray *apps = [NSMutableArray array]; + NSArray *lines = [output componentsSeparatedByCharactersInSet: + [NSCharacterSet newlineCharacterSet]]; + for (NSString *rawLine in lines) { + NSString *line = [rawLine stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (line.length == 0) { + continue; + } + NSScanner *scanner = [NSScanner scannerWithString:line]; + int pid = 0; + if (![scanner scanInt:&pid]) { + continue; + } + NSString *path = [[line substringFromIndex:scanner.scanLocation] + stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]]; + NSRange appRange = [path rangeOfString:@".app/" options:NSBackwardsSearch]; + if (appRange.location == NSNotFound) { + continue; + } + NSString *appPath = [path substringToIndex:appRange.location + 4]; + NSString *infoPath = [appPath stringByAppendingPathComponent:@"Info.plist"]; + NSDictionary *info = [NSDictionary dictionaryWithContentsOfFile:infoPath] ?: @{}; + NSString *bundleId = [info[@"CFBundleIdentifier"] isKindOfClass:[NSString class]] + ? info[@"CFBundleIdentifier"] : nil; + NSString *displayName = [info[@"CFBundleDisplayName"] isKindOfClass:[NSString class]] + ? info[@"CFBundleDisplayName"] : nil; + if (displayName.length == 0 && [info[@"CFBundleName"] isKindOfClass:[NSString class]]) { + displayName = info[@"CFBundleName"]; + } + if (bundleId.length == 0) { + bundleId = [[appPath lastPathComponent] stringByDeletingPathExtension]; + } + [apps addObject:@{ + @"pid": @(pid), + @"bundle_id": bundleId ?: @"unknown", + @"display_name": displayName ?: [[appPath lastPathComponent] stringByDeletingPathExtension], + @"app_path": appPath, + @"executable_path": path + }]; + } + [apps sortUsingDescriptors:@[ + [NSSortDescriptor sortDescriptorWithKey:@"pid" ascending:NO] + ]]; + return apps; +} + +static NSDictionary *OPApplicationInfoForPath(NSString *appPath) { + NSString *infoPath = [appPath stringByAppendingPathComponent:@"Info.plist"]; + NSDictionary *info = [NSDictionary dictionaryWithContentsOfFile:infoPath] ?: @{}; + NSString *bundleId = [info[@"CFBundleIdentifier"] isKindOfClass:[NSString class]] + ? info[@"CFBundleIdentifier"] : nil; + if (bundleId.length == 0) { + return nil; + } + NSString *displayName = [info[@"CFBundleDisplayName"] isKindOfClass:[NSString class]] + ? info[@"CFBundleDisplayName"] : nil; + if (displayName.length == 0 && [info[@"CFBundleName"] isKindOfClass:[NSString class]]) { + displayName = info[@"CFBundleName"]; + } + NSString *executableName = [info[@"CFBundleExecutable"] isKindOfClass:[NSString class]] + ? info[@"CFBundleExecutable"] : nil; + NSMutableDictionary *app = [NSMutableDictionary dictionary]; + app[@"bundle_id"] = bundleId; + app[@"display_name"] = displayName ?: [appPath.lastPathComponent stringByDeletingPathExtension]; + app[@"app_path"] = appPath; + if (executableName.length > 0) { + app[@"executable_path"] = [appPath stringByAppendingPathComponent:executableName]; + } + NSString *bundleVersion = [info[@"CFBundleShortVersionString"] isKindOfClass:[NSString class]] + ? info[@"CFBundleShortVersionString"] : nil; + if (bundleVersion.length > 0) { + app[@"version"] = bundleVersion; + } + return app; +} + +static NSArray *OPInstalledApplications(NSUInteger limit) { + NSArray *roots = @[ + @"/Applications", + @"/System/Applications", + @"/System/Library/CoreServices", + @"/var/jb/Applications", + @"/var/containers/Bundle/Application" + ]; + NSFileManager *fileManager = [NSFileManager defaultManager]; + NSMutableDictionary *appsByBundleId = [NSMutableDictionary dictionary]; + for (NSString *root in roots) { + BOOL isDirectory = NO; + if (![fileManager fileExistsAtPath:root isDirectory:&isDirectory] || !isDirectory) { + continue; + } + NSDirectoryEnumerator *enumerator = [fileManager enumeratorAtPath:root]; + for (NSString *relativePath in enumerator) { + if (![relativePath.pathExtension isEqualToString:@"app"]) { + continue; + } + NSString *appPath = [root stringByAppendingPathComponent:relativePath]; + NSDictionary *app = OPApplicationInfoForPath(appPath); + NSString *bundleId = app[@"bundle_id"]; + if (bundleId.length > 0 && !appsByBundleId[bundleId]) { + appsByBundleId[bundleId] = app; + } + [enumerator skipDescendants]; + if (limit > 0 && appsByBundleId.count >= limit) { + break; + } + } + if (limit > 0 && appsByBundleId.count >= limit) { + break; + } + } + NSMutableArray *apps = [[appsByBundleId allValues] mutableCopy]; + [apps sortUsingDescriptors:@[ + [NSSortDescriptor sortDescriptorWithKey:@"display_name" ascending:YES selector:@selector(localizedCaseInsensitiveCompare:)] + ]]; + return apps; +} + +static NSData *OPLZFSEDecodedData(NSData *compressed) { + if (compressed.length == 0) { + return nil; + } + const uint8_t *input = compressed.bytes; + size_t hintedSize = 0; + if (compressed.length >= 8 && memcmp(input, "bvx2", 4) == 0) { + hintedSize = (size_t)input[4] + | ((size_t)input[5] << 8) + | ((size_t)input[6] << 16) + | ((size_t)input[7] << 24); + } + size_t outputSize = hintedSize > 0 ? hintedSize : MAX((size_t)4096, compressed.length * 8); + const size_t maxOutputSize = 1024 * 1024; + for (NSUInteger attempt = 0; attempt < 8 && outputSize <= maxOutputSize; attempt++) { + NSMutableData *decoded = [NSMutableData dataWithLength:outputSize]; + size_t written = compression_decode_buffer(decoded.mutableBytes, + decoded.length, + input, + compressed.length, + NULL, + COMPRESSION_LZFSE); + if (written > 0 && written <= decoded.length) { + decoded.length = written; + return decoded; + } + if (hintedSize > 0) { + break; + } + outputSize *= 2; + } + return nil; +} + +static NSString *OPBundleIdentifierFromSceneIdentifier(NSString *value) { + if (![value hasPrefix:@"sceneID:"]) { + return nil; + } + NSString *identifier = [value substringFromIndex:@"sceneID:".length]; + NSRange paren = [identifier rangeOfString:@"("]; + if (paren.location != NSNotFound) { + identifier = [identifier substringToIndex:paren.location]; + } + if ([identifier hasSuffix:@"-default"]) { + identifier = [identifier substringToIndex:identifier.length - @"-default".length]; + } else if (identifier.length > 37) { + NSString *suffix = [identifier substringFromIndex:identifier.length - 37]; + NSRegularExpression *uuidSuffix = [NSRegularExpression regularExpressionWithPattern: + @"^-[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$" + options:0 error:nil]; + if ([uuidSuffix numberOfMatchesInString:suffix + options:0 + range:NSMakeRange(0, suffix.length)] > 0) { + identifier = [identifier substringToIndex:identifier.length - 37]; + } + } + return identifier; +} + +static BOOL OPBundleIdentifierLooksValid(NSString *value) { + if (![value isKindOfClass:[NSString class]] || value.length < 3 || value.length > 200) { + return NO; + } + if (![value containsString:@"."]) { + return NO; + } + NSArray *prefixes = @[@"com.", @"org.", @"net.", @"io.", @"app."]; + BOOL hasKnownPrefix = NO; + for (NSString *prefix in prefixes) { + if ([value hasPrefix:prefix]) { + hasKnownPrefix = YES; + break; + } + } + if (!hasKnownPrefix) { + return NO; + } + NSCharacterSet *allowed = [NSCharacterSet characterSetWithCharactersInString: + @"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_"]; + for (NSUInteger i = 0; i < value.length; i++) { + if (![allowed characterIsMember:[value characterAtIndex:i]]) { + return NO; + } + } + return YES; +} + +static NSString *OPSafeFilenameForBundleId(NSString *bundleId) { + if (![bundleId isKindOfClass:[NSString class]] || bundleId.length == 0) { + return @"unknown"; + } + NSMutableString *safe = [NSMutableString string]; + NSCharacterSet *allowed = [NSCharacterSet characterSetWithCharactersInString: + @"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-"]; + for (NSUInteger index = 0; index < bundleId.length; index++) { + unichar ch = [bundleId characterAtIndex:index]; + if ([allowed characterIsMember:ch]) { + [safe appendFormat:@"%C", ch]; + } else { + [safe appendString:@"_"]; + } + } + return safe.length > 0 ? safe : @"unknown"; +} + +static NSString *OPAppUIStatePath(NSString *bundleId) { + NSString *filename = [OPSafeFilenameForBundleId(bundleId) + stringByAppendingPathExtension:@"json"]; + return [OPAppUIPath() stringByAppendingPathComponent:filename]; +} + +static NSDictionary *OPAppUIIntakeStatus(void) { + return @{ + @"status": OPAppUIIntakeReady ? @"ready" : + (OPAppUIIntakeThreadStarted ? @"starting_or_unavailable" : @"not_started"), + @"provider": @"openphone-agentd.app_ui_intake", + @"transport": @"tcp_loopback", + @"host": @"127.0.0.1", + @"port": @27631, + @"thread_started": @(OPAppUIIntakeThreadStarted != 0), + @"listen_ready": @(OPAppUIIntakeReady != 0), + @"publish_count": @((unsigned long long)OPAppUIIntakePublishCount), + @"last_publish_ms": @((long long)OPAppUIIntakeLastPublishMs), + @"last_bundle_id": OPAppUIIntakeLastBundleId ?: @"", + @"start_error": OPAppUIIntakeStartError ?: @"" + }; +} + +static NSDictionary *OPAppUIPublish(NSDictionary *request) { + NSDictionary *state = [request[@"state"] isKindOfClass:[NSDictionary class]] + ? request[@"state"] : nil; + if (!state && [request[@"app_ui_state"] isKindOfClass:[NSDictionary class]]) { + state = request[@"app_ui_state"]; + } + if (!state && [request[@"schema"] isEqualToString:@"openphone.app_ui_state.v1"]) { + state = request; + } + if (![state isKindOfClass:[NSDictionary class]]) { + return OPError(@"app_ui_state_missing"); + } + NSString *bundleId = [state[@"bundle_id"] isKindOfClass:[NSString class]] + ? state[@"bundle_id"] : @""; + if (!OPBundleIdentifierLooksValid(bundleId)) { + return OPError(@"invalid_app_ui_bundle_id"); + } + NSMutableDictionary *stored = [state mutableCopy]; + stored[@"schema"] = @"openphone.app_ui_state.v1"; + stored[@"status"] = stored[@"status"] ?: @"ok"; + stored[@"provider"] = stored[@"provider"] ?: @"OpenPhoneAppIntrospector.UIKitAccessibility"; + stored[@"bundle_id"] = bundleId; + if (![stored[@"timestamp_ms"] respondsToSelector:@selector(longLongValue)]) { + stored[@"timestamp_ms"] = @(OPNowMs()); + } + stored[@"received_at_ms"] = @(OPNowMs()); + stored[@"received_transport"] = OPStringFromRequest(request, @"transport", @"daemon_command"); + stored[@"storage_owner"] = @"openphone-agentd"; + NSString *path = OPAppUIStatePath(bundleId); + OPEnsureDirectories(); + BOOL ok = OPWriteProtectedJSONFile(path, stored); + if (!ok) { + return OPError(@"app_ui_state_write_failed"); + } + @synchronized(@"OPAppUIIntakeMetrics") { + OPAppUIIntakePublishCount++; + OPAppUIIntakeLastPublishMs = OPNowMs(); + OPAppUIIntakeLastBundleId = [bundleId copy]; + } + return @{ + @"status": @"ok", + @"schema": @"openphone.app_ui_publish_result.v1", + @"bundle_id": bundleId, + @"path": path, + @"ui_tree_status": [stored[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? (stored[@"ui_tree"][@"status"] ?: @"unknown") : @"missing", + @"source": @"openphone.agentd.app_ui_intake" + }; +} + +static void OPEnsureAppInputRequests(void) { + @synchronized(@"OPAppInputRequests") { + if (!OPAppInputRequests) { + OPAppInputRequests = [NSMutableDictionary dictionary]; + } + } +} + +static NSDictionary *OPAppInputPoll(NSDictionary *request) { + NSString *bundleId = OPStringFromRequest(request, @"bundle_id", @""); + NSString *pollScope = OPStringFromRequest(request, @"scope", @""); + if (!OPBundleIdentifierLooksValid(bundleId)) { + return OPError(@"invalid_app_input_bundle_id"); + } + OPEnsureAppInputRequests(); + long long now = OPNowMs(); + @synchronized(@"OPAppInputRequests") { + for (NSString *requestId in [OPAppInputRequests.allKeys copy]) { + NSMutableDictionary *entry = OPAppInputRequests[requestId]; + long long expiresAtMs = [entry[@"expires_at_ms"] respondsToSelector:@selector(longLongValue)] + ? [entry[@"expires_at_ms"] longLongValue] : 0; + if (expiresAtMs > 0 && expiresAtMs < now) { + [OPAppInputRequests removeObjectForKey:requestId]; + } + } + for (NSString *requestId in [OPAppInputRequests.allKeys copy]) { + NSMutableDictionary *entry = OPAppInputRequests[requestId]; + NSString *entryBundleId = [entry[@"bundle_id"] isKindOfClass:[NSString class]] + ? entry[@"bundle_id"] : @""; + NSString *status = [entry[@"status"] isKindOfClass:[NSString class]] + ? entry[@"status"] : @""; + NSDictionary *action = [entry[@"action"] isKindOfClass:[NSDictionary class]] + ? entry[@"action"] : @{}; + NSString *preferredScope = [action[@"preferred_input_scope"] isKindOfClass:[NSString class]] + ? action[@"preferred_input_scope"] : @""; + if (preferredScope.length > 0 && + (pollScope.length == 0 || ![preferredScope isEqualToString:pollScope])) { + continue; + } + if ([entryBundleId isEqualToString:bundleId] && [status isEqualToString:@"pending"]) { + entry[@"delivered_at_ms"] = @(now); + return @{ + @"status": @"ok", + @"schema": @"openphone.app_input_poll_result.v1", + @"provider": @"openphone-agentd.app_input_bridge", + @"request": [entry copy], + @"source": @"openphone.agentd.app_input" + }; + } + } + } + return @{ + @"status": @"idle", + @"schema": @"openphone.app_input_poll_result.v1", + @"provider": @"openphone-agentd.app_input_bridge", + @"bundle_id": bundleId, + @"source": @"openphone.agentd.app_input" + }; +} + +static NSDictionary *OPAppInputComplete(NSDictionary *request) { + NSString *requestId = OPStringFromRequest(request, @"request_id", @""); + NSString *bundleId = OPStringFromRequest(request, @"bundle_id", @""); + NSDictionary *response = [request[@"response"] isKindOfClass:[NSDictionary class]] + ? request[@"response"] : @{}; + if (requestId.length == 0) { + return OPError(@"missing_app_input_request_id"); + } + if (!OPBundleIdentifierLooksValid(bundleId)) { + return OPError(@"invalid_app_input_bundle_id"); + } + OPEnsureAppInputRequests(); + @synchronized(@"OPAppInputRequests") { + NSMutableDictionary *entry = OPAppInputRequests[requestId]; + if (![entry isKindOfClass:[NSMutableDictionary class]]) { + return OPError(@"app_input_request_not_found"); + } + NSString *entryBundleId = [entry[@"bundle_id"] isKindOfClass:[NSString class]] + ? entry[@"bundle_id"] : @""; + if (![entryBundleId isEqualToString:bundleId]) { + return OPError(@"app_input_bundle_mismatch"); + } + entry[@"status"] = @"completed"; + entry[@"completed_at_ms"] = @(OPNowMs()); + entry[@"response"] = response ?: @{}; + } + return @{ + @"status": @"ok", + @"schema": @"openphone.app_input_complete_result.v1", + @"provider": @"openphone-agentd.app_input_bridge", + @"request_id": requestId, + @"bundle_id": bundleId, + @"source": @"openphone.agentd.app_input" + }; +} + +static NSDictionary *OPAppInputInfo(NSDictionary *action, NSString *bundleId) { + NSString *actionType = [action[@"type"] isKindOfClass:[NSString class]] + ? action[@"type"] : @""; + if (!OPBundleIdentifierLooksValid(bundleId)) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneAppIntrospector.AppInput", + @"reason": @"invalid_app_input_bundle_id", + @"action_type": actionType, + @"bundle_id": bundleId ?: @"" + }; + } + long long timeoutMs = OPLongLongFromRequest(action, @"input_timeout_ms", 2500, 250, 5000); + NSString *requestId = [NSString stringWithFormat:@"app-input-%lld-%d", OPNowMs(), getpid()]; + long long now = OPNowMs(); + NSMutableDictionary *entry = [@{ + @"schema": @"openphone.app_input_request.v1", + @"status": @"pending", + @"provider": @"openphone-agentd.app_input_bridge", + @"request_id": requestId, + @"bundle_id": bundleId, + @"action": action ?: @{}, + @"created_at_ms": @(now), + @"expires_at_ms": @(now + timeoutMs), + @"timeout_ms": @(timeoutMs), + @"source": @"openphone.agentd.input.perform" + } mutableCopy]; + OPEnsureAppInputRequests(); + @synchronized(@"OPAppInputRequests") { + OPAppInputRequests[requestId] = entry; + } + + long long start = OPNowMs(); + while (OPNowMs() - start <= timeoutMs) { + NSDictionary *response = nil; + BOOL completed = NO; + @synchronized(@"OPAppInputRequests") { + NSMutableDictionary *current = OPAppInputRequests[requestId]; + NSString *status = [current[@"status"] isKindOfClass:[NSString class]] + ? current[@"status"] : @""; + if ([status isEqualToString:@"completed"]) { + response = [current[@"response"] isKindOfClass:[NSDictionary class]] + ? [current[@"response"] copy] : @{}; + [OPAppInputRequests removeObjectForKey:requestId]; + completed = YES; + } + } + if (completed) { + NSMutableDictionary *result = [response mutableCopy] ?: [NSMutableDictionary dictionary]; + result[@"provider"] = result[@"provider"] ?: @"OpenPhoneAppIntrospector.AppInput"; + result[@"request_id"] = requestId; + result[@"bundle_id"] = bundleId; + result[@"timeout_ms"] = @(timeoutMs); + result[@"action_type"] = actionType ?: @""; + result[@"source"] = result[@"source"] ?: @"app_process"; + return result; + } + usleep(100000); + } + + @synchronized(@"OPAppInputRequests") { + [OPAppInputRequests removeObjectForKey:requestId]; + } + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneAppIntrospector.AppInput", + @"reason": @"response_timeout", + @"request_id": requestId, + @"bundle_id": bundleId, + @"action_type": actionType, + @"timeout_ms": @(timeoutMs) + }; +} + +static NSArray *OPBundleIdentifiersFromRecentLayoutData(NSData *data) { + if (data.length == 0) { + return @[]; + } + const uint8_t *bytes = data.bytes; + NSMutableArray *identifiers = [NSMutableArray array]; + NSMutableSet *seen = [NSMutableSet set]; + NSUInteger i = 0; + while (i < data.length) { + uint8_t byte = bytes[i]; + if (byte < 32 || byte > 126) { + i++; + continue; + } + NSUInteger start = i; + while (i < data.length && bytes[i] >= 32 && bytes[i] <= 126) { + i++; + } + if (i - start < 3) { + continue; + } + NSData *stringData = [NSData dataWithBytes:bytes + start length:i - start]; + NSString *candidate = [[NSString alloc] initWithData:stringData encoding:NSASCIIStringEncoding]; + if (candidate.length == 0) { + continue; + } + NSString *bundleId = [candidate hasPrefix:@"sceneID:"] + ? OPBundleIdentifierFromSceneIdentifier(candidate) : candidate; + if (OPBundleIdentifierLooksValid(bundleId) && ![seen containsObject:bundleId]) { + [seen addObject:bundleId]; + [identifiers addObject:bundleId]; + } + } + return identifiers; +} + +static NSDictionary *OPInstalledAppsByBundleIdentifier(void) { + NSMutableDictionary *appsByBundleId = [NSMutableDictionary dictionary]; + for (NSDictionary *app in OPInstalledApplications(0)) { + NSString *bundleId = [app[@"bundle_id"] isKindOfClass:[NSString class]] + ? app[@"bundle_id"] : nil; + if (bundleId.length > 0) { + appsByBundleId[bundleId] = app; + } + } + return appsByBundleId; +} + +static NSDictionary *OPSpringBoardRecentLayoutInfo(NSUInteger limit) { + NSString *path = @"/var/mobile/Library/SpringBoard/RecentAppLayouts.pb.lzfse"; + NSData *compressed = [NSData dataWithContentsOfFile:path]; + if (compressed.length == 0) { + return @{ + @"status": @"unavailable", + @"provider": @"SpringBoard.RecentAppLayouts", + @"reason": @"recent_layout_missing", + @"path": path + }; + } + NSData *decoded = OPLZFSEDecodedData(compressed); + if (decoded.length == 0) { + return @{ + @"status": @"unavailable", + @"provider": @"SpringBoard.RecentAppLayouts", + @"reason": @"lzfse_decode_failed", + @"path": path, + @"compressed_bytes": @(compressed.length) + }; + } + + NSArray *bundleIds = OPBundleIdentifiersFromRecentLayoutData(decoded); + NSDictionary *installedApps = OPInstalledAppsByBundleIdentifier(); + NSMutableArray *apps = [NSMutableArray array]; + NSUInteger count = 0; + for (NSString *bundleId in bundleIds) { + NSDictionary *installed = installedApps[bundleId]; + NSMutableDictionary *entry = [@{ + @"bundle_id": bundleId, + @"rank": @(count), + @"source": @"SpringBoard.RecentAppLayouts" + } mutableCopy]; + NSString *displayName = [installed[@"display_name"] isKindOfClass:[NSString class]] + ? installed[@"display_name"] : nil; + NSString *appPath = [installed[@"app_path"] isKindOfClass:[NSString class]] + ? installed[@"app_path"] : nil; + if (displayName.length > 0) { + entry[@"display_name"] = displayName; + } + if (appPath.length > 0) { + entry[@"app_path"] = appPath; + } + [apps addObject:entry]; + count++; + if (limit > 0 && count >= limit) { + break; + } + } + + return @{ + @"status": apps.count > 0 ? @"ok" : @"empty", + @"provider": @"SpringBoard.RecentAppLayouts", + @"path": path, + @"compressed_bytes": @(compressed.length), + @"decoded_bytes": @(decoded.length), + @"apps": apps, + @"count": @(apps.count), + @"first_bundle_id": apps.count > 0 ? apps[0][@"bundle_id"] : @"" + }; +} + +static NSDictionary *OPSpringBoardPublishedState(void) { + NSString *path = OPSpringBoardStatePath(); + NSDictionary *state = OPReadJSONFile(path); + if (!state) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardState", + @"path": path, + @"reason": @"state_missing" + }; + } + long long now = OPNowMs(); + long long timestamp = 0; + id timestampValue = state[@"timestamp_ms"]; + if ([timestampValue respondsToSelector:@selector(longLongValue)]) { + timestamp = [timestampValue longLongValue]; + } + long long age = timestamp > 0 ? MAX(0, now - timestamp) : 9223372036854775807LL; + NSMutableDictionary *result = [state mutableCopy]; + result[@"status"] = age <= 30000 ? @"ok" : @"stale"; + result[@"provider"] = result[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardState"; + result[@"path"] = path; + result[@"age_ms"] = @(age); + if (age > 30000) { + result[@"reason"] = @"state_stale"; + } else { + NSString *publishedForeground = [result[@"foreground_app"] isKindOfClass:[NSString class]] + ? result[@"foreground_app"] : @""; + NSString *servicesForeground = OPForegroundBundleIdentifier(); + if (publishedForeground.length == 0 && + OPBundleIdentifierLooksValid(servicesForeground) && + ![servicesForeground isEqualToString:@"com.apple.springboard"]) { + result[@"foreground_app"] = servicesForeground; + result[@"foreground_source"] = @"agentd_springboardservices_enriched"; + result[@"foreground_enriched"] = @YES; + result[@"foreground_enrichment_source"] = @"SpringBoardServices"; + } + } + return result; +} + +static NSDictionary *OPSpringBoardTriggerStatus(void) { + NSString *path = OPSpringBoardTriggerStatusPath(); + NSDictionary *state = OPReadJSONFile(path); + if (!state) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardVolumeHooks", + @"path": path, + @"reason": @"trigger_status_missing" + }; + } + long long now = OPNowMs(); + long long timestamp = 0; + id timestampValue = state[@"timestamp_ms"]; + if ([timestampValue respondsToSelector:@selector(longLongValue)]) { + timestamp = [timestampValue longLongValue]; + } + long long age = timestamp > 0 ? MAX(0, now - timestamp) : 9223372036854775807LL; + NSMutableDictionary *result = [state mutableCopy]; + NSString *publishedStatus = [result[@"status"] isKindOfClass:[NSString class]] + ? result[@"status"] : @"unknown"; + result[@"status"] = age <= 30000 ? publishedStatus : @"stale"; + result[@"provider"] = result[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardVolumeHooks"; + result[@"path"] = path; + result[@"age_ms"] = @(age); + if (age > 30000) { + result[@"reason"] = @"trigger_status_stale"; + } + return result; +} + +static NSDictionary *OPAppUIPublishedState(NSString *foregroundBundleId) { + NSString *bundleId = [foregroundBundleId isKindOfClass:[NSString class]] + ? foregroundBundleId : @""; + NSString *path = OPAppUIStatePath(bundleId); + if (!OPBundleIdentifierLooksValid(bundleId)) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneAppIntrospector.UIKitAccessibility", + @"path": path, + @"reason": @"foreground_bundle_unavailable" + }; + } + NSDictionary *state = OPReadJSONFile(path); + if (!state) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneAppIntrospector.UIKitAccessibility", + @"bundle_id": bundleId, + @"path": path, + @"reason": @"state_missing" + }; + } + NSString *publishedBundleId = [state[@"bundle_id"] isKindOfClass:[NSString class]] + ? state[@"bundle_id"] : @""; + NSMutableDictionary *result = [state mutableCopy]; + result[@"provider"] = result[@"provider"] ?: @"OpenPhoneAppIntrospector.UIKitAccessibility"; + result[@"path"] = path; + if (![publishedBundleId isEqualToString:bundleId]) { + result[@"status"] = @"unavailable"; + result[@"reason"] = @"bundle_mismatch"; + result[@"requested_bundle_id"] = bundleId; + return result; + } + long long now = OPNowMs(); + long long timestamp = 0; + id timestampValue = state[@"timestamp_ms"]; + if ([timestampValue respondsToSelector:@selector(longLongValue)]) { + timestamp = [timestampValue longLongValue]; + } + long long age = timestamp > 0 ? MAX(0, now - timestamp) : 9223372036854775807LL; + result[@"age_ms"] = @(age); + long long receivedAt = [state[@"received_at_ms"] respondsToSelector:@selector(longLongValue)] + ? [state[@"received_at_ms"] longLongValue] : 0; + NSString *applicationState = [state[@"application_state_name"] isKindOfClass:[NSString class]] + ? state[@"application_state_name"] : @"unknown"; + NSDictionary *uiTree = [state[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? state[@"ui_tree"] : @{}; + if (OPProcessStartMs > 0 && receivedAt > 0 && receivedAt < OPProcessStartMs) { + // Published by a prior daemon process (before this restart). Even if the + // timestamp is recent, the UI may have changed while we were down, so a + // resumed task must force a fresh observation rather than trust it. + result[@"status"] = @"stale"; + result[@"reason"] = @"state_pre_restart"; + } else if (age > 10000) { + result[@"status"] = @"stale"; + result[@"reason"] = @"state_stale"; + } else if ([applicationState isEqualToString:@"background"]) { + result[@"status"] = @"unavailable"; + result[@"reason"] = @"app_background"; + } else if (![uiTree[@"status"] isEqualToString:@"ok"]) { + result[@"status"] = @"unavailable"; + result[@"reason"] = @"ui_tree_unavailable"; + } else { + result[@"status"] = @"ok"; + } + return result; +} + +static NSDictionary *OPScreenDisplayInfo(void) { + typedef CGSize (*OPGSMainScreenPixelSizeFunc)(void); + typedef CGSize (*OPGSMainScreenPointSizeFunc)(void); + typedef float (*OPGSMainScreenScaleFactorFunc)(void); + typedef int (*OPGSMainScreenOrientationFunc)(void); + typedef CGSize (*OPGSMainScreenSizeFunc)(void); + static OPGSMainScreenPixelSizeFunc pixelSizeFunc = NULL; + static OPGSMainScreenPointSizeFunc pointSizeFunc = NULL; + static OPGSMainScreenScaleFactorFunc scaleFactorFunc = NULL; + static OPGSMainScreenOrientationFunc orientationFunc = NULL; + static OPGSMainScreenSizeFunc screenSizeFunc = NULL; + static BOOL loaded = NO; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + NSArray *frameworks = @[ + @"/System/Library/PrivateFrameworks/GraphicsServices.framework/GraphicsServices", + @"/var/jb/System/Library/PrivateFrameworks/GraphicsServices.framework/GraphicsServices" + ]; + for (NSString *framework in frameworks) { + void *handle = dlopen(framework.UTF8String, RTLD_LAZY); + if (!handle) { + continue; + } + loaded = YES; + pixelSizeFunc = (OPGSMainScreenPixelSizeFunc)dlsym(handle, "GSMainScreenPixelSize"); + pointSizeFunc = (OPGSMainScreenPointSizeFunc)dlsym(handle, "GSMainScreenPointSize"); + scaleFactorFunc = (OPGSMainScreenScaleFactorFunc)dlsym(handle, "GSMainScreenScaleFactor"); + orientationFunc = (OPGSMainScreenOrientationFunc)dlsym(handle, "GSMainScreenOrientation"); + screenSizeFunc = (OPGSMainScreenSizeFunc)dlsym(handle, "GSMainScreenSize"); + if (pixelSizeFunc || pointSizeFunc || scaleFactorFunc || orientationFunc || screenSizeFunc) { + break; + } + } + }); + NSMutableDictionary *info = [NSMutableDictionary dictionary]; + info[@"status"] = loaded ? @"available" : @"unavailable"; + info[@"provider"] = @"GraphicsServices"; + NSNumber *pixelWidth = nil; + NSNumber *pixelHeight = nil; + NSNumber *pointWidth = nil; + NSNumber *pointHeight = nil; + if (pixelSizeFunc) { + CGSize size = pixelSizeFunc(); + pixelWidth = @((double)size.width); + pixelHeight = @((double)size.height); + info[@"pixel_width"] = pixelWidth; + info[@"pixel_height"] = pixelHeight; + } + if (pointSizeFunc) { + CGSize size = pointSizeFunc(); + pointWidth = @((double)size.width); + pointHeight = @((double)size.height); + info[@"point_width"] = pointWidth; + info[@"point_height"] = pointHeight; + } + if (screenSizeFunc) { + CGSize size = screenSizeFunc(); + info[@"screen_width"] = @((double)size.width); + info[@"screen_height"] = @((double)size.height); + } + if (scaleFactorFunc) { + float scale = scaleFactorFunc(); + if (scale >= 0.5f && scale <= 10.0f) { + info[@"scale"] = @((double)scale); + } + } + if (!info[@"scale"] && pixelWidth.doubleValue > 0.0 && pointWidth.doubleValue > 0.0) { + info[@"scale"] = @(pixelWidth.doubleValue / pointWidth.doubleValue); + } + if (orientationFunc) { + int orientation = orientationFunc(); + NSString *name = nil; + switch (orientation) { + case 1: + name = @"portrait"; + break; + case 2: + name = @"portrait_upside_down"; + break; + case 3: + name = @"landscape_left"; + break; + case 4: + name = @"landscape_right"; + break; + default: + break; + } + if (name) { + info[@"orientation"] = @(orientation); + info[@"orientation_name"] = name; + } + } + return info; +} + +static NSDictionary *OPScreenLockInfo(void) { + typedef mach_port_t (*OPSpringBoardServerPortFunc)(void); + typedef void (*OPSBGetScreenLockStatusFunc)(mach_port_t port, BOOL *lockStatus, BOOL *passcodeEnabled); + static OPSpringBoardServerPortFunc springBoardPortFunc = NULL; + static OPSBGetScreenLockStatusFunc lockStatusFunc = NULL; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + NSArray *frameworks = @[ + @"/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices", + @"/var/jb/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices" + ]; + for (NSString *framework in frameworks) { + void *handle = dlopen(framework.UTF8String, RTLD_LAZY); + if (!handle) { + continue; + } + springBoardPortFunc = (OPSpringBoardServerPortFunc)dlsym(handle, "SBSSpringBoardServerPort"); + lockStatusFunc = (OPSBGetScreenLockStatusFunc)dlsym(handle, "SBGetScreenLockStatus"); + if (springBoardPortFunc && lockStatusFunc) { + break; + } + } + }); + if (!springBoardPortFunc || !lockStatusFunc) { + return @{ + @"status": @"unavailable", + @"provider": @"SpringBoardServices", + @"reason": @"SBGetScreenLockStatus_missing" + }; + } + mach_port_t port = springBoardPortFunc(); + if (port == MACH_PORT_NULL) { + return @{ + @"status": @"unavailable", + @"provider": @"SpringBoardServices", + @"reason": @"springboard_port_missing" + }; + } + BOOL locked = NO; + BOOL passcodeEnabled = NO; + lockStatusFunc(port, &locked, &passcodeEnabled); + return @{ + @"status": @"available", + @"provider": @"SpringBoardServices.SBGetScreenLockStatus", + @"locked": @(locked), + @"passcode_enabled": @(passcodeEnabled) + }; +} + +typedef CFTypeRef OPHIDEventSystemClientRef; +typedef CFTypeRef OPHIDEventRef; +typedef CFTypeRef OPHIDEventQueueRef; +typedef OPHIDEventSystemClientRef (*OPIOHIDEventSystemClientCreateFunc)(CFAllocatorRef allocator); +typedef OPHIDEventSystemClientRef (*OPIOHIDEventSystemClientCreateSimpleClientFunc)(CFAllocatorRef allocator); +typedef void (*OPIOHIDEventSystemClientActivateFunc)(OPHIDEventSystemClientRef client); +typedef void (*OPIOHIDEventSystemClientDispatchEventFunc)(OPHIDEventSystemClientRef client, OPHIDEventRef event); +typedef void (*OPIOHIDEventSystemClientEventCallback)(void *target, void *refcon, + OPHIDEventQueueRef queue, OPHIDEventRef event); +typedef void (*OPIOHIDEventSystemClientRegisterEventCallbackFunc)( + OPHIDEventSystemClientRef client, OPIOHIDEventSystemClientEventCallback callback, + void *target, void *refcon); +typedef void (*OPIOHIDEventSystemClientScheduleWithRunLoopFunc)( + OPHIDEventSystemClientRef client, CFRunLoopRef runLoop, CFStringRef mode); +typedef uint32_t (*OPIOHIDEventGetTypeFunc)(OPHIDEventRef event); +typedef int (*OPIOHIDEventGetIntegerValueFunc)(OPHIDEventRef event, uint32_t field); +typedef OPHIDEventRef (*OPIOHIDEventCreateDigitizerEventFunc)(CFAllocatorRef allocator, + uint64_t timestamp, uint32_t type, uint32_t index, uint32_t identity, + uint32_t eventMask, uint32_t buttonMask, double x, double y, double z, + double tipPressure, double barrelPressure, Boolean range, Boolean touch, + uint32_t options); +typedef OPHIDEventRef (*OPIOHIDEventCreateDigitizerFingerEventFunc)(CFAllocatorRef allocator, + uint64_t timestamp, uint32_t index, uint32_t identity, uint32_t eventMask, + double x, double y, double z, double tipPressure, double twist, + Boolean range, Boolean touch, uint32_t options); +typedef OPHIDEventRef (*OPIOHIDEventCreateKeyboardEventFunc)(CFAllocatorRef allocator, + uint64_t timestamp, uint16_t usagePage, uint16_t usage, Boolean down, + uint32_t options); +typedef void (*OPIOHIDEventAppendEventFunc)(OPHIDEventRef parent, OPHIDEventRef child); + +static const uint32_t OPIOHIDDigitizerTransducerTypeHand = 0x23; +static const uint32_t OPIOHIDDigitizerEventRange = 0x00000001; +static const uint32_t OPIOHIDDigitizerEventTouch = 0x00000002; +static const uint32_t OPIOHIDDigitizerEventPosition = 0x00000004; +static const uint32_t OPIOHIDDigitizerEventIdentity = 0x00000020; +static const uint32_t OPIOHIDDigitizerEventStart = 0x00000100; +static const uint32_t OPIOHIDDigitizerEventStop = 0x00000008; +static const uint32_t OPIOHIDEventOptionIsAbsolute = 0x00000001; +static const uint32_t OPIOHIDEventTypeKeyboard = 3; +static const uint32_t OPIOHIDEventFieldKeyboardUsagePage = (OPIOHIDEventTypeKeyboard << 16); +static const uint32_t OPIOHIDEventFieldKeyboardUsage = (OPIOHIDEventTypeKeyboard << 16) + 1; +static const uint32_t OPIOHIDEventFieldKeyboardDown = (OPIOHIDEventTypeKeyboard << 16) + 2; +static const uint32_t OPIOHIDUsagePageConsumer = 0x0c; +static const uint32_t OPIOHIDUsageConsumerVolumeIncrement = 0xe9; +static const uint32_t OPIOHIDUsageConsumerVolumeDecrement = 0xea; + +static BOOL OPHIDLoaded = NO; +static BOOL OPHIDLoadAttempted = NO; +static BOOL OPHIDClientAttempted = NO; +static NSMutableArray *OPHIDMissingSymbols = nil; +static NSMutableArray *OPHIDClientErrors = nil; +static OPHIDEventSystemClientRef OPHIDClient = NULL; +static OPIOHIDEventSystemClientCreateFunc OPHIDCreateClient = NULL; +static OPIOHIDEventSystemClientCreateSimpleClientFunc OPHIDCreateSimpleClient = NULL; +static OPIOHIDEventSystemClientActivateFunc OPHIDActivateClient = NULL; +static OPIOHIDEventSystemClientDispatchEventFunc OPHIDDispatchEvent = NULL; +static OPIOHIDEventSystemClientRegisterEventCallbackFunc OPHIDRegisterEventCallback = NULL; +static OPIOHIDEventSystemClientScheduleWithRunLoopFunc OPHIDScheduleWithRunLoop = NULL; +static OPIOHIDEventGetTypeFunc OPHIDEventGetType = NULL; +static OPIOHIDEventGetIntegerValueFunc OPHIDEventGetIntegerValue = NULL; +static OPIOHIDEventCreateDigitizerEventFunc OPHIDCreateDigitizerEvent = NULL; +static OPIOHIDEventCreateDigitizerFingerEventFunc OPHIDCreateFingerEvent = NULL; +static OPIOHIDEventCreateKeyboardEventFunc OPHIDCreateKeyboardEvent = NULL; +static OPIOHIDEventAppendEventFunc OPHIDAppendEvent = NULL; + +static void OPEnsureHIDInputLoaded(void) { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + OPHIDLoadAttempted = YES; + OPHIDMissingSymbols = [NSMutableArray array]; + NSArray *frameworks = @[ + @"/System/Library/Frameworks/IOKit.framework/IOKit", + @"/var/jb/System/Library/Frameworks/IOKit.framework/IOKit" + ]; + void *handle = NULL; + for (NSString *framework in frameworks) { + handle = dlopen(framework.UTF8String, RTLD_LAZY); + if (handle) { + break; + } + } + if (!handle) { + [OPHIDMissingSymbols addObject:@"IOKit.framework"]; + return; + } + OPHIDCreateClient = (OPIOHIDEventSystemClientCreateFunc)dlsym( + handle, "IOHIDEventSystemClientCreate"); + OPHIDCreateSimpleClient = (OPIOHIDEventSystemClientCreateSimpleClientFunc)dlsym( + handle, "IOHIDEventSystemClientCreateSimpleClient"); + OPHIDActivateClient = (OPIOHIDEventSystemClientActivateFunc)dlsym( + handle, "IOHIDEventSystemClientActivate"); + OPHIDDispatchEvent = (OPIOHIDEventSystemClientDispatchEventFunc)dlsym( + handle, "IOHIDEventSystemClientDispatchEvent"); + OPHIDRegisterEventCallback = (OPIOHIDEventSystemClientRegisterEventCallbackFunc)dlsym( + handle, "IOHIDEventSystemClientRegisterEventCallback"); + OPHIDScheduleWithRunLoop = (OPIOHIDEventSystemClientScheduleWithRunLoopFunc)dlsym( + handle, "IOHIDEventSystemClientScheduleWithRunLoop"); + OPHIDEventGetType = (OPIOHIDEventGetTypeFunc)dlsym(handle, "IOHIDEventGetType"); + OPHIDEventGetIntegerValue = (OPIOHIDEventGetIntegerValueFunc)dlsym( + handle, "IOHIDEventGetIntegerValue"); + OPHIDCreateDigitizerEvent = (OPIOHIDEventCreateDigitizerEventFunc)dlsym( + handle, "IOHIDEventCreateDigitizerEvent"); + OPHIDCreateFingerEvent = (OPIOHIDEventCreateDigitizerFingerEventFunc)dlsym( + handle, "IOHIDEventCreateDigitizerFingerEvent"); + OPHIDCreateKeyboardEvent = (OPIOHIDEventCreateKeyboardEventFunc)dlsym( + handle, "IOHIDEventCreateKeyboardEvent"); + OPHIDAppendEvent = (OPIOHIDEventAppendEventFunc)dlsym(handle, "IOHIDEventAppendEvent"); + + if (!OPHIDCreateSimpleClient && !OPHIDCreateClient) { + [OPHIDMissingSymbols addObject:@"IOHIDEventSystemClientCreate"]; + } + if (!OPHIDDispatchEvent) { + [OPHIDMissingSymbols addObject:@"IOHIDEventSystemClientDispatchEvent"]; + } + if (!OPHIDCreateFingerEvent) { + [OPHIDMissingSymbols addObject:@"IOHIDEventCreateDigitizerFingerEvent"]; + } + OPHIDLoaded = OPHIDMissingSymbols.count == 0; + }); +} + +static BOOL OPEnsureHIDClient(void) { + OPEnsureHIDInputLoaded(); + if (!OPHIDLoaded) { + return NO; + } + if (OPHIDClient) { + return YES; + } + if (!OPHIDClientErrors) { + OPHIDClientErrors = [NSMutableArray array]; + } + OPHIDClientAttempted = YES; + OPHIDClient = OPHIDCreateSimpleClient + ? OPHIDCreateSimpleClient(kCFAllocatorDefault) + : OPHIDCreateClient(kCFAllocatorDefault); + if (!OPHIDClient) { + [OPHIDClientErrors addObject:@"IOHIDEventSystemClient"]; + return NO; + } + return YES; +} + +static NSDictionary *OPHIDInputProviderInfo(void) { + OPEnsureHIDInputLoaded(); + return @{ + @"available": @(OPHIDLoaded), + @"provider": @"IOKit.IOHIDEventSystemClient", + @"attempted": @(OPHIDLoadAttempted), + @"client_attempted": @(OPHIDClientAttempted), + @"client_created": @(OPHIDClient != NULL), + @"keyboard_available": @(OPHIDCreateKeyboardEvent != NULL), + @"missing": OPHIDMissingSymbols ?: @[], + @"client_errors": OPHIDClientErrors ?: @[] + }; +} + +static BOOL OPHIDCoordinatesAreReasonable(double x, double y) { + if (!isfinite(x) || !isfinite(y) || x < 0.0 || y < 0.0) { + return NO; + } + NSDictionary *display = OPScreenDisplayInfo(); + double pointWidth = [display[@"point_width"] doubleValue]; + double pointHeight = [display[@"point_height"] doubleValue]; + double pixelWidth = [display[@"pixel_width"] doubleValue]; + double pixelHeight = [display[@"pixel_height"] doubleValue]; + double maxWidth = MAX(pointWidth, pixelWidth); + double maxHeight = MAX(pointHeight, pixelHeight); + if (maxWidth <= 0.0 || maxHeight <= 0.0) { + return YES; + } + return x <= maxWidth * 1.1 && y <= maxHeight * 1.1; +} + +static NSDictionary *OPDispatchHIDDigitizer(double x, double y, BOOL range, BOOL touch, + uint32_t eventMask) { + NSDictionary *provider = OPHIDInputProviderInfo(); + if (![provider[@"available"] boolValue]) { + return @{ + @"ok": @NO, + @"reason": @"iohid_provider_unavailable", + @"provider": provider + }; + } + if (!OPEnsureHIDClient()) { + return @{ + @"ok": @NO, + @"reason": @"iohid_client_unavailable", + @"provider": OPHIDInputProviderInfo() + }; + } + if (!OPHIDCoordinatesAreReasonable(x, y)) { + return @{ + @"ok": @NO, + @"reason": @"coordinates_out_of_range", + @"x": @(x), + @"y": @(y), + @"provider": provider + }; + } + + uint64_t timestamp = mach_absolute_time(); + OPHIDEventRef finger = OPHIDCreateFingerEvent(kCFAllocatorDefault, timestamp, + 1, 2, eventMask, x, y, 0.0, touch ? 1.0 : 0.0, 0.0, + range, touch, OPIOHIDEventOptionIsAbsolute); + if (!finger) { + return @{ + @"ok": @NO, + @"reason": @"finger_event_create_failed", + @"provider": provider + }; + } + + OPHIDEventRef eventToDispatch = finger; + OPHIDEventRef hand = NULL; + if (OPHIDCreateDigitizerEvent && OPHIDAppendEvent) { + hand = OPHIDCreateDigitizerEvent(kCFAllocatorDefault, timestamp, + OPIOHIDDigitizerTransducerTypeHand, 0, 1, eventMask, 0, + x, y, 0.0, touch ? 1.0 : 0.0, 0.0, range, touch, + OPIOHIDEventOptionIsAbsolute); + if (hand) { + OPHIDAppendEvent(hand, finger); + eventToDispatch = hand; + } + } + + OPHIDDispatchEvent(OPHIDClient, eventToDispatch); + if (hand) { + CFRelease(hand); + } + CFRelease(finger); + return @{ + @"ok": @YES, + @"provider": provider[@"provider"] ?: @"IOKit.IOHIDEventSystemClient", + @"x": @(x), + @"y": @(y), + @"range": @(range), + @"touch": @(touch), + @"event_mask": @(eventMask) + }; +} + +static NSDictionary *OPPerformHIDTap(double x, double y, long holdMs) { + holdMs = MAX(20, MIN(holdMs, 5000)); + uint32_t downMask = OPIOHIDDigitizerEventRange | + OPIOHIDDigitizerEventTouch | + OPIOHIDDigitizerEventPosition | + OPIOHIDDigitizerEventIdentity | + OPIOHIDDigitizerEventStart; + NSDictionary *down = OPDispatchHIDDigitizer(x, y, YES, YES, downMask); + if (![down[@"ok"] boolValue]) { + return down; + } + usleep((useconds_t)(holdMs * 1000)); + uint32_t upMask = OPIOHIDDigitizerEventRange | + OPIOHIDDigitizerEventTouch | + OPIOHIDDigitizerEventPosition | + OPIOHIDDigitizerEventStop; + NSDictionary *up = OPDispatchHIDDigitizer(x, y, NO, NO, upMask); + BOOL ok = [up[@"ok"] boolValue]; + return @{ + @"ok": @(ok), + @"provider": @"IOKit.IOHIDEventSystemClient", + @"kind": holdMs >= 500 ? @"long_press" : @"tap", + @"x": @(x), + @"y": @(y), + @"duration_ms": @(holdMs), + @"down": down, + @"up": up + }; +} + +static NSDictionary *OPPerformHIDSwipe(double startX, double startY, double endX, double endY, + long durationMs) { + durationMs = MAX(50, MIN(durationMs, 5000)); + if (!OPHIDCoordinatesAreReasonable(startX, startY) || + !OPHIDCoordinatesAreReasonable(endX, endY)) { + return @{ + @"ok": @NO, + @"reason": @"coordinates_out_of_range", + @"start_x": @(startX), + @"start_y": @(startY), + @"end_x": @(endX), + @"end_y": @(endY), + @"provider": OPHIDInputProviderInfo() + }; + } + uint32_t downMask = OPIOHIDDigitizerEventRange | + OPIOHIDDigitizerEventTouch | + OPIOHIDDigitizerEventPosition | + OPIOHIDDigitizerEventIdentity | + OPIOHIDDigitizerEventStart; + NSDictionary *down = OPDispatchHIDDigitizer(startX, startY, YES, YES, downMask); + if (![down[@"ok"] boolValue]) { + return down; + } + NSMutableArray *moves = [NSMutableArray array]; + int steps = 8; + long stepDelayUs = (durationMs * 1000) / steps; + for (int step = 1; step < steps; step++) { + double t = (double)step / (double)steps; + double x = startX + ((endX - startX) * t); + double y = startY + ((endY - startY) * t); + NSDictionary *move = OPDispatchHIDDigitizer(x, y, YES, YES, + OPIOHIDDigitizerEventPosition); + [moves addObject:move]; + if (![move[@"ok"] boolValue]) { + return @{ + @"ok": @NO, + @"reason": @"move_event_failed", + @"failed_move": move, + @"moves": moves, + @"provider": @"IOKit.IOHIDEventSystemClient" + }; + } + usleep((useconds_t)stepDelayUs); + } + uint32_t upMask = OPIOHIDDigitizerEventRange | + OPIOHIDDigitizerEventTouch | + OPIOHIDDigitizerEventPosition | + OPIOHIDDigitizerEventStop; + NSDictionary *up = OPDispatchHIDDigitizer(endX, endY, NO, NO, upMask); + BOOL ok = [up[@"ok"] boolValue]; + return @{ + @"ok": @(ok), + @"provider": @"IOKit.IOHIDEventSystemClient", + @"kind": @"swipe", + @"start_x": @(startX), + @"start_y": @(startY), + @"end_x": @(endX), + @"end_y": @(endY), + @"duration_ms": @(durationMs), + @"steps": @(steps), + @"down": down, + @"moves": moves, + @"up": up + }; +} + +static NSDictionary *OPDispatchHIDKeyboardUsage(uint16_t usage, BOOL down) { + NSDictionary *provider = OPHIDInputProviderInfo(); + if (!OPHIDCreateKeyboardEvent) { + return @{ + @"ok": @NO, + @"reason": @"keyboard_event_create_missing", + @"provider": provider + }; + } + if (![provider[@"available"] boolValue]) { + return @{ + @"ok": @NO, + @"reason": @"iohid_provider_unavailable", + @"provider": provider + }; + } + if (!OPEnsureHIDClient()) { + return @{ + @"ok": @NO, + @"reason": @"iohid_client_unavailable", + @"provider": OPHIDInputProviderInfo() + }; + } + + OPHIDEventRef event = OPHIDCreateKeyboardEvent(kCFAllocatorDefault, + mach_absolute_time(), 0x07, usage, down, 0); + if (!event) { + return @{ + @"ok": @NO, + @"reason": @"keyboard_event_create_failed", + @"usage": @(usage), + @"down": @(down), + @"provider": provider + }; + } + OPHIDDispatchEvent(OPHIDClient, event); + CFRelease(event); + return @{ + @"ok": @YES, + @"provider": provider[@"provider"] ?: @"IOKit.IOHIDEventSystemClient", + @"usage_page": @0x07, + @"usage": @(usage), + @"down": @(down) + }; +} + +static NSDictionary *OPTapHIDKeyboardUsage(uint16_t usage) { + NSDictionary *down = OPDispatchHIDKeyboardUsage(usage, YES); + if (![down[@"ok"] boolValue]) { + return down; + } + usleep(12000); + NSDictionary *up = OPDispatchHIDKeyboardUsage(usage, NO); + return @{ + @"ok": @([up[@"ok"] boolValue]), + @"provider": @"IOKit.IOHIDEventSystemClient", + @"usage": @(usage), + @"down": down, + @"up": up + }; +} + +static BOOL OPKeyboardUsageForCharacter(unichar c, uint16_t *usage, BOOL *shift) { + *shift = NO; + if (c >= 'a' && c <= 'z') { + *usage = (uint16_t)(0x04 + (c - 'a')); + return YES; + } + if (c >= 'A' && c <= 'Z') { + *usage = (uint16_t)(0x04 + (c - 'A')); + *shift = YES; + return YES; + } + if (c >= '1' && c <= '9') { + *usage = (uint16_t)(0x1e + (c - '1')); + return YES; + } + if (c == '0') { + *usage = 0x27; + return YES; + } + switch (c) { + case '\n': + case '\r': + *usage = 0x28; + return YES; + case '\t': + *usage = 0x2b; + return YES; + case ' ': + *usage = 0x2c; + return YES; + case '-': + *usage = 0x2d; + return YES; + case '_': + *usage = 0x2d; + *shift = YES; + return YES; + case '=': + *usage = 0x2e; + return YES; + case '+': + *usage = 0x2e; + *shift = YES; + return YES; + case '[': + *usage = 0x2f; + return YES; + case '{': + *usage = 0x2f; + *shift = YES; + return YES; + case ']': + *usage = 0x30; + return YES; + case '}': + *usage = 0x30; + *shift = YES; + return YES; + case '\\': + *usage = 0x31; + return YES; + case '|': + *usage = 0x31; + *shift = YES; + return YES; + case ';': + *usage = 0x33; + return YES; + case ':': + *usage = 0x33; + *shift = YES; + return YES; + case '\'': + *usage = 0x34; + return YES; + case '"': + *usage = 0x34; + *shift = YES; + return YES; + case '`': + *usage = 0x35; + return YES; + case '~': + *usage = 0x35; + *shift = YES; + return YES; + case ',': + *usage = 0x36; + return YES; + case '<': + *usage = 0x36; + *shift = YES; + return YES; + case '.': + *usage = 0x37; + return YES; + case '>': + *usage = 0x37; + *shift = YES; + return YES; + case '/': + *usage = 0x38; + return YES; + case '?': + *usage = 0x38; + *shift = YES; + return YES; + case '!': + *usage = 0x1e; + *shift = YES; + return YES; + case '@': + *usage = 0x1f; + *shift = YES; + return YES; + case '#': + *usage = 0x20; + *shift = YES; + return YES; + case '$': + *usage = 0x21; + *shift = YES; + return YES; + case '%': + *usage = 0x22; + *shift = YES; + return YES; + case '^': + *usage = 0x23; + *shift = YES; + return YES; + case '&': + *usage = 0x24; + *shift = YES; + return YES; + case '*': + *usage = 0x25; + *shift = YES; + return YES; + case '(': + *usage = 0x26; + *shift = YES; + return YES; + case ')': + *usage = 0x27; + *shift = YES; + return YES; + default: + return NO; + } +} + +static NSDictionary *OPPerformHIDTypeText(NSString *text) { + if (![text isKindOfClass:[NSString class]] || text.length == 0) { + return @{@"ok": @NO, @"reason": @"missing_text"}; + } + NSUInteger maxLength = MIN(text.length, (NSUInteger)4096); + NSUInteger typed = 0; + NSMutableArray *unsupported = [NSMutableArray array]; + for (NSUInteger i = 0; i < maxLength; i++) { + unichar c = [text characterAtIndex:i]; + uint16_t usage = 0; + BOOL shift = NO; + if (!OPKeyboardUsageForCharacter(c, &usage, &shift)) { + [unsupported addObject:@(c)]; + continue; + } + if (shift) { + NSDictionary *shiftDown = OPDispatchHIDKeyboardUsage(0xe1, YES); + if (![shiftDown[@"ok"] boolValue]) { + return shiftDown; + } + usleep(6000); + } + NSDictionary *key = OPTapHIDKeyboardUsage(usage); + if (shift) { + NSDictionary *shiftUp = OPDispatchHIDKeyboardUsage(0xe1, NO); + if (![shiftUp[@"ok"] boolValue]) { + return shiftUp; + } + } + if (![key[@"ok"] boolValue]) { + return key; + } + typed += 1; + usleep(12000); + } + return @{ + @"ok": @(unsupported.count == 0 && typed > 0), + @"provider": @"IOKit.IOHIDEventSystemClient", + @"kind": @"type_text", + @"requested_length": @(text.length), + @"attempted_length": @(maxLength), + @"typed_characters": @(typed), + @"unsupported_characters": @(unsupported.count), + @"truncated": @(text.length > maxLength) + }; +} + +typedef struct { + BOOL loaded; + BOOL hasWake; + BOOL hasHome; + BOOL hasCoordinateInput; + BOOL hasTextInput; +} OPInputProviderStatus; + +static OPInputProviderStatus OPInputStatus(void) { + typedef void (*OPSBSUndimScreenFunc)(void); + typedef mach_port_t (*OPSpringBoardServerPortFunc)(void); + typedef void (*OPGSSendSimpleEventFunc)(int type, mach_port_t port); + static BOOL loaded = NO; + static OPSBSUndimScreenFunc undimFunc = NULL; + static OPSpringBoardServerPortFunc springBoardPortFunc = NULL; + static OPGSSendSimpleEventFunc sendSimpleEventFunc = NULL; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + NSArray *springBoardFrameworks = @[ + @"/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices", + @"/var/jb/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices" + ]; + for (NSString *framework in springBoardFrameworks) { + void *handle = dlopen(framework.UTF8String, RTLD_LAZY); + if (!handle) { + continue; + } + loaded = YES; + undimFunc = (OPSBSUndimScreenFunc)dlsym(handle, "SBSUndimScreen"); + springBoardPortFunc = (OPSpringBoardServerPortFunc)dlsym(handle, "SBSSpringBoardServerPort"); + if (undimFunc || springBoardPortFunc) { + break; + } + } + + NSArray *graphicsFrameworks = @[ + @"/System/Library/PrivateFrameworks/GraphicsServices.framework/GraphicsServices", + @"/var/jb/System/Library/PrivateFrameworks/GraphicsServices.framework/GraphicsServices" + ]; + for (NSString *framework in graphicsFrameworks) { + void *handle = dlopen(framework.UTF8String, RTLD_LAZY); + if (!handle) { + continue; + } + loaded = YES; + sendSimpleEventFunc = (OPGSSendSimpleEventFunc)dlsym(handle, "GSSendSimpleEvent"); + if (sendSimpleEventFunc) { + break; + } + } + }); + return (OPInputProviderStatus){ + .loaded = loaded, + .hasWake = undimFunc != NULL, + .hasHome = springBoardPortFunc != NULL && sendSimpleEventFunc != NULL, + .hasCoordinateInput = [OPHIDInputProviderInfo()[@"available"] boolValue], + .hasTextInput = [OPHIDInputProviderInfo()[@"available"] boolValue] && + [OPHIDInputProviderInfo()[@"keyboard_available"] boolValue] + }; +} + +static NSDictionary *OPWakeScreen(void) { + typedef void (*OPSBSUndimScreenFunc)(void); + static OPSBSUndimScreenFunc undimFunc = NULL; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + NSArray *frameworks = @[ + @"/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices", + @"/var/jb/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices" + ]; + for (NSString *framework in frameworks) { + void *handle = dlopen(framework.UTF8String, RTLD_LAZY); + if (!handle) { + continue; + } + undimFunc = (OPSBSUndimScreenFunc)dlsym(handle, "SBSUndimScreen"); + if (undimFunc) { + break; + } + } + }); + if (!undimFunc) { + return @{@"ok": @NO, @"reason": @"SBSUndimScreen_missing"}; + } + undimFunc(); + return @{@"ok": @YES, @"provider": @"SpringBoardServices.SBSUndimScreen"}; +} + +static NSDictionary *OPPressHome(void) { + typedef mach_port_t (*OPSpringBoardServerPortFunc)(void); + typedef void (*OPGSSendSimpleEventFunc)(int type, mach_port_t port); + static OPSpringBoardServerPortFunc springBoardPortFunc = NULL; + static OPGSSendSimpleEventFunc sendSimpleEventFunc = NULL; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + NSArray *springBoardFrameworks = @[ + @"/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices", + @"/var/jb/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices" + ]; + for (NSString *framework in springBoardFrameworks) { + void *handle = dlopen(framework.UTF8String, RTLD_LAZY); + if (!handle) { + continue; + } + springBoardPortFunc = (OPSpringBoardServerPortFunc)dlsym(handle, "SBSSpringBoardServerPort"); + if (springBoardPortFunc) { + break; + } + } + + NSArray *graphicsFrameworks = @[ + @"/System/Library/PrivateFrameworks/GraphicsServices.framework/GraphicsServices", + @"/var/jb/System/Library/PrivateFrameworks/GraphicsServices.framework/GraphicsServices" + ]; + for (NSString *framework in graphicsFrameworks) { + void *handle = dlopen(framework.UTF8String, RTLD_LAZY); + if (!handle) { + continue; + } + sendSimpleEventFunc = (OPGSSendSimpleEventFunc)dlsym(handle, "GSSendSimpleEvent"); + if (sendSimpleEventFunc) { + break; + } + } + }); + if (!springBoardPortFunc) { + return @{@"ok": @NO, @"reason": @"SBSSpringBoardServerPort_missing"}; + } + if (!sendSimpleEventFunc) { + return @{@"ok": @NO, @"reason": @"GSSendSimpleEvent_missing"}; + } + mach_port_t port = springBoardPortFunc(); + if (port == MACH_PORT_NULL) { + return @{@"ok": @NO, @"reason": @"springboard_port_missing"}; + } + sendSimpleEventFunc(1000, port); + usleep(80000); + sendSimpleEventFunc(1001, port); + return @{@"ok": @YES, @"provider": @"GraphicsServices.GSSendSimpleEvent", @"port": @(port)}; +} + +static NSDictionary *OPUiOpen(NSArray *uiopenArguments) { + NSString *uiopen = OPExecutablePath(@[ + @"/var/jb/usr/bin/uiopen", + @"/var/jb/bin/uiopen", + @"/usr/bin/uiopen" + ]); + if (!uiopen) { + return @{@"ok": @NO, @"reason": @"uiopen_missing"}; + } + NSMutableArray *arguments = [NSMutableArray arrayWithObject:uiopen]; + [arguments addObjectsFromArray:uiopenArguments]; + return OPSpawn(arguments); +} + +static BOOL OPBoolFromRequest(NSDictionary *request, NSString *key, BOOL defaultValue) { + id value = request[key]; + if ([value isKindOfClass:[NSNumber class]]) { + return [value boolValue]; + } + if ([value isKindOfClass:[NSString class]]) { + NSString *lower = [(NSString *)value lowercaseString]; + if ([lower isEqualToString:@"true"] || [lower isEqualToString:@"yes"] + || [lower isEqualToString:@"1"]) { + return YES; + } + if ([lower isEqualToString:@"false"] || [lower isEqualToString:@"no"] + || [lower isEqualToString:@"0"]) { + return NO; + } + } + return defaultValue; +} + +static NSDictionary *OPVolumeTriggerPreferences(void) { + NSDictionary *preferences = [NSDictionary dictionaryWithContentsOfFile:OPVolumeTriggerPreferencesPath]; + return [preferences isKindOfClass:[NSDictionary class]] ? preferences : @{}; +} + +static NSString *OPVolumeTriggerGoalFromPreferences(NSDictionary *preferences) { + NSString *goal = OPStringFromRequest(preferences ?: @{}, @"TriggerGoal", + OPDefaultHardwareTriggerGoal); + goal = [goal stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (goal.length == 0 || [goal isEqualToString:OPLegacyHardwareTriggerGoal]) { + return OPDefaultHardwareTriggerGoal; + } + return goal; +} + +static long long OPVolumeTriggerWindowMs(NSDictionary *preferences) { + return OPLongLongFromRequest(preferences ?: @{}, @"WindowMs", 1200, 100, 5000); +} + +static long long OPVolumeTriggerCooldownMs(NSDictionary *preferences) { + return OPLongLongFromRequest(preferences ?: @{}, @"CooldownMs", 10000, 0, 60000); +} + +static NSDictionary *OPDefaultModelConfig(void) { + return @{ + @"schema": @"openphone.model_config.v1", + @"enabled": @NO, + @"mode": @"broker", + @"endpoint_url": @"", + @"model": @"", + @"region": @"us-east-1", + @"timeout_ms": @30000, + @"max_steps": @5, + @"max_duration_ms": @120000, + @"credential_required": @YES, + @"credential_source": @"external" + }; +} + +static BOOL OPModelModeIsOpenAIRealtime(NSString *mode) { + return [mode isEqualToString:@"openai_realtime"] || + [mode isEqualToString:@"openai_realtime2"]; +} + +static NSString *OPModelDefaultModelForMode(NSString *mode) { + if ([mode isEqualToString:@"openai_realtime2"]) { + return OPOpenAIRealtime2Model; + } + if ([mode isEqualToString:@"openai_realtime"]) { + return OPOpenAIRealtimeModel; + } + return @""; +} + +static NSString *OPModelEffectiveModel(NSDictionary *config) { + NSString *model = [config[@"model"] isKindOfClass:[NSString class]] + ? config[@"model"] : @""; + if (model.length > 0) { + return model; + } + NSString *mode = [config[@"mode"] isKindOfClass:[NSString class]] + ? config[@"mode"] : @"broker"; + return OPModelDefaultModelForMode(mode); +} + +static BOOL OPModelModeHasDefaultEndpoint(NSString *mode) { + return [mode isEqualToString:@"bedrock_converse"] || + OPModelModeIsOpenAIRealtime(mode); +} + +static BOOL OPModelModeIsValid(NSString *mode) { + return [mode isEqualToString:@"broker"] || + [mode isEqualToString:@"direct_dev"] || + [mode isEqualToString:@"bedrock_converse"] || + OPModelModeIsOpenAIRealtime(mode); +} + +static NSDictionary *OPModelConfig(void) { + NSMutableDictionary *config = [OPDefaultModelConfig() mutableCopy]; + NSDictionary *stored = OPReadJSONFile(OPModelConfigPath()); + if ([stored isKindOfClass:[NSDictionary class]]) { + for (NSString *key in stored) { + if (!OPSensitiveKey(key) && stored[key]) { + config[key] = stored[key]; + } + } + } + NSString *mode = [config[@"mode"] isKindOfClass:[NSString class]] + ? [config[@"mode"] lowercaseString] : @"broker"; + if (!OPModelModeIsValid(mode)) { + mode = @"broker"; + } + config[@"mode"] = mode; + NSString *region = [config[@"region"] isKindOfClass:[NSString class]] + ? config[@"region"] : @"us-east-1"; + config[@"region"] = region.length > 0 ? region : @"us-east-1"; + long long timeoutMs = OPLongLongFromRequest(config, @"timeout_ms", 30000, 1000, 120000); + long long maxSteps = OPLongLongFromRequest(config, @"max_steps", 5, 1, 120); + long long maxDurationMs = OPLongLongFromRequest(config, @"max_duration_ms", 120000, 1000, 3300000); + config[@"timeout_ms"] = @(timeoutMs); + config[@"max_steps"] = @(maxSteps); + config[@"max_duration_ms"] = @(maxDurationMs); + config[@"model"] = OPModelEffectiveModel(config) ?: @""; + config[@"enabled"] = @(OPBoolFromRequest(config, @"enabled", NO)); + BOOL credentialRequired = OPBoolFromRequest(config, @"credential_required", YES); + if (OPModelModeIsOpenAIRealtime(mode)) { + credentialRequired = YES; + } + config[@"credential_required"] = @(credentialRequired); + return config; +} + +static NSDictionary *OPDefaultAgentControlConfig(void) { + return @{ + @"schema": @"openphone.agent_control.v1", + @"autonomy_mode": @"yolo", + @"yolo_enabled": @YES, + @"hardware_triggers_enabled": @YES, + @"paused": @NO, + @"pause_reason": @"", + @"updated_at_ms": @0, + @"updated_by": @"default" + }; +} + +static NSDictionary *OPAgentControlConfig(void) { + NSMutableDictionary *config = [OPDefaultAgentControlConfig() mutableCopy]; + NSDictionary *stored = OPReadJSONFile(OPAgentControlPath()); + if ([stored isKindOfClass:[NSDictionary class]]) { + for (NSString *key in stored) { + if (!OPSensitiveKey(key) && stored[key]) { + config[key] = stored[key]; + } + } + } + NSString *mode = [config[@"autonomy_mode"] isKindOfClass:[NSString class]] + ? [config[@"autonomy_mode"] lowercaseString] : @"yolo"; + if (![mode isEqualToString:@"yolo"] && ![mode isEqualToString:@"reviewed"] && + ![mode isEqualToString:@"dry_run"]) { + mode = @"yolo"; + } + config[@"schema"] = @"openphone.agent_control.v1"; + config[@"autonomy_mode"] = mode; + config[@"yolo_enabled"] = @(OPBoolFromRequest(config, @"yolo_enabled", YES)); + config[@"hardware_triggers_enabled"] = @(OPBoolFromRequest(config, @"hardware_triggers_enabled", YES)); + config[@"paused"] = @(OPBoolFromRequest(config, @"paused", NO)); + if (![config[@"pause_reason"] isKindOfClass:[NSString class]]) { + config[@"pause_reason"] = @""; + } + if (![config[@"updated_by"] isKindOfClass:[NSString class]]) { + config[@"updated_by"] = @"unknown"; + } + long long updatedAt = [config[@"updated_at_ms"] respondsToSelector:@selector(longLongValue)] + ? [config[@"updated_at_ms"] longLongValue] : 0; + config[@"updated_at_ms"] = @(MAX(0, updatedAt)); + return config; +} + +static NSDictionary *OPAgentControlSummary(void) { + NSDictionary *config = OPAgentControlConfig(); + BOOL paused = [config[@"paused"] boolValue]; + BOOL yoloEnabled = [config[@"yolo_enabled"] boolValue]; + BOOL hardwareTriggersEnabled = [config[@"hardware_triggers_enabled"] boolValue]; + NSString *state = paused ? @"paused" + : ((yoloEnabled && hardwareTriggersEnabled) ? @"running" : @"limited"); + NSString *triggerPolicy = paused ? @"paused" + : (!hardwareTriggersEnabled ? @"disabled" + : (!yoloEnabled ? @"manual_only" : @"allow_yolo")); + return @{ + @"schema": @"openphone.agent_control.v1", + @"state": state, + @"autonomy_mode": config[@"autonomy_mode"] ?: @"yolo", + @"yolo_enabled": @(yoloEnabled), + @"hardware_triggers_enabled": @(hardwareTriggersEnabled), + @"paused": @(paused), + @"pause_reason": config[@"pause_reason"] ?: @"", + @"trigger_policy": triggerPolicy, + @"updated_at_ms": config[@"updated_at_ms"] ?: @0, + @"updated_by": config[@"updated_by"] ?: @"unknown", + @"config_path": OPAgentControlPath(), + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPModelCredentialStatus(void) { + const char *envToken = getenv("OPENPHONE_MODEL_BEARER_TOKEN"); + BOOL envPresent = envToken && envToken[0] != '\0'; + BOOL credentialFilePresent = [[NSFileManager defaultManager] fileExistsAtPath:OPModelCredentialPath()]; + return @{ + @"status": (envPresent || credentialFilePresent) ? @"present" : @"missing", + @"source": envPresent ? @"env" : (credentialFilePresent ? @"credential_file" : @"none"), + @"env_present": @(envPresent), + @"credential_file": OPModelCredentialPath(), + @"credential_file_present": @(credentialFilePresent) + }; +} + +static NSString *OPModelCredentialValue(void) { + const char *envToken = getenv("OPENPHONE_MODEL_BEARER_TOKEN"); + if (envToken && envToken[0] != '\0') { + return [NSString stringWithUTF8String:envToken] ?: @""; + } + NSData *data = [NSData dataWithContentsOfFile:OPModelCredentialPath()]; + if (!data) { + return @""; + } + NSString *text = [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @""; + text = [text stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (text.length == 0) { + return @""; + } + NSData *jsonData = [text dataUsingEncoding:NSUTF8StringEncoding]; + id parsed = jsonData ? [NSJSONSerialization JSONObjectWithData:jsonData options:0 error:nil] : nil; + if ([parsed isKindOfClass:[NSDictionary class]]) { + NSDictionary *object = parsed; + for (NSString *key in @[@"credential", @"bearer", @"value"]) { + if ([object[key] isKindOfClass:[NSString class]] && [object[key] length] > 0) { + return object[key]; + } + } + return @""; + } + return text; +} + +static NSDictionary *OPModelStatusDictionary(void) { + NSDictionary *config = OPModelConfig(); + NSString *endpoint = [config[@"endpoint_url"] isKindOfClass:[NSString class]] + ? config[@"endpoint_url"] : @""; + NSString *model = [config[@"model"] isKindOfClass:[NSString class]] + ? config[@"model"] : @""; + NSString *mode = [config[@"mode"] isKindOfClass:[NSString class]] + ? config[@"mode"] : @"broker"; + NSDictionary *credential = OPModelCredentialStatus(); + BOOL enabled = [config[@"enabled"] boolValue]; + BOOL credentialRequired = [config[@"credential_required"] boolValue]; + BOOL endpointConfigured = OPModelModeHasDefaultEndpoint(mode) ? YES : endpoint.length > 0; + BOOL configured = OPModelModeHasDefaultEndpoint(mode) + ? model.length > 0 : (endpoint.length > 0 && model.length > 0); + BOOL credentialReady = ![credential[@"status"] isEqualToString:@"missing"]; + BOOL ready = enabled && configured && (!credentialRequired || credentialReady); + return @{ + @"status": ready ? @"ready" : (enabled ? @"configured_incomplete" : @"disabled"), + @"schema": @"openphone.model_status.v1", + @"enabled": @(enabled), + @"mode": mode ?: @"broker", + @"endpoint_configured": @(endpointConfigured), + @"endpoint_url_configured": @(endpoint.length > 0), + @"model_configured": @(model.length > 0), + @"model": model ?: @"", + @"region": config[@"region"] ?: @"us-east-1", + @"timeout_ms": config[@"timeout_ms"] ?: @30000, + @"max_steps": config[@"max_steps"] ?: @5, + @"max_duration_ms": config[@"max_duration_ms"] ?: @120000, + @"screenshot_max_dimension_px": config[@"screenshot_max_dimension_px"] ?: @1024, + @"screenshot_jpeg_quality_x100": config[@"screenshot_jpeg_quality_x100"] ?: @60, + @"credential_required": @(credentialRequired), + @"credential": credential, + @"config_path": OPModelConfigPath(), + @"runtime_authority": @"openphone-agentd", + @"notes": @"Broker/direct/realtime credentials are external only; credential values are never included in status, audit, or trajectory.", + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPModelStatus(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSDictionary *status = OPModelStatusDictionary(); + OPRecordAudit(@"model_status_read", taskId, @"tasks.observe", @"allow_task_scoped", + @{}, status[@"status"] ?: @"unknown"); + return status; +} + +static NSDictionary *OPModelConfigure(NSDictionary *request) { + for (NSString *key in request) { + if (OPSensitiveKey(key)) { + return OPError(@"inline_model_credential_rejected"); + } + } + NSString *mode = [OPStringFromRequest(request, @"mode", @"broker") lowercaseString]; + if (!OPModelModeIsValid(mode)) { + return OPError(@"invalid_model_mode"); + } + NSString *endpoint = OPStringFromRequest(request, @"endpoint_url", @""); + NSString *model = OPStringFromRequest(request, @"model", @""); + NSString *region = OPStringFromRequest(request, @"region", @"us-east-1"); + long long timeoutMs = OPLongLongFromRequest(request, @"timeout_ms", 30000, 1000, 120000); + if (model.length == 0) { + model = OPModelDefaultModelForMode(mode); + } + long long maxSteps = OPLongLongFromRequest(request, @"max_steps", 5, 1, 120); + long long maxDurationMs = OPLongLongFromRequest(request, @"max_duration_ms", 120000, 1000, 3300000); + BOOL credentialRequired = OPBoolFromRequest(request, @"credential_required", YES); + if (OPModelModeIsOpenAIRealtime(mode)) { + credentialRequired = YES; + } + // Preserve screenshot tuning across model config writes: the screenshot + // path reads these keys from model config, so a config write that dropped + // them would silently reset the values. + NSDictionary *existingModelConfig = OPModelConfig(); + long long screenshotMaxDim = OPLongLongFromRequest(request, @"screenshot_max_dimension_px", + OPLongLongFromRequest(existingModelConfig, @"screenshot_max_dimension_px", 1024, 0, 4096), + 0, 4096); + long long screenshotQuality = OPLongLongFromRequest(request, @"screenshot_jpeg_quality_x100", + OPLongLongFromRequest(existingModelConfig, @"screenshot_jpeg_quality_x100", 60, 1, 100), + 1, 100); + BOOL defaultEnabled = OPModelModeHasDefaultEndpoint(mode) + ? model.length > 0 : (endpoint.length > 0 && model.length > 0); + NSDictionary *config = @{ + @"schema": @"openphone.model_config.v1", + @"enabled": @(OPBoolFromRequest(request, @"enabled", defaultEnabled)), + @"mode": mode, + @"endpoint_url": endpoint ?: @"", + @"model": model ?: @"", + @"region": region.length > 0 ? region : @"us-east-1", + @"timeout_ms": @(timeoutMs), + @"max_steps": @(maxSteps), + @"max_duration_ms": @(maxDurationMs), + @"screenshot_max_dimension_px": @(screenshotMaxDim), + @"screenshot_jpeg_quality_x100": @(screenshotQuality), + @"credential_required": @(credentialRequired), + @"credential_source": @"external", + @"updated_at_ms": @(OPNowMs()) + }; + BOOL ok = OPWriteProtectedJSONFile(OPModelConfigPath(), config); + if (!ok) { + return OPError(@"model_config_write_failed"); + } + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + OPRecordAudit(@"model_configured", taskId, @"tasks.observe", @"allow_yolo", + OPRedactedObject(request ?: @{}, 0), [NSString stringWithFormat:@"mode:%@", mode]); + return OPModelStatusDictionary(); +} + +static NSDictionary *OPJSONDictionaryFromString(NSString *string) { + if (string.length == 0) { + return nil; + } + NSData *data = [string dataUsingEncoding:NSUTF8StringEncoding]; + if (!data) { + return nil; + } + id object = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + return [object isKindOfClass:[NSDictionary class]] ? object : nil; +} + +static NSString *OPScreenshotHelperPath(void) { + return OPExecutablePath(@[ + @"/var/jb/usr/local/bin/openphone-screencap-helper", + @"/usr/local/bin/openphone-screencap-helper" + ]); +} + +static NSDictionary *OPScreenshotHelperStatus(void) { + NSString *helper = OPScreenshotHelperPath(); + return @{ + @"status": helper ? @"implemented_experimental" : @"not_installed", + @"provider": @"openphone-screencap-helper", + @"path": helper ?: @"", + @"fallback_provider": @"OpenPhoneVolumeTrigger.SpringBoardScreenshot", + @"fallback_request_path": OPSpringBoardScreenshotRequestPath(), + @"fallback_response_path": OPSpringBoardScreenshotResponsePath(), + @"bytes_returned": @"never", + @"storage": OPScreenshotsPath() + }; +} + +static NSDictionary *OPScreenshotProviderSummary(NSDictionary *info) { + NSMutableDictionary *summary = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"provider", @"status", @"reason", @"path", @"helper"]) { + id value = info[key]; + if (value) { + summary[key] = value; + } + } + id kernReturn = info[@"kern_return"]; + if (kernReturn) { + summary[@"kern_return"] = kernReturn; + } + return summary.count > 0 ? summary : @{}; +} + +static NSDictionary *OPSpringBoardScreenshotBridgeInfo(void) { + NSDictionary *state = OPSpringBoardPublishedState(); + if (![state[@"status"] isEqualToString:@"ok"]) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardScreenshot", + @"reason": state[@"reason"] ?: @"springboard_state_unavailable", + @"springboard_state_status": state[@"status"] ?: @"unknown" + }; + } + NSDictionary *bridge = [state[@"screenshot_bridge"] isKindOfClass:[NSDictionary class]] + ? state[@"screenshot_bridge"] : @{}; + if (![bridge[@"status"] isEqualToString:@"ready"]) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardScreenshot", + @"reason": @"springboard_screenshot_bridge_not_ready", + @"springboard_state_status": state[@"status"] ?: @"unknown", + @"bridge_status": bridge[@"status"] ?: @"missing" + }; + } + NSMutableDictionary *result = [bridge mutableCopy]; + result[@"provider"] = result[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardScreenshot"; + return result; +} + +static NSDictionary *OPSpringBoardScreenshotInfo(NSDictionary *request, + NSString *path, + NSDictionary *fallbackFrom) { + NSDictionary *bridge = OPSpringBoardScreenshotBridgeInfo(); + if (![bridge[@"status"] isEqualToString:@"ready"]) { + NSMutableDictionary *unavailable = [@{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardScreenshot", + @"reason": bridge[@"reason"] ?: @"springboard_screenshot_bridge_unavailable", + @"path": path ?: @"", + @"bytes_returned": @"never", + @"storage": OPScreenshotsPath(), + @"fallback_from": OPScreenshotProviderSummary(fallbackFrom ?: @{}) + } mutableCopy]; + unavailable[@"bridge"] = bridge ?: @{}; + return unavailable; + } + + NSString *requestId = [NSString stringWithFormat:@"screenshot-%lld-%d", OPNowMs(), getpid()]; + NSString *requestPath = OPSpringBoardScreenshotRequestPath(); + NSString *responsePath = OPSpringBoardScreenshotResponsePath(); + long long timeoutMs = OPLongLongFromRequest(request, @"screenshot_timeout_ms", 1800, 250, 5000); + NSDictionary *modelConfig = OPModelConfig(); + long long maxDimension = OPLongLongFromRequest(request, @"screenshot_max_dimension_px", + OPLongLongFromRequest(modelConfig, @"screenshot_max_dimension_px", 1024, 0, 4096), + 0, 4096); + long long jpegQualityX100 = OPLongLongFromRequest(request, @"screenshot_jpeg_quality_x100", + OPLongLongFromRequest(modelConfig, @"screenshot_jpeg_quality_x100", 60, 10, 100), + 10, 100); + NSDictionary *payload = @{ + @"schema": @"openphone.springboard_screenshot_request.v1", + @"request_id": requestId, + @"timestamp_ms": @(OPNowMs()), + @"provider": @"openphone.agentd", + @"path": path ?: @"", + @"timeout_ms": @(timeoutMs), + @"max_dimension_px": @(maxDimension), + @"jpeg_quality_x100": @(jpegQualityX100), + @"source": @"openphone.agentd.screen.screenshot" + }; + if (!OPWriteJSONFile(requestPath, payload)) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardScreenshot", + @"reason": @"request_write_failed", + @"path": path ?: @"", + @"request_path": requestPath, + @"bytes_returned": @"never", + @"storage": OPScreenshotsPath(), + @"fallback_from": OPScreenshotProviderSummary(fallbackFrom ?: @{}) + }; + } + chmod(requestPath.UTF8String, 0600); + + long long start = OPNowMs(); + while (OPNowMs() - start <= timeoutMs) { + NSDictionary *response = OPReadJSONFile(responsePath); + NSString *responseRequestId = [response[@"request_id"] isKindOfClass:[NSString class]] + ? response[@"request_id"] : @""; + if ([responseRequestId isEqualToString:requestId]) { + NSMutableDictionary *result = [response mutableCopy]; + result[@"provider"] = result[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardScreenshot"; + result[@"path"] = [result[@"path"] isKindOfClass:[NSString class]] ? result[@"path"] : path ?: @""; + result[@"request_path"] = requestPath; + result[@"response_path"] = responsePath; + result[@"bytes_returned"] = @"never"; + result[@"storage"] = OPScreenshotsPath(); + result[@"fallback_from"] = OPScreenshotProviderSummary(fallbackFrom ?: @{}); + if ([result[@"status"] isEqualToString:@"ok"]) { + NSString *resultPath = [result[@"path"] isKindOfClass:[NSString class]] + ? result[@"path"] : @""; + NSData *png = [NSData dataWithContentsOfFile:resultPath]; + if (png.length == 0) { + result[@"status"] = @"unavailable"; + result[@"reason"] = @"springboard_screenshot_file_missing"; + } else { + if (!result[@"bytes"]) { + result[@"bytes"] = @(png.length); + } + if (!result[@"sha256"]) { + result[@"sha256"] = OPSHA256Hex(png); + } + } + } + return result; + } + usleep(100000); + } + + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardScreenshot", + @"reason": @"response_timeout", + @"request_id": requestId, + @"path": path ?: @"", + @"request_path": requestPath, + @"response_path": responsePath, + @"timeout_ms": @(timeoutMs), + @"bytes_returned": @"never", + @"storage": OPScreenshotsPath(), + @"fallback_from": OPScreenshotProviderSummary(fallbackFrom ?: @{}) + }; +} + +// Trim the screenshots directory when it grows too large. Kept small so that +// disk-write pressure (jetsam bug_type 145) and simple disk usage stay in check. +static void OPPruneScreenshotsDir(NSUInteger keepCount) { + NSFileManager *fm = [NSFileManager defaultManager]; + NSString *dir = OPScreenshotsPath(); + NSError *err = nil; + NSArray *files = [fm contentsOfDirectoryAtPath:dir error:&err]; + if (!files || files.count <= keepCount) { + return; + } + NSMutableArray *entries = [NSMutableArray array]; + for (NSString *name in files) { + NSString *lower = [name lowercaseString]; + if (![lower hasSuffix:@".png"] && ![lower hasSuffix:@".jpg"] + && ![lower hasSuffix:@".jpeg"]) { + continue; + } + NSString *path = [dir stringByAppendingPathComponent:name]; + NSDictionary *attrs = [fm attributesOfItemAtPath:path error:nil]; + if (!attrs) { + continue; + } + NSDate *modified = attrs.fileModificationDate ?: attrs.fileCreationDate ?: [NSDate distantPast]; + [entries addObject:@{@"path": path, @"date": modified}]; + } + if (entries.count <= keepCount) { + return; + } + [entries sortUsingComparator:^NSComparisonResult(NSDictionary *a, NSDictionary *b) { + return [b[@"date"] compare:a[@"date"]]; + }]; + for (NSUInteger i = keepCount; i < entries.count; i++) { + [fm removeItemAtPath:entries[i][@"path"] error:nil]; + } +} + +static NSDictionary *OPScreenScreenshotInfo(NSDictionary *request) { + if (!OPBoolFromRequest(request, @"include_screenshot", NO)) { + return @{ + @"status": @"not_requested", + @"provider": @"openphone-screencap-helper" + }; + } + + OPEnsureDirectories(); + OPPruneScreenshotsDir(60); + // Compress screenshots to cut memory/disk pressure ~4x. Default: JPEG q=0.6, + // longest edge <= 1024px. Config keys screenshot_max_dimension_px and + // screenshot_jpeg_quality (0 dimension disables scaling); per-request + // overrides accepted via the same keys. + NSDictionary *modelConfig = OPModelConfig(); + long long maxDimension = OPLongLongFromRequest(request, @"screenshot_max_dimension_px", + OPLongLongFromRequest(modelConfig, @"screenshot_max_dimension_px", 1024, 0, 4096), + 0, 4096); + double jpegQuality = (double)OPLongLongFromRequest(request, @"screenshot_jpeg_quality_x100", + OPLongLongFromRequest(modelConfig, @"screenshot_jpeg_quality_x100", 60, 10, 100), + 10, 100) / 100.0; + NSString *filename = [NSString stringWithFormat:@"screen-%lld-%d.jpg", OPNowMs(), getpid()]; + NSString *path = [OPScreenshotsPath() stringByAppendingPathComponent:filename]; + NSString *helper = OPScreenshotHelperPath(); + NSDictionary *helperResult = helper ? nil : @{ + @"status": @"unavailable", + @"provider": @"openphone-screencap-helper", + @"reason": @"helper_not_installed", + @"path": path + }; + + if (helper) { + // Ask SpringBoard to hide our island overlay briefly so it doesn't + // appear in the captured screenshot (which would confuse the model + // into thinking the app UI is obscured). Give it ~150ms to fade. + notify_post("com.openphone.island.hide-for-capture"); + usleep(150000); + NSString *output = OPSpawnCapture(@[helper, path, + [NSString stringWithFormat:@"%lld", maxDimension], + [NSString stringWithFormat:@"%.2f", jpegQuality]], 32 * 1024); + notify_post("com.openphone.island.show-after-capture"); + if (output.length == 0) { + helperResult = @{ + @"status": @"unavailable", + @"provider": @"openphone-screencap-helper", + @"reason": @"helper_failed_or_crashed", + @"path": path + }; + } else { + NSDictionary *parsed = OPJSONDictionaryFromString(output); + if (!parsed) { + helperResult = @{ + @"status": @"unavailable", + @"provider": @"openphone-screencap-helper", + @"reason": @"helper_json_invalid", + @"path": path, + @"raw_length": @(output.length) + }; + } else { + NSMutableDictionary *result = [parsed mutableCopy]; + if (![result[@"path"] isKindOfClass:[NSString class]]) { + result[@"path"] = path; + } + result[@"helper"] = helper; + result[@"bytes_returned"] = @"never"; + if ([result[@"status"] isEqualToString:@"ok"]) { + return result; + } + helperResult = result; + } + } + } + + if (!helperResult) { + helperResult = @{ + @"status": @"unavailable", + @"provider": @"openphone-screencap-helper", + @"reason": @"helper_unavailable", + @"path": path + }; + } + return OPSpringBoardScreenshotInfo(request, path, helperResult); +} + +static NSDictionary *OPSpringBoardInputBridgeInfo(void) { + NSDictionary *state = OPSpringBoardPublishedState(); + if (![state[@"status"] isEqualToString:@"ok"]) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardInput", + @"reason": state[@"reason"] ?: @"springboard_state_unavailable", + @"springboard_state_status": state[@"status"] ?: @"unknown" + }; + } + NSDictionary *bridge = [state[@"input_bridge"] isKindOfClass:[NSDictionary class]] + ? state[@"input_bridge"] : @{}; + if (![bridge[@"status"] isEqualToString:@"ready"]) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardInput", + @"reason": @"springboard_input_bridge_not_ready", + @"springboard_state_status": state[@"status"] ?: @"unknown", + @"bridge_status": bridge[@"status"] ?: @"missing" + }; + } + NSMutableDictionary *result = [bridge mutableCopy]; + result[@"provider"] = result[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardInput"; + return result; +} + +static NSDictionary *OPSpringBoardInputInfo(NSDictionary *action) { + NSDictionary *bridge = OPSpringBoardInputBridgeInfo(); + NSString *actionType = [action[@"type"] isKindOfClass:[NSString class]] + ? action[@"type"] : @""; + if (![bridge[@"status"] isEqualToString:@"ready"]) { + NSMutableDictionary *unavailable = [@{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardInput", + @"reason": bridge[@"reason"] ?: @"springboard_input_bridge_unavailable", + @"action_type": actionType, + @"request_path": OPSpringBoardInputRequestPath(), + @"response_path": OPSpringBoardInputResponsePath() + } mutableCopy]; + unavailable[@"bridge"] = bridge ?: @{}; + return unavailable; + } + + NSString *requestId = [NSString stringWithFormat:@"input-%lld-%d", OPNowMs(), getpid()]; + NSString *requestPath = OPSpringBoardInputRequestPath(); + NSString *responsePath = OPSpringBoardInputResponsePath(); + long long timeoutMs = OPLongLongFromRequest(action, @"input_timeout_ms", 1200, 250, 5000); + NSDictionary *payload = @{ + @"schema": @"openphone.springboard_input_request.v1", + @"request_id": requestId, + @"timestamp_ms": @(OPNowMs()), + @"provider": @"openphone.agentd", + @"action": action ?: @{}, + @"timeout_ms": @(timeoutMs), + @"source": @"openphone.agentd.input.perform" + }; + if (!OPWriteJSONFile(requestPath, payload)) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardInput", + @"reason": @"request_write_failed", + @"request_path": requestPath, + @"response_path": responsePath, + @"action_type": actionType + }; + } + chmod(requestPath.UTF8String, 0644); + + long long start = OPNowMs(); + while (OPNowMs() - start <= timeoutMs) { + NSDictionary *response = OPReadJSONFile(responsePath); + NSString *responseRequestId = [response[@"request_id"] isKindOfClass:[NSString class]] + ? response[@"request_id"] : @""; + if ([responseRequestId isEqualToString:requestId]) { + NSMutableDictionary *result = [response mutableCopy]; + result[@"provider"] = result[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardInput"; + result[@"request_path"] = requestPath; + result[@"response_path"] = responsePath; + result[@"bridge"] = bridge ?: @{}; + [[NSFileManager defaultManager] removeItemAtPath:requestPath error:nil]; + return result; + } + usleep(100000); + } + + [[NSFileManager defaultManager] removeItemAtPath:requestPath error:nil]; + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardInput", + @"reason": @"response_timeout", + @"request_id": requestId, + @"action_type": actionType, + @"request_path": requestPath, + @"response_path": responsePath, + @"timeout_ms": @(timeoutMs), + @"bridge": bridge ?: @{} + }; +} + +static long long OPSQLiteCount(sqlite3 *db, NSString *table) { + NSString *sql = [NSString stringWithFormat:@"SELECT count(*) FROM %@", table]; + sqlite3_stmt *statement = NULL; + long long count = 0; + if (sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL) == SQLITE_OK) { + if (sqlite3_step(statement) == SQLITE_ROW) { + count = sqlite3_column_int64(statement, 0); + } + } + sqlite3_finalize(statement); + return count; +} + +static long long OPAgentJobCountForStatus(sqlite3 *db, NSString *status) { + sqlite3_stmt *statement = NULL; + long long count = 0; + if (sqlite3_prepare_v2(db, + "SELECT count(*) FROM agent_job WHERE status = ?", + -1, &statement, NULL) == SQLITE_OK) { + OPSQLiteBindText(statement, 1, status ?: @""); + if (sqlite3_step(statement) == SQLITE_ROW) { + count = sqlite3_column_int64(statement, 0); + } + } + sqlite3_finalize(statement); + return count; +} + +static long long OPAgentJobDueCount(sqlite3 *db, long long nowMs) { + sqlite3_stmt *statement = NULL; + long long count = 0; + if (sqlite3_prepare_v2(db, + "SELECT count(*) FROM agent_job " + "WHERE scheduler_enabled = 1 AND status = 'queued' " + "AND (next_run_at_ms = 0 OR next_run_at_ms <= ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, nowMs); + if (sqlite3_step(statement) == SQLITE_ROW) { + count = sqlite3_column_int64(statement, 0); + } + } + sqlite3_finalize(statement); + return count; +} + +static long long OPWatcherDueCount(sqlite3 *db, long long nowMs) { + sqlite3_stmt *statement = NULL; + long long count = 0; + if (sqlite3_prepare_v2(db, + "SELECT count(*) FROM watcher " + "WHERE status = 'active' AND next_run_at_ms > 0 AND next_run_at_ms <= ? " + "AND (lower(source) IN ('time', 'timer', 'deadline') " + "OR lower(type) IN ('time', 'timer', 'deadline'))", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, nowMs); + if (sqlite3_step(statement) == SQLITE_ROW) { + count = sqlite3_column_int64(statement, 0); + } + } + sqlite3_finalize(statement); + return count; +} + +static long long OPCommitmentDueCount(sqlite3 *db, long long nowMs) { + sqlite3_stmt *statement = NULL; + long long count = 0; + if (sqlite3_prepare_v2(db, + "SELECT count(*) FROM commitment " + "WHERE status = 'active' AND due_at_ms > 0 AND due_at_ms <= ? " + "AND (expires_at_ms = 0 OR expires_at_ms >= ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, nowMs); + sqlite3_bind_int64(statement, 2, nowMs); + if (sqlite3_step(statement) == SQLITE_ROW) { + count = sqlite3_column_int64(statement, 0); + } + } + sqlite3_finalize(statement); + return count; +} + +static long long OPWatcherStaleRunningCount(sqlite3 *db, long long nowMs, long long staleAfterMs) { + sqlite3_stmt *statement = NULL; + long long count = 0; + long long cutoff = nowMs - staleAfterMs; + if (sqlite3_prepare_v2(db, + "SELECT count(*) FROM watcher " + "WHERE status = 'running' AND updated_at_ms <= ? " + "AND (lower(source) IN ('time', 'timer', 'deadline') " + "OR lower(type) IN ('time', 'timer', 'deadline'))", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, cutoff); + if (sqlite3_step(statement) == SQLITE_ROW) { + count = sqlite3_column_int64(statement, 0); + } + } + sqlite3_finalize(statement); + return count; +} + +static long long OPAgentJobSchedulerEnabledCount(sqlite3 *db) { + sqlite3_stmt *statement = NULL; + long long count = 0; + if (sqlite3_prepare_v2(db, + "SELECT count(*) FROM agent_job WHERE scheduler_enabled = 1", + -1, &statement, NULL) == SQLITE_OK) { + if (sqlite3_step(statement) == SQLITE_ROW) { + count = sqlite3_column_int64(statement, 0); + } + } + sqlite3_finalize(statement); + return count; +} + +static long long OPAgentJobStaleRunningCount(sqlite3 *db, long long nowMs, long long staleAfterMs) { + sqlite3_stmt *statement = NULL; + long long count = 0; + long long cutoff = nowMs - staleAfterMs; + if (sqlite3_prepare_v2(db, + "SELECT count(*) FROM agent_job " + "WHERE scheduler_enabled = 1 AND status = 'running' AND updated_at_ms <= ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, cutoff); + if (sqlite3_step(statement) == SQLITE_ROW) { + count = sqlite3_column_int64(statement, 0); + } + } + sqlite3_finalize(statement); + return count; +} + +static NSDictionary *OPLocalStoresStatus(void) { + sqlite3 *db = NULL; + NSString *error = nil; + NSDictionary *modelStatus = OPModelStatusDictionary(); + if (!OPSQLiteOpen(&db, &error)) { + return @{ + @"status": @"unavailable", + @"provider": @"sqlite", + @"path": OPDatabasePath(), + @"reason": error ?: @"sqlite_open_failed" + }; + } + NSDictionary *result = @{ + @"status": @"ok", + @"provider": @"sqlite", + @"path": OPDatabasePath(), + @"memory": @{ + @"status": @"implemented_partial", + @"rows": @(OPSQLiteCount(db, @"memory")), + @"fts": @(OPSQLiteTableExists(db, @"memory_fts")) + }, + @"context_index": @{ + @"status": @"implemented_partial", + @"rows": @(OPSQLiteCount(db, @"context_event")), + @"fts": @(OPSQLiteTableExists(db, @"context_event_fts")) + }, + @"commitments": @{ + @"status": @"implemented_partial", + @"rows": @(OPSQLiteCount(db, @"commitment")), + @"fts": @(OPSQLiteTableExists(db, @"commitment_fts")), + @"due": @(OPCommitmentDueCount(db, OPNowMs())), + @"scheduler": OPCommitmentSchedulerStatus() + }, + @"watchers": @{ + @"status": @"implemented_partial", + @"rows": @(OPSQLiteCount(db, @"watcher")), + @"fts": @(OPSQLiteTableExists(db, @"watcher_fts")), + @"due": @(OPWatcherDueCount(db, OPNowMs())), + @"stale_running": @(OPWatcherStaleRunningCount(db, OPNowMs(), 300000)), + @"scheduler": OPWatcherSchedulerStatus() + }, + @"background_jobs": @{ + @"status": @"implemented_partial", + @"rows": @(OPSQLiteCount(db, @"agent_job")), + @"fts": @(OPSQLiteTableExists(db, @"agent_job_fts")), + @"queued": @(OPAgentJobCountForStatus(db, @"queued")), + @"running": @(OPAgentJobCountForStatus(db, @"running")), + @"due": @(OPAgentJobDueCount(db, OPNowMs())), + @"stale_running": @(OPAgentJobStaleRunningCount(db, OPNowMs(), 300000)), + @"scheduler_enabled": @(OPAgentJobSchedulerEnabledCount(db)), + @"scheduler": OPBackgroundJobSchedulerStatus(), + @"runner": @"deterministic", + @"model_loop": modelStatus[@"status"] ?: @"disabled" + } + }; + sqlite3_close(db); + return result; +} + +static NSDictionary *OPHealth(NSDate *startedAt) { + NSTimeInterval uptime = [[NSDate date] timeIntervalSinceDate:startedAt]; + struct utsname info; + memset(&info, 0, sizeof(info)); + uname(&info); + + BOOL rootlessPrefixAvailable = [[NSFileManager defaultManager] fileExistsAtPath:@"/var/jb"]; + NSString *foregroundBundleId = OPForegroundBundleIdentifier() ?: @"unknown"; + NSArray *runningApps = OPRunningApplications(); + OPInputProviderStatus inputStatus = OPInputStatus(); + NSDictionary *displayInfo = OPScreenDisplayInfo(); + NSDictionary *lockInfo = OPScreenLockInfo(); + NSDictionary *screenshotStatus = OPScreenshotHelperStatus(); + NSDictionary *recentLayoutInfo = OPSpringBoardRecentLayoutInfo(8); + NSDictionary *springBoardState = OPSpringBoardPublishedState(); + NSDictionary *springBoardUITree = [springBoardState[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? springBoardState[@"ui_tree"] : @{}; + NSDictionary *springBoardInputBridge = [springBoardState[@"input_bridge"] isKindOfClass:[NSDictionary class]] + ? springBoardState[@"input_bridge"] : @{}; + NSDictionary *storeStatus = OPLocalStoresStatus(); + NSDictionary *modelStatus = OPModelStatusDictionary(); + NSDictionary *agentControl = OPAgentControlSummary(); + NSString *publishedForeground = [springBoardState[@"foreground_app"] isKindOfClass:[NSString class]] + ? springBoardState[@"foreground_app"] : @""; + NSString *screenForeground = foregroundBundleId; + if ((screenForeground.length == 0 || [screenForeground isEqualToString:@"unknown"]) && + [springBoardState[@"status"] isEqualToString:@"ok"] && + publishedForeground.length > 0) { + screenForeground = publishedForeground; + } + if (screenForeground.length == 0) { + screenForeground = @"unknown"; + } + NSDictionary *appUIState = OPAppUIPublishedState(screenForeground); + NSDictionary *appUITree = [appUIState[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? appUIState[@"ui_tree"] : @{}; + BOOL hasAppUITree = [appUIState[@"status"] isEqualToString:@"ok"] && + [appUITree[@"status"] isEqualToString:@"ok"]; + NSDictionary *healthUITree = hasAppUITree ? appUITree : springBoardUITree; + NSString *healthUITreeScope = hasAppUITree ? @"app_process" : + (springBoardUITree[@"scope"] ?: @"springboard_only"); + return @{ + @"status": @"ok", + @"service": @"openphone-agentd", + @"version": OPAgentVersion, + @"pid": @(getpid()), + @"uptime_ms": @((long long)(uptime * 1000.0)), + @"autonomy_mode": agentControl[@"autonomy_mode"] ?: @"yolo", + @"agent": agentControl, + @"default_approved_capabilities": OPFullYoloCapabilities(), + @"paths": @{ + @"store": OPStorePath(), + @"socket": OPSocketPath(), + @"log": OPLogPath() + }, + @"device": @{ + @"rootless_prefix_available": @(rootlessPrefixAvailable), + @"rootless_prefix": @"/var/jb", + @"uname": [NSString stringWithFormat:@"%s %s %s %s %s", + info.sysname, info.nodename, info.release, info.version, info.machine] + }, + @"providers": @{ + @"screen": @{ + @"status": [screenshotStatus[@"status"] isEqualToString:@"implemented_experimental"] + ? @"metadata_with_screenshot_helper" : @"metadata_only", + @"foreground_app": screenForeground, + @"foreground_source": [screenForeground isEqualToString:publishedForeground] + ? @"springboard_state" : @"springboardservices", + @"display": displayInfo, + @"lock": lockInfo, + @"screenshot": screenshotStatus, + @"springboard_state": @{ + @"status": springBoardState[@"status"] ?: @"unknown", + @"provider": springBoardState[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardState", + @"age_ms": springBoardState[@"age_ms"] ?: @0, + @"foreground_app": springBoardState[@"foreground_app"] ?: @"", + @"active_scene_count": springBoardState[@"active_scene_count"] ?: @0 + }, + @"app_ui_state": @{ + @"status": appUIState[@"status"] ?: @"unavailable", + @"provider": appUIState[@"provider"] ?: @"OpenPhoneAppIntrospector.UIKitAccessibility", + @"bundle_id": appUIState[@"bundle_id"] ?: screenForeground, + @"age_ms": appUIState[@"age_ms"] ?: @0, + @"application_state_name": appUIState[@"application_state_name"] ?: @"unknown", + @"reason": appUIState[@"reason"] ?: @"" + }, + @"app_ui_intake": OPAppUIIntakeStatus(), + @"recent_layout": @{ + @"status": recentLayoutInfo[@"status"] ?: @"unknown", + @"provider": recentLayoutInfo[@"provider"] ?: @"SpringBoard.RecentAppLayouts", + @"count": recentLayoutInfo[@"count"] ?: @0, + @"first_bundle_id": recentLayoutInfo[@"first_bundle_id"] ?: @"" + }, + @"ui_tree": @{ + @"status": healthUITree[@"status"] ?: @"unavailable", + @"provider": healthUITree[@"provider"] ?: @"SpringBoard.UIKitAccessibility", + @"scope": healthUITreeScope ?: @"springboard_only", + @"window_count": healthUITree[@"window_count"] ?: @0, + @"element_count": healthUITree[@"element_count"] ?: @0, + @"text_count": healthUITree[@"text_count"] ?: @0 + } + }, + @"input": @{ + @"status": inputStatus.hasHome || inputStatus.hasWake + || inputStatus.hasCoordinateInput + ? @"implemented_partial" : @"not_implemented", + @"provider_loaded": @(inputStatus.loaded), + @"coordinate_provider": OPHIDInputProviderInfo(), + @"springboard_bridge": @{ + @"status": springBoardInputBridge[@"status"] ?: @"unavailable", + @"provider": springBoardInputBridge[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardInput", + @"request_path": springBoardInputBridge[@"request_path"] ?: OPSpringBoardInputRequestPath(), + @"response_path": springBoardInputBridge[@"response_path"] ?: OPSpringBoardInputResponsePath(), + @"scope": springBoardInputBridge[@"scope"] ?: @"springboard_windows" + }, + @"home": inputStatus.hasHome ? @"implemented" : @"not_available", + @"wake_and_home": (inputStatus.hasWake && inputStatus.hasHome) + ? @"implemented" : @"not_available", + @"tap": inputStatus.hasCoordinateInput ? @"implemented" : @"not_available", + @"tap_element": inputStatus.hasCoordinateInput + ? @"implemented_partial" : @"not_available", + @"long_press": inputStatus.hasCoordinateInput ? @"implemented" : @"not_available", + @"swipe": inputStatus.hasCoordinateInput ? @"implemented" : @"not_available", + @"type_text": inputStatus.hasTextInput ? @"implemented" : @"not_available" + }, + @"apps": @{ + @"status": @"implemented_partial", + @"foreground_app": foregroundBundleId, + @"running_apps_count": @(runningApps.count), + @"installed_apps": @"implemented", + @"open_app": @"implemented", + @"open_url": @"implemented" + }, + @"notifications": OPNotificationProviderStatus(), + @"messages": OPMessagesProviderStatus(), + @"calls": OPCallsProviderStatus(), + @"triggers": @{ + @"volume_combo": OPVolumeTriggerStatus() + }, + @"calendar": OPCalendarProviderStatus(), + @"contacts": OPContactsProviderStatus(), + @"model": modelStatus, + @"stores": storeStatus, + @"memory": storeStatus[@"memory"] ?: @{@"status": @"unknown"}, + @"context_index": storeStatus[@"context_index"] ?: @{@"status": @"unknown"} + }, + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPStartTask(NSDictionary *request) { + NSString *goal = [request[@"goal"] isKindOfClass:[NSString class]] ? request[@"goal"] : @""; + NSArray *approved = [request[@"approved_capabilities"] isKindOfClass:[NSArray class]] + ? request[@"approved_capabilities"] : OPFullYoloCapabilities(); + NSString *taskId = OPTaskId(); + NSDictionary *task = @{ + @"state": @"task.accepted", + @"task_id": taskId, + @"goal": goal, + @"autonomy_mode": @"yolo", + @"user_visible": @YES, + @"background_allowed": @NO, + @"approved_capabilities": approved, + @"status": @"active", + @"created_at": @(OPNowMs()), + @"updated_at": @(OPNowMs()), + @"owner_pid": @(getpid()), + @"source": @"openphone.agentd" + }; + OPWriteJSONFile(OPTaskPath(taskId), task); + OPRecordAudit(@"task_started", taskId, @"tasks.observe", @"allow_task_scoped", request, + @"openphone-agentd"); + OPRecordTrajectory(taskId, @"task_started", task); + return task; +} + +static NSDictionary *OPStopTask(NSDictionary *request) { + NSString *taskId = [request[@"task_id"] isKindOfClass:[NSString class]] ? request[@"task_id"] : @""; + NSString *reason = OPStringFromRequest(request, @"reason", @""); + if (reason.length == 0) { + reason = @"stopped"; + } + NSDictionary *result = @{ + @"state": @"task.stopped", + @"task_id": taskId, + @"cancel_requested": @YES, + @"reason": reason, + @"source": @"openphone.agentd", + @"detail": @"stopped" + }; + OPUpdateTask(taskId, @"stopped", @{ + @"stop_reason": reason, + @"cancel_reason": reason, + @"cancel_requested": @YES, + @"stopped_at": @(OPNowMs()) + }); + OPRecordAudit(@"task_stopped", taskId, @"tasks.observe", @"allow_task_scoped", request, + @"stopped"); + OPRecordTrajectory(taskId, @"task_stopped", result); + OPIslandPublishTerminal(taskId, NO, @"Cancelled"); + return result; +} + +static BOOL OPTaskCancellationRequested(NSString *taskId, NSString **reasonOut) { + if (![taskId isKindOfClass:[NSString class]] || taskId.length == 0) { + return NO; + } + NSDictionary *task = OPReadJSONFile(OPTaskPath(taskId)); + if (![task isKindOfClass:[NSDictionary class]]) { + return NO; + } + NSString *status = [task[@"status"] isKindOfClass:[NSString class]] ? task[@"status"] : @""; + BOOL cancelled = [task[@"cancel_requested"] boolValue] || + [status isEqualToString:@"stopped"] || + [status isEqualToString:@"cancelled"]; + if (cancelled && reasonOut) { + NSString *reason = OPStringFromRequest(task, @"cancel_reason", + OPStringFromRequest(task, @"stop_reason", @"cancelled")); + *reasonOut = reason.length > 0 ? reason : @"cancelled"; + } + return cancelled; +} + +static NSDictionary *OPFinishTask(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + if (taskId.length == 0) { + return OPError(@"missing_task_id"); + } + // Model may name the user-facing message any of: summary, answer, reply, + // response, message, text. Accept them all. + NSString *summary = OPStringFromRequest(request, @"summary", @""); + for (NSString *key in @[@"answer", @"reply", @"response", @"message", @"text"]) { + if (summary.length > 0) break; + summary = OPStringFromRequest(request, key, @""); + } + if (summary.length == 0) { + summary = @"Task finished."; + } + NSDictionary *result = @{ + @"status": @"ok", + @"state": @"task.finished", + @"task_id": taskId, + @"summary": summary, + @"source": @"openphone.agentd" + }; + OPUpdateTask(taskId, @"completed", @{ + @"result": result, + @"completed_at": @(OPNowMs()) + }); + OPRecordAudit(@"task_finished", taskId, @"tasks.observe", @"allow_task_scoped", + request, @"finish_task"); + OPRecordTrajectory(taskId, @"task_finished", result); + OPIslandPublishTerminal(taskId, YES, summary); + OPRecordAssistantTurn(summary, taskId, YES); + return result; +} + +static NSDictionary *OPFailTask(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + if (taskId.length == 0) { + return OPError(@"missing_task_id"); + } + NSString *reason = OPStringFromRequest(request, @"reason", @""); + for (NSString *key in @[@"summary", @"answer", @"reply", @"message", @"text", @"detail"]) { + if (reason.length > 0) break; + reason = OPStringFromRequest(request, key, @""); + } + if (reason.length == 0) { + reason = @"Task failed."; + } + NSDictionary *result = @{ + @"status": @"ok", + @"state": @"task.failed", + @"task_id": taskId, + @"reason": reason, + @"source": @"openphone.agentd" + }; + OPUpdateTask(taskId, @"failed", @{ + @"result": result, + @"completed_at": @(OPNowMs()) + }); + OPRecordAudit(@"task_failed", taskId, @"tasks.observe", @"failed", + request, @"fail_task"); + OPRecordTrajectory(taskId, @"task_failed", result); + OPIslandPublishTerminal(taskId, NO, reason); + OPRecordAssistantTurn(reason, taskId, NO); + return result; +} + +static NSDictionary *OPGetTask(NSDictionary *request) { + NSString *taskId = [request[@"task_id"] isKindOfClass:[NSString class]] ? request[@"task_id"] : @""; + if (taskId.length == 0) { + return OPError(@"missing_task_id"); + } + NSDictionary *task = OPReadJSONFile(OPTaskPath(taskId)); + if (!task) { + return OPError(@"task_not_found"); + } + return @{ + @"status": @"ok", + @"task_id": taskId, + @"task": task, + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPListTasks(NSDictionary *request) { + NSUInteger limit = OPLimitFromRequest(request, 50, 500); + NSArray *files = [[NSFileManager defaultManager] + contentsOfDirectoryAtPath:OPTasksPath() error:nil] ?: @[]; + NSMutableArray *tasks = [NSMutableArray array]; + for (NSString *file in files) { + if (![file.pathExtension isEqualToString:@"json"]) { + continue; + } + NSDictionary *task = OPReadJSONFile([OPTasksPath() stringByAppendingPathComponent:file]); + if (task) { + [tasks addObject:task]; + } + } + [tasks sortUsingDescriptors:@[ + [NSSortDescriptor sortDescriptorWithKey:@"updated_at" ascending:NO] + ]]; + if (limit > 0 && tasks.count > limit) { + [tasks removeObjectsInRange:NSMakeRange(limit, tasks.count - limit)]; + } + return @{ + @"status": @"ok", + @"tasks": tasks, + @"count": @(tasks.count), + @"source": @"openphone.agentd" + }; +} + +static NSArray *OPRecentTaskDictionaries(NSUInteger limit) { + NSArray *files = [[NSFileManager defaultManager] + contentsOfDirectoryAtPath:OPTasksPath() error:nil] ?: @[]; + NSMutableArray *tasks = [NSMutableArray array]; + for (NSString *file in files) { + if (![file.pathExtension isEqualToString:@"json"]) { + continue; + } + NSDictionary *task = OPReadJSONFile([OPTasksPath() stringByAppendingPathComponent:file]); + if ([task isKindOfClass:[NSDictionary class]]) { + [tasks addObject:task]; + } + } + [tasks sortUsingDescriptors:@[ + [NSSortDescriptor sortDescriptorWithKey:@"updated_at" ascending:NO] + ]]; + if (limit > 0 && tasks.count > limit) { + [tasks removeObjectsInRange:NSMakeRange(limit, tasks.count - limit)]; + } + return tasks; +} + +static NSDictionary *OPTaskStatusSummary(NSDictionary *task) { + if (![task isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSString *goal = [task[@"goal"] isKindOfClass:[NSString class]] + ? task[@"goal"] : @""; + if (goal.length > 160) { + goal = [[goal substringToIndex:160] stringByAppendingString:@"..."]; + } + NSDictionary *modelLoopCurrent = [task[@"model_loop_current"] isKindOfClass:[NSDictionary class]] + ? task[@"model_loop_current"] : @{}; + NSDictionary *modelLoopSummary = [task[@"model_loop_summary"] isKindOfClass:[NSDictionary class]] + ? task[@"model_loop_summary"] : @{}; + id currentStep = modelLoopCurrent[@"step"] ?: modelLoopSummary[@"steps_used"] ?: @0; + return @{ + @"task_id": task[@"task_id"] ?: @"", + @"status": task[@"status"] ?: @"unknown", + @"goal": goal ?: @"", + @"runner": task[@"runner"] ?: @"", + @"model_provider": task[@"model_provider"] ?: @"", + @"created_at": task[@"created_at"] ?: @0, + @"updated_at": task[@"updated_at"] ?: @0, + @"completed_at": task[@"completed_at"] ?: @0, + @"stop_reason": task[@"stop_reason"] ?: modelLoopSummary[@"stop_reason"] ?: @"", + @"cancel_requested": task[@"cancel_requested"] ?: @NO, + @"current_step": currentStep ?: @0, + @"model_loop_status": modelLoopCurrent[@"status"] ?: modelLoopSummary[@"status"] ?: @"", + @"model_loop_tool": modelLoopCurrent[@"tool"] ?: @"", + @"steps_used": modelLoopSummary[@"steps_used"] ?: @0, + @"tool_errors": modelLoopSummary[@"tool_errors"] ?: @0, + @"unverified_ui_actions": modelLoopSummary[@"unverified_ui_actions"] ?: @0, + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPAgentStatus(NSDictionary *request) { + NSUInteger limit = OPLimitFromRequest(request, 5, 25); + NSArray *tasks = OPRecentTaskDictionaries(limit > 0 ? limit : 5); + NSMutableArray *taskSummaries = [NSMutableArray array]; + NSMutableArray *activeSummaries = [NSMutableArray array]; + for (NSDictionary *task in tasks) { + NSDictionary *summary = OPTaskStatusSummary(task); + if (summary.count == 0) { + continue; + } + [taskSummaries addObject:summary]; + NSString *status = [summary[@"status"] isKindOfClass:[NSString class]] + ? summary[@"status"] : @""; + if ([status isEqualToString:@"active"]) { + [activeSummaries addObject:summary]; + } + } + NSDictionary *control = OPAgentControlSummary(); + NSDictionary *latestTask = taskSummaries.count > 0 ? taskSummaries[0] : @{}; + NSDictionary *currentTask = activeSummaries.count > 0 ? activeSummaries[0] : @{}; + NSString *agentState = [control[@"paused"] boolValue] ? @"paused" + : (activeSummaries.count > 0 ? @"running_task" : @"idle"); + NSDictionary *modelStatus = OPModelStatusDictionary(); + NSDictionary *springBoardState = OPSpringBoardPublishedState(); + NSDictionary *status = @{ + @"status": @"ok", + @"schema": @"openphone.agent_status.v1", + @"state": agentState, + @"autonomy_mode": control[@"autonomy_mode"] ?: @"yolo", + @"control": control, + @"current_task": currentTask, + @"latest_task": latestTask, + @"recent_tasks": taskSummaries, + @"active_task_count": @(activeSummaries.count), + @"current_model_loop_step": currentTask[@"current_step"] ?: @0, + @"current_model_loop_status": currentTask[@"model_loop_status"] ?: @"", + @"model": modelStatus, + @"triggers": @{ + @"volume_combo": OPVolumeTriggerStatus(), + @"last_accepted_at_ms": @(OPHardwareTriggerLastAcceptedMs) + }, + @"springboard_state": @{ + @"status": springBoardState[@"status"] ?: @"unknown", + @"age_ms": springBoardState[@"age_ms"] ?: @0, + @"foreground_app": springBoardState[@"foreground_app"] ?: @"", + @"foreground_source": springBoardState[@"foreground_source"] ?: @"" + }, + @"user_facing": @{ + @"summary": [control[@"paused"] boolValue] + ? @"Agent trigger execution is paused." + : (activeSummaries.count > 0 ? @"Agent task is running." : @"Agent is idle."), + @"last_task_id": latestTask[@"task_id"] ?: @"", + @"last_stop_reason": latestTask[@"stop_reason"] ?: @"" + }, + @"source": @"openphone.agentd" + }; + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + OPRecordAudit(@"agent_status_read", taskId, @"tasks.observe", @"allow_task_scoped", + @{@"limit": @(limit)}, agentState); + return status; +} + +static NSDictionary *OPAgentControl(NSDictionary *request) { + NSString *action = [OPStringFromRequest(request, @"action", + OPStringFromRequest(request, @"state", @"")) lowercaseString]; + NSMutableDictionary *config = [OPAgentControlConfig() mutableCopy]; + BOOL changed = NO; + if ([action isEqualToString:@"pause"] || [action isEqualToString:@"paused"] || + [action isEqualToString:@"suspend"]) { + config[@"paused"] = @YES; + changed = YES; + } else if ([action isEqualToString:@"resume"] || [action isEqualToString:@"running"] || + [action isEqualToString:@"unpause"]) { + config[@"paused"] = @NO; + config[@"pause_reason"] = @""; + changed = YES; + } else if (action.length > 0 && ![action isEqualToString:@"status"]) { + return OPError(@"invalid_agent_control_action"); + } + if (request[@"paused"] != nil) { + config[@"paused"] = @(OPBoolFromRequest(request, @"paused", [config[@"paused"] boolValue])); + changed = YES; + } + if (request[@"hardware_triggers_enabled"] != nil) { + config[@"hardware_triggers_enabled"] = @(OPBoolFromRequest(request, + @"hardware_triggers_enabled", [config[@"hardware_triggers_enabled"] boolValue])); + changed = YES; + } + if (request[@"yolo_enabled"] != nil) { + config[@"yolo_enabled"] = @(OPBoolFromRequest(request, + @"yolo_enabled", [config[@"yolo_enabled"] boolValue])); + changed = YES; + } + if ([request[@"autonomy_mode"] isKindOfClass:[NSString class]]) { + NSString *requestedMode = [request[@"autonomy_mode"] lowercaseString]; + if (!OPAutonomyModeValid(requestedMode)) { + return OPError(@"invalid_autonomy_mode"); + } + config[@"autonomy_mode"] = requestedMode; + changed = YES; + } + NSString *reason = OPStringFromRequest(request, @"reason", @""); + if (reason.length > 0 || [config[@"paused"] boolValue]) { + config[@"pause_reason"] = reason.length > 0 ? reason : (config[@"pause_reason"] ?: @""); + } + if (changed) { + config[@"updated_at_ms"] = @(OPNowMs()); + config[@"updated_by"] = OPStringFromRequest(request, @"source", @"openphone-agentctl"); + config[@"schema"] = @"openphone.agent_control.v1"; + BOOL ok = OPWriteProtectedJSONFile(OPAgentControlPath(), config); + if (!ok) { + return OPError(@"agent_control_write_failed"); + } + OPRecordAudit(@"agent_control_updated", OPStringFromRequest(request, @"task_id", @""), + @"background.run", [config[@"paused"] boolValue] ? @"paused" : @"allow_yolo", + OPRedactedObject(request ?: @{}, 0), action.length > 0 ? action : @"updated"); + OPRecordContextEvent(@"agent_control_updated", @"openphone.agentd", + OPStringFromRequest(request, @"task_id", @""), + action.length > 0 ? action : @"updated", + [config[@"paused"] boolValue] ? @"agent paused" : @"agent running", + OPAgentControlSummary()); + } + return @{ + @"status": @"ok", + @"schema": @"openphone.agent_control_result.v1", + @"changed": @(changed), + @"control": OPAgentControlSummary(), + @"source": @"openphone.agentd" + }; +} + +static NSArray *OPStaleActiveTaskFiles(NSUInteger limit, long long cutoffMs) { + NSArray *files = [[NSFileManager defaultManager] + contentsOfDirectoryAtPath:OPTasksPath() error:nil] ?: @[]; + NSMutableArray *tasks = [NSMutableArray array]; + for (NSString *file in files) { + if (![file.pathExtension isEqualToString:@"json"]) { + continue; + } + NSDictionary *task = OPReadJSONFile([OPTasksPath() stringByAppendingPathComponent:file]); + if (![task isKindOfClass:[NSDictionary class]]) { + continue; + } + NSString *status = [task[@"status"] isKindOfClass:[NSString class]] + ? task[@"status"] : @""; + if (![status isEqualToString:@"active"]) { + continue; + } + long long updatedAt = [task[@"updated_at"] respondsToSelector:@selector(longLongValue)] + ? [task[@"updated_at"] longLongValue] : 0; + long long createdAt = [task[@"created_at"] respondsToSelector:@selector(longLongValue)] + ? [task[@"created_at"] longLongValue] : 0; + long long activityAt = updatedAt > 0 ? updatedAt : createdAt; + if (activityAt > cutoffMs) { + continue; + } + [tasks addObject:task]; + } + [tasks sortUsingDescriptors:@[ + [NSSortDescriptor sortDescriptorWithKey:@"updated_at" ascending:YES] + ]]; + if (limit > 0 && tasks.count > limit) { + [tasks removeObjectsInRange:NSMakeRange(limit, tasks.count - limit)]; + } + return tasks; +} + +static NSDictionary *OPRepairStaleActiveTasks(NSDictionary *request) { + NSUInteger limit = OPLimitFromRequest(request, 25, 100); + if (limit == 0) { + limit = 25; + } + long long staleAfterMs = OPLongLongFromRequest(request, + @"stale_after_ms", 300000, 0, 86400000LL); + NSString *source = OPStringFromRequest(request, @"source", @"task_repair"); + NSString *repairReason = OPStringFromRequest(request, @"reason", + @"stale active task repaired after daemon restart"); + NSString *repairTaskId = OPStringFromRequest(request, @"task_id", @""); + long long now = OPNowMs(); + long long cutoff = now - staleAfterMs; + NSArray *staleTasks = OPStaleActiveTaskFiles(limit, cutoff); + + NSMutableArray *entries = [NSMutableArray array]; + NSUInteger repairedCount = 0; + for (NSDictionary *task in staleTasks) { + NSString *taskId = [task[@"task_id"] isKindOfClass:[NSString class]] + ? task[@"task_id"] : @""; + if (taskId.length == 0) { + continue; + } + long long updatedAt = [task[@"updated_at"] respondsToSelector:@selector(longLongValue)] + ? [task[@"updated_at"] longLongValue] : 0; + long long createdAt = [task[@"created_at"] respondsToSelector:@selector(longLongValue)] + ? [task[@"created_at"] longLongValue] : 0; + long long activityAt = updatedAt > 0 ? updatedAt : createdAt; + long long ageMs = activityAt > 0 ? MAX(0, now - activityAt) : 0; + NSString *runner = [task[@"runner"] isKindOfClass:[NSString class]] + ? task[@"runner"] : @"unknown"; + NSString *stopReason = [source isEqualToString:@"daemon_startup"] + ? @"stale_active_repaired_after_daemon_restart" + : @"stale_active_repaired"; + NSDictionary *result = @{ + @"status": @"ok", + @"state": @"task.failed", + @"task_id": taskId, + @"reason": stopReason, + @"detail": repairReason.length > 0 ? repairReason : stopReason, + @"source": @"openphone.agentd" + }; + NSDictionary *repair = @{ + @"status": @"failed", + @"repair_action": @"failed", + @"repair_policy": @"fail_stale_active", + @"previous_status": @"active", + @"runner": runner.length > 0 ? runner : @"unknown", + @"stale_after_ms": @(staleAfterMs), + @"stale_active_age_ms": @(ageMs), + @"last_activity_at_ms": @(activityAt), + @"repaired_at_ms": @(now), + @"repair_source": source.length > 0 ? source : @"task_repair", + @"daemon_pid": @(getpid()) + }; + NSDictionary *summary = @{ + @"status": @"task.failed", + @"task_id": taskId, + @"runner": runner.length > 0 ? runner : @"unknown", + @"duration_ms": @(ageMs), + @"stop_reason": stopReason, + @"last_tool_result": result, + @"trajectory": OPTrajectoryPath(taskId), + @"repair": repair, + @"source": @"openphone.agentd" + }; + OPUpdateTask(taskId, @"failed", @{ + @"result": result, + @"completed_at": @(now), + @"stop_reason": stopReason, + @"recovery": repair, + @"model_loop_summary": [runner isEqualToString:@"model"] ? summary : @{} + }); + OPRecordContextEvent(@"task_repaired", @"openphone.agentd", + repairTaskId.length > 0 ? repairTaskId : taskId, + task[@"goal"] ?: taskId, stopReason, repair); + OPRecordAudit(@"task_repaired", taskId, @"tasks.observe", @"failed", + @{@"task_id": taskId, @"source": source ?: @"", @"stale_after_ms": @(staleAfterMs)}, + stopReason); + OPRecordTrajectory(taskId, @"task_repaired", summary); + if ([runner isEqualToString:@"model"]) { + OPRecordTrajectory(taskId, @"model_loop_finished", summary); + } else { + OPRecordTrajectory(taskId, @"task_failed", summary); + } + // Ensure any island in a mid-task state reflects the failure so the + // pill doesn't stay stuck on "Acting …" after a daemon restart. + NSString *msg = [source containsString:@"startup"] + ? @"Daemon restarted mid-task — try again" + : @"Interrupted — try again"; + OPIslandPublishTerminal(taskId, NO, msg); + repairedCount++; + [entries addObject:@{ + @"task_id": taskId, + @"status": @"failed", + @"repair_action": @"failed", + @"runner": runner.length > 0 ? runner : @"unknown", + @"stale_active_age_ms": @(ageMs), + @"stop_reason": stopReason + }]; + } + + return @{ + @"status": @"ok", + @"repair_policy": @"fail_stale_active", + @"stale_after_ms": @(staleAfterMs), + @"cutoff_ms": @(cutoff), + @"limit": @(limit), + @"stale_count": @(staleTasks.count), + @"repaired_count": @(repairedCount), + @"tasks": entries, + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPGetAudit(NSDictionary *request) { + NSUInteger limit = OPLimitFromRequest(request, 100, 1000); + NSArray *events = OPReadJSONLines(OPAuditPath(), limit); + NSArray *redactedEvents = OPRedactedEvents(events); + return @{ + @"status": @"ok", + @"audit_path": OPAuditPath(), + @"events": redactedEvents, + @"count": @(redactedEvents.count), + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPGetTrajectory(NSDictionary *request) { + NSString *taskId = [request[@"task_id"] isKindOfClass:[NSString class]] ? request[@"task_id"] : @""; + if (taskId.length == 0) { + return OPError(@"missing_task_id"); + } + NSUInteger limit = OPLimitFromRequest(request, 200, 2000); + NSString *path = OPTrajectoryPath(taskId); + if (![[NSFileManager defaultManager] fileExistsAtPath:path]) { + return OPError(@"trajectory_not_found"); + } + NSArray *events = OPReadJSONLines(path, limit); + NSArray *redactedEvents = OPRedactedEvents(events); + return @{ + @"status": @"ok", + @"task_id": taskId, + @"trajectory_path": path, + @"events": redactedEvents, + @"count": @(redactedEvents.count), + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPMemorySave(NSDictionary *request) { + NSString *taskId = [request[@"task_id"] isKindOfClass:[NSString class]] ? request[@"task_id"] : @""; + NSString *text = [request[@"text"] isKindOfClass:[NSString class]] ? request[@"text"] : @""; + text = [text stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (text.length == 0) { + return OPError(@"missing_memory_text"); + } + NSString *type = [request[@"type"] isKindOfClass:[NSString class]] ? request[@"type"] : @"fact"; + if (type.length == 0) { + type = @"fact"; + } + NSString *subject = [request[@"subject"] isKindOfClass:[NSString class]] ? request[@"subject"] : @"user"; + if (subject.length == 0) { + subject = @"user"; + } + NSString *reason = [request[@"reason"] isKindOfClass:[NSString class]] ? request[@"reason"] : @""; + double confidence = OPDoubleFromRequest(request, @"confidence", 1.0, 0.0, 1.0); + NSDictionary *metadata = [request[@"metadata"] isKindOfClass:[NSDictionary class]] + ? OPRedactedObject(request[@"metadata"], 0) : @{}; + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + + long long now = OPNowMs(); + long long memoryId = 0; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "INSERT INTO memory(created_at_ms, updated_at_ms, type, subject, text, confidence, source, reason, metadata_json) " + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, now); + sqlite3_bind_int64(statement, 2, now); + OPSQLiteBindText(statement, 3, type); + OPSQLiteBindText(statement, 4, subject); + OPSQLiteBindText(statement, 5, text); + sqlite3_bind_double(statement, 6, confidence); + OPSQLiteBindText(statement, 7, @"openphone.agentd"); + OPSQLiteBindText(statement, 8, reason); + OPSQLiteBindText(statement, 9, OPJSONString(metadata)); + if (sqlite3_step(statement) == SQLITE_DONE) { + memoryId = sqlite3_last_insert_rowid(db); + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + + NSDictionary *memory = OPMemoryReadById(db, memoryId); + if (memory) { + OPMemoryIndexFTS(db, memory); + } + sqlite3_close(db); + + if (memoryId == 0 || !memory) { + return OPError([NSString stringWithFormat:@"memory_save_failed:%@", error ?: @"unknown"]); + } + + long long contextId = OPRecordContextEvent(@"memory_saved", @"openphone.agentd", taskId, + subject, text, @{ + @"memory_id": memory[@"memory_id"] ?: @"", + @"memory_type": type, + @"reason": reason ?: @"" + }); + NSMutableDictionary *result = [@{ + @"status": @"ok", + @"memory": memory, + @"context_event_id": @(contextId), + @"db_path": OPDatabasePath(), + @"source": @"openphone.agentd" + } mutableCopy]; + OPRecordAudit(@"memory_saved", taskId, @"memory.write", @"allow_yolo", + request, [NSString stringWithFormat:@"memory_id:%@", memory[@"memory_id"] ?: @""]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"memory_save", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSDictionary *OPMemoryUpdate(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + long long memoryId = OPRecordIdFromRequest(request, @[@"memory_id", @"id"]); + if (memoryId <= 0) { + return OPError(@"missing_memory_id"); + } + NSString *reason = OPStringFromRequest(request, @"reason", @""); + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSDictionary *before = OPMemoryReadById(db, memoryId); + if (!before) { + sqlite3_close(db); + return OPError(@"memory_not_found"); + } + + BOOL hasText = request[@"text"] != nil; + BOOL hasType = request[@"type"] != nil; + BOOL hasSubject = request[@"subject"] != nil; + BOOL hasConfidence = request[@"confidence"] != nil; + BOOL hasMetadata = request[@"metadata"] != nil; + if (!hasText && !hasType && !hasSubject && !hasConfidence && !hasMetadata) { + sqlite3_close(db); + return OPError(@"missing_memory_update_fields"); + } + + NSString *text = hasText + ? [OPStringFromRequest(request, @"text", @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]] + : before[@"text"]; + if (text.length == 0) { + sqlite3_close(db); + return OPError(@"missing_memory_text"); + } + NSString *type = hasType ? OPStringFromRequest(request, @"type", before[@"type"]) : before[@"type"]; + if (type.length == 0) { + type = @"fact"; + } + NSString *subject = hasSubject + ? OPStringFromRequest(request, @"subject", before[@"subject"]) : before[@"subject"]; + if (subject.length == 0) { + subject = @"user"; + } + double confidence = hasConfidence + ? OPDoubleFromRequest(request, @"confidence", [before[@"confidence"] doubleValue], 0.0, 1.0) + : [before[@"confidence"] doubleValue]; + NSDictionary *metadata = hasMetadata + ? OPJSONObjectFromRequest(request, @"metadata", @{}) + : (before[@"metadata"] ?: @{}); + + long long now = OPNowMs(); + BOOL updated = NO; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "UPDATE memory SET updated_at_ms = ?, type = ?, subject = ?, text = ?, confidence = ?, reason = ?, metadata_json = ? WHERE id = ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, now); + OPSQLiteBindText(statement, 2, type); + OPSQLiteBindText(statement, 3, subject); + OPSQLiteBindText(statement, 4, text); + sqlite3_bind_double(statement, 5, confidence); + OPSQLiteBindText(statement, 6, reason); + OPSQLiteBindText(statement, 7, OPJSONString(metadata)); + sqlite3_bind_int64(statement, 8, memoryId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + + NSDictionary *memory = updated ? OPMemoryReadById(db, memoryId) : nil; + if (memory) { + OPMemoryIndexFTS(db, memory); + } + sqlite3_close(db); + + if (!updated || !memory) { + return OPError([NSString stringWithFormat:@"memory_update_failed:%@", error ?: @"unknown"]); + } + + long long contextId = OPRecordContextEvent(@"memory_updated", @"openphone.agentd", taskId, + subject, text, @{ + @"memory_id": memory[@"memory_id"] ?: @"", + @"reason": reason ?: @"" + }); + NSDictionary *result = @{ + @"status": @"ok", + @"memory_id": memory[@"memory_id"] ?: @"", + @"memory": memory, + @"previous_memory": before, + @"context_event_id": @(contextId), + @"db_path": OPDatabasePath(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"memory_updated", taskId, @"memory.write", @"allow_yolo", + request, [NSString stringWithFormat:@"memory_id:%@", memory[@"memory_id"] ?: @""]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"memory_update", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSDictionary *OPMemoryDelete(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + long long memoryId = OPRecordIdFromRequest(request, @[@"memory_id", @"id"]); + if (memoryId <= 0) { + return OPError(@"missing_memory_id"); + } + NSString *reason = OPStringFromRequest(request, @"reason", @""); + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSDictionary *before = OPMemoryReadById(db, memoryId); + if (!before) { + sqlite3_close(db); + return OPError(@"memory_not_found"); + } + + BOOL deleted = NO; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, "DELETE FROM memory WHERE id = ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, memoryId); + deleted = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + if (deleted) { + OPMemoryDeleteFTS(db, memoryId); + } + sqlite3_close(db); + + if (!deleted) { + return OPError([NSString stringWithFormat:@"memory_delete_failed:%@", error ?: @"unknown"]); + } + + long long contextId = OPRecordContextEvent(@"memory_deleted", @"openphone.agentd", taskId, + before[@"subject"], before[@"text"], @{ + @"memory_id": before[@"memory_id"] ?: @"", + @"reason": reason ?: @"" + }); + NSDictionary *result = @{ + @"status": @"ok", + @"memory_id": before[@"memory_id"] ?: @"", + @"deleted": @YES, + @"memory": before, + @"context_event_id": @(contextId), + @"db_path": OPDatabasePath(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"memory_deleted", taskId, @"memory.write", @"allow_yolo", + request, [NSString stringWithFormat:@"memory_id:%@", before[@"memory_id"] ?: @""]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"memory_delete", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSDictionary *OPMemoryMerge(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + long long targetId = OPRecordIdFromRequest(request, @[@"target_memory_id", @"target_id", @"memory_id", @"id"]); + long long sourceId = OPRecordIdFromRequest(request, @[@"source_memory_id", @"source_id", @"merge_memory_id"]); + if (targetId <= 0 || sourceId <= 0) { + return OPError(@"missing_memory_merge_ids"); + } + if (targetId == sourceId) { + return OPError(@"memory_merge_same_id"); + } + NSString *reason = OPStringFromRequest(request, @"reason", @""); + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSDictionary *target = OPMemoryReadById(db, targetId); + NSDictionary *source = OPMemoryReadById(db, sourceId); + if (!target || !source) { + sqlite3_close(db); + return OPError(!target ? @"target_memory_not_found" : @"source_memory_not_found"); + } + + BOOL hasText = request[@"text"] != nil; + NSString *text = hasText + ? [OPStringFromRequest(request, @"text", @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]] + : [NSString stringWithFormat:@"%@\n%@", target[@"text"] ?: @"", source[@"text"] ?: @""]; + text = [text stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (text.length == 0) { + sqlite3_close(db); + return OPError(@"missing_memory_text"); + } + NSString *type = request[@"type"] != nil + ? OPStringFromRequest(request, @"type", target[@"type"]) : target[@"type"]; + NSString *subject = request[@"subject"] != nil + ? OPStringFromRequest(request, @"subject", target[@"subject"]) : target[@"subject"]; + double targetConfidence = [target[@"confidence"] doubleValue]; + double sourceConfidence = [source[@"confidence"] doubleValue]; + double confidence = request[@"confidence"] != nil + ? OPDoubleFromRequest(request, @"confidence", MAX(targetConfidence, sourceConfidence), 0.0, 1.0) + : MAX(targetConfidence, sourceConfidence); + NSMutableDictionary *metadata = [(target[@"metadata"] ?: @{}) mutableCopy]; + metadata[@"merged_from"] = source[@"memory_id"] ?: @""; + metadata[@"merged_from_subject"] = source[@"subject"] ?: @""; + metadata[@"merged_from_type"] = source[@"type"] ?: @""; + metadata[@"merged_at_ms"] = @(OPNowMs()); + if (request[@"metadata"] != nil) { + id patch = OPJSONObjectFromRequest(request, @"metadata", @{}); + if ([patch isKindOfClass:[NSDictionary class]]) { + [metadata addEntriesFromDictionary:patch]; + } + } + + long long now = OPNowMs(); + BOOL updated = NO; + BOOL deleted = NO; + OPSQLiteExec(db, @"BEGIN IMMEDIATE", &error); + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "UPDATE memory SET updated_at_ms = ?, type = ?, subject = ?, text = ?, confidence = ?, reason = ?, metadata_json = ? WHERE id = ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, now); + OPSQLiteBindText(statement, 2, type); + OPSQLiteBindText(statement, 3, subject); + OPSQLiteBindText(statement, 4, text); + sqlite3_bind_double(statement, 5, confidence); + OPSQLiteBindText(statement, 6, reason); + OPSQLiteBindText(statement, 7, OPJSONString(metadata)); + sqlite3_bind_int64(statement, 8, targetId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + + if (updated) { + OPMemoryDeleteFTS(db, sourceId); + sqlite3_stmt *deleteStatement = NULL; + if (sqlite3_prepare_v2(db, "DELETE FROM memory WHERE id = ?", + -1, &deleteStatement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(deleteStatement, 1, sourceId); + deleted = sqlite3_step(deleteStatement) == SQLITE_DONE && sqlite3_changes(db) > 0; + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(deleteStatement); + } + + NSDictionary *memory = updated && deleted ? OPMemoryReadById(db, targetId) : nil; + if (memory) { + OPMemoryIndexFTS(db, memory); + OPSQLiteExec(db, @"COMMIT", &error); + } else { + OPSQLiteExec(db, @"ROLLBACK", &error); + } + sqlite3_close(db); + + if (!memory) { + return OPError([NSString stringWithFormat:@"memory_merge_failed:%@", error ?: @"unknown"]); + } + + long long contextId = OPRecordContextEvent(@"memory_merged", @"openphone.agentd", taskId, + subject, text, @{ + @"target_memory_id": memory[@"memory_id"] ?: @"", + @"source_memory_id": source[@"memory_id"] ?: @"", + @"reason": reason ?: @"" + }); + NSDictionary *result = @{ + @"status": @"ok", + @"memory_id": memory[@"memory_id"] ?: @"", + @"memory": memory, + @"merged_from": source[@"memory_id"] ?: @"", + @"deleted_memory": source, + @"context_event_id": @(contextId), + @"db_path": OPDatabasePath(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"memory_merged", taskId, @"memory.write", @"allow_yolo", + request, [NSString stringWithFormat:@"target:%@ source:%@", + memory[@"memory_id"] ?: @"", source[@"memory_id"] ?: @""]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"memory_merge", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSArray *OPMemorySearchRows(sqlite3 *db, NSString *query, + NSUInteger limit, NSString **providerOut) { + NSMutableArray *memories = [NSMutableArray array]; + BOOL hasQuery = query.length > 0; + BOOL ftsFailed = NO; + if (hasQuery && OPSQLiteTableExists(db, @"memory_fts")) { + NSString *ftsQuery = OPFTSQuery(query); + if (ftsQuery.length > 0) { + sqlite3_stmt *statement = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT m.id, m.created_at_ms, m.updated_at_ms, m.type, m.subject, m.text, " + "m.confidence, m.source, m.reason, m.metadata_json " + "FROM memory_fts f JOIN memory m ON f.memory_id = m.id " + "WHERE memory_fts MATCH ? ORDER BY bm25(memory_fts) LIMIT ?", + -1, &statement, NULL); + if (rc == SQLITE_OK) { + OPSQLiteBindText(statement, 1, ftsQuery); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)limit); + while ((rc = sqlite3_step(statement)) == SQLITE_ROW) { + [memories addObject:OPMemoryFromStatement(statement)]; + } + if (rc != SQLITE_DONE) { + ftsFailed = YES; + [memories removeAllObjects]; + } + } else { + ftsFailed = YES; + } + sqlite3_finalize(statement); + if (!ftsFailed) { + if (providerOut) { + *providerOut = @"sqlite_fts5"; + } + return memories; + } + } + } + + sqlite3_stmt *statement = NULL; + NSString *sql = hasQuery + ? @"SELECT id, created_at_ms, updated_at_ms, type, subject, text, confidence, source, reason, metadata_json " + "FROM memory WHERE lower(text) LIKE ? OR lower(subject) LIKE ? OR lower(type) LIKE ? " + "ORDER BY updated_at_ms DESC LIMIT ?" + : @"SELECT id, created_at_ms, updated_at_ms, type, subject, text, confidence, source, reason, metadata_json " + "FROM memory ORDER BY updated_at_ms DESC LIMIT ?"; + if (sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL) == SQLITE_OK) { + if (hasQuery) { + NSString *like = [NSString stringWithFormat:@"%%%@%%", query.lowercaseString ?: @""]; + OPSQLiteBindText(statement, 1, like); + OPSQLiteBindText(statement, 2, like); + OPSQLiteBindText(statement, 3, like); + sqlite3_bind_int64(statement, 4, (sqlite3_int64)limit); + } else { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)limit); + } + while (sqlite3_step(statement) == SQLITE_ROW) { + [memories addObject:OPMemoryFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + if (providerOut) { + *providerOut = hasQuery ? @"sqlite_like" : @"sqlite_latest"; + } + return memories; +} + +static NSDictionary *OPMemorySearch(NSDictionary *request) { + NSString *taskId = [request[@"task_id"] isKindOfClass:[NSString class]] ? request[@"task_id"] : @""; + NSString *query = [request[@"query"] isKindOfClass:[NSString class]] ? request[@"query"] : @""; + query = [query stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSUInteger limit = OPLimitFromRequest(request, 20, 200); + if (limit == 0) { + limit = 20; + } + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSString *provider = nil; + NSArray *memories = OPMemorySearchRows(db, query, limit, &provider); + BOOL ftsAvailable = OPSQLiteTableExists(db, @"memory_fts"); + sqlite3_close(db); + NSDictionary *result = @{ + @"status": @"ok", + @"query": query ?: @"", + @"limit": @(limit), + @"memories": memories, + @"count": @(memories.count), + @"provider": provider ?: @"sqlite", + @"fts_available": @(ftsAvailable), + @"db_path": OPDatabasePath(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"memory_searched", taskId, @"memory.read", @"allow_task_scoped", + request, [NSString stringWithFormat:@"count:%lu", (unsigned long)memories.count]); + if (!OPBoolFromRequest(request, @"suppress_trajectory", NO)) { + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"memory_search", + @"arguments": request ?: @{}, + @"result": result + }); + } + return result; +} + +static NSDictionary *OPContextEventFromStatement(sqlite3_stmt *statement) { + NSString *metadataJSON = OPSQLiteColumnString(statement, 7); + long long rowId = sqlite3_column_int64(statement, 0); + return @{ + @"id": @(rowId), + @"event_id": [NSString stringWithFormat:@"ios-context-%lld", rowId], + @"created_at_ms": @(sqlite3_column_int64(statement, 1)), + @"type": OPSQLiteColumnString(statement, 2), + @"source": OPSQLiteColumnString(statement, 3), + @"task_id": OPSQLiteColumnString(statement, 4), + @"title": OPSQLiteColumnString(statement, 5), + @"body": OPSQLiteColumnString(statement, 6), + @"metadata": OPJSONDictionary(metadataJSON) + }; +} + +static NSDictionary *OPContextSearch(NSDictionary *request) { + NSString *taskId = [request[@"task_id"] isKindOfClass:[NSString class]] ? request[@"task_id"] : @""; + NSString *query = [request[@"query"] isKindOfClass:[NSString class]] ? request[@"query"] : @""; + query = [query stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSUInteger limit = OPLimitFromRequest(request, 20, 200); + if (limit == 0) { + limit = 20; + } + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + + NSMutableArray *events = [NSMutableArray array]; + NSString *provider = @"sqlite_latest"; + BOOL hasQuery = query.length > 0; + BOOL ftsFailed = NO; + if (hasQuery && OPSQLiteTableExists(db, @"context_event_fts")) { + NSString *ftsQuery = OPFTSQuery(query); + if (ftsQuery.length > 0) { + sqlite3_stmt *statement = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT e.id, e.created_at_ms, e.type, e.source, e.task_id, e.title, e.body, e.metadata_json " + "FROM context_event_fts f JOIN context_event e ON f.event_id = e.id " + "WHERE context_event_fts MATCH ? ORDER BY bm25(context_event_fts) LIMIT ?", + -1, &statement, NULL); + if (rc == SQLITE_OK) { + OPSQLiteBindText(statement, 1, ftsQuery); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)limit); + while ((rc = sqlite3_step(statement)) == SQLITE_ROW) { + [events addObject:OPContextEventFromStatement(statement)]; + } + if (rc != SQLITE_DONE) { + [events removeAllObjects]; + ftsFailed = YES; + } + } else { + ftsFailed = YES; + } + sqlite3_finalize(statement); + if (!ftsFailed) { + provider = @"sqlite_fts5"; + } + } + } + if (!hasQuery || ftsFailed || ![provider isEqualToString:@"sqlite_fts5"]) { + [events removeAllObjects]; + sqlite3_stmt *statement = NULL; + NSString *sql = hasQuery + ? @"SELECT id, created_at_ms, type, source, task_id, title, body, metadata_json " + "FROM context_event WHERE lower(title) LIKE ? OR lower(body) LIKE ? OR lower(type) LIKE ? " + "ORDER BY created_at_ms DESC LIMIT ?" + : @"SELECT id, created_at_ms, type, source, task_id, title, body, metadata_json " + "FROM context_event ORDER BY created_at_ms DESC LIMIT ?"; + if (sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL) == SQLITE_OK) { + if (hasQuery) { + NSString *like = [NSString stringWithFormat:@"%%%@%%", query.lowercaseString ?: @""]; + OPSQLiteBindText(statement, 1, like); + OPSQLiteBindText(statement, 2, like); + OPSQLiteBindText(statement, 3, like); + sqlite3_bind_int64(statement, 4, (sqlite3_int64)limit); + provider = @"sqlite_like"; + } else { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)limit); + provider = @"sqlite_latest"; + } + while (sqlite3_step(statement) == SQLITE_ROW) { + [events addObject:OPContextEventFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + } + BOOL ftsAvailable = OPSQLiteTableExists(db, @"context_event_fts"); + sqlite3_close(db); + NSDictionary *result = @{ + @"status": @"ok", + @"query": query ?: @"", + @"limit": @(limit), + @"events": events, + @"count": @(events.count), + @"provider": provider, + @"fts_available": @(ftsAvailable), + @"db_path": OPDatabasePath(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"context_searched", taskId, @"memory.read", @"allow_task_scoped", + request, [NSString stringWithFormat:@"count:%lu", (unsigned long)events.count]); + if (!OPBoolFromRequest(request, @"suppress_trajectory", NO)) { + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"context_search", + @"arguments": request ?: @{}, + @"result": result + }); + } + return result; +} + +static NSDictionary *OPSpringBoardClipboardBridgeInfo(void) { + NSDictionary *state = OPSpringBoardPublishedState(); + if (![state[@"status"] isEqualToString:@"ok"]) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardClipboard", + @"reason": state[@"reason"] ?: @"springboard_state_unavailable", + @"springboard_state_status": state[@"status"] ?: @"unknown" + }; + } + NSDictionary *bridge = [state[@"clipboard_bridge"] isKindOfClass:[NSDictionary class]] + ? state[@"clipboard_bridge"] : @{}; + if (![bridge[@"status"] isEqualToString:@"ready"]) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardClipboard", + @"reason": @"springboard_clipboard_bridge_not_ready", + @"springboard_state_status": state[@"status"] ?: @"unknown", + @"bridge_status": bridge[@"status"] ?: @"missing" + }; + } + NSMutableDictionary *result = [bridge mutableCopy]; + result[@"provider"] = result[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardClipboard"; + return result; +} + +static NSDictionary *OPSpringBoardClipboardPerform(NSString *operation, + NSString *text, + NSDictionary *request) { + NSDictionary *bridge = OPSpringBoardClipboardBridgeInfo(); + if (![bridge[@"status"] isEqualToString:@"ready"]) { + NSMutableDictionary *unavailable = [@{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardClipboard", + @"reason": bridge[@"reason"] ?: @"springboard_clipboard_bridge_unavailable", + @"operation": operation ?: @"", + @"request_path": OPSpringBoardClipboardRequestPath(), + @"response_path": OPSpringBoardClipboardResponsePath() + } mutableCopy]; + unavailable[@"bridge"] = bridge ?: @{}; + return unavailable; + } + + NSString *requestId = [NSString stringWithFormat:@"clipboard-%lld-%d", OPNowMs(), getpid()]; + NSString *requestPath = OPSpringBoardClipboardRequestPath(); + NSString *responsePath = OPSpringBoardClipboardResponsePath(); + long long timeoutMs = OPLongLongFromRequest(request ?: @{}, @"clipboard_timeout_ms", 1200, 250, 5000); + NSMutableDictionary *payload = [@{ + @"schema": @"openphone.springboard_clipboard_request.v1", + @"request_id": requestId, + @"timestamp_ms": @(OPNowMs()), + @"provider": @"openphone.agentd", + @"operation": operation ?: @"read", + @"timeout_ms": @(timeoutMs), + @"source": @"openphone.agentd.clipboard" + } mutableCopy]; + if (text) { + payload[@"text"] = text; + } + if (!OPWriteJSONFile(requestPath, payload)) { + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardClipboard", + @"reason": @"request_write_failed", + @"operation": operation ?: @"", + @"request_path": requestPath, + @"response_path": responsePath + }; + } + chmod(requestPath.UTF8String, 0600); + + long long start = OPNowMs(); + while (OPNowMs() - start <= timeoutMs) { + NSDictionary *response = OPReadJSONFile(responsePath); + NSString *responseRequestId = [response[@"request_id"] isKindOfClass:[NSString class]] + ? response[@"request_id"] : @""; + if ([responseRequestId isEqualToString:requestId]) { + NSMutableDictionary *result = [response mutableCopy]; + result[@"provider"] = result[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardClipboard"; + result[@"operation"] = result[@"operation"] ?: operation ?: @""; + result[@"request_path"] = requestPath; + result[@"response_path"] = responsePath; + result[@"bridge"] = bridge ?: @{}; + [[NSFileManager defaultManager] removeItemAtPath:requestPath error:nil]; + return result; + } + usleep(100000); + } + + [[NSFileManager defaultManager] removeItemAtPath:requestPath error:nil]; + return @{ + @"status": @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardClipboard", + @"reason": @"response_timeout", + @"operation": operation ?: @"", + @"request_id": requestId, + @"request_path": requestPath, + @"response_path": responsePath, + @"timeout_ms": @(timeoutMs), + @"bridge": bridge ?: @{} + }; +} + +static NSString *OPClipboardFallbackPath(void) { + return [OPConfigPath() stringByAppendingPathComponent:@"clipboard-fallback.json"]; +} + +static NSString *OPClipboardLimitedText(NSString *value, NSUInteger limit, BOOL *truncatedOut) { + NSString *text = [value isKindOfClass:[NSString class]] ? value : @""; + if (limit == 0 || text.length <= limit) { + if (truncatedOut) { + *truncatedOut = NO; + } + return text ?: @""; + } + if (truncatedOut) { + *truncatedOut = YES; + } + return [text substringToIndex:limit] ?: @""; +} + +static NSString *OPClipboardTextHash(NSString *text) { + NSData *data = [(text ?: @"") dataUsingEncoding:NSUTF8StringEncoding] ?: [NSData data]; + return OPSHA256Hex(data); +} + +static NSDictionary *OPClipboardSystemRead(NSString **textOut) { + NSDictionary *bridge = OPSpringBoardClipboardPerform(@"read", nil, @{}); + if ([bridge[@"status"] isEqualToString:@"ok"]) { + NSString *text = [bridge[@"text"] isKindOfClass:[NSString class]] + ? bridge[@"text"] : @""; + if (textOut) { + *textOut = text ?: @""; + } + return @{ + @"provider": bridge[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardClipboard", + @"system_clipboard": @YES, + @"bridge": bridge + }; + } + + NSDictionary *fallback = OPReadJSONFile(OPClipboardFallbackPath()); + NSString *text = [fallback[@"text"] isKindOfClass:[NSString class]] + ? fallback[@"text"] : @""; + if (textOut) { + *textOut = text ?: @""; + } + return @{ + @"provider": @"openphone.clipboard_fallback_file", + @"system_clipboard": @NO, + @"fallback_path": OPClipboardFallbackPath(), + @"bridge": bridge ?: @{} + }; +} + +static NSDictionary *OPClipboardSystemWrite(NSString *text) { + NSDictionary *bridge = OPSpringBoardClipboardPerform(@"write", text ?: @"", @{}); + if ([bridge[@"status"] isEqualToString:@"ok"]) { + return @{ + @"provider": bridge[@"provider"] ?: @"OpenPhoneVolumeTrigger.SpringBoardClipboard", + @"system_clipboard": @YES, + @"bridge": bridge + }; + } + + NSDictionary *fallback = @{ + @"schema": @"openphone.clipboard_fallback.v1", + @"updated_at_ms": @(OPNowMs()), + @"text": text ?: @"" + }; + BOOL wrote = OPWriteJSONFile(OPClipboardFallbackPath(), fallback); + chmod(OPClipboardFallbackPath().UTF8String, 0600); + return @{ + @"provider": @"openphone.clipboard_fallback_file", + @"system_clipboard": @NO, + @"fallback_path": OPClipboardFallbackPath(), + @"fallback_write_ok": @(wrote), + @"bridge": bridge ?: @{} + }; +} + +static NSDictionary *OPClipboardTraceSummary(NSDictionary *result) { + if (![result isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSMutableDictionary *summary = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"status", @"tool", @"provider", @"system_clipboard", + @"text_length", @"text_sha256", @"truncated", @"max_chars", + @"context_event_id", @"source"]) { + id value = result[key]; + if (value) { + summary[key] = value; + } + } + NSString *text = [result[@"text"] isKindOfClass:[NSString class]] ? result[@"text"] : @""; + if (text.length > 0) { + BOOL truncated = NO; + summary[@"text_preview"] = OPClipboardLimitedText(text, 160, &truncated); + summary[@"text_preview_truncated"] = @(truncated); + } + return summary; +} + +static NSDictionary *OPClipboardRead(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *reason = OPStringFromRequest(request, @"reason", @""); + NSUInteger maxChars = (NSUInteger)OPLongLongFromRequest(request, @"max_chars", + (long long)OPLimitFromRequest(request, 4096, 32768), 0, 32768); + if (maxChars == 0) { + maxChars = 4096; + } + + NSString *rawText = @""; + NSDictionary *provider = OPClipboardSystemRead(&rawText); + BOOL truncated = NO; + NSString *text = OPClipboardLimitedText(rawText, maxChars, &truncated); + NSString *hash = OPClipboardTextHash(rawText); + long long contextId = OPRecordContextEvent(@"clipboard_read", @"openphone.agentd", taskId, + @"Clipboard", OPClipboardLimitedText(rawText, 240, NULL), @{ + @"provider": provider[@"provider"] ?: @"unknown", + @"system_clipboard": provider[@"system_clipboard"] ?: @NO, + @"text_length": @(rawText.length), + @"text_sha256": hash ?: @"", + @"truncated": @(truncated), + @"reason": reason ?: @"" + }); + NSMutableDictionary *result = [@{ + @"status": @"ok", + @"tool": @"clipboard_read", + @"text": text ?: @"", + @"text_length": @(rawText.length), + @"text_sha256": hash ?: @"", + @"truncated": @(truncated), + @"max_chars": @(maxChars), + @"provider": provider[@"provider"] ?: @"unknown", + @"system_clipboard": provider[@"system_clipboard"] ?: @NO, + @"context_event_id": @(contextId), + @"source": @"openphone.agentd" + } mutableCopy]; + if (provider[@"fallback_path"]) { + result[@"fallback_path"] = provider[@"fallback_path"]; + } + OPRecordAudit(@"clipboard_read", taskId, @"clipboard.read", @"allow_task_scoped", + @{ + @"reason": reason ?: @"", + @"provider": result[@"provider"] ?: @"", + @"system_clipboard": result[@"system_clipboard"] ?: @NO, + @"text_length": result[@"text_length"] ?: @0, + @"text_sha256": result[@"text_sha256"] ?: @"", + @"truncated": result[@"truncated"] ?: @NO + }, + [NSString stringWithFormat:@"chars:%@", result[@"text_length"] ?: @0]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"clipboard_read", + @"arguments": @{ + @"reason": reason ?: @"", + @"max_chars": @(maxChars) + }, + @"result": OPClipboardTraceSummary(result) + }); + return result; +} + +static NSDictionary *OPClipboardWrite(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *reason = OPStringFromRequest(request, @"reason", @""); + NSString *text = OPStringFromRequest(request, @"text", + OPStringFromRequest(request, @"clipboard_text", @"")); + if (text.length == 0 && request[@"text"] == nil && request[@"clipboard_text"] == nil) { + return OPError(@"missing_clipboard_text"); + } + + NSDictionary *provider = OPClipboardSystemWrite(text ?: @""); + BOOL wrote = !provider[@"fallback_write_ok"] || [provider[@"fallback_write_ok"] boolValue]; + if (!wrote) { + return OPError(@"clipboard_write_failed"); + } + NSString *hash = OPClipboardTextHash(text); + long long contextId = OPRecordContextEvent(@"clipboard_written", @"openphone.agentd", taskId, + @"Clipboard", OPClipboardLimitedText(text, 240, NULL), @{ + @"provider": provider[@"provider"] ?: @"unknown", + @"system_clipboard": provider[@"system_clipboard"] ?: @NO, + @"text_length": @(text.length), + @"text_sha256": hash ?: @"", + @"reason": reason ?: @"" + }); + NSMutableDictionary *result = [@{ + @"status": @"ok", + @"tool": @"clipboard_write", + @"text_length": @(text.length), + @"text_sha256": hash ?: @"", + @"provider": provider[@"provider"] ?: @"unknown", + @"system_clipboard": provider[@"system_clipboard"] ?: @NO, + @"context_event_id": @(contextId), + @"source": @"openphone.agentd" + } mutableCopy]; + if (provider[@"fallback_path"]) { + result[@"fallback_path"] = provider[@"fallback_path"]; + } + OPRecordAudit(@"clipboard_written", taskId, @"clipboard.write", @"allow_yolo", + @{ + @"reason": reason ?: @"", + @"provider": result[@"provider"] ?: @"", + @"system_clipboard": result[@"system_clipboard"] ?: @NO, + @"text_length": result[@"text_length"] ?: @0, + @"text_sha256": result[@"text_sha256"] ?: @"" + }, + [NSString stringWithFormat:@"chars:%@", result[@"text_length"] ?: @0]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"clipboard_write", + @"arguments": @{ + @"reason": reason ?: @"", + @"text_length": @(text.length), + @"text_sha256": hash ?: @"" + }, + @"result": OPClipboardTraceSummary(result) + }); + return result; +} + +static NSString *OPContactsAddressBookPath(void) { + const char *override = getenv("OPENPHONE_CONTACTS_DB_PATH"); + if (override && override[0] != '\0') { + return [NSString stringWithUTF8String:override]; + } + return @"/var/mobile/Library/AddressBook/AddressBook.sqlitedb"; +} + +static NSDictionary *OPContactsProviderStatus(void) { + NSString *addressBookPath = OPContactsAddressBookPath(); + BOOL addressBookAvailable = [[NSFileManager defaultManager] fileExistsAtPath:addressBookPath]; + BOOL fixtureAvailable = [[NSFileManager defaultManager] fileExistsAtPath:OPContactsFixturePath()]; + return @{ + @"status": @"implemented_partial", + @"provider": @"AddressBook.sqlite", + @"path": addressBookPath ?: @"", + @"addressbook_available": @(addressBookAvailable), + @"fixture_provider": @"openphone.contacts_fixture_file", + @"fixture_path": OPContactsFixturePath(), + @"fixture_available": @(fixtureAvailable), + @"access": @"read_only", + @"retention": @"search results are returned to caller; context/audit store counts and query hashes only" + }; +} + +static BOOL OPSQLiteColumnExists(sqlite3 *db, NSString *table, NSString *column) { + if (!db || table.length == 0 || column.length == 0) { + return NO; + } + sqlite3_stmt *statement = NULL; + NSString *sql = [NSString stringWithFormat:@"PRAGMA table_info(%@)", table]; + BOOL exists = NO; + if (sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL) == SQLITE_OK) { + while (sqlite3_step(statement) == SQLITE_ROW) { + NSString *name = OPSQLiteColumnString(statement, 1); + if ([name caseInsensitiveCompare:column] == NSOrderedSame) { + exists = YES; + break; + } + } + } + sqlite3_finalize(statement); + return exists; +} + +static NSString *OPContactsPersonColumnExpr(sqlite3 *db, NSString *column) { + if (OPSQLiteColumnExists(db, @"ABPerson", column)) { + return [NSString stringWithFormat:@"COALESCE(p.%@, '')", column]; + } + return @"''"; +} + +static NSArray *OPContactsStringArray(id value, NSUInteger maxItems) { + NSMutableArray *items = [NSMutableArray array]; + if ([value isKindOfClass:[NSArray class]]) { + for (id item in (NSArray *)value) { + if (![item isKindOfClass:[NSString class]]) { + continue; + } + NSString *text = [(NSString *)item stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (text.length == 0 || [items containsObject:text]) { + continue; + } + [items addObject:text]; + if (items.count >= maxItems) { + break; + } + } + } else if ([value isKindOfClass:[NSString class]]) { + NSString *text = [(NSString *)value stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (text.length > 0) { + [items addObject:text]; + } + } + return items; +} + +static NSString *OPContactsDisplayName(NSString *first, NSString *middle, NSString *last, + NSString *organization, NSString *nickname, long long rowId) { + NSMutableArray *nameParts = [NSMutableArray array]; + for (NSString *part in @[first ?: @"", middle ?: @"", last ?: @""]) { + NSString *trimmed = [part stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (trimmed.length > 0) { + [nameParts addObject:trimmed]; + } + } + NSString *displayName = [nameParts componentsJoinedByString:@" "]; + if (displayName.length == 0) { + displayName = [organization stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + } + if (displayName.length == 0) { + displayName = [nickname stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + } + if (displayName.length == 0) { + displayName = [NSString stringWithFormat:@"Contact %lld", rowId]; + } + return displayName ?: @""; +} + +static BOOL OPContactsValueMatches(NSDictionary *contact, NSString *query) { + if (query.length == 0) { + return YES; + } + NSString *needle = query.lowercaseString ?: @""; + NSMutableArray *haystack = [NSMutableArray array]; + for (NSString *key in @[@"display_name", @"given_name", @"middle_name", @"family_name", + @"organization", @"department", @"job_title", @"nickname"]) { + id value = contact[key]; + if ([value isKindOfClass:[NSString class]]) { + [haystack addObject:value]; + } + } + for (NSString *key in @[@"phone_numbers", @"emails"]) { + id value = contact[key]; + if ([value isKindOfClass:[NSArray class]]) { + for (id item in (NSArray *)value) { + if ([item isKindOfClass:[NSString class]]) { + [haystack addObject:item]; + } + } + } + } + for (NSString *value in haystack) { + if ([value.lowercaseString containsString:needle]) { + return YES; + } + } + return NO; +} + +static NSDictionary *OPContactsNormalizedFixtureContact(NSDictionary *contact, NSUInteger index) { + NSString *displayName = OPStringFromRequest(contact, @"display_name", + OPStringFromRequest(contact, @"name", @"")); + NSString *givenName = OPStringFromRequest(contact, @"given_name", + OPStringFromRequest(contact, @"first_name", @"")); + NSString *middleName = OPStringFromRequest(contact, @"middle_name", @""); + NSString *familyName = OPStringFromRequest(contact, @"family_name", + OPStringFromRequest(contact, @"last_name", @"")); + NSString *organization = OPStringFromRequest(contact, @"organization", @""); + NSString *nickname = OPStringFromRequest(contact, @"nickname", @""); + if (displayName.length == 0) { + displayName = OPContactsDisplayName(givenName, middleName, familyName, + organization, nickname, (long long)index + 1); + } + return @{ + @"contact_id": OPStringFromRequest(contact, @"contact_id", + [NSString stringWithFormat:@"fixture-contact-%lu", (unsigned long)index + 1]), + @"display_name": displayName ?: @"", + @"given_name": givenName ?: @"", + @"middle_name": middleName ?: @"", + @"family_name": familyName ?: @"", + @"organization": organization ?: @"", + @"department": OPStringFromRequest(contact, @"department", @""), + @"job_title": OPStringFromRequest(contact, @"job_title", @""), + @"nickname": nickname ?: @"", + @"phone_numbers": OPContactsStringArray(contact[@"phone_numbers"] ?: contact[@"phones"], 8), + @"emails": OPContactsStringArray(contact[@"emails"] ?: contact[@"email_addresses"], 8), + @"provider": @"openphone.contacts_fixture_file" + }; +} + +static NSArray *OPContactsFixtureSearch(NSString *query, NSUInteger limit) { + NSDictionary *fixture = OPReadJSONFile(OPContactsFixturePath()); + NSArray *contacts = [fixture[@"contacts"] isKindOfClass:[NSArray class]] + ? fixture[@"contacts"] : @[]; + NSMutableArray *results = [NSMutableArray array]; + NSUInteger index = 0; + for (id value in contacts) { + if (![value isKindOfClass:[NSDictionary class]]) { + index++; + continue; + } + NSDictionary *contact = OPContactsNormalizedFixtureContact((NSDictionary *)value, index); + index++; + if (!OPContactsValueMatches(contact, query)) { + continue; + } + [results addObject:contact]; + if (results.count >= limit) { + break; + } + } + return results; +} + +static BOOL OPContactsLooksLikePhone(NSString *value) { + NSUInteger digits = 0; + for (NSUInteger i = 0; i < value.length; i++) { + unichar c = [value characterAtIndex:i]; + if (c >= '0' && c <= '9') { + digits++; + } + } + return digits >= 5; +} + +static NSDictionary *OPContactsMultiValues(sqlite3 *db, long long rowId) { + NSMutableArray *phones = [NSMutableArray array]; + NSMutableArray *emails = [NSMutableArray array]; + if (!OPSQLiteTableExists(db, @"ABMultiValue") || + !OPSQLiteColumnExists(db, @"ABMultiValue", @"record_id") || + !OPSQLiteColumnExists(db, @"ABMultiValue", @"value")) { + return @{@"phone_numbers": phones, @"emails": emails}; + } + NSString *propertyExpr = OPSQLiteColumnExists(db, @"ABMultiValue", @"property") + ? @"COALESCE(property, 0)" : @"0"; + NSString *orderBy = OPSQLiteColumnExists(db, @"ABMultiValue", @"identifier") + ? @" ORDER BY identifier" : @""; + sqlite3_stmt *statement = NULL; + NSString *sql = [NSString stringWithFormat: + @"SELECT %@, COALESCE(value, '') FROM ABMultiValue " + "WHERE record_id = ?%@ LIMIT 32", propertyExpr, orderBy]; + if (sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, rowId); + while (sqlite3_step(statement) == SQLITE_ROW) { + long long property = sqlite3_column_int64(statement, 0); + NSString *value = [OPSQLiteColumnString(statement, 1) stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (value.length == 0) { + continue; + } + if ((property == 4 || [value containsString:@"@"]) && ![emails containsObject:value]) { + [emails addObject:value]; + } else if ((property == 3 || OPContactsLooksLikePhone(value)) && + ![phones containsObject:value]) { + [phones addObject:value]; + } + if (phones.count >= 8 && emails.count >= 8) { + break; + } + } + } + sqlite3_finalize(statement); + return @{ + @"phone_numbers": phones.count > 8 ? [phones subarrayWithRange:NSMakeRange(0, 8)] : phones, + @"emails": emails.count > 8 ? [emails subarrayWithRange:NSMakeRange(0, 8)] : emails + }; +} + +static NSArray *OPContactsSystemSearch(NSString *query, NSUInteger limit, + NSString **errorOut) { + NSString *path = OPContactsAddressBookPath(); + sqlite3 *db = NULL; + int rc = sqlite3_open_v2(path.UTF8String, &db, + SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, NULL); + if (rc != SQLITE_OK) { + if (errorOut) { + *errorOut = db ? [NSString stringWithUTF8String:sqlite3_errmsg(db)] : @"sqlite_open_failed"; + } + if (db) { + sqlite3_close(db); + } + return nil; + } + sqlite3_busy_timeout(db, 5000); + if (!OPSQLiteTableExists(db, @"ABPerson")) { + if (errorOut) { + *errorOut = @"abperson_table_missing"; + } + sqlite3_close(db); + return nil; + } + + NSArray *columns = @[@"First", @"Middle", @"Last", @"Organization", + @"Department", @"JobTitle", @"Nickname"]; + NSMutableArray *selectParts = [NSMutableArray array]; + NSMutableArray *searchParts = [NSMutableArray array]; + for (NSString *column in columns) { + NSString *expr = OPContactsPersonColumnExpr(db, column); + [selectParts addObject:expr]; + [searchParts addObject:expr]; + } + NSString *select = [selectParts componentsJoinedByString:@", "]; + NSString *nameExpr = [searchParts componentsJoinedByString:@" || ' ' || "]; + BOOL hasQuery = query.length > 0; + BOOL hasMultiValueSearch = OPSQLiteTableExists(db, @"ABMultiValue") && + OPSQLiteColumnExists(db, @"ABMultiValue", @"record_id") && + OPSQLiteColumnExists(db, @"ABMultiValue", @"value"); + NSString *where = @""; + if (hasQuery) { + NSString *multi = hasMultiValueSearch + ? @" OR EXISTS (SELECT 1 FROM ABMultiValue mv WHERE mv.record_id = p.ROWID AND lower(COALESCE(mv.value, '')) LIKE ?)" + : @""; + where = [NSString stringWithFormat:@"WHERE lower(%@) LIKE ?%@", nameExpr, multi]; + } + NSString *sql = [NSString stringWithFormat: + @"SELECT p.ROWID, %@ FROM ABPerson p %@ ORDER BY p.ROWID DESC LIMIT ?", + select, where]; + sqlite3_stmt *statement = NULL; + NSMutableArray *contacts = [NSMutableArray array]; + rc = sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL); + if (rc == SQLITE_OK) { + int bindIndex = 1; + if (hasQuery) { + NSString *like = [NSString stringWithFormat:@"%%%@%%", query.lowercaseString ?: @""]; + OPSQLiteBindText(statement, bindIndex++, like); + if (hasMultiValueSearch) { + OPSQLiteBindText(statement, bindIndex++, like); + } + } + sqlite3_bind_int64(statement, bindIndex, (sqlite3_int64)limit); + while ((rc = sqlite3_step(statement)) == SQLITE_ROW) { + long long rowId = sqlite3_column_int64(statement, 0); + NSString *first = OPSQLiteColumnString(statement, 1); + NSString *middle = OPSQLiteColumnString(statement, 2); + NSString *last = OPSQLiteColumnString(statement, 3); + NSString *organization = OPSQLiteColumnString(statement, 4); + NSString *department = OPSQLiteColumnString(statement, 5); + NSString *jobTitle = OPSQLiteColumnString(statement, 6); + NSString *nickname = OPSQLiteColumnString(statement, 7); + NSDictionary *multi = OPContactsMultiValues(db, rowId); + [contacts addObject:@{ + @"contact_id": [NSString stringWithFormat:@"ios-contact-%lld", rowId], + @"row_id": @(rowId), + @"display_name": OPContactsDisplayName(first, middle, last, + organization, nickname, rowId), + @"given_name": first ?: @"", + @"middle_name": middle ?: @"", + @"family_name": last ?: @"", + @"organization": organization ?: @"", + @"department": department ?: @"", + @"job_title": jobTitle ?: @"", + @"nickname": nickname ?: @"", + @"phone_numbers": multi[@"phone_numbers"] ?: @[], + @"emails": multi[@"emails"] ?: @[], + @"provider": @"AddressBook.sqlite" + }]; + } + if (rc != SQLITE_DONE && errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"contacts_query_failed"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"contacts_query_prepare_failed"; + } + sqlite3_finalize(statement); + sqlite3_close(db); + if (rc != SQLITE_DONE && contacts.count == 0 && errorOut && *errorOut) { + return nil; + } + return contacts; +} + +static NSArray *OPContactsSummaryContacts(NSArray *contacts, NSUInteger limit) { + NSMutableArray *summary = [NSMutableArray array]; + for (NSDictionary *contact in contacts) { + NSMutableDictionary *item = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"contact_id", @"display_name", @"organization", @"job_title"]) { + id value = contact[key]; + if ([value isKindOfClass:[NSString class]] && [value length] > 0) { + item[key] = value; + } + } + NSArray *phones = [contact[@"phone_numbers"] isKindOfClass:[NSArray class]] + ? contact[@"phone_numbers"] : @[]; + NSArray *emails = [contact[@"emails"] isKindOfClass:[NSArray class]] + ? contact[@"emails"] : @[]; + if (phones.count > 0) { + item[@"phone_numbers"] = phones.count > 4 ? [phones subarrayWithRange:NSMakeRange(0, 4)] : phones; + } + if (emails.count > 0) { + item[@"emails"] = emails.count > 4 ? [emails subarrayWithRange:NSMakeRange(0, 4)] : emails; + } + if (item.count > 0) { + [summary addObject:item]; + } + if (summary.count >= limit) { + break; + } + } + return summary; +} + +static NSDictionary *OPContactsTraceSummary(NSDictionary *result) { + if (![result isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSArray *contacts = [result[@"contacts"] isKindOfClass:[NSArray class]] + ? result[@"contacts"] : @[]; + return @{ + @"status": result[@"status"] ?: @"unknown", + @"tool": @"contacts_search", + @"provider": result[@"provider"] ?: @"unknown", + @"query_length": result[@"query_length"] ?: @0, + @"query_sha256": result[@"query_sha256"] ?: @"", + @"count": result[@"count"] ?: @0, + @"limit": result[@"limit"] ?: @0, + @"contacts": OPContactsSummaryContacts(contacts, 8), + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPContactsSearch(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *reason = OPStringFromRequest(request, @"reason", @""); + NSString *query = OPStringFromRequest(request, @"query", + OPStringFromRequest(request, @"name", @"")); + query = [query stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + BOOL allowEmptyQuery = OPBoolFromRequest(request, @"allow_empty_query", NO); + if (query.length == 0 && !allowEmptyQuery) { + return OPError(@"missing_contacts_query"); + } + NSUInteger limit = OPLimitFromRequest(request, 10, 50); + if (limit == 0) { + limit = 10; + } + + NSString *provider = @""; + NSString *systemError = nil; + NSArray *contacts = nil; + BOOL addressBookAvailable = [[NSFileManager defaultManager] fileExistsAtPath:OPContactsAddressBookPath()]; + if (addressBookAvailable) { + contacts = OPContactsSystemSearch(query, limit, &systemError); + if (contacts) { + provider = @"AddressBook.sqlite"; + } + } + if (!contacts) { + if ([[NSFileManager defaultManager] fileExistsAtPath:OPContactsFixturePath()]) { + contacts = OPContactsFixtureSearch(query, limit); + provider = @"openphone.contacts_fixture_file"; + } else { + NSString *reasonText = addressBookAvailable + ? [NSString stringWithFormat:@"contacts_system_query_failed:%@", systemError ?: @"unknown"] + : @"contacts_provider_unavailable"; + return OPError(reasonText); + } + } + + NSString *queryHash = OPClipboardTextHash(query ?: @""); + long long contextId = OPRecordContextEvent(@"contacts_searched", @"openphone.agentd", taskId, + @"Contacts search", + [NSString stringWithFormat:@"Contacts search returned %lu result(s)", + (unsigned long)contacts.count], + @{ + @"provider": provider ?: @"unknown", + @"count": @(contacts.count), + @"limit": @(limit), + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"reason": reason ?: @"" + }); + NSDictionary *result = @{ + @"status": @"ok", + @"tool": @"contacts_search", + @"query": query ?: @"", + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"limit": @(limit), + @"contacts": contacts ?: @[], + @"count": @(contacts.count), + @"provider": provider ?: @"unknown", + @"addressbook_path": OPContactsAddressBookPath(), + @"fixture_path": OPContactsFixturePath(), + @"context_event_id": @(contextId), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"contacts_searched", taskId, @"contacts.read", @"allow_task_scoped", + @{ + @"reason": reason ?: @"", + @"provider": provider ?: @"unknown", + @"limit": @(limit), + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"count": @(contacts.count) + }, + [NSString stringWithFormat:@"count:%lu", (unsigned long)contacts.count]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"contacts_search", + @"arguments": @{ + @"reason": reason ?: @"", + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"limit": @(limit) + }, + @"result": OPContactsTraceSummary(result) + }); + return result; +} + +static NSArray *OPCalendarDatabasePaths(void) { + const char *override = getenv("OPENPHONE_CALENDAR_DB_PATH"); + if (override && override[0] != '\0') { + return @[[NSString stringWithUTF8String:override]]; + } + return @[ + @"/var/mobile/Library/Calendar/Calendar.sqlitedb", + @"/private/var/mobile/Library/Calendar/Calendar.sqlitedb" + ]; +} + +static NSString *OPCalendarDatabasePath(void) { + NSArray *paths = OPCalendarDatabasePaths(); + for (NSString *path in paths) { + if (OPPathExists(path)) { + return path; + } + } + return paths.count > 0 ? paths[0] : @"/var/mobile/Library/Calendar/Calendar.sqlitedb"; +} + +static NSDictionary *OPCalendarProviderStatus(void) { + NSString *calendarPath = OPCalendarDatabasePath(); + BOOL calendarAvailable = OPPathExists(calendarPath); + BOOL fixtureAvailable = OPPathExists(OPCalendarFixturePath()); + return @{ + @"status": @"implemented_partial", + @"provider": @"Calendar.sqlitedb", + @"path": calendarPath ?: @"", + @"calendar_available": @(calendarAvailable), + @"fixture_provider": @"openphone.calendar_fixture_file", + @"fixture_path": OPCalendarFixturePath(), + @"fixture_available": @(fixtureAvailable), + @"protected_helper_socket": OPProtectedDataHelperSocketPath(), + @"protected_helper_available": @(OPPathExists(OPProtectedDataHelperSocketPath())), + @"protected_helper_manager": @"launchd_sh_wrapper_setuid_helper", + @"protected_helper_binary": OPProtectedDataHelperBinaryPath() ?: @"", + @"protected_helper_spawn_pid": @((int)OPProtectedDataHelperPid), + @"protected_helper_last_spawn_ms": @((long long)OPProtectedDataHelperLastSpawnMs), + @"protected_helper_last_spawn_error": OPProtectedDataHelperLastSpawnError ?: @"", + @"access": @"read_only", + @"retention": @"event summaries are returned to caller; context/audit store counts, query hashes, and time ranges only" + }; +} + +static long long OPCalendarLongLongFromValue(id value, long long defaultValue) { + if ([value isKindOfClass:[NSNumber class]]) { + return [value longLongValue]; + } + if ([value isKindOfClass:[NSString class]]) { + return [(NSString *)value longLongValue]; + } + return defaultValue; +} + +static NSString *OPCalendarISO8601FromMs(long long ms) { + if (ms <= 0) { + return @""; + } + NSDate *date = [NSDate dateWithTimeIntervalSince1970:((NSTimeInterval)ms / 1000.0)]; + NSDateFormatter *formatter = [[NSDateFormatter alloc] init]; + formatter.locale = [NSLocale localeWithLocaleIdentifier:@"en_US_POSIX"]; + formatter.timeZone = [NSTimeZone timeZoneForSecondsFromGMT:0]; + formatter.dateFormat = @"yyyy-MM-dd'T'HH:mm:ss'Z'"; + return [formatter stringFromDate:date] ?: @""; +} + +static long long OPCalendarMsFromSQLiteValue(double value) { + if (value <= 0.0) { + return 0; + } + if (value > 1000000000000.0) { + return (long long)value; + } + if (value > 1000000000.0) { + return (long long)(value * 1000.0); + } + return (long long)((value + 978307200.0) * 1000.0); +} + +static double OPCalendarSQLiteValueFromUnixMs(long long ms) { + if (ms <= 0) { + return 0.0; + } + return ((double)ms / 1000.0) - 978307200.0; +} + +static NSString *OPCalendarFirstExistingColumn(sqlite3 *db, NSString *table, NSArray *columns) { + for (NSString *column in columns) { + if (OPSQLiteColumnExists(db, table, column)) { + return column; + } + } + return @""; +} + +static NSString *OPCalendarTextExpr(sqlite3 *db, NSString *table, NSString *alias, + NSArray *columns) { + NSString *column = OPCalendarFirstExistingColumn(db, table, columns); + if (column.length == 0) { + return @"''"; + } + return [NSString stringWithFormat:@"COALESCE(%@.%@, '')", alias, column]; +} + +static NSString *OPCalendarNumberExpr(sqlite3 *db, NSString *table, NSString *alias, + NSArray *columns) { + NSString *column = OPCalendarFirstExistingColumn(db, table, columns); + if (column.length == 0) { + return @"0"; + } + return [NSString stringWithFormat:@"COALESCE(%@.%@, 0)", alias, column]; +} + +static BOOL OPCalendarValueMatches(NSDictionary *event, NSString *query) { + if (query.length == 0) { + return YES; + } + NSString *needle = query.lowercaseString ?: @""; + for (NSString *key in @[@"title", @"calendar_title", @"location", @"notes_preview"]) { + id value = event[key]; + if ([value isKindOfClass:[NSString class]] && + [[(NSString *)value lowercaseString] containsString:needle]) { + return YES; + } + } + return NO; +} + +static NSDictionary *OPCalendarNormalizedFixtureEvent(NSDictionary *event, NSUInteger index) { + NSString *title = OPStringFromRequest(event, @"title", + OPStringFromRequest(event, @"summary", @"")); + if (title.length == 0) { + title = [NSString stringWithFormat:@"Calendar event %lu", (unsigned long)index + 1]; + } + NSString *notes = OPStringFromRequest(event, @"notes", + OPStringFromRequest(event, @"description", @"")); + BOOL notesTruncated = NO; + NSString *notesPreview = OPClipboardLimitedText(notes, 240, ¬esTruncated); + long long startMs = OPCalendarLongLongFromValue(event[@"start_at_ms"] ?: event[@"start_ms"], 0); + long long endMs = OPCalendarLongLongFromValue(event[@"end_at_ms"] ?: event[@"end_ms"], 0); + return @{ + @"event_id": OPStringFromRequest(event, @"event_id", + [NSString stringWithFormat:@"fixture-calendar-event-%lu", (unsigned long)index + 1]), + @"title": title ?: @"", + @"calendar_title": OPStringFromRequest(event, @"calendar_title", + OPStringFromRequest(event, @"calendar", @"")), + @"location": OPStringFromRequest(event, @"location", @""), + @"notes_preview": notesPreview ?: @"", + @"notes_truncated": @(notesTruncated), + @"start_at_ms": @(startMs), + @"start_at": OPCalendarISO8601FromMs(startMs), + @"end_at_ms": @(endMs), + @"end_at": OPCalendarISO8601FromMs(endMs), + @"all_day": @(OPBoolFromRequest(event, @"all_day", NO)), + @"provider": @"openphone.calendar_fixture_file" + }; +} + +static NSArray *OPCalendarFixtureSearch(NSString *query, NSUInteger limit, + long long startAtMs, long long endAtMs) { + NSDictionary *fixture = OPReadJSONFile(OPCalendarFixturePath()); + NSArray *events = [fixture[@"events"] isKindOfClass:[NSArray class]] + ? fixture[@"events"] : @[]; + NSMutableArray *results = [NSMutableArray array]; + NSUInteger index = 0; + for (id value in events) { + if (![value isKindOfClass:[NSDictionary class]]) { + index++; + continue; + } + NSDictionary *event = OPCalendarNormalizedFixtureEvent((NSDictionary *)value, index); + index++; + long long eventStart = OPCalendarLongLongFromValue(event[@"start_at_ms"], 0); + if (startAtMs > 0 && eventStart > 0 && eventStart < startAtMs) { + continue; + } + if (endAtMs > 0 && eventStart > 0 && eventStart > endAtMs) { + continue; + } + if (!OPCalendarValueMatches(event, query)) { + continue; + } + [results addObject:event]; + if (results.count >= limit) { + break; + } + } + return results; +} + +static NSArray *OPCalendarSystemSearchAtPath(NSString *path, NSString *query, + NSUInteger limit, long long startAtMs, long long endAtMs, NSString **errorOut) { + sqlite3 *db = NULL; + int rc = sqlite3_open_v2(path.UTF8String, &db, + SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, NULL); + if (rc != SQLITE_OK) { + if (errorOut) { + *errorOut = db ? [NSString stringWithUTF8String:sqlite3_errmsg(db)] : @"sqlite_open_failed"; + } + if (db) { + sqlite3_close(db); + } + return nil; + } + sqlite3_busy_timeout(db, 5000); + if (!OPSQLiteTableExists(db, @"CalendarItem")) { + if (errorOut) { + *errorOut = @"calendaritem_table_missing"; + } + sqlite3_close(db); + return nil; + } + + NSString *calendarIdColumn = OPCalendarFirstExistingColumn(db, @"CalendarItem", + @[@"calendar_id", @"calendarID", @"calendar"]); + NSString *locationIdColumn = OPCalendarFirstExistingColumn(db, @"CalendarItem", + @[@"location_id", @"locationID", @"structured_location_id"]); + BOOL hasCalendarJoin = calendarIdColumn.length > 0 && OPSQLiteTableExists(db, @"Calendar"); + BOOL hasLocationJoin = locationIdColumn.length > 0 && OPSQLiteTableExists(db, @"Location"); + + NSString *titleExpr = OPCalendarTextExpr(db, @"CalendarItem", @"i", + @[@"summary", @"title", @"name"]); + NSString *directLocationExpr = OPCalendarTextExpr(db, @"CalendarItem", @"i", + @[@"location", @"location_title", @"locationTitle"]); + NSString *notesExpr = OPCalendarTextExpr(db, @"CalendarItem", @"i", + @[@"description", @"notes", @"comment"]); + NSString *startExpr = OPCalendarNumberExpr(db, @"CalendarItem", @"i", + @[@"start_date", @"startDate", @"start_time", @"startTime"]); + NSString *endExpr = OPCalendarNumberExpr(db, @"CalendarItem", @"i", + @[@"end_date", @"endDate", @"end_time", @"endTime"]); + NSString *allDayExpr = OPCalendarNumberExpr(db, @"CalendarItem", @"i", + @[@"all_day", @"allDay", @"is_all_day", @"isAllDay"]); + NSString *calendarTitleExpr = hasCalendarJoin + ? OPCalendarTextExpr(db, @"Calendar", @"c", @[@"title", @"displayName", @"summary", @"name"]) + : @"''"; + NSString *joinedLocationExpr = hasLocationJoin + ? OPCalendarTextExpr(db, @"Location", @"l", @[@"title", @"address", @"displayName", @"location"]) + : @"''"; + NSString *locationExpr = [directLocationExpr isEqualToString:@"''"] + ? joinedLocationExpr + : [NSString stringWithFormat:@"trim(%@ || ' ' || %@)", directLocationExpr, joinedLocationExpr]; + + NSMutableString *from = [NSMutableString stringWithString:@"FROM CalendarItem i"]; + if (hasCalendarJoin) { + [from appendFormat:@" LEFT JOIN Calendar c ON c.ROWID = i.%@", calendarIdColumn]; + } + if (hasLocationJoin) { + [from appendFormat:@" LEFT JOIN Location l ON l.ROWID = i.%@", locationIdColumn]; + } + + NSMutableArray *whereParts = [NSMutableArray array]; + BOOL hasQuery = query.length > 0; + if (hasQuery) { + [whereParts addObject:[NSString stringWithFormat: + @"lower(%@ || ' ' || %@ || ' ' || %@ || ' ' || %@) LIKE ?", + titleExpr, locationExpr, notesExpr, calendarTitleExpr]]; + } + if (startAtMs > 0 && ![startExpr isEqualToString:@"0"]) { + [whereParts addObject:[NSString stringWithFormat:@"%@ >= ?", startExpr]]; + } + if (endAtMs > 0 && ![startExpr isEqualToString:@"0"]) { + [whereParts addObject:[NSString stringWithFormat:@"%@ <= ?", startExpr]]; + } + NSString *where = whereParts.count > 0 + ? [NSString stringWithFormat:@"WHERE %@", [whereParts componentsJoinedByString:@" AND "]] + : @""; + NSString *orderBy = [startExpr isEqualToString:@"0"] ? @"i.ROWID DESC" : [NSString stringWithFormat:@"%@ ASC", startExpr]; + NSString *sql = [NSString stringWithFormat: + @"SELECT i.ROWID, %@, %@, %@, %@, %@, %@, %@ " + "%@ %@ ORDER BY %@ LIMIT ?", + titleExpr, locationExpr, notesExpr, startExpr, endExpr, allDayExpr, + calendarTitleExpr, from, where, orderBy]; + + sqlite3_stmt *statement = NULL; + NSMutableArray *events = [NSMutableArray array]; + rc = sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL); + if (rc == SQLITE_OK) { + int bindIndex = 1; + if (hasQuery) { + NSString *like = [NSString stringWithFormat:@"%%%@%%", query.lowercaseString ?: @""]; + OPSQLiteBindText(statement, bindIndex++, like); + } + if (startAtMs > 0 && ![startExpr isEqualToString:@"0"]) { + sqlite3_bind_double(statement, bindIndex++, OPCalendarSQLiteValueFromUnixMs(startAtMs)); + } + if (endAtMs > 0 && ![startExpr isEqualToString:@"0"]) { + sqlite3_bind_double(statement, bindIndex++, OPCalendarSQLiteValueFromUnixMs(endAtMs)); + } + sqlite3_bind_int64(statement, bindIndex, (sqlite3_int64)limit); + while ((rc = sqlite3_step(statement)) == SQLITE_ROW) { + long long rowId = sqlite3_column_int64(statement, 0); + NSString *title = OPSQLiteColumnString(statement, 1); + NSString *location = OPSQLiteColumnString(statement, 2); + NSString *notes = OPSQLiteColumnString(statement, 3); + long long startMs = OPCalendarMsFromSQLiteValue(sqlite3_column_double(statement, 4)); + long long endMs = OPCalendarMsFromSQLiteValue(sqlite3_column_double(statement, 5)); + BOOL allDay = sqlite3_column_int64(statement, 6) != 0; + NSString *calendarTitle = OPSQLiteColumnString(statement, 7); + BOOL notesTruncated = NO; + NSString *notesPreview = OPClipboardLimitedText(notes, 240, ¬esTruncated); + [events addObject:@{ + @"event_id": [NSString stringWithFormat:@"ios-calendar-event-%lld", rowId], + @"row_id": @(rowId), + @"title": title ?: @"", + @"calendar_title": calendarTitle ?: @"", + @"location": location ?: @"", + @"notes_preview": notesPreview ?: @"", + @"notes_truncated": @(notesTruncated), + @"start_at_ms": @(startMs), + @"start_at": OPCalendarISO8601FromMs(startMs), + @"end_at_ms": @(endMs), + @"end_at": OPCalendarISO8601FromMs(endMs), + @"all_day": @(allDay), + @"provider": @"Calendar.sqlitedb" + }]; + } + if (rc != SQLITE_DONE && errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"calendar_query_failed"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"calendar_query_prepare_failed"; + } + sqlite3_finalize(statement); + sqlite3_close(db); + if (rc != SQLITE_DONE && events.count == 0 && errorOut && *errorOut) { + return nil; + } + return events; +} + +static NSArray *OPCalendarSystemSearch(NSString *query, NSUInteger limit, + long long startAtMs, long long endAtMs, NSString **errorOut) { + NSMutableArray *errors = [NSMutableArray array]; + for (NSString *path in OPCalendarDatabasePaths()) { + NSString *pathError = nil; + NSArray *events = OPCalendarSystemSearchAtPath(path, query, limit, + startAtMs, endAtMs, &pathError); + if (events) { + return events; + } + if (pathError.length > 0) { + [errors addObject:[NSString stringWithFormat:@"%@:%@", path.lastPathComponent ?: @"calendar", pathError]]; + } + } + if (errorOut) { + *errorOut = errors.count > 0 ? [errors componentsJoinedByString:@";"] : @"calendar_query_failed"; + } + return nil; +} + +static NSArray *OPCalendarSummaryEvents(NSArray *events, NSUInteger limit) { + NSMutableArray *summary = [NSMutableArray array]; + for (NSDictionary *event in events) { + NSMutableDictionary *item = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"event_id", @"title", @"calendar_title", @"location", + @"start_at_ms", @"start_at", @"end_at_ms", @"end_at", @"all_day"]) { + id value = event[key]; + if (value) { + item[key] = value; + } + } + NSString *notesPreview = [event[@"notes_preview"] isKindOfClass:[NSString class]] + ? event[@"notes_preview"] : @""; + if (notesPreview.length > 0) { + item[@"notes_preview"] = notesPreview; + item[@"notes_truncated"] = event[@"notes_truncated"] ?: @NO; + } + if (item.count > 0) { + [summary addObject:item]; + } + if (summary.count >= limit) { + break; + } + } + return summary; +} + +static NSDictionary *OPCalendarTraceSummary(NSDictionary *result) { + if (![result isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSArray *events = [result[@"events"] isKindOfClass:[NSArray class]] + ? result[@"events"] : @[]; + return @{ + @"status": result[@"status"] ?: @"unknown", + @"tool": @"calendar_search", + @"provider": result[@"provider"] ?: @"unknown", + @"query_length": result[@"query_length"] ?: @0, + @"query_sha256": result[@"query_sha256"] ?: @"", + @"start_at_ms": result[@"start_at_ms"] ?: @0, + @"end_at_ms": result[@"end_at_ms"] ?: @0, + @"count": result[@"count"] ?: @0, + @"limit": result[@"limit"] ?: @0, + @"events": OPCalendarSummaryEvents(events, 8), + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPCalendarSearch(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *reason = OPStringFromRequest(request, @"reason", @""); + NSString *query = OPStringFromRequest(request, @"query", + OPStringFromRequest(request, @"title", @"")); + query = [query stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + long long startAtMs = OPCalendarLongLongFromValue(request[@"start_at_ms"] ?: request[@"start_ms"], 0); + long long endAtMs = OPCalendarLongLongFromValue(request[@"end_at_ms"] ?: request[@"end_ms"], 0); + BOOL allowEmptyQuery = OPBoolFromRequest(request, @"allow_empty_query", NO); + if (query.length == 0 && startAtMs <= 0 && endAtMs <= 0 && !allowEmptyQuery) { + return OPError(@"missing_calendar_query_or_range"); + } + NSUInteger limit = OPLimitFromRequest(request, 10, 50); + if (limit == 0) { + limit = 10; + } + + NSString *provider = @""; + NSString *systemError = nil; + NSArray *events = nil; + BOOL calendarAvailable = OPPathExists(OPCalendarDatabasePath()); + events = OPCalendarSystemSearch(query, limit, startAtMs, endAtMs, &systemError); + if (events) { + provider = @"Calendar.sqlitedb"; + } + if (!events && !OPProtectedDataHelperRole()) { + NSMutableDictionary *helperRequest = [request mutableCopy] ?: [NSMutableDictionary dictionary]; + helperRequest[@"command"] = @"calendar_search"; + helperRequest[@"task_id"] = @""; + NSDictionary *helperResult = OPProtectedDataHelperRequest(helperRequest); + if ([helperResult[@"status"] isEqualToString:@"ok"] && + [helperResult[@"events"] isKindOfClass:[NSArray class]]) { + events = helperResult[@"events"]; + provider = [helperResult[@"provider"] isKindOfClass:[NSString class]] + ? helperResult[@"provider"] : @"Calendar.sqlitedb"; + } else if ([helperResult[@"reason"] isKindOfClass:[NSString class]]) { + systemError = systemError.length > 0 + ? [NSString stringWithFormat:@"%@;protected_helper:%@", systemError, helperResult[@"reason"]] + : [NSString stringWithFormat:@"protected_helper:%@", helperResult[@"reason"]]; + } + } + if (!events) { + if (OPPathExists(OPCalendarFixturePath())) { + events = OPCalendarFixtureSearch(query, limit, startAtMs, endAtMs); + provider = @"openphone.calendar_fixture_file"; + } else { + NSString *reasonText = (calendarAvailable || systemError.length > 0) + ? [NSString stringWithFormat:@"calendar_system_query_failed:%@", systemError ?: @"unknown"] + : @"calendar_provider_unavailable"; + return OPError(reasonText); + } + } + + NSString *queryHash = OPClipboardTextHash(query ?: @""); + long long contextId = OPRecordContextEvent(@"calendar_searched", @"openphone.agentd", taskId, + @"Calendar search", + [NSString stringWithFormat:@"Calendar search returned %lu event(s)", + (unsigned long)events.count], + @{ + @"provider": provider ?: @"unknown", + @"count": @(events.count), + @"limit": @(limit), + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"end_at_ms": @(endAtMs), + @"reason": reason ?: @"" + }); + NSDictionary *result = @{ + @"status": @"ok", + @"tool": @"calendar_search", + @"query": query ?: @"", + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"start_at": OPCalendarISO8601FromMs(startAtMs), + @"end_at_ms": @(endAtMs), + @"end_at": OPCalendarISO8601FromMs(endAtMs), + @"limit": @(limit), + @"events": events ?: @[], + @"count": @(events.count), + @"provider": provider ?: @"unknown", + @"calendar_path": OPCalendarDatabasePath(), + @"fixture_path": OPCalendarFixturePath(), + @"context_event_id": @(contextId), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"calendar_searched", taskId, @"calendar.read", @"allow_task_scoped", + @{ + @"reason": reason ?: @"", + @"provider": provider ?: @"unknown", + @"limit": @(limit), + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"end_at_ms": @(endAtMs), + @"count": @(events.count) + }, + [NSString stringWithFormat:@"count:%lu", (unsigned long)events.count]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"calendar_search", + @"arguments": @{ + @"reason": reason ?: @"", + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"end_at_ms": @(endAtMs), + @"limit": @(limit) + }, + @"result": OPCalendarTraceSummary(result) + }); + return result; +} + +static NSArray *OPCallsDatabasePaths(void) { + const char *override = getenv("OPENPHONE_CALLS_DB_PATH"); + if (override && override[0] != '\0') { + return @[[NSString stringWithUTF8String:override]]; + } + return @[ + @"/var/mobile/Library/CallHistoryDB/CallHistory.storedata", + @"/private/var/mobile/Library/CallHistoryDB/CallHistory.storedata" + ]; +} + +static NSString *OPCallsDatabasePath(void) { + NSArray *paths = OPCallsDatabasePaths(); + for (NSString *path in paths) { + if (OPPathExists(path)) { + return path; + } + } + return paths.count > 0 ? paths[0] : @"/var/mobile/Library/CallHistoryDB/CallHistory.storedata"; +} + +static NSDictionary *OPCallsProviderStatus(void) { + NSString *callsPath = OPCallsDatabasePath(); + BOOL callHistoryAvailable = OPPathExists(callsPath); + BOOL fixtureAvailable = OPPathExists(OPCallsFixturePath()); + return @{ + @"status": @"implemented_partial", + @"provider": @"CallHistory.storedata", + @"path": callsPath ?: @"", + @"call_history_available": @(callHistoryAvailable), + @"fixture_provider": @"openphone.calls_fixture_file", + @"fixture_path": OPCallsFixturePath(), + @"fixture_available": @(fixtureAvailable), + @"protected_helper_socket": OPProtectedDataHelperSocketPath(), + @"protected_helper_available": @(OPPathExists(OPProtectedDataHelperSocketPath())), + @"protected_helper_manager": @"launchd_sh_wrapper_setuid_helper", + @"protected_helper_binary": OPProtectedDataHelperBinaryPath() ?: @"", + @"protected_helper_spawn_pid": @((int)OPProtectedDataHelperPid), + @"protected_helper_last_spawn_ms": @((long long)OPProtectedDataHelperLastSpawnMs), + @"protected_helper_last_spawn_error": OPProtectedDataHelperLastSpawnError ?: @"", + @"access": @"read_only", + @"retention": @"call summaries are returned to caller; context/audit store counts, query hashes, and time ranges only" + }; +} + +static NSString *OPCallsTableName(sqlite3 *db) { + for (NSString *table in @[@"ZCALLRECORD", @"CallRecord", @"call_history", @"calls"]) { + if (OPSQLiteTableExists(db, table)) { + return table; + } + } + return @""; +} + +static NSString *OPCallsTextExpr(sqlite3 *db, NSString *table, NSString *alias, + NSArray *columns) { + NSString *column = OPCalendarFirstExistingColumn(db, table, columns); + if (column.length == 0) { + return @"''"; + } + return [NSString stringWithFormat:@"COALESCE(%@.%@, '')", alias, column]; +} + +static NSString *OPCallsNumberExpr(sqlite3 *db, NSString *table, NSString *alias, + NSArray *columns) { + NSString *column = OPCalendarFirstExistingColumn(db, table, columns); + if (column.length == 0) { + return @"0"; + } + return [NSString stringWithFormat:@"COALESCE(%@.%@, 0)", alias, column]; +} + +static NSString *OPCallsDirectionFromOriginated(long long originated) { + if (originated == 1) { + return @"outgoing"; + } + if (originated == 0) { + return @"incoming"; + } + return @"unknown"; +} + +static BOOL OPCallsValueMatches(NSDictionary *call, NSString *query) { + if (query.length == 0) { + return YES; + } + NSString *needle = query.lowercaseString ?: @""; + for (NSString *key in @[@"address", @"display_name", @"service", @"direction", @"call_type"]) { + id value = call[key]; + if ([value isKindOfClass:[NSString class]] && + [[(NSString *)value lowercaseString] containsString:needle]) { + return YES; + } + } + return NO; +} + +static NSDictionary *OPCallsNormalizedFixtureCall(NSDictionary *call, NSUInteger index) { + long long startMs = OPCalendarLongLongFromValue(call[@"start_at_ms"] ?: call[@"start_ms"], 0); + long long durationSeconds = OPCalendarLongLongFromValue( + call[@"duration_seconds"] ?: call[@"duration"], 0); + NSString *direction = OPStringFromRequest(call, @"direction", @"unknown"); + if (direction.length == 0) { + direction = @"unknown"; + } + return @{ + @"call_id": OPStringFromRequest(call, @"call_id", + [NSString stringWithFormat:@"fixture-call-%lu", (unsigned long)index + 1]), + @"address": OPStringFromRequest(call, @"address", + OPStringFromRequest(call, @"phone_number", @"")), + @"display_name": OPStringFromRequest(call, @"display_name", + OPStringFromRequest(call, @"name", @"")), + @"service": OPStringFromRequest(call, @"service", @""), + @"direction": direction ?: @"unknown", + @"answered": @(OPBoolFromRequest(call, @"answered", YES)), + @"duration_seconds": @(durationSeconds), + @"start_at_ms": @(startMs), + @"start_at": OPCalendarISO8601FromMs(startMs), + @"call_type": OPStringFromRequest(call, @"call_type", @""), + @"provider": @"openphone.calls_fixture_file" + }; +} + +static NSArray *OPCallsFixtureSearch(NSString *query, NSUInteger limit, + long long startAtMs, long long endAtMs) { + NSDictionary *fixture = OPReadJSONFile(OPCallsFixturePath()); + NSArray *calls = [fixture[@"calls"] isKindOfClass:[NSArray class]] + ? fixture[@"calls"] : @[]; + NSMutableArray *results = [NSMutableArray array]; + NSUInteger index = 0; + for (id value in calls) { + if (![value isKindOfClass:[NSDictionary class]]) { + index++; + continue; + } + NSDictionary *call = OPCallsNormalizedFixtureCall((NSDictionary *)value, index); + index++; + long long callStart = OPCalendarLongLongFromValue(call[@"start_at_ms"], 0); + if (startAtMs > 0 && callStart > 0 && callStart < startAtMs) { + continue; + } + if (endAtMs > 0 && callStart > 0 && callStart > endAtMs) { + continue; + } + if (!OPCallsValueMatches(call, query)) { + continue; + } + [results addObject:call]; + if (results.count >= limit) { + break; + } + } + return results; +} + +static NSArray *OPCallsSystemSearchAtPath(NSString *path, NSString *query, + NSUInteger limit, long long startAtMs, long long endAtMs, NSString **errorOut) { + sqlite3 *db = NULL; + int rc = sqlite3_open_v2(path.UTF8String, &db, + SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, NULL); + if (rc != SQLITE_OK) { + if (errorOut) { + *errorOut = db ? [NSString stringWithUTF8String:sqlite3_errmsg(db)] : @"sqlite_open_failed"; + } + if (db) { + sqlite3_close(db); + } + return nil; + } + sqlite3_busy_timeout(db, 5000); + NSString *table = OPCallsTableName(db); + if (table.length == 0) { + if (errorOut) { + *errorOut = @"call_table_missing"; + } + sqlite3_close(db); + return nil; + } + + NSString *addressExpr = OPCallsTextExpr(db, table, @"c", + @[@"ZADDRESS", @"address", @"phone_number", @"number", @"sender"]); + NSString *displayNameExpr = OPCallsTextExpr(db, table, @"c", + @[@"ZNAME", @"ZDISPLAYNAME", @"display_name", @"name", @"caller_name"]); + NSString *serviceExpr = OPCallsTextExpr(db, table, @"c", + @[@"ZSERVICE_PROVIDER", @"ZSERVICE", @"service", @"provider"]); + NSString *startExpr = OPCallsNumberExpr(db, table, @"c", + @[@"ZDATE", @"date", @"timestamp", @"start_date", @"start_time"]); + NSString *durationExpr = OPCallsNumberExpr(db, table, @"c", + @[@"ZDURATION", @"duration", @"duration_seconds"]); + NSString *originatedExpr = OPCallsNumberExpr(db, table, @"c", + @[@"ZORIGINATED", @"originated", @"outgoing", @"is_outgoing"]); + NSString *answeredExpr = OPCallsNumberExpr(db, table, @"c", + @[@"ZANSWERED", @"answered", @"is_answered"]); + NSString *callTypeExpr = OPCallsTextExpr(db, table, @"c", + @[@"ZCALLTYPE", @"call_type", @"type"]); + + NSMutableArray *whereParts = [NSMutableArray array]; + BOOL hasQuery = query.length > 0; + if (hasQuery) { + [whereParts addObject:[NSString stringWithFormat: + @"lower(%@ || ' ' || %@ || ' ' || %@ || ' ' || %@) LIKE ?", + addressExpr, displayNameExpr, serviceExpr, callTypeExpr]]; + } + if (startAtMs > 0 && ![startExpr isEqualToString:@"0"]) { + [whereParts addObject:[NSString stringWithFormat:@"%@ >= ?", startExpr]]; + } + if (endAtMs > 0 && ![startExpr isEqualToString:@"0"]) { + [whereParts addObject:[NSString stringWithFormat:@"%@ <= ?", startExpr]]; + } + NSString *where = whereParts.count > 0 + ? [NSString stringWithFormat:@"WHERE %@", [whereParts componentsJoinedByString:@" AND "]] + : @""; + NSString *orderBy = [startExpr isEqualToString:@"0"] ? @"c.ROWID DESC" : [NSString stringWithFormat:@"%@ DESC", startExpr]; + NSString *sql = [NSString stringWithFormat: + @"SELECT c.ROWID, %@, %@, %@, %@, %@, %@, %@, %@ " + "FROM %@ c %@ ORDER BY %@ LIMIT ?", + addressExpr, displayNameExpr, serviceExpr, startExpr, durationExpr, + originatedExpr, answeredExpr, callTypeExpr, table, where, orderBy]; + + sqlite3_stmt *statement = NULL; + NSMutableArray *calls = [NSMutableArray array]; + rc = sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL); + if (rc == SQLITE_OK) { + int bindIndex = 1; + if (hasQuery) { + NSString *like = [NSString stringWithFormat:@"%%%@%%", query.lowercaseString ?: @""]; + OPSQLiteBindText(statement, bindIndex++, like); + } + if (startAtMs > 0 && ![startExpr isEqualToString:@"0"]) { + sqlite3_bind_double(statement, bindIndex++, OPCalendarSQLiteValueFromUnixMs(startAtMs)); + } + if (endAtMs > 0 && ![startExpr isEqualToString:@"0"]) { + sqlite3_bind_double(statement, bindIndex++, OPCalendarSQLiteValueFromUnixMs(endAtMs)); + } + sqlite3_bind_int64(statement, bindIndex, (sqlite3_int64)limit); + while ((rc = sqlite3_step(statement)) == SQLITE_ROW) { + long long rowId = sqlite3_column_int64(statement, 0); + NSString *address = OPSQLiteColumnString(statement, 1); + NSString *displayName = OPSQLiteColumnString(statement, 2); + NSString *service = OPSQLiteColumnString(statement, 3); + long long startMs = OPCalendarMsFromSQLiteValue(sqlite3_column_double(statement, 4)); + long long durationSeconds = sqlite3_column_int64(statement, 5); + long long originated = sqlite3_column_int64(statement, 6); + BOOL answered = sqlite3_column_int64(statement, 7) != 0; + NSString *callType = OPSQLiteColumnString(statement, 8); + [calls addObject:@{ + @"call_id": [NSString stringWithFormat:@"ios-call-%lld", rowId], + @"row_id": @(rowId), + @"address": address ?: @"", + @"display_name": displayName ?: @"", + @"service": service ?: @"", + @"direction": OPCallsDirectionFromOriginated(originated), + @"answered": @(answered), + @"duration_seconds": @(durationSeconds), + @"start_at_ms": @(startMs), + @"start_at": OPCalendarISO8601FromMs(startMs), + @"call_type": callType ?: @"", + @"provider": @"CallHistory.storedata" + }]; + } + if (rc != SQLITE_DONE && errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"calls_query_failed"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"calls_query_prepare_failed"; + } + sqlite3_finalize(statement); + sqlite3_close(db); + if (rc != SQLITE_DONE && calls.count == 0 && errorOut && *errorOut) { + return nil; + } + return calls; +} + +static NSArray *OPCallsSystemSearch(NSString *query, NSUInteger limit, + long long startAtMs, long long endAtMs, NSString **errorOut) { + NSMutableArray *errors = [NSMutableArray array]; + for (NSString *path in OPCallsDatabasePaths()) { + NSString *pathError = nil; + NSArray *calls = OPCallsSystemSearchAtPath(path, query, limit, + startAtMs, endAtMs, &pathError); + if (calls) { + return calls; + } + if (pathError.length > 0) { + [errors addObject:[NSString stringWithFormat:@"%@:%@", path.lastPathComponent ?: @"calls", pathError]]; + } + } + if (errorOut) { + *errorOut = errors.count > 0 ? [errors componentsJoinedByString:@";"] : @"calls_query_failed"; + } + return nil; +} + +static NSArray *OPCallsSummaryCalls(NSArray *calls, NSUInteger limit) { + NSMutableArray *summary = [NSMutableArray array]; + for (NSDictionary *call in calls) { + NSMutableDictionary *item = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"call_id", @"address", @"display_name", @"service", @"direction", + @"answered", @"duration_seconds", @"start_at_ms", @"start_at", @"call_type"]) { + id value = call[key]; + if (value) { + item[key] = value; + } + } + if (item.count > 0) { + [summary addObject:item]; + } + if (summary.count >= limit) { + break; + } + } + return summary; +} + +static NSDictionary *OPCallsTraceSummary(NSDictionary *result) { + if (![result isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSArray *calls = [result[@"calls"] isKindOfClass:[NSArray class]] + ? result[@"calls"] : @[]; + return @{ + @"status": result[@"status"] ?: @"unknown", + @"tool": @"calls_search", + @"provider": result[@"provider"] ?: @"unknown", + @"query_length": result[@"query_length"] ?: @0, + @"query_sha256": result[@"query_sha256"] ?: @"", + @"start_at_ms": result[@"start_at_ms"] ?: @0, + @"end_at_ms": result[@"end_at_ms"] ?: @0, + @"count": result[@"count"] ?: @0, + @"limit": result[@"limit"] ?: @0, + @"calls": OPCallsSummaryCalls(calls, 8), + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPCallsSearch(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *reason = OPStringFromRequest(request, @"reason", @""); + NSString *query = OPStringFromRequest(request, @"query", + OPStringFromRequest(request, @"phone_number", + OPStringFromRequest(request, @"address", @""))); + query = [query stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + long long startAtMs = OPCalendarLongLongFromValue(request[@"start_at_ms"] ?: request[@"start_ms"], 0); + long long endAtMs = OPCalendarLongLongFromValue(request[@"end_at_ms"] ?: request[@"end_ms"], 0); + BOOL allowEmptyQuery = OPBoolFromRequest(request, @"allow_empty_query", NO); + if (query.length == 0 && startAtMs <= 0 && endAtMs <= 0 && !allowEmptyQuery) { + return OPError(@"missing_calls_query_or_range"); + } + NSUInteger limit = OPLimitFromRequest(request, 10, 50); + if (limit == 0) { + limit = 10; + } + + NSString *provider = @""; + NSString *systemError = nil; + NSArray *calls = nil; + BOOL callHistoryAvailable = OPPathExists(OPCallsDatabasePath()); + calls = OPCallsSystemSearch(query, limit, startAtMs, endAtMs, &systemError); + if (calls) { + provider = @"CallHistory.storedata"; + } + if (!calls && !OPProtectedDataHelperRole()) { + NSMutableDictionary *helperRequest = [request mutableCopy] ?: [NSMutableDictionary dictionary]; + helperRequest[@"command"] = @"calls_search"; + helperRequest[@"task_id"] = @""; + NSDictionary *helperResult = OPProtectedDataHelperRequest(helperRequest); + if ([helperResult[@"status"] isEqualToString:@"ok"] && + [helperResult[@"calls"] isKindOfClass:[NSArray class]]) { + calls = helperResult[@"calls"]; + provider = [helperResult[@"provider"] isKindOfClass:[NSString class]] + ? helperResult[@"provider"] : @"CallHistory.storedata"; + } else if ([helperResult[@"reason"] isKindOfClass:[NSString class]]) { + systemError = systemError.length > 0 + ? [NSString stringWithFormat:@"%@;protected_helper:%@", systemError, helperResult[@"reason"]] + : [NSString stringWithFormat:@"protected_helper:%@", helperResult[@"reason"]]; + } + } + if (!calls) { + if (OPPathExists(OPCallsFixturePath())) { + calls = OPCallsFixtureSearch(query, limit, startAtMs, endAtMs); + provider = @"openphone.calls_fixture_file"; + } else { + NSString *reasonText = (callHistoryAvailable || systemError.length > 0) + ? [NSString stringWithFormat:@"calls_system_query_failed:%@", systemError ?: @"unknown"] + : @"calls_provider_unavailable"; + return OPError(reasonText); + } + } + + NSString *queryHash = OPClipboardTextHash(query ?: @""); + long long contextId = OPRecordContextEvent(@"calls_searched", @"openphone.agentd", taskId, + @"Calls search", + [NSString stringWithFormat:@"Calls search returned %lu call(s)", + (unsigned long)calls.count], + @{ + @"provider": provider ?: @"unknown", + @"count": @(calls.count), + @"limit": @(limit), + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"end_at_ms": @(endAtMs), + @"reason": reason ?: @"" + }); + NSDictionary *result = @{ + @"status": @"ok", + @"tool": @"calls_search", + @"query": query ?: @"", + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"start_at": OPCalendarISO8601FromMs(startAtMs), + @"end_at_ms": @(endAtMs), + @"end_at": OPCalendarISO8601FromMs(endAtMs), + @"limit": @(limit), + @"calls": calls ?: @[], + @"count": @(calls.count), + @"provider": provider ?: @"unknown", + @"calls_path": OPCallsDatabasePath(), + @"fixture_path": OPCallsFixturePath(), + @"context_event_id": @(contextId), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"calls_searched", taskId, @"calls.read", @"allow_task_scoped", + @{ + @"reason": reason ?: @"", + @"provider": provider ?: @"unknown", + @"limit": @(limit), + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"end_at_ms": @(endAtMs), + @"count": @(calls.count) + }, + [NSString stringWithFormat:@"count:%lu", (unsigned long)calls.count]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"calls_search", + @"arguments": @{ + @"reason": reason ?: @"", + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"end_at_ms": @(endAtMs), + @"limit": @(limit) + }, + @"result": OPCallsTraceSummary(result) + }); + return result; +} + +static NSArray *OPMessagesDatabasePaths(void) { + const char *override = getenv("OPENPHONE_MESSAGES_DB_PATH"); + if (override && override[0] != '\0') { + return @[[NSString stringWithUTF8String:override]]; + } + return @[ + @"/var/mobile/Library/SMS/sms.db", + @"/private/var/mobile/Library/SMS/sms.db" + ]; +} + +static NSString *OPMessagesDatabasePath(void) { + NSArray *paths = OPMessagesDatabasePaths(); + for (NSString *path in paths) { + if (OPPathExists(path)) { + return path; + } + } + return paths.count > 0 ? paths[0] : @"/var/mobile/Library/SMS/sms.db"; +} + +static NSDictionary *OPMessagesProviderStatus(void) { + NSString *messagesPath = OPMessagesDatabasePath(); + BOOL smsAvailable = OPPathExists(messagesPath); + BOOL fixtureAvailable = OPPathExists(OPMessagesFixturePath()); + return @{ + @"status": @"implemented_partial", + @"provider": @"SMS.sqlite", + @"path": messagesPath ?: @"", + @"sms_available": @(smsAvailable), + @"fixture_provider": @"openphone.messages_fixture_file", + @"fixture_path": OPMessagesFixturePath(), + @"fixture_available": @(fixtureAvailable), + @"protected_helper_socket": OPProtectedDataHelperSocketPath(), + @"protected_helper_available": @(OPPathExists(OPProtectedDataHelperSocketPath())), + @"protected_helper_manager": @"launchd_sh_wrapper_setuid_helper", + @"protected_helper_binary": OPProtectedDataHelperBinaryPath() ?: @"", + @"protected_helper_spawn_pid": @((int)OPProtectedDataHelperPid), + @"protected_helper_last_spawn_ms": @((long long)OPProtectedDataHelperLastSpawnMs), + @"protected_helper_last_spawn_error": OPProtectedDataHelperLastSpawnError ?: @"", + @"access": @"read_only", + @"retention": @"bounded message previews are returned to caller; context/audit store counts, query hashes, and time ranges only" + }; +} + +static NSString *OPMessagesTextExpr(sqlite3 *db, NSString *table, NSString *alias, + NSArray *columns) { + NSString *column = OPCalendarFirstExistingColumn(db, table, columns); + if (column.length == 0) { + return @"''"; + } + return [NSString stringWithFormat:@"COALESCE(%@.%@, '')", alias, column]; +} + +static NSString *OPMessagesNumberExpr(sqlite3 *db, NSString *table, NSString *alias, + NSArray *columns) { + NSString *column = OPCalendarFirstExistingColumn(db, table, columns); + if (column.length == 0) { + return @"0"; + } + return [NSString stringWithFormat:@"COALESCE(%@.%@, 0)", alias, column]; +} + +static NSString *OPMessagesDirectionFromIsFromMe(long long isFromMe) { + if (isFromMe == 1) { + return @"outgoing"; + } + if (isFromMe == 0) { + return @"incoming"; + } + return @"unknown"; +} + +static long long OPMessagesMsFromSQLiteValue(double value) { + if (value <= 0.0) { + return 0; + } + if (value > 100000000000000000.0) { + return (long long)(value / 1000000.0) + 978307200000LL; + } + if (value > 1000000000000.0) { + return (long long)value; + } + if (value > 1000000000.0) { + return (long long)(value * 1000.0); + } + return (long long)((value + 978307200.0) * 1000.0); +} + +static NSDictionary *OPMessagesPreviewFields(NSString *text, NSString *prefix, NSUInteger maxChars) { + NSString *value = text ?: @""; + BOOL truncated = NO; + NSString *preview = OPClipboardLimitedText(value, maxChars, &truncated); + NSString *hash = value.length > 0 ? OPClipboardTextHash(value) : @""; + return @{ + [NSString stringWithFormat:@"%@_preview", prefix]: preview ?: @"", + [NSString stringWithFormat:@"%@_truncated", prefix]: @(truncated), + [NSString stringWithFormat:@"%@_length", prefix]: @(value.length), + [NSString stringWithFormat:@"%@_sha256", prefix]: hash ?: @"" + }; +} + +static BOOL OPMessagesValueMatches(NSDictionary *message, NSString *query) { + if (query.length == 0) { + return YES; + } + NSString *needle = query.lowercaseString ?: @""; + for (NSString *key in @[@"handle", @"service", @"direction", @"text_preview", @"subject_preview"]) { + id value = message[key]; + if ([value isKindOfClass:[NSString class]] && + [[(NSString *)value lowercaseString] containsString:needle]) { + return YES; + } + } + return NO; +} + +static NSDictionary *OPMessagesNormalizedFixtureMessage(NSDictionary *message, NSUInteger index) { + NSString *text = OPStringFromRequest(message, @"text", + OPStringFromRequest(message, @"body", OPStringFromRequest(message, @"text_preview", @""))); + NSString *subject = OPStringFromRequest(message, @"subject", + OPStringFromRequest(message, @"subject_preview", @"")); + long long sentAtMs = OPCalendarLongLongFromValue(message[@"sent_at_ms"] ?: message[@"date_ms"], 0); + NSString *direction = OPStringFromRequest(message, @"direction", + OPBoolFromRequest(message, @"is_from_me", NO) ? @"outgoing" : @"incoming"); + NSMutableDictionary *result = [NSMutableDictionary dictionaryWithDictionary:@{ + @"message_id": OPStringFromRequest(message, @"message_id", + [NSString stringWithFormat:@"fixture-message-%lu", (unsigned long)index + 1]), + @"guid": OPStringFromRequest(message, @"guid", @""), + @"handle": OPStringFromRequest(message, @"handle", + OPStringFromRequest(message, @"address", @"")), + @"service": OPStringFromRequest(message, @"service", @""), + @"direction": direction.length > 0 ? direction : @"unknown", + @"sent_at_ms": @(sentAtMs), + @"sent_at": OPCalendarISO8601FromMs(sentAtMs), + @"read": @(OPBoolFromRequest(message, @"read", NO)), + @"delivered": @(OPBoolFromRequest(message, @"delivered", NO)), + @"provider": @"openphone.messages_fixture_file" + }]; + [result addEntriesFromDictionary:OPMessagesPreviewFields(text, @"text", 500)]; + [result addEntriesFromDictionary:OPMessagesPreviewFields(subject, @"subject", 240)]; + return result; +} + +static NSArray *OPMessagesFixtureSearch(NSString *query, NSUInteger limit, + long long startAtMs, long long endAtMs) { + NSDictionary *fixture = OPReadJSONFile(OPMessagesFixturePath()); + NSArray *messages = [fixture[@"messages"] isKindOfClass:[NSArray class]] + ? fixture[@"messages"] : @[]; + NSMutableArray *results = [NSMutableArray array]; + NSUInteger index = 0; + for (id value in messages) { + if (![value isKindOfClass:[NSDictionary class]]) { + index++; + continue; + } + NSDictionary *message = OPMessagesNormalizedFixtureMessage((NSDictionary *)value, index); + index++; + long long sentAtMs = OPCalendarLongLongFromValue(message[@"sent_at_ms"], 0); + if (startAtMs > 0 && sentAtMs > 0 && sentAtMs < startAtMs) { + continue; + } + if (endAtMs > 0 && sentAtMs > 0 && sentAtMs > endAtMs) { + continue; + } + if (!OPMessagesValueMatches(message, query)) { + continue; + } + [results addObject:message]; + if (results.count >= limit) { + break; + } + } + return results; +} + +static NSArray *OPMessagesSystemSearchAtPath(NSString *path, NSString *query, + NSUInteger limit, long long startAtMs, long long endAtMs, NSString **errorOut) { + sqlite3 *db = NULL; + int rc = sqlite3_open_v2(path.UTF8String, &db, + SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, NULL); + if (rc != SQLITE_OK) { + if (errorOut) { + *errorOut = db ? [NSString stringWithUTF8String:sqlite3_errmsg(db)] : @"sqlite_open_failed"; + } + if (db) { + sqlite3_close(db); + } + return nil; + } + sqlite3_busy_timeout(db, 5000); + if (!OPSQLiteTableExists(db, @"message")) { + if (errorOut) { + *errorOut = @"message_table_missing"; + } + sqlite3_close(db); + return nil; + } + + BOOL hasHandleJoin = OPSQLiteTableExists(db, @"handle") && + OPSQLiteColumnExists(db, @"message", @"handle_id"); + NSString *guidExpr = OPMessagesTextExpr(db, @"message", @"m", @[@"guid"]); + NSString *textExpr = OPMessagesTextExpr(db, @"message", @"m", @[@"text"]); + NSString *subjectExpr = OPMessagesTextExpr(db, @"message", @"m", @[@"subject"]); + NSString *serviceExpr = OPMessagesTextExpr(db, @"message", @"m", @[@"service"]); + NSString *dateExpr = OPMessagesNumberExpr(db, @"message", @"m", @[@"date"]); + NSString *isFromMeExpr = OPMessagesNumberExpr(db, @"message", @"m", @[@"is_from_me"]); + NSString *readExpr = OPMessagesNumberExpr(db, @"message", @"m", @[@"date_read", @"read"]); + NSString *deliveredExpr = OPMessagesNumberExpr(db, @"message", @"m", @[@"date_delivered", @"delivered"]); + NSString *handleExpr = hasHandleJoin + ? OPMessagesTextExpr(db, @"handle", @"h", @[@"id", @"uncanonicalized_id"]) + : OPMessagesTextExpr(db, @"message", @"m", @[@"handle", @"cache_roomnames", @"address"]); + + NSMutableString *from = [NSMutableString stringWithString:@"FROM message m"]; + if (hasHandleJoin) { + [from appendString:@" LEFT JOIN handle h ON h.ROWID = m.handle_id"]; + } + NSMutableArray *whereParts = [NSMutableArray array]; + BOOL hasQuery = query.length > 0; + if (hasQuery) { + [whereParts addObject:[NSString stringWithFormat: + @"lower(%@ || ' ' || %@ || ' ' || %@ || ' ' || %@) LIKE ?", + textExpr, subjectExpr, handleExpr, serviceExpr]]; + } + NSString *where = whereParts.count > 0 + ? [NSString stringWithFormat:@"WHERE %@", [whereParts componentsJoinedByString:@" AND "]] + : @""; + NSString *orderBy = [dateExpr isEqualToString:@"0"] ? @"m.ROWID DESC" : [NSString stringWithFormat:@"%@ DESC", dateExpr]; + NSUInteger fetchLimit = MIN(MAX(limit * 8, (NSUInteger)50), (NSUInteger)250); + NSString *sql = [NSString stringWithFormat: + @"SELECT m.ROWID, %@, %@, %@, %@, %@, %@, %@, %@, %@ " + "%@ %@ ORDER BY %@ LIMIT ?", + guidExpr, textExpr, subjectExpr, handleExpr, serviceExpr, dateExpr, + isFromMeExpr, readExpr, deliveredExpr, from, where, orderBy]; + + sqlite3_stmt *statement = NULL; + NSMutableArray *messages = [NSMutableArray array]; + rc = sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL); + if (rc == SQLITE_OK) { + int bindIndex = 1; + if (hasQuery) { + NSString *like = [NSString stringWithFormat:@"%%%@%%", query.lowercaseString ?: @""]; + OPSQLiteBindText(statement, bindIndex++, like); + } + sqlite3_bind_int64(statement, bindIndex, (sqlite3_int64)fetchLimit); + while ((rc = sqlite3_step(statement)) == SQLITE_ROW) { + long long rowId = sqlite3_column_int64(statement, 0); + NSString *guid = OPSQLiteColumnString(statement, 1); + NSString *text = OPSQLiteColumnString(statement, 2); + NSString *subject = OPSQLiteColumnString(statement, 3); + NSString *handle = OPSQLiteColumnString(statement, 4); + NSString *service = OPSQLiteColumnString(statement, 5); + long long sentAtMs = OPMessagesMsFromSQLiteValue(sqlite3_column_double(statement, 6)); + if (startAtMs > 0 && sentAtMs > 0 && sentAtMs < startAtMs) { + continue; + } + if (endAtMs > 0 && sentAtMs > 0 && sentAtMs > endAtMs) { + continue; + } + long long isFromMe = sqlite3_column_int64(statement, 7); + BOOL read = sqlite3_column_int64(statement, 8) > 0; + BOOL delivered = sqlite3_column_int64(statement, 9) > 0; + NSMutableDictionary *message = [NSMutableDictionary dictionaryWithDictionary:@{ + @"message_id": [NSString stringWithFormat:@"ios-message-%lld", rowId], + @"row_id": @(rowId), + @"guid": guid ?: @"", + @"handle": handle ?: @"", + @"service": service ?: @"", + @"direction": OPMessagesDirectionFromIsFromMe(isFromMe), + @"sent_at_ms": @(sentAtMs), + @"sent_at": OPCalendarISO8601FromMs(sentAtMs), + @"read": @(read), + @"delivered": @(delivered), + @"provider": @"SMS.sqlite" + }]; + [message addEntriesFromDictionary:OPMessagesPreviewFields(text, @"text", 500)]; + [message addEntriesFromDictionary:OPMessagesPreviewFields(subject, @"subject", 240)]; + [messages addObject:message]; + if (messages.count >= limit) { + break; + } + } + if (rc != SQLITE_DONE && errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"messages_query_failed"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"messages_query_prepare_failed"; + } + sqlite3_finalize(statement); + sqlite3_close(db); + if (rc != SQLITE_DONE && messages.count == 0 && errorOut && *errorOut) { + return nil; + } + return messages; +} + +static NSArray *OPMessagesSystemSearch(NSString *query, NSUInteger limit, + long long startAtMs, long long endAtMs, NSString **errorOut) { + NSMutableArray *errors = [NSMutableArray array]; + for (NSString *path in OPMessagesDatabasePaths()) { + NSString *pathError = nil; + NSArray *messages = OPMessagesSystemSearchAtPath(path, query, limit, + startAtMs, endAtMs, &pathError); + if (messages) { + return messages; + } + if (pathError.length > 0) { + [errors addObject:[NSString stringWithFormat:@"%@:%@", path.lastPathComponent ?: @"messages", pathError]]; + } + } + if (errorOut) { + *errorOut = errors.count > 0 ? [errors componentsJoinedByString:@";"] : @"messages_query_failed"; + } + return nil; +} + +static NSArray *OPMessagesSummaryMessages(NSArray *messages, NSUInteger limit, + BOOL includeText) { + NSMutableArray *summary = [NSMutableArray array]; + for (NSDictionary *message in messages) { + NSMutableDictionary *item = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"message_id", @"handle", @"service", @"direction", + @"sent_at_ms", @"sent_at", @"read", @"delivered"]) { + id value = message[key]; + if (value) { + item[key] = value; + } + } + if (includeText) { + for (NSString *key in @[@"text_preview", @"text_truncated", @"text_length", + @"subject_preview", @"subject_truncated", @"subject_length"]) { + id value = message[key]; + if (value) { + item[key] = value; + } + } + } else { + for (NSString *key in @[@"text_length", @"text_sha256", @"subject_length", @"subject_sha256"]) { + id value = message[key]; + if (value) { + item[key] = value; + } + } + } + if (item.count > 0) { + [summary addObject:item]; + } + if (summary.count >= limit) { + break; + } + } + return summary; +} + +static NSDictionary *OPMessagesTraceSummary(NSDictionary *result) { + if (![result isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSArray *messages = [result[@"messages"] isKindOfClass:[NSArray class]] + ? result[@"messages"] : @[]; + return @{ + @"status": result[@"status"] ?: @"unknown", + @"tool": @"messages_search", + @"provider": result[@"provider"] ?: @"unknown", + @"query_length": result[@"query_length"] ?: @0, + @"query_sha256": result[@"query_sha256"] ?: @"", + @"start_at_ms": result[@"start_at_ms"] ?: @0, + @"end_at_ms": result[@"end_at_ms"] ?: @0, + @"count": result[@"count"] ?: @0, + @"limit": result[@"limit"] ?: @0, + @"messages": OPMessagesSummaryMessages(messages, 8, NO), + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPMessagesSearch(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *reason = OPStringFromRequest(request, @"reason", @""); + NSString *query = OPStringFromRequest(request, @"query", + OPStringFromRequest(request, @"text", + OPStringFromRequest(request, @"handle", @""))); + query = [query stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + long long startAtMs = OPCalendarLongLongFromValue(request[@"start_at_ms"] ?: request[@"start_ms"], 0); + long long endAtMs = OPCalendarLongLongFromValue(request[@"end_at_ms"] ?: request[@"end_ms"], 0); + BOOL allowEmptyQuery = OPBoolFromRequest(request, @"allow_empty_query", NO); + if (query.length == 0 && startAtMs <= 0 && endAtMs <= 0 && !allowEmptyQuery) { + return OPError(@"missing_messages_query_or_range"); + } + NSUInteger limit = OPLimitFromRequest(request, 10, 50); + if (limit == 0) { + limit = 10; + } + + NSString *provider = @""; + NSString *systemError = nil; + NSArray *messages = nil; + BOOL smsAvailable = OPPathExists(OPMessagesDatabasePath()); + messages = OPMessagesSystemSearch(query, limit, startAtMs, endAtMs, &systemError); + if (messages) { + provider = @"SMS.sqlite"; + } + if (!messages && !OPProtectedDataHelperRole()) { + NSMutableDictionary *helperRequest = [request mutableCopy] ?: [NSMutableDictionary dictionary]; + helperRequest[@"command"] = @"messages_search"; + helperRequest[@"task_id"] = @""; + NSDictionary *helperResult = OPProtectedDataHelperRequest(helperRequest); + if ([helperResult[@"status"] isEqualToString:@"ok"] && + [helperResult[@"messages"] isKindOfClass:[NSArray class]]) { + messages = helperResult[@"messages"]; + provider = [helperResult[@"provider"] isKindOfClass:[NSString class]] + ? helperResult[@"provider"] : @"SMS.sqlite"; + } else if ([helperResult[@"reason"] isKindOfClass:[NSString class]]) { + systemError = systemError.length > 0 + ? [NSString stringWithFormat:@"%@;protected_helper:%@", systemError, helperResult[@"reason"]] + : [NSString stringWithFormat:@"protected_helper:%@", helperResult[@"reason"]]; + } + } + if (!messages) { + if (OPPathExists(OPMessagesFixturePath())) { + messages = OPMessagesFixtureSearch(query, limit, startAtMs, endAtMs); + provider = @"openphone.messages_fixture_file"; + } else { + NSString *reasonText = (smsAvailable || systemError.length > 0) + ? [NSString stringWithFormat:@"messages_system_query_failed:%@", systemError ?: @"unknown"] + : @"messages_provider_unavailable"; + return OPError(reasonText); + } + } + + NSString *queryHash = OPClipboardTextHash(query ?: @""); + long long contextId = OPRecordContextEvent(@"messages_searched", @"openphone.agentd", taskId, + @"Messages search", + [NSString stringWithFormat:@"Messages search returned %lu message(s)", + (unsigned long)messages.count], + @{ + @"provider": provider ?: @"unknown", + @"count": @(messages.count), + @"limit": @(limit), + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"end_at_ms": @(endAtMs), + @"reason": reason ?: @"" + }); + NSDictionary *result = @{ + @"status": @"ok", + @"tool": @"messages_search", + @"query": query ?: @"", + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"start_at": OPCalendarISO8601FromMs(startAtMs), + @"end_at_ms": @(endAtMs), + @"end_at": OPCalendarISO8601FromMs(endAtMs), + @"limit": @(limit), + @"messages": messages ?: @[], + @"count": @(messages.count), + @"provider": provider ?: @"unknown", + @"messages_path": OPMessagesDatabasePath(), + @"fixture_path": OPMessagesFixturePath(), + @"context_event_id": @(contextId), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"messages_searched", taskId, @"messages.read", @"allow_task_scoped", + @{ + @"reason": reason ?: @"", + @"provider": provider ?: @"unknown", + @"limit": @(limit), + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"end_at_ms": @(endAtMs), + @"count": @(messages.count) + }, + [NSString stringWithFormat:@"count:%lu", (unsigned long)messages.count]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"messages_search", + @"arguments": @{ + @"reason": reason ?: @"", + @"query_length": @(query.length), + @"query_sha256": queryHash ?: @"", + @"start_at_ms": @(startAtMs), + @"end_at_ms": @(endAtMs), + @"limit": @(limit) + }, + @"result": OPMessagesTraceSummary(result) + }); + return result; +} + +static NSDictionary *OPCommitmentFromStatement(sqlite3_stmt *statement) { + NSString *triggerSpecJSON = OPSQLiteColumnString(statement, 6); + NSString *evidenceJSON = OPSQLiteColumnString(statement, 11); + long long rowId = sqlite3_column_int64(statement, 0); + return @{ + @"id": @(rowId), + @"commitment_id": [NSString stringWithFormat:@"ios-commitment-%lld", rowId], + @"created_at_ms": @(sqlite3_column_int64(statement, 1)), + @"updated_at_ms": @(sqlite3_column_int64(statement, 2)), + @"title": OPSQLiteColumnString(statement, 3), + @"description": OPSQLiteColumnString(statement, 4), + @"trigger_type": OPSQLiteColumnString(statement, 5), + @"trigger_spec": OPJSONDictionary(triggerSpecJSON), + @"due_at_ms": @(sqlite3_column_int64(statement, 7)), + @"due_at": @(sqlite3_column_int64(statement, 7)), + @"expires_at_ms": @(sqlite3_column_int64(statement, 8)), + @"expires_at": @(sqlite3_column_int64(statement, 8)), + @"status": OPSQLiteColumnString(statement, 9), + @"confidence": @(sqlite3_column_double(statement, 10)), + @"evidence": OPJSONDictionary(evidenceJSON), + @"source": OPSQLiteColumnString(statement, 12), + @"reason": OPSQLiteColumnString(statement, 13) + }; +} + +static BOOL OPWatcherSourceIsLocalTimer(NSString *source, NSString *type) { + NSString *sourceLower = [source isKindOfClass:[NSString class]] ? source.lowercaseString : @""; + NSString *typeLower = [type isKindOfClass:[NSString class]] ? type.lowercaseString : @""; + NSSet *timerTypes = [NSSet setWithObjects:@"time", @"timer", @"deadline", nil]; + return [timerTypes containsObject:sourceLower] || [timerTypes containsObject:typeLower]; +} + +static BOOL OPWatcherFiresLocally(NSString *source, NSString *type, long long nextRunAtMs) { + return nextRunAtMs > 0 && OPWatcherSourceIsLocalTimer(source, type); +} + +static NSDictionary *OPWatcherFromStatement(sqlite3_stmt *statement) { + NSString *conditionJSON = OPSQLiteColumnString(statement, 12); + NSString *scheduleJSON = OPSQLiteColumnString(statement, 13); + NSString *deliveryJSON = OPSQLiteColumnString(statement, 14); + NSString *metadataJSON = OPSQLiteColumnString(statement, 19); + long long rowId = sqlite3_column_int64(statement, 0); + NSString *source = OPSQLiteColumnString(statement, 4); + NSString *type = OPSQLiteColumnString(statement, 5); + long long nextRunAtMs = sqlite3_column_int64(statement, 15); + BOOL firesLocally = OPWatcherFiresLocally(source, type, nextRunAtMs); + return @{ + @"id": @(rowId), + @"watcher_id": [NSString stringWithFormat:@"ios-watcher-%lld", rowId], + @"created_at_ms": @(sqlite3_column_int64(statement, 1)), + @"updated_at_ms": @(sqlite3_column_int64(statement, 2)), + @"status": OPSQLiteColumnString(statement, 3), + @"source": source, + @"type": type, + @"evaluator": OPSQLiteColumnString(statement, 6), + @"title": OPSQLiteColumnString(statement, 7), + @"query": OPSQLiteColumnString(statement, 8), + @"url": OPSQLiteColumnString(statement, 9), + @"address": OPSQLiteColumnString(statement, 10), + @"number": OPSQLiteColumnString(statement, 11), + @"condition": OPJSONDictionary(conditionJSON), + @"schedule": OPJSONDictionary(scheduleJSON), + @"delivery": OPJSONDictionary(deliveryJSON), + @"next_run_at_ms": @(nextRunAtMs), + @"interval_ms": @(sqlite3_column_int64(statement, 16)), + @"recurring": @(sqlite3_column_int64(statement, 17) != 0), + @"reason": OPSQLiteColumnString(statement, 18), + @"metadata": OPJSONDictionary(metadataJSON), + @"scheduler_status": firesLocally ? OPWatcherSchedulerStatus() : @"not_started", + @"fires_locally": @(firesLocally) + }; +} + +static NSDictionary *OPAgentJobFromStatement(sqlite3_stmt *statement) { + NSString *scheduleJSON = OPSQLiteColumnString(statement, 7); + NSString *deliveryJSON = OPSQLiteColumnString(statement, 11); + NSString *payloadJSON = OPSQLiteColumnString(statement, 13); + long long rowId = sqlite3_column_int64(statement, 0); + BOOL schedulerEnabled = sqlite3_column_int64(statement, 16) != 0; + return @{ + @"id": @(rowId), + @"job_id": [NSString stringWithFormat:@"ios-job-%lld", rowId], + @"created_at_ms": @(sqlite3_column_int64(statement, 1)), + @"updated_at_ms": @(sqlite3_column_int64(statement, 2)), + @"status": OPSQLiteColumnString(statement, 3), + @"type": OPSQLiteColumnString(statement, 4), + @"title": OPSQLiteColumnString(statement, 5), + @"prompt": OPSQLiteColumnString(statement, 6), + @"schedule": OPJSONDictionary(scheduleJSON), + @"next_run_at_ms": @(sqlite3_column_int64(statement, 8)), + @"interval_ms": @(sqlite3_column_int64(statement, 9)), + @"session_target": OPSQLiteColumnString(statement, 10), + @"delivery": OPJSONDictionary(deliveryJSON), + @"notification_text": OPSQLiteColumnString(statement, 12), + @"payload": OPJSONDictionary(payloadJSON), + @"reason": OPSQLiteColumnString(statement, 14), + @"source": OPSQLiteColumnString(statement, 15), + @"scheduler_enabled": @(schedulerEnabled), + @"scheduler_status": OPBackgroundJobSchedulerStatus(), + @"runner": @"deterministic", + @"runs_locally": @(schedulerEnabled), + @"model_loop_status": @"not_started" + }; +} + +static NSDictionary *OPCommitmentRead(sqlite3 *db, long long commitmentId) { + sqlite3_stmt *read = NULL; + NSDictionary *commitment = nil; + if (sqlite3_prepare_v2(db, + "SELECT id, created_at_ms, updated_at_ms, title, description, trigger_type, " + "trigger_spec_json, due_at_ms, expires_at_ms, status, confidence, evidence_json, source, reason " + "FROM commitment WHERE id = ?", + -1, &read, NULL) == SQLITE_OK) { + sqlite3_bind_int64(read, 1, commitmentId); + if (sqlite3_step(read) == SQLITE_ROW) { + commitment = OPCommitmentFromStatement(read); + } + } + sqlite3_finalize(read); + return commitment; +} + +static NSDictionary *OPWatcherRead(sqlite3 *db, long long watcherId) { + sqlite3_stmt *read = NULL; + NSDictionary *watcher = nil; + if (sqlite3_prepare_v2(db, + "SELECT id, created_at_ms, updated_at_ms, status, source, type, evaluator, title, " + "query, url, address, number, condition_json, schedule_json, delivery_json, " + "next_run_at_ms, interval_ms, recurring, reason, metadata_json " + "FROM watcher WHERE id = ?", + -1, &read, NULL) == SQLITE_OK) { + sqlite3_bind_int64(read, 1, watcherId); + if (sqlite3_step(read) == SQLITE_ROW) { + watcher = OPWatcherFromStatement(read); + } + } + sqlite3_finalize(read); + return watcher; +} + +static NSDictionary *OPAgentJobRead(sqlite3 *db, long long jobId) { + sqlite3_stmt *read = NULL; + NSDictionary *job = nil; + if (sqlite3_prepare_v2(db, + "SELECT id, created_at_ms, updated_at_ms, status, type, title, prompt, schedule_json, " + "next_run_at_ms, interval_ms, session_target, delivery_json, notification_text, " + "payload_json, reason, source, scheduler_enabled FROM agent_job WHERE id = ?", + -1, &read, NULL) == SQLITE_OK) { + sqlite3_bind_int64(read, 1, jobId); + if (sqlite3_step(read) == SQLITE_ROW) { + job = OPAgentJobFromStatement(read); + } + } + sqlite3_finalize(read); + return job; +} + +static NSDictionary *OPCommitmentCreate(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *title = [OPStringFromRequest(request, @"title", @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (title.length == 0) { + return OPError(@"missing_commitment_title"); + } + NSString *description = OPStringFromRequest(request, @"description", @""); + long long dueAt = OPLongLongFromRequest(request, @"due_at_ms", 0, 0, 4102444800000LL); + if (dueAt == 0) { + dueAt = OPLongLongFromRequest(request, @"due_at", 0, 0, 4102444800000LL); + } + long long expiresAt = OPLongLongFromRequest(request, @"expires_at_ms", 0, 0, 4102444800000LL); + if (expiresAt == 0) { + expiresAt = OPLongLongFromRequest(request, @"expires_at", 0, 0, 4102444800000LL); + } + NSString *triggerType = OPStringFromRequest(request, @"trigger_type", dueAt > 0 ? @"time" : @"manual"); + if (triggerType.length == 0) { + triggerType = dueAt > 0 ? @"time" : @"manual"; + } + NSString *reason = OPStringFromRequest(request, @"reason", @""); + double confidence = OPDoubleFromRequest(request, @"confidence", 1.0, 0.0, 1.0); + id triggerSpec = OPJSONObjectFromRequest(request, @"trigger_spec", @{}); + id evidence = OPJSONObjectFromRequest(request, @"evidence", OPJSONObjectFromRequest(request, @"metadata", @{})); + NSString *source = OPStringFromRequest(request, @"source", @"openphone.agentd"); + if (source.length == 0) { + source = @"openphone.agentd"; + } + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + + long long now = OPNowMs(); + long long commitmentId = 0; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "INSERT INTO commitment(created_at_ms, updated_at_ms, title, description, trigger_type, " + "trigger_spec_json, due_at_ms, expires_at_ms, status, confidence, evidence_json, source, reason) " + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, now); + sqlite3_bind_int64(statement, 2, now); + OPSQLiteBindText(statement, 3, title); + OPSQLiteBindText(statement, 4, description); + OPSQLiteBindText(statement, 5, triggerType); + OPSQLiteBindText(statement, 6, OPJSONString(triggerSpec)); + sqlite3_bind_int64(statement, 7, dueAt); + sqlite3_bind_int64(statement, 8, expiresAt); + OPSQLiteBindText(statement, 9, @"active"); + sqlite3_bind_double(statement, 10, confidence); + OPSQLiteBindText(statement, 11, OPJSONString(evidence)); + OPSQLiteBindText(statement, 12, source); + OPSQLiteBindText(statement, 13, reason); + if (sqlite3_step(statement) == SQLITE_DONE) { + commitmentId = sqlite3_last_insert_rowid(db); + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + + if (commitmentId > 0 && OPSQLiteTableExists(db, @"commitment_fts")) { + sqlite3_stmt *fts = NULL; + if (sqlite3_prepare_v2(db, + "INSERT INTO commitment_fts(commitment_id, title, description, trigger_type) " + "VALUES(?, ?, ?, ?)", + -1, &fts, NULL) == SQLITE_OK) { + sqlite3_bind_int64(fts, 1, commitmentId); + OPSQLiteBindText(fts, 2, title); + OPSQLiteBindText(fts, 3, description); + OPSQLiteBindText(fts, 4, triggerType); + sqlite3_step(fts); + } + sqlite3_finalize(fts); + } + NSDictionary *commitment = commitmentId > 0 ? OPCommitmentRead(db, commitmentId) : nil; + sqlite3_close(db); + if (commitmentId == 0 || !commitment) { + return OPError([NSString stringWithFormat:@"commitment_create_failed:%@", error ?: @"unknown"]); + } + + OPRecordContextEvent(@"commitment_created", @"openphone.agentd", taskId, + title, description.length > 0 ? description : title, @{ + @"commitment_id": commitment[@"commitment_id"] ?: @"", + @"due_at_ms": @(dueAt), + @"trigger_type": triggerType + }); + NSDictionary *result = @{ + @"status": @"ok", + @"commitment_id": commitment[@"id"] ?: @(commitmentId), + @"commitment": commitment, + @"db_path": OPDatabasePath(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"commitment_created", taskId, @"commitments.write", @"allow_yolo", + request, [NSString stringWithFormat:@"commitment_id:%@", commitment[@"commitment_id"] ?: @""]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"commitment_create", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSDictionary *OPCommitmentSearch(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *query = [OPStringFromRequest(request, @"query", @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSUInteger limit = OPLimitFromRequest(request, 20, 200); + if (limit == 0) { + limit = 20; + } + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + + NSMutableArray *commitments = [NSMutableArray array]; + NSString *provider = @"sqlite_latest"; + BOOL hasQuery = query.length > 0; + BOOL ftsFailed = NO; + if (hasQuery && OPSQLiteTableExists(db, @"commitment_fts")) { + NSString *ftsQuery = OPFTSQuery(query); + if (ftsQuery.length > 0) { + sqlite3_stmt *statement = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT c.id, c.created_at_ms, c.updated_at_ms, c.title, c.description, " + "c.trigger_type, c.trigger_spec_json, c.due_at_ms, c.expires_at_ms, c.status, " + "c.confidence, c.evidence_json, c.source, c.reason " + "FROM commitment_fts f JOIN commitment c ON f.commitment_id = c.id " + "WHERE commitment_fts MATCH ? ORDER BY bm25(commitment_fts) LIMIT ?", + -1, &statement, NULL); + if (rc == SQLITE_OK) { + OPSQLiteBindText(statement, 1, ftsQuery); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)limit); + while ((rc = sqlite3_step(statement)) == SQLITE_ROW) { + [commitments addObject:OPCommitmentFromStatement(statement)]; + } + if (rc != SQLITE_DONE) { + [commitments removeAllObjects]; + ftsFailed = YES; + } + } else { + ftsFailed = YES; + } + sqlite3_finalize(statement); + if (!ftsFailed) { + provider = @"sqlite_fts5"; + } + } + } + if (!hasQuery || ftsFailed || ![provider isEqualToString:@"sqlite_fts5"]) { + [commitments removeAllObjects]; + sqlite3_stmt *statement = NULL; + NSString *sql = hasQuery + ? @"SELECT id, created_at_ms, updated_at_ms, title, description, trigger_type, " + "trigger_spec_json, due_at_ms, expires_at_ms, status, confidence, evidence_json, source, reason " + "FROM commitment WHERE lower(title) LIKE ? OR lower(description) LIKE ? OR lower(trigger_type) LIKE ? " + "ORDER BY updated_at_ms DESC LIMIT ?" + : @"SELECT id, created_at_ms, updated_at_ms, title, description, trigger_type, " + "trigger_spec_json, due_at_ms, expires_at_ms, status, confidence, evidence_json, source, reason " + "FROM commitment ORDER BY updated_at_ms DESC LIMIT ?"; + if (sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL) == SQLITE_OK) { + if (hasQuery) { + NSString *like = [NSString stringWithFormat:@"%%%@%%", query.lowercaseString ?: @""]; + OPSQLiteBindText(statement, 1, like); + OPSQLiteBindText(statement, 2, like); + OPSQLiteBindText(statement, 3, like); + sqlite3_bind_int64(statement, 4, (sqlite3_int64)limit); + provider = @"sqlite_like"; + } else { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)limit); + provider = @"sqlite_latest"; + } + while (sqlite3_step(statement) == SQLITE_ROW) { + [commitments addObject:OPCommitmentFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + } + BOOL ftsAvailable = OPSQLiteTableExists(db, @"commitment_fts"); + sqlite3_close(db); + NSDictionary *result = @{ + @"status": @"ok", + @"query": query ?: @"", + @"limit": @(limit), + @"commitments": commitments, + @"count": @(commitments.count), + @"provider": provider, + @"fts_available": @(ftsAvailable), + @"db_path": OPDatabasePath(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"commitments_searched", taskId, @"commitments.read", @"allow_task_scoped", + request, [NSString stringWithFormat:@"count:%lu", (unsigned long)commitments.count]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"commitment_search", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSDictionary *OPCommitmentUpdateStatus(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + long long commitmentId = OPRecordIdFromRequest(request, @[@"commitment_id", @"id"]); + if (commitmentId <= 0) { + return OPError(@"missing_commitment_id"); + } + NSString *status = [OPStringFromRequest(request, @"status", @"") lowercaseString]; + status = [status stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (status.length == 0) { + return OPError(@"missing_commitment_status"); + } + NSString *reason = OPStringFromRequest(request, @"reason", @""); + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE commitment SET status = ?, updated_at_ms = ?, reason = ? WHERE id = ?", + -1, &statement, NULL) == SQLITE_OK) { + OPSQLiteBindText(statement, 1, status); + sqlite3_bind_int64(statement, 2, OPNowMs()); + OPSQLiteBindText(statement, 3, reason); + sqlite3_bind_int64(statement, 4, commitmentId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + NSDictionary *commitment = updated ? OPCommitmentRead(db, commitmentId) : nil; + sqlite3_close(db); + if (!updated || !commitment) { + return OPError([NSString stringWithFormat:@"commitment_update_failed:%@", error ?: @"not_found"]); + } + OPRecordContextEvent(@"commitment_status_updated", @"openphone.agentd", taskId, + commitment[@"title"] ?: @"", status, @{ + @"commitment_id": commitment[@"commitment_id"] ?: @"", + @"status": status, + @"reason": reason ?: @"" + }); + NSDictionary *result = @{ + @"status": @"ok", + @"commitment_id": commitment[@"id"] ?: @(commitmentId), + @"commitment": commitment, + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"commitment_status_updated", taskId, @"commitments.write", @"allow_yolo", + request, [NSString stringWithFormat:@"commitment_id:%lld status:%@", commitmentId, status]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"commitment_update_status", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static id OPDictionaryNumberValue(NSDictionary *dictionary, NSString *key) { + id value = dictionary[key]; + return [value isKindOfClass:[NSNumber class]] ? value : nil; +} + +static NSDictionary *OPWatcherCreate(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *title = [OPStringFromRequest(request, @"title", @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (title.length == 0) { + return OPError(@"missing_watcher_title"); + } + NSString *source = OPStringFromRequest(request, @"source", @""); + NSString *type = OPStringFromRequest(request, @"type", @""); + if (source.length == 0 && type.length > 0) { + source = type; + } + if (source.length == 0) { + source = @"time"; + } + if (type.length == 0) { + type = source; + } + NSString *evaluator = OPStringFromRequest(request, @"evaluator", @""); + if (evaluator.length == 0) { + evaluator = [source isEqualToString:@"web"] ? @"hash_change" : @"event_match"; + } + NSString *query = OPStringFromRequest(request, @"query", @""); + NSString *url = OPStringFromRequest(request, @"url", @""); + NSString *address = OPStringFromRequest(request, @"address", @""); + NSString *number = OPStringFromRequest(request, @"number", @""); + NSString *reason = OPStringFromRequest(request, @"reason", @""); + long long nextRunAt = OPLongLongFromRequest(request, @"next_run_at", 0, 0, 4102444800000LL); + long long deadlineAt = OPLongLongFromRequest(request, @"deadline_at", 0, 0, 4102444800000LL); + if (nextRunAt == 0 && deadlineAt > 0) { + nextRunAt = deadlineAt; + } + long long intervalMs = OPLongLongFromRequest(request, @"interval_ms", 0, 0, 31536000000LL); + BOOL recurring = OPBoolFromRequest(request, @"recurring", intervalMs > 0); + + NSMutableDictionary *condition = [OPJSONObjectFromRequest(request, @"condition", @{}) mutableCopy]; + if (!condition) { + condition = [NSMutableDictionary dictionary]; + } + if (query.length > 0) { + condition[@"query"] = query; + } + if (url.length > 0) { + condition[@"url"] = url; + } + if (address.length > 0) { + condition[@"address"] = address; + } + if (number.length > 0) { + condition[@"number"] = number; + } + if (deadlineAt > 0) { + condition[@"deadline_at"] = @(deadlineAt); + } + NSString *notifyOn = OPStringFromRequest(request, @"notify_on", @""); + if (notifyOn.length > 0) { + condition[@"notify_on"] = notifyOn; + } + NSString *direction = OPStringFromRequest(request, @"direction", @""); + if (direction.length > 0) { + condition[@"direction"] = direction; + } + if (OPBoolFromRequest(request, @"match_any", NO)) { + condition[@"match_any"] = @YES; + } + long long threadId = OPLongLongFromRequest(request, @"thread_id", 0, 0, 9223372036854775807LL); + if (threadId > 0) { + condition[@"thread_id"] = @(threadId); + } + NSString *smsBody = OPStringFromRequest(request, @"sms_body", @""); + if (smsBody.length > 0) { + condition[@"sms_body"] = smsBody; + } + + NSMutableDictionary *schedule = [OPJSONObjectFromRequest(request, @"schedule", @{}) mutableCopy]; + if (!schedule) { + schedule = [NSMutableDictionary dictionary]; + } + if (nextRunAt > 0) { + schedule[@"next_run_at"] = @(nextRunAt); + } else { + id scheduledNext = OPDictionaryNumberValue(schedule, @"next_run_at"); + if (scheduledNext) { + nextRunAt = [scheduledNext longLongValue]; + } + } + if (intervalMs > 0) { + schedule[@"interval_ms"] = @(intervalMs); + } else { + id scheduledInterval = OPDictionaryNumberValue(schedule, @"interval_ms"); + if (scheduledInterval) { + intervalMs = [scheduledInterval longLongValue]; + } + } + if (recurring) { + schedule[@"recurring"] = @YES; + } + + NSMutableDictionary *delivery = [OPJSONObjectFromRequest(request, @"delivery", @{}) mutableCopy]; + if (!delivery) { + delivery = [NSMutableDictionary dictionary]; + } + NSString *tool = OPStringFromRequest(request, @"tool", @""); + NSString *prompt = OPStringFromRequest(request, @"prompt", @""); + id arguments = OPJSONObjectFromRequest(request, @"arguments", nil); + if (tool.length > 0) { + delivery[@"tool"] = tool; + } + if (prompt.length > 0) { + delivery[@"prompt"] = prompt; + if (![delivery[@"mode"] isKindOfClass:[NSString class]]) { + delivery[@"mode"] = @"background_job"; + } + } + if (arguments) { + delivery[@"arguments"] = arguments; + } + if (delivery.count == 0) { + delivery[@"mode"] = @"notification"; + } + BOOL firesLocally = OPWatcherFiresLocally(source, type, nextRunAt); + NSDictionary *metadata = @{ + @"scheduler_status": firesLocally ? OPWatcherSchedulerStatus() : @"not_started", + @"fires_locally": @(firesLocally) + }; + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + long long now = OPNowMs(); + long long watcherId = 0; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "INSERT INTO watcher(created_at_ms, updated_at_ms, status, source, type, evaluator, title, query, url, " + "address, number, condition_json, schedule_json, delivery_json, next_run_at_ms, interval_ms, " + "recurring, reason, metadata_json) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, now); + sqlite3_bind_int64(statement, 2, now); + OPSQLiteBindText(statement, 3, @"active"); + OPSQLiteBindText(statement, 4, source); + OPSQLiteBindText(statement, 5, type); + OPSQLiteBindText(statement, 6, evaluator); + OPSQLiteBindText(statement, 7, title); + OPSQLiteBindText(statement, 8, query); + OPSQLiteBindText(statement, 9, url); + OPSQLiteBindText(statement, 10, address); + OPSQLiteBindText(statement, 11, number); + OPSQLiteBindText(statement, 12, OPJSONString(condition)); + OPSQLiteBindText(statement, 13, OPJSONString(schedule)); + OPSQLiteBindText(statement, 14, OPJSONString(delivery)); + sqlite3_bind_int64(statement, 15, nextRunAt); + sqlite3_bind_int64(statement, 16, intervalMs); + sqlite3_bind_int64(statement, 17, recurring ? 1 : 0); + OPSQLiteBindText(statement, 18, reason); + OPSQLiteBindText(statement, 19, OPJSONString(metadata)); + if (sqlite3_step(statement) == SQLITE_DONE) { + watcherId = sqlite3_last_insert_rowid(db); + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + if (watcherId > 0 && OPSQLiteTableExists(db, @"watcher_fts")) { + sqlite3_stmt *fts = NULL; + if (sqlite3_prepare_v2(db, + "INSERT INTO watcher_fts(watcher_id, title, query, source, type) VALUES(?, ?, ?, ?, ?)", + -1, &fts, NULL) == SQLITE_OK) { + sqlite3_bind_int64(fts, 1, watcherId); + OPSQLiteBindText(fts, 2, title); + OPSQLiteBindText(fts, 3, query); + OPSQLiteBindText(fts, 4, source); + OPSQLiteBindText(fts, 5, type); + sqlite3_step(fts); + } + sqlite3_finalize(fts); + } + NSDictionary *watcher = watcherId > 0 ? OPWatcherRead(db, watcherId) : nil; + sqlite3_close(db); + if (watcherId == 0 || !watcher) { + return OPError([NSString stringWithFormat:@"watcher_create_failed:%@", error ?: @"unknown"]); + } + OPRecordContextEvent(@"watcher_created", @"openphone.agentd", taskId, + title, query.length > 0 ? query : source, @{ + @"watcher_id": watcher[@"watcher_id"] ?: @"", + @"source": source, + @"type": type, + @"scheduler_status": metadata[@"scheduler_status"] ?: @"not_started" + }); + NSDictionary *result = @{ + @"status": @"ok", + @"watcher_id": watcher[@"id"] ?: @(watcherId), + @"watcher": watcher, + @"scheduler_status": metadata[@"scheduler_status"] ?: @"not_started", + @"fires_locally": @(firesLocally), + @"db_path": OPDatabasePath(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"watcher_created", taskId, @"watchers.write", @"allow_yolo", + request, [NSString stringWithFormat:@"watcher_id:%@", watcher[@"watcher_id"] ?: @""]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"watcher_create", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSArray *OPWatcherListRows(sqlite3 *db, NSString *query, + NSUInteger limit, NSString **providerOut) { + NSMutableArray *watchers = [NSMutableArray array]; + BOOL hasQuery = query.length > 0; + BOOL ftsFailed = NO; + if (hasQuery && OPSQLiteTableExists(db, @"watcher_fts")) { + NSString *ftsQuery = OPFTSQuery(query); + if (ftsQuery.length > 0) { + sqlite3_stmt *statement = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT w.id, w.created_at_ms, w.updated_at_ms, w.status, w.source, w.type, " + "w.evaluator, w.title, w.query, w.url, w.address, w.number, w.condition_json, " + "w.schedule_json, w.delivery_json, w.next_run_at_ms, w.interval_ms, w.recurring, " + "w.reason, w.metadata_json FROM watcher_fts f JOIN watcher w ON f.watcher_id = w.id " + "WHERE watcher_fts MATCH ? ORDER BY bm25(watcher_fts) LIMIT ?", + -1, &statement, NULL); + if (rc == SQLITE_OK) { + OPSQLiteBindText(statement, 1, ftsQuery); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)limit); + while ((rc = sqlite3_step(statement)) == SQLITE_ROW) { + [watchers addObject:OPWatcherFromStatement(statement)]; + } + if (rc != SQLITE_DONE) { + [watchers removeAllObjects]; + ftsFailed = YES; + } + } else { + ftsFailed = YES; + } + sqlite3_finalize(statement); + if (!ftsFailed) { + if (providerOut) { + *providerOut = @"sqlite_fts5"; + } + return watchers; + } + } + } + + sqlite3_stmt *statement = NULL; + NSString *sql = hasQuery + ? @"SELECT id, created_at_ms, updated_at_ms, status, source, type, evaluator, title, query, url, " + "address, number, condition_json, schedule_json, delivery_json, next_run_at_ms, interval_ms, " + "recurring, reason, metadata_json FROM watcher WHERE lower(title) LIKE ? OR lower(query) LIKE ? " + "OR lower(source) LIKE ? OR lower(type) LIKE ? ORDER BY updated_at_ms DESC LIMIT ?" + : @"SELECT id, created_at_ms, updated_at_ms, status, source, type, evaluator, title, query, url, " + "address, number, condition_json, schedule_json, delivery_json, next_run_at_ms, interval_ms, " + "recurring, reason, metadata_json FROM watcher ORDER BY updated_at_ms DESC LIMIT ?"; + if (sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL) == SQLITE_OK) { + if (hasQuery) { + NSString *like = [NSString stringWithFormat:@"%%%@%%", query.lowercaseString ?: @""]; + OPSQLiteBindText(statement, 1, like); + OPSQLiteBindText(statement, 2, like); + OPSQLiteBindText(statement, 3, like); + OPSQLiteBindText(statement, 4, like); + sqlite3_bind_int64(statement, 5, (sqlite3_int64)limit); + } else { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)limit); + } + while (sqlite3_step(statement) == SQLITE_ROW) { + [watchers addObject:OPWatcherFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + if (providerOut) { + *providerOut = hasQuery ? @"sqlite_like" : @"sqlite_latest"; + } + return watchers; +} + +static NSDictionary *OPWatcherList(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *query = [OPStringFromRequest(request, @"query", @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSUInteger limit = OPLimitFromRequest(request, 20, 200); + if (limit == 0) { + limit = 20; + } + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSString *provider = nil; + NSArray *watchers = OPWatcherListRows(db, query, limit, &provider); + BOOL ftsAvailable = OPSQLiteTableExists(db, @"watcher_fts"); + sqlite3_close(db); + NSDictionary *result = @{ + @"status": @"ok", + @"query": query ?: @"", + @"limit": @(limit), + @"watchers": watchers, + @"count": @(watchers.count), + @"provider": provider ?: @"sqlite", + @"fts_available": @(ftsAvailable), + @"scheduler_status": OPWatcherSchedulerStatus(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"watchers_listed", taskId, @"watchers.read", @"allow_task_scoped", + request, [NSString stringWithFormat:@"count:%lu", (unsigned long)watchers.count]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"watcher_list", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSDictionary *OPWatcherStop(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + long long watcherId = OPRecordIdFromRequest(request, @[@"watcher_id", @"id"]); + BOOL stopAll = OPBoolFromRequest(request, @"all", NO) || + [OPStringFromRequest(request, @"scope", @"") isEqualToString:@"all"]; + NSString *query = [OPStringFromRequest(request, @"query", @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (watcherId <= 0 && !stopAll && query.length == 0) { + return OPError(@"missing_watcher_target"); + } + NSString *reason = OPStringFromRequest(request, @"reason", @""); + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSMutableArray *ids = [NSMutableArray array]; + sqlite3_stmt *select = NULL; + NSString *sql = nil; + if (watcherId > 0) { + sql = @"SELECT id FROM watcher WHERE id = ? AND status != 'stopped'"; + } else if (stopAll) { + sql = @"SELECT id FROM watcher WHERE status != 'stopped'"; + } else { + sql = @"SELECT id FROM watcher WHERE status != 'stopped' AND " + "(lower(title) LIKE ? OR lower(query) LIKE ? OR lower(source) LIKE ? OR lower(type) LIKE ?)"; + } + if (sqlite3_prepare_v2(db, sql.UTF8String, -1, &select, NULL) == SQLITE_OK) { + if (watcherId > 0) { + sqlite3_bind_int64(select, 1, watcherId); + } else if (!stopAll) { + NSString *like = [NSString stringWithFormat:@"%%%@%%", query.lowercaseString ?: @""]; + OPSQLiteBindText(select, 1, like); + OPSQLiteBindText(select, 2, like); + OPSQLiteBindText(select, 3, like); + OPSQLiteBindText(select, 4, like); + } + while (sqlite3_step(select) == SQLITE_ROW) { + [ids addObject:@(sqlite3_column_int64(select, 0))]; + } + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(select); + + NSMutableArray *stopped = [NSMutableArray array]; + for (NSNumber *recordId in ids) { + sqlite3_stmt *update = NULL; + if (sqlite3_prepare_v2(db, + "UPDATE watcher SET status = 'stopped', updated_at_ms = ?, reason = ? WHERE id = ?", + -1, &update, NULL) == SQLITE_OK) { + sqlite3_bind_int64(update, 1, OPNowMs()); + OPSQLiteBindText(update, 2, reason); + sqlite3_bind_int64(update, 3, [recordId longLongValue]); + sqlite3_step(update); + } + sqlite3_finalize(update); + NSDictionary *watcher = OPWatcherRead(db, [recordId longLongValue]); + if (watcher) { + [stopped addObject:watcher]; + } + } + sqlite3_close(db); + NSDictionary *result = @{ + @"status": @"ok", + @"stopped_count": @(stopped.count), + @"watchers": stopped, + @"scheduler_status": OPWatcherSchedulerStatus(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"watchers_stopped", taskId, @"watchers.write", @"allow_yolo", + request, [NSString stringWithFormat:@"count:%lu error:%@", (unsigned long)stopped.count, error ?: @""]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"watcher_stop", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSDictionary *OPBackgroundJobCreate(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *title = [OPStringFromRequest(request, @"title", @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSString *prompt = [OPStringFromRequest(request, @"prompt", @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (title.length == 0) { + return OPError(@"missing_background_job_title"); + } + if (prompt.length == 0) { + return OPError(@"missing_background_job_prompt"); + } + NSString *type = OPStringFromRequest(request, @"type", @"agent_turn"); + if (type.length == 0) { + type = @"agent_turn"; + } + id schedule = OPJSONObjectFromRequest(request, @"schedule", @{}); + long long nextRunAt = OPLongLongFromRequest(request, @"next_run_at", 0, 0, 4102444800000LL); + if (nextRunAt == 0) { + nextRunAt = OPLongLongFromRequest(request, @"run_at", 0, 0, 4102444800000LL); + } + long long intervalMs = OPLongLongFromRequest(request, @"interval_ms", 0, 0, 31536000000LL); + NSMutableDictionary *scheduleDict = [[schedule mutableCopy] ?: [NSMutableDictionary dictionary] mutableCopy]; + scheduleDict[@"scheduler_status"] = OPBackgroundJobSchedulerStatus(); + scheduleDict[@"scheduler_enabled"] = @YES; + if (nextRunAt > 0) { + scheduleDict[@"next_run_at"] = @(nextRunAt); + } else { + id scheduledNext = OPDictionaryNumberValue(scheduleDict, @"next_run_at"); + if (scheduledNext) { + nextRunAt = [scheduledNext longLongValue]; + } + } + if (intervalMs > 0) { + scheduleDict[@"interval_ms"] = @(intervalMs); + } else { + id scheduledInterval = OPDictionaryNumberValue(scheduleDict, @"interval_ms"); + if (scheduledInterval) { + intervalMs = [scheduledInterval longLongValue]; + } + } + BOOL recurring = OPBoolFromRequest(request, @"recurring", + intervalMs > 0 && ![scheduleDict[@"recurring"] isEqual:@NO]); + scheduleDict[@"recurring"] = @(recurring); + NSString *sessionTarget = OPStringFromRequest(request, @"session_target", @"main"); + if (sessionTarget.length == 0) { + sessionTarget = @"main"; + } + id delivery = OPJSONObjectFromRequest(request, @"delivery", @{@"mode": @"notification"}); + NSString *notificationText = OPStringFromRequest(request, @"notification_text", @""); + id payload = OPJSONObjectFromRequest(request, @"payload", @{}); + NSString *reason = OPStringFromRequest(request, @"reason", @""); + NSString *source = OPStringFromRequest(request, @"source", @"openphone.agentd"); + if (source.length == 0) { + source = @"openphone.agentd"; + } + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + long long now = OPNowMs(); + long long jobId = 0; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "INSERT INTO agent_job(created_at_ms, updated_at_ms, status, type, title, prompt, schedule_json, " + "next_run_at_ms, interval_ms, session_target, delivery_json, notification_text, payload_json, " + "reason, source, scheduler_enabled) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, now); + sqlite3_bind_int64(statement, 2, now); + OPSQLiteBindText(statement, 3, @"queued"); + OPSQLiteBindText(statement, 4, type); + OPSQLiteBindText(statement, 5, title); + OPSQLiteBindText(statement, 6, prompt); + OPSQLiteBindText(statement, 7, OPJSONString(scheduleDict)); + sqlite3_bind_int64(statement, 8, nextRunAt); + sqlite3_bind_int64(statement, 9, intervalMs); + OPSQLiteBindText(statement, 10, sessionTarget); + OPSQLiteBindText(statement, 11, OPJSONString(delivery)); + OPSQLiteBindText(statement, 12, notificationText); + OPSQLiteBindText(statement, 13, OPJSONString(payload)); + OPSQLiteBindText(statement, 14, reason); + OPSQLiteBindText(statement, 15, source); + sqlite3_bind_int64(statement, 16, 1); + if (sqlite3_step(statement) == SQLITE_DONE) { + jobId = sqlite3_last_insert_rowid(db); + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + if (jobId > 0 && OPSQLiteTableExists(db, @"agent_job_fts")) { + sqlite3_stmt *fts = NULL; + if (sqlite3_prepare_v2(db, + "INSERT INTO agent_job_fts(job_id, title, prompt, type) VALUES(?, ?, ?, ?)", + -1, &fts, NULL) == SQLITE_OK) { + sqlite3_bind_int64(fts, 1, jobId); + OPSQLiteBindText(fts, 2, title); + OPSQLiteBindText(fts, 3, prompt); + OPSQLiteBindText(fts, 4, type); + sqlite3_step(fts); + } + sqlite3_finalize(fts); + } + NSDictionary *job = jobId > 0 ? OPAgentJobRead(db, jobId) : nil; + sqlite3_close(db); + if (jobId == 0 || !job) { + return OPError([NSString stringWithFormat:@"background_job_create_failed:%@", error ?: @"unknown"]); + } + OPRecordContextEvent(@"background_job_created", @"openphone.agentd", taskId, + title, prompt, @{ + @"job_id": job[@"job_id"] ?: @"", + @"type": type, + @"scheduler_status": OPBackgroundJobSchedulerStatus() + }); + NSDictionary *result = @{ + @"status": @"ok", + @"job_id": job[@"id"] ?: @(jobId), + @"job": job, + @"scheduler_status": OPBackgroundJobSchedulerStatus(), + @"runner": @"deterministic", + @"runs_locally": @YES, + @"model_loop_status": @"not_started", + @"db_path": OPDatabasePath(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"background_job_created", taskId, @"background.run", @"allow_yolo", + request, [NSString stringWithFormat:@"job_id:%@", job[@"job_id"] ?: @""]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"background_job_create", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSArray *OPBackgroundJobListRows(sqlite3 *db, NSString *query, + NSUInteger limit, NSString **providerOut) { + NSMutableArray *jobs = [NSMutableArray array]; + BOOL hasQuery = query.length > 0; + BOOL ftsFailed = NO; + if (hasQuery && OPSQLiteTableExists(db, @"agent_job_fts")) { + NSString *ftsQuery = OPFTSQuery(query); + if (ftsQuery.length > 0) { + sqlite3_stmt *statement = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT j.id, j.created_at_ms, j.updated_at_ms, j.status, j.type, j.title, j.prompt, " + "j.schedule_json, j.next_run_at_ms, j.interval_ms, j.session_target, j.delivery_json, " + "j.notification_text, j.payload_json, j.reason, j.source, j.scheduler_enabled " + "FROM agent_job_fts f JOIN agent_job j ON f.job_id = j.id " + "WHERE agent_job_fts MATCH ? ORDER BY bm25(agent_job_fts) LIMIT ?", + -1, &statement, NULL); + if (rc == SQLITE_OK) { + OPSQLiteBindText(statement, 1, ftsQuery); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)limit); + while ((rc = sqlite3_step(statement)) == SQLITE_ROW) { + [jobs addObject:OPAgentJobFromStatement(statement)]; + } + if (rc != SQLITE_DONE) { + [jobs removeAllObjects]; + ftsFailed = YES; + } + } else { + ftsFailed = YES; + } + sqlite3_finalize(statement); + if (!ftsFailed) { + if (providerOut) { + *providerOut = @"sqlite_fts5"; + } + return jobs; + } + } + } + + sqlite3_stmt *statement = NULL; + NSString *sql = hasQuery + ? @"SELECT id, created_at_ms, updated_at_ms, status, type, title, prompt, schedule_json, " + "next_run_at_ms, interval_ms, session_target, delivery_json, notification_text, payload_json, " + "reason, source, scheduler_enabled FROM agent_job WHERE lower(title) LIKE ? OR lower(prompt) LIKE ? OR lower(type) LIKE ? " + "ORDER BY updated_at_ms DESC LIMIT ?" + : @"SELECT id, created_at_ms, updated_at_ms, status, type, title, prompt, schedule_json, " + "next_run_at_ms, interval_ms, session_target, delivery_json, notification_text, payload_json, " + "reason, source, scheduler_enabled FROM agent_job ORDER BY updated_at_ms DESC LIMIT ?"; + if (sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL) == SQLITE_OK) { + if (hasQuery) { + NSString *like = [NSString stringWithFormat:@"%%%@%%", query.lowercaseString ?: @""]; + OPSQLiteBindText(statement, 1, like); + OPSQLiteBindText(statement, 2, like); + OPSQLiteBindText(statement, 3, like); + sqlite3_bind_int64(statement, 4, (sqlite3_int64)limit); + } else { + sqlite3_bind_int64(statement, 1, (sqlite3_int64)limit); + } + while (sqlite3_step(statement) == SQLITE_ROW) { + [jobs addObject:OPAgentJobFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + if (providerOut) { + *providerOut = hasQuery ? @"sqlite_like" : @"sqlite_latest"; + } + return jobs; +} + +static NSDictionary *OPBackgroundJobList(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *query = [OPStringFromRequest(request, @"query", @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSUInteger limit = OPLimitFromRequest(request, 20, 200); + if (limit == 0) { + limit = 20; + } + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSString *provider = nil; + NSArray *jobs = OPBackgroundJobListRows(db, query, limit, &provider); + BOOL ftsAvailable = OPSQLiteTableExists(db, @"agent_job_fts"); + sqlite3_close(db); + NSDictionary *result = @{ + @"status": @"ok", + @"query": query ?: @"", + @"limit": @(limit), + @"jobs": jobs, + @"count": @(jobs.count), + @"provider": provider ?: @"sqlite", + @"fts_available": @(ftsAvailable), + @"scheduler_status": OPBackgroundJobSchedulerStatus(), + @"runner": @"deterministic", + @"model_loop_status": @"not_started", + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"background_jobs_listed", taskId, @"tasks.observe", @"allow_task_scoped", + request, [NSString stringWithFormat:@"count:%lu", (unsigned long)jobs.count]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"background_job_list", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSDictionary *OPBackgroundJobStop(NSDictionary *request) { + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + long long jobId = OPRecordIdFromRequest(request, @[@"job_id", @"id"]); + if (jobId <= 0) { + return OPError(@"missing_background_job_id"); + } + NSString *reason = OPStringFromRequest(request, @"reason", @""); + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE agent_job SET status = 'stopped', updated_at_ms = ?, reason = ? WHERE id = ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, OPNowMs()); + OPSQLiteBindText(statement, 2, reason); + sqlite3_bind_int64(statement, 3, jobId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + NSDictionary *job = updated ? OPAgentJobRead(db, jobId) : nil; + sqlite3_close(db); + if (!updated || !job) { + return OPError([NSString stringWithFormat:@"background_job_stop_failed:%@", error ?: @"not_found"]); + } + NSDictionary *result = @{ + @"status": @"ok", + @"stopped_count": @1, + @"job_id": job[@"id"] ?: @(jobId), + @"job": job, + @"scheduler_status": OPBackgroundJobSchedulerStatus(), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"background_job_stopped", taskId, @"background.run", @"allow_yolo", + request, [NSString stringWithFormat:@"job_id:%lld", jobId]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"background_job_stop", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSArray *OPCommitmentDueRows(sqlite3 *db, NSUInteger limit, long long nowMs) { + NSMutableArray *commitments = [NSMutableArray array]; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "SELECT id, created_at_ms, updated_at_ms, title, description, trigger_type, " + "trigger_spec_json, due_at_ms, expires_at_ms, status, confidence, evidence_json, source, reason " + "FROM commitment " + "WHERE status = 'active' AND due_at_ms > 0 AND due_at_ms <= ? " + "AND (expires_at_ms = 0 OR expires_at_ms >= ?) " + "ORDER BY due_at_ms ASC LIMIT ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, nowMs); + sqlite3_bind_int64(statement, 2, nowMs); + sqlite3_bind_int64(statement, 3, (sqlite3_int64)limit); + while (sqlite3_step(statement) == SQLITE_ROW) { + [commitments addObject:OPCommitmentFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + return commitments; +} + +static NSString *OPCommitmentPrompt(NSDictionary *commitment) { + NSDictionary *triggerSpec = [commitment[@"trigger_spec"] isKindOfClass:[NSDictionary class]] + ? commitment[@"trigger_spec"] : @{}; + for (NSString *key in @[@"prompt", @"goal"]) { + NSString *value = [triggerSpec[key] isKindOfClass:[NSString class]] + ? [triggerSpec[key] stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]] : @""; + if (value.length > 0) { + return value; + } + } + NSString *title = [commitment[@"title"] isKindOfClass:[NSString class]] + ? commitment[@"title"] : @"OpenPhone commitment"; + NSString *description = [commitment[@"description"] isKindOfClass:[NSString class]] + ? [commitment[@"description"] stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]] : @""; + if (description.length > 0) { + return [NSString stringWithFormat: + @"Handle due OpenPhone commitment '%@'. Description: %@. Use the current phone context. If no safe phone action is clear, finish with a concise status.", + title, description]; + } + return [NSString stringWithFormat: + @"Handle due OpenPhone commitment '%@' using the current phone context. If no safe phone action is clear, finish with a concise status.", + title]; +} + +static NSDictionary *OPCommitmentClaimDue(NSDictionary *commitment, long long claimedAtMs, + NSString *source, NSString **errorOut) { + long long commitmentId = [commitment[@"id"] longLongValue]; + if (commitmentId <= 0) { + if (errorOut) { + *errorOut = @"missing_commitment_id"; + } + return nil; + } + NSDictionary *existingEvidence = [commitment[@"evidence"] isKindOfClass:[NSDictionary class]] + ? commitment[@"evidence"] : @{}; + NSMutableDictionary *evidence = [existingEvidence mutableCopy]; + NSDictionary *existingScheduler = [evidence[@"scheduler"] isKindOfClass:[NSDictionary class]] + ? evidence[@"scheduler"] : @{}; + NSMutableDictionary *scheduler = [existingScheduler mutableCopy]; + long long attemptCount = [scheduler[@"trigger_attempt_count"] respondsToSelector:@selector(longLongValue)] + ? [scheduler[@"trigger_attempt_count"] longLongValue] + 1 : 1; + scheduler[@"scheduler_status"] = OPCommitmentSchedulerStatus(); + scheduler[@"last_claimed_at_ms"] = @(claimedAtMs); + scheduler[@"last_trigger_status"] = @"running"; + scheduler[@"last_trigger_source"] = source ?: @"commitment_scheduler"; + scheduler[@"trigger_attempt_count"] = @(attemptCount); + evidence[@"scheduler"] = scheduler; + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]; + } + return nil; + } + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE commitment SET status = 'running', updated_at_ms = ?, evidence_json = ? " + "WHERE id = ? AND status = 'active' AND due_at_ms > 0 AND due_at_ms <= ? " + "AND (expires_at_ms = 0 OR expires_at_ms >= ?)", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, claimedAtMs); + OPSQLiteBindText(statement, 2, OPJSONString(evidence)); + sqlite3_bind_int64(statement, 3, commitmentId); + sqlite3_bind_int64(statement, 4, claimedAtMs); + sqlite3_bind_int64(statement, 5, claimedAtMs); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + if (!updated && errorOut) { + *errorOut = @"commitment_claim_missed_active_due_row"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"commitment_claim_prepare_failed"; + } + sqlite3_finalize(statement); + NSDictionary *claimedCommitment = updated ? OPCommitmentRead(db, commitmentId) : nil; + sqlite3_close(db); + return claimedCommitment; +} + +static NSDictionary *OPCommitmentUpdateAfterTrigger(NSDictionary *commitment, + NSString *jobPublicId, long long triggeredAtMs, NSString *source, NSString **errorOut) { + long long commitmentId = [commitment[@"id"] longLongValue]; + if (commitmentId <= 0) { + if (errorOut) { + *errorOut = @"missing_commitment_id"; + } + return nil; + } + NSDictionary *existingEvidence = [commitment[@"evidence"] isKindOfClass:[NSDictionary class]] + ? commitment[@"evidence"] : @{}; + NSMutableDictionary *evidence = [existingEvidence mutableCopy]; + NSDictionary *existingScheduler = [evidence[@"scheduler"] isKindOfClass:[NSDictionary class]] + ? evidence[@"scheduler"] : @{}; + NSMutableDictionary *scheduler = [existingScheduler mutableCopy]; + long long triggerCount = [scheduler[@"trigger_count"] respondsToSelector:@selector(longLongValue)] + ? [scheduler[@"trigger_count"] longLongValue] + 1 : 1; + scheduler[@"scheduler_status"] = OPCommitmentSchedulerStatus(); + scheduler[@"last_trigger_status"] = @"background_job_queued"; + scheduler[@"last_triggered_at_ms"] = @(triggeredAtMs); + scheduler[@"last_job_id"] = jobPublicId ?: @""; + scheduler[@"trigger_count"] = @(triggerCount); + scheduler[@"source"] = source ?: @"commitment_scheduler"; + evidence[@"scheduler"] = scheduler; + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]; + } + return nil; + } + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE commitment SET status = 'triggered', updated_at_ms = ?, evidence_json = ?, " + "reason = ? WHERE id = ? AND status IN ('active', 'running')", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, triggeredAtMs); + OPSQLiteBindText(statement, 2, OPJSONString(evidence)); + OPSQLiteBindText(statement, 3, [NSString stringWithFormat:@"commitment triggered:%@", + source ?: @"commitment_scheduler"]); + sqlite3_bind_int64(statement, 4, commitmentId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + if (!updated && errorOut) { + *errorOut = @"commitment_update_missed_running_row"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"commitment_update_prepare_failed"; + } + sqlite3_finalize(statement); + NSDictionary *updatedCommitment = updated ? OPCommitmentRead(db, commitmentId) : nil; + sqlite3_close(db); + return updatedCommitment; +} + +static NSDictionary *OPCommitmentRequeueAfterTriggerFailure(NSDictionary *commitment, + NSString *failureReason, long long failedAtMs, NSString *source, NSString **errorOut) { + long long commitmentId = [commitment[@"id"] longLongValue]; + if (commitmentId <= 0) { + if (errorOut) { + *errorOut = @"missing_commitment_id"; + } + return nil; + } + NSDictionary *existingEvidence = [commitment[@"evidence"] isKindOfClass:[NSDictionary class]] + ? commitment[@"evidence"] : @{}; + NSMutableDictionary *evidence = [existingEvidence mutableCopy]; + NSDictionary *existingScheduler = [evidence[@"scheduler"] isKindOfClass:[NSDictionary class]] + ? evidence[@"scheduler"] : @{}; + NSMutableDictionary *scheduler = [existingScheduler mutableCopy]; + long long failureCount = [scheduler[@"trigger_failure_count"] respondsToSelector:@selector(longLongValue)] + ? [scheduler[@"trigger_failure_count"] longLongValue] + 1 : 1; + scheduler[@"scheduler_status"] = OPCommitmentSchedulerStatus(); + scheduler[@"last_trigger_status"] = @"background_job_create_failed"; + scheduler[@"last_trigger_error"] = failureReason ?: @"background_job_create_failed"; + scheduler[@"trigger_failure_count"] = @(failureCount); + scheduler[@"last_failed_at_ms"] = @(failedAtMs); + scheduler[@"source"] = source ?: @"commitment_scheduler"; + evidence[@"scheduler"] = scheduler; + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]; + } + return nil; + } + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE commitment SET status = 'active', updated_at_ms = ?, evidence_json = ?, " + "reason = ? WHERE id = ? AND status = 'running'", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, failedAtMs); + OPSQLiteBindText(statement, 2, OPJSONString(evidence)); + OPSQLiteBindText(statement, 3, [NSString stringWithFormat:@"commitment trigger failed:%@", + failureReason ?: @"background_job_create_failed"]); + sqlite3_bind_int64(statement, 4, commitmentId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + if (!updated && errorOut) { + *errorOut = @"commitment_failure_requeue_missed_running_row"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"commitment_failure_requeue_prepare_failed"; + } + sqlite3_finalize(statement); + NSDictionary *updatedCommitment = updated ? OPCommitmentRead(db, commitmentId) : nil; + sqlite3_close(db); + return updatedCommitment; +} + +static NSDictionary *OPCommitmentMaterializeDue(NSUInteger limit, long long nowMs, + NSString *taskId, NSString *source, NSDictionary *requestDelivery) { + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSArray *dueCommitments = OPCommitmentDueRows(db, limit, nowMs); + sqlite3_close(db); + + NSMutableArray *entries = [NSMutableArray array]; + NSUInteger triggeredCount = 0; + NSUInteger jobCount = 0; + for (NSDictionary *commitment in dueCommitments) { + NSString *commitmentPublicId = [commitment[@"commitment_id"] isKindOfClass:[NSString class]] + ? commitment[@"commitment_id"] : [NSString stringWithFormat:@"ios-commitment-%lld", + [commitment[@"id"] longLongValue]]; + NSString *title = [commitment[@"title"] isKindOfClass:[NSString class]] + ? commitment[@"title"] : commitmentPublicId; + NSDictionary *triggerSpec = [commitment[@"trigger_spec"] isKindOfClass:[NSDictionary class]] + ? commitment[@"trigger_spec"] : @{}; + NSDictionary *delivery = [triggerSpec[@"delivery"] isKindOfClass:[NSDictionary class]] + ? triggerSpec[@"delivery"] + : ([requestDelivery isKindOfClass:[NSDictionary class]] ? requestDelivery : @{@"mode": @"notification"}); + NSDictionary *event = @{ + @"type": @"time", + @"commitment_id": commitmentPublicId, + @"commitment_row_id": commitment[@"id"] ?: @0, + @"title": title ?: commitmentPublicId, + @"trigger_type": commitment[@"trigger_type"] ?: @"time", + @"triggered_at_ms": @(nowMs), + @"due_at_ms": commitment[@"due_at_ms"] ?: @0 + }; + NSString *claimError = nil; + NSDictionary *claimedCommitment = OPCommitmentClaimDue(commitment, nowMs, + source ?: @"commitment_scheduler", &claimError); + if (!claimedCommitment) { + [entries addObject:@{ + @"commitment_id": commitmentPublicId, + @"status": @"skipped", + @"reason": claimError ?: @"commitment_claim_failed", + @"event": event + }]; + continue; + } + + NSString *prompt = OPCommitmentPrompt(claimedCommitment); + NSMutableDictionary *payload = [@{ + @"source": source ?: @"commitment_scheduler", + @"commitment_id": commitmentPublicId, + @"commitment_row_id": claimedCommitment[@"id"] ?: @0, + @"event": event, + @"trigger_spec": triggerSpec, + @"delivery": delivery + } mutableCopy]; + NSMutableDictionary *jobRequest = [@{ + @"command": @"background_job_create", + @"task_id": taskId ?: @"", + @"title": [NSString stringWithFormat:@"Commitment due: %@", title ?: commitmentPublicId], + @"prompt": prompt, + @"type": @"commitment_due", + @"next_run_at": @(nowMs), + @"source": @"commitment_scheduler", + @"reason": [NSString stringWithFormat:@"commitment due:%@", commitmentPublicId], + @"payload": payload, + @"delivery": delivery + } mutableCopy]; + NSString *notificationText = [delivery[@"notification_text"] isKindOfClass:[NSString class]] + ? delivery[@"notification_text"] : @""; + if (notificationText.length > 0) { + jobRequest[@"notification_text"] = notificationText; + } + + NSDictionary *jobResult = OPBackgroundJobCreate(jobRequest); + NSMutableDictionary *entry = [@{ + @"commitment_id": commitmentPublicId, + @"status": @"failed", + @"event": event + } mutableCopy]; + if ([jobResult[@"status"] isEqualToString:@"ok"]) { + triggeredCount++; + jobCount++; + NSDictionary *job = [jobResult[@"job"] isKindOfClass:[NSDictionary class]] + ? jobResult[@"job"] : @{}; + NSString *jobPublicId = [job[@"job_id"] isKindOfClass:[NSString class]] + ? job[@"job_id"] : @""; + NSString *updateError = nil; + NSDictionary *updatedCommitment = OPCommitmentUpdateAfterTrigger(claimedCommitment, + jobPublicId, nowMs, source ?: @"commitment_scheduler", &updateError); + entry[@"status"] = @"background_job_queued"; + entry[@"job_id"] = jobPublicId; + entry[@"job"] = job; + if (updatedCommitment) { + entry[@"commitment"] = updatedCommitment; + } + if (updateError) { + entry[@"update_error"] = updateError; + } + OPRecordContextEvent(@"commitment_triggered", @"openphone.agentd", taskId, + title ?: commitmentPublicId, prompt, @{ + @"commitment_id": commitmentPublicId, + @"job_id": jobPublicId ?: @"", + @"scheduler_status": OPCommitmentSchedulerStatus(), + @"source": source ?: @"commitment_scheduler" + }); + OPRecordAudit(@"commitment_triggered", taskId, @"commitments.write", @"allow_yolo", + jobRequest, [NSString stringWithFormat:@"commitment_id:%@ job_id:%@", + commitmentPublicId, jobPublicId ?: @""]); + } else { + NSString *failureReason = [jobResult[@"reason"] isKindOfClass:[NSString class]] + ? jobResult[@"reason"] : @"background_job_create_failed"; + entry[@"reason"] = failureReason; + entry[@"job_result"] = jobResult ?: @{}; + NSString *failureUpdateError = nil; + NSDictionary *requeuedCommitment = OPCommitmentRequeueAfterTriggerFailure(claimedCommitment, + failureReason, nowMs, source ?: @"commitment_scheduler", &failureUpdateError); + if (requeuedCommitment) { + entry[@"commitment"] = requeuedCommitment; + } + if (failureUpdateError) { + entry[@"update_error"] = failureUpdateError; + } + } + [entries addObject:entry]; + } + + return @{ + @"status": @"ok", + @"scheduler_status": OPCommitmentSchedulerStatus(), + @"source": @"openphone.agentd", + @"limit": @(limit), + @"due_count": @(dueCommitments.count), + @"triggered_count": @(triggeredCount), + @"job_count": @(jobCount), + @"commitments": entries + }; +} + +static pthread_mutex_t OPBackgroundJobSchedulerMutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_t OPBackgroundJobSchedulerThread; +static volatile int OPBackgroundJobSchedulerThreadStarted = 0; +static const int OPBackgroundJobSchedulerStartupDelaySeconds = 15; + +static NSArray *OPBackgroundJobDueRows(sqlite3 *db, NSUInteger limit, + long long nowMs, long long targetJobId) { + NSMutableArray *jobs = [NSMutableArray array]; + sqlite3_stmt *statement = NULL; + NSString *sql = targetJobId > 0 + ? @"SELECT id, created_at_ms, updated_at_ms, status, type, title, prompt, schedule_json, " + "next_run_at_ms, interval_ms, session_target, delivery_json, notification_text, payload_json, " + "reason, source, scheduler_enabled FROM agent_job " + "WHERE id = ? AND scheduler_enabled = 1 AND status = 'queued' " + "AND (next_run_at_ms = 0 OR next_run_at_ms <= ?) LIMIT 1" + : @"SELECT id, created_at_ms, updated_at_ms, status, type, title, prompt, schedule_json, " + "next_run_at_ms, interval_ms, session_target, delivery_json, notification_text, payload_json, " + "reason, source, scheduler_enabled FROM agent_job " + "WHERE scheduler_enabled = 1 AND status = 'queued' AND (next_run_at_ms = 0 OR next_run_at_ms <= ?) " + "ORDER BY CASE WHEN next_run_at_ms = 0 THEN created_at_ms ELSE next_run_at_ms END ASC " + "LIMIT ?"; + if (sqlite3_prepare_v2(db, sql.UTF8String, -1, &statement, NULL) == SQLITE_OK) { + if (targetJobId > 0) { + sqlite3_bind_int64(statement, 1, targetJobId); + sqlite3_bind_int64(statement, 2, nowMs); + } else { + sqlite3_bind_int64(statement, 1, nowMs); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)limit); + } + while (sqlite3_step(statement) == SQLITE_ROW) { + [jobs addObject:OPAgentJobFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + return jobs; +} + +static BOOL OPBackgroundJobClaim(long long jobId, NSString **errorOut) { + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]; + } + return NO; + } + sqlite3_stmt *statement = NULL; + BOOL claimed = NO; + if (sqlite3_prepare_v2(db, + "UPDATE agent_job SET status = 'running', updated_at_ms = ? " + "WHERE id = ? AND status = 'queued' AND scheduler_enabled = 1", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, OPNowMs()); + sqlite3_bind_int64(statement, 2, jobId); + claimed = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + if (!claimed && errorOut) { + *errorOut = @"not_queued"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"claim_prepare_failed"; + } + sqlite3_finalize(statement); + sqlite3_close(db); + return claimed; +} + +static long long OPBackgroundJobRetryBackoffMs(long long failureCount) { + long long backoffMs = 30000; + long long shifts = failureCount > 1 ? MIN(failureCount - 1, 6) : 0; + for (long long i = 0; i < shifts; i++) { + backoffMs *= 2; + } + return MIN(backoffMs, 3600000); +} + +static NSDictionary *OPBackgroundJobFinish(NSDictionary *job, NSDictionary *runResult, + NSString **errorOut) { + long long jobId = [job[@"id"] longLongValue]; + if (jobId <= 0) { + if (errorOut) { + *errorOut = @"missing_job_id"; + } + return nil; + } + BOOL taskFinished = [runResult[@"status"] isEqualToString:@"task.finished"]; + long long intervalMs = [job[@"interval_ms"] isKindOfClass:[NSNumber class]] + ? [job[@"interval_ms"] longLongValue] : 0; + NSDictionary *schedule = [job[@"schedule"] isKindOfClass:[NSDictionary class]] + ? job[@"schedule"] : @{}; + BOOL recurring = intervalMs > 0 && + (![schedule[@"recurring"] respondsToSelector:@selector(boolValue)] || + [schedule[@"recurring"] boolValue]); + long long now = OPNowMs(); + NSDictionary *existingPayload = [job[@"payload"] isKindOfClass:[NSDictionary class]] + ? job[@"payload"] : @{}; + NSDictionary *existingScheduler = [existingPayload[@"scheduler"] isKindOfClass:[NSDictionary class]] + ? existingPayload[@"scheduler"] : @{}; + long long previousFailureCount = [existingScheduler[@"failure_count"] respondsToSelector:@selector(longLongValue)] + ? [existingScheduler[@"failure_count"] longLongValue] : 0; + long long failureCount = taskFinished ? 0 : previousFailureCount + 1; + long long retryBackoffMs = (!taskFinished && recurring) + ? OPBackgroundJobRetryBackoffMs(failureCount) : 0; + NSString *runPolicy = recurring + ? (taskFinished ? @"recurring_interval" : @"recurring_failure_backoff") + : @"terminal"; + NSString *finalStatus = recurring ? @"queued" : (taskFinished ? @"completed" : @"failed"); + long long nextRunAt = recurring + ? (now + (taskFinished ? intervalMs : retryBackoffMs)) : 0; + NSString *runner = [runResult[@"runner"] isKindOfClass:[NSString class]] + ? runResult[@"runner"] : @"auto"; + NSString *modelLoopStatus = [runResult[@"model_loop_status"] isKindOfClass:[NSString class]] + ? runResult[@"model_loop_status"] + : ([runner isEqualToString:@"model"] ? (taskFinished ? @"finished" : @"failed") : @"not_started"); + + NSMutableDictionary *payload = [existingPayload mutableCopy]; + payload[@"scheduler"] = @{ + @"status": OPBackgroundJobSchedulerStatus(), + @"runner": runner ?: @"auto", + @"model_loop_status": modelLoopStatus ?: @"unknown", + @"last_run_at_ms": @(now), + @"last_status": runResult[@"status"] ?: @"unknown", + @"last_task_id": runResult[@"task_id"] ?: @"", + @"last_stop_reason": runResult[@"stop_reason"] ?: @"", + @"recurring": @(recurring), + @"interval_ms": @(intervalMs), + @"next_run_at_ms": @(nextRunAt), + @"run_policy": runPolicy, + @"failure_count": @(failureCount), + @"retry_backoff_ms": @(retryBackoffMs) + }; + payload[@"last_run_task"] = runResult ?: @{}; + + NSMutableDictionary *updatedSchedule = [schedule mutableCopy]; + updatedSchedule[@"scheduler_status"] = OPBackgroundJobSchedulerStatus(); + updatedSchedule[@"scheduler_enabled"] = @YES; + updatedSchedule[@"last_run_at_ms"] = @(now); + updatedSchedule[@"last_status"] = runResult[@"status"] ?: @"unknown"; + updatedSchedule[@"last_stop_reason"] = runResult[@"stop_reason"] ?: @""; + updatedSchedule[@"run_policy"] = runPolicy; + updatedSchedule[@"recurring"] = @(recurring); + updatedSchedule[@"interval_ms"] = @(intervalMs); + updatedSchedule[@"next_run_at"] = @(nextRunAt); + updatedSchedule[@"failure_count"] = @(failureCount); + updatedSchedule[@"retry_backoff_ms"] = @(retryBackoffMs); + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]; + } + return nil; + } + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE agent_job SET status = ?, updated_at_ms = ?, next_run_at_ms = ?, " + "schedule_json = ?, payload_json = ? " + "WHERE id = ? AND status = 'running'", + -1, &statement, NULL) == SQLITE_OK) { + OPSQLiteBindText(statement, 1, finalStatus); + sqlite3_bind_int64(statement, 2, now); + sqlite3_bind_int64(statement, 3, nextRunAt); + OPSQLiteBindText(statement, 4, OPJSONString(updatedSchedule)); + OPSQLiteBindText(statement, 5, OPJSONString(payload)); + sqlite3_bind_int64(statement, 6, jobId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + if (!updated && errorOut) { + *errorOut = @"finish_update_missed_running_row"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"finish_prepare_failed"; + } + sqlite3_finalize(statement); + NSDictionary *updatedJob = updated ? OPAgentJobRead(db, jobId) : nil; + sqlite3_close(db); + return updatedJob; +} + +static NSArray *OPBackgroundJobStaleRunningRows(sqlite3 *db, + NSUInteger limit, long long cutoffMs) { + NSMutableArray *jobs = [NSMutableArray array]; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "SELECT id, created_at_ms, updated_at_ms, status, type, title, prompt, schedule_json, " + "next_run_at_ms, interval_ms, session_target, delivery_json, notification_text, payload_json, " + "reason, source, scheduler_enabled FROM agent_job " + "WHERE scheduler_enabled = 1 AND status = 'running' AND updated_at_ms <= ? " + "ORDER BY updated_at_ms ASC LIMIT ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, cutoffMs); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)limit); + while (sqlite3_step(statement) == SQLITE_ROW) { + [jobs addObject:OPAgentJobFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + return jobs; +} + +static NSDictionary *OPBackgroundJobRepairStuckInternal(NSUInteger limit, + long long staleAfterMs, NSString *taskId, NSString *source) { + if (limit == 0) { + limit = 1; + } + long long now = OPNowMs(); + long long cutoff = now - staleAfterMs; + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSArray *staleJobs = OPBackgroundJobStaleRunningRows(db, limit, cutoff); + + NSMutableArray *entries = [NSMutableArray array]; + NSUInteger repairedCount = 0; + for (NSDictionary *job in staleJobs) { + long long jobId = [job[@"id"] longLongValue]; + NSString *jobPublicId = [job[@"job_id"] isKindOfClass:[NSString class]] + ? job[@"job_id"] : [NSString stringWithFormat:@"ios-job-%lld", jobId]; + long long updatedAt = [job[@"updated_at_ms"] respondsToSelector:@selector(longLongValue)] + ? [job[@"updated_at_ms"] longLongValue] : 0; + long long ageMs = updatedAt > 0 ? MAX(0, now - updatedAt) : 0; + + NSDictionary *existingPayload = [job[@"payload"] isKindOfClass:[NSDictionary class]] + ? job[@"payload"] : @{}; + NSMutableDictionary *payload = [existingPayload mutableCopy]; + NSDictionary *existingRepair = [payload[@"stuck_repair"] isKindOfClass:[NSDictionary class]] + ? payload[@"stuck_repair"] : @{}; + long long repairCount = [existingRepair[@"repair_count"] respondsToSelector:@selector(longLongValue)] + ? [existingRepair[@"repair_count"] longLongValue] + 1 : 1; + payload[@"stuck_repair"] = @{ + @"status": @"requeued", + @"repair_action": @"requeued", + @"repair_count": @(repairCount), + @"last_repair_at_ms": @(now), + @"stale_after_ms": @(staleAfterMs), + @"stale_running_age_ms": @(ageMs), + @"previous_status": @"running", + @"source": source ?: @"background_job_repair" + }; + + NSDictionary *existingSchedule = [job[@"schedule"] isKindOfClass:[NSDictionary class]] + ? job[@"schedule"] : @{}; + NSMutableDictionary *schedule = [existingSchedule mutableCopy]; + schedule[@"last_repair_at_ms"] = @(now); + schedule[@"stuck_repair_count"] = @(repairCount); + schedule[@"scheduler_status"] = OPBackgroundJobSchedulerStatus(); + schedule[@"next_run_at"] = @(now); + + sqlite3_stmt *update = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE agent_job SET status = 'queued', updated_at_ms = ?, next_run_at_ms = ?, " + "schedule_json = ?, payload_json = ?, reason = ? " + "WHERE id = ? AND status = 'running' AND scheduler_enabled = 1", + -1, &update, NULL) == SQLITE_OK) { + sqlite3_bind_int64(update, 1, now); + sqlite3_bind_int64(update, 2, now); + OPSQLiteBindText(update, 3, OPJSONString(schedule)); + OPSQLiteBindText(update, 4, OPJSONString(payload)); + OPSQLiteBindText(update, 5, [NSString stringWithFormat:@"stuck repair:%@", + source ?: @"background_job_repair"]); + sqlite3_bind_int64(update, 6, jobId); + updated = sqlite3_step(update) == SQLITE_DONE && sqlite3_changes(db) > 0; + } + sqlite3_finalize(update); + + NSDictionary *updatedJob = updated ? OPAgentJobRead(db, jobId) : nil; + NSMutableDictionary *entry = [@{ + @"job_id": jobPublicId, + @"status": updated ? @"requeued" : @"skipped", + @"repair_action": updated ? @"requeued" : @"not_repaired", + @"stale_running_age_ms": @(ageMs) + } mutableCopy]; + if (updatedJob) { + entry[@"job"] = updatedJob; + } + if (updated) { + repairedCount++; + OPRecordContextEvent(@"background_job_repaired", @"openphone.agentd", taskId, + job[@"title"] ?: jobPublicId, @"stale running job requeued", @{ + @"job_id": jobPublicId, + @"repair_action": @"requeued", + @"stale_running_age_ms": @(ageMs), + @"source": source ?: @"background_job_repair" + }); + OPRecordAudit(@"background_job_repaired", taskId, @"background.run", + @"allow_yolo", @{@"job_id": jobPublicId, @"source": source ?: @""}, + [NSString stringWithFormat:@"job_id:%@ action:requeued", jobPublicId]); + } + [entries addObject:entry]; + } + sqlite3_close(db); + + return @{ + @"status": @"ok", + @"scheduler_status": OPBackgroundJobSchedulerStatus(), + @"repair_policy": @"requeue_stale_running", + @"stale_after_ms": @(staleAfterMs), + @"cutoff_ms": @(cutoff), + @"limit": @(limit), + @"stale_count": @(staleJobs.count), + @"repaired_count": @(repairedCount), + @"jobs": entries, + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPBackgroundJobRepairStuck(NSDictionary *request) { + if (pthread_mutex_trylock(&OPBackgroundJobSchedulerMutex) != 0) { + return @{ + @"status": @"ok", + @"scheduler_status": OPBackgroundJobSchedulerStatus(), + @"busy": @YES, + @"repaired_count": @0, + @"source": @"openphone.agentd" + }; + } + @try { + NSUInteger limit = OPLimitFromRequest(request, 5, 25); + if (limit == 0) { + limit = 5; + } + long long staleAfterMs = OPLongLongFromRequest(request, + @"stale_after_ms", 300000, 1000, 86400000LL); + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *source = OPStringFromRequest(request, @"source", @"background_job_repair"); + return OPBackgroundJobRepairStuckInternal(limit, staleAfterMs, taskId, source); + } @finally { + pthread_mutex_unlock(&OPBackgroundJobSchedulerMutex); + } +} + +static NSDictionary *OPBackgroundJobDebugMarkRunning(NSDictionary *request) { + if (!OPBoolFromRequest(request, @"validation", NO)) { + return OPError(@"validation_required"); + } + long long jobId = OPRecordIdFromRequest(request, @[@"job_id", @"id"]); + if (jobId <= 0) { + return OPError(@"missing_background_job_id"); + } + long long ageMs = OPLongLongFromRequest(request, @"age_ms", 600000, 1000, 86400000LL); + long long now = OPNowMs(); + long long updatedAt = now - ageMs; + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *source = OPStringFromRequest(request, @"source", @"validation_stuck_fixture"); + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSDictionary *job = OPAgentJobRead(db, jobId); + if (!job) { + sqlite3_close(db); + return OPError(@"background_job_not_found"); + } + NSDictionary *existingPayload = [job[@"payload"] isKindOfClass:[NSDictionary class]] + ? job[@"payload"] : @{}; + NSMutableDictionary *payload = [existingPayload mutableCopy]; + payload[@"validation_stuck_fixture"] = @{ + @"status": @"marked_running", + @"marked_at_ms": @(now), + @"updated_at_ms": @(updatedAt), + @"age_ms": @(ageMs), + @"source": source ?: @"validation_stuck_fixture" + }; + + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE agent_job SET status = 'running', updated_at_ms = ?, payload_json = ?, " + "reason = ? WHERE id = ? AND scheduler_enabled = 1", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, updatedAt); + OPSQLiteBindText(statement, 2, OPJSONString(payload)); + OPSQLiteBindText(statement, 3, [NSString stringWithFormat:@"validation stuck fixture:%@", + source ?: @"validation"]); + sqlite3_bind_int64(statement, 4, jobId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + NSDictionary *updatedJob = updated ? OPAgentJobRead(db, jobId) : nil; + sqlite3_close(db); + if (!updated || !updatedJob) { + return OPError([NSString stringWithFormat:@"background_job_debug_mark_running_failed:%@", + error ?: @"not_updated"]); + } + + NSDictionary *result = @{ + @"status": @"ok", + @"job_id": updatedJob[@"id"] ?: @(jobId), + @"job": updatedJob, + @"age_ms": @(ageMs), + @"updated_at_ms": @(updatedAt), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"background_job_debug_marked_running", taskId, @"background.run", + @"allow_yolo", request, [NSString stringWithFormat:@"job_id:%lld age_ms:%lld", + jobId, ageMs]); + return result; +} + +static long long OPWatcherRetryBackoffMs(long long failureCount) { + long long backoffMs = 30000; + long long shifts = failureCount > 1 ? MIN(failureCount - 1, 6) : 0; + for (long long i = 0; i < shifts; i++) { + backoffMs *= 2; + } + return MIN(backoffMs, 3600000); +} + +static NSDictionary *OPWatcherClaimForFire(NSDictionary *watcher, long long claimedAtMs, + NSString *source, NSString **errorOut) { + long long watcherId = [watcher[@"id"] longLongValue]; + if (watcherId <= 0) { + if (errorOut) { + *errorOut = @"missing_watcher_id"; + } + return nil; + } + NSDictionary *existingSchedule = [watcher[@"schedule"] isKindOfClass:[NSDictionary class]] + ? watcher[@"schedule"] : @{}; + NSMutableDictionary *schedule = [existingSchedule mutableCopy]; + long long fireAttemptCount = [schedule[@"fire_attempt_count"] respondsToSelector:@selector(longLongValue)] + ? [schedule[@"fire_attempt_count"] longLongValue] + 1 : 1; + schedule[@"last_claimed_at_ms"] = @(claimedAtMs); + schedule[@"fire_attempt_count"] = @(fireAttemptCount); + schedule[@"scheduler_status"] = OPWatcherSchedulerStatus(); + + NSDictionary *existingMetadata = [watcher[@"metadata"] isKindOfClass:[NSDictionary class]] + ? watcher[@"metadata"] : @{}; + NSMutableDictionary *metadata = [existingMetadata mutableCopy]; + metadata[@"scheduler_status"] = OPWatcherSchedulerStatus(); + metadata[@"fires_locally"] = @YES; + metadata[@"last_claimed_at_ms"] = @(claimedAtMs); + metadata[@"last_fire_status"] = @"running"; + metadata[@"fire_attempt_count"] = @(fireAttemptCount); + metadata[@"source"] = source ?: @"watcher_scheduler"; + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]; + } + return nil; + } + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE watcher SET status = 'running', updated_at_ms = ?, schedule_json = ?, " + "metadata_json = ? WHERE id = ? AND status = 'active'", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, claimedAtMs); + OPSQLiteBindText(statement, 2, OPJSONString(schedule)); + OPSQLiteBindText(statement, 3, OPJSONString(metadata)); + sqlite3_bind_int64(statement, 4, watcherId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + if (!updated && errorOut) { + *errorOut = @"watcher_claim_missed_active_row"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"watcher_claim_prepare_failed"; + } + sqlite3_finalize(statement); + NSDictionary *claimedWatcher = updated ? OPWatcherRead(db, watcherId) : nil; + sqlite3_close(db); + return claimedWatcher; +} + +static NSDictionary *OPWatcherRequeueAfterFireFailure(NSDictionary *watcher, NSString *failureReason, + long long failedAtMs, NSString *source, NSString **errorOut) { + long long watcherId = [watcher[@"id"] longLongValue]; + if (watcherId <= 0) { + if (errorOut) { + *errorOut = @"missing_watcher_id"; + } + return nil; + } + NSDictionary *existingMetadata = [watcher[@"metadata"] isKindOfClass:[NSDictionary class]] + ? watcher[@"metadata"] : @{}; + NSMutableDictionary *metadata = [existingMetadata mutableCopy]; + long long failureCount = [metadata[@"fire_failure_count"] respondsToSelector:@selector(longLongValue)] + ? [metadata[@"fire_failure_count"] longLongValue] + 1 : 1; + long long backoffMs = OPWatcherRetryBackoffMs(failureCount); + long long nextRunAt = failedAtMs + backoffMs; + metadata[@"scheduler_status"] = OPWatcherSchedulerStatus(); + metadata[@"fires_locally"] = @YES; + metadata[@"last_fire_status"] = @"background_job_create_failed"; + metadata[@"last_fire_error"] = failureReason ?: @"background_job_create_failed"; + metadata[@"fire_failure_count"] = @(failureCount); + metadata[@"retry_backoff_ms"] = @(backoffMs); + metadata[@"next_run_at_ms"] = @(nextRunAt); + metadata[@"source"] = source ?: @"watcher_scheduler"; + + NSDictionary *existingSchedule = [watcher[@"schedule"] isKindOfClass:[NSDictionary class]] + ? watcher[@"schedule"] : @{}; + NSMutableDictionary *schedule = [existingSchedule mutableCopy]; + schedule[@"last_fire_status"] = @"background_job_create_failed"; + schedule[@"last_fire_error"] = failureReason ?: @"background_job_create_failed"; + schedule[@"fire_failure_count"] = @(failureCount); + schedule[@"retry_backoff_ms"] = @(backoffMs); + schedule[@"next_run_at"] = @(nextRunAt); + schedule[@"scheduler_status"] = OPWatcherSchedulerStatus(); + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]; + } + return nil; + } + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE watcher SET status = 'active', updated_at_ms = ?, schedule_json = ?, " + "next_run_at_ms = ?, metadata_json = ? WHERE id = ? AND status = 'running'", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, failedAtMs); + OPSQLiteBindText(statement, 2, OPJSONString(schedule)); + sqlite3_bind_int64(statement, 3, nextRunAt); + OPSQLiteBindText(statement, 4, OPJSONString(metadata)); + sqlite3_bind_int64(statement, 5, watcherId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + if (!updated && errorOut) { + *errorOut = @"watcher_failure_requeue_missed_running_row"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"watcher_failure_requeue_prepare_failed"; + } + sqlite3_finalize(statement); + NSDictionary *updatedWatcher = updated ? OPWatcherRead(db, watcherId) : nil; + sqlite3_close(db); + return updatedWatcher; +} + +static NSArray *OPWatcherStaleRunningRows(sqlite3 *db, + NSUInteger limit, long long cutoffMs) { + NSMutableArray *watchers = [NSMutableArray array]; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "SELECT id, created_at_ms, updated_at_ms, status, source, type, evaluator, title, " + "query, url, address, number, condition_json, schedule_json, delivery_json, " + "next_run_at_ms, interval_ms, recurring, reason, metadata_json " + "FROM watcher " + "WHERE status = 'running' AND updated_at_ms <= ? " + "AND (lower(source) IN ('time', 'timer', 'deadline') " + "OR lower(type) IN ('time', 'timer', 'deadline')) " + "ORDER BY updated_at_ms ASC LIMIT ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, cutoffMs); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)limit); + while (sqlite3_step(statement) == SQLITE_ROW) { + [watchers addObject:OPWatcherFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + return watchers; +} + +static NSDictionary *OPWatcherRepairStuckInternal(NSUInteger limit, + long long staleAfterMs, NSString *taskId, NSString *source) { + if (limit == 0) { + limit = 1; + } + long long now = OPNowMs(); + long long cutoff = now - staleAfterMs; + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSArray *staleWatchers = OPWatcherStaleRunningRows(db, limit, cutoff); + + NSMutableArray *entries = [NSMutableArray array]; + NSUInteger repairedCount = 0; + for (NSDictionary *watcher in staleWatchers) { + long long watcherId = [watcher[@"id"] longLongValue]; + NSString *watcherPublicId = [watcher[@"watcher_id"] isKindOfClass:[NSString class]] + ? watcher[@"watcher_id"] : [NSString stringWithFormat:@"ios-watcher-%lld", watcherId]; + long long updatedAt = [watcher[@"updated_at_ms"] respondsToSelector:@selector(longLongValue)] + ? [watcher[@"updated_at_ms"] longLongValue] : 0; + long long ageMs = updatedAt > 0 ? MAX(0, now - updatedAt) : 0; + + NSDictionary *existingMetadata = [watcher[@"metadata"] isKindOfClass:[NSDictionary class]] + ? watcher[@"metadata"] : @{}; + NSMutableDictionary *metadata = [existingMetadata mutableCopy]; + NSDictionary *existingRepair = [metadata[@"stuck_repair"] isKindOfClass:[NSDictionary class]] + ? metadata[@"stuck_repair"] : @{}; + long long repairCount = [existingRepair[@"repair_count"] respondsToSelector:@selector(longLongValue)] + ? [existingRepair[@"repair_count"] longLongValue] + 1 : 1; + metadata[@"scheduler_status"] = OPWatcherSchedulerStatus(); + metadata[@"fires_locally"] = @YES; + metadata[@"last_fire_status"] = @"requeued_stale_running"; + metadata[@"next_run_at_ms"] = @(now); + metadata[@"stuck_repair"] = @{ + @"status": @"requeued", + @"repair_action": @"requeued", + @"repair_count": @(repairCount), + @"last_repair_at_ms": @(now), + @"stale_after_ms": @(staleAfterMs), + @"stale_running_age_ms": @(ageMs), + @"previous_status": @"running", + @"source": source ?: @"watcher_repair" + }; + + NSDictionary *existingSchedule = [watcher[@"schedule"] isKindOfClass:[NSDictionary class]] + ? watcher[@"schedule"] : @{}; + NSMutableDictionary *schedule = [existingSchedule mutableCopy]; + schedule[@"last_repair_at_ms"] = @(now); + schedule[@"stuck_repair_count"] = @(repairCount); + schedule[@"scheduler_status"] = OPWatcherSchedulerStatus(); + schedule[@"next_run_at"] = @(now); + + sqlite3_stmt *update = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE watcher SET status = 'active', updated_at_ms = ?, next_run_at_ms = ?, " + "schedule_json = ?, metadata_json = ?, reason = ? " + "WHERE id = ? AND status = 'running'", + -1, &update, NULL) == SQLITE_OK) { + sqlite3_bind_int64(update, 1, now); + sqlite3_bind_int64(update, 2, now); + OPSQLiteBindText(update, 3, OPJSONString(schedule)); + OPSQLiteBindText(update, 4, OPJSONString(metadata)); + OPSQLiteBindText(update, 5, [NSString stringWithFormat:@"stuck repair:%@", + source ?: @"watcher_repair"]); + sqlite3_bind_int64(update, 6, watcherId); + updated = sqlite3_step(update) == SQLITE_DONE && sqlite3_changes(db) > 0; + } + sqlite3_finalize(update); + + NSDictionary *updatedWatcher = updated ? OPWatcherRead(db, watcherId) : nil; + NSMutableDictionary *entry = [@{ + @"watcher_id": watcherPublicId, + @"status": updated ? @"requeued" : @"skipped", + @"repair_action": updated ? @"requeued" : @"not_repaired", + @"stale_running_age_ms": @(ageMs) + } mutableCopy]; + if (updatedWatcher) { + entry[@"watcher"] = updatedWatcher; + } + if (updated) { + repairedCount++; + OPRecordContextEvent(@"watcher_repaired", @"openphone.agentd", taskId, + watcher[@"title"] ?: watcherPublicId, @"stale running watcher requeued", @{ + @"watcher_id": watcherPublicId, + @"repair_action": @"requeued", + @"stale_running_age_ms": @(ageMs), + @"source": source ?: @"watcher_repair" + }); + OPRecordAudit(@"watcher_repaired", taskId, @"watchers.write", + @"allow_yolo", @{@"watcher_id": watcherPublicId, @"source": source ?: @""}, + [NSString stringWithFormat:@"watcher_id:%@ action:requeued", watcherPublicId]); + } + [entries addObject:entry]; + } + sqlite3_close(db); + + return @{ + @"status": @"ok", + @"scheduler_status": OPWatcherSchedulerStatus(), + @"repair_policy": @"requeue_stale_running", + @"stale_after_ms": @(staleAfterMs), + @"cutoff_ms": @(cutoff), + @"limit": @(limit), + @"stale_count": @(staleWatchers.count), + @"repaired_count": @(repairedCount), + @"watchers": entries, + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPWatcherRepairStuck(NSDictionary *request) { + NSUInteger limit = OPLimitFromRequest(request, 5, 25); + if (limit == 0) { + limit = 5; + } + long long staleAfterMs = OPLongLongFromRequest(request, + @"stale_after_ms", 300000, 1000, 86400000LL); + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *source = OPStringFromRequest(request, @"source", @"watcher_repair"); + return OPWatcherRepairStuckInternal(limit, staleAfterMs, taskId, source); +} + +static NSDictionary *OPWatcherDebugMarkRunning(NSDictionary *request) { + if (!OPBoolFromRequest(request, @"validation", NO)) { + return OPError(@"validation_required"); + } + long long watcherId = OPRecordIdFromRequest(request, @[@"watcher_id", @"id"]); + if (watcherId <= 0) { + return OPError(@"missing_watcher_id"); + } + long long ageMs = OPLongLongFromRequest(request, @"age_ms", 600000, 1000, 86400000LL); + long long now = OPNowMs(); + long long updatedAt = now - ageMs; + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *source = OPStringFromRequest(request, @"source", @"validation_watcher_stuck_fixture"); + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSDictionary *watcher = OPWatcherRead(db, watcherId); + if (!watcher) { + sqlite3_close(db); + return OPError(@"watcher_not_found"); + } + NSDictionary *existingMetadata = [watcher[@"metadata"] isKindOfClass:[NSDictionary class]] + ? watcher[@"metadata"] : @{}; + NSMutableDictionary *metadata = [existingMetadata mutableCopy]; + metadata[@"validation_stuck_fixture"] = @{ + @"status": @"marked_running", + @"marked_at_ms": @(now), + @"updated_at_ms": @(updatedAt), + @"age_ms": @(ageMs), + @"source": source ?: @"validation_watcher_stuck_fixture" + }; + metadata[@"scheduler_status"] = OPWatcherSchedulerStatus(); + metadata[@"fires_locally"] = @YES; + metadata[@"last_fire_status"] = @"running"; + + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE watcher SET status = 'running', updated_at_ms = ?, metadata_json = ?, " + "reason = ? WHERE id = ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, updatedAt); + OPSQLiteBindText(statement, 2, OPJSONString(metadata)); + OPSQLiteBindText(statement, 3, [NSString stringWithFormat:@"validation stuck fixture:%@", + source ?: @"validation"]); + sqlite3_bind_int64(statement, 4, watcherId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + } else { + error = [NSString stringWithUTF8String:sqlite3_errmsg(db)]; + } + sqlite3_finalize(statement); + NSDictionary *updatedWatcher = updated ? OPWatcherRead(db, watcherId) : nil; + sqlite3_close(db); + if (!updated || !updatedWatcher) { + return OPError([NSString stringWithFormat:@"watcher_debug_mark_running_failed:%@", + error ?: @"not_updated"]); + } + + NSDictionary *result = @{ + @"status": @"ok", + @"watcher_id": updatedWatcher[@"id"] ?: @(watcherId), + @"watcher": updatedWatcher, + @"age_ms": @(ageMs), + @"updated_at_ms": @(updatedAt), + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"watcher_debug_marked_running", taskId, @"watchers.write", + @"allow_yolo", request, [NSString stringWithFormat:@"watcher_id:%lld age_ms:%lld", + watcherId, ageMs]); + return result; +} + +static NSArray *OPWatcherDueRows(sqlite3 *db, NSUInteger limit, long long nowMs) { + NSMutableArray *watchers = [NSMutableArray array]; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "SELECT id, created_at_ms, updated_at_ms, status, source, type, evaluator, title, " + "query, url, address, number, condition_json, schedule_json, delivery_json, " + "next_run_at_ms, interval_ms, recurring, reason, metadata_json " + "FROM watcher " + "WHERE status = 'active' AND next_run_at_ms > 0 AND next_run_at_ms <= ? " + "AND (lower(source) IN ('time', 'timer', 'deadline') " + "OR lower(type) IN ('time', 'timer', 'deadline')) " + "ORDER BY next_run_at_ms ASC LIMIT ?", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_int64(statement, 1, nowMs); + sqlite3_bind_int64(statement, 2, (sqlite3_int64)limit); + while (sqlite3_step(statement) == SQLITE_ROW) { + [watchers addObject:OPWatcherFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + return watchers; +} + +static NSString *OPWatcherPrompt(NSDictionary *watcher) { + NSDictionary *delivery = [watcher[@"delivery"] isKindOfClass:[NSDictionary class]] + ? watcher[@"delivery"] : @{}; + NSString *prompt = [delivery[@"prompt"] isKindOfClass:[NSString class]] + ? [delivery[@"prompt"] stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]] : @""; + if (prompt.length > 0) { + return prompt; + } + NSString *query = [watcher[@"query"] isKindOfClass:[NSString class]] + ? [watcher[@"query"] stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]] : @""; + NSString *title = [watcher[@"title"] isKindOfClass:[NSString class]] + ? watcher[@"title"] : @"OpenPhone watcher"; + if (query.length > 0) { + return [NSString stringWithFormat:@"Handle fired OpenPhone watcher '%@': %@", + title, query]; + } + return [NSString stringWithFormat:@"Handle fired OpenPhone watcher '%@' using the current phone context.", + title]; +} + +static NSDictionary *OPWatcherUpdateAfterFire(NSDictionary *watcher, NSString *jobPublicId, + long long firedAtMs, NSString **errorOut) { + long long watcherId = [watcher[@"id"] longLongValue]; + if (watcherId <= 0) { + if (errorOut) { + *errorOut = @"missing_watcher_id"; + } + return nil; + } + long long intervalMs = [watcher[@"interval_ms"] respondsToSelector:@selector(longLongValue)] + ? [watcher[@"interval_ms"] longLongValue] : 0; + BOOL recurring = intervalMs > 0 && + [watcher[@"recurring"] respondsToSelector:@selector(boolValue)] && + [watcher[@"recurring"] boolValue]; + long long nextRunAt = recurring ? (firedAtMs + intervalMs) : 0; + NSString *finalStatus = recurring ? @"active" : @"fired"; + + NSDictionary *existingSchedule = [watcher[@"schedule"] isKindOfClass:[NSDictionary class]] + ? watcher[@"schedule"] : @{}; + NSMutableDictionary *schedule = [existingSchedule mutableCopy]; + schedule[@"last_fired_at_ms"] = @(firedAtMs); + schedule[@"last_job_id"] = jobPublicId ?: @""; + schedule[@"next_run_at"] = @(nextRunAt); + schedule[@"scheduler_status"] = OPWatcherSchedulerStatus(); + if (recurring) { + schedule[@"recurring"] = @YES; + schedule[@"interval_ms"] = @(intervalMs); + } + + NSDictionary *existingMetadata = [watcher[@"metadata"] isKindOfClass:[NSDictionary class]] + ? watcher[@"metadata"] : @{}; + NSMutableDictionary *metadata = [existingMetadata mutableCopy]; + metadata[@"scheduler_status"] = OPWatcherSchedulerStatus(); + metadata[@"fires_locally"] = @YES; + metadata[@"last_fired_at_ms"] = @(firedAtMs); + metadata[@"last_job_id"] = jobPublicId ?: @""; + metadata[@"last_fire_status"] = @"background_job_queued"; + metadata[@"recurring"] = @(recurring); + metadata[@"next_run_at_ms"] = @(nextRunAt); + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]; + } + return nil; + } + sqlite3_stmt *statement = NULL; + BOOL updated = NO; + if (sqlite3_prepare_v2(db, + "UPDATE watcher SET status = ?, updated_at_ms = ?, schedule_json = ?, " + "next_run_at_ms = ?, metadata_json = ? WHERE id = ? " + "AND status IN ('active', 'running')", + -1, &statement, NULL) == SQLITE_OK) { + OPSQLiteBindText(statement, 1, finalStatus); + sqlite3_bind_int64(statement, 2, firedAtMs); + OPSQLiteBindText(statement, 3, OPJSONString(schedule)); + sqlite3_bind_int64(statement, 4, nextRunAt); + OPSQLiteBindText(statement, 5, OPJSONString(metadata)); + sqlite3_bind_int64(statement, 6, watcherId); + updated = sqlite3_step(statement) == SQLITE_DONE && sqlite3_changes(db) > 0; + if (!updated && errorOut) { + *errorOut = @"watcher_update_missed_active_row"; + } + } else if (errorOut) { + *errorOut = [NSString stringWithUTF8String:sqlite3_errmsg(db)] ?: @"watcher_update_prepare_failed"; + } + sqlite3_finalize(statement); + NSDictionary *updatedWatcher = updated ? OPWatcherRead(db, watcherId) : nil; + sqlite3_close(db); + return updatedWatcher; +} + +static NSDictionary *OPWatcherMaterializeDue(NSUInteger limit, long long nowMs, + NSString *taskId, NSString *source) { + NSDictionary *repairResult = OPWatcherRepairStuckInternal(limit, 300000, taskId, + source ?: @"watcher_scheduler"); + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSArray *dueWatchers = OPWatcherDueRows(db, limit, nowMs); + sqlite3_close(db); + + NSMutableArray *entries = [NSMutableArray array]; + NSUInteger firedCount = 0; + NSUInteger jobCount = 0; + for (NSDictionary *watcher in dueWatchers) { + NSString *watcherPublicId = [watcher[@"watcher_id"] isKindOfClass:[NSString class]] + ? watcher[@"watcher_id"] : [NSString stringWithFormat:@"ios-watcher-%lld", + [watcher[@"id"] longLongValue]]; + NSString *title = [watcher[@"title"] isKindOfClass:[NSString class]] + ? watcher[@"title"] : watcherPublicId; + NSDictionary *delivery = [watcher[@"delivery"] isKindOfClass:[NSDictionary class]] + ? watcher[@"delivery"] : @{}; + NSDictionary *event = @{ + @"type": @"timer", + @"watcher_id": watcherPublicId, + @"watcher_row_id": watcher[@"id"] ?: @0, + @"title": title ?: watcherPublicId, + @"fired_at_ms": @(nowMs), + @"scheduled_at_ms": watcher[@"next_run_at_ms"] ?: @0 + }; + NSString *claimError = nil; + NSDictionary *claimedWatcher = OPWatcherClaimForFire(watcher, nowMs, + source ?: @"watcher_scheduler", &claimError); + if (!claimedWatcher) { + NSMutableDictionary *claimEntry = [@{ + @"watcher_id": watcherPublicId, + @"status": @"skipped", + @"reason": claimError ?: @"watcher_claim_failed", + @"event": event + } mutableCopy]; + [entries addObject:claimEntry]; + continue; + } + NSDictionary *materializedWatcher = claimedWatcher; + NSMutableDictionary *payload = [@{ + @"source": @"watcher_scheduler", + @"watcher_id": watcherPublicId, + @"watcher_row_id": materializedWatcher[@"id"] ?: @0, + @"event": event, + @"delivery": delivery + } mutableCopy]; + NSString *prompt = OPWatcherPrompt(materializedWatcher); + NSMutableDictionary *jobRequest = [@{ + @"command": @"background_job_create", + @"task_id": taskId ?: @"", + @"title": [NSString stringWithFormat:@"Watcher fired: %@", title ?: watcherPublicId], + @"prompt": prompt, + @"type": @"watcher_fire", + @"next_run_at": @(nowMs), + @"source": @"watcher_scheduler", + @"reason": [NSString stringWithFormat:@"timer watcher fired:%@", watcherPublicId], + @"payload": payload, + @"delivery": delivery + } mutableCopy]; + NSString *notificationText = [delivery[@"notification_text"] isKindOfClass:[NSString class]] + ? delivery[@"notification_text"] : @""; + if (notificationText.length > 0) { + jobRequest[@"notification_text"] = notificationText; + } + + NSDictionary *jobResult = OPBackgroundJobCreate(jobRequest); + NSMutableDictionary *entry = [@{ + @"watcher_id": watcherPublicId, + @"status": @"failed", + @"event": event + } mutableCopy]; + if ([jobResult[@"status"] isEqualToString:@"ok"]) { + firedCount++; + jobCount++; + NSDictionary *job = [jobResult[@"job"] isKindOfClass:[NSDictionary class]] + ? jobResult[@"job"] : @{}; + NSString *jobPublicId = [job[@"job_id"] isKindOfClass:[NSString class]] + ? job[@"job_id"] : @""; + NSString *updateError = nil; + NSDictionary *updatedWatcher = OPWatcherUpdateAfterFire(materializedWatcher, jobPublicId, nowMs, &updateError); + entry[@"status"] = @"background_job_queued"; + entry[@"job_id"] = jobPublicId; + entry[@"job"] = job; + if (updatedWatcher) { + entry[@"watcher"] = updatedWatcher; + } + if (updateError) { + entry[@"update_error"] = updateError; + } + OPRecordContextEvent(@"watcher_fired", @"openphone.agentd", taskId, + title ?: watcherPublicId, prompt, @{ + @"watcher_id": watcherPublicId, + @"job_id": jobPublicId ?: @"", + @"scheduler_status": OPWatcherSchedulerStatus(), + @"source": source ?: @"watcher_scheduler" + }); + OPRecordAudit(@"watcher_fired", taskId, @"watchers.write", @"allow_yolo", + jobRequest, [NSString stringWithFormat:@"watcher_id:%@ job_id:%@", + watcherPublicId, jobPublicId ?: @""]); + } else { + NSString *failureReason = [jobResult[@"reason"] isKindOfClass:[NSString class]] + ? jobResult[@"reason"] : @"background_job_create_failed"; + entry[@"reason"] = failureReason; + entry[@"job_result"] = jobResult ?: @{}; + NSString *failureUpdateError = nil; + NSDictionary *requeuedWatcher = OPWatcherRequeueAfterFireFailure(materializedWatcher, + failureReason, nowMs, source ?: @"watcher_scheduler", &failureUpdateError); + entry[@"retry_backoff"] = @"exponential"; + if (requeuedWatcher) { + entry[@"watcher"] = requeuedWatcher; + } + if (failureUpdateError) { + entry[@"update_error"] = failureUpdateError; + } + } + [entries addObject:entry]; + } + + return @{ + @"status": @"ok", + @"scheduler_status": OPWatcherSchedulerStatus(), + @"stuck_repair_count": repairResult[@"repaired_count"] ?: @0, + @"stuck_repair_result": repairResult ?: @{}, + @"source": @"openphone.agentd", + @"limit": @(limit), + @"due_count": @(dueWatchers.count), + @"fired_count": @(firedCount), + @"job_count": @(jobCount), + @"watchers": entries + }; +} + +// --------------------------------------------------------------------------- +// Notification provider. The OpenPhoneVolumeTrigger tweak (injected into +// SpringBoard) hooks NCNotificationDispatcher and posts each incoming banner +// to the daemon via the `notification_ingest` command. The daemon keeps a +// bounded, redacted rolling log and fires any active `notification`-source +// watchers whose condition matches, turning the assistant reactive. +// --------------------------------------------------------------------------- + +static const NSUInteger OPNotificationLogMax = 100; + +static NSString *OPNotificationBoundedText(NSString *text, NSUInteger maxLen) { + if (![text isKindOfClass:[NSString class]]) { + return @""; + } + NSString *trimmed = [text stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (trimmed.length <= maxLen) { + return trimmed; + } + return [[trimmed substringToIndex:maxLen] stringByAppendingString:@"…"]; +} + +static NSDictionary *OPNotificationProviderStatus(void) { + NSDictionary *log = OPReadJSONFile(OPNotificationLogPath()); + NSArray *items = [log[@"notifications"] isKindOfClass:[NSArray class]] + ? log[@"notifications"] : @[]; + return @{ + @"status": @"implemented", + @"provider": @"OpenPhoneVolumeTrigger.NCNotificationDispatcher", + @"transport": @"unix_socket", + @"log_path": OPNotificationLogPath(), + @"stored_count": @(items.count), + @"ingest_count": @((long long)OPNotificationIngestCount), + @"last_ingest_ms": @((long long)OPNotificationLastIngestMs), + @"last_bundle_id": OPNotificationLastBundleId ?: @"", + @"watcher_source": @"notification", + @"retention": @"bounded title/body previews only, newest 100 kept; no attachments or full payloads" + }; +} + +// Active notification-source watchers whose condition matches the notification +// fire a background job. A watcher matches when its bundle_id condition (if any) +// equals the notification bundle and its query/keyword (if any) appears in the +// notification title or body (case-insensitive). Empty conditions match all. +static NSDictionary *OPNotificationFireWatchers(NSDictionary *notification) { + NSString *bundleId = [notification[@"bundle_id"] isKindOfClass:[NSString class]] + ? notification[@"bundle_id"] : @""; + NSString *haystack = [[NSString stringWithFormat:@"%@ %@ %@", + notification[@"title"] ?: @"", notification[@"subtitle"] ?: @"", + notification[@"body"] ?: @""] lowercaseString]; + long long nowMs = OPNowMs(); + + sqlite3 *db = NULL; + NSString *error = nil; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSMutableArray *candidates = [NSMutableArray array]; + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, + "SELECT id, created_at_ms, updated_at_ms, status, source, type, evaluator, title, " + "query, url, address, number, condition_json, schedule_json, delivery_json, " + "next_run_at_ms, interval_ms, recurring, reason, metadata_json " + "FROM watcher WHERE status = 'active' AND lower(source) = 'notification'", + -1, &statement, NULL) == SQLITE_OK) { + while (sqlite3_step(statement) == SQLITE_ROW) { + [candidates addObject:OPWatcherFromStatement(statement)]; + } + } + sqlite3_finalize(statement); + sqlite3_close(db); + + NSMutableArray *fired = [NSMutableArray array]; + NSUInteger firedCount = 0; + for (NSDictionary *watcher in candidates) { + NSDictionary *condition = [watcher[@"condition"] isKindOfClass:[NSDictionary class]] + ? watcher[@"condition"] : @{}; + NSString *wantBundle = [condition[@"bundle_id"] isKindOfClass:[NSString class]] + ? condition[@"bundle_id"] : ([watcher[@"address"] isKindOfClass:[NSString class]] + ? watcher[@"address"] : @""); + if (wantBundle.length > 0 && ![wantBundle isEqualToString:bundleId]) { + continue; + } + NSString *keyword = [condition[@"query"] isKindOfClass:[NSString class]] + ? condition[@"query"] : ([watcher[@"query"] isKindOfClass:[NSString class]] + ? watcher[@"query"] : @""); + keyword = [keyword stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (keyword.length > 0 && + [haystack rangeOfString:keyword.lowercaseString].location == NSNotFound) { + continue; + } + NSString *watcherPublicId = [watcher[@"watcher_id"] isKindOfClass:[NSString class]] + ? watcher[@"watcher_id"] : [NSString stringWithFormat:@"ios-watcher-%lld", + [watcher[@"id"] longLongValue]]; + NSString *title = [watcher[@"title"] isKindOfClass:[NSString class]] + ? watcher[@"title"] : watcherPublicId; + NSString *claimError = nil; + NSDictionary *claimedWatcher = OPWatcherClaimForFire(watcher, nowMs, + @"notification_provider", &claimError); + if (!claimedWatcher) { + [fired addObject:@{@"watcher_id": watcherPublicId, @"status": @"skipped", + @"reason": claimError ?: @"watcher_claim_failed"}]; + continue; + } + NSDictionary *delivery = [claimedWatcher[@"delivery"] isKindOfClass:[NSDictionary class]] + ? claimedWatcher[@"delivery"] : @{}; + NSDictionary *event = @{ + @"type": @"notification", + @"watcher_id": watcherPublicId, + @"watcher_row_id": claimedWatcher[@"id"] ?: @0, + @"fired_at_ms": @(nowMs), + @"notification": notification + }; + NSString *prompt = OPWatcherPrompt(claimedWatcher); + NSString *notificationSummary = [NSString stringWithFormat: + @"Notification from %@: %@ %@", bundleId, + notification[@"title"] ?: @"", notification[@"body"] ?: @""]; + NSMutableDictionary *jobRequest = [@{ + @"command": @"background_job_create", + @"title": [NSString stringWithFormat:@"Notification watcher fired: %@", title], + @"prompt": [NSString stringWithFormat:@"%@\n\nTriggering notification: %@", + prompt, notificationSummary], + @"type": @"watcher_fire", + @"next_run_at": @(nowMs), + @"source": @"notification_provider", + @"reason": [NSString stringWithFormat:@"notification watcher fired:%@", watcherPublicId], + @"payload": @{@"source": @"notification_provider", @"watcher_id": watcherPublicId, + @"event": event, @"delivery": delivery}, + @"delivery": delivery + } mutableCopy]; + NSDictionary *jobResult = OPBackgroundJobCreate(jobRequest); + if ([jobResult[@"status"] isEqualToString:@"ok"]) { + firedCount++; + NSDictionary *job = [jobResult[@"job"] isKindOfClass:[NSDictionary class]] + ? jobResult[@"job"] : @{}; + NSString *jobPublicId = [job[@"job_id"] isKindOfClass:[NSString class]] + ? job[@"job_id"] : @""; + NSString *updateError = nil; + OPWatcherUpdateAfterFire(claimedWatcher, jobPublicId, nowMs, &updateError); + OPRecordAudit(@"watcher_fired", @"", @"watchers.write", @"allow_yolo", + jobRequest, [NSString stringWithFormat:@"watcher_id:%@ job_id:%@ source:notification", + watcherPublicId, jobPublicId ?: @""]); + [fired addObject:@{@"watcher_id": watcherPublicId, + @"status": @"background_job_queued", @"job_id": jobPublicId}]; + } else { + NSString *failureReason = [jobResult[@"reason"] isKindOfClass:[NSString class]] + ? jobResult[@"reason"] : @"background_job_create_failed"; + NSString *failureUpdateError = nil; + OPWatcherRequeueAfterFireFailure(claimedWatcher, failureReason, nowMs, + @"notification_provider", &failureUpdateError); + [fired addObject:@{@"watcher_id": watcherPublicId, @"status": @"failed", + @"reason": failureReason}]; + } + } + return @{ + @"status": @"ok", + @"candidate_count": @(candidates.count), + @"fired_count": @(firedCount), + @"watchers": fired + }; +} + +static NSDictionary *OPNotificationIngest(NSDictionary *request) { + NSString *bundleId = OPStringFromRequest(request, @"bundle_id", @""); + if (!OPBundleIdentifierLooksValid(bundleId)) { + return OPError(@"invalid_notification_bundle_id"); + } + long long nowMs = OPNowMs(); + NSDictionary *record = @{ + @"schema": @"openphone.notification.v1", + @"bundle_id": bundleId, + @"title": OPNotificationBoundedText(OPStringFromRequest(request, @"title", @""), 200), + @"subtitle": OPNotificationBoundedText(OPStringFromRequest(request, @"subtitle", @""), 200), + @"body": OPNotificationBoundedText(OPStringFromRequest(request, @"body", @""), 1000), + @"notification_id": OPNotificationBoundedText(OPStringFromRequest(request, @"notification_id", @""), 200), + @"thread_id": OPNotificationBoundedText(OPStringFromRequest(request, @"thread_id", @""), 200), + @"received_at_ms": @(nowMs), + @"source": @"OpenPhoneVolumeTrigger.NCNotificationDispatcher" + }; + + OPEnsureDirectories(); + NSDictionary *existing = OPReadJSONFile(OPNotificationLogPath()); + NSArray *prior = [existing[@"notifications"] isKindOfClass:[NSArray class]] + ? existing[@"notifications"] : @[]; + NSMutableArray *items = [prior mutableCopy]; + [items addObject:record]; + while (items.count > OPNotificationLogMax) { + [items removeObjectAtIndex:0]; + } + OPWriteJSONFile(OPNotificationLogPath(), @{ + @"schema": @"openphone.notification_log.v1", + @"updated_at_ms": @(nowMs), + @"notifications": items, + @"source": @"openphone.agentd" + }); + chmod(OPNotificationLogPath().UTF8String, 0644); + + @synchronized(@"OPNotificationMetrics") { + OPNotificationIngestCount++; + OPNotificationLastIngestMs = nowMs; + OPNotificationLastBundleId = [bundleId copy]; + } + OPRecordContextEvent(@"notification_received", @"openphone.agentd", @"", + record[@"title"], record[@"body"], @{ + @"bundle_id": bundleId, + @"source": @"notification_provider" + }); + + NSDictionary *fireResult = OPNotificationFireWatchers(record); + return @{ + @"status": @"ok", + @"schema": @"openphone.notification_ingest_result.v1", + @"bundle_id": bundleId, + @"stored_count": @(items.count), + @"watcher_fire": fireResult ?: @{}, + @"source": @"openphone.agentd.notification_provider" + }; +} + +static NSDictionary *OPNotificationList(NSDictionary *request) { + long long limit = OPLongLongFromRequest(request, @"limit", 20, 1, OPNotificationLogMax); + NSString *bundleFilter = OPStringFromRequest(request, @"bundle_id", @""); + NSDictionary *log = OPReadJSONFile(OPNotificationLogPath()); + NSArray *items = [log[@"notifications"] isKindOfClass:[NSArray class]] + ? log[@"notifications"] : @[]; + NSMutableArray *filtered = [NSMutableArray array]; + for (NSDictionary *item in [items reverseObjectEnumerator]) { + if (![item isKindOfClass:[NSDictionary class]]) continue; + if (bundleFilter.length > 0 && + ![item[@"bundle_id"] isEqualToString:bundleFilter]) { + continue; + } + [filtered addObject:item]; + if ((long long)filtered.count >= limit) break; + } + return @{ + @"status": @"ok", + @"schema": @"openphone.notification_list.v1", + @"count": @(filtered.count), + @"total_stored": @(items.count), + @"notifications": filtered, + @"provider_status": OPNotificationProviderStatus(), + @"source": @"openphone.agentd" + }; +} + +// Promote durable voice turns from the persisted chat history into the memory +// store so a follow-up voice turn days later can reference them. The in-memory +// OPRecentVoiceTurns ring buffer only survives the process; chat-history.json +// survives restarts and is the source of truth here. A watermark file tracks +// the last-promoted turn timestamp so each turn is saved at most once. +static NSDictionary *OPPromoteVoiceTurnsToMemory(NSDictionary *request) { + long long minChars = OPLongLongFromRequest(request, @"min_chars", 12, 1, 4000); + long long maxPromote = OPLongLongFromRequest(request, @"max_promote", 10, 1, 100); + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + + NSString *historyPath = [[OPStorePath() stringByAppendingPathComponent:@"springboard"] + stringByAppendingPathComponent:@"chat-history.json"]; + NSDictionary *history = OPReadJSONFile(historyPath); + NSArray *turns = [history[@"turns"] isKindOfClass:[NSArray class]] ? history[@"turns"] : @[]; + + NSDictionary *watermark = OPReadJSONFile(OPVoiceMemoryWatermarkPath()); + long long lastPromotedMs = [watermark[@"last_promoted_at_ms"] respondsToSelector:@selector(longLongValue)] + ? [watermark[@"last_promoted_at_ms"] longLongValue] : 0; + + long long highestSeenMs = lastPromotedMs; + long long promoted = 0; + NSMutableArray *saved = [NSMutableArray array]; + for (NSDictionary *turn in turns) { + if (![turn isKindOfClass:[NSDictionary class]]) continue; + long long atMs = [turn[@"at_ms"] respondsToSelector:@selector(longLongValue)] + ? [turn[@"at_ms"] longLongValue] : 0; + if (atMs <= lastPromotedMs) continue; + if (atMs > highestSeenMs) highestSeenMs = atMs; + NSString *role = [turn[@"role"] isKindOfClass:[NSString class]] ? turn[@"role"] : @""; + if (![role isEqualToString:@"user"]) continue; + NSString *text = [turn[@"text"] isKindOfClass:[NSString class]] ? turn[@"text"] : @""; + text = [text stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if ((long long)text.length < minChars) continue; + if (promoted >= maxPromote) break; + NSString *turnTaskId = [turn[@"task_id"] isKindOfClass:[NSString class]] + ? turn[@"task_id"] : @""; + NSDictionary *saveResult = OPMemorySave(@{ + @"text": text, + @"type": @"voice_turn", + @"subject": @"user", + @"confidence": @0.7, + @"reason": @"auto-promoted from voice chat history", + @"task_id": turnTaskId, + @"metadata": @{ + @"origin": @"voice_turn_auto_promote", + @"turn_at_ms": @(atMs) + } + }); + if ([saveResult[@"status"] isEqualToString:@"ok"]) { + promoted += 1; + [saved addObject:@{ + @"turn_at_ms": @(atMs), + @"memory_id": saveResult[@"memory"][@"memory_id"] ?: @"" + }]; + } + } + + if (highestSeenMs > lastPromotedMs) { + OPWriteJSONFile(OPVoiceMemoryWatermarkPath(), @{ + @"last_promoted_at_ms": @(highestSeenMs), + @"updated_at_ms": @(OPNowMs()), + @"promoted_count": @(promoted), + @"source": @"openphone.agentd" + }); + chmod(OPVoiceMemoryWatermarkPath().UTF8String, 0644); + } + if (promoted > 0) { + OPRecordAudit(@"voice_turns_promoted_to_memory", taskId, @"memory.write", + @"allow_yolo", request, [NSString stringWithFormat:@"promoted:%lld", promoted]); + } + return @{ + @"status": @"ok", + @"promoted_count": @(promoted), + @"turns_scanned": @(turns.count), + @"last_promoted_at_ms": @(highestSeenMs), + @"saved": saved, + @"source": @"openphone.agentd" + }; +} + +// Unattended background jobs run without a human watching the Approve/Deny +// island, so by default they may only observe and navigate, never commit +// irreversible or externally-visible mutations. Every state-changing action +// on iOS routes through the UI (tap/type_text/swipe/long_press) which all map +// to the "input.perform" capability; mutating provider tools map to their own +// write/send/delete/place capabilities. The read/navigate-only grant below +// omits all of those, so OPModelExecuteDecision denies mutations with +// "capability_not_approved". A job opts back into full state-changing power +// only by carrying an explicit grant (payload.background_state_changing = true +// or payload.autonomy = "yolo"), mirroring the Android "explicit policy grant" +// model (IOS_PLAN.md Phase 10, Option B). +static NSArray *OPBackgroundReadOnlyCapabilities(void) { + return @[ + @"screen.read.visible", + @"screen.capture", + @"apps.read", + @"apps.launch", + @"tasks.observe", + @"memory.read", + @"memory.write", + @"commitments.read", + @"watchers.read", + @"notifications.read", + @"clipboard.read", + @"contacts.read", + @"calendar.read", + @"messages.read", + @"calls.read", + @"settings.read", + @"files.read.scoped", + @"background.run", + @"network.use" + ]; +} + +static BOOL OPBackgroundJobHasStateChangingGrant(NSDictionary *job) { + if (![job isKindOfClass:[NSDictionary class]]) { + return NO; + } + NSDictionary *payload = [job[@"payload"] isKindOfClass:[NSDictionary class]] + ? job[@"payload"] : @{}; + if ([payload[@"background_state_changing"] isEqual:@YES]) { + return YES; + } + NSString *autonomy = [payload[@"autonomy"] isKindOfClass:[NSString class]] + ? [payload[@"autonomy"] lowercaseString] : @""; + if ([autonomy isEqualToString:@"yolo"] || [autonomy isEqualToString:@"full"]) { + return YES; + } + return NO; +} + +// Resolve the capability grant a background job runs under. Returns the +// read/navigate-only set unless the job carries an explicit state-changing +// grant, in which case it gets full YOLO capabilities. policyOut (optional) +// receives a short label for audit/trajectory. +static NSArray *OPBackgroundJobRunCapabilities(NSDictionary *job, + NSString **policyOut) { + if (OPBackgroundJobHasStateChangingGrant(job)) { + if (policyOut) { + *policyOut = @"explicit_state_changing_grant"; + } + return OPFullYoloCapabilities(); + } + if (policyOut) { + *policyOut = @"read_navigate_only"; + } + return OPBackgroundReadOnlyCapabilities(); +} + +static NSDictionary *OPBackgroundJobRunDue(NSDictionary *request) { + if (pthread_mutex_trylock(&OPBackgroundJobSchedulerMutex) != 0) { + return @{ + @"status": @"ok", + @"scheduler_status": OPBackgroundJobSchedulerStatus(), + @"commitment_scheduler_status": OPCommitmentSchedulerStatus(), + @"watcher_scheduler_status": OPWatcherSchedulerStatus(), + @"busy": @YES, + @"ran_count": @0, + @"source": @"openphone.agentd" + }; + } + + NSMutableArray *results = [NSMutableArray array]; + NSUInteger claimedCount = 0; + NSUInteger skippedCount = 0; + NSString *error = nil; + @try { + NSUInteger limit = OPLimitFromRequest(request, 1, 10); + if (limit == 0) { + limit = 1; + } + long long maxSteps = OPLongLongFromRequest(request, @"max_steps", 1, 1, 5); + long long maxDurationMs = OPLongLongFromRequest(request, @"max_duration_ms", 15000, 1000, 120000); + NSString *source = OPStringFromRequest(request, @"source", @"background_job_scheduler"); + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + long long targetJobId = OPRecordIdFromRequest(request, @[@"job_id", @"id"]); + BOOL repairStuck = OPBoolFromRequest(request, @"repair_stuck", targetJobId <= 0); + BOOL materializeCommitments = OPBoolFromRequest(request, @"materialize_commitments", targetJobId <= 0); + BOOL materializeWatchers = OPBoolFromRequest(request, @"materialize_watchers", targetJobId <= 0); + BOOL runJobs = OPBoolFromRequest(request, @"run_jobs", YES); + long long staleAfterMs = OPLongLongFromRequest(request, + @"stale_after_ms", 300000, 1000, 86400000LL); + NSDictionary *repairResult = repairStuck + ? OPBackgroundJobRepairStuckInternal(limit, staleAfterMs, taskId, source) + : @{@"status": @"skipped", @"reason": @"repair_stuck_disabled", @"repaired_count": @0}; + id deliveryRequest = OPJSONObjectFromRequest(request, @"delivery", nil); + NSDictionary *commitmentResult = materializeCommitments + ? OPCommitmentMaterializeDue(limit, OPNowMs(), taskId, source, + [deliveryRequest isKindOfClass:[NSDictionary class]] ? deliveryRequest : nil) + : @{@"status": @"skipped", @"reason": @"materialize_commitments_disabled", + @"triggered_count": @0, @"job_count": @0}; + NSDictionary *watcherResult = materializeWatchers + ? OPWatcherMaterializeDue(limit, OPNowMs(), taskId, source) + : @{@"status": @"skipped", @"reason": @"materialize_watchers_disabled", + @"fired_count": @0, @"job_count": @0}; + if (!runJobs) { + return @{ + @"status": @"ok", + @"scheduler_status": OPBackgroundJobSchedulerStatus(), + @"commitment_scheduler_status": OPCommitmentSchedulerStatus(), + @"watcher_scheduler_status": OPWatcherSchedulerStatus(), + @"stuck_repair_count": repairResult[@"repaired_count"] ?: @0, + @"stuck_repair_result": repairResult ?: @{}, + @"commitment_triggered_count": commitmentResult[@"triggered_count"] ?: @0, + @"commitment_jobs_created": commitmentResult[@"job_count"] ?: @0, + @"commitment_result": commitmentResult ?: @{}, + @"watcher_fire_count": watcherResult[@"fired_count"] ?: @0, + @"watcher_jobs_created": watcherResult[@"job_count"] ?: @0, + @"watcher_result": watcherResult ?: @{}, + @"runner": @"deterministic", + @"model_loop_status": @"not_started", + @"run_jobs": @NO, + @"run_policy": @"repair_and_materialize_only", + @"target_job_id": @(targetJobId), + @"limit": @(limit), + @"claimed_count": @0, + @"skipped_count": @0, + @"ran_count": @0, + @"jobs": @[], + @"source": @"openphone.agentd" + }; + } + + sqlite3 *db = NULL; + if (!OPSQLiteOpen(&db, &error)) { + return OPError([NSString stringWithFormat:@"sqlite_open_failed:%@", error ?: @"unknown"]); + } + NSArray *jobs = OPBackgroundJobDueRows(db, limit, OPNowMs(), targetJobId); + sqlite3_close(db); + + for (NSDictionary *job in jobs) { + long long jobId = [job[@"id"] longLongValue]; + NSString *jobPublicId = [job[@"job_id"] isKindOfClass:[NSString class]] + ? job[@"job_id"] : [NSString stringWithFormat:@"ios-job-%lld", jobId]; + NSString *claimError = nil; + if (!OPBackgroundJobClaim(jobId, &claimError)) { + skippedCount++; + [results addObject:@{ + @"job_id": jobPublicId, + @"status": @"skipped", + @"reason": claimError ?: @"claim_failed" + }]; + continue; + } + claimedCount++; + + NSString *prompt = [job[@"prompt"] isKindOfClass:[NSString class]] ? job[@"prompt"] : @""; + if (prompt.length == 0) { + prompt = [job[@"title"] isKindOfClass:[NSString class]] ? job[@"title"] : @"background job"; + } + NSString *runPolicy = nil; + NSArray *runCapabilities = OPBackgroundJobRunCapabilities(job, &runPolicy); + NSDictionary *runRequest = @{ + @"mode": OPStringFromRequest(request, @"mode", @"auto"), + @"goal": prompt, + @"reason": [NSString stringWithFormat:@"background job scheduler:%@", jobPublicId], + @"max_steps": @(maxSteps), + @"max_duration_ms": @(maxDurationMs), + @"include_screenshot": @(![source isEqualToString:@"background_job_scheduler"]), + @"background_job_id": jobPublicId, + @"background_job_row_id": @(jobId), + @"approved_capabilities": runCapabilities, + @"background_capability_policy": runPolicy ?: @"read_navigate_only", + @"source": source ?: @"background_job_scheduler" + }; + OPRecordContextEvent(@"background_job_started", @"openphone.agentd", taskId, + job[@"title"] ?: jobPublicId, prompt, @{ + @"job_id": jobPublicId, + @"runner": runRequest[@"mode"] ?: @"auto", + @"model_loop_status": @"auto", + @"capability_policy": runPolicy ?: @"read_navigate_only" + }); + OPRecordAudit(@"background_job_started", taskId, @"background.run", + [runPolicy isEqualToString:@"explicit_state_changing_grant"] + ? @"allow_yolo" : @"allow_read_navigate_only", + runRequest, [NSString stringWithFormat:@"job_id:%@ policy:%@", + jobPublicId, runPolicy ?: @"read_navigate_only"]); + + NSDictionary *runResult = nil; + @try { + runResult = OPRunTask(runRequest); + } @catch (NSException *exception) { + OPLog(@"background job run exception job_id=%@ exception=%@ reason=%@", + jobPublicId ?: @"", exception.name ?: @"NSException", + exception.reason ?: @""); + runResult = @{ + @"status": @"task.failed", + @"runner": @"deterministic", + @"task_id": @"", + @"stop_reason": @"exception", + @"exception_name": exception.name ?: @"NSException", + @"exception_reason": exception.reason ?: @"", + @"source": @"openphone.agentd" + }; + } + BOOL taskFinished = [runResult[@"status"] isEqualToString:@"task.finished"]; + NSString *runner = [runResult[@"runner"] isKindOfClass:[NSString class]] + ? runResult[@"runner"] : @"auto"; + NSString *modelLoopStatus = [runner isEqualToString:@"model"] ? @"finished" : @"not_started"; + NSString *finishError = nil; + NSDictionary *updatedJob = OPBackgroundJobFinish(job, runResult, &finishError); + NSString *jobStatus = taskFinished ? @"completed" : @"failed"; + if ([updatedJob[@"status"] isEqualToString:@"queued"]) { + jobStatus = @"queued"; + } else if ([updatedJob[@"status"] isKindOfClass:[NSString class]]) { + jobStatus = updatedJob[@"status"]; + } + NSMutableDictionary *entry = [@{ + @"job_id": jobPublicId, + @"status": jobStatus ?: @"unknown", + @"run_task": runResult ?: @{}, + @"runner": runner ?: @"auto", + @"model_loop_status": modelLoopStatus ?: @"unknown" + } mutableCopy]; + if (updatedJob) { + entry[@"job"] = updatedJob; + } + if (finishError) { + entry[@"finish_error"] = finishError; + } + [results addObject:entry]; + + OPRecordContextEvent(taskFinished ? @"background_job_completed" : @"background_job_failed", + @"openphone.agentd", runResult[@"task_id"] ?: taskId, + job[@"title"] ?: jobPublicId, runResult[@"stop_reason"] ?: jobStatus, @{ + @"job_id": jobPublicId, + @"status": jobStatus ?: @"unknown", + @"task_id": runResult[@"task_id"] ?: @"" + }); + OPRecordAudit(taskFinished ? @"background_job_completed" : @"background_job_failed", + runResult[@"task_id"] ?: taskId, @"background.run", + taskFinished ? @"allow_task_scoped" : @"failed", + runRequest, [NSString stringWithFormat:@"job_id:%@ status:%@", jobPublicId, jobStatus]); + } + + NSString *aggregateRunner = @"auto"; + NSString *aggregateModelLoopStatus = @"auto"; + if (results.count > 0) { + NSDictionary *firstResult = results[0]; + aggregateRunner = [firstResult[@"runner"] isKindOfClass:[NSString class]] + ? firstResult[@"runner"] : aggregateRunner; + aggregateModelLoopStatus = [firstResult[@"model_loop_status"] isKindOfClass:[NSString class]] + ? firstResult[@"model_loop_status"] : aggregateModelLoopStatus; + } + NSDictionary *result = @{ + @"status": @"ok", + @"scheduler_status": OPBackgroundJobSchedulerStatus(), + @"commitment_scheduler_status": OPCommitmentSchedulerStatus(), + @"watcher_scheduler_status": OPWatcherSchedulerStatus(), + @"stuck_repair_count": repairResult[@"repaired_count"] ?: @0, + @"stuck_repair_result": repairResult ?: @{}, + @"commitment_triggered_count": commitmentResult[@"triggered_count"] ?: @0, + @"commitment_jobs_created": commitmentResult[@"job_count"] ?: @0, + @"commitment_result": commitmentResult ?: @{}, + @"watcher_fire_count": watcherResult[@"fired_count"] ?: @0, + @"watcher_jobs_created": watcherResult[@"job_count"] ?: @0, + @"watcher_result": watcherResult ?: @{}, + @"runner": aggregateRunner ?: @"auto", + @"model_loop_status": aggregateModelLoopStatus ?: @"auto", + @"target_job_id": @(targetJobId), + @"limit": @(limit), + @"claimed_count": @(claimedCount), + @"skipped_count": @(skippedCount), + @"ran_count": @(results.count - skippedCount), + @"jobs": results, + @"source": @"openphone.agentd" + }; + if (results.count > 0) { + OPRecordAudit(@"background_job_scheduler_tick", taskId, @"background.run", + @"allow_task_scoped", request, [NSString stringWithFormat:@"ran:%lu skipped:%lu", + (unsigned long)(results.count - skippedCount), (unsigned long)skippedCount]); + } + return result; + } @finally { + pthread_mutex_unlock(&OPBackgroundJobSchedulerMutex); + } +} + +static void *OPBackgroundJobSchedulerMain(void *context) { + (void)context; + OPLog(@"background job scheduler startup delay seconds=%d", + OPBackgroundJobSchedulerStartupDelaySeconds); + for (int i = 0; i < OPBackgroundJobSchedulerStartupDelaySeconds && OPRunning; i++) { + sleep(1); + } + long long lastVoicePromoteMs = 0; + while (OPRunning) { + @autoreleasepool { + NSDictionary *result = OPBackgroundJobRunDue(@{ + @"command": @"background_job_run_due", + @"limit": @1, + @"max_steps": @1, + @"max_duration_ms": @15000, + @"run_jobs": @NO, + @"source": @"background_job_scheduler", + @"reason": @"periodic background job scheduler repair/materialization tick" + }); + // Promote durable voice turns into memory on a slow cadence (~60s) + // so follow-up voice turns days later can reference them. + long long nowMs = OPNowMs(); + if (nowMs - lastVoicePromoteMs >= 60000) { + lastVoicePromoteMs = nowMs; + NSDictionary *promote = OPPromoteVoiceTurnsToMemory(@{ + @"source": @"background_job_scheduler" + }); + long long promoted = [promote[@"promoted_count"] respondsToSelector:@selector(longLongValue)] + ? [promote[@"promoted_count"] longLongValue] : 0; + if (promoted > 0) { + OPLog(@"voice turns promoted to memory count=%lld", promoted); + } + } + long long ran = [result[@"ran_count"] respondsToSelector:@selector(longLongValue)] + ? [result[@"ran_count"] longLongValue] : 0; + long long repaired = [result[@"stuck_repair_count"] respondsToSelector:@selector(longLongValue)] + ? [result[@"stuck_repair_count"] longLongValue] : 0; + long long fired = [result[@"watcher_fire_count"] respondsToSelector:@selector(longLongValue)] + ? [result[@"watcher_fire_count"] longLongValue] : 0; + long long triggered = [result[@"commitment_triggered_count"] respondsToSelector:@selector(longLongValue)] + ? [result[@"commitment_triggered_count"] longLongValue] : 0; + if (ran > 0) { + OPLog(@"background job scheduler ran_count=%lld", ran); + } else if (repaired > 0 || fired > 0 || triggered > 0) { + OPLog(@"background job scheduler recovery repaired=%lld commitment_triggered_count=%lld watcher_fire_count=%lld", + repaired, triggered, fired); + } + } + for (int i = 0; i < 5 && OPRunning; i++) { + sleep(1); + } + } + return NULL; +} + +static void OPStartBackgroundJobScheduler(void) { + if (OPBackgroundJobSchedulerThreadStarted) { + return; + } + int rc = pthread_create(&OPBackgroundJobSchedulerThread, NULL, + OPBackgroundJobSchedulerMain, NULL); + if (rc == 0) { + OPBackgroundJobSchedulerThreadStarted = 1; + pthread_detach(OPBackgroundJobSchedulerThread); + OPLog(@"background job scheduler started status=%@", OPBackgroundJobSchedulerStatus()); + } else { + OPLog(@"background job scheduler start failed: %s", strerror(rc)); + } +} + +static NSDictionary *OPListApps(NSDictionary *request) { + NSUInteger limit = OPLimitFromRequest(request, 200, 2000); + NSString *taskId = [request[@"task_id"] isKindOfClass:[NSString class]] ? request[@"task_id"] : @""; + NSString *query = [request[@"query"] isKindOfClass:[NSString class]] + ? [request[@"query"] lowercaseString] : @""; + NSArray *allApps = OPInstalledApplications(query.length > 0 ? 0 : limit); + NSMutableArray *apps = [NSMutableArray array]; + for (NSDictionary *app in allApps) { + if (query.length > 0) { + NSString *bundleId = [app[@"bundle_id"] isKindOfClass:[NSString class]] + ? [app[@"bundle_id"] lowercaseString] : @""; + NSString *displayName = [app[@"display_name"] isKindOfClass:[NSString class]] + ? [app[@"display_name"] lowercaseString] : @""; + if (![bundleId containsString:query] && ![displayName containsString:query]) { + continue; + } + } + [apps addObject:app]; + if (limit > 0 && apps.count >= limit) { + break; + } + } + NSDictionary *result = @{ + @"status": @"ok", + @"apps": apps, + @"count": @(apps.count), + @"query": query ?: @"", + @"source": @"openphone.agentd" + }; + OPRecordAudit(@"apps_listed", taskId, @"apps.read", @"allow_task_scoped", + request, [NSString stringWithFormat:@"count:%lu", (unsigned long)apps.count]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"list_apps", + @"arguments": request ?: @{}, + @"result": result + }); + return result; +} + +static NSDictionary *OPGetScreen(NSDictionary *request) { + NSString *taskId = [request[@"task_id"] isKindOfClass:[NSString class]] ? request[@"task_id"] : @""; + NSString *foregroundBundleId = OPForegroundBundleIdentifier(); + NSString *foregroundSource = foregroundBundleId.length > 0 ? @"springboardservices" : @"unavailable"; + NSArray *runningApps = OPRunningApplications(); + NSMutableDictionary *displayInfo = [OPScreenDisplayInfo() mutableCopy]; + NSDictionary *lockInfo = OPScreenLockInfo(); + NSDictionary *springBoardState = OPSpringBoardPublishedState(); + NSDictionary *recentLayoutInfo = OPSpringBoardRecentLayoutInfo(20); + NSDictionary *screenshotInfo = OPScreenScreenshotInfo(request ?: @{}); + BOOL isLocked = [lockInfo[@"locked"] boolValue]; + BOOL hasFreshSpringBoardState = [springBoardState[@"status"] isEqualToString:@"ok"]; + NSDictionary *springBoardUITree = [springBoardState[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? springBoardState[@"ui_tree"] : @{}; + BOOL hasSpringBoardUITree = hasFreshSpringBoardState && + [springBoardUITree[@"status"] isEqualToString:@"ok"]; + NSString *springBoardForeground = [springBoardState[@"foreground_app"] isKindOfClass:[NSString class]] + ? springBoardState[@"foreground_app"] : @""; + BOOL foregroundDisagreesWithSpringBoardState = NO; + if (hasFreshSpringBoardState && springBoardForeground.length > 0) { + if (foregroundBundleId.length == 0 || + [foregroundBundleId isEqualToString:@"unknown"] || + [foregroundBundleId isEqualToString:springBoardForeground]) { + foregroundBundleId = springBoardForeground; + foregroundSource = @"springboard_state"; + } else { + foregroundDisagreesWithSpringBoardState = YES; + } + } + NSDictionary *springBoardDisplay = [springBoardState[@"display"] isKindOfClass:[NSDictionary class]] + ? springBoardState[@"display"] : @{}; + if (hasFreshSpringBoardState && !displayInfo[@"orientation_name"] && + [springBoardDisplay[@"orientation_name"] isKindOfClass:[NSString class]]) { + displayInfo[@"orientation_name"] = springBoardDisplay[@"orientation_name"]; + displayInfo[@"orientation"] = springBoardDisplay[@"orientation"] ?: @0; + displayInfo[@"orientation_provider"] = @"OpenPhoneVolumeTrigger.SpringBoardState"; + } + NSString *recentFirstBundleId = [recentLayoutInfo[@"first_bundle_id"] isKindOfClass:[NSString class]] + ? recentLayoutInfo[@"first_bundle_id"] : @""; + if (foregroundBundleId.length == 0 && !isLocked && recentFirstBundleId.length > 0) { + foregroundBundleId = recentFirstBundleId; + foregroundSource = @"springboard_recent_layout_inferred"; + } + if (foregroundBundleId.length == 0) { + foregroundBundleId = @"unknown"; + } + NSDictionary *appUIState = OPAppUIPublishedState(foregroundBundleId); + NSDictionary *appUITree = [appUIState[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? appUIState[@"ui_tree"] : @{}; + BOOL hasAppUITree = !isLocked && + [appUIState[@"status"] isEqualToString:@"ok"] && + [appUITree[@"status"] isEqualToString:@"ok"]; + NSDictionary *selectedUITree = hasAppUITree ? appUITree : springBoardUITree; + BOOL hasSelectedUITree = hasAppUITree || hasSpringBoardUITree; + NSArray *visibleText = hasSelectedUITree && + [selectedUITree[@"visible_text"] isKindOfClass:[NSArray class]] + ? selectedUITree[@"visible_text"] : @[]; + NSArray *interactiveElements = hasSelectedUITree && + [selectedUITree[@"interactive_elements"] isKindOfClass:[NSArray class]] + ? selectedUITree[@"interactive_elements"] : @[]; + NSArray *windows = hasSelectedUITree && + [selectedUITree[@"windows"] isKindOfClass:[NSArray class]] + ? selectedUITree[@"windows"] : @[]; + NSString *uiTreeSource = hasAppUITree ? @"app_process" : + (hasSpringBoardUITree ? @"springboard_state" : @"unavailable"); + BOOL screenshotOK = [screenshotInfo[@"status"] isEqualToString:@"ok"]; + NSMutableArray *riskFlags = [NSMutableArray array]; + if (hasAppUITree) { + [riskFlags addObject:@"ui_tree_app_process"]; + if (interactiveElements.count == 0) { + [riskFlags addObject:@"interactive_elements_empty"]; + } + if (visibleText.count == 0) { + [riskFlags addObject:@"visible_text_empty"]; + } + } else if (hasSpringBoardUITree) { + [riskFlags addObject:@"ui_tree_springboard_only"]; + if (interactiveElements.count == 0) { + [riskFlags addObject:@"interactive_elements_empty"]; + } + if (visibleText.count == 0) { + [riskFlags addObject:@"visible_text_empty"]; + } + } else { + [riskFlags addObject:@"ui_tree_unavailable"]; + } + if (!isLocked && OPBundleIdentifierLooksValid(foregroundBundleId) && !hasAppUITree) { + NSString *appUIStatus = [appUIState[@"status"] isKindOfClass:[NSString class]] + ? appUIState[@"status"] : @"unavailable"; + NSString *appUIReason = [appUIState[@"reason"] isKindOfClass:[NSString class]] + ? appUIState[@"reason"] : @""; + if ([appUIStatus isEqualToString:@"stale"]) { + [riskFlags addObject:@"app_ui_state_stale"]; + } else if (appUIReason.length > 0) { + [riskFlags addObject:[NSString stringWithFormat:@"app_ui_%@", appUIReason]]; + } else { + [riskFlags addObject:@"app_ui_unavailable"]; + } + } + if ([foregroundSource isEqualToString:@"unavailable"]) { + [riskFlags addObject:@"foreground_app_unavailable"]; + } else if ([foregroundSource isEqualToString:@"springboard_recent_layout_inferred"]) { + [riskFlags addObject:@"foreground_app_inferred"]; + } + if (foregroundDisagreesWithSpringBoardState) { + [riskFlags addObject:@"foreground_app_springboard_state_disagreement"]; + } + if ([springBoardState[@"status"] isEqualToString:@"stale"]) { + [riskFlags addObject:@"springboard_state_stale"]; + } else if ([springBoardState[@"status"] isEqualToString:@"unavailable"]) { + [riskFlags addObject:@"springboard_state_unavailable"]; + } + if (!screenshotOK) { + [riskFlags addObject:@"screen_metadata_only"]; + NSString *screenshotStatus = [screenshotInfo[@"status"] isKindOfClass:[NSString class]] + ? screenshotInfo[@"status"] : @"unknown"; + NSString *screenshotReason = [screenshotInfo[@"reason"] isKindOfClass:[NSString class]] + ? screenshotInfo[@"reason"] : @""; + if ([screenshotReason isEqualToString:@"helper_not_installed"]) { + [riskFlags addObject:@"screenshot_provider_missing"]; + } else if ([screenshotStatus isEqualToString:@"not_requested"]) { + [riskFlags addObject:@"screenshot_not_requested"]; + } else { + [riskFlags addObject:@"screenshot_unavailable"]; + } + } + if (isLocked) { + [riskFlags insertObject:@"device_locked" atIndex:0]; + } + NSString *captureMode = screenshotOK ? @"screenshot_file" : @"metadata_only"; + NSString *source = screenshotOK ? @"openphone.agentd.screen.screenshot" + : @"openphone.agentd.screen.metadata"; + NSString *state = nil; + if (isLocked) { + state = screenshotOK ? @"screen.locked.screenshot" : @"screen.locked.metadata_only"; + } else if (screenshotOK) { + state = @"screen.observed.screenshot"; + } else { + state = [foregroundBundleId isEqualToString:@"unknown"] + ? @"screen.metadata_only" : @"screen.observed.metadata_only"; + } + NSDictionary *result = @{ + @"status": @"ok", + @"state": state, + @"task_id": taskId, + @"timestamp_ms": @(OPNowMs()), + @"capture_mode": captureMode, + @"source": source, + @"screenshot": screenshotInfo, + @"context": @{ + @"foreground_app": foregroundBundleId, + @"foreground_package": foregroundBundleId, + @"foreground_source": foregroundSource, + @"springboard_state": springBoardState, + @"app_ui_state": appUIState, + @"last_known_foreground_candidate": recentFirstBundleId ?: @"", + @"recent_apps": recentLayoutInfo[@"apps"] ?: @[], + @"recent_apps_source": recentLayoutInfo[@"provider"] ?: @"SpringBoard.RecentAppLayouts", + @"recent_apps_status": recentLayoutInfo[@"status"] ?: @"unknown", + @"running_apps": runningApps, + @"display": displayInfo, + @"lock": lockInfo, + @"screenshot": screenshotInfo, + @"ui_tree": selectedUITree.count > 0 ? selectedUITree : @{ + @"status": @"unavailable", + @"provider": @"SpringBoard.UIKitAccessibility", + @"reason": @"screen_missing_ui_tree" + }, + @"ui_tree_source": uiTreeSource, + @"visible_text": visibleText, + @"interactive_elements": interactiveElements, + @"windows": windows, + @"notifications": @[], + @"risk_flags": riskFlags, + @"source": source + }, + @"request": request ?: @{} + }; + NSString *auditDetail = screenshotOK ? @"screenshot_ok" : @"metadata_only"; + if (!screenshotOK) { + NSString *status = [screenshotInfo[@"status"] isKindOfClass:[NSString class]] + ? screenshotInfo[@"status"] : @"unknown"; + NSString *reason = [screenshotInfo[@"reason"] isKindOfClass:[NSString class]] + ? screenshotInfo[@"reason"] : @""; + auditDetail = reason.length > 0 + ? [NSString stringWithFormat:@"screenshot_%@:%@", status, reason] + : [NSString stringWithFormat:@"screenshot_%@", status]; + } + OPRecordAudit(@"screen_capture", taskId, @"screen.read.visible", @"allow_task_scoped", + request, auditDetail); + NSDictionary *trajectoryPayload = OPBoolFromRequest(request, @"compact_trajectory", NO) + ? OPModelScreenTraceSummary(result) : result; + OPRecordTrajectory(taskId, @"screen_observed", trajectoryPayload); + return result; +} + +static BOOL OPDoubleForKey(NSDictionary *dictionary, NSString *key, double *outValue) { + id value = dictionary[key]; + if ([value isKindOfClass:[NSNumber class]]) { + *outValue = [value doubleValue]; + return YES; + } + if ([value isKindOfClass:[NSString class]]) { + NSString *string = value; + if (string.length == 0) { + return NO; + } + *outValue = [string doubleValue]; + return YES; + } + return NO; +} + +static BOOL OPCenterFromBoundsObject(id boundsObject, double *outX, double *outY) { + if ([boundsObject isKindOfClass:[NSDictionary class]]) { + NSDictionary *bounds = boundsObject; + double x = 0.0; + double y = 0.0; + double width = 0.0; + double height = 0.0; + if (OPDoubleForKey(bounds, @"x", &x) && + OPDoubleForKey(bounds, @"y", &y) && + OPDoubleForKey(bounds, @"width", &width) && + OPDoubleForKey(bounds, @"height", &height)) { + *outX = x + (width / 2.0); + *outY = y + (height / 2.0); + return YES; + } + if (OPDoubleForKey(bounds, @"left", &x) && + OPDoubleForKey(bounds, @"top", &y) && + OPDoubleForKey(bounds, @"right", &width) && + OPDoubleForKey(bounds, @"bottom", &height)) { + *outX = x + ((width - x) / 2.0); + *outY = y + ((height - y) / 2.0); + return YES; + } + } + if ([boundsObject isKindOfClass:[NSArray class]]) { + NSArray *bounds = boundsObject; + if (bounds.count >= 4) { + double x = [bounds[0] doubleValue]; + double y = [bounds[1] doubleValue]; + double width = [bounds[2] doubleValue]; + double height = [bounds[3] doubleValue]; + *outX = x + (width / 2.0); + *outY = y + (height / 2.0); + return YES; + } + } + return NO; +} + +static NSDictionary *OPInteractiveElementFromScreen(NSDictionary *screen, NSString *elementId) { + if (![elementId isKindOfClass:[NSString class]] || elementId.length == 0) { + return nil; + } + NSDictionary *context = [screen[@"context"] isKindOfClass:[NSDictionary class]] + ? screen[@"context"] : @{}; + NSMutableArray *candidateArrays = [NSMutableArray array]; + if ([context[@"interactive_elements"] isKindOfClass:[NSArray class]]) { + [candidateArrays addObject:context[@"interactive_elements"]]; + } + NSDictionary *uiTree = [context[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? context[@"ui_tree"] : @{}; + if ([uiTree[@"interactive_elements"] isKindOfClass:[NSArray class]]) { + [candidateArrays addObject:uiTree[@"interactive_elements"]]; + } + + for (NSArray *elements in candidateArrays) { + for (id object in elements) { + if (![object isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *element = object; + NSString *identifier = [element[@"id"] isKindOfClass:[NSString class]] + ? element[@"id"] : @""; + NSString *viewIdentifier = [element[@"view_id"] isKindOfClass:[NSString class]] + ? element[@"view_id"] : @""; + if ([identifier isEqualToString:elementId] || + (viewIdentifier.length > 0 && [viewIdentifier isEqualToString:elementId])) { + return element; + } + } + } + return nil; +} + +static NSDictionary *OPResolvedElementSummary(NSDictionary *element) { + if (![element isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSMutableDictionary *summary = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"id", @"view_id", @"kind", @"label", @"bounds", @"enabled", + @"focused", @"sensitive", @"risk_hint", @"scope", @"input_scope", + @"source_bundle_id", @"dom_index", @"tag", @"input_type"]) { + id value = element[key]; + if (value) { + summary[key] = value; + } + } + return summary; +} + +static NSString *OPInputActionType(NSDictionary *result, NSString *fallback) { + NSString *actionType = [result[@"action_type"] isKindOfClass:[NSString class]] + ? result[@"action_type"] : @""; + return actionType.length > 0 ? actionType : (fallback ?: @""); +} + +static NSString *OPInputProviderScope(NSDictionary *result, NSString *provider) { + NSString *scope = [result[@"scope"] isKindOfClass:[NSString class]] + ? result[@"scope"] : @""; + if (scope.length > 0) { + return scope; + } + NSDictionary *bridge = [result[@"bridge"] isKindOfClass:[NSDictionary class]] + ? result[@"bridge"] : @{}; + scope = [bridge[@"scope"] isKindOfClass:[NSString class]] + ? bridge[@"scope"] : @""; + if (scope.length > 0) { + return scope; + } + if ([provider containsString:@"OpenPhoneVolumeTrigger.SpringBoardInput"]) { + return @"springboard_windows"; + } + if ([provider containsString:@"OpenPhoneAppIntrospector.AppInput"] || + [provider containsString:@"OpenPhoneAppIntrospector.WebContentInput"]) { + if ([provider containsString:@"OpenPhoneAppIntrospector.WebContentInput"]) { + return @"web_content_process"; + } + return @"app_process"; + } + if ([provider containsString:@"IOHIDEventSystemClient"]) { + return @"daemon_hid"; + } + if ([provider containsString:@"SpringBoardServices"] || + [provider containsString:@"GraphicsServices"]) { + return @"springboard_services"; + } + return @"unknown"; +} + +static BOOL OPInputPasscodeVisible(NSDictionary *result) { + if ([result[@"passcode_visible"] respondsToSelector:@selector(boolValue)] && + [result[@"passcode_visible"] boolValue]) { + return YES; + } + NSDictionary *fallback = [result[@"lockscreen_fallback"] isKindOfClass:[NSDictionary class]] + ? result[@"lockscreen_fallback"] : @{}; + NSArray *attempts = [fallback[@"attempts"] isKindOfClass:[NSArray class]] + ? fallback[@"attempts"] : @[]; + for (id object in attempts) { + if (![object isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *attempt = object; + if ([attempt[@"passcode_visible"] respondsToSelector:@selector(boolValue)] && + [attempt[@"passcode_visible"] boolValue]) { + return YES; + } + } + return NO; +} + +static BOOL OPInputUnlockVerified(NSDictionary *result) { + NSDictionary *unlock = [result[@"lockscreen_unlock"] isKindOfClass:[NSDictionary class]] + ? result[@"lockscreen_unlock"] : @{}; + if (![unlock[@"status"] isEqualToString:@"ok"]) { + return NO; + } + if ([unlock[@"locked_after_attempt"] respondsToSelector:@selector(boolValue)]) { + return ![unlock[@"locked_after_attempt"] boolValue]; + } + return NO; +} + +static NSDictionary *OPInputAttemptVerification(NSDictionary *result, NSString *actionType) { + NSString *provider = [result[@"provider"] isKindOfClass:[NSString class]] + ? result[@"provider"] : @""; + NSString *status = [result[@"status"] isKindOfClass:[NSString class]] + ? result[@"status"] : @""; + if (status.length == 0 && [result[@"ok"] respondsToSelector:@selector(boolValue)]) { + status = [result[@"ok"] boolValue] ? @"ok" : @"unavailable"; + } + NSString *reason = [result[@"reason"] isKindOfClass:[NSString class]] + ? result[@"reason"] : @""; + if (![status isEqualToString:@"ok"]) { + return @{ + @"status": @"failed", + @"source": @"provider_status", + @"reason": reason.length > 0 ? reason : (status.length > 0 ? status : @"not_attempted") + }; + } + if ([actionType isEqualToString:@"show_passcode"] && OPInputPasscodeVisible(result)) { + return @{ + @"status": @"verified", + @"source": @"springboard_passcode_visible", + @"reason": @"passcode_visible_after_invocation" + }; + } + if ([actionType isEqualToString:@"unlock_with_passcode"] && OPInputUnlockVerified(result)) { + return @{ + @"status": @"verified", + @"source": @"springboard_lock_state", + @"reason": @"unlocked_after_invocation" + }; + } + if ([provider containsString:@"IOHIDEventSystemClient"]) { + return @{ + @"status": @"unverified", + @"source": @"iohid_dispatch", + @"reason": @"dispatch_only_no_visible_verification" + }; + } + if ([provider containsString:@"OpenPhoneAppIntrospector.AppInput"] || + [provider containsString:@"OpenPhoneAppIntrospector.WebContentInput"]) { + NSString *activationMethod = [result[@"activation_method"] isKindOfClass:[NSString class]] + ? result[@"activation_method"] : @""; + long long textLength = [result[@"text_length"] respondsToSelector:@selector(longLongValue)] + ? [result[@"text_length"] longLongValue] : 0; + long long beforeTextLength = [result[@"before_text_length"] respondsToSelector:@selector(longLongValue)] + ? [result[@"before_text_length"] longLongValue] : -1; + long long afterTextLength = [result[@"after_text_length"] respondsToSelector:@selector(longLongValue)] + ? [result[@"after_text_length"] longLongValue] : -1; + if ([actionType isEqualToString:@"type_text"] && + [activationMethod isEqualToString:@"text_input_insert"] && + textLength > 0 && + beforeTextLength >= 0 && + afterTextLength > beforeTextLength) { + return @{ + @"status": @"verified", + @"source": @"app_process_text_state", + @"reason": @"text_length_changed_after_insert" + }; + } + if ([actionType isEqualToString:@"type_text"] && + [activationMethod isEqualToString:@"webkit_dom_text_input"] && + textLength > 0 && + beforeTextLength >= 0 && + afterTextLength > beforeTextLength) { + return @{ + @"status": @"verified", + @"source": @"web_content_dom_state", + @"reason": @"dom_text_length_changed_after_insert" + }; + } + return @{ + @"status": @"unverified", + @"source": @"app_process_activation", + @"reason": @"app_process_activation_without_visible_verification" + }; + } + if ([result[@"activation_method"] isKindOfClass:[NSString class]]) { + return @{ + @"status": @"unverified", + @"source": @"springboard_activation", + @"reason": @"activation_without_visible_verification" + }; + } + return @{ + @"status": @"unverified", + @"source": @"provider_dispatch", + @"reason": @"no_visible_verification" + }; +} + +static NSDictionary *OPInputAttemptSummaryForAction(NSDictionary *result, NSString *fallbackActionType) { + if (![result isKindOfClass:[NSDictionary class]]) { + return @{ + @"provider": @"none", + @"scope": @"none", + @"action_type": fallbackActionType ?: @"", + @"status": @"not_attempted", + @"verification": @{ + @"status": @"failed", + @"source": @"none", + @"reason": @"no_provider_result" + } + }; + } + NSMutableDictionary *summary = [NSMutableDictionary dictionary]; + NSString *status = [result[@"status"] isKindOfClass:[NSString class]] + ? result[@"status"] : nil; + if (!status && [result[@"ok"] respondsToSelector:@selector(boolValue)]) { + status = [result[@"ok"] boolValue] ? @"ok" : @"unavailable"; + } + NSString *provider = [result[@"provider"] isKindOfClass:[NSString class]] + ? result[@"provider"] : @"unknown"; + NSString *actionType = OPInputActionType(result, fallbackActionType); + summary[@"provider"] = provider; + summary[@"scope"] = OPInputProviderScope(result, provider); + summary[@"action_type"] = actionType ?: @""; + if (status.length > 0) { + summary[@"status"] = status; + } + for (NSString *key in @[@"reason", @"request_id", @"kind", @"strategy", + @"activation_method", @"activated_class", @"target_class", @"preferred_input_scope", + @"dom_index", @"tag", @"input_type", @"attempts", + @"diagnostics", @"text_length", @"before_text_length", @"after_text_length"]) { + id value = result[key]; + if (value) { + summary[key] = value; + } + } + NSMutableDictionary *dispatchMetadata = [NSMutableDictionary dictionary]; + for (NSString *key in @[ + @"x", @"y", @"start_x", @"start_y", @"end_x", @"end_y", + @"duration_ms", @"timeout_ms", @"usage", @"usage_page"]) { + id value = result[key]; + if (value) { + dispatchMetadata[key] = value; + } + } + if (dispatchMetadata.count > 0) { + summary[@"dispatch_metadata"] = dispatchMetadata; + } else if ([result[@"dispatch_metadata"] isKindOfClass:[NSDictionary class]]) { + summary[@"dispatch_metadata"] = result[@"dispatch_metadata"]; + } + summary[@"verification"] = OPInputAttemptVerification(result, actionType); + return summary; +} + +static NSArray *OPInputProviderAttemptsForResult(NSDictionary *result, + NSString *actionType) { + if (![result isKindOfClass:[NSDictionary class]]) { + return @[OPInputAttemptSummaryForAction(nil, actionType)]; + } + NSArray *existing = [result[@"provider_attempts"] isKindOfClass:[NSArray class]] + ? result[@"provider_attempts"] : nil; + if (existing.count > 0) { + NSMutableArray *attempts = [NSMutableArray array]; + for (id object in existing) { + if ([object isKindOfClass:[NSDictionary class]]) { + [attempts addObject:OPInputAttemptSummaryForAction(object, actionType)]; + } + } + if (attempts.count > 0) { + return attempts; + } + } + NSDictionary *providerResult = [result[@"provider_result"] isKindOfClass:[NSDictionary class]] + ? result[@"provider_result"] : nil; + if (providerResult) { + NSDictionary *wake = [providerResult[@"wake"] isKindOfClass:[NSDictionary class]] + ? providerResult[@"wake"] : nil; + NSDictionary *home = [providerResult[@"home"] isKindOfClass:[NSDictionary class]] + ? providerResult[@"home"] : nil; + if (wake || home) { + NSMutableArray *attempts = [NSMutableArray array]; + if (wake) { + [attempts addObject:OPInputAttemptSummaryForAction(wake, actionType ?: @"wake_and_home")]; + } + if (home) { + [attempts addObject:OPInputAttemptSummaryForAction(home, actionType ?: @"wake_and_home")]; + } + return attempts; + } + return @[OPInputAttemptSummaryForAction(providerResult, actionType)]; + } + return @[OPInputAttemptSummaryForAction(nil, actionType)]; +} + +static NSDictionary *OPInputResultVerification(NSArray *attempts, + NSDictionary *result) { + for (NSDictionary *attempt in attempts) { + NSDictionary *verification = [attempt[@"verification"] isKindOfClass:[NSDictionary class]] + ? attempt[@"verification"] : @{}; + if ([verification[@"status"] isEqualToString:@"verified"]) { + return @{ + @"status": @"verified", + @"source": verification[@"source"] ?: @"provider_attempt", + @"reason": verification[@"reason"] ?: @"visible_state_verified" + }; + } + } + NSString *state = [result[@"state"] isKindOfClass:[NSString class]] + ? result[@"state"] : @""; + if ([state isEqualToString:@"action.executed"]) { + return @{ + @"status": @"unverified", + @"source": @"provider_attempts", + @"reason": @"dispatch_without_visible_state_change" + }; + } + return @{ + @"status": @"failed", + @"source": @"provider_attempts", + @"reason": result[@"detail"] ?: @"input_failed" + }; +} + +static NSDictionary *OPFinalizeInputResult(NSDictionary *result, NSString *actionType) { + if (![result isKindOfClass:[NSDictionary class]]) { + return result ?: @{}; + } + NSMutableDictionary *finalResult = [result mutableCopy]; + NSArray *attempts = OPInputProviderAttemptsForResult(finalResult, actionType); + NSDictionary *verification = OPInputResultVerification(attempts, finalResult); + finalResult[@"provider_attempts"] = attempts ?: @[]; + finalResult[@"verification"] = verification; + NSString *state = [finalResult[@"state"] isKindOfClass:[NSString class]] + ? finalResult[@"state"] : @""; + if ([verification[@"status"] isEqualToString:@"verified"]) { + finalResult[@"user_facing_status"] = @"verified"; + } else if ([state isEqualToString:@"action.executed"]) { + finalResult[@"user_facing_status"] = @"dispatch_unverified"; + } else { + finalResult[@"user_facing_status"] = @"failed"; + } + return finalResult; +} + +static BOOL OPPointFromAction(NSDictionary *action, double *outX, double *outY) { + if (OPDoubleForKey(action, @"x", outX) && OPDoubleForKey(action, @"y", outY)) { + return YES; + } + id centerObject = action[@"center"]; + if ([centerObject isKindOfClass:[NSDictionary class]]) { + NSDictionary *center = centerObject; + if (OPDoubleForKey(center, @"x", outX) && OPDoubleForKey(center, @"y", outY)) { + return YES; + } + } + if (OPCenterFromBoundsObject(action[@"bounds"], outX, outY)) { + return YES; + } + id elementObject = action[@"element"]; + if ([elementObject isKindOfClass:[NSDictionary class]]) { + NSDictionary *element = elementObject; + if (OPDoubleForKey(element, @"x", outX) && OPDoubleForKey(element, @"y", outY)) { + return YES; + } + if (OPCenterFromBoundsObject(element[@"bounds"], outX, outY)) { + return YES; + } + } + return NO; +} + +static NSDictionary *OPActionForRecording(NSDictionary *action) { + if (![action isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSString *type = [action[@"type"] isKindOfClass:[NSString class]] ? action[@"type"] : @""; + NSMutableDictionary *recorded = [NSMutableDictionary dictionary]; + for (id key in action) { + NSString *keyString = [key isKindOfClass:[NSString class]] + ? key : [key description]; + if (OPSensitiveKey(keyString)) { + recorded[OPRedactedKeyName(keyString)] = @""; + } else { + recorded[keyString] = OPRedactedObject(action[key], 1); + } + } + if ([type isEqualToString:@"type_text"]) { + NSString *text = [action[@"text"] isKindOfClass:[NSString class]] ? action[@"text"] : @""; + recorded[@"text"] = [NSString stringWithFormat:@"", + (unsigned long)text.length]; + } + return recorded; +} + +static NSDictionary *OPExecuteAction(NSDictionary *request) { + NSDictionary *action = [request[@"action"] isKindOfClass:[NSDictionary class]] + ? request[@"action"] : request; + NSString *type = [action[@"type"] isKindOfClass:[NSString class]] ? action[@"type"] : @""; + NSString *taskId = [request[@"task_id"] isKindOfClass:[NSString class]] ? request[@"task_id"] : @""; + NSDictionary *recordedRequest = request; + NSDictionary *result = nil; + NSString *capability = @"unknown"; + if ([type isEqualToString:@"wait"]) { + capability = @"tasks.observe"; + NSNumber *durationNumber = [action[@"duration_ms"] isKindOfClass:[NSNumber class]] + ? action[@"duration_ms"] : @1000; + long durationMs = MAX(0, MIN(durationNumber.longValue, 30000)); + usleep((useconds_t)(durationMs * 1000)); + result = @{ + @"state": @"action.executed", + @"task_id": taskId, + @"capability": capability, + @"detail": [NSString stringWithFormat:@"wait:%ldms", durationMs], + @"source": @"openphone.agentd" + }; + } else if ([type isEqualToString:@"open_url"]) { + capability = @"network.use"; + NSString *url = [action[@"url"] isKindOfClass:[NSString class]] ? action[@"url"] : @""; + if (url.length == 0) { + result = @{ + @"state": @"action.denied.missing_argument", + @"task_id": taskId, + @"capability": capability, + @"detail": @"missing_url", + @"source": @"openphone.agentd" + }; + } else { + NSDictionary *spawn = OPUiOpen(@[@"--url", url]); + result = @{ + @"state": [spawn[@"ok"] boolValue] ? @"action.executed" : @"action.denied.input_failed", + @"task_id": taskId, + @"capability": capability, + @"detail": @"open_url", + @"provider_result": spawn, + @"source": @"openphone.agentd" + }; + } + } else if ([type isEqualToString:@"open_app"]) { + capability = @"apps.launch"; + NSDictionary *target = [action[@"target"] isKindOfClass:[NSDictionary class]] + ? action[@"target"] : @{}; + NSString *bundleId = [action[@"bundle_id"] isKindOfClass:[NSString class]] + ? action[@"bundle_id"] : nil; + if (bundleId.length == 0 && [target[@"package"] isKindOfClass:[NSString class]]) { + bundleId = target[@"package"]; + } + if (bundleId.length == 0 && [target[@"bundle_id"] isKindOfClass:[NSString class]]) { + bundleId = target[@"bundle_id"]; + } + if (bundleId.length == 0) { + result = @{ + @"state": @"action.denied.missing_argument", + @"task_id": taskId, + @"capability": capability, + @"detail": @"missing_bundle_id", + @"source": @"openphone.agentd" + }; + } else { + NSDictionary *spawn = OPUiOpen(@[@"--bundleid", bundleId]); + result = @{ + @"state": [spawn[@"ok"] boolValue] ? @"action.executed" : @"action.denied.input_failed", + @"task_id": taskId, + @"capability": capability, + @"detail": [NSString stringWithFormat:@"open_app:%@", bundleId], + @"provider_result": spawn, + @"source": @"openphone.agentd" + }; + } + } else if ([type isEqualToString:@"home"]) { + capability = @"input.perform"; + NSDictionary *home = OPPressHome(); + result = @{ + @"state": [home[@"ok"] boolValue] ? @"action.executed" : @"action.denied.input_failed", + @"task_id": taskId, + @"capability": capability, + @"detail": @"home", + @"provider_result": home, + @"source": @"openphone.agentd" + }; + } else if ([type isEqualToString:@"wake_and_home"]) { + capability = @"input.perform"; + NSDictionary *wake = OPWakeScreen(); + NSDictionary *home = OPPressHome(); + BOOL ok = [wake[@"ok"] boolValue] && [home[@"ok"] boolValue]; + result = @{ + @"state": ok ? @"action.executed" : @"action.denied.input_failed", + @"task_id": taskId, + @"capability": capability, + @"detail": @"wake_and_home", + @"provider_result": @{ + @"wake": wake, + @"home": home + }, + @"source": @"openphone.agentd" + }; + } else if ([type isEqualToString:@"show_passcode"]) { + capability = @"input.perform"; + NSMutableDictionary *bridgeAction = [action mutableCopy] ?: [NSMutableDictionary dictionary]; + bridgeAction[@"type"] = @"show_passcode"; + NSDictionary *springBoardInput = OPSpringBoardInputInfo(bridgeAction); + BOOL springBoardOK = [springBoardInput[@"status"] isEqualToString:@"ok"]; + result = @{ + @"state": springBoardOK ? @"action.executed" : @"action.denied.input_failed", + @"task_id": taskId, + @"capability": capability, + @"detail": @"show_passcode", + @"provider_result": springBoardInput ?: @{}, + @"provider_attempts": @[OPInputAttemptSummaryForAction(springBoardInput, type)], + @"source": @"openphone.agentd" + }; + } else if ([type isEqualToString:@"unlock_with_passcode"]) { + capability = @"input.perform"; + NSString *passcode = [action[@"passcode"] isKindOfClass:[NSString class]] + ? action[@"passcode"] : @""; + if (passcode.length == 0) { + result = @{ + @"state": @"action.denied.missing_argument", + @"task_id": taskId, + @"capability": capability, + @"detail": @"missing_passcode", + @"source": @"openphone.agentd" + }; + } else { + NSMutableDictionary *bridgeAction = [action mutableCopy] ?: [NSMutableDictionary dictionary]; + bridgeAction[@"type"] = @"unlock_with_passcode"; + if (![bridgeAction[@"input_timeout_ms"] respondsToSelector:@selector(longLongValue)]) { + bridgeAction[@"input_timeout_ms"] = @3000; + } + NSDictionary *springBoardInput = OPSpringBoardInputInfo(bridgeAction); + BOOL springBoardOK = [springBoardInput[@"status"] isEqualToString:@"ok"]; + result = @{ + @"state": springBoardOK ? @"action.executed" : @"action.denied.input_failed", + @"task_id": taskId, + @"capability": capability, + @"detail": @"unlock_with_passcode", + @"provider_result": springBoardInput ?: @{}, + @"provider_attempts": @[OPInputAttemptSummaryForAction(springBoardInput, type)], + @"source": @"openphone.agentd" + }; + } + } else if ([type isEqualToString:@"tap"] || [type isEqualToString:@"tap_element"]) { + capability = @"input.perform"; + double x = 0.0; + double y = 0.0; + BOOL hasPoint = OPPointFromAction(action, &x, &y); + NSDictionary *resolvedElement = nil; + NSDictionary *resolutionScreen = nil; + NSDictionary *appInput = nil; + NSString *elementId = [action[@"element_id"] isKindOfClass:[NSString class]] + ? action[@"element_id"] : @""; + NSString *coordinateSource = hasPoint ? @"action_coordinates" : @""; + if (!hasPoint && [type isEqualToString:@"tap_element"] && elementId.length > 0) { + resolutionScreen = OPGetScreen(@{ + @"command": @"get_screen", + @"task_id": taskId ?: @"", + @"include_screenshot": @NO, + @"compact_trajectory": @YES, + @"reason": @"tap_element_resolve" + }); + resolvedElement = OPInteractiveElementFromScreen(resolutionScreen, elementId); + if (resolvedElement) { + id enabledValue = resolvedElement[@"enabled"]; + BOOL enabled = ![enabledValue respondsToSelector:@selector(boolValue)] || + [enabledValue boolValue]; + if (!enabled) { + result = @{ + @"state": @"action.denied.element_disabled", + @"task_id": taskId, + @"capability": capability, + @"detail": [NSString stringWithFormat:@"element_disabled:%@", elementId], + @"target": OPResolvedElementSummary(resolvedElement), + @"source": @"openphone.agentd" + }; + } else if (OPCenterFromBoundsObject(resolvedElement[@"bounds"], &x, &y)) { + hasPoint = YES; + coordinateSource = @"ui_tree.bounds_center"; + } else { + result = @{ + @"state": @"action.denied.missing_coordinates", + @"task_id": taskId, + @"capability": capability, + @"detail": [NSString stringWithFormat:@"element_missing_bounds:%@", elementId], + @"target": OPResolvedElementSummary(resolvedElement), + @"source": @"openphone.agentd" + }; + } + } else { + hasPoint = NO; + } + } + if (!result && resolvedElement) { + NSString *scope = [resolvedElement[@"scope"] isKindOfClass:[NSString class]] + ? resolvedElement[@"scope"] : @""; + NSString *inputScope = [resolvedElement[@"input_scope"] isKindOfClass:[NSString class]] + ? resolvedElement[@"input_scope"] : scope; + NSString *riskHint = [resolvedElement[@"risk_hint"] isKindOfClass:[NSString class]] + ? resolvedElement[@"risk_hint"] : @""; + if ([scope isEqualToString:@"app_process"] || + [scope isEqualToString:@"web_content_process"] || + [riskHint isEqualToString:@"app_process"] || + [riskHint isEqualToString:@"web_content_process"]) { + NSString *bundleId = [resolvedElement[@"source_bundle_id"] isKindOfClass:[NSString class]] + ? resolvedElement[@"source_bundle_id"] : @""; + if (bundleId.length == 0) { + NSDictionary *context = [resolutionScreen[@"context"] isKindOfClass:[NSDictionary class]] + ? resolutionScreen[@"context"] : @{}; + bundleId = [context[@"foreground_app"] isKindOfClass:[NSString class]] + ? context[@"foreground_app"] : @""; + } + NSMutableDictionary *appAction = [action mutableCopy] ?: [NSMutableDictionary dictionary]; + appAction[@"type"] = type; + if (elementId.length > 0) { + appAction[@"element_id"] = elementId; + } + appAction[@"x"] = @(x); + appAction[@"y"] = @(y); + appAction[@"duration_ms"] = [type isEqualToString:@"long_press"] ? @700 : @80; + appAction[@"target"] = OPResolvedElementSummary(resolvedElement); + if (inputScope.length > 0) { + appAction[@"preferred_input_scope"] = inputScope; + } + appInput = OPAppInputInfo(appAction, bundleId); + BOOL appInputOK = [appInput[@"status"] isEqualToString:@"ok"]; + if (appInputOK) { + NSMutableDictionary *mutableResult = [@{ + @"state": @"action.executed", + @"task_id": taskId, + @"capability": capability, + @"detail": [NSString stringWithFormat:@"%@:%@:app_process", type, elementId], + @"provider_result": appInput ?: @{}, + @"provider_attempts": @[OPInputAttemptSummaryForAction(appInput, type)], + @"source": @"openphone.agentd" + } mutableCopy]; + if (coordinateSource.length > 0) { + mutableResult[@"coordinate_source"] = coordinateSource; + } + mutableResult[@"target"] = OPResolvedElementSummary(resolvedElement); + result = mutableResult; + } + } + } + if (!hasPoint && !result && [type isEqualToString:@"tap_element"] && elementId.length > 0) { + NSMutableDictionary *bridgeAction = [action mutableCopy] ?: [NSMutableDictionary dictionary]; + bridgeAction[@"type"] = @"tap_element"; + bridgeAction[@"element_id"] = elementId; + NSDictionary *springBoardInput = OPSpringBoardInputInfo(bridgeAction); + BOOL springBoardOK = [springBoardInput[@"status"] isEqualToString:@"ok"]; + result = @{ + @"state": springBoardOK ? @"action.executed" : @"action.denied.element_not_found", + @"task_id": taskId, + @"capability": capability, + @"detail": springBoardOK + ? [NSString stringWithFormat:@"tap_element:%@:springboard_bridge", elementId] + : [NSString stringWithFormat:@"element_not_found:%@", elementId], + @"provider_result": springBoardInput ?: @{}, + @"provider_attempts": @[OPInputAttemptSummaryForAction(springBoardInput, type)], + @"source": @"openphone.agentd" + }; + } + if (!hasPoint && !result) { + NSString *detail = [type isEqualToString:@"tap_element"] + ? @"missing_element_coordinates" : @"missing_coordinates"; + result = @{ + @"state": @"action.denied.missing_coordinates", + @"task_id": taskId, + @"capability": capability, + @"detail": detail, + @"source": @"openphone.agentd" + }; + } else if (!result) { + NSMutableDictionary *bridgeAction = [action mutableCopy] ?: [NSMutableDictionary dictionary]; + bridgeAction[@"type"] = type; + bridgeAction[@"x"] = @(x); + bridgeAction[@"y"] = @(y); + bridgeAction[@"duration_ms"] = @80; + NSDictionary *springBoardInput = OPSpringBoardInputInfo(bridgeAction); + BOOL springBoardOK = [springBoardInput[@"status"] isEqualToString:@"ok"]; + NSDictionary *tap = springBoardOK ? nil : OPPerformHIDTap(x, y, 80); + BOOL ok = springBoardOK || [tap[@"ok"] boolValue]; + NSMutableDictionary *mutableResult = [@{ + @"state": ok ? @"action.executed" : @"action.denied.input_failed", + @"task_id": taskId, + @"capability": capability, + @"detail": elementId.length > 0 + ? [NSString stringWithFormat:@"%@:%@:%0.1f,%0.1f", type, elementId, x, y] + : [NSString stringWithFormat:@"%@:%0.1f,%0.1f", type, x, y], + @"provider_result": springBoardOK ? springBoardInput : (tap ?: @{}), + @"source": @"openphone.agentd" + } mutableCopy]; + NSMutableArray *providerAttempts = [NSMutableArray array]; + if (appInput) { + [providerAttempts addObject:OPInputAttemptSummaryForAction(appInput, type)]; + } + [providerAttempts addObject:OPInputAttemptSummaryForAction(springBoardInput, type)]; + if (!springBoardOK && tap) { + [providerAttempts addObject:OPInputAttemptSummaryForAction(tap, type)]; + mutableResult[@"springboard_provider_result"] = springBoardInput ?: @{}; + } + mutableResult[@"provider_attempts"] = providerAttempts; + if (coordinateSource.length > 0) { + mutableResult[@"coordinate_source"] = coordinateSource; + } + if (resolvedElement) { + mutableResult[@"target"] = OPResolvedElementSummary(resolvedElement); + } + result = mutableResult; + } + } else if ([type isEqualToString:@"long_press"]) { + capability = @"input.perform"; + double x = 0.0; + double y = 0.0; + if (!OPPointFromAction(action, &x, &y)) { + result = @{ + @"state": @"action.denied.missing_coordinates", + @"task_id": taskId, + @"capability": capability, + @"detail": @"missing_coordinates", + @"source": @"openphone.agentd" + }; + } else { + long durationMs = OPLongLongFromRequest(action, @"duration_ms", 700, 100, 5000); + NSMutableDictionary *bridgeAction = [action mutableCopy] ?: [NSMutableDictionary dictionary]; + bridgeAction[@"type"] = @"long_press"; + bridgeAction[@"x"] = @(x); + bridgeAction[@"y"] = @(y); + bridgeAction[@"duration_ms"] = @(durationMs); + NSDictionary *springBoardInput = OPSpringBoardInputInfo(bridgeAction); + BOOL springBoardOK = [springBoardInput[@"status"] isEqualToString:@"ok"]; + NSDictionary *press = springBoardOK ? nil : OPPerformHIDTap(x, y, durationMs); + BOOL ok = springBoardOK || [press[@"ok"] boolValue]; + NSMutableDictionary *mutableResult = [@{ + @"state": ok ? @"action.executed" : @"action.denied.input_failed", + @"task_id": taskId, + @"capability": capability, + @"detail": [NSString stringWithFormat:@"long_press:%0.1f,%0.1f:%ldms", + x, y, durationMs], + @"provider_result": springBoardOK ? springBoardInput : (press ?: @{}), + @"source": @"openphone.agentd" + } mutableCopy]; + NSMutableArray *providerAttempts = [NSMutableArray array]; + [providerAttempts addObject:OPInputAttemptSummaryForAction(springBoardInput, type)]; + if (!springBoardOK && press) { + [providerAttempts addObject:OPInputAttemptSummaryForAction(press, type)]; + mutableResult[@"springboard_provider_result"] = springBoardInput ?: @{}; + } + mutableResult[@"provider_attempts"] = providerAttempts; + result = mutableResult; + } + } else if ([type isEqualToString:@"swipe"]) { + capability = @"input.perform"; + double startX = 0.0; + double startY = 0.0; + double endX = 0.0; + double endY = 0.0; + BOOL hasStart = OPDoubleForKey(action, @"start_x", &startX) && + OPDoubleForKey(action, @"start_y", &startY); + BOOL hasEnd = OPDoubleForKey(action, @"end_x", &endX) && + OPDoubleForKey(action, @"end_y", &endY); + if (!hasStart || !hasEnd) { + result = @{ + @"state": @"action.denied.missing_coordinates", + @"task_id": taskId, + @"capability": capability, + @"detail": @"missing_swipe_coordinates", + @"source": @"openphone.agentd" + }; + } else { + long durationMs = OPLongLongFromRequest(action, @"duration_ms", 300, 50, 5000); + NSDictionary *swipe = OPPerformHIDSwipe(startX, startY, endX, endY, durationMs); + result = @{ + @"state": [swipe[@"ok"] boolValue] + ? @"action.executed" : @"action.denied.input_failed", + @"task_id": taskId, + @"capability": capability, + @"detail": [NSString stringWithFormat:@"swipe:%0.1f,%0.1f:%0.1f,%0.1f:%ldms", + startX, startY, endX, endY, durationMs], + @"provider_result": swipe, + @"source": @"openphone.agentd" + }; + } + } else if ([type isEqualToString:@"type_text"]) { + capability = @"input.perform"; + NSString *text = [action[@"text"] isKindOfClass:[NSString class]] ? action[@"text"] : @""; + NSDictionary *appInput = nil; + recordedRequest = @{ + @"command": request[@"command"] ?: @"execute_action", + @"task_id": taskId ?: @"", + @"action": OPActionForRecording(action) + }; + if (text.length == 0) { + result = @{ + @"state": @"action.denied.missing_argument", + @"task_id": taskId, + @"capability": capability, + @"detail": @"missing_text", + @"source": @"openphone.agentd" + }; + } else { + NSDictionary *screen = OPGetScreen(@{ + @"command": @"get_screen", + @"task_id": taskId ?: @"", + @"include_screenshot": @NO, + @"compact_trajectory": @YES, + @"reason": @"type_text_app_input" + }); + NSDictionary *context = [screen[@"context"] isKindOfClass:[NSDictionary class]] + ? screen[@"context"] : @{}; + NSString *foregroundBundleId = [context[@"foreground_app"] isKindOfClass:[NSString class]] + ? context[@"foreground_app"] : @""; + NSDictionary *appUIState = [context[@"app_ui_state"] isKindOfClass:[NSDictionary class]] + ? context[@"app_ui_state"] : @{}; + NSString *elementId = [action[@"element_id"] isKindOfClass:[NSString class]] + ? action[@"element_id"] : @""; + NSDictionary *resolvedElement = elementId.length > 0 + ? OPInteractiveElementFromScreen(screen, elementId) : nil; + BOOL appUIReady = [context[@"ui_tree_source"] isEqualToString:@"app_process"] && + [appUIState[@"status"] isEqualToString:@"ok"] && + OPBundleIdentifierLooksValid(foregroundBundleId); + if (appUIReady) { + NSDictionary *uiTree = [context[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? context[@"ui_tree"] : @{}; + NSString *preferredScope = [uiTree[@"scope"] isKindOfClass:[NSString class]] + ? uiTree[@"scope"] : @""; + if (resolvedElement) { + NSString *elementInputScope = [resolvedElement[@"input_scope"] isKindOfClass:[NSString class]] + ? resolvedElement[@"input_scope"] : @""; + NSString *elementScope = [resolvedElement[@"scope"] isKindOfClass:[NSString class]] + ? resolvedElement[@"scope"] : @""; + preferredScope = elementInputScope.length > 0 ? elementInputScope : elementScope; + NSString *elementBundleId = [resolvedElement[@"source_bundle_id"] isKindOfClass:[NSString class]] + ? resolvedElement[@"source_bundle_id"] : @""; + if (OPBundleIdentifierLooksValid(elementBundleId)) { + foregroundBundleId = elementBundleId; + } + } + NSMutableDictionary *appAction = [action mutableCopy] ?: [NSMutableDictionary dictionary]; + appAction[@"type"] = @"type_text"; + appAction[@"text_length"] = @(text.length); + if (resolvedElement) { + appAction[@"target"] = OPResolvedElementSummary(resolvedElement); + } + appAction[@"preferred_input_scope"] = preferredScope.length > 0 + ? preferredScope : @"app_process"; + if (![appAction[@"input_timeout_ms"] respondsToSelector:@selector(longLongValue)]) { + appAction[@"input_timeout_ms"] = @3000; + } + appInput = OPAppInputInfo(appAction, foregroundBundleId); + if ([appInput[@"status"] isEqualToString:@"ok"]) { + result = @{ + @"state": @"action.executed", + @"task_id": taskId, + @"capability": capability, + @"detail": [NSString stringWithFormat:@"type_text:%lu:app_process", + (unsigned long)text.length], + @"provider_result": appInput ?: @{}, + @"provider_attempts": @[OPInputAttemptSummaryForAction(appInput, type)], + @"source": @"openphone.agentd" + }; + } + } + if (!result) { + NSDictionary *typed = OPPerformHIDTypeText(text); + NSMutableDictionary *mutableResult = [@{ + @"state": [typed[@"ok"] boolValue] + ? @"action.executed" : @"action.denied.input_failed", + @"task_id": taskId, + @"capability": capability, + @"detail": [NSString stringWithFormat:@"type_text:%lu", + (unsigned long)text.length], + @"provider_result": typed, + @"source": @"openphone.agentd" + } mutableCopy]; + NSMutableArray *providerAttempts = [NSMutableArray array]; + if (appInput) { + [providerAttempts addObject:OPInputAttemptSummaryForAction(appInput, type)]; + } + [providerAttempts addObject:OPInputAttemptSummaryForAction(typed, type)]; + mutableResult[@"provider_attempts"] = providerAttempts; + result = mutableResult; + } + } + } else { + result = @{ + @"state": @"action.denied.unsupported", + @"task_id": taskId, + @"capability": capability, + @"detail": type.length > 0 ? type : @"missing_action_type", + @"source": @"openphone.agentd" + }; + } + if ([capability isEqualToString:@"input.perform"]) { + result = OPFinalizeInputResult(result, type); + } + NSString *decision = [result[@"state"] isEqualToString:@"action.executed"] + ? @"allow_task_scoped" : @"deny"; + OPRecordAudit([result[@"state"] isEqualToString:@"action.executed"] + ? @"action_executed" : @"action_rejected", taskId, capability, decision, + recordedRequest, result[@"detail"]); + OPRecordTrajectory(taskId, @"tool_result", @{ + @"tool": @"execute_action", + @"arguments": recordedRequest ?: @{}, + @"result": result ?: @{} + }); + return result; +} + +static NSDictionary *OPActionForGoal(NSString *goal) { + NSString *trimmedGoal = [goal stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSString *lower = [trimmedGoal.lowercaseString stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if ([lower hasPrefix:@"type_text "]) { + NSString *text = [trimmedGoal substringFromIndex:10]; + return @{ + @"type": @"type_text", + @"text": text, + @"reason": @"deterministic run_task matched type_text" + }; + } + if ([lower hasPrefix:@"type "]) { + NSString *text = [trimmedGoal substringFromIndex:5]; + return @{ + @"type": @"type_text", + @"text": text, + @"reason": @"deterministic run_task matched type" + }; + } + NSMutableArray *parts = [NSMutableArray array]; + NSArray *rawParts = [lower componentsSeparatedByCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + for (NSString *part in rawParts) { + if (part.length > 0) { + [parts addObject:part]; + } + } + if (parts.count >= 3 && [parts[0] isEqualToString:@"tap"]) { + return @{ + @"type": @"tap", + @"x": @([parts[1] doubleValue]), + @"y": @([parts[2] doubleValue]), + @"reason": @"deterministic run_task matched tap coordinates" + }; + } + if (parts.count >= 3 && ([parts[0] isEqualToString:@"long_press"] || + ([parts[0] isEqualToString:@"long"] && parts.count >= 4 && + [parts[1] isEqualToString:@"press"]))) { + NSUInteger xIndex = [parts[0] isEqualToString:@"long_press"] ? 1 : 2; + NSUInteger yIndex = xIndex + 1; + NSUInteger durationIndex = yIndex + 1; + NSMutableDictionary *action = [@{ + @"type": @"long_press", + @"x": @([parts[xIndex] doubleValue]), + @"y": @([parts[yIndex] doubleValue]), + @"reason": @"deterministic run_task matched long press coordinates" + } mutableCopy]; + if (parts.count > durationIndex) { + action[@"duration_ms"] = @([parts[durationIndex] integerValue]); + } + return action; + } + if (parts.count >= 5 && [parts[0] isEqualToString:@"swipe"]) { + NSMutableDictionary *action = [@{ + @"type": @"swipe", + @"start_x": @([parts[1] doubleValue]), + @"start_y": @([parts[2] doubleValue]), + @"end_x": @([parts[3] doubleValue]), + @"end_y": @([parts[4] doubleValue]), + @"reason": @"deterministic run_task matched swipe coordinates" + } mutableCopy]; + if (parts.count >= 6) { + action[@"duration_ms"] = @([parts[5] integerValue]); + } + return action; + } + if ([lower containsString:@"wake"] || [lower containsString:@"unlock"]) { + return @{ + @"type": @"wake_and_home", + @"reason": @"deterministic run_task matched wake/home" + }; + } + if ([lower isEqualToString:@"home"] || [lower containsString:@"home screen"]) { + return @{ + @"type": @"home", + @"reason": @"deterministic run_task matched home" + }; + } + NSString *explicitURL = OPExplicitURLFromText(goal ?: @""); + if (explicitURL.length > 0) { + return @{ + @"type": @"open_url", + @"url": explicitURL, + @"reason": @"deterministic run_task matched URL" + }; + } + if ([lower containsString:@"safari"]) { + return @{ + @"type": @"open_app", + @"target": @{@"package": @"com.apple.mobilesafari"}, + @"reason": @"deterministic run_task matched Safari" + }; + } + return @{ + @"type": @"wait", + @"duration_ms": @1000, + @"reason": @"deterministic run_task fallback" + }; +} + +static NSArray *OPModelToolNames(void) { + return @[ + @"get_screen", + @"tap", + @"tap_element", + @"long_press", + @"swipe", + @"type_text", + @"open_app", + @"open_url", + @"home", + @"wake_and_home", + @"wait", + @"clipboard_read", + @"clipboard_write", + @"contacts_search", + @"calendar_search", + @"calls_search", + @"messages_search", + @"memory_save", + @"memory_search", + @"context_search", + @"finish_task", + @"fail_task" + ]; +} + +static NSString *OPModelToolCapability(NSString *tool) { + if ([tool isEqualToString:@"get_screen"]) { + return @"screen.read.visible"; + } + if ([tool isEqualToString:@"memory_save"]) { + return @"memory.write"; + } + if ([tool isEqualToString:@"clipboard_read"]) { + return @"clipboard.read"; + } + if ([tool isEqualToString:@"clipboard_write"]) { + return @"clipboard.write"; + } + if ([tool isEqualToString:@"contacts_search"]) { + return @"contacts.read"; + } + if ([tool isEqualToString:@"calendar_search"]) { + return @"calendar.read"; + } + if ([tool isEqualToString:@"calls_search"]) { + return @"calls.read"; + } + if ([tool isEqualToString:@"messages_search"]) { + return @"messages.read"; + } + if ([tool isEqualToString:@"memory_search"] || [tool isEqualToString:@"context_search"]) { + return @"memory.read"; + } + if ([tool isEqualToString:@"open_app"]) { + return @"apps.launch"; + } + if ([tool isEqualToString:@"open_url"]) { + return @"network.use"; + } + if ([tool isEqualToString:@"finish_task"] || [tool isEqualToString:@"fail_task"] || + [tool isEqualToString:@"wait"]) { + return @"tasks.observe"; + } + if ([OPModelToolNames() containsObject:tool]) { + return @"input.perform"; + } + return @"unknown"; +} + +static BOOL OPModelToolDrivesUI(NSString *tool) { + static NSSet *tools = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + tools = [NSSet setWithArray:@[ + @"tap", + @"tap_element", + @"long_press", + @"swipe", + @"type_text", + @"open_app", + @"open_url", + @"home", + @"wake_and_home" + ]]; + }); + return [tools containsObject:tool ?: @""]; +} + +static NSString *OPExplicitURLFromText(NSString *text) { + if (![text isKindOfClass:[NSString class]] || text.length == 0) { + return @""; + } + NSArray *parts = [text componentsSeparatedByCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSCharacterSet *trim = [NSCharacterSet characterSetWithCharactersInString:@"\"'“”‘’.,;!?)]}>"]; + for (NSString *part in parts) { + NSString *candidate = [part stringByTrimmingCharactersInSet:trim]; + NSString *lower = candidate.lowercaseString; + if ([lower hasPrefix:@"http://"] || [lower hasPrefix:@"https://"]) { + return candidate; + } + } + return @""; +} + +static BOOL OPModelToolShouldYieldToExplicitURL(NSString *tool) { + static NSSet *tools = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + tools = [NSSet setWithArray:@[ + @"get_screen", + @"tap", + @"tap_element", + @"long_press", + @"swipe", + @"open_app", + @"home", + @"wake_and_home", + @"wait" + ]]; + }); + return [tools containsObject:tool ?: @""]; +} + +static NSDictionary *OPModelDecisionByApplyingGuardrails(NSDictionary *decision, + NSString *goal, NSInteger step, NSDictionary **guardrailOut) { + if (guardrailOut) { + *guardrailOut = nil; + } + if (![decision isKindOfClass:[NSDictionary class]]) { + return decision ?: @{}; + } + NSString *tool = [decision[@"tool"] isKindOfClass:[NSString class]] + ? decision[@"tool"] : @""; + NSString *url = OPExplicitURLFromText(goal ?: @""); + if (step == 1 && url.length > 0 && ![tool isEqualToString:@"open_url"] && + OPModelToolShouldYieldToExplicitURL(tool)) { + NSMutableDictionary *rewritten = [decision mutableCopy]; + rewritten[@"tool"] = @"open_url"; + rewritten[@"arguments"] = @{ + @"url": url, + @"reason": @"explicit URL in user goal" + }; + rewritten[@"expected_visible_change"] = @"The requested URL opens in Safari or the active browser."; + NSString *thought = [decision[@"thought"] isKindOfClass:[NSString class]] + ? decision[@"thought"] : @""; + rewritten[@"thought"] = thought.length > 0 + ? [NSString stringWithFormat:@"%@ Guardrail: explicit URL goals open the URL directly before tapping app icons.", thought] + : @"Guardrail: explicit URL goals open the URL directly before tapping app icons."; + if (guardrailOut) { + *guardrailOut = @{ + @"reason": @"explicit_url_first_action", + @"original_tool": tool ?: @"", + @"rewritten_tool": @"open_url", + @"url": url, + @"step": @(step) + }; + } + return rewritten; + } + return decision; +} + +static NSArray *OPModelCompactStrings(NSArray *values, NSUInteger limit) { + NSMutableArray *result = [NSMutableArray array]; + for (id value in values) { + if (![value isKindOfClass:[NSString class]]) { + continue; + } + NSString *text = [(NSString *)value stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (text.length == 0) { + continue; + } + if (text.length > 160) { + text = [[text substringToIndex:160] stringByAppendingString:@"..."]; + } + [result addObject:text]; + if (result.count >= limit) { + break; + } + } + return result; +} + +static NSInteger OPModelElementPriority(NSDictionary *element) { + NSMutableArray *parts = [NSMutableArray array]; + for (NSString *key in @[@"label", @"value", @"view_id", @"kind", @"tag", @"input_type", + @"scope", @"input_scope", @"risk_hint", @"class"]) { + NSString *part = [element[key] isKindOfClass:[NSString class]] ? element[key] : @""; + if (part.length > 0) { + [parts addObject:part.lowercaseString]; + } + } + NSString *haystack = [parts componentsJoinedByString:@" "]; + if ([haystack containsString:@"search"]) { + return 0; + } + NSString *tag = [element[@"tag"] isKindOfClass:[NSString class]] + ? [element[@"tag"] lowercaseString] : @""; + NSString *kind = [element[@"kind"] isKindOfClass:[NSString class]] + ? [element[@"kind"] lowercaseString] : @""; + NSString *inputType = [element[@"input_type"] isKindOfClass:[NSString class]] + ? element[@"input_type"] : @""; + if ([kind containsString:@"input"] || [tag isEqualToString:@"input"] || + [tag isEqualToString:@"textarea"] || [tag isEqualToString:@"select"] || + inputType.length > 0) { + return 1; + } + if ([haystack containsString:@"web_content_process"]) { + return 2; + } + NSString *label = [element[@"label"] isKindOfClass:[NSString class]] ? element[@"label"] : @""; + if (label.length > 0 && [element[@"enabled"] boolValue]) { + return 3; + } + if (kind.length > 0 && ![kind isEqualToString:@"view"]) { + return 4; + } + return 5; +} + +static NSArray *OPModelCompactElements(NSArray *elements, NSUInteger limit) { + NSMutableArray *ranked = [NSMutableArray array]; + NSUInteger index = 0; + for (id value in elements) { + if (![value isKindOfClass:[NSDictionary class]]) { + index += 1; + continue; + } + NSDictionary *element = value; + [ranked addObject:@{ + @"priority": @(OPModelElementPriority(element)), + @"index": @(index), + @"element": element + }]; + index += 1; + } + [ranked sortUsingComparator:^NSComparisonResult(NSDictionary *a, NSDictionary *b) { + NSInteger priorityA = [a[@"priority"] integerValue]; + NSInteger priorityB = [b[@"priority"] integerValue]; + if (priorityA < priorityB) { + return NSOrderedAscending; + } + if (priorityA > priorityB) { + return NSOrderedDescending; + } + NSUInteger indexA = [a[@"index"] unsignedIntegerValue]; + NSUInteger indexB = [b[@"index"] unsignedIntegerValue]; + if (indexA < indexB) { + return NSOrderedAscending; + } + if (indexA > indexB) { + return NSOrderedDescending; + } + return NSOrderedSame; + }]; + + NSMutableArray *result = [NSMutableArray array]; + NSMutableSet *seen = [NSMutableSet set]; + for (NSDictionary *rankedElement in ranked) { + NSDictionary *element = [rankedElement[@"element"] isKindOfClass:[NSDictionary class]] + ? rankedElement[@"element"] : @{}; + NSMutableDictionary *compact = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"id", @"view_id", @"kind", @"label", @"value", @"enabled", + @"bounds", @"risk_hint", @"scope", @"input_scope", @"tag", @"input_type", + @"source_bundle_id", @"focused"]) { + if (element[key]) { + compact[key] = element[key]; + } + } + NSString *identifier = [compact[@"id"] isKindOfClass:[NSString class]] + ? compact[@"id"] : @""; + if (identifier.length > 0 && [seen containsObject:identifier]) { + continue; + } + if (identifier.length > 0) { + [seen addObject:identifier]; + } + if (compact.count > 0) { + [result addObject:compact]; + } + if (result.count >= limit) { + break; + } + } + return result; +} + +static NSDictionary *OPModelScreenSummary(NSDictionary *screen) { + NSDictionary *context = [screen[@"context"] isKindOfClass:[NSDictionary class]] + ? screen[@"context"] : @{}; + NSArray *visibleText = [context[@"visible_text"] isKindOfClass:[NSArray class]] + ? context[@"visible_text"] : @[]; + NSArray *interactiveElements = [context[@"interactive_elements"] isKindOfClass:[NSArray class]] + ? context[@"interactive_elements"] : @[]; + NSDictionary *lock = [context[@"lock"] isKindOfClass:[NSDictionary class]] + ? context[@"lock"] : @{}; + NSDictionary *screenshot = [context[@"screenshot"] isKindOfClass:[NSDictionary class]] + ? context[@"screenshot"] : @{}; + return @{ + @"state": screen[@"state"] ?: @"", + @"foreground_app": context[@"foreground_app"] ?: @"unknown", + @"foreground_source": context[@"foreground_source"] ?: @"unknown", + @"locked": lock[@"locked"] ?: @NO, + @"risk_flags": context[@"risk_flags"] ?: @[], + @"visible_text": OPModelCompactStrings(visibleText, 20), + @"interactive_elements": OPModelCompactElements(interactiveElements, 30), + @"screenshot": @{ + @"status": screenshot[@"status"] ?: @"unknown", + @"path": screenshot[@"path"] ?: @"", + @"sha256": screenshot[@"sha256"] ?: @"", + @"width": screenshot[@"width"] ?: @0, + @"height": screenshot[@"height"] ?: @0 + } + }; +} + +// Deterministic signature of a screen so we can detect "did the UI actually +// change" between two observations. Concatenates foreground bundle id + a +// stable digest of visible-text and interactive-element ids. +static NSString *OPModelScreenSignature(NSDictionary *screen) { + if (![screen isKindOfClass:[NSDictionary class]]) return @""; + NSDictionary *context = [screen[@"context"] isKindOfClass:[NSDictionary class]] + ? screen[@"context"] : @{}; + NSString *fg = [context[@"foreground_app"] isKindOfClass:[NSString class]] + ? context[@"foreground_app"] : @""; + NSArray *vt = [context[@"visible_text"] isKindOfClass:[NSArray class]] + ? context[@"visible_text"] : @[]; + NSArray *ie = [context[@"interactive_elements"] isKindOfClass:[NSArray class]] + ? context[@"interactive_elements"] : @[]; + NSMutableArray *textParts = [NSMutableArray array]; + NSUInteger textCount = MIN(vt.count, (NSUInteger)15); + for (NSUInteger i = 0; i < textCount; i++) { + id v = vt[i]; + if ([v isKindOfClass:[NSString class]]) [textParts addObject:v]; + else if ([v isKindOfClass:[NSDictionary class]] && + [v[@"text"] isKindOfClass:[NSString class]]) { + [textParts addObject:v[@"text"]]; + } + } + NSMutableArray *elemParts = [NSMutableArray array]; + NSUInteger elemCount = MIN(ie.count, (NSUInteger)15); + for (NSUInteger i = 0; i < elemCount; i++) { + id v = ie[i]; + if ([v isKindOfClass:[NSDictionary class]] && + [v[@"id"] isKindOfClass:[NSString class]]) { + [elemParts addObject:v[@"id"]]; + } + } + return [NSString stringWithFormat:@"%@|%@|%@", + fg, + [textParts componentsJoinedByString:@"§"], + [elemParts componentsJoinedByString:@"§"]]; +} + +static NSDictionary *OPModelToolResultSummary(NSDictionary *toolResult) { + if (![toolResult isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSMutableDictionary *summary = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"status", @"state", @"reason", @"summary", @"source", + @"detail", @"user_facing_status", @"tool", @"provider", @"system_clipboard", + @"text_length", @"text_sha256", @"truncated", @"max_chars"]) { + id value = toolResult[key]; + if (value) { + summary[key] = value; + } + } + if ([toolResult[@"tool"] isEqualToString:@"clipboard_read"] && + [toolResult[@"text"] isKindOfClass:[NSString class]]) { + summary[@"text"] = toolResult[@"text"]; + } + if ([toolResult[@"tool"] isEqualToString:@"contacts_search"] && + [toolResult[@"contacts"] isKindOfClass:[NSArray class]]) { + summary[@"contacts"] = OPContactsSummaryContacts(toolResult[@"contacts"], 8); + summary[@"count"] = toolResult[@"count"] ?: @0; + summary[@"query_length"] = toolResult[@"query_length"] ?: @0; + summary[@"query_sha256"] = toolResult[@"query_sha256"] ?: @""; + } + if ([toolResult[@"tool"] isEqualToString:@"calendar_search"] && + [toolResult[@"events"] isKindOfClass:[NSArray class]]) { + summary[@"events"] = OPCalendarSummaryEvents(toolResult[@"events"], 8); + summary[@"count"] = toolResult[@"count"] ?: @0; + summary[@"query_length"] = toolResult[@"query_length"] ?: @0; + summary[@"query_sha256"] = toolResult[@"query_sha256"] ?: @""; + summary[@"start_at_ms"] = toolResult[@"start_at_ms"] ?: @0; + summary[@"end_at_ms"] = toolResult[@"end_at_ms"] ?: @0; + } + if ([toolResult[@"tool"] isEqualToString:@"calls_search"] && + [toolResult[@"calls"] isKindOfClass:[NSArray class]]) { + summary[@"calls"] = OPCallsSummaryCalls(toolResult[@"calls"], 8); + summary[@"count"] = toolResult[@"count"] ?: @0; + summary[@"query_length"] = toolResult[@"query_length"] ?: @0; + summary[@"query_sha256"] = toolResult[@"query_sha256"] ?: @""; + summary[@"start_at_ms"] = toolResult[@"start_at_ms"] ?: @0; + summary[@"end_at_ms"] = toolResult[@"end_at_ms"] ?: @0; + } + if ([toolResult[@"tool"] isEqualToString:@"messages_search"] && + [toolResult[@"messages"] isKindOfClass:[NSArray class]]) { + summary[@"messages"] = OPMessagesSummaryMessages(toolResult[@"messages"], 8, YES); + summary[@"count"] = toolResult[@"count"] ?: @0; + summary[@"query_length"] = toolResult[@"query_length"] ?: @0; + summary[@"query_sha256"] = toolResult[@"query_sha256"] ?: @""; + summary[@"start_at_ms"] = toolResult[@"start_at_ms"] ?: @0; + summary[@"end_at_ms"] = toolResult[@"end_at_ms"] ?: @0; + } + if ([toolResult[@"verification"] isKindOfClass:[NSDictionary class]]) { + summary[@"verification"] = toolResult[@"verification"]; + } + NSArray *providerAttempts = [toolResult[@"provider_attempts"] isKindOfClass:[NSArray class]] + ? toolResult[@"provider_attempts"] : @[]; + if (providerAttempts.count > 0) { + NSMutableArray *attempts = [NSMutableArray array]; + for (id object in providerAttempts) { + if (![object isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *attempt = (NSDictionary *)object; + NSMutableDictionary *compact = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"provider", @"scope", @"action_type", @"status", @"reason", + @"activation_method", @"target_class", @"text_length", + @"before_text_length", @"after_text_length"]) { + id value = attempt[key]; + if (value) { + compact[key] = value; + } + } + if ([attempt[@"verification"] isKindOfClass:[NSDictionary class]]) { + compact[@"verification"] = attempt[@"verification"]; + } + if (compact.count > 0) { + [attempts addObject:compact]; + } + } + if (attempts.count > 0) { + summary[@"provider_attempts"] = attempts; + } + } + return summary; +} + +static NSDictionary *OPModelProviderVerification(NSDictionary *toolResult) { + NSDictionary *verification = [toolResult[@"verification"] isKindOfClass:[NSDictionary class]] + ? toolResult[@"verification"] : @{}; + if ([verification[@"status"] isEqualToString:@"verified"]) { + return verification; + } + NSArray *providerAttempts = [toolResult[@"provider_attempts"] isKindOfClass:[NSArray class]] + ? toolResult[@"provider_attempts"] : @[]; + for (id object in providerAttempts) { + if (![object isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *attempt = (NSDictionary *)object; + NSDictionary *attemptVerification = [attempt[@"verification"] isKindOfClass:[NSDictionary class]] + ? attempt[@"verification"] : @{}; + if ([attemptVerification[@"status"] isEqualToString:@"verified"]) { + return attemptVerification; + } + } + return @{}; +} + +static BOOL OPModelToolResultIsProviderVerified(NSDictionary *toolResult) { + if (![toolResult isKindOfClass:[NSDictionary class]]) { + return NO; + } + NSString *userFacingStatus = [toolResult[@"user_facing_status"] isKindOfClass:[NSString class]] + ? toolResult[@"user_facing_status"] : @""; + if ([userFacingStatus isEqualToString:@"verified"] || [userFacingStatus isEqualToString:@"success"]) { + return YES; + } + return OPModelProviderVerification(toolResult).count > 0; +} + +static BOOL OPModelShouldUseProviderVerificationOnly(NSString *tool, NSDictionary *toolResult) { + if (![tool isEqualToString:@"type_text"]) { + return NO; + } + return OPModelToolResultIsProviderVerified(toolResult); +} + +static NSDictionary *OPModelProviderVerificationState(NSString *tool, NSDictionary *toolResult, + NSDictionary *beforeScreen, NSString *expectedVisibleChange) { + NSDictionary *providerVerification = OPModelProviderVerification(toolResult ?: @{}); + NSString *source = [providerVerification[@"source"] isKindOfClass:[NSString class]] + ? providerVerification[@"source"] : @"provider_verification"; + NSString *reason = [providerVerification[@"reason"] isKindOfClass:[NSString class]] + ? providerVerification[@"reason"] : @"provider_verified_visible_effect"; + NSMutableDictionary *result = [@{ + @"status": @"verified", + @"reason": reason, + @"source": source, + @"tool": tool ?: @"", + @"expected_visible_change": expectedVisibleChange ?: @"", + @"screen_state": beforeScreen[@"state"] ?: @"", + @"post_action_screen_capture": @"skipped_provider_verified_type_text" + } mutableCopy]; + if (providerVerification.count > 0) { + result[@"provider_verification"] = providerVerification; + } + return result; +} + +static BOOL OPStringContainsCaseInsensitive(NSString *haystack, NSString *needle) { + if (haystack.length == 0 || needle.length == 0) { + return NO; + } + return [haystack rangeOfString:needle options:NSCaseInsensitiveSearch].location != NSNotFound; +} + +static BOOL OPModelVerifiedTypeTextCompletesGoal(NSString *goal, NSDictionary *decision, + NSDictionary *verification) { + if (![decision[@"tool"] isEqualToString:@"type_text"] || + ![verification[@"status"] isEqualToString:@"verified"]) { + return NO; + } + NSDictionary *arguments = [decision[@"arguments"] isKindOfClass:[NSDictionary class]] + ? decision[@"arguments"] : @{}; + NSString *text = [arguments[@"text"] isKindOfClass:[NSString class]] + ? arguments[@"text"] : @""; + if (text.length == 0 || !OPStringContainsCaseInsensitive(goal ?: @"", text)) { + return NO; + } + NSArray *terminalHints = @[ + @"then finish", + @"finish only after", + @"type the exact text", + @"enter the exact text" + ]; + BOOL hasTerminalHint = NO; + for (NSString *hint in terminalHints) { + if (OPStringContainsCaseInsensitive(goal ?: @"", hint)) { + hasTerminalHint = YES; + break; + } + } + if (!hasTerminalHint) { + return NO; + } + NSString *lowerGoal = [(goal ?: @"") lowercaseString]; + NSArray *negatedSubmitHints = @[ + @"do not submit", + @"don't submit", + @"dont submit", + @"without submitting", + @"without submit", + @"not submit", + @"never submit" + ]; + BOOL submitIsNegated = NO; + for (NSString *hint in negatedSubmitHints) { + if ([lowerGoal containsString:hint]) { + submitIsNegated = YES; + break; + } + } + for (NSString *continuationHint in @[@"then send", @"then submit", @"then tap", + @"tap send", @"press send", @"submit", @"post"]) { + if (OPStringContainsCaseInsensitive(goal ?: @"", continuationHint)) { + if (submitIsNegated && + ([continuationHint isEqualToString:@"submit"] || + [continuationHint isEqualToString:@"then submit"])) { + continue; + } + return NO; + } + } + return YES; +} + +static NSDictionary *OPModelScreenTraceSummary(NSDictionary *screen) { + NSDictionary *summary = OPModelScreenSummary(screen ?: @{}); + NSArray *elements = [summary[@"interactive_elements"] isKindOfClass:[NSArray class]] + ? summary[@"interactive_elements"] : @[]; + return @{ + @"state": summary[@"state"] ?: @"", + @"foreground_app": summary[@"foreground_app"] ?: @"unknown", + @"foreground_source": summary[@"foreground_source"] ?: @"unknown", + @"locked": summary[@"locked"] ?: @NO, + @"risk_flags": summary[@"risk_flags"] ?: @[], + @"visible_text": summary[@"visible_text"] ?: @[], + @"interactive_element_count": @(elements.count), + @"screenshot": summary[@"screenshot"] ?: @{} + }; +} + +static NSDictionary *OPModelCompactScreenForLoop(NSDictionary *screen) { + if (![screen isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSDictionary *context = [screen[@"context"] isKindOfClass:[NSDictionary class]] + ? screen[@"context"] : @{}; + NSDictionary *summary = OPModelScreenSummary(screen); + NSDictionary *uiTree = [context[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? context[@"ui_tree"] : @{}; + NSDictionary *appUIState = [context[@"app_ui_state"] isKindOfClass:[NSDictionary class]] + ? context[@"app_ui_state"] : @{}; + NSDictionary *springBoardState = [context[@"springboard_state"] isKindOfClass:[NSDictionary class]] + ? context[@"springboard_state"] : @{}; + NSDictionary *display = [context[@"display"] isKindOfClass:[NSDictionary class]] + ? context[@"display"] : @{}; + NSArray *recentApps = [context[@"recent_apps"] isKindOfClass:[NSArray class]] + ? context[@"recent_apps"] : @[]; + NSMutableArray *compactRecentApps = [NSMutableArray array]; + for (id value in recentApps) { + if (![value isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *app = value; + NSMutableDictionary *compact = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"bundle_id", @"display_name", @"rank", @"source"]) { + if (app[key]) { + compact[key] = app[key]; + } + } + if (compact.count > 0) { + [compactRecentApps addObject:compact]; + } + if (compactRecentApps.count >= 8) { + break; + } + } + NSMutableDictionary *compactContext = [NSMutableDictionary dictionary]; + compactContext[@"foreground_app"] = summary[@"foreground_app"] ?: @"unknown"; + compactContext[@"foreground_package"] = summary[@"foreground_app"] ?: @"unknown"; + compactContext[@"foreground_source"] = summary[@"foreground_source"] ?: @"unknown"; + compactContext[@"lock"] = @{ + @"locked": summary[@"locked"] ?: @NO, + @"status": [context[@"lock"] isKindOfClass:[NSDictionary class]] + ? (context[@"lock"][@"status"] ?: @"unknown") : @"unknown" + }; + compactContext[@"risk_flags"] = summary[@"risk_flags"] ?: @[]; + compactContext[@"visible_text"] = summary[@"visible_text"] ?: @[]; + compactContext[@"interactive_elements"] = summary[@"interactive_elements"] ?: @[]; + compactContext[@"screenshot"] = summary[@"screenshot"] ?: @{}; + compactContext[@"ui_tree_source"] = context[@"ui_tree_source"] ?: @"unknown"; + compactContext[@"ui_tree"] = @{ + @"status": uiTree[@"status"] ?: @"unknown", + @"provider": uiTree[@"provider"] ?: @"unknown", + @"scope": uiTree[@"scope"] ?: @"", + @"text_count": uiTree[@"text_count"] ?: @0, + @"element_count": uiTree[@"element_count"] ?: @0, + @"visible_text": summary[@"visible_text"] ?: @[], + @"interactive_elements": summary[@"interactive_elements"] ?: @[] + }; + compactContext[@"app_ui_state"] = @{ + @"status": appUIState[@"status"] ?: @"unknown", + @"effective_bundle_id": appUIState[@"effective_bundle_id"] ?: @"", + @"reason": appUIState[@"reason"] ?: @"", + @"age_ms": appUIState[@"age_ms"] ?: @0, + @"received_transport": appUIState[@"received_transport"] ?: @"" + }; + compactContext[@"springboard_state"] = @{ + @"status": springBoardState[@"status"] ?: @"unknown", + @"foreground_app": springBoardState[@"foreground_app"] ?: @"", + @"foreground_source": springBoardState[@"foreground_source"] ?: @"", + @"age_ms": springBoardState[@"age_ms"] ?: @0, + @"scene_count": springBoardState[@"scene_count"] ?: @0 + }; + compactContext[@"display"] = @{ + @"status": display[@"status"] ?: @"unknown", + @"point_width": display[@"point_width"] ?: @0, + @"point_height": display[@"point_height"] ?: @0, + @"pixel_width": display[@"pixel_width"] ?: @0, + @"pixel_height": display[@"pixel_height"] ?: @0, + @"scale": display[@"scale"] ?: @0 + }; + compactContext[@"recent_apps"] = compactRecentApps; + compactContext[@"recent_apps_source"] = context[@"recent_apps_source"] ?: @""; + return @{ + @"status": screen[@"status"] ?: @"ok", + @"state": screen[@"state"] ?: @"", + @"task_id": screen[@"task_id"] ?: @"", + @"timestamp_ms": screen[@"timestamp_ms"] ?: @(OPNowMs()), + @"capture_mode": screen[@"capture_mode"] ?: @"metadata_only", + @"source": screen[@"source"] ?: @"openphone.agentd.screen.compact", + @"screenshot": summary[@"screenshot"] ?: @{}, + @"context": compactContext, + @"request": screen[@"request"] ?: @{} + }; +} + +static NSDictionary *OPModelVerificationTraceSummary(NSDictionary *verification) { + if (![verification isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSMutableDictionary *summary = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"status", @"reason", @"source", @"tool", + @"expected_visible_change", @"screen_state", @"post_action_screen_capture"]) { + id value = verification[key]; + if (value) { + summary[key] = value; + } + } + if ([verification[@"provider_verification"] isKindOfClass:[NSDictionary class]]) { + summary[@"provider_verification"] = verification[@"provider_verification"]; + } + NSDictionary *delta = [verification[@"screen_delta"] isKindOfClass:[NSDictionary class]] + ? verification[@"screen_delta"] : nil; + if (delta) { + NSMutableDictionary *compactDelta = [NSMutableDictionary dictionary]; + for (NSString *key in @[@"status", @"signals", @"signal_count", @"strong_signal", + @"fields"]) { + id value = delta[key]; + if (value) { + compactDelta[key] = value; + } + } + summary[@"screen_delta"] = compactDelta; + } + return summary; +} + +static NSDictionary *OPModelScreenDelta(NSDictionary *beforeScreen, NSDictionary *afterScreen) { + NSDictionary *before = OPModelScreenSummary(beforeScreen ?: @{}); + NSDictionary *after = OPModelScreenSummary(afterScreen ?: @{}); + NSMutableArray *signals = [NSMutableArray array]; + NSMutableDictionary *fields = [NSMutableDictionary dictionary]; + + NSString *beforeState = [before[@"state"] isKindOfClass:[NSString class]] + ? before[@"state"] : @""; + NSString *afterState = [after[@"state"] isKindOfClass:[NSString class]] + ? after[@"state"] : @""; + if (beforeState.length > 0 && afterState.length > 0 && + ![beforeState isEqualToString:afterState]) { + [signals addObject:@"screen_state"]; + fields[@"screen_state"] = @{@"before": beforeState, @"after": afterState}; + } + + NSString *beforeForeground = [before[@"foreground_app"] isKindOfClass:[NSString class]] + ? before[@"foreground_app"] : @""; + NSString *afterForeground = [after[@"foreground_app"] isKindOfClass:[NSString class]] + ? after[@"foreground_app"] : @""; + if (beforeForeground.length > 0 && afterForeground.length > 0 && + ![beforeForeground isEqualToString:afterForeground]) { + [signals addObject:@"foreground_app"]; + fields[@"foreground_app"] = @{@"before": beforeForeground, @"after": afterForeground}; + } + + BOOL beforeLocked = [before[@"locked"] respondsToSelector:@selector(boolValue)] && + [before[@"locked"] boolValue]; + BOOL afterLocked = [after[@"locked"] respondsToSelector:@selector(boolValue)] && + [after[@"locked"] boolValue]; + if (beforeLocked != afterLocked) { + [signals addObject:@"lock_state"]; + fields[@"lock_state"] = @{@"before": @(beforeLocked), @"after": @(afterLocked)}; + } + + NSArray *beforeText = [before[@"visible_text"] isKindOfClass:[NSArray class]] + ? before[@"visible_text"] : @[]; + NSArray *afterText = [after[@"visible_text"] isKindOfClass:[NSArray class]] + ? after[@"visible_text"] : @[]; + if (![beforeText isEqualToArray:afterText]) { + [signals addObject:@"visible_text"]; + fields[@"visible_text"] = @{ + @"before_count": @(beforeText.count), + @"after_count": @(afterText.count), + @"before": beforeText, + @"after": afterText + }; + } + + NSArray *beforeElements = [before[@"interactive_elements"] isKindOfClass:[NSArray class]] + ? before[@"interactive_elements"] : @[]; + NSArray *afterElements = [after[@"interactive_elements"] isKindOfClass:[NSArray class]] + ? after[@"interactive_elements"] : @[]; + if (![beforeElements isEqualToArray:afterElements]) { + [signals addObject:@"ui_tree"]; + fields[@"ui_tree"] = @{ + @"before_element_count": @(beforeElements.count), + @"after_element_count": @(afterElements.count) + }; + } + + NSDictionary *beforeScreenshot = [before[@"screenshot"] isKindOfClass:[NSDictionary class]] + ? before[@"screenshot"] : @{}; + NSDictionary *afterScreenshot = [after[@"screenshot"] isKindOfClass:[NSDictionary class]] + ? after[@"screenshot"] : @{}; + NSString *beforeHash = [beforeScreenshot[@"sha256"] isKindOfClass:[NSString class]] + ? beforeScreenshot[@"sha256"] : @""; + NSString *afterHash = [afterScreenshot[@"sha256"] isKindOfClass:[NSString class]] + ? afterScreenshot[@"sha256"] : @""; + if (beforeHash.length > 0 && afterHash.length > 0 && ![beforeHash isEqualToString:afterHash]) { + [signals addObject:@"screenshot_hash"]; + fields[@"screenshot_hash"] = @{ + @"before": beforeHash, + @"after": afterHash, + @"before_status": beforeScreenshot[@"status"] ?: @"unknown", + @"after_status": afterScreenshot[@"status"] ?: @"unknown" + }; + } + + BOOL strong = [signals containsObject:@"foreground_app"] || + [signals containsObject:@"lock_state"] || + [signals containsObject:@"visible_text"] || + [signals containsObject:@"ui_tree"] || + [signals containsObject:@"screen_state"]; + return @{ + @"status": signals.count > 0 ? @"changed" : @"unchanged", + @"signals": signals, + @"signal_count": @(signals.count), + @"strong_signal": @(strong), + @"fields": fields, + @"before": before, + @"after": after, + @"source": @"openphone.agentd.visible_effect" + }; +} + +static NSDictionary *OPModelPromptContext(NSString *goal, NSString *taskId, + NSDictionary *screen, NSDictionary *modelStatus, NSArray *approvedCapabilities, + NSDictionary *loopState) { + NSDictionary *memory = OPMemorySearch(@{ + @"task_id": taskId ?: @"", + @"query": goal ?: @"", + @"limit": @5, + @"suppress_trajectory": @YES, + @"reason": @"model prompt memory context" + }); + NSDictionary *context = OPContextSearch(@{ + @"task_id": taskId ?: @"", + @"query": goal ?: @"", + @"limit": @5, + @"suppress_trajectory": @YES, + @"reason": @"model prompt continuity context" + }); + return @{ + @"schema": @"openphone.model_prompt_context.v1", + @"goal": goal ?: @"", + @"autonomy_mode": @"yolo", + @"approved_capabilities": approvedCapabilities ?: @[], + @"model": @{ + @"status": modelStatus[@"status"] ?: @"unknown", + @"mode": modelStatus[@"mode"] ?: @"broker", + @"model": modelStatus[@"model"] ?: @"" + }, + @"tools": OPModelToolNames(), + @"loop": loopState ?: @{}, + @"screen": OPModelScreenSummary(screen ?: @{}), + @"memory": @{ + @"count": memory[@"count"] ?: @0, + @"memories": memory[@"memories"] ?: @[] + }, + @"context": @{ + @"count": context[@"count"] ?: @0, + @"events": context[@"events"] ?: @[] + } + }; +} + +static NSDictionary *OPModelPromptContextTraceSummary(NSDictionary *context) { + if (![context isKindOfClass:[NSDictionary class]]) { + return @{}; + } + NSDictionary *memory = [context[@"memory"] isKindOfClass:[NSDictionary class]] + ? context[@"memory"] : @{}; + NSDictionary *continuity = [context[@"context"] isKindOfClass:[NSDictionary class]] + ? context[@"context"] : @{}; + NSArray *tools = [context[@"tools"] isKindOfClass:[NSArray class]] + ? context[@"tools"] : @[]; + NSArray *approved = [context[@"approved_capabilities"] isKindOfClass:[NSArray class]] + ? context[@"approved_capabilities"] : @[]; + NSString *goal = [context[@"goal"] isKindOfClass:[NSString class]] ? context[@"goal"] : @""; + return @{ + @"schema": context[@"schema"] ?: @"openphone.model_prompt_context.v1", + @"goal_length": @(goal.length), + @"autonomy_mode": context[@"autonomy_mode"] ?: @"", + @"tool_count": @(tools.count), + @"approved_capability_count": @(approved.count), + @"loop": context[@"loop"] ?: @{}, + @"screen": context[@"screen"] ?: @{}, + @"memory": @{ + @"count": memory[@"count"] ?: @0 + }, + @"context": @{ + @"count": continuity[@"count"] ?: @0 + }, + @"model": context[@"model"] ?: @{} + }; +} + +static NSString *OPTruncatedString(NSString *value, NSUInteger limit) { + if (value.length <= limit) { + return value ?: @""; + } + return [[value substringToIndex:limit] stringByAppendingFormat:@"...", + (unsigned long)value.length]; +} + +static NSString *OPModelJSONStringForPrompt(id object, NSUInteger limit) { + NSString *json = OPJSONString(OPRedactedObject(object ?: @{}, 0)); + return OPTruncatedString(json, limit); +} + +static NSString *OPModelPromptText(NSDictionary *requestBody) { + NSString *goal = [requestBody[@"goal"] isKindOfClass:[NSString class]] + ? requestBody[@"goal"] : @""; + NSArray *tools = [requestBody[@"tools"] isKindOfClass:[NSArray class]] + ? requestBody[@"tools"] : OPModelToolNames(); + NSDictionary *context = [requestBody[@"context"] isKindOfClass:[NSDictionary class]] + ? requestBody[@"context"] : @{}; + NSString *toolList = [tools componentsJoinedByString:@" | "]; + return [NSString stringWithFormat: + @"You are OpenPhone — a voice assistant inside the user's iPhone.\n" + @"Speak like a friend. Short. Human. Never recap raw context fields.\n\n" + @"You route every request into ONE mode:\n\n" + @" answer — user asked a question you can answer from what's already visible or from common knowledge. Fill `reply` with a short human answer. Done in one turn.\n" + @" act — user wants something done on the phone. Fill `proposed_actions[]` with the exact tool calls needed, IN ORDER. If ONE tool call finishes the whole goal (open X, dial Y, open URL Z) put JUST that one call. Only include multiple actions if they are actually required for that goal (e.g. open Messages → tap Alex → type → send).\n" + @" stop — user asked to cancel or stop the assistant.\n\n" + @"HARD RULES:\n" + @"- The current phone context (foreground app, visible text, tappable elements) is ALREADY in this prompt. NEVER pick mode=inspect just to look again.\n" + @"- For questions like 'what app am I in', 'can you see my screen', 'what's on my calendar today' → mode=answer with a direct sentence in `reply`.\n" + @"- For 'open X', 'search Y on Z', 'call mom' → mode=act with the minimal proposed_actions[] to finish the goal.\n" + @"- Explicit URLs (goal contains http/https): the FIRST proposed_action MUST be open_url with that URL. Never tap Safari icons first.\n" + @"- Autonomy is full YOLO. Never ask permission. Never say 'I would' or 'shall I'. Just do it.\n\n" + @"HOW TO WRITE `reply`:\n" + @"- Under 20 words. Talk TO the user. Never mention 'foreground_app', 'SpringBoard', 'element id', 'UI tree', 'metadata'.\n" + @"- For act mode: describe what you're doing right now, present tense. Example: \"Opening Wikipedia for Adam.\"\n" + @"- For answer mode: the direct answer. Example: \"Settings.\"\n\n" + @"EXAMPLES:\n" + @" User: \"Can you see my screen?\" →\n" + @" { \"mode\":\"answer\", \"reply\":\"Yes — you're on the Home screen.\" }\n" + @" User: \"What app am I in?\" →\n" + @" { \"mode\":\"answer\", \"reply\":\"Settings.\" }\n" + @" User: \"Open Wikipedia\" →\n" + @" { \"mode\":\"act\", \"reply\":\"Opening Wikipedia.\",\n" + @" \"proposed_actions\":[{\"tool\":\"open_url\",\"arguments\":{\"url\":\"https://en.wikipedia.org/\"}}] }\n" + @" User: \"Search Adam on Wikipedia\" →\n" + @" { \"mode\":\"act\", \"reply\":\"Opening the Wikipedia page for Adam.\",\n" + @" \"proposed_actions\":[{\"tool\":\"open_url\",\"arguments\":{\"url\":\"https://en.wikipedia.org/wiki/Adam\"}}] }\n" + @" User: \"Text Alex on my way\" →\n" + @" { \"mode\":\"act\", \"reply\":\"Texting Alex 'on my way'.\", \"task_goal\":\"send SMS 'on my way' to contact Alex\",\n" + @" \"proposed_actions\":[{\"tool\":\"open_app\",\"arguments\":{\"bundle_id\":\"com.apple.MobileSMS\"}}] }\n\n" + @"TOOL SCHEMAS (for proposed_actions):\n" + @"- open_url: {\"url\":\"https://...\"}\n" + @"- open_app: {\"bundle_id\":\"com.example.App\"}\n" + @"- tap_element: {\"element_id\":\"exact visible id from the screen context\"}\n" + @"- tap: {\"x\":num,\"y\":num}\n" + @"- type_text: {\"text\":\"...\",\"element_id\":\"field id\"}\n" + @"- swipe: {\"start_x\",\"start_y\",\"end_x\",\"end_y\"}\n" + @"- home: {}\n\n" + @"Return ONE JSON object matching this schema — no markdown, no prose outside JSON:\n" + @"{\"schema\":\"openphone.model_decision.v3\",\"mode\":\"answer|act|stop\",\"reply\":\"user-facing text\",\"task_goal\":\"only if multi-step\",\"proposed_actions\":[{\"tool\":\"...\",\"arguments\":{...}}],\"reason\":\"private, brief\"}\n\n" + @"Allowed tools inside proposed_actions: %@\n" + @"User said: %@\n\n" + @"Current phone context:\n%@", + toolList, + goal, + OPModelJSONStringForPrompt(context, 6000)]; +} + +static NSString *OPModelBedrockEndpoint(NSDictionary *config, NSString *model) { + NSString *endpoint = [config[@"endpoint_url"] isKindOfClass:[NSString class]] + ? config[@"endpoint_url"] : @""; + if (endpoint.length > 0 && [endpoint rangeOfString:@"/model/"].location != NSNotFound) { + return endpoint; + } + NSString *region = [config[@"region"] isKindOfClass:[NSString class]] + ? config[@"region"] : @"us-east-1"; + NSString *base = endpoint.length > 0 + ? [endpoint stringByTrimmingCharactersInSet: + [NSCharacterSet characterSetWithCharactersInString:@"/"]] + : [NSString stringWithFormat:@"https://bedrock-runtime.%@.amazonaws.com", + region.length > 0 ? region : @"us-east-1"]; + NSMutableCharacterSet *allowed = [[NSCharacterSet URLPathAllowedCharacterSet] mutableCopy]; + [allowed removeCharactersInString:@":/?#[]@!$&'()*+,;="]; + NSString *encodedModel = [model stringByAddingPercentEncodingWithAllowedCharacters:allowed] ?: model ?: @""; + return [NSString stringWithFormat:@"%@/model/%@/converse", base, encodedModel]; +} + +static NSDictionary *OPModelBedrockConverseDecision(NSDictionary *modelStatus, + NSDictionary *requestBody) { + NSDictionary *config = OPModelConfig(); + NSString *model = [config[@"model"] isKindOfClass:[NSString class]] + ? config[@"model"] : @""; + if (model.length == 0) { + return OPError(@"model_not_configured"); + } + NSString *credential = OPModelCredentialValue(); + if (credential.length == 0) { + return OPError(@"model_credential_missing"); + } + NSString *endpoint = OPModelBedrockEndpoint(config, model); + NSURL *url = [NSURL URLWithString:endpoint ?: @""]; + if (!url || !url.scheme || !url.host) { + return OPError(@"bedrock_endpoint_invalid"); + } + long long timeoutMs = [modelStatus[@"timeout_ms"] respondsToSelector:@selector(longLongValue)] + ? [modelStatus[@"timeout_ms"] longLongValue] : 30000; + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url + cachePolicy:NSURLRequestReloadIgnoringLocalCacheData + timeoutInterval:MAX(1.0, (NSTimeInterval)timeoutMs / 1000.0)]; + request.HTTPMethod = @"POST"; + [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"]; + [request setValue:@"application/json" forHTTPHeaderField:@"Accept"]; + [request setValue:[NSString stringWithFormat:@"Bearer %@", credential] + forHTTPHeaderField:@"Authorization"]; + + NSString *instructions = [requestBody[@"instructions"] isKindOfClass:[NSString class]] + ? requestBody[@"instructions"] : @"Return exactly one JSON object."; + NSDictionary *body = @{ + @"system": @[@{@"text": instructions}], + @"messages": @[@{ + @"role": @"user", + @"content": @[@{@"text": OPModelPromptText(requestBody)}] + }], + @"inferenceConfig": @{ + @"maxTokens": @800, + @"temperature": @0 + } + }; + request.HTTPBody = OPCanonicalJSONData(body); + + NSURLResponse *response = nil; + NSError *error = nil; + long long startedMs = OPNowMs(); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + NSData *data = [NSURLConnection sendSynchronousRequest:request + returningResponse:&response + error:&error]; +#pragma clang diagnostic pop + long long latencyMs = OPNowMs() - startedMs; + if (error || !data) { + return OPError([NSString stringWithFormat:@"bedrock_request_failed:%@", + error.localizedDescription ?: @"unknown"]); + } + NSInteger statusCode = 0; + if ([response isKindOfClass:[NSHTTPURLResponse class]]) { + statusCode = [(NSHTTPURLResponse *)response statusCode]; + } + if (statusCode < 200 || statusCode >= 300) { + return @{ + @"status": @"error", + @"reason": [NSString stringWithFormat:@"bedrock_http_status:%ld", (long)statusCode], + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"source": @"openphone.agentd" + }; + } + id parsed = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + if (![parsed isKindOfClass:[NSDictionary class]]) { + return @{ + @"status": @"error", + @"reason": @"bedrock_response_not_object", + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"source": @"openphone.agentd" + }; + } + NSDictionary *object = parsed; + NSDictionary *output = [object[@"output"] isKindOfClass:[NSDictionary class]] + ? object[@"output"] : @{}; + NSDictionary *message = [output[@"message"] isKindOfClass:[NSDictionary class]] + ? output[@"message"] : @{}; + NSArray *content = [message[@"content"] isKindOfClass:[NSArray class]] + ? message[@"content"] : @[]; + NSMutableArray *textParts = [NSMutableArray array]; + for (id blockValue in content) { + if (![blockValue isKindOfClass:[NSDictionary class]]) { + continue; + } + NSString *text = [blockValue[@"text"] isKindOfClass:[NSString class]] + ? blockValue[@"text"] : @""; + if (text.length > 0) { + [textParts addObject:text]; + } + } + NSString *decisionText = [[textParts componentsJoinedByString:@"\n"] + stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (decisionText.length == 0) { + return @{ + @"status": @"error", + @"reason": @"bedrock_empty_text", + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"source": @"openphone.agentd" + }; + } + NSMutableDictionary *result = [@{ + @"status": @"ok", + @"provider": @"bedrock_converse", + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"decision": decisionText, + @"source": @"openphone.agentd", + @"metadata": @{ + @"provider": @"bedrock_converse", + @"provider_backed": @YES, + @"model": model, + @"region": config[@"region"] ?: @"us-east-1", + @"latency_ms": @(latencyMs) + } + } mutableCopy]; + if ([object[@"usage"] isKindOfClass:[NSDictionary class]]) { + result[@"usage"] = object[@"usage"]; + } + return result; +} + +static NSDictionary *OPModelBrokerDecision(NSDictionary *modelStatus, NSDictionary *requestBody) { + NSDictionary *config = OPModelConfig(); + NSString *endpoint = [config[@"endpoint_url"] isKindOfClass:[NSString class]] + ? config[@"endpoint_url"] : @""; + NSURL *url = [NSURL URLWithString:endpoint ?: @""]; + if (!url || !url.scheme || !url.host) { + return OPError(@"model_endpoint_invalid"); + } + long long timeoutMs = [modelStatus[@"timeout_ms"] respondsToSelector:@selector(longLongValue)] + ? [modelStatus[@"timeout_ms"] longLongValue] : 30000; + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url + cachePolicy:NSURLRequestReloadIgnoringLocalCacheData + timeoutInterval:MAX(1.0, (NSTimeInterval)timeoutMs / 1000.0)]; + request.HTTPMethod = @"POST"; + [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"]; + [request setValue:@"application/json" forHTTPHeaderField:@"Accept"]; + NSString *credential = OPModelCredentialValue(); + if (credential.length > 0) { + [request setValue:[NSString stringWithFormat:@"Bearer %@", credential] + forHTTPHeaderField:@"Authorization"]; + } + request.HTTPBody = OPCanonicalJSONData(requestBody ?: @{}); + + NSURLResponse *response = nil; + NSError *error = nil; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + NSData *data = [NSURLConnection sendSynchronousRequest:request + returningResponse:&response + error:&error]; +#pragma clang diagnostic pop + if (error || !data) { + return OPError([NSString stringWithFormat:@"model_broker_request_failed:%@", + error.localizedDescription ?: @"unknown"]); + } + NSInteger statusCode = 0; + if ([response isKindOfClass:[NSHTTPURLResponse class]]) { + statusCode = [(NSHTTPURLResponse *)response statusCode]; + } + if (statusCode < 200 || statusCode >= 300) { + return @{ + @"status": @"error", + @"reason": [NSString stringWithFormat:@"model_broker_http_status:%ld", (long)statusCode], + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"source": @"openphone.agentd" + }; + } + id object = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + if (![object isKindOfClass:[NSDictionary class]]) { + return @{ + @"status": @"error", + @"reason": @"model_broker_response_not_object", + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"source": @"openphone.agentd" + }; + } + NSDictionary *envelope = object; + id decision = envelope[@"decision"]; + if (!decision && [envelope[@"decision_json"] isKindOfClass:[NSString class]]) { + decision = envelope[@"decision_json"]; + } + if (!decision && ([envelope[@"schema"] isEqualToString:@"openphone.model_decision.v1"] || + [envelope[@"schema"] isEqualToString:@"openphone.model_decision.v2"] || + [envelope[@"schema"] isEqualToString:@"openphone.model_decision.v3"])) { + decision = envelope; + } + if (!decision) { + return @{ + @"status": @"error", + @"reason": @"model_broker_missing_decision", + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"source": @"openphone.agentd" + }; + } + NSMutableDictionary *result = [@{ + @"status": @"ok", + @"provider": modelStatus[@"mode"] ?: @"broker", + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"decision": decision, + @"source": @"openphone.agentd" + } mutableCopy]; + if ([envelope[@"usage"] isKindOfClass:[NSDictionary class]]) { + result[@"usage"] = envelope[@"usage"]; + } + if ([envelope[@"metadata"] isKindOfClass:[NSDictionary class]]) { + result[@"metadata"] = envelope[@"metadata"]; + } + return result; +} + +static NSDictionary *OPModelProviderDecision(NSDictionary *modelStatus, NSDictionary *requestBody) { + NSString *mode = [modelStatus[@"mode"] isKindOfClass:[NSString class]] + ? modelStatus[@"mode"] : @"broker"; + if ([mode isEqualToString:@"bedrock_converse"]) { + return OPModelBedrockConverseDecision(modelStatus, requestBody); + } + return OPModelBrokerDecision(modelStatus, requestBody); +} + +static NSError *OPRealtimeError(NSString *message) { + return [NSError errorWithDomain:@"OpenPhoneRealtime" + code:1 + userInfo:@{NSLocalizedDescriptionKey: message ?: @"realtime_error"}]; +} + +@interface OPRealtimeWebSocket : NSObject +@property(nonatomic, strong) NSURLSession *session; +@property(nonatomic, strong) NSURLSessionWebSocketTask *task; +@end + +@implementation OPRealtimeWebSocket + ++ (instancetype)connectWithURL:(NSURL *)url bearerToken:(NSString *)bearerToken + timeoutMs:(long long)timeoutMs error:(NSError **)errorOut { + if (!url || bearerToken.length == 0) { + if (errorOut) { + *errorOut = OPRealtimeError(@"realtime_url_or_credential_missing"); + } + return nil; + } + NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration]; + configuration.timeoutIntervalForRequest = MAX(1.0, (NSTimeInterval)timeoutMs / 1000.0); + configuration.timeoutIntervalForResource = MAX(1.0, (NSTimeInterval)timeoutMs / 1000.0); + NSURLSession *session = [NSURLSession sessionWithConfiguration:configuration]; + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url + cachePolicy:NSURLRequestReloadIgnoringLocalCacheData + timeoutInterval:configuration.timeoutIntervalForRequest]; + [request setValue:[NSString stringWithFormat:@"Bearer %@", bearerToken] + forHTTPHeaderField:@"Authorization"]; + [request setValue:@"openphone-ios-agentd" forHTTPHeaderField:@"User-Agent"]; + [request setValue:@"openphone-ios-local-user" forHTTPHeaderField:@"OpenAI-Safety-Identifier"]; + NSURLSessionWebSocketTask *task = [session webSocketTaskWithRequest:request]; + OPRealtimeWebSocket *socket = [OPRealtimeWebSocket new]; + socket.session = session; + socket.task = task; + [task resume]; + return socket; +} + +- (BOOL)sendEvent:(NSDictionary *)event timeoutMs:(long long)timeoutMs error:(NSError **)errorOut { + NSString *text = OPJSONString(event ?: @{}); + NSURLSessionWebSocketMessage *message = [[NSURLSessionWebSocketMessage alloc] initWithString:text]; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + __block NSError *sendError = nil; + [self.task sendMessage:message completionHandler:^(NSError *error) { + sendError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_time_t deadline = dispatch_time(DISPATCH_TIME_NOW, + (int64_t)(MAX(1LL, timeoutMs) * NSEC_PER_MSEC)); + if (dispatch_semaphore_wait(semaphore, deadline) != 0) { + if (errorOut) { + *errorOut = OPRealtimeError(@"realtime_send_timeout"); + } + return NO; + } + if (sendError) { + if (errorOut) { + *errorOut = sendError; + } + return NO; + } + return YES; +} + +- (NSDictionary *)readEventWithTimeoutMs:(long long)timeoutMs error:(NSError **)errorOut { + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + __block NSURLSessionWebSocketMessage *message = nil; + __block NSError *receiveError = nil; + [self.task receiveMessageWithCompletionHandler: + ^(NSURLSessionWebSocketMessage *incoming, NSError *error) { + message = incoming; + receiveError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_time_t deadline = dispatch_time(DISPATCH_TIME_NOW, + (int64_t)(MAX(1LL, timeoutMs) * NSEC_PER_MSEC)); + if (dispatch_semaphore_wait(semaphore, deadline) != 0) { + if (errorOut) { + *errorOut = OPRealtimeError(@"realtime_receive_timeout"); + } + return nil; + } + if (receiveError || !message) { + if (errorOut) { + *errorOut = receiveError ?: OPRealtimeError(@"realtime_receive_failed"); + } + return nil; + } + NSString *text = nil; + if (message.type == NSURLSessionWebSocketMessageTypeString) { + text = message.string; + } else if (message.data) { + text = [[NSString alloc] initWithData:message.data encoding:NSUTF8StringEncoding]; + } + if (text.length == 0) { + if (errorOut) { + *errorOut = OPRealtimeError(@"realtime_empty_message"); + } + return nil; + } + NSData *data = [text dataUsingEncoding:NSUTF8StringEncoding]; + id object = data ? [NSJSONSerialization JSONObjectWithData:data options:0 error:nil] : nil; + if (![object isKindOfClass:[NSDictionary class]]) { + if (errorOut) { + *errorOut = OPRealtimeError(@"realtime_message_not_json_object"); + } + return nil; + } + return object; +} + +- (void)close { + [self.task cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil]; + [self.session invalidateAndCancel]; +} + +@end + +static NSDictionary *OPRealtimeProperty(NSString *type, NSString *description) { + return @{ + @"type": type ?: @"string", + @"description": description ?: @"" + }; +} + +static NSDictionary *OPRealtimeParameters(NSDictionary *properties, NSArray *required) { + return @{ + @"type": @"object", + @"properties": properties ?: @{}, + @"required": required ?: @[], + @"additionalProperties": @YES + }; +} + +static NSDictionary *OPRealtimeToolDefinition(NSString *name, NSString *description, + NSDictionary *properties, NSArray *required) { + return @{ + @"type": @"function", + @"name": name ?: @"", + @"description": description ?: @"", + @"parameters": OPRealtimeParameters(properties, required) + }; +} + +static NSArray *OPRealtimeToolDefinitions(void) { + NSMutableArray *tools = [NSMutableArray array]; + [tools addObject:OPRealtimeToolDefinition(@"get_screen", + @"Observe the current iPhone screen, foreground app, UI tree, visible text, and screenshot metadata.", + @{ + @"include_screenshot": OPRealtimeProperty(@"boolean", @"Include screenshot metadata when available."), + @"include_ui_tree": OPRealtimeProperty(@"boolean", @"Include the daemon's best UI tree."), + @"include_activity": OPRealtimeProperty(@"boolean", @"Include foreground/activity metadata."), + @"reason": OPRealtimeProperty(@"string", @"Why this observation is needed.") + }, @[@"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"tap", + @"Tap raw screen coordinates.", + @{ + @"x": OPRealtimeProperty(@"number", @"Screen x coordinate in points."), + @"y": OPRealtimeProperty(@"number", @"Screen y coordinate in points."), + @"reason": OPRealtimeProperty(@"string", @"Why this tap should progress the task.") + }, @[@"x", @"y", @"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"tap_element", + @"Tap a visible UI element by id from get_screen.", + @{ + @"element_id": OPRealtimeProperty(@"string", @"Element id from the current UI tree."), + @"reason": OPRealtimeProperty(@"string", @"Why this element should be tapped.") + }, @[@"element_id", @"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"long_press", + @"Long-press raw screen coordinates.", + @{ + @"x": OPRealtimeProperty(@"number", @"Screen x coordinate in points."), + @"y": OPRealtimeProperty(@"number", @"Screen y coordinate in points."), + @"duration_ms": OPRealtimeProperty(@"number", @"Press duration in milliseconds."), + @"reason": OPRealtimeProperty(@"string", @"Why this long press should progress the task.") + }, @[@"x", @"y", @"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"swipe", + @"Swipe from one coordinate to another.", + @{ + @"start_x": OPRealtimeProperty(@"number", @"Start x coordinate in points."), + @"start_y": OPRealtimeProperty(@"number", @"Start y coordinate in points."), + @"end_x": OPRealtimeProperty(@"number", @"End x coordinate in points."), + @"end_y": OPRealtimeProperty(@"number", @"End y coordinate in points."), + @"duration_ms": OPRealtimeProperty(@"number", @"Swipe duration in milliseconds."), + @"reason": OPRealtimeProperty(@"string", @"Why this swipe should progress the task.") + }, @[@"start_x", @"start_y", @"end_x", @"end_y", @"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"type_text", + @"Type text into the focused or specified editable field.", + @{ + @"text": OPRealtimeProperty(@"string", @"Text to enter."), + @"element_id": OPRealtimeProperty(@"string", @"Optional editable element id from get_screen."), + @"reason": OPRealtimeProperty(@"string", @"Why this text entry should progress the task.") + }, @[@"text", @"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"open_app", + @"Open an installed app by bundle id.", + @{ + @"bundle_id": OPRealtimeProperty(@"string", @"iOS bundle identifier, for example com.apple.mobilesafari."), + @"reason": OPRealtimeProperty(@"string", @"Why this app should be opened.") + }, @[@"bundle_id", @"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"open_url", + @"Open a URL through iOS.", + @{ + @"url": OPRealtimeProperty(@"string", @"Absolute URL to open."), + @"reason": OPRealtimeProperty(@"string", @"Why this URL should be opened.") + }, @[@"url", @"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"home", + @"Press the Home gesture/button equivalent.", + @{@"reason": OPRealtimeProperty(@"string", @"Why returning home should progress the task.")}, + @[@"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"wake_and_home", + @"Wake the display and return to Home.", + @{@"reason": OPRealtimeProperty(@"string", @"Why waking/home should progress the task.")}, + @[@"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"wait", + @"Wait briefly before observing or acting again.", + @{ + @"duration_ms": OPRealtimeProperty(@"number", @"Wait duration in milliseconds."), + @"reason": OPRealtimeProperty(@"string", @"Why waiting should progress the task.") + }, @[@"duration_ms", @"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"clipboard_read", + @"Read bounded text from the iPhone clipboard.", + @{ + @"max_chars": OPRealtimeProperty(@"number", @"Maximum clipboard characters to return."), + @"reason": OPRealtimeProperty(@"string", @"Why copied text is needed for this task.") + }, @[@"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"clipboard_write", + @"Write text to the iPhone clipboard.", + @{ + @"text": OPRealtimeProperty(@"string", @"Text to copy onto the iPhone clipboard."), + @"reason": OPRealtimeProperty(@"string", @"Why this text should be copied.") + }, @[@"text", @"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"contacts_search", + @"Search the iPhone contacts provider for people, organizations, phone numbers, or email addresses.", + @{ + @"query": OPRealtimeProperty(@"string", @"Contact name, organization, phone, or email to search for."), + @"limit": OPRealtimeProperty(@"number", @"Maximum result count."), + @"reason": OPRealtimeProperty(@"string", @"Why contact lookup is needed for this task.") + }, @[@"query", @"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"calendar_search", + @"Search the iPhone calendar provider for saved events or schedule context.", + @{ + @"query": OPRealtimeProperty(@"string", @"Event title, calendar name, location, or notes text to search for."), + @"start_at_ms": OPRealtimeProperty(@"number", @"Optional Unix epoch milliseconds lower bound for event start time."), + @"end_at_ms": OPRealtimeProperty(@"number", @"Optional Unix epoch milliseconds upper bound for event start time."), + @"limit": OPRealtimeProperty(@"number", @"Maximum result count."), + @"reason": OPRealtimeProperty(@"string", @"Why calendar lookup is needed for this task.") + }, @[@"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"calls_search", + @"Search the iPhone call-history provider for recent phone or FaceTime calls.", + @{ + @"query": OPRealtimeProperty(@"string", @"Phone number, caller name, service, direction, or call type to search for."), + @"start_at_ms": OPRealtimeProperty(@"number", @"Optional Unix epoch milliseconds lower bound for call start time."), + @"end_at_ms": OPRealtimeProperty(@"number", @"Optional Unix epoch milliseconds upper bound for call start time."), + @"limit": OPRealtimeProperty(@"number", @"Maximum result count."), + @"reason": OPRealtimeProperty(@"string", @"Why call-history lookup is needed for this task.") + }, @[@"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"messages_search", + @"Search the iPhone SMS/iMessage provider for saved message context.", + @{ + @"query": OPRealtimeProperty(@"string", @"Message text, phone number, email, handle, or service to search for."), + @"start_at_ms": OPRealtimeProperty(@"number", @"Optional Unix epoch milliseconds lower bound for message time."), + @"end_at_ms": OPRealtimeProperty(@"number", @"Optional Unix epoch milliseconds upper bound for message time."), + @"limit": OPRealtimeProperty(@"number", @"Maximum result count."), + @"reason": OPRealtimeProperty(@"string", @"Why message lookup is needed for this task.") + }, @[@"reason"])]; + [tools addObject:OPRealtimeToolDefinition(@"memory_save", + @"Save an explicit durable user preference or fact.", + @{ + @"text": OPRealtimeProperty(@"string", @"Memory text to save."), + @"type": OPRealtimeProperty(@"string", @"Memory type such as preference, fact, or instruction."), + @"subject": OPRealtimeProperty(@"string", @"Optional memory subject."), + @"reason": OPRealtimeProperty(@"string", @"Why this should be remembered.") + }, @[@"text"])]; + [tools addObject:OPRealtimeToolDefinition(@"memory_search", + @"Search durable memories for task-relevant user preferences or facts.", + @{ + @"query": OPRealtimeProperty(@"string", @"Search query."), + @"limit": OPRealtimeProperty(@"number", @"Maximum result count."), + @"reason": OPRealtimeProperty(@"string", @"Why memory context may help.") + }, @[@"query"])]; + [tools addObject:OPRealtimeToolDefinition(@"context_search", + @"Search recent task/conversation context.", + @{ + @"query": OPRealtimeProperty(@"string", @"Search query."), + @"limit": OPRealtimeProperty(@"number", @"Maximum result count."), + @"reason": OPRealtimeProperty(@"string", @"Why prior context may help.") + }, @[@"query"])]; + [tools addObject:OPRealtimeToolDefinition(@"finish_task", + @"Mark the task complete only when the requested result is visible or otherwise verified.", + @{@"summary": OPRealtimeProperty(@"string", @"Brief completion summary.")}, + @[@"summary"])]; + [tools addObject:OPRealtimeToolDefinition(@"fail_task", + @"Mark the task blocked or failed with a precise reason.", + @{@"reason": OPRealtimeProperty(@"string", @"Precise blocker or failure reason.")}, + @[@"reason"])]; + return tools; +} + +static NSString *OPRealtimeDeviceTimeContext(void) { + NSDateFormatter *formatter = [NSDateFormatter new]; + formatter.locale = [NSLocale localeWithLocaleIdentifier:@"en_US_POSIX"]; + formatter.dateFormat = @"EEE yyyy-MM-dd HH:mm zzz"; + return [NSString stringWithFormat:@"%@ (unix_ms %lld)", + [formatter stringFromDate:[NSDate date]], OPNowMs()]; +} + +static NSString *OPRealtimeInstructions(NSString *mode) { + BOOL realtime2 = [mode isEqualToString:@"openai_realtime2"]; + return [NSString stringWithFormat: + @"You are OpenPhone Agent, a persistent mobile GUI agent running inside an iPhone. " + @"The iPhone daemon is the runtime authority. You can control the phone only through the provided function tools. " + @"Autonomy mode is full YOLO: the user's task authorizes registered tools for this task. Do not ask for OpenPhone confirmation. " + @"Be execution-biased: observe, choose one useful action, inspect the result, recover from no-ops, and continue until the task is visibly complete or truly blocked. " + @"First call get_screen with include_ui_tree=true, include_activity=true, and include_screenshot=true before operating visible UI unless the next tool is obviously read-only memory/context. " + @"Use tap_element when an element id exists; use raw coordinates only when the UI tree is sparse or unlabeled. " + @"Use open_url for URLs instead of typing them into a browser field. Use memory_search/context_search when prior phone context would help. " + @"Use clipboard_read when the task depends on copied text, clipboard_write when the user asks to copy text, contacts_search when the task needs a saved person, organization, phone, or email, calendar_search when the task needs saved events or schedule context, calls_search when the task needs recent call-history context, and messages_search when the task needs saved SMS/iMessage context. " + @"Do not narrate a plan or answer with plain text when a phone tool can progress the task. " + @"For broad choices like random, any, best, or surprise me, choose a reasonable visible/default option yourself. " + @"Call finish_task only when the visible screen or tool result verifies the requested outcome. " + @"Never say Done unless finish_task has been called.%@", + realtime2 ? @" Realtime 2 should use low reasoning effort and concise tool selection." : @""]; +} + +static NSString *OPRealtimeInitialTaskPrompt(NSString *goal) { + return [NSString stringWithFormat: + @"Start this iPhone task and keep working until it is visibly complete or blocked. " + @"Device time: %@. Compute dates and deadlines from this device time. " + @"User goal: %@", + OPRealtimeDeviceTimeContext(), goal ?: @""]; +} + +static NSString *OPRealtimeTextOnlyCorrectionPrompt(NSString *modelText) { + return [NSString stringWithFormat: + @"You replied with text instead of taking a phone-agent step: %@\n\n" + @"Choose exactly one tool call now. If you have not observed the phone, call get_screen. " + @"If the task is already complete, call finish_task. If no concrete step exists, call fail_task.", + modelText ?: @""]; +} + +static NSURL *OPRealtimeURL(NSDictionary *config, NSString *model) { + NSString *endpoint = [config[@"endpoint_url"] isKindOfClass:[NSString class]] + ? config[@"endpoint_url"] : @""; + if (endpoint.length > 0) { + return [NSURL URLWithString:endpoint]; + } + NSMutableCharacterSet *allowed = [[NSCharacterSet URLQueryAllowedCharacterSet] mutableCopy]; + [allowed removeCharactersInString:@"&=+"]; + NSString *encoded = [model stringByAddingPercentEncodingWithAllowedCharacters:allowed] ?: model ?: @""; + return [NSURL URLWithString:[NSString stringWithFormat: + @"wss://api.openai.com/v1/realtime?model=%@", encoded]]; +} + +static NSDictionary *OPRealtimeSessionUpdateEvent(NSString *mode, NSString *model) { + NSMutableDictionary *session = [@{ + @"type": @"realtime", + @"model": model ?: @"", + @"instructions": OPRealtimeInstructions(mode ?: @"openai_realtime"), + @"output_modalities": @[@"text"], + @"tool_choice": @"auto", + @"tools": OPRealtimeToolDefinitions() + } mutableCopy]; + if ([mode isEqualToString:@"openai_realtime2"] || + [model isEqualToString:OPOpenAIRealtime2Model]) { + session[@"reasoning"] = @{@"effort": @"low"}; + } + return @{ + @"type": @"session.update", + @"session": session + }; +} + +// Streaming session config: accepts pcm16 mic audio and lets the server run +// voice-activity detection so end-of-turn is decided server-side instead of by +// our local record-then-transcribe VAD. Output stays text so the existing tool +// loop is unchanged; the agent speaks through the island, not TTS. +static NSDictionary *OPRealtimeStreamingSessionUpdateEvent(NSString *mode, NSString *model, + long long sampleRateHz) { + NSMutableDictionary *session = [@{ + @"type": @"realtime", + @"model": model ?: @"", + @"instructions": OPRealtimeInstructions(mode ?: @"openai_realtime2"), + @"output_modalities": @[@"text"], + @"tool_choice": @"auto", + @"tools": OPRealtimeToolDefinitions(), + @"audio": @{ + @"input": @{ + @"format": @{ + @"type": @"audio/pcm", + @"rate": @(sampleRateHz > 0 ? sampleRateHz : 16000) + }, + @"turn_detection": @{ + @"type": @"server_vad", + @"threshold": @0.5, + @"prefix_padding_ms": @300, + @"silence_duration_ms": @700, + @"create_response": @YES + } + } + } + } mutableCopy]; + if ([mode isEqualToString:@"openai_realtime2"] || + [model isEqualToString:OPOpenAIRealtime2Model]) { + session[@"reasoning"] = @{@"effort": @"low"}; + } + return @{ + @"type": @"session.update", + @"session": session + }; +} + +static NSDictionary *OPRealtimeAudioAppendEvent(NSString *base64Audio) { + return @{ + @"type": @"input_audio_buffer.append", + @"audio": base64Audio ?: @"" + }; +} + +static NSDictionary *OPRealtimeResponseCreateEvent(BOOL requireTool) { + NSMutableDictionary *response = [@{ + @"output_modalities": @[@"text"] + } mutableCopy]; + if (requireTool) { + response[@"tool_choice"] = @"required"; + } + return @{ + @"type": @"response.create", + @"response": response + }; +} + +static BOOL OPRealtimeSendUserMessage(OPRealtimeWebSocket *socket, NSString *text, + long long timeoutMs, NSError **errorOut) { + NSDictionary *event = @{ + @"type": @"conversation.item.create", + @"item": @{ + @"type": @"message", + @"role": @"user", + @"content": @[@{ + @"type": @"input_text", + @"text": text ?: @"" + }] + } + }; + return [socket sendEvent:event timeoutMs:timeoutMs error:errorOut]; +} + +static BOOL OPRealtimeSendFunctionOutput(OPRealtimeWebSocket *socket, NSString *callId, + NSDictionary *toolResult, long long timeoutMs, NSError **errorOut) { + NSString *output = OPJSONString(OPRedactedObject(toolResult ?: @{}, 0)); + NSDictionary *event = @{ + @"type": @"conversation.item.create", + @"item": @{ + @"type": @"function_call_output", + @"call_id": callId ?: @"", + @"output": output ?: @"{}" + } + }; + return [socket sendEvent:event timeoutMs:timeoutMs error:errorOut]; +} + +static BOOL OPRealtimeSendScreenFollowupIfUseful(OPRealtimeWebSocket *socket, + NSString *toolName, NSDictionary *toolResult, long long timeoutMs, NSError **errorOut) { + if (![toolName isEqualToString:@"get_screen"]) { + return YES; + } + NSString *text = [NSString stringWithFormat:@"Latest iPhone screen observation:\n%@", + OPJSONString(OPRedactedObject(toolResult ?: @{}, 0))]; + return OPRealtimeSendUserMessage(socket, text, timeoutMs, errorOut); +} + +static NSString *OPRealtimeStringFromValue(id value) { + if ([value isKindOfClass:[NSString class]]) { + return value; + } + if (value) { + return OPJSONString(value); + } + return @""; +} + +static NSDictionary *OPRealtimeFunctionCallFromEvent(NSDictionary *event) { + NSString *type = [event[@"type"] isKindOfClass:[NSString class]] ? event[@"type"] : @""; + if ([type isEqualToString:@"response.function_call_arguments.done"]) { + return @{ + @"call_id": OPRealtimeStringFromValue(event[@"call_id"]), + @"name": OPRealtimeStringFromValue(event[@"name"]), + @"arguments": OPRealtimeStringFromValue(event[@"arguments"]) + }; + } + if ([type isEqualToString:@"response.output_item.done"]) { + NSDictionary *item = [event[@"item"] isKindOfClass:[NSDictionary class]] + ? event[@"item"] : @{}; + if ([item[@"type"] isEqualToString:@"function_call"]) { + return @{ + @"call_id": OPRealtimeStringFromValue(item[@"call_id"]), + @"name": OPRealtimeStringFromValue(item[@"name"]), + @"arguments": OPRealtimeStringFromValue(item[@"arguments"]) + }; + } + } + return nil; +} + +static void OPRealtimeAddOrUpgradeCall(NSMutableArray *calls, NSDictionary *incoming) { + NSString *callId = [incoming[@"call_id"] isKindOfClass:[NSString class]] + ? incoming[@"call_id"] : @""; + NSString *name = [incoming[@"name"] isKindOfClass:[NSString class]] + ? incoming[@"name"] : @""; + if (callId.length == 0 || name.length == 0) { + return; + } + NSString *incomingArguments = [incoming[@"arguments"] isKindOfClass:[NSString class]] + ? incoming[@"arguments"] : @""; + for (NSUInteger i = 0; i < calls.count; i++) { + NSDictionary *existing = [calls[i] isKindOfClass:[NSDictionary class]] ? calls[i] : @{}; + NSString *existingCallId = [existing[@"call_id"] isKindOfClass:[NSString class]] + ? existing[@"call_id"] : @""; + NSString *existingArguments = [existing[@"arguments"] isKindOfClass:[NSString class]] + ? existing[@"arguments"] : @""; + if ([existingCallId isEqualToString:callId]) { + if (existingArguments.length == 0 && incomingArguments.length > 0) { + calls[i] = incoming; + } + return; + } + } + [calls addObject:incoming]; +} + +static NSString *OPRealtimeTextFromMessageItem(NSDictionary *item) { + NSArray *content = [item[@"content"] isKindOfClass:[NSArray class]] + ? item[@"content"] : @[]; + NSMutableArray *parts = [NSMutableArray array]; + for (id value in content) { + if (![value isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *part = value; + NSString *text = [part[@"text"] isKindOfClass:[NSString class]] + ? part[@"text"] : ([part[@"transcript"] isKindOfClass:[NSString class]] + ? part[@"transcript"] : @""); + if (text.length > 0) { + [parts addObject:text]; + } + } + return [parts componentsJoinedByString:@"\n"]; +} + +static void OPRealtimeCollectResponseDone(NSDictionary *response, NSMutableArray *calls, + NSMutableString *finalText) { + NSArray *output = [response[@"output"] isKindOfClass:[NSArray class]] + ? response[@"output"] : @[]; + for (id value in output) { + if (![value isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *item = value; + if ([item[@"type"] isEqualToString:@"function_call"]) { + OPRealtimeAddOrUpgradeCall(calls, @{ + @"call_id": OPRealtimeStringFromValue(item[@"call_id"]), + @"name": OPRealtimeStringFromValue(item[@"name"]), + @"arguments": OPRealtimeStringFromValue(item[@"arguments"]) + }); + } else if ([item[@"type"] isEqualToString:@"message"]) { + NSString *text = OPRealtimeTextFromMessageItem(item); + if (text.length > 0) { + if (finalText.length > 0) { + [finalText appendString:@"\n"]; + } + [finalText appendString:text]; + } + } + } +} + +static NSDictionary *OPRealtimeWaitForEventType(OPRealtimeWebSocket *socket, + NSString *expectedType, long long timeoutMs) { + long long deadlineMs = OPNowMs() + timeoutMs; + while (OPNowMs() < deadlineMs) { + NSError *error = nil; + NSDictionary *event = [socket readEventWithTimeoutMs:MAX(1, deadlineMs - OPNowMs()) + error:&error]; + if (!event) { + return OPError([NSString stringWithFormat:@"realtime_read_failed:%@", + error.localizedDescription ?: @"unknown"]); + } + NSString *type = [event[@"type"] isKindOfClass:[NSString class]] ? event[@"type"] : @""; + if ([type isEqualToString:expectedType]) { + return @{@"status": @"ok", @"event": event}; + } + if ([type isEqualToString:@"error"]) { + return @{ + @"status": @"error", + @"reason": @"realtime_error_event", + @"event": event, + @"source": @"openphone.agentd" + }; + } + } + return OPError([NSString stringWithFormat:@"realtime_wait_timeout:%@", expectedType ?: @""]); +} + +static NSDictionary *OPRealtimeWaitForTurn(OPRealtimeWebSocket *socket, long long timeoutMs) { + NSMutableArray *calls = [NSMutableArray array]; + NSMutableString *finalText = [NSMutableString string]; + NSMutableString *inputTranscript = [NSMutableString string]; + long long deadlineMs = OPNowMs() + timeoutMs; + while (OPNowMs() < deadlineMs) { + NSError *error = nil; + NSDictionary *event = [socket readEventWithTimeoutMs:MAX(1, deadlineMs - OPNowMs()) + error:&error]; + if (!event) { + return OPError([NSString stringWithFormat:@"realtime_read_failed:%@", + error.localizedDescription ?: @"unknown"]); + } + NSString *type = [event[@"type"] isKindOfClass:[NSString class]] ? event[@"type"] : @""; + if ([type isEqualToString:@"error"]) { + return @{ + @"status": @"error", + @"reason": @"realtime_error_event", + @"event": event, + @"source": @"openphone.agentd" + }; + } + // Server-VAD transcript of the user's own speech (streaming mode). Used + // to persist the voice turn and to seed the island transcript line. + if ([type isEqualToString:@"conversation.item.input_audio_transcription.completed"]) { + NSString *transcript = [event[@"transcript"] isKindOfClass:[NSString class]] + ? event[@"transcript"] : @""; + if (transcript.length > 0) { + if (inputTranscript.length > 0) { + [inputTranscript appendString:@" "]; + } + [inputTranscript appendString:transcript]; + } + } + NSDictionary *call = OPRealtimeFunctionCallFromEvent(event); + if (call) { + OPRealtimeAddOrUpgradeCall(calls, call); + } + if ([type isEqualToString:@"response.output_text.done"] || + [type isEqualToString:@"response.text.done"]) { + NSString *text = [event[@"text"] isKindOfClass:[NSString class]] + ? event[@"text"] : ([event[@"content"] isKindOfClass:[NSString class]] + ? event[@"content"] : @""); + if (text.length > 0) { + if (finalText.length > 0) { + [finalText appendString:@"\n"]; + } + [finalText appendString:text]; + } + } + if ([type isEqualToString:@"response.done"]) { + NSDictionary *response = [event[@"response"] isKindOfClass:[NSDictionary class]] + ? event[@"response"] : @{}; + OPRealtimeCollectResponseDone(response, calls, finalText); + return @{ + @"status": @"ok", + @"function_calls": calls, + @"final_text": finalText ?: @"", + @"input_transcript": inputTranscript ?: @"" + }; + } + } + return OPError(@"realtime_turn_timeout"); +} + +static NSDictionary *OPRealtimeArgumentsFromString(NSString *arguments) { + if (arguments.length == 0) { + return @{}; + } + NSData *data = [arguments dataUsingEncoding:NSUTF8StringEncoding]; + id object = data ? [NSJSONSerialization JSONObjectWithData:data options:0 error:nil] : nil; + return [object isKindOfClass:[NSDictionary class]] ? object : @{}; +} + +static NSDictionary *OPRealtimeDecisionFromCall(NSDictionary *call) { + NSString *tool = [call[@"name"] isKindOfClass:[NSString class]] ? call[@"name"] : @""; + NSString *argumentsString = [call[@"arguments"] isKindOfClass:[NSString class]] + ? call[@"arguments"] : @""; + NSMutableDictionary *arguments = [OPRealtimeArgumentsFromString(argumentsString) mutableCopy]; + if (![tool isEqualToString:@"finish_task"] && ![tool isEqualToString:@"fail_task"] && + ![arguments[@"reason"] isKindOfClass:[NSString class]]) { + arguments[@"reason"] = @"Continue the active iPhone task."; + } + return @{ + @"schema": @"openphone.model_decision.v1", + @"thought": @"Realtime tool call.", + @"tool": tool ?: @"", + @"arguments": arguments ?: @{}, + @"expected_visible_change": [arguments[@"expected_visible_change"] isKindOfClass:[NSString class]] + ? arguments[@"expected_visible_change"] : @"", + @"confidence": @0.8 + }; +} + +static NSDictionary *OPModelDecisionFromObject(id object, NSString **errorOut) { + if (![object isKindOfClass:[NSDictionary class]]) { + if (errorOut) { + *errorOut = @"decision_not_object"; + } + return nil; + } + NSDictionary *decision = object; + NSString *schema = [decision[@"schema"] isKindOfClass:[NSString class]] + ? decision[@"schema"] : @""; + if (![schema isEqualToString:@"openphone.model_decision.v1"] && + ![schema isEqualToString:@"openphone.model_decision.v2"] && + ![schema isEqualToString:@"openphone.model_decision.v3"]) { + if (errorOut) { + *errorOut = @"invalid_decision_schema"; + } + return nil; + } + // v3 router decisions don't need a `tool` field — they carry mode/reply + // instead. Short-circuit here so the tool validation below doesn't reject. + if ([schema isEqualToString:@"openphone.model_decision.v3"]) { + return decision; + } + NSString *tool = [decision[@"tool"] isKindOfClass:[NSString class]] ? decision[@"tool"] : @""; + if (![OPModelToolNames() containsObject:tool]) { + if (errorOut) { + *errorOut = tool.length > 0 ? @"unknown_model_tool" : @"missing_model_tool"; + } + return nil; + } + NSDictionary *arguments = [decision[@"arguments"] isKindOfClass:[NSDictionary class]] + ? decision[@"arguments"] : @{}; + NSString *expected = [decision[@"expected_visible_change"] isKindOfClass:[NSString class]] + ? decision[@"expected_visible_change"] : @""; + if (OPModelToolDrivesUI(tool) && expected.length == 0) { + if (errorOut) { + *errorOut = @"missing_expected_visible_change"; + } + return nil; + } + NSMutableDictionary *parsed = [NSMutableDictionary dictionary]; + parsed[@"schema"] = schema; + parsed[@"tool"] = tool; + parsed[@"arguments"] = arguments; + parsed[@"expected_visible_change"] = expected; + if ([decision[@"thought"] isKindOfClass:[NSString class]]) { + NSString *thought = decision[@"thought"]; + parsed[@"thought"] = thought.length > 400 + ? [[thought substringToIndex:400] stringByAppendingString:@"..."] : thought; + } + if ([decision[@"confidence"] respondsToSelector:@selector(doubleValue)]) { + double confidence = [decision[@"confidence"] doubleValue]; + parsed[@"confidence"] = @(MAX(0.0, MIN(confidence, 1.0))); + } + if ([decision[@"memory_relevance"] isKindOfClass:[NSString class]]) { + parsed[@"memory_relevance"] = decision[@"memory_relevance"]; + } + return parsed; +} + +static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { + if (string.length == 0) { + if (errorOut) { + *errorOut = @"empty_json"; + } + return nil; + } + NSData *data = [string dataUsingEncoding:NSUTF8StringEncoding]; + if (!data) { + if (errorOut) { + *errorOut = @"json_encoding_failed"; + } + return nil; + } + NSError *jsonError = nil; + id object = [NSJSONSerialization JSONObjectWithData:data options:0 error:&jsonError]; + if (!object && errorOut) { + *errorOut = jsonError.localizedDescription.length > 0 + ? [NSString stringWithFormat:@"invalid_json:%@", jsonError.localizedDescription] + : @"invalid_json"; + } + return object; +} + +static NSString *OPModelExtractBalancedJSONObject(NSString *string) { + if (string.length == 0) { + return @""; + } + BOOL inString = NO; + BOOL escaped = NO; + NSInteger depth = 0; + NSUInteger start = NSNotFound; + for (NSUInteger i = 0; i < string.length; i++) { + unichar ch = [string characterAtIndex:i]; + if (inString) { + if (escaped) { + escaped = NO; + } else if (ch == '\\') { + escaped = YES; + } else if (ch == '"') { + inString = NO; + } + continue; + } + if (ch == '"') { + inString = YES; + continue; + } + if (ch == '{') { + if (depth == 0) { + start = i; + } + depth += 1; + continue; + } + if (ch == '}' && depth > 0) { + depth -= 1; + if (depth == 0 && start != NSNotFound) { + return [string substringWithRange:NSMakeRange(start, i - start + 1)]; + } + } + } + return @""; +} + +static NSDictionary *OPModelDecisionFromString(NSString *string, NSString **errorOut, + NSString **repairStrategyOut) { + if (repairStrategyOut) { + *repairStrategyOut = @""; + } + NSString *rawError = nil; + id rawObject = OPModelJSONObjectFromString(string, &rawError); + if (rawObject) { + return OPModelDecisionFromObject(rawObject, errorOut); + } + NSString *candidate = OPModelExtractBalancedJSONObject(string); + if (candidate.length == 0 || [candidate isEqualToString:string]) { + if (errorOut) { + *errorOut = rawError.length > 0 ? rawError : @"invalid_json"; + } + return nil; + } + NSString *repairJsonError = nil; + id repairedObject = OPModelJSONObjectFromString(candidate, &repairJsonError); + NSString *decisionError = nil; + NSDictionary *decision = OPModelDecisionFromObject(repairedObject, &decisionError); + if (!decision) { + if (errorOut) { + *errorOut = decisionError.length > 0 + ? decisionError + : (repairJsonError.length > 0 ? repairJsonError : @"repair_failed"); + } + return nil; + } + if (repairStrategyOut) { + *repairStrategyOut = @"balanced_json_object"; + } + return decision; +} + +static NSDictionary *OPModelExecuteDecision(NSDictionary *decision, NSString *taskId, + NSArray *approvedCapabilities) { + NSString *tool = decision[@"tool"] ?: @""; + NSString *capability = OPModelToolCapability(tool); + if (![approvedCapabilities containsObject:capability]) { + return @{ + @"status": @"error", + @"state": @"tool.denied.capability", + @"tool": tool, + @"capability": capability, + @"reason": @"capability_not_approved", + @"source": @"openphone.agentd" + }; + } + NSDictionary *arguments = [decision[@"arguments"] isKindOfClass:[NSDictionary class]] + ? decision[@"arguments"] : @{}; + NSMutableDictionary *toolRequest = [arguments mutableCopy]; + toolRequest[@"task_id"] = taskId ?: @""; + if ([tool isEqualToString:@"get_screen"]) { + toolRequest[@"include_screenshot"] = toolRequest[@"include_screenshot"] ?: @YES; + toolRequest[@"include_activity"] = @YES; + toolRequest[@"include_ui_tree"] = @YES; + toolRequest[@"compact_trajectory"] = @YES; + toolRequest[@"reason"] = toolRequest[@"reason"] ?: @"model decision observation"; + return OPModelCompactScreenForLoop(OPGetScreen(toolRequest)); + } + if ([tool isEqualToString:@"memory_save"]) { + toolRequest[@"reason"] = toolRequest[@"reason"] ?: @"model decision memory_save"; + return OPMemorySave(toolRequest); + } + if ([tool isEqualToString:@"memory_search"]) { + toolRequest[@"reason"] = toolRequest[@"reason"] ?: @"model decision memory_search"; + return OPMemorySearch(toolRequest); + } + if ([tool isEqualToString:@"context_search"]) { + toolRequest[@"reason"] = toolRequest[@"reason"] ?: @"model decision context_search"; + return OPContextSearch(toolRequest); + } + if ([tool isEqualToString:@"clipboard_read"]) { + toolRequest[@"reason"] = toolRequest[@"reason"] ?: @"model decision clipboard_read"; + return OPClipboardRead(toolRequest); + } + if ([tool isEqualToString:@"clipboard_write"]) { + toolRequest[@"reason"] = toolRequest[@"reason"] ?: @"model decision clipboard_write"; + return OPClipboardWrite(toolRequest); + } + if ([tool isEqualToString:@"contacts_search"]) { + toolRequest[@"reason"] = toolRequest[@"reason"] ?: @"model decision contacts_search"; + return OPContactsSearch(toolRequest); + } + if ([tool isEqualToString:@"calendar_search"]) { + toolRequest[@"reason"] = toolRequest[@"reason"] ?: @"model decision calendar_search"; + return OPCalendarSearch(toolRequest); + } + if ([tool isEqualToString:@"calls_search"]) { + toolRequest[@"reason"] = toolRequest[@"reason"] ?: @"model decision calls_search"; + return OPCallsSearch(toolRequest); + } + if ([tool isEqualToString:@"messages_search"]) { + toolRequest[@"reason"] = toolRequest[@"reason"] ?: @"model decision messages_search"; + return OPMessagesSearch(toolRequest); + } + if ([tool isEqualToString:@"finish_task"]) { + NSString *summary = OPStringFromRequest(toolRequest, @"summary", + OPStringFromRequest(toolRequest, @"result", @"")); + if (summary.length == 0) { + summary = @"Task finished."; + } + return @{ + @"status": @"ok", + @"state": @"task.finished", + @"task_id": taskId ?: @"", + @"summary": summary, + @"source": @"openphone.agentd" + }; + } + if ([tool isEqualToString:@"fail_task"]) { + NSString *reason = OPStringFromRequest(toolRequest, @"reason", + OPStringFromRequest(toolRequest, @"summary", + OPStringFromRequest(toolRequest, @"result", @""))); + if (reason.length == 0) { + reason = @"Task failed."; + } + return @{ + @"status": @"ok", + @"state": @"task.failed", + @"task_id": taskId ?: @"", + @"reason": reason, + @"source": @"openphone.agentd" + }; + } + NSMutableDictionary *action = [arguments mutableCopy]; + action[@"type"] = tool; + return OPExecuteAction(@{ + @"command": @"execute_action", + @"task_id": taskId ?: @"", + @"action": action + }); +} + +static NSDictionary *OPModelVerificationState(NSString *tool, NSDictionary *toolResult, + NSDictionary *beforeScreen, NSDictionary *afterScreen, NSString *expectedVisibleChange) { + if (!OPModelToolDrivesUI(tool)) { + return @{ + @"status": @"not_required", + @"reason": @"non_ui_tool" + }; + } + NSString *state = [toolResult[@"state"] isKindOfClass:[NSString class]] + ? toolResult[@"state"] : @""; + BOOL toolOK = [toolResult[@"status"] isEqualToString:@"ok"] || + [state isEqualToString:@"action.executed"]; + NSDictionary *delta = OPModelScreenDelta(beforeScreen ?: @{}, afterScreen ?: @{}); + NSDictionary *providerVerification = [toolResult[@"verification"] isKindOfClass:[NSDictionary class]] + ? toolResult[@"verification"] : @{}; + NSString *userFacingStatus = [toolResult[@"user_facing_status"] isKindOfClass:[NSString class]] + ? toolResult[@"user_facing_status"] : @""; + if (!toolOK) { + return @{ + @"status": @"failed", + @"reason": toolResult[@"detail"] ?: toolResult[@"reason"] ?: @"tool_failed", + @"source": @"tool_result", + @"expected_visible_change": expectedVisibleChange ?: @"", + @"screen_state": afterScreen[@"state"] ?: @"", + @"screen_delta": delta + }; + } + if ([userFacingStatus isEqualToString:@"verified"] || + [userFacingStatus isEqualToString:@"success"] || + [providerVerification[@"status"] isEqualToString:@"verified"]) { + return @{ + @"status": @"verified", + @"reason": @"provider_verified_visible_effect", + @"source": providerVerification[@"source"] ?: @"provider_verification", + @"expected_visible_change": expectedVisibleChange ?: @"", + @"screen_state": afterScreen[@"state"] ?: @"", + @"screen_delta": delta + }; + } + BOOL changed = [delta[@"status"] isEqualToString:@"changed"]; + BOOL strongSignal = [delta[@"strong_signal"] boolValue]; + if (changed && strongSignal) { + return @{ + @"status": @"verified", + @"reason": @"observed_visible_screen_change", + @"source": @"before_after_screen_delta", + @"expected_visible_change": expectedVisibleChange ?: @"", + @"screen_state": afterScreen[@"state"] ?: @"", + @"screen_delta": delta + }; + } + if (changed) { + return @{ + @"status": @"observed_change", + @"reason": @"observed_screenshot_or_metadata_change", + @"source": @"before_after_screen_delta", + @"expected_visible_change": expectedVisibleChange ?: @"", + @"screen_state": afterScreen[@"state"] ?: @"", + @"screen_delta": delta + }; + } + return @{ + @"status": @"unverified_dispatch_only", + @"reason": userFacingStatus.length > 0 ? userFacingStatus : @"visible_effect_not_proven", + @"source": providerVerification[@"source"] ?: @"provider_dispatch", + @"expected_visible_change": expectedVisibleChange ?: @"", + @"screen_state": afterScreen[@"state"] ?: @"", + @"screen_delta": delta + }; +} + +static NSDictionary *OPRunOpenAIRealtimeTask(NSDictionary *request) { + long long startedMs = OPNowMs(); + NSString *goal = OPStringFromRequest(request, @"goal", @""); + NSArray *approved = [request[@"approved_capabilities"] isKindOfClass:[NSArray class]] + ? request[@"approved_capabilities"] : OPFullYoloCapabilities(); + NSDictionary *modelStatus = OPModelStatusDictionary(); + NSString *mode = [modelStatus[@"mode"] isKindOfClass:[NSString class]] + ? modelStatus[@"mode"] : @"broker"; + if (!OPModelModeIsOpenAIRealtime(mode)) { + return OPError(@"model_mode_not_openai_realtime"); + } + if (![modelStatus[@"status"] isEqualToString:@"ready"]) { + return OPError(@"model_provider_not_configured"); + } + + long long configuredMaxSteps = [modelStatus[@"max_steps"] respondsToSelector:@selector(longLongValue)] + ? [modelStatus[@"max_steps"] longLongValue] : 25; + long long configuredMaxDuration = [modelStatus[@"max_duration_ms"] respondsToSelector:@selector(longLongValue)] + ? [modelStatus[@"max_duration_ms"] longLongValue] : 120000; + long long timeoutMs = [modelStatus[@"timeout_ms"] respondsToSelector:@selector(longLongValue)] + ? [modelStatus[@"timeout_ms"] longLongValue] : 30000; + long long maxSteps = OPLongLongFromRequest(request, @"max_steps", configuredMaxSteps, 1, 120); + long long maxDurationMs = OPLongLongFromRequest(request, @"max_duration_ms", + configuredMaxDuration, 1000, 3300000); + NSDictionary *limits = @{ + @"max_steps": @(maxSteps), + @"max_duration_ms": @(maxDurationMs), + @"max_tool_errors": @2, + @"max_unverified_ui_actions": @2, + @"max_text_only_turns": @3 + }; + + NSString *requestedTaskId = OPStringFromRequest(request, @"task_id", @""); + BOOL adoptedTask = NO; + NSDictionary *task = nil; + if (requestedTaskId.length > 0) { + task = OPReadJSONFile(OPTaskPath(requestedTaskId)); + if (![task isKindOfClass:[NSDictionary class]]) { + return OPError(@"task_not_found"); + } + adoptedTask = YES; + } else { + task = OPStartTask(@{@"goal": goal, @"approved_capabilities": approved}); + } + NSString *taskId = adoptedTask ? requestedTaskId : task[@"task_id"]; + if (adoptedTask && goal.length == 0 && [task[@"goal"] isKindOfClass:[NSString class]]) { + goal = task[@"goal"]; + } + + NSString *cancelReason = @""; + if (OPTaskCancellationRequested(taskId, &cancelReason)) { + NSDictionary *cancelledSummary = @{ + @"status": @"task.cancelled", + @"goal": goal ?: @"", + @"task_id": taskId ?: @"", + @"runner": @"model", + @"model_provider": mode ?: @"openai_realtime", + @"model_runtime": @"openai_realtime_websocket", + @"model": modelStatus[@"model"] ?: @"", + @"limits": limits, + @"steps_used": @0, + @"tool_errors": @0, + @"unverified_ui_actions": @0, + @"duration_ms": @(OPNowMs() - startedMs), + @"stop_reason": @"cancelled", + @"cancel_reason": cancelReason.length > 0 ? cancelReason : @"cancelled", + @"last_tool_result": @{ + @"status": @"ok", + @"state": @"task.cancelled", + @"reason": cancelReason.length > 0 ? cancelReason : @"cancelled", + @"task_id": taskId ?: @"", + @"source": @"openphone.agentd" + }, + @"trajectory": OPTrajectoryPath(taskId ?: @""), + @"source": @"openphone.agentd" + }; + OPUpdateTask(taskId, @"stopped", @{ + @"model_loop_summary": cancelledSummary, + @"completed_at": @(OPNowMs()), + @"cancel_requested": @YES, + @"cancel_reason": cancelReason.length > 0 ? cancelReason : @"cancelled" + }); + OPRecordTrajectory(taskId, @"model_loop_cancelled", cancelledSummary); + OPRecordTrajectory(taskId, @"model_loop_finished", cancelledSummary); + return cancelledSummary; + } + + OPUpdateTask(taskId, @"active", @{ + @"runner": @"model", + @"model_provider": mode ?: @"openai_realtime", + @"model_runtime": @"openai_realtime_websocket", + @"model": modelStatus[@"model"] ?: @"", + @"runner_pid": @(getpid()), + @"runner_started_at": @(OPNowMs()), + @"limits": limits + }); + OPRecordTrajectory(taskId, adoptedTask ? @"model_loop_attached" : @"model_loop_started", @{ + @"runner": @"model", + @"provider": mode ?: @"openai_realtime", + @"runtime": @"openai_realtime_websocket", + @"model": modelStatus[@"model"] ?: @"", + @"goal": goal ?: @"" + }); + + OPRealtimeWebSocket *socket = nil; + NSDictionary *screen = @{}; + NSDictionary *lastToolResult = @{}; + NSString *lastModelTool = @""; + NSString *stopReason = @"unknown"; + BOOL terminal = NO; + BOOL succeeded = NO; + BOOL cancelled = NO; + long long stepsUsed = 0; + long long toolErrors = 0; + long long unverifiedUIActions = 0; + long long textOnlyTurns = 0; + + NSDictionary *config = OPModelConfig(); + NSString *model = [modelStatus[@"model"] isKindOfClass:[NSString class]] + ? modelStatus[@"model"] : OPModelEffectiveModel(config); + NSString *credential = OPModelCredentialValue(); + NSURL *url = OPRealtimeURL(config, model); + if (!url || !url.scheme || !url.host) { + stopReason = @"realtime_endpoint_invalid"; + toolErrors = 1; + lastToolResult = OPError(stopReason); + } else if (credential.length == 0) { + stopReason = @"model_credential_missing"; + toolErrors = 1; + lastToolResult = OPError(stopReason); + } else { + NSError *error = nil; + OPRecordTrajectory(taskId, @"realtime_session_starting", @{ + @"provider": mode ?: @"openai_realtime", + @"model": model ?: @"", + @"endpoint_host": url.host ?: @"", + @"timeout_ms": @(timeoutMs) + }); + socket = [OPRealtimeWebSocket connectWithURL:url bearerToken:credential + timeoutMs:timeoutMs error:&error]; + if (!socket) { + stopReason = @"realtime_connect_failed"; + toolErrors = 1; + lastToolResult = OPError(error.localizedDescription ?: stopReason); + } else if (![socket sendEvent:OPRealtimeSessionUpdateEvent(mode, model) + timeoutMs:timeoutMs error:&error]) { + stopReason = @"realtime_session_update_send_failed"; + toolErrors = 1; + lastToolResult = OPError(error.localizedDescription ?: stopReason); + } else { + NSDictionary *sessionUpdated = OPRealtimeWaitForEventType(socket, @"session.updated", timeoutMs); + if (![sessionUpdated[@"status"] isEqualToString:@"ok"]) { + stopReason = sessionUpdated[@"reason"] ?: @"realtime_session_update_failed"; + toolErrors = 1; + lastToolResult = sessionUpdated ?: OPError(stopReason); + } else if (!OPRealtimeSendUserMessage(socket, OPRealtimeInitialTaskPrompt(goal), + timeoutMs, &error)) { + stopReason = @"realtime_initial_message_failed"; + toolErrors = 1; + lastToolResult = OPError(error.localizedDescription ?: stopReason); + } else if ((screen = OPModelCompactScreenForLoop(OPGetScreen(@{ + @"include_screenshot": @YES, + @"include_ui_tree": @YES, + @"include_activity": @YES, + @"compact_trajectory": @YES, + @"task_id": taskId ?: @"", + @"reason": @"realtime initial forced observation" + }))) && !OPRealtimeSendUserMessage(socket, + [NSString stringWithFormat:@"Current iPhone screen observation:\n%@", + OPJSONString(OPRedactedObject(screen, 0))], + timeoutMs, &error)) { + stopReason = @"realtime_initial_observation_failed"; + toolErrors = 1; + lastToolResult = OPError(error.localizedDescription ?: stopReason); + } else if (![socket sendEvent:OPRealtimeResponseCreateEvent(YES) + timeoutMs:timeoutMs error:&error]) { + stopReason = @"realtime_response_create_failed"; + toolErrors = 1; + lastToolResult = OPError(error.localizedDescription ?: stopReason); + } else { + OPRecordTrajectory(taskId, @"realtime_session_ready", @{ + @"provider": mode ?: @"openai_realtime", + @"model": model ?: @"", + @"runtime": @"openai_realtime_websocket" + }); + + for (long long step = 1; step <= maxSteps; step++) { + if (OPTaskCancellationRequested(taskId, &cancelReason)) { + cancelled = YES; + stopReason = @"cancelled"; + break; + } + if (OPNowMs() - startedMs > maxDurationMs) { + stopReason = @"duration_limit"; + break; + } + stepsUsed = step; + OPUpdateTask(taskId, @"active", @{ + @"model_loop_current": @{ + @"status": @"realtime_waiting_for_turn", + @"step": @(step), + @"max_steps": @(maxSteps), + @"tool": lastModelTool ?: @"", + @"updated_at_ms": @(OPNowMs()) + } + }); + NSDictionary *turn = OPRealtimeWaitForTurn(socket, timeoutMs); + if (![turn[@"status"] isEqualToString:@"ok"]) { + toolErrors += 1; + stopReason = turn[@"reason"] ?: @"realtime_turn_failed"; + lastToolResult = turn ?: OPError(stopReason); + break; + } + NSString *finalText = [turn[@"final_text"] isKindOfClass:[NSString class]] + ? turn[@"final_text"] : @""; + if (finalText.length > 0) { + OPRecordTrajectory(taskId, @"realtime_model_text", @{ + @"step": @(step), + @"provider": mode ?: @"openai_realtime", + @"text": OPTruncatedString(finalText, 1000) + }); + } + NSArray *calls = [turn[@"function_calls"] isKindOfClass:[NSArray class]] + ? turn[@"function_calls"] : @[]; + if (calls.count == 0) { + textOnlyTurns += 1; + if (textOnlyTurns >= 3) { + stopReason = @"agent.blocked"; + lastToolResult = @{ + @"status": @"error", + @"state": @"task.failed", + @"reason": @"realtime_returned_text_without_tool_calls", + @"model_text": OPTruncatedString(finalText, 1000), + @"source": @"openphone.agentd" + }; + break; + } + NSError *correctionError = nil; + if (!OPRealtimeSendUserMessage(socket, + OPRealtimeTextOnlyCorrectionPrompt(finalText), + timeoutMs, &correctionError) || + ![socket sendEvent:OPRealtimeResponseCreateEvent(YES) + timeoutMs:timeoutMs error:&correctionError]) { + toolErrors += 1; + stopReason = @"realtime_text_correction_failed"; + lastToolResult = OPError(correctionError.localizedDescription ?: stopReason); + break; + } + continue; + } + textOnlyTurns = 0; + + for (id value in calls) { + if (![value isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *call = value; + if (OPTaskCancellationRequested(taskId, &cancelReason)) { + cancelled = YES; + stopReason = @"cancelled"; + break; + } + NSDictionary *decision = OPRealtimeDecisionFromCall(call); + NSDictionary *guardrail = nil; + decision = OPModelDecisionByApplyingGuardrails(decision, goal, step, &guardrail); + NSString *tool = [decision[@"tool"] isKindOfClass:[NSString class]] + ? decision[@"tool"] : @""; + lastModelTool = tool ?: @""; + OPUpdateTask(taskId, @"active", @{ + @"model_loop_current": @{ + @"status": @"decision_received", + @"step": @(step), + @"max_steps": @(maxSteps), + @"tool": tool ?: @"", + @"updated_at_ms": @(OPNowMs()) + } + }); + OPIslandPublishToolStep(taskId, tool, @"decision_received", step, maxSteps); + { + NSString *msg = [decision[@"assistant_message"] isKindOfClass:[NSString class]] + ? decision[@"assistant_message"] : @""; + if (msg.length > 0) OPIslandPublishAssistantMessage(msg, taskId); + } + OPRecordTrajectory(taskId, @"model_decision", @{ + @"step": @(step), + @"provider": mode ?: @"openai_realtime", + @"runtime": @"openai_realtime_websocket", + @"call_id": call[@"call_id"] ?: @"", + @"decision": decision + }); + if (guardrail.count > 0) { + OPRecordTrajectory(taskId, @"model_decision_guardrail", guardrail); + } + if (![OPModelToolNames() containsObject:tool]) { + toolErrors += 1; + lastToolResult = @{ + @"status": @"error", + @"state": @"task.failed", + @"reason": [NSString stringWithFormat:@"unknown_model_tool:%@", tool ?: @""], + @"source": @"openphone.agentd" + }; + NSError *outputError = nil; + OPRealtimeSendFunctionOutput(socket, call[@"call_id"], lastToolResult, + timeoutMs, &outputError); + stopReason = @"unknown_model_tool"; + break; + } + + if (screen.count == 0 && OPModelToolDrivesUI(tool)) { + screen = OPModelCompactScreenForLoop(OPGetScreen(@{ + @"task_id": taskId ?: @"", + @"include_screenshot": @YES, + @"include_activity": @YES, + @"include_ui_tree": @YES, + @"compact_trajectory": @YES, + @"reason": @"realtime model pre-action observation" + })); + } + OPRecordTrajectory(taskId, @"tool_call", @{ + @"tool": tool ?: @"", + @"step": @(step), + @"arguments": decision[@"arguments"] ?: @{} + }); + OPUpdateTask(taskId, @"active", @{ + @"model_loop_current": @{ + @"status": @"tool_running", + @"step": @(step), + @"max_steps": @(maxSteps), + @"tool": tool ?: @"", + @"updated_at_ms": @(OPNowMs()) + } + }); + OPIslandPublishToolStep(taskId, tool, @"tool_running", step, maxSteps); + + NSDictionary *toolResult = OPModelExecuteDecision(decision, taskId, approved); + lastToolResult = toolResult ?: @{}; + NSString *state = [toolResult[@"state"] isKindOfClass:[NSString class]] + ? toolResult[@"state"] : @""; + BOOL toolOK = [toolResult[@"status"] isEqualToString:@"ok"] || + [state isEqualToString:@"action.executed"] || + [state isEqualToString:@"task.finished"] || + [state isEqualToString:@"task.failed"]; + if (!toolOK || [state hasPrefix:@"action.denied"] || + [toolResult[@"status"] isEqualToString:@"error"]) { + toolErrors += 1; + } + + NSDictionary *afterScreen = @{}; + NSDictionary *verification = @{@"status": @"not_required", @"reason": @"non_ui_tool"}; + BOOL skippedPostActionScreen = NO; + if (OPModelToolDrivesUI(tool)) { + if (!toolOK) { + skippedPostActionScreen = YES; + verification = @{ + @"status": @"failed", + @"reason": toolResult[@"detail"] ?: toolResult[@"reason"] ?: @"tool_failed", + @"source": @"tool_result", + @"expected_visible_change": decision[@"expected_visible_change"] ?: @"", + @"screen_state": screen[@"state"] ?: @"" + }; + } else if (OPModelShouldUseProviderVerificationOnly(tool, toolResult)) { + skippedPostActionScreen = YES; + verification = OPModelProviderVerificationState(tool, toolResult, screen, + decision[@"expected_visible_change"] ?: @""); + } else { + afterScreen = OPModelCompactScreenForLoop(OPGetScreen(@{ + @"task_id": taskId ?: @"", + @"include_screenshot": @YES, + @"include_activity": @YES, + @"include_ui_tree": @YES, + @"compact_trajectory": @YES, + @"reason": @"realtime model post-action verification" + })); + verification = OPModelVerificationState(tool, toolResult, screen, + afterScreen, decision[@"expected_visible_change"] ?: @""); + } + if ([verification[@"status"] isEqualToString:@"unverified_dispatch_only"]) { + unverifiedUIActions += 1; + } + } + OPRecordTrajectory(taskId, @"model_step_verified", @{ + @"step": @(step), + @"tool": tool ?: @"", + @"tool_result": OPModelToolResultSummary(toolResult ?: @{}), + @"verification": OPModelVerificationTraceSummary(verification), + @"before_screen": screen.count > 0 ? OPModelScreenTraceSummary(screen) : @{}, + @"after_screen": (!skippedPostActionScreen && afterScreen.count > 0) + ? OPModelScreenTraceSummary(afterScreen) : @{} + }); + OPUpdateTask(taskId, @"active", @{ + @"model_loop_current": @{ + @"status": @"step_verified", + @"step": @(step), + @"max_steps": @(maxSteps), + @"tool": tool ?: @"", + @"verification": verification[@"status"] ?: @"not_required", + @"updated_at_ms": @(OPNowMs()) + } + }); + + NSError *outputError = nil; + if (!OPRealtimeSendFunctionOutput(socket, call[@"call_id"], toolResult ?: @{}, + timeoutMs, &outputError) || + !OPRealtimeSendScreenFollowupIfUseful(socket, tool, toolResult ?: @{}, + timeoutMs, &outputError)) { + toolErrors += 1; + stopReason = @"realtime_function_output_failed"; + lastToolResult = OPError(outputError.localizedDescription ?: stopReason); + break; + } + + if (OPModelVerifiedTypeTextCompletesGoal(goal, decision, verification)) { + terminal = YES; + succeeded = YES; + stopReason = @"verified_type_text_goal_complete"; + lastToolResult = @{ + @"status": @"ok", + @"state": @"task.finished", + @"task_id": taskId ?: @"", + @"summary": @"Verified text entry completed.", + @"reason": stopReason, + @"action_result": OPModelToolResultSummary(toolResult ?: @{}), + @"source": @"openphone.agentd" + }; + break; + } + if ([tool isEqualToString:@"finish_task"] || [state isEqualToString:@"task.finished"]) { + terminal = YES; + succeeded = YES; + stopReason = @"finish_task"; + break; + } + if ([tool isEqualToString:@"fail_task"] || [state isEqualToString:@"task.failed"]) { + terminal = YES; + succeeded = NO; + stopReason = @"fail_task"; + break; + } + if (toolErrors >= 2) { + stopReason = @"tool_error_limit"; + break; + } + if (unverifiedUIActions >= 2) { + stopReason = @"no_visible_progress"; + break; + } + screen = afterScreen.count > 0 ? afterScreen : screen; + } + if (terminal || cancelled || + ![stopReason isEqualToString:@"unknown"] || + toolErrors >= 2 || unverifiedUIActions >= 2) { + break; + } + NSError *nextError = nil; + if (![socket sendEvent:OPRealtimeResponseCreateEvent(YES) + timeoutMs:timeoutMs error:&nextError]) { + toolErrors += 1; + stopReason = @"realtime_response_create_failed"; + lastToolResult = OPError(nextError.localizedDescription ?: stopReason); + break; + } + } + } + } + } + + if (socket) { + [socket close]; + } + + if (cancelled) { + lastToolResult = @{ + @"status": @"ok", + @"state": @"task.cancelled", + @"reason": cancelReason.length > 0 ? cancelReason : @"cancelled", + @"task_id": taskId ?: @"", + @"source": @"openphone.agentd" + }; + succeeded = NO; + } else if (!terminal) { + if ([stopReason isEqualToString:@"unknown"]) { + stopReason = stepsUsed >= maxSteps ? @"step_limit" : @"model_stopped"; + } + NSDictionary *failure = OPFailTask(@{ + @"task_id": taskId ?: @"", + @"reason": stopReason ?: @"model_stopped" + }); + lastToolResult = failure ?: lastToolResult; + succeeded = NO; + } + + long long durationMs = OPNowMs() - startedMs; + NSDictionary *summary = @{ + @"status": succeeded ? @"task.finished" : (cancelled ? @"task.cancelled" : @"task.failed"), + @"goal": goal ?: @"", + @"task_id": taskId ?: @"", + @"runner": @"model", + @"model_provider": mode ?: @"openai_realtime", + @"model_runtime": @"openai_realtime_websocket", + @"model": model ?: @"", + @"limits": limits, + @"steps_used": @(stepsUsed), + @"tool_errors": @(toolErrors), + @"unverified_ui_actions": @(unverifiedUIActions), + @"text_only_turns": @(textOnlyTurns), + @"duration_ms": @(durationMs), + @"stop_reason": stopReason ?: @"unknown", + @"cancel_reason": cancelled ? (cancelReason.length > 0 ? cancelReason : @"cancelled") : @"", + @"last_tool_result": lastToolResult ?: @{}, + @"trajectory": OPTrajectoryPath(taskId ?: @""), + @"source": @"openphone.agentd" + }; + OPUpdateTask(taskId, succeeded ? @"completed" : (cancelled ? @"stopped" : @"failed"), @{ + @"result": lastToolResult ?: @{}, + @"model_loop_summary": summary, + @"model_loop_current": @{ + @"status": summary[@"status"] ?: @"task.finished", + @"step": @(stepsUsed), + @"max_steps": @(maxSteps), + @"tool": lastModelTool ?: @"", + @"stop_reason": stopReason ?: @"unknown", + @"updated_at_ms": @(OPNowMs()) + }, + @"completed_at": @(OPNowMs()), + @"cancel_requested": cancelled ? @YES : @NO, + @"cancel_reason": cancelled ? (cancelReason.length > 0 ? cancelReason : @"cancelled") : @"" + }); + OPRecordAudit(succeeded ? @"model_task_finished" : + (cancelled ? @"model_task_cancelled" : @"model_task_failed"), taskId, + @"tasks.observe", succeeded ? @"allow_task_scoped" : (cancelled ? @"cancelled" : @"failed"), + @{ + @"command": @"run_task", + @"mode": @"model", + @"goal": goal ?: @"", + @"status": summary[@"status"] ?: @"unknown", + @"stop_reason": stopReason ?: @"unknown", + @"model_provider": summary[@"model_provider"] ?: @"unknown", + @"model_runtime": summary[@"model_runtime"] ?: @"unknown", + @"model": model ?: @"", + @"steps_used": summary[@"steps_used"] ?: @0 + }, + stopReason ?: @"unknown"); + if (cancelled) { + OPRecordTrajectory(taskId, @"model_loop_cancelled", summary); + } + OPRecordTrajectory(taskId, @"model_loop_finished", summary); + return summary; +} + +static NSDictionary *OPRunModelTask(NSDictionary *request) { + long long startedMs = OPNowMs(); + NSString *goal = OPStringFromRequest(request, @"goal", @""); + NSArray *approved = [request[@"approved_capabilities"] isKindOfClass:[NSArray class]] + ? request[@"approved_capabilities"] : OPFullYoloCapabilities(); + NSDictionary *modelStatus = OPModelStatusDictionary(); + NSArray *fixtureDecisions = [request[@"model_decisions"] isKindOfClass:[NSArray class]] + ? request[@"model_decisions"] : @[]; + BOOL hasFixture = fixtureDecisions.count > 0; + if (!hasFixture && ![modelStatus[@"status"] isEqualToString:@"ready"]) { + return OPError(@"model_provider_not_configured"); + } + long long configuredMaxSteps = [modelStatus[@"max_steps"] respondsToSelector:@selector(longLongValue)] + ? [modelStatus[@"max_steps"] longLongValue] : 5; + long long configuredMaxDuration = [modelStatus[@"max_duration_ms"] respondsToSelector:@selector(longLongValue)] + ? [modelStatus[@"max_duration_ms"] longLongValue] : 120000; + long long maxSteps = OPLongLongFromRequest(request, @"max_steps", configuredMaxSteps, 1, 25); + long long maxDurationMs = OPLongLongFromRequest(request, @"max_duration_ms", configuredMaxDuration, 1000, 600000); + NSDictionary *limits = @{ + @"max_steps": @(maxSteps), + @"max_duration_ms": @(maxDurationMs), + @"max_parser_failures": @2, + @"max_tool_errors": @2, + @"max_unverified_ui_actions": @2 + }; + NSString *requestedTaskId = OPStringFromRequest(request, @"task_id", @""); + BOOL adoptedTask = NO; + NSDictionary *task = nil; + if (requestedTaskId.length > 0) { + task = OPReadJSONFile(OPTaskPath(requestedTaskId)); + if (![task isKindOfClass:[NSDictionary class]]) { + return OPError(@"task_not_found"); + } + adoptedTask = YES; + } else { + task = OPStartTask(@{@"goal": goal, @"approved_capabilities": approved}); + } + NSString *taskId = adoptedTask ? requestedTaskId : task[@"task_id"]; + if (adoptedTask && goal.length == 0 && [task[@"goal"] isKindOfClass:[NSString class]]) { + goal = task[@"goal"]; + } + NSString *cancelReason = @""; + if (OPTaskCancellationRequested(taskId, &cancelReason)) { + long long durationMs = OPNowMs() - startedMs; + NSDictionary *summary = @{ + @"status": @"task.cancelled", + @"goal": goal ?: @"", + @"task_id": taskId ?: @"", + @"runner": @"model", + @"model_provider": hasFixture ? @"fixture" : (modelStatus[@"mode"] ?: @"broker"), + @"limits": limits, + @"steps_used": @0, + @"parser_failures": @0, + @"tool_errors": @0, + @"unverified_ui_actions": @0, + @"duration_ms": @(durationMs), + @"stop_reason": @"cancelled", + @"cancel_reason": cancelReason.length > 0 ? cancelReason : @"cancelled", + @"last_tool_result": @{ + @"status": @"ok", + @"state": @"task.cancelled", + @"reason": cancelReason.length > 0 ? cancelReason : @"cancelled", + @"task_id": taskId ?: @"", + @"source": @"openphone.agentd" + }, + @"trajectory": OPTrajectoryPath(taskId ?: @""), + @"source": @"openphone.agentd" + }; + OPUpdateTask(taskId, @"stopped", @{ + @"model_loop_summary": summary, + @"completed_at": @(OPNowMs()), + @"cancel_requested": @YES, + @"cancel_reason": cancelReason.length > 0 ? cancelReason : @"cancelled" + }); + OPRecordAudit(@"model_task_cancelled", taskId, @"tasks.observe", @"cancelled", + @{@"command": @"run_task", @"mode": @"model", @"goal": goal ?: @""}, cancelReason); + OPRecordTrajectory(taskId, @"model_loop_cancelled", summary); + OPRecordTrajectory(taskId, @"model_loop_finished", summary); + return summary; + } + OPUpdateTask(taskId, @"active", @{ + @"runner": @"model", + @"model_provider": hasFixture ? @"fixture" : (modelStatus[@"mode"] ?: @"broker"), + @"runner_pid": @(getpid()), + @"runner_started_at": @(OPNowMs()), + @"limits": limits + }); + if (adoptedTask) { + OPRecordAudit(@"model_task_attached", taskId, @"tasks.observe", @"allow_task_scoped", + @{@"command": @"run_task", @"mode": @"model", @"goal": goal ?: @""}, + @"adopted_existing_task"); + OPRecordTrajectory(taskId, @"model_loop_attached", @{ + @"runner": @"model", + @"provider": hasFixture ? @"fixture" : (modelStatus[@"mode"] ?: @"broker"), + @"goal": goal ?: @"" + }); + } + + NSDictionary *screen = OPModelCompactScreenForLoop(OPGetScreen(@{ + @"task_id": taskId ?: @"", + @"include_screenshot": @YES, + @"include_activity": @YES, + @"include_ui_tree": @YES, + @"compact_trajectory": @YES, + @"reason": @"model run_task initial observation" + })); + NSDictionary *promptContext = OPModelPromptContext(goal, taskId, screen, modelStatus, approved, @{}); + OPRecordTrajectory(taskId, @"model_prompt_prepared", @{ + @"provider": hasFixture ? @"fixture" : (modelStatus[@"mode"] ?: @"broker"), + @"context": OPModelPromptContextTraceSummary(promptContext ?: @{}) + }); + + long long parserFailures = 0; + long long toolErrors = 0; + long long unverifiedUIActions = 0; + long long stepsUsed = 0; + NSString *stopReason = @"unknown"; + NSDictionary *lastToolResult = @{}; + NSString *lastModelTool = @""; + long long consecutiveGetScreenCount = 0; + NSString *lastDecisionSig = @""; + long long lastDecisionRepeats = 0; + // Router-mode queue: when the model returns mode=act with proposed_actions, + // we drain them one by one *without* re-prompting the model. Massively + // reduces roundtrips + eliminates the source of loop bugs. + NSMutableArray *routerActionQueue = [NSMutableArray array]; + NSString *routerReply = @""; + NSString *routerLastScreenSignature = @""; + long long consecutiveNoProgress = 0; + BOOL terminal = NO; + BOOL succeeded = NO; + BOOL cancelled = NO; + + for (long long step = 1; step <= maxSteps; step++) { + @autoreleasepool { + if (OPTaskCancellationRequested(taskId, &cancelReason)) { + cancelled = YES; + stopReason = @"cancelled"; + break; + } + stepsUsed = step; + OPUpdateTask(taskId, @"active", @{ + @"model_loop_current": @{ + @"status": @"step_started", + @"step": @(step), + @"max_steps": @(maxSteps), + @"tool": lastModelTool ?: @"", + @"updated_at_ms": @(OPNowMs()) + } + }); + if (OPNowMs() - startedMs > maxDurationMs) { + stopReason = @"duration_limit"; + break; + } + id rawDecision = nil; + if (hasFixture && (NSUInteger)(step - 1) < fixtureDecisions.count) { + rawDecision = fixtureDecisions[(NSUInteger)(step - 1)]; + } else if (routerActionQueue.count > 0) { + // Router pre-planned action queue: drain the next action WITHOUT + // hitting the model again. This is what kills the loop bug. + NSDictionary *action = routerActionQueue.firstObject; + [routerActionQueue removeObjectAtIndex:0]; + NSMutableDictionary *synth = [NSMutableDictionary dictionary]; + synth[@"schema"] = @"openphone.model_decision.v2"; + synth[@"tool"] = action[@"tool"] ?: @""; + synth[@"arguments"] = action[@"arguments"] ?: @{}; + synth[@"expected_visible_change"] = @"progress toward goal"; + synth[@"confidence"] = @(0.95); + synth[@"assistant_message"] = routerReply ?: @""; + synth[@"reasoning"] = @"pre-planned action from router"; + rawDecision = synth; + OPRecordTrajectory(taskId, @"router_action_dequeued", @{ + @"step": @(step), + @"tool": synth[@"tool"], + @"queue_remaining": @(routerActionQueue.count) + }); + } else { + // Loop guidance: tell the model exactly what it did last and, + // if that action already succeeded, that it should finish now. + NSString *guidance = @""; + if ([lastModelTool isEqualToString:@"get_screen"]) { + guidance = @"HARD RULE: last_tool was get_screen. You MUST choose an action tool (tap, tap_element, type_text, open_url, open_app, swipe, home, finish_task, fail_task) this step. Do NOT choose get_screen again — the observation is already in this prompt."; + } else if (lastModelTool.length > 0) { + NSString *lastState = [lastToolResult[@"state"] isKindOfClass:[NSString class]] + ? lastToolResult[@"state"] : @""; + if ([lastState isEqualToString:@"action.executed"] || + [lastState isEqualToString:@"task.finished"]) { + guidance = [NSString stringWithFormat: + @"HARD RULE: last_tool was %@ and it EXECUTED SUCCESSFULLY. Do NOT repeat the same action. The user's request is now satisfied by that action. Call finish_task with a short human summary of what you did.", + lastModelTool]; + } + } + NSDictionary *loopState = @{ + @"step": @(step), + @"last_tool": lastModelTool ?: @"", + @"last_tool_result": OPModelToolResultSummary(lastToolResult), + @"last_screen": OPModelScreenSummary(screen ?: @{}), + @"guidance": guidance + }; + promptContext = step == 1 ? promptContext + : OPModelPromptContext(goal, taskId, screen, modelStatus, approved, loopState); + NSDictionary *brokerRequest = @{ + @"schema": @"openphone.model_request.v1", + @"task_id": taskId ?: @"", + @"goal": goal ?: @"", + @"step": @(step), + @"autonomy_mode": @"yolo", + @"decision_schema": @"openphone.model_decision.v1", + @"model": modelStatus[@"model"] ?: @"", + @"mode": modelStatus[@"mode"] ?: @"broker", + @"tools": OPModelToolNames(), + @"context": promptContext ?: @{}, + @"instructions": @"Return exactly one JSON object matching openphone.model_decision.v1. Do not include prose outside JSON." + }; + OPRecordTrajectory(taskId, @"model_request", @{ + @"step": @(step), + @"provider": modelStatus[@"mode"] ?: @"broker", + @"model": modelStatus[@"model"] ?: @"", + @"timeout_ms": modelStatus[@"timeout_ms"] ?: @30000, + @"request_schema": @"openphone.model_request.v1", + @"endpoint_configured": modelStatus[@"endpoint_configured"] ?: @NO, + @"context": OPModelPromptContextTraceSummary(promptContext ?: @{}) + }); + // Publish an island update BEFORE the blocking Bedrock call so the + // user sees "Thinking · N/25" immediately instead of a frozen pill. + OPIslandUpdate(@{ + @"mode": @"thinking", + @"subtitle": @"Thinking", + @"step": @(step), + @"max_steps": @(maxSteps), + @"task_id": taskId ?: @"", + @"accent": @"blue" + }); + NSDictionary *brokerResponse = OPModelProviderDecision(modelStatus, brokerRequest); + OPRecordTrajectory(taskId, @"model_response", @{ + @"step": @(step), + @"provider": modelStatus[@"mode"] ?: @"broker", + @"status": brokerResponse[@"status"] ?: @"unknown", + @"http_status": brokerResponse[@"http_status"] ?: @0, + @"response_bytes": brokerResponse[@"response_bytes"] ?: @0, + @"usage": brokerResponse[@"usage"] ?: @{}, + @"metadata": brokerResponse[@"metadata"] ?: @{}, + @"reason": brokerResponse[@"reason"] ?: @"" + }); + if (![brokerResponse[@"status"] isEqualToString:@"ok"]) { + toolErrors += 1; + lastToolResult = brokerResponse ?: @{}; + stopReason = @"model_provider_error"; + break; + } + rawDecision = brokerResponse[@"decision"]; + } + + if (OPTaskCancellationRequested(taskId, &cancelReason)) { + cancelled = YES; + stopReason = @"cancelled"; + break; + } + + NSString *parseError = nil; + NSString *repairStrategy = nil; + NSDictionary *decision = [rawDecision isKindOfClass:[NSString class]] + ? OPModelDecisionFromString(rawDecision, &parseError, &repairStrategy) + : OPModelDecisionFromObject(rawDecision, &parseError); + if (!decision) { + parserFailures += 1; + OPRecordTrajectory(taskId, @"model_parse_error", @{ + @"step": @(step), + @"provider": hasFixture ? @"fixture" : (modelStatus[@"mode"] ?: @"broker"), + @"reason": parseError ?: @"parse_failed" + }); + if (parserFailures >= 2) { + stopReason = @"parser_failure_limit"; + break; + } + continue; + } + if (repairStrategy.length > 0) { + OPRecordTrajectory(taskId, @"model_parse_repaired", @{ + @"step": @(step), + @"provider": hasFixture ? @"fixture" : (modelStatus[@"mode"] ?: @"broker"), + @"strategy": repairStrategy, + @"source_type": @"string", + @"source_length": @([(NSString *)rawDecision length]) + }); + } + + // Router: if the model returned a v3 mode-router decision, translate + // it into the internal single-tool decision + optionally queue more + // actions to run without another model call. + NSString *rMode = [decision[@"mode"] isKindOfClass:[NSString class]] + ? decision[@"mode"] : @""; + NSString *rSchema = [decision[@"schema"] isKindOfClass:[NSString class]] + ? decision[@"schema"] : @""; + BOOL isRouter = rMode.length > 0 || [rSchema isEqualToString:@"openphone.model_decision.v3"]; + if (isRouter) { + NSString *reply = [decision[@"reply"] isKindOfClass:[NSString class]] + ? decision[@"reply"] : @""; + NSArray *proposed = [decision[@"proposed_actions"] isKindOfClass:[NSArray class]] + ? decision[@"proposed_actions"] : @[]; + routerReply = reply; + OPRecordTrajectory(taskId, @"router_decision", @{ + @"step": @(step), + @"mode": rMode ?: @"", + @"reply": reply ?: @"", + @"proposed_action_count": @(proposed.count) + }); + if ([rMode isEqualToString:@"answer"] || [rMode isEqualToString:@"clarify"] || + proposed.count == 0) { + // Single-turn: fill finish_task and go. + NSMutableDictionary *synth = [NSMutableDictionary dictionary]; + synth[@"schema"] = @"openphone.model_decision.v2"; + synth[@"tool"] = @"finish_task"; + synth[@"arguments"] = @{@"summary": reply.length > 0 ? reply : @"Done."}; + synth[@"assistant_message"] = reply; + synth[@"expected_visible_change"] = @"none"; + synth[@"confidence"] = @(0.95); + decision = synth; + } else if ([rMode isEqualToString:@"stop"]) { + NSMutableDictionary *synth = [NSMutableDictionary dictionary]; + synth[@"schema"] = @"openphone.model_decision.v2"; + synth[@"tool"] = @"fail_task"; + synth[@"arguments"] = @{@"reason": reply.length > 0 ? reply : @"Stopped."}; + synth[@"assistant_message"] = reply; + synth[@"expected_visible_change"] = @"none"; + synth[@"confidence"] = @(0.95); + decision = synth; + } else { + // mode=act: pull first proposed_action as this step's decision, + // queue the rest, and ALWAYS append a synthetic finish_task at + // the end so the loop terminates after the last planned action + // — no re-prompting the model, which is where loops happen. + NSDictionary *first = proposed.firstObject; + if ([first isKindOfClass:[NSDictionary class]]) { + NSMutableDictionary *synth = [NSMutableDictionary dictionary]; + synth[@"schema"] = @"openphone.model_decision.v2"; + synth[@"tool"] = [first[@"tool"] isKindOfClass:[NSString class]] ? first[@"tool"] : @""; + synth[@"arguments"] = [first[@"arguments"] isKindOfClass:[NSDictionary class]] ? first[@"arguments"] : @{}; + synth[@"assistant_message"] = reply; + synth[@"expected_visible_change"] = @"progress toward goal"; + synth[@"confidence"] = @(0.95); + decision = synth; + [routerActionQueue removeAllObjects]; + for (NSUInteger i = 1; i < proposed.count; i++) { + id a = proposed[i]; + if ([a isKindOfClass:[NSDictionary class]]) { + [routerActionQueue addObject:a]; + } + } + // Sentinel finish_task at end of the queue. + NSString *finishMsg = reply.length > 0 ? reply : @"Done."; + [routerActionQueue addObject:@{ + @"tool": @"finish_task", + @"arguments": @{@"summary": finishMsg} + }]; + } + } + } + + NSDictionary *guardrail = nil; + decision = OPModelDecisionByApplyingGuardrails(decision, goal, step, &guardrail); + NSString *tool = decision[@"tool"] ?: @""; + // NOTE: the aggressive "one-action-then-finish" guard was removed here + // — the router handles single-vs-multi-step upfront via + // proposed_actions[]. If we ever want to add it back for legacy + // decisions, only trigger when no routerActionQueue was ever populated + // (i.e. the model refused the router pattern). + OPUpdateTask(taskId, @"active", @{ + @"model_loop_current": @{ + @"status": @"decision_received", + @"step": @(step), + @"max_steps": @(maxSteps), + @"tool": tool ?: @"", + @"updated_at_ms": @(OPNowMs()) + } + }); + OPIslandPublishToolStep(taskId, tool, @"decision_received", step, maxSteps); + { + NSString *msg = [decision[@"assistant_message"] isKindOfClass:[NSString class]] + ? decision[@"assistant_message"] : @""; + if (msg.length == 0) { + // Fall back to arguments.summary / .answer / .reply so the user + // still hears the model's words even if it forgot the top-level + // assistant_message field. + NSDictionary *args = [decision[@"arguments"] isKindOfClass:[NSDictionary class]] + ? decision[@"arguments"] : @{}; + for (NSString *key in @[@"summary", @"answer", @"reply", @"response", @"message", @"text"]) { + NSString *v = OPStringFromRequest(args, key, @""); + if (v.length > 0) { msg = v; break; } + } + } + if (msg.length > 0) OPIslandPublishAssistantMessage(msg, taskId); + } + OPRecordTrajectory(taskId, @"model_decision", @{ + @"step": @(step), + @"provider": hasFixture ? @"fixture" : (modelStatus[@"mode"] ?: @"broker"), + @"decision": decision + }); + if (guardrail.count > 0) { + OPRecordTrajectory(taskId, @"model_decision_guardrail", guardrail); + } + // Track consecutive read-only observations. A page can take a beat to + // render (App Store loading, Safari navigation) so let the model + // re-observe up to 3 times before failing. + if ([tool isEqualToString:@"get_screen"]) { + consecutiveGetScreenCount += 1; + } else { + consecutiveGetScreenCount = 0; + } + if (consecutiveGetScreenCount >= 4) { + stopReason = @"repeated_read_only_observation"; + lastToolResult = @{ + @"status": @"ok", + @"state": @"task.failed", + @"reason": stopReason, + @"tool": tool, + @"task_id": taskId ?: @"", + @"source": @"openphone.agentd" + }; + OPRecordTrajectory(taskId, @"model_no_progress", @{ + @"step": @(step), + @"tool": tool, + @"consecutive_get_screen": @(consecutiveGetScreenCount), + @"reason": stopReason, + @"last_tool_result": OPModelToolResultSummary(lastToolResult) + }); + break; + } + // Same-action loop guard: if the model picks the identical tool+args + // twice in a row (e.g. open_url with the same URL), force finish. The + // action already succeeded once; repeating it is dead loop. + NSDictionary *decArgs = [decision[@"arguments"] isKindOfClass:[NSDictionary class]] + ? decision[@"arguments"] : @{}; + NSData *argsData = [NSJSONSerialization dataWithJSONObject:decArgs + options:NSJSONWritingSortedKeys + error:nil]; + NSString *argsSig = argsData ? [[NSString alloc] initWithData:argsData + encoding:NSUTF8StringEncoding] : @""; + NSString *sig = [NSString stringWithFormat:@"%@|%@", tool ?: @"", argsSig ?: @""]; + if ([sig isEqualToString:lastDecisionSig] && + ![tool isEqualToString:@"finish_task"] && ![tool isEqualToString:@"fail_task"] && + ![tool isEqualToString:@"get_screen"]) { + lastDecisionRepeats += 1; + } else { + lastDecisionRepeats = 0; + } + lastDecisionSig = sig; + if (lastDecisionRepeats >= 1) { + // 2nd identical action in a row → treat previous run as successful + // and finish. Better UX than looping forever. + NSString *msg = @"Done."; + NSString *url = [decArgs[@"url"] isKindOfClass:[NSString class]] + ? decArgs[@"url"] : @""; + if (url.length > 0) { + msg = [NSString stringWithFormat:@"Opened %@.", url]; + } + OPRecordTrajectory(taskId, @"model_loop_guard_same_action", @{ + @"step": @(step), + @"tool": tool, + @"arguments": decArgs, + @"reason": @"same_action_repeated" + }); + lastToolResult = @{ + @"status": @"ok", + @"state": @"task.finished", + @"task_id": taskId ?: @"", + @"summary": msg, + @"source": @"openphone.agentd" + }; + terminal = YES; + succeeded = YES; + stopReason = @"same_action_repeated"; + break; + } + OPRecordTrajectory(taskId, @"tool_call", @{ + @"tool": tool, + @"step": @(step), + @"arguments": decision[@"arguments"] ?: @{} + }); + OPUpdateTask(taskId, @"active", @{ + @"model_loop_current": @{ + @"status": @"tool_running", + @"step": @(step), + @"max_steps": @(maxSteps), + @"tool": tool ?: @"", + @"updated_at_ms": @(OPNowMs()) + } + }); + OPIslandPublishToolStep(taskId, tool, @"tool_running", step, maxSteps); + + // Autonomy modes: reviewed pauses on UI-driving tools; dry_run refuses + // outright. yolo (default) proceeds as before. + NSString *autonomy = OPAutonomyMode(); + if (![autonomy isEqualToString:@"yolo"] && OPModelToolDrivesUI(tool)) { + if ([autonomy isEqualToString:@"dry_run"]) { + lastToolResult = @{ + @"status": @"ok", + @"state": @"action.denied", + @"reason": @"dry_run_mode", + @"tool": tool, + @"task_id": taskId ?: @"", + @"source": @"openphone.agentd" + }; + OPRecordTrajectory(taskId, @"tool_denied_dry_run", @{@"tool": tool}); + stopReason = @"dry_run_denied"; + break; + } + // reviewed + NSString *thought = [decision[@"thought"] isKindOfClass:[NSString class]] + ? decision[@"thought"] : @""; + NSString *summary = thought.length > 0 ? thought + : [NSString stringWithFormat:@"Run %@?", tool]; + BOOL approvedNow = OPRequestUserConfirmation(taskId, tool, summary, 30.0); + if (!approvedNow) { + lastToolResult = @{ + @"status": @"ok", + @"state": @"action.denied", + @"reason": @"user_denied", + @"tool": tool, + @"task_id": taskId ?: @"", + @"source": @"openphone.agentd" + }; + OPRecordTrajectory(taskId, @"tool_denied_by_user", @{@"tool": tool}); + stopReason = @"user_denied"; + break; + } + // Reset island back to action mode after approval. + OPIslandPublishToolStep(taskId, tool, @"tool_running", step, maxSteps); + } + + // Step-1 get_screen is wasted work: the current screen is already in + // the step-1 prompt. Skip the second observation, mark last_tool as + // get_screen so the loop's guidance forces an action on step 2. + NSDictionary *toolResult; + if (step == 1 && [tool isEqualToString:@"get_screen"]) { + toolResult = @{ + @"status": @"ok", + @"state": @"observation.reused_from_step1_prompt", + @"tool": @"get_screen", + @"task_id": taskId ?: @"", + @"reason": @"screen already observed in step 1 prompt", + @"source": @"openphone.agentd" + }; + OPRecordTrajectory(taskId, @"model_step1_get_screen_shortcircuit", @{ + @"step": @(step), + @"tool": tool + }); + } else { + toolResult = OPModelExecuteDecision(decision, taskId, approved); + } + lastToolResult = toolResult ?: @{}; + NSString *state = [toolResult[@"state"] isKindOfClass:[NSString class]] + ? toolResult[@"state"] : @""; + BOOL toolOK = [toolResult[@"status"] isEqualToString:@"ok"] || + [state isEqualToString:@"action.executed"] || + [state isEqualToString:@"task.finished"] || + [state isEqualToString:@"task.failed"]; + if (!toolOK || [state hasPrefix:@"action.denied"] || + [toolResult[@"status"] isEqualToString:@"error"]) { + toolErrors += 1; + } + + NSDictionary *afterScreen = @{}; + NSDictionary *verification = @{@"status": @"not_required", @"reason": @"non_ui_tool"}; + BOOL skippedPostActionScreen = NO; + if (OPModelToolDrivesUI(tool)) { + if (!toolOK) { + skippedPostActionScreen = YES; + verification = @{ + @"status": @"failed", + @"reason": toolResult[@"detail"] ?: toolResult[@"reason"] ?: @"tool_failed", + @"source": @"tool_result", + @"expected_visible_change": decision[@"expected_visible_change"] ?: @"", + @"screen_state": screen[@"state"] ?: @"" + }; + } else if (OPModelShouldUseProviderVerificationOnly(tool, toolResult)) { + skippedPostActionScreen = YES; + verification = OPModelProviderVerificationState(tool, toolResult, screen, + decision[@"expected_visible_change"] ?: @""); + } else { + afterScreen = OPModelCompactScreenForLoop(OPGetScreen(@{ + @"task_id": taskId ?: @"", + @"include_screenshot": @YES, + @"include_activity": @YES, + @"include_ui_tree": @YES, + @"compact_trajectory": @YES, + @"reason": @"model run_task post-action verification" + })); + verification = OPModelVerificationState(tool, toolResult, screen, afterScreen, + decision[@"expected_visible_change"] ?: @""); + } + if ([verification[@"status"] isEqualToString:@"unverified_dispatch_only"]) { + unverifiedUIActions += 1; + } + } + if (OPTaskCancellationRequested(taskId, &cancelReason)) { + cancelled = YES; + stopReason = @"cancelled"; + } + OPRecordTrajectory(taskId, @"model_step_verified", @{ + @"step": @(step), + @"tool": tool, + @"tool_result": OPModelToolResultSummary(toolResult ?: @{}), + @"verification": OPModelVerificationTraceSummary(verification), + @"before_screen": screen.count > 0 ? OPModelScreenTraceSummary(screen) : @{}, + @"after_screen": (!skippedPostActionScreen && afterScreen.count > 0) + ? OPModelScreenTraceSummary(afterScreen) : @{} + }); + OPUpdateTask(taskId, @"active", @{ + @"model_loop_current": @{ + @"status": @"step_verified", + @"step": @(step), + @"max_steps": @(maxSteps), + @"tool": tool ?: @"", + @"verification": verification[@"status"] ?: @"not_required", + @"updated_at_ms": @(OPNowMs()) + } + }); + if (cancelled) { + break; + } + lastModelTool = tool; + + if (OPModelVerifiedTypeTextCompletesGoal(goal, decision, verification)) { + terminal = YES; + succeeded = YES; + stopReason = @"verified_type_text_goal_complete"; + lastToolResult = @{ + @"status": @"ok", + @"state": @"task.finished", + @"task_id": taskId ?: @"", + @"summary": @"Verified text entry completed.", + @"reason": stopReason, + @"action_result": OPModelToolResultSummary(toolResult ?: @{}), + @"source": @"openphone.agentd" + }; + break; + } + if ([tool isEqualToString:@"finish_task"]) { + terminal = YES; + succeeded = YES; + stopReason = @"finish_task"; + break; + } + if ([tool isEqualToString:@"fail_task"]) { + terminal = YES; + succeeded = NO; + stopReason = @"fail_task"; + break; + } + if (toolErrors >= 2) { + stopReason = @"tool_error_limit"; + break; + } + if (unverifiedUIActions >= 2) { + stopReason = @"no_visible_progress"; + break; + } + // Screen-signature no-progress detection: if we ran a UI-driving tool + // and the screen didn't materially change, count as "no progress". + // Two consecutive no-progress steps → terminate as done. + if (OPModelToolDrivesUI(tool) && afterScreen.count > 0) { + NSString *newSig = OPModelScreenSignature(afterScreen); + if (routerLastScreenSignature.length > 0 && + [newSig isEqualToString:routerLastScreenSignature]) { + consecutiveNoProgress += 1; + } else { + consecutiveNoProgress = 0; + } + routerLastScreenSignature = newSig; + if (consecutiveNoProgress >= 2) { + OPRecordTrajectory(taskId, @"model_no_progress_by_signature", @{ + @"step": @(step), + @"signature": newSig + }); + terminal = YES; + succeeded = YES; + stopReason = @"no_progress_signature"; + if (routerReply.length > 0) { + lastToolResult = @{ + @"status": @"ok", + @"state": @"task.finished", + @"task_id": taskId ?: @"", + @"summary": routerReply, + @"source": @"openphone.agentd" + }; + } + break; + } + } + screen = afterScreen.count > 0 ? afterScreen : screen; + } // @autoreleasepool + } + + if (cancelled) { + lastToolResult = @{ + @"status": @"ok", + @"state": @"task.cancelled", + @"reason": cancelReason.length > 0 ? cancelReason : @"cancelled", + @"task_id": taskId ?: @"", + @"source": @"openphone.agentd" + }; + succeeded = NO; + } else if (!terminal) { + if ([stopReason isEqualToString:@"unknown"]) { + stopReason = stepsUsed >= maxSteps ? @"step_limit" : @"model_stopped"; + } + NSDictionary *failure = OPFailTask(@{ + @"task_id": taskId ?: @"", + @"reason": stopReason + }); + lastToolResult = failure ?: lastToolResult; + succeeded = NO; + } + + long long durationMs = OPNowMs() - startedMs; + NSDictionary *summary = @{ + @"status": succeeded ? @"task.finished" : (cancelled ? @"task.cancelled" : @"task.failed"), + @"goal": goal ?: @"", + @"task_id": taskId ?: @"", + @"runner": @"model", + @"model_provider": hasFixture ? @"fixture" : (modelStatus[@"mode"] ?: @"broker"), + @"limits": limits, + @"steps_used": @(stepsUsed), + @"parser_failures": @(parserFailures), + @"tool_errors": @(toolErrors), + @"unverified_ui_actions": @(unverifiedUIActions), + @"duration_ms": @(durationMs), + @"stop_reason": stopReason ?: @"unknown", + @"cancel_reason": cancelled ? (cancelReason.length > 0 ? cancelReason : @"cancelled") : @"", + @"last_tool_result": lastToolResult ?: @{}, + @"trajectory": OPTrajectoryPath(taskId ?: @""), + @"source": @"openphone.agentd" + }; + OPUpdateTask(taskId, succeeded ? @"completed" : (cancelled ? @"stopped" : @"failed"), @{ + @"result": lastToolResult ?: @{}, + @"model_loop_summary": summary, + @"model_loop_current": @{ + @"status": summary[@"status"] ?: @"task.finished", + @"step": @(stepsUsed), + @"max_steps": @(maxSteps), + @"tool": lastModelTool ?: @"", + @"stop_reason": stopReason ?: @"unknown", + @"updated_at_ms": @(OPNowMs()) + }, + @"completed_at": @(OPNowMs()), + @"cancel_requested": cancelled ? @YES : @NO, + @"cancel_reason": cancelled ? (cancelReason.length > 0 ? cancelReason : @"cancelled") : @"" + }); + OPRecordAudit(succeeded ? @"model_task_finished" : + (cancelled ? @"model_task_cancelled" : @"model_task_failed"), taskId, + @"tasks.observe", succeeded ? @"allow_task_scoped" : (cancelled ? @"cancelled" : @"failed"), + @{ + @"command": @"run_task", + @"mode": @"model", + @"goal": goal ?: @"", + @"status": summary[@"status"] ?: @"unknown", + @"stop_reason": stopReason ?: @"unknown", + @"model_provider": summary[@"model_provider"] ?: @"unknown", + @"steps_used": summary[@"steps_used"] ?: @0 + }, + stopReason ?: @"unknown"); + if (cancelled) { + OPRecordTrajectory(taskId, @"model_loop_cancelled", summary); + } + OPRecordTrajectory(taskId, @"model_loop_finished", summary); + + // Publish terminal island state now that the loop is truly done. The + // per-tool OPFinishTask / OPFailTask handlers only fire for explicit + // tool calls, not for the model's finish_task decision — the loop just + // sets succeeded=YES and returns. Without this, the island stays stuck + // in "action / Finishing" forever. + NSString *terminalMsg = @""; + if (lastToolResult && [lastToolResult isKindOfClass:[NSDictionary class]]) { + for (NSString *key in @[@"summary", @"answer", @"reply", @"message", @"text", @"reason"]) { + NSString *v = OPStringFromRequest(lastToolResult, key, @""); + if (v.length > 0) { terminalMsg = v; break; } + } + } + if (terminalMsg.length == 0) { + terminalMsg = succeeded ? @"Done." : (cancelled ? @"Cancelled" : @"Couldn't finish that."); + } + OPIslandPublishTerminal(taskId, succeeded, terminalMsg); + OPRecordAssistantTurn(terminalMsg, taskId, succeeded); + + return summary; +} + +static NSDictionary *OPRunDeterministicTask(NSDictionary *request) { + long long startedMs = OPNowMs(); + NSString *goal = [request[@"goal"] isKindOfClass:[NSString class]] ? request[@"goal"] : @""; + long long maxSteps = OPLongLongFromRequest(request, @"max_steps", 1, 1, 25); + long long maxDurationMs = OPLongLongFromRequest(request, @"max_duration_ms", 60000, 1000, 600000); + BOOL includeScreenshot = OPBoolFromRequest(request, @"include_screenshot", YES); + NSDictionary *limits = @{ + @"max_steps": @(maxSteps), + @"max_duration_ms": @(maxDurationMs), + @"max_tool_errors": @1, + @"no_progress_detection": @"pending_screen_provider", + @"include_screenshot": @(includeScreenshot) + }; + NSDictionary *task = OPStartTask(@{@"goal": goal, @"approved_capabilities": OPFullYoloCapabilities()}); + NSString *taskId = task[@"task_id"]; + OPUpdateTask(taskId, @"active", @{ + @"runner": @"deterministic", + @"runner_pid": @(getpid()), + @"runner_started_at": @(OPNowMs()), + @"limits": limits + }); + NSDictionary *beforeScreen = OPGetScreen(@{ + @"task_id": taskId ?: @"", + @"include_screenshot": @(includeScreenshot), + @"include_activity": @YES, + @"include_ui_tree": @YES, + @"reason": @"deterministic run_task preflight" + }); + NSDictionary *action = OPActionForGoal(goal); + NSDictionary *recordedAction = OPActionForRecording(action); + long long stepsUsed = 1; + OPRecordTrajectory(taskId, @"tool_call", @{ + @"tool": @"execute_action", + @"step": @(stepsUsed), + @"arguments": recordedAction + }); + NSDictionary *actionResult = OPExecuteAction(@{@"task_id": taskId ?: @"", @"action": action}); + NSDictionary *afterScreen = OPGetScreen(@{ + @"task_id": taskId ?: @"", + @"include_screenshot": @(includeScreenshot), + @"include_activity": @YES, + @"include_ui_tree": @YES, + @"reason": @"deterministic run_task verification" + }); + BOOL executed = [actionResult[@"state"] isEqualToString:@"action.executed"]; + long long durationMs = OPNowMs() - startedMs; + BOOL durationLimited = durationMs > maxDurationMs; + long long toolErrors = executed ? 0 : 1; + NSString *stopReason = executed ? @"finished" : @"action_failed"; + if (durationLimited) { + stopReason = @"duration_limit"; + } else if (stepsUsed >= maxSteps && !executed) { + stopReason = @"step_limit_or_action_failed"; + } + BOOL succeeded = executed && !durationLimited; + NSString *finalStatus = succeeded ? @"completed" : @"failed"; + NSDictionary *summary = @{ + @"status": succeeded ? @"task.finished" : @"task.failed", + @"goal": goal, + @"task_id": taskId ?: @"", + @"runner": @"deterministic", + @"limits": limits, + @"steps_used": @(stepsUsed), + @"tool_errors": @(toolErrors), + @"duration_ms": @(durationMs), + @"stop_reason": stopReason, + @"action": recordedAction, + @"action_result": actionResult, + @"before_screen_state": beforeScreen[@"state"] ?: @"", + @"after_screen_state": afterScreen[@"state"] ?: @"", + @"trajectory": OPTrajectoryPath(taskId ?: @""), + @"source": @"openphone.agentd" + }; + OPUpdateTask(taskId, finalStatus, @{ + @"result": summary, + @"completed_at": @(OPNowMs()) + }); + OPRecordAudit(succeeded ? @"task_finished" : @"task_failed", taskId, @"tasks.observe", + succeeded ? @"allow_task_scoped" : @"failed", request, stopReason); + OPRecordTrajectory(taskId, succeeded ? @"task_finished" : @"task_failed", summary); + return summary; +} + +static NSString *OPEffectiveRunTaskMode(NSDictionary *request) { + NSString *mode = [OPStringFromRequest(request, @"mode", + OPStringFromRequest(request, @"runner", @"auto")) lowercaseString]; + if (mode.length == 0) { + mode = @"auto"; + } + if ([mode isEqualToString:@"auto"]) { + NSDictionary *modelStatus = OPModelStatusDictionary(); + NSArray *fixtureDecisions = [request[@"model_decisions"] isKindOfClass:[NSArray class]] + ? request[@"model_decisions"] : @[]; + mode = ([modelStatus[@"status"] isEqualToString:@"ready"] || fixtureDecisions.count > 0) + ? @"model" : @"deterministic"; + } + return mode; +} + +static NSDictionary *OPRunTask(NSDictionary *request) { + NSString *mode = OPEffectiveRunTaskMode(request ?: @{}); + if ([mode isEqualToString:@"model"]) { + NSDictionary *modelStatus = OPModelStatusDictionary(); + NSArray *fixtureDecisions = [request[@"model_decisions"] isKindOfClass:[NSArray class]] + ? request[@"model_decisions"] : @[]; + NSString *providerMode = [modelStatus[@"mode"] isKindOfClass:[NSString class]] + ? modelStatus[@"mode"] : @"broker"; + if (fixtureDecisions.count == 0 && OPModelModeIsOpenAIRealtime(providerMode)) { + return OPRunOpenAIRealtimeTask(request); + } + return OPRunModelTask(request); + } + if (![mode isEqualToString:@"deterministic"]) { + return OPError(@"invalid_run_task_mode"); + } + return OPRunDeterministicTask(request); +} + +static void *OPAsyncRunTaskMain(void *context) { + @autoreleasepool { + NSDictionary *request = (__bridge_transfer NSDictionary *)context; + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *source = OPStringFromRequest(request, @"source", @"async_run_task"); + NSString *goal = OPStringFromRequest(request, @"goal", @""); + OPLog(@"async run_task started task_id=%@ source=%@ mode=%@", + taskId ?: @"", source ?: @"", request[@"mode"] ?: @"auto"); + NSDictionary *result = OPRunTask(request ?: @{}); + NSString *resultTaskId = [result[@"task_id"] isKindOfClass:[NSString class]] + ? result[@"task_id"] : taskId; + NSString *status = [result[@"status"] isKindOfClass:[NSString class]] + ? result[@"status"] : @"unknown"; + OPRecordContextEvent(@"async_run_task_finished", @"openphone.agentd", + resultTaskId ?: @"", goal ?: @"", status ?: @"unknown", @{ + @"source": source ?: @"async_run_task", + @"mode": request[@"mode"] ?: @"auto", + @"runner": result[@"runner"] ?: @"unknown", + @"stop_reason": result[@"stop_reason"] ?: @"" + }); + OPRecordAudit(@"async_run_task_finished", resultTaskId ?: @"", @"tasks.observe", + [status isEqualToString:@"task.finished"] ? @"allow_task_scoped" : @"failed", + request ?: @{}, status ?: @"unknown"); + OPLog(@"async run_task finished task_id=%@ status=%@ runner=%@ stop_reason=%@", + resultTaskId ?: @"", status ?: @"unknown", + result[@"runner"] ?: @"unknown", result[@"stop_reason"] ?: @""); + } + return NULL; +} + +static NSDictionary *OPStartAsyncRunTask(NSDictionary *request) { + pthread_t thread; + NSDictionary *ownedRequest = [request copy] ?: @{}; + int rc = pthread_create(&thread, NULL, OPAsyncRunTaskMain, + (__bridge_retained void *)ownedRequest); + if (rc != 0) { + return @{ + @"status": @"error", + @"reason": [NSString stringWithFormat:@"pthread_create_failed:%d", rc], + @"source": @"openphone.agentd" + }; + } + pthread_detach(thread); + return @{ + @"status": @"ok", + @"state": @"task.started_async", + @"task_id": ownedRequest[@"task_id"] ?: @"", + @"mode": ownedRequest[@"mode"] ?: @"auto", + @"runner": @"async_agent_loop", + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPHardwareTrigger(NSDictionary *request) { + NSString *trigger = OPStringFromRequest(request, @"trigger", @"hardware"); + NSString *source = OPStringFromRequest(request, @"source", @"springboard"); + NSString *reason = OPStringFromRequest(request, @"reason", @"hardware trigger received"); + NSString *taskId = [NSString stringWithFormat:@"ios-trigger-%lld-%d", OPNowMs(), getpid()]; + NSDictionary *control = OPAgentControlSummary(); + BOOL controlAllowsTrigger = ![control[@"paused"] boolValue] && + [control[@"hardware_triggers_enabled"] boolValue] && + [control[@"yolo_enabled"] boolValue]; + if (!controlAllowsTrigger && !OPBoolFromRequest(request, @"bypass_agent_control", NO)) { + NSString *state = [control[@"paused"] boolValue] ? @"trigger.paused" + : (![control[@"hardware_triggers_enabled"] boolValue] + ? @"trigger.disabled" : @"trigger.yolo_disabled"); + NSDictionary *result = @{ + @"status": @"ok", + @"state": state, + @"trigger": trigger ?: @"hardware", + @"task_id": taskId, + @"source": @"openphone.agentd", + @"runtime_authority": @"phone_local", + @"model_loop_status": @"paused", + @"control": control, + @"deduped": @NO + }; + OPRecordContextEvent(@"hardware_trigger_suppressed", source, taskId, + trigger, state, result); + OPRecordAudit(@"hardware_trigger_suppressed", taskId, @"background.run", + state, request, [NSString stringWithFormat:@"trigger:%@ control:%@", + trigger, control[@"trigger_policy"] ?: @"unknown"]); + OPRecordTrajectory(taskId, @"hardware_trigger_suppressed", result); + return result; + } + long long now = OPNowMs(); + BOOL dedupe = OPBoolFromRequest(request, @"dedupe", YES); + long long cooldownMs = OPLongLongFromRequest(request, @"cooldown_ms", 10000, 0, 60000); + if (dedupe && cooldownMs > 0) { + pthread_mutex_lock(&OPHardwareTriggerMutex); + long long ageMs = OPHardwareTriggerLastAcceptedMs > 0 + ? MAX(0, now - OPHardwareTriggerLastAcceptedMs) : cooldownMs + 1; + BOOL duplicate = OPHardwareTriggerLastAcceptedMs > 0 && ageMs < cooldownMs; + if (!duplicate) { + OPHardwareTriggerLastAcceptedMs = now; + } + pthread_mutex_unlock(&OPHardwareTriggerMutex); + if (duplicate) { + NSDictionary *result = @{ + @"status": @"ok", + @"state": @"trigger.ignored_duplicate", + @"trigger": trigger ?: @"hardware", + @"task_id": taskId, + @"source": @"openphone.agentd", + @"runtime_authority": @"phone_local", + @"model_loop_status": @"suppressed_duplicate", + @"deduped": @YES, + @"cooldown_ms": @(cooldownMs), + @"last_trigger_age_ms": @(ageMs) + }; + OPRecordContextEvent(@"hardware_trigger_suppressed", source, taskId, + trigger, @"duplicate hardware trigger suppressed", result); + OPRecordAudit(@"hardware_trigger_suppressed", taskId, @"background.run", + @"allow_yolo", request, [NSString stringWithFormat:@"trigger:%@ duplicate_age_ms:%lld", + trigger, ageMs]); + OPRecordTrajectory(taskId, @"hardware_trigger_suppressed", result); + return result; + } + } + + long long preObserveDelayMs = OPLongLongFromRequest(request, + @"pre_observe_delay_ms", 0, 0, 5000); + if (preObserveDelayMs > 0) { + OPLog(@"hardware trigger pre-observe delay ms=%lld source=%@ trigger=%@", + preObserveDelayMs, source ?: @"", trigger ?: @""); + usleep((useconds_t)(preObserveDelayMs * 1000)); + } + + NSDictionary *screen = OPGetScreen(@{ + @"task_id": taskId, + @"include_screenshot": @NO, + @"include_activity": @YES, + @"include_ui_tree": @YES, + @"reason": reason + }); + long long contextId = OPRecordContextEvent(@"hardware_trigger", source, taskId, + trigger, reason, @{ + @"trigger": trigger ?: @"hardware", + @"screen_state": screen[@"state"] ?: @"", + @"foreground_app": screen[@"context"][@"foreground_app"] ?: @"unknown" + }); + OPRecordAudit(@"hardware_trigger", taskId, @"background.run", @"allow_yolo", + request, [NSString stringWithFormat:@"trigger:%@ source:%@", trigger, source]); + + NSDictionary *modelStatus = OPModelStatusDictionary(); + NSMutableDictionary *result = [@{ + @"status": @"ok", + @"trigger": trigger ?: @"hardware", + @"task_id": taskId, + @"context_event_id": @(contextId), + @"screen_state": screen[@"state"] ?: @"", + @"source": @"openphone.agentd", + @"runtime_authority": @"phone_local", + @"model_loop_status": @"not_started", + @"model_status": modelStatus[@"status"] ?: @"unknown" + } mutableCopy]; + + BOOL modelReady = [modelStatus[@"status"] isEqualToString:@"ready"]; + BOOL runTask = OPBoolFromRequest(request, @"run_task", modelReady); + BOOL createBackgroundJob = OPBoolFromRequest(request, @"create_background_job", !runTask); + NSString *goal = OPStringFromRequest(request, @"goal", OPDefaultHardwareTriggerGoal); + if (goal.length == 0 || [goal isEqualToString:OPLegacyHardwareTriggerGoal]) { + goal = OPDefaultHardwareTriggerGoal; + } + if (createBackgroundJob) { + NSDictionary *jobResult = OPBackgroundJobCreate(@{ + @"task_id": taskId, + @"title": @"Hardware volume trigger", + @"prompt": goal, + @"type": @"agent_turn", + @"reason": reason, + @"source": source, + @"payload": @{ + @"trigger": trigger ?: @"hardware", + @"context_event_id": @(contextId), + @"screen_state": screen[@"state"] ?: @"" + } + }); + result[@"background_job"] = jobResult; + if (OPBoolFromRequest(request, @"run_background_jobs", YES) && + [jobResult[@"status"] isEqualToString:@"ok"]) { + NSDictionary *createdJob = [jobResult[@"job"] isKindOfClass:[NSDictionary class]] + ? jobResult[@"job"] : @{}; + id createdJobId = createdJob[@"id"] ?: jobResult[@"job_id"]; + NSMutableDictionary *schedulerRequest = [@{ + @"command": @"background_job_run_due", + @"task_id": taskId, + @"limit": @1, + @"max_steps": @1, + @"max_duration_ms": @10000, + @"source": source, + @"reason": @"hardware trigger scheduler tick", + @"repair_stuck": @NO, + @"materialize_watchers": @NO, + @"run_jobs": @NO + } mutableCopy]; + if ([createdJobId respondsToSelector:@selector(longLongValue)] && + [createdJobId longLongValue] > 0) { + schedulerRequest[@"job_id"] = createdJobId; + } + result[@"scheduler"] = OPBackgroundJobRunDue(schedulerRequest); + } + } + + if (runTask) { + NSString *requestedMode = OPStringFromRequest(request, @"mode", @"auto"); + NSDictionary *modeProbe = @{ + @"mode": requestedMode.length > 0 ? requestedMode : @"auto", + @"goal": goal ?: @"", + @"reason": reason ?: @"hardware trigger" + }; + NSString *effectiveMode = OPEffectiveRunTaskMode(modeProbe); + result[@"requested_run_mode"] = requestedMode.length > 0 ? requestedMode : @"auto"; + result[@"effective_run_mode"] = effectiveMode ?: @"unknown"; + if (![effectiveMode isEqualToString:@"model"]) { + result[@"run_task"] = @{ + @"status": @"error", + @"reason": @"model_provider_not_configured", + @"requested_mode": requestedMode.length > 0 ? requestedMode : @"auto", + @"effective_mode": effectiveMode ?: @"unknown", + @"source": @"openphone.agentd" + }; + result[@"model_loop_status"] = @"provider_not_ready"; + } else { + NSDictionary *task = OPStartTask(@{ + @"goal": goal, + @"approved_capabilities": OPFullYoloCapabilities() + }); + NSString *agentTaskId = [task[@"task_id"] isKindOfClass:[NSString class]] + ? task[@"task_id"] : @""; + NSMutableDictionary *runRequest = [@{ + @"task_id": agentTaskId, + @"goal": goal, + @"reason": reason, + @"mode": @"model", + @"max_steps": @(OPLongLongFromRequest(request, @"max_steps", 5, 1, 25)), + @"max_duration_ms": @(OPLongLongFromRequest(request, @"max_duration_ms", 120000, 1000, 600000)), + @"source": source ?: @"hardware_trigger", + @"trigger_task_id": taskId + } mutableCopy]; + result[@"agent_task_id"] = agentTaskId ?: @""; + result[@"run_task"] = OPStartAsyncRunTask(runRequest); + result[@"model_loop_status"] = [result[@"run_task"][@"status"] isEqualToString:@"ok"] + ? @"started_async" : @"start_failed"; + } + } + + OPRecordTrajectory(taskId, @"hardware_trigger", result); + return result; +} + +typedef struct { + AudioQueueRef queue; + NSMutableData *__unsafe_unretained pcmData; + pthread_mutex_t mutex; + pthread_cond_t cond; + BOOL done; + BOOL heardSpeech; + BOOL stopOnVAD; + long long startedMs; + long long lastSpeechMs; + long long maxMs; + long long minRecordMs; + long long initialSpeechTimeoutMs; + long long endSilenceMs; + double rmsThreshold; + double lastRms; + double peakRms; + UInt32 sampleRate; + char stopReason[64]; +} OPVoiceCaptureState; + +static void OPVoiceSetLast(NSString *state, NSString *transcript, + NSString *error, NSString *provider, BOOL running) { + pthread_mutex_lock(&OPVoiceTriggerMutex); + OPVoiceTriggerRunning = running; + OPVoiceTriggerLastState = [state copy] ?: @"unknown"; + if (transcript != nil) { + OPVoiceTriggerLastTranscript = [transcript copy]; + } + if (error != nil) { + OPVoiceTriggerLastError = [error copy]; + } + if (provider != nil) { + OPVoiceTriggerLastProvider = [provider copy]; + } + if (!running) { + OPVoiceTriggerLastFinishedMs = OPNowMs(); + } + pthread_mutex_unlock(&OPVoiceTriggerMutex); +} + +static NSDictionary *OPVoiceStatus(NSDictionary *request) { + (void)request; + pthread_mutex_lock(&OPVoiceTriggerMutex); + NSDictionary *status = @{ + @"status": @"ok", + @"state": OPVoiceTriggerRunning ? @"voice.listening_or_transcribing" : + (OPVoiceTriggerLastState ?: @"voice.idle"), + @"running": @(OPVoiceTriggerRunning), + @"runtime_authority": @"phone_local", + @"microphone_owner": @"openphone-agentd", + @"default_transcription_provider": @"openai_transcription", + @"apple_speech_fallback": @"explicit_debug_only", + @"last_started_at_ms": @(OPVoiceTriggerLastStartedMs), + @"last_finished_at_ms": @(OPVoiceTriggerLastFinishedMs), + @"last_transcript": OPVoiceTriggerLastTranscript ?: @"", + @"last_error": OPVoiceTriggerLastError ?: @"", + @"last_provider": OPVoiceTriggerLastProvider ?: @"", + @"voice_credential_file": OPVoiceCredentialPath(), + @"audio_path": [OPVoicePath() stringByAppendingPathComponent:@"last-command.wav"], + @"source": @"openphone.agentd" + }; + pthread_mutex_unlock(&OPVoiceTriggerMutex); + return status; +} + +static void OPVoiceSetDoneLocked(OPVoiceCaptureState *state, const char *reason) { + if (!state || state->done) { + return; + } + state->done = YES; + strlcpy(state->stopReason, reason ?: "done", sizeof(state->stopReason)); + pthread_cond_signal(&state->cond); +} + +static double OPVoiceRMSForPCM16(const void *bytes, UInt32 byteCount) { + if (!bytes || byteCount < sizeof(int16_t)) { + return 0.0; + } + const int16_t *samples = (const int16_t *)bytes; + UInt32 count = byteCount / sizeof(int16_t); + double sum = 0.0; + for (UInt32 i = 0; i < count; i++) { + double sample = (double)samples[i]; + sum += sample * sample; + } + return sqrt(sum / (double)count); +} + +static void OPVoiceAudioQueueInputCallback(void *userData, AudioQueueRef queue, + AudioQueueBufferRef buffer, const AudioTimeStamp *startTime, + UInt32 packetCount, const AudioStreamPacketDescription *packetDescriptions) { + (void)startTime; + (void)packetCount; + (void)packetDescriptions; + OPVoiceCaptureState *state = (OPVoiceCaptureState *)userData; + if (!state || !buffer) { + return; + } + + long long now = OPNowMs(); + double rms = OPVoiceRMSForPCM16(buffer->mAudioData, buffer->mAudioDataByteSize); + BOOL shouldRequeue = NO; + pthread_mutex_lock(&state->mutex); + if (!state->done) { + if (buffer->mAudioDataByteSize > 0) { + [state->pcmData appendBytes:buffer->mAudioData + length:buffer->mAudioDataByteSize]; + } + state->lastRms = rms; + if (rms > state->peakRms) { + state->peakRms = rms; + } + if (rms >= state->rmsThreshold) { + state->heardSpeech = YES; + state->lastSpeechMs = now; + } + + long long elapsedMs = MAX(0, now - state->startedMs); + if (elapsedMs >= state->maxMs) { + OPVoiceSetDoneLocked(state, "max_duration"); + } else if (state->stopOnVAD && !state->heardSpeech && + elapsedMs >= state->initialSpeechTimeoutMs) { + OPVoiceSetDoneLocked(state, "initial_speech_timeout"); + } else if (state->stopOnVAD && state->heardSpeech && + elapsedMs >= state->minRecordMs && + state->lastSpeechMs > 0 && + (now - state->lastSpeechMs) >= state->endSilenceMs) { + OPVoiceSetDoneLocked(state, "end_silence"); + } + shouldRequeue = !state->done; + } + if (state->done) { + pthread_cond_signal(&state->cond); + } + pthread_mutex_unlock(&state->mutex); + + if (shouldRequeue) { + AudioQueueEnqueueBuffer(queue, buffer, 0, NULL); + } +} + +static void OPVoiceAppendLE16(NSMutableData *data, uint16_t value) { + uint8_t bytes[2] = { + (uint8_t)(value & 0xff), + (uint8_t)((value >> 8) & 0xff) + }; + [data appendBytes:bytes length:sizeof(bytes)]; +} + +static void OPVoiceAppendLE32(NSMutableData *data, uint32_t value) { + uint8_t bytes[4] = { + (uint8_t)(value & 0xff), + (uint8_t)((value >> 8) & 0xff), + (uint8_t)((value >> 16) & 0xff), + (uint8_t)((value >> 24) & 0xff) + }; + [data appendBytes:bytes length:sizeof(bytes)]; +} + +static NSData *OPVoiceWAVDataFromPCM16(NSData *pcmData, UInt32 sampleRate) { + NSMutableData *wav = [NSMutableData data]; + UInt32 dataSize = (UInt32)MIN((NSUInteger)UINT32_MAX, pcmData.length); + UInt32 riffSize = 36 + dataSize; + [wav appendData:[@"RIFF" dataUsingEncoding:NSASCIIStringEncoding]]; + OPVoiceAppendLE32(wav, riffSize); + [wav appendData:[@"WAVEfmt " dataUsingEncoding:NSASCIIStringEncoding]]; + OPVoiceAppendLE32(wav, 16); + OPVoiceAppendLE16(wav, 1); + OPVoiceAppendLE16(wav, 1); + OPVoiceAppendLE32(wav, sampleRate); + OPVoiceAppendLE32(wav, sampleRate * 2); + OPVoiceAppendLE16(wav, 2); + OPVoiceAppendLE16(wav, 16); + [wav appendData:[@"data" dataUsingEncoding:NSASCIIStringEncoding]]; + OPVoiceAppendLE32(wav, dataSize); + [wav appendData:pcmData ?: [NSData data]]; + return wav; +} + +static NSString *OPVoiceActivateAudioSession(void) { +#if TARGET_OS_IPHONE + NSError *error = nil; + AVAudioSession *session = [AVAudioSession sharedInstance]; + if (!session) { + return @"audio_session_unavailable"; + } + AVAudioSessionCategoryOptions options = + AVAudioSessionCategoryOptionAllowBluetoothHFP | + AVAudioSessionCategoryOptionDuckOthers | + AVAudioSessionCategoryOptionDefaultToSpeaker; + if (![session setCategory:AVAudioSessionCategoryPlayAndRecord + mode:AVAudioSessionModeMeasurement + options:options + error:&error]) { + return [NSString stringWithFormat:@"set_category_failed:%@", + error.localizedDescription ?: @"unknown"]; + } + error = nil; + [session setPreferredSampleRate:16000.0 error:&error]; + error = nil; + [session setActive:YES error:&error]; + if (error) { + return [NSString stringWithFormat:@"set_active_failed:%@", + error.localizedDescription ?: @"unknown"]; + } + return @""; +#else + return @"audio_session_unavailable_on_macos"; +#endif +} + +static NSData *OPVoiceRecordWAV(NSDictionary *request, NSDictionary **metadataOut, + NSString **errorOut) { + NSString *sessionWarning = OPVoiceActivateAudioSession(); + UInt32 sampleRate = (UInt32)OPLongLongFromRequest(request, + @"sample_rate_hz", 16000, 8000, 48000); + // Generous defaults tuned for real conversation. People naturally pause + // 1.5-2s between clauses when explaining a task; 1.7s of silence was + // cutting them off mid-sentence. + long long maxMs = OPLongLongFromRequest(request, + @"record_max_ms", 45000, 1000, 90000); + long long minRecordMs = OPLongLongFromRequest(request, + @"record_min_ms", 900, 100, 10000); + long long initialSpeechTimeoutMs = OPLongLongFromRequest(request, + @"initial_speech_timeout_ms", 8000, 500, 20000); + long long endSilenceMs = OPLongLongFromRequest(request, + @"end_silence_ms", 3000, 300, 10000); + double rmsThreshold = (double)OPLongLongFromRequest(request, + @"rms_threshold", 700, 50, 15000); + BOOL stopOnVAD = OPBoolFromRequest(request, @"vad", YES); + + NSMutableData *pcm = [NSMutableData data]; + OPVoiceCaptureState state; + memset(&state, 0, sizeof(state)); + state.pcmData = pcm; + state.stopOnVAD = stopOnVAD; + state.startedMs = OPNowMs(); + state.maxMs = maxMs; + state.minRecordMs = minRecordMs; + state.initialSpeechTimeoutMs = initialSpeechTimeoutMs; + state.endSilenceMs = endSilenceMs; + state.rmsThreshold = rmsThreshold; + state.sampleRate = sampleRate; + strlcpy(state.stopReason, "unknown", sizeof(state.stopReason)); + pthread_mutex_init(&state.mutex, NULL); + pthread_cond_init(&state.cond, NULL); + + AudioStreamBasicDescription format; + memset(&format, 0, sizeof(format)); + format.mSampleRate = sampleRate; + format.mFormatID = kAudioFormatLinearPCM; + format.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + format.mBytesPerPacket = 2; + format.mFramesPerPacket = 1; + format.mBytesPerFrame = 2; + format.mChannelsPerFrame = 1; + format.mBitsPerChannel = 16; + + OSStatus status = AudioQueueNewInput(&format, OPVoiceAudioQueueInputCallback, + &state, NULL, NULL, 0, &state.queue); + if (status != noErr || !state.queue) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"audio_queue_new_input_failed:%d", + (int)status]; + } + pthread_mutex_destroy(&state.mutex); + pthread_cond_destroy(&state.cond); + return nil; + } + + UInt32 bufferBytes = sampleRate * 2 / 10; + if (bufferBytes < 2048) { + bufferBytes = 2048; + } else if (bufferBytes > 8192) { + bufferBytes = 8192; + } + for (int i = 0; i < 3; i++) { + AudioQueueBufferRef buffer = NULL; + status = AudioQueueAllocateBuffer(state.queue, bufferBytes, &buffer); + if (status != noErr || !buffer) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"audio_queue_allocate_buffer_failed:%d", + (int)status]; + } + AudioQueueDispose(state.queue, true); + pthread_mutex_destroy(&state.mutex); + pthread_cond_destroy(&state.cond); + return nil; + } + AudioQueueEnqueueBuffer(state.queue, buffer, 0, NULL); + } + + status = AudioQueueStart(state.queue, NULL); + if (status != noErr) { + if (errorOut) { + *errorOut = [NSString stringWithFormat:@"audio_queue_start_failed:%d", + (int)status]; + } + AudioQueueDispose(state.queue, true); + pthread_mutex_destroy(&state.mutex); + pthread_cond_destroy(&state.cond); + return nil; + } + + pthread_mutex_lock(&state.mutex); + while (!state.done) { + struct timeval tv; + gettimeofday(&tv, NULL); + struct timespec ts; + ts.tv_sec = tv.tv_sec + 1; + ts.tv_nsec = tv.tv_usec * 1000; + pthread_cond_timedwait(&state.cond, &state.mutex, &ts); + long long now = OPNowMs(); + long long elapsedMs = MAX(0, now - state.startedMs); + if (elapsedMs >= maxMs + 2000) { + OPVoiceSetDoneLocked(&state, "watchdog_timeout"); + } + if (OPVoiceCancelRequested) { + OPVoiceSetDoneLocked(&state, "cancelled"); + } + } + NSString *stopReason = [NSString stringWithUTF8String:state.stopReason]; + BOOL heardSpeech = state.heardSpeech; + double peakRms = state.peakRms; + double lastRms = state.lastRms; + long long durationMs = MAX(0, OPNowMs() - state.startedMs); + pthread_mutex_unlock(&state.mutex); + + AudioQueueStop(state.queue, true); + AudioQueueDispose(state.queue, true); + pthread_mutex_destroy(&state.mutex); + pthread_cond_destroy(&state.cond); + + NSData *wav = OPVoiceWAVDataFromPCM16(pcm, sampleRate); + if (metadataOut) { + *metadataOut = @{ + @"sample_rate_hz": @(sampleRate), + @"pcm_bytes": @(pcm.length), + @"wav_bytes": @(wav.length), + @"duration_ms": @(durationMs), + @"stop_reason": stopReason ?: @"unknown", + @"heard_speech": @(heardSpeech), + @"peak_rms": @(peakRms), + @"last_rms": @(lastRms), + @"rms_threshold": @(rmsThreshold), + @"vad": @(stopOnVAD), + @"audio_session_warning": sessionWarning ?: @"" + }; + } + return wav; +} + +static NSString *OPVoiceCredentialValue(NSString **sourceOut) { + for (NSString *envName in @[@"OPENPHONE_VOICE_BEARER_TOKEN", + @"OPENPHONE_OPENAI_API_KEY", + @"OPENAI_API_KEY"]) { + const char *value = getenv(envName.UTF8String); + if (value && value[0] != '\0') { + if (sourceOut) { + *sourceOut = [NSString stringWithFormat:@"env:%@", envName]; + } + return [NSString stringWithUTF8String:value] ?: @""; + } + } + + NSData *data = [NSData dataWithContentsOfFile:OPVoiceCredentialPath()]; + if (data) { + NSString *text = [[NSString alloc] initWithData:data + encoding:NSUTF8StringEncoding] ?: @""; + text = [text stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (text.length > 0) { + id parsed = [NSJSONSerialization JSONObjectWithData: + [text dataUsingEncoding:NSUTF8StringEncoding] + options:0 error:nil]; + if ([parsed isKindOfClass:[NSDictionary class]]) { + NSDictionary *object = parsed; + for (NSString *key in @[@"openai_api_key", @"api_key", + @"credential", @"bearer", @"value"]) { + if ([object[key] isKindOfClass:[NSString class]] && + [object[key] length] > 0) { + if (sourceOut) { + *sourceOut = @"voice_credential_file"; + } + return object[key]; + } + } + } else { + if (sourceOut) { + *sourceOut = @"voice_credential_file"; + } + return text; + } + } + } + + NSDictionary *modelConfig = OPModelConfig(); + NSString *mode = [modelConfig[@"mode"] isKindOfClass:[NSString class]] + ? modelConfig[@"mode"] : @""; + if (OPModelModeIsOpenAIRealtime(mode)) { + NSString *credential = OPModelCredentialValue(); + if (credential.length > 0) { + if (sourceOut) { + *sourceOut = @"model_credential_file"; + } + return credential; + } + } + if (sourceOut) { + *sourceOut = @"none"; + } + return @""; +} + +static void OPVoiceAppendMultipartText(NSMutableData *body, NSString *boundary, + NSString *name, NSString *value) { + NSString *part = [NSString stringWithFormat: + @"--%@\r\nContent-Disposition: form-data; name=\"%@\"\r\n\r\n%@\r\n", + boundary ?: @"", name ?: @"", value ?: @""]; + [body appendData:[part dataUsingEncoding:NSUTF8StringEncoding]]; +} + +static void OPVoiceAppendMultipartFile(NSMutableData *body, NSString *boundary, + NSString *name, NSString *filename, NSString *contentType, NSData *data) { + NSString *header = [NSString stringWithFormat: + @"--%@\r\nContent-Disposition: form-data; name=\"%@\"; filename=\"%@\"\r\n" + @"Content-Type: %@\r\n\r\n", + boundary ?: @"", name ?: @"file", filename ?: @"audio.wav", + contentType ?: @"application/octet-stream"]; + [body appendData:[header dataUsingEncoding:NSUTF8StringEncoding]]; + [body appendData:data ?: [NSData data]]; + [body appendData:[@"\r\n" dataUsingEncoding:NSUTF8StringEncoding]]; +} + +static NSDictionary *OPVoiceTranscribeOpenAI(NSData *wavData, NSDictionary *request) { + NSString *credentialSource = nil; + NSString *credential = OPVoiceCredentialValue(&credentialSource); + if (credential.length == 0) { + return OPError(@"openai_voice_credential_missing"); + } + NSString *model = OPStringFromRequest(request, + @"transcription_model", @"gpt-4o-mini-transcribe"); + NSString *endpoint = OPStringFromRequest(request, + @"transcription_endpoint", @"https://api.openai.com/v1/audio/transcriptions"); + NSURL *url = [NSURL URLWithString:endpoint ?: @""]; + if (!url || !url.scheme || !url.host) { + return OPError(@"openai_transcription_endpoint_invalid"); + } + + NSString *boundary = [NSString stringWithFormat:@"openphone-%lld-%d", + OPNowMs(), getpid()]; + NSString *language = OPStringFromRequest(request, @"language", @"en"); + NSMutableData *body = [NSMutableData data]; + OPVoiceAppendMultipartText(body, boundary, @"model", model); + OPVoiceAppendMultipartText(body, boundary, @"response_format", @"json"); + if (language.length > 0) { + OPVoiceAppendMultipartText(body, boundary, @"language", language); + } + OPVoiceAppendMultipartFile(body, boundary, @"file", @"command.wav", + @"audio/wav", wavData ?: [NSData data]); + NSString *footer = [NSString stringWithFormat:@"--%@--\r\n", boundary]; + [body appendData:[footer dataUsingEncoding:NSUTF8StringEncoding]]; + + long long timeoutMs = OPLongLongFromRequest(request, + @"transcription_timeout_ms", 30000, 5000, 120000); + NSMutableURLRequest *httpRequest = [NSMutableURLRequest requestWithURL:url + cachePolicy:NSURLRequestReloadIgnoringLocalCacheData + timeoutInterval:MAX(1.0, (NSTimeInterval)timeoutMs / 1000.0)]; + httpRequest.HTTPMethod = @"POST"; + [httpRequest setValue:[NSString stringWithFormat:@"Bearer %@", credential] + forHTTPHeaderField:@"Authorization"]; + [httpRequest setValue:[NSString stringWithFormat:@"multipart/form-data; boundary=%@", + boundary] + forHTTPHeaderField:@"Content-Type"]; + [httpRequest setValue:@"openphone-ios-agentd" forHTTPHeaderField:@"User-Agent"]; + [httpRequest setValue:@"openphone-ios-local-user" + forHTTPHeaderField:@"OpenAI-Safety-Identifier"]; + httpRequest.HTTPBody = body; + + NSURLResponse *response = nil; + NSError *error = nil; + long long startedMs = OPNowMs(); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + NSData *data = [NSURLConnection sendSynchronousRequest:httpRequest + returningResponse:&response + error:&error]; +#pragma clang diagnostic pop + long long latencyMs = OPNowMs() - startedMs; + if (error || !data) { + return OPError([NSString stringWithFormat:@"openai_transcription_failed:%@", + error.localizedDescription ?: @"unknown"]); + } + NSInteger statusCode = 0; + if ([response isKindOfClass:[NSHTTPURLResponse class]]) { + statusCode = [(NSHTTPURLResponse *)response statusCode]; + } + if (statusCode < 200 || statusCode >= 300) { + return @{ + @"status": @"error", + @"reason": [NSString stringWithFormat:@"openai_transcription_http_status:%ld", + (long)statusCode], + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"provider": @"openai_transcription", + @"source": @"openphone.agentd" + }; + } + id parsed = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + if (![parsed isKindOfClass:[NSDictionary class]]) { + return OPError(@"openai_transcription_response_not_object"); + } + NSDictionary *object = parsed; + NSString *text = [object[@"text"] isKindOfClass:[NSString class]] + ? object[@"text"] : @""; + text = [text stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (text.length == 0) { + return OPError(@"openai_transcription_empty"); + } + return @{ + @"status": @"ok", + @"provider": @"openai_transcription", + @"credential_source": credentialSource ?: @"unknown", + @"model": model ?: @"", + @"transcript": text, + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"latency_ms": @(latencyMs), + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPVoiceTranscribeAppleSpeech(NSURL *audioURL, NSDictionary *request) { + if (!audioURL) { + return OPError(@"apple_speech_audio_url_missing"); + } + if (!NSClassFromString(@"SFSpeechRecognizer")) { + return OPError(@"apple_speech_framework_unavailable"); + } + + __block SFSpeechRecognizerAuthorizationStatus auth = + [SFSpeechRecognizer authorizationStatus]; + if (auth == SFSpeechRecognizerAuthorizationStatusNotDetermined) { + dispatch_semaphore_t authSemaphore = dispatch_semaphore_create(0); + [SFSpeechRecognizer requestAuthorization:^(SFSpeechRecognizerAuthorizationStatus status) { + auth = status; + dispatch_semaphore_signal(authSemaphore); + }]; + dispatch_semaphore_wait(authSemaphore, + dispatch_time(DISPATCH_TIME_NOW, 5000 * NSEC_PER_MSEC)); + } + if (auth != SFSpeechRecognizerAuthorizationStatusAuthorized) { + return OPError([NSString stringWithFormat:@"apple_speech_not_authorized:%ld", + (long)auth]); + } + + NSLocale *locale = [NSLocale currentLocale] ?: [NSLocale localeWithLocaleIdentifier:@"en_US"]; + SFSpeechRecognizer *recognizer = [[SFSpeechRecognizer alloc] initWithLocale:locale]; + if (!recognizer || !recognizer.available) { + recognizer = [[SFSpeechRecognizer alloc] initWithLocale: + [NSLocale localeWithLocaleIdentifier:@"en_US"]]; + } + if (!recognizer || !recognizer.available) { + return OPError(@"apple_speech_recognizer_unavailable"); + } + + SFSpeechURLRecognitionRequest *speechRequest = + [[SFSpeechURLRecognitionRequest alloc] initWithURL:audioURL]; + speechRequest.shouldReportPartialResults = YES; + + long long timeoutMs = OPLongLongFromRequest(request, + @"transcription_timeout_ms", 30000, 5000, 120000); + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + __block NSString *transcript = @""; + __block NSError *finalError = nil; + __block BOOL finished = NO; + SFSpeechRecognitionTask *task = [recognizer recognitionTaskWithRequest:speechRequest + resultHandler:^(SFSpeechRecognitionResult *result, NSError *error) { + if (result.bestTranscription.formattedString.length > 0) { + transcript = result.bestTranscription.formattedString; + } + if (result.isFinal || error) { + finalError = error; + finished = YES; + dispatch_semaphore_signal(semaphore); + } + }]; + if (!task) { + return OPError(@"apple_speech_task_create_failed"); + } + long long startedMs = OPNowMs(); + intptr_t waitResult = dispatch_semaphore_wait(semaphore, + dispatch_time(DISPATCH_TIME_NOW, timeoutMs * NSEC_PER_MSEC)); + if (waitResult != 0) { + [task cancel]; + } + long long latencyMs = OPNowMs() - startedMs; + transcript = [transcript stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (transcript.length == 0) { + if (finalError) { + return OPError([NSString stringWithFormat:@"apple_speech_failed:%@", + finalError.localizedDescription ?: @"unknown"]); + } + return OPError(waitResult == 0 && finished + ? @"apple_speech_empty" + : @"apple_speech_timeout"); + } + return @{ + @"status": @"ok", + @"provider": @"apple_speech_daemon", + @"transcript": transcript, + @"latency_ms": @(latencyMs), + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPVoiceTranscribe(NSData *wavData, NSURL *audioURL, + NSDictionary *request) { + NSString *provider = [OPStringFromRequest(request, + @"transcription_provider", @"auto") lowercaseString]; + if ([provider isEqualToString:@"openai"]) { + return OPVoiceTranscribeOpenAI(wavData, request); + } + if ([provider isEqualToString:@"apple"] || + [provider isEqualToString:@"apple_speech"]) { + return OPVoiceTranscribeAppleSpeech(audioURL, request); + } + NSString *credentialSource = nil; + NSString *credential = OPVoiceCredentialValue(&credentialSource); + if (credential.length > 0) { + NSDictionary *openAI = OPVoiceTranscribeOpenAI(wavData, request); + if ([openAI[@"status"] isEqualToString:@"ok"]) { + return openAI; + } + if (!OPBoolFromRequest(request, @"apple_speech_fallback", NO)) { + return openAI; + } + OPLog(@"openai transcription explicit fallback provider=apple_speech reason=%@", + openAI[@"reason"] ?: @"unknown"); + return OPVoiceTranscribeAppleSpeech(audioURL, request); + } + if (OPBoolFromRequest(request, @"apple_speech_fallback", NO)) { + return OPVoiceTranscribeAppleSpeech(audioURL, request); + } + return OPError(@"openai_voice_credential_missing"); +} + +// ---- Realtime-2 streaming voice pipeline ----------------------------------- +// Streams mic PCM16 straight to the OpenAI Realtime WebSocket so the server +// runs VAD and decides end-of-turn, instead of our local record-then-transcribe +// path. Output modality stays text: the model's tool calls drive the phone and +// its replies surface on the island. Barge-in: while the agent is mid-turn, a +// sustained loud RMS from the mic cancels the in-flight response so the user can +// interrupt (Android parity: RMS >= 1700 sustained for a 240ms guard). +typedef struct { + AudioQueueRef queue; + __unsafe_unretained OPRealtimeWebSocket *socket; + pthread_mutex_t mutex; + BOOL streaming; // pushing audio to the socket + BOOL stopped; + BOOL bargeInArmed; // agent is mid-turn; watch for interruption + volatile int bargeInDetected; + long long bargeInStartMs; + long long sendTimeoutMs; + double bargeInRmsThreshold; + long long bargeInGuardMs; + UInt32 sampleRate; +} OPRealtimeStreamState; + +static void OPRealtimeStreamInputCallback(void *userData, AudioQueueRef queue, + AudioQueueBufferRef buffer, const AudioTimeStamp *startTime, + UInt32 packetCount, const AudioStreamPacketDescription *packetDescriptions) { + (void)startTime; + (void)packetCount; + (void)packetDescriptions; + OPRealtimeStreamState *state = (OPRealtimeStreamState *)userData; + if (!state || !buffer) { + return; + } + pthread_mutex_lock(&state->mutex); + BOOL streaming = state->streaming && !state->stopped; + OPRealtimeWebSocket *socket = state->socket; + BOOL bargeArmed = state->bargeInArmed; + double threshold = state->bargeInRmsThreshold; + long long guardMs = state->bargeInGuardMs; + long long sendTimeout = state->sendTimeoutMs; + pthread_mutex_unlock(&state->mutex); + + if (!streaming || !socket) { + if (!state->stopped) { + AudioQueueEnqueueBuffer(queue, buffer, 0, NULL); + } + return; + } + + // Barge-in detection: sustained loud input while the agent is talking. + if (bargeArmed && buffer->mAudioDataByteSize > 0) { + double rms = OPVoiceRMSForPCM16(buffer->mAudioData, buffer->mAudioDataByteSize); + long long now = OPNowMs(); + pthread_mutex_lock(&state->mutex); + if (rms >= threshold) { + if (state->bargeInStartMs == 0) { + state->bargeInStartMs = now; + } else if (now - state->bargeInStartMs >= guardMs) { + state->bargeInDetected = 1; + } + } else { + state->bargeInStartMs = 0; + } + pthread_mutex_unlock(&state->mutex); + } + + if (buffer->mAudioDataByteSize > 0) { + NSData *pcm = [NSData dataWithBytesNoCopy:buffer->mAudioData + length:buffer->mAudioDataByteSize + freeWhenDone:NO]; + NSString *b64 = [pcm base64EncodedStringWithOptions:0]; + NSError *sendError = nil; + [socket sendEvent:OPRealtimeAudioAppendEvent(b64) + timeoutMs:sendTimeout > 0 ? sendTimeout : 5000 + error:&sendError]; + } + + if (!state->stopped) { + AudioQueueEnqueueBuffer(queue, buffer, 0, NULL); + } +} + +// Start a mic AudioQueue that streams pcm16 to the socket. Returns "" on +// success or an error string. Caller owns the returned queue via state. +static NSString *OPRealtimeStreamStart(OPRealtimeStreamState *state) { + NSString *sessionWarning = OPVoiceActivateAudioSession(); + (void)sessionWarning; + AudioStreamBasicDescription format; + memset(&format, 0, sizeof(format)); + format.mSampleRate = state->sampleRate; + format.mFormatID = kAudioFormatLinearPCM; + format.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + format.mBytesPerPacket = 2; + format.mFramesPerPacket = 1; + format.mBytesPerFrame = 2; + format.mChannelsPerFrame = 1; + format.mBitsPerChannel = 16; + OSStatus status = AudioQueueNewInput(&format, OPRealtimeStreamInputCallback, + state, NULL, NULL, 0, &state->queue); + if (status != noErr || !state->queue) { + return [NSString stringWithFormat:@"audio_queue_new_input_failed:%d", (int)status]; + } + UInt32 bufferBytes = state->sampleRate * 2 / 10; // ~100ms chunks + if (bufferBytes < 2048) bufferBytes = 2048; + else if (bufferBytes > 8192) bufferBytes = 8192; + for (int i = 0; i < 3; i++) { + AudioQueueBufferRef buffer = NULL; + status = AudioQueueAllocateBuffer(state->queue, bufferBytes, &buffer); + if (status != noErr || !buffer) { + AudioQueueDispose(state->queue, true); + state->queue = NULL; + return [NSString stringWithFormat:@"audio_queue_allocate_buffer_failed:%d", (int)status]; + } + AudioQueueEnqueueBuffer(state->queue, buffer, 0, NULL); + } + status = AudioQueueStart(state->queue, NULL); + if (status != noErr) { + AudioQueueDispose(state->queue, true); + state->queue = NULL; + return [NSString stringWithFormat:@"audio_queue_start_failed:%d", (int)status]; + } + return @""; +} + +static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { + if (!state || !state->queue) { + return; + } + pthread_mutex_lock(&state->mutex); + state->stopped = YES; + state->streaming = NO; + pthread_mutex_unlock(&state->mutex); + AudioQueueStop(state->queue, true); + AudioQueueDispose(state->queue, true); + state->queue = NULL; +} + +// Drive the streaming realtime voice loop. Mirrors OPRunOpenAIRealtimeTask's +// tool execution, but the first user turn arrives from the mic (server-VAD) +// rather than a pre-transcribed goal. Returns a summary dict. +static NSDictionary *OPRunStreamingRealtimeVoice(NSString *voiceTaskId, + NSDictionary *request) { + long long startedMs = OPNowMs(); + NSDictionary *modelStatus = OPModelStatusDictionary(); + NSString *mode = [modelStatus[@"mode"] isKindOfClass:[NSString class]] + ? modelStatus[@"mode"] : @"openai_realtime2"; + NSDictionary *config = OPModelConfig(); + NSString *model = [modelStatus[@"model"] isKindOfClass:[NSString class]] + ? modelStatus[@"model"] : OPModelEffectiveModel(config); + // Realtime is always OpenAI, so prefer the OpenAI voice credential. Only + // fall back to the model credential when a realtime2 model mode is actually + // configured (in which case model-credential.json holds the OpenAI key). + NSString *credential = OPVoiceCredentialValue(NULL); + if (credential.length == 0) { + credential = OPModelCredentialValue(); + } + long long timeoutMs = [modelStatus[@"timeout_ms"] respondsToSelector:@selector(longLongValue)] + ? [modelStatus[@"timeout_ms"] longLongValue] : 30000; + long long maxSteps = OPLongLongFromRequest(request, @"max_steps", 25, 1, 120); + long long maxDurationMs = OPLongLongFromRequest(request, @"max_duration_ms", + 600000, 1000, 3300000); + // OpenAI Realtime requires the input PCM rate >= 24000 Hz. + UInt32 sampleRate = (UInt32)OPLongLongFromRequest(request, @"sample_rate_hz", + 24000, 24000, 48000); + NSArray *approved = OPFullYoloCapabilities(); + + NSURL *url = OPRealtimeURL(config, model); + if (!url || !url.scheme || credential.length == 0) { + return OPError(@"realtime_stream_endpoint_or_credential_invalid"); + } + + NSDictionary *task = OPStartTask(@{@"goal": @"Voice conversation (streaming).", + @"approved_capabilities": approved}); + NSString *taskId = [task[@"task_id"] isKindOfClass:[NSString class]] + ? task[@"task_id"] : voiceTaskId; + OPUpdateTask(taskId, @"active", @{ + @"runner": @"model", + @"model_provider": mode ?: @"openai_realtime2", + @"model_runtime": @"openai_realtime_streaming", + @"model": model ?: @"", + @"runner_pid": @(getpid()) + }); + OPRecordTrajectory(taskId, @"realtime_stream_started", @{ + @"provider": mode ?: @"openai_realtime2", + @"model": model ?: @"", + @"sample_rate_hz": @(sampleRate) + }); + + NSError *error = nil; + OPRealtimeWebSocket *socket = [OPRealtimeWebSocket connectWithURL:url + bearerToken:credential timeoutMs:timeoutMs error:&error]; + if (!socket) { + OPUpdateTask(taskId, @"failed", @{@"stop_reason": @"realtime_connect_failed"}); + return OPError(error.localizedDescription ?: @"realtime_connect_failed"); + } + if (![socket sendEvent:OPRealtimeStreamingSessionUpdateEvent(mode, model, sampleRate) + timeoutMs:timeoutMs error:&error]) { + [socket close]; + OPUpdateTask(taskId, @"failed", @{@"stop_reason": @"realtime_session_update_failed"}); + return OPError(error.localizedDescription ?: @"realtime_session_update_failed"); + } + NSDictionary *sessionUpdated = OPRealtimeWaitForEventType(socket, @"session.updated", timeoutMs); + if (![sessionUpdated[@"status"] isEqualToString:@"ok"]) { + OPRecordTrajectory(taskId, @"realtime_session_update_failed", + OPRedactedObject(sessionUpdated ?: @{}, 0)); + [socket close]; + OPUpdateTask(taskId, @"failed", @{@"stop_reason": @"realtime_session_update_failed"}); + return sessionUpdated ?: OPError(@"realtime_session_update_failed"); + } + + OPRealtimeStreamState stream; + memset(&stream, 0, sizeof(stream)); + pthread_mutex_init(&stream.mutex, NULL); + stream.socket = socket; + stream.sampleRate = sampleRate; + stream.streaming = YES; + stream.sendTimeoutMs = MIN(timeoutMs, 5000); + stream.bargeInRmsThreshold = (double)OPLongLongFromRequest(request, + @"barge_in_rms", 1700, 200, 15000); + stream.bargeInGuardMs = OPLongLongFromRequest(request, @"barge_in_guard_ms", + 240, 60, 2000); + NSString *streamStartError = OPRealtimeStreamStart(&stream); + if (streamStartError.length > 0) { + [socket close]; + pthread_mutex_destroy(&stream.mutex); + OPUpdateTask(taskId, @"failed", @{@"stop_reason": streamStartError}); + return OPError(streamStartError); + } + + // Yellow realtime island: server-VAD is now listening on the live mic. + OPIslandReset(@"realtime", @"Listening", @"yellow"); + OPIslandUpdate(@{@"task_id": taskId ?: @""}); + OPVoiceSetLast(@"voice.realtime_listening", @"", @"", mode ?: @"openai_realtime2", YES); + + NSString *stopReason = @"unknown"; + NSString *lastTranscript = @""; + BOOL cancelled = NO; + BOOL terminal = NO; + BOOL succeeded = NO; + long long stepsUsed = 0; + long long toolErrors = 0; + NSDictionary *screen = @{}; + NSString *cancelReason = @""; + + for (long long step = 1; step <= maxSteps; step++) { + if (OPVoiceCancelRequested || OPTaskCancellationRequested(taskId, &cancelReason)) { + cancelled = YES; + stopReason = @"cancelled"; + break; + } + if (OPNowMs() - startedMs > maxDurationMs) { + stopReason = @"duration_limit"; + break; + } + stepsUsed = step; + + // Wait for a server-VAD-delimited model turn. Long timeout: the user + // may take a while to start speaking. + pthread_mutex_lock(&stream.mutex); + stream.bargeInArmed = NO; + stream.bargeInStartMs = 0; + stream.bargeInDetected = 0; + pthread_mutex_unlock(&stream.mutex); + + NSDictionary *turn = OPRealtimeWaitForTurn(socket, MAX(timeoutMs, 60000)); + if (![turn[@"status"] isEqualToString:@"ok"]) { + // A read/turn timeout with no speech is a natural end of conversation, + // not an error. The socket's own receive timeout surfaces as + // "realtime_read_failed:realtime_receive_timeout"; treat any timeout + // flavor as a graceful idle end. + stopReason = [turn[@"reason"] isKindOfClass:[NSString class]] + ? turn[@"reason"] : @"realtime_turn_failed"; + if ([stopReason isEqualToString:@"realtime_turn_timeout"] || + [stopReason rangeOfString:@"timeout"].location != NSNotFound) { + stopReason = @"conversation_idle_timeout"; + terminal = YES; + succeeded = YES; + } else { + toolErrors += 1; + } + break; + } + NSString *transcript = [turn[@"input_transcript"] isKindOfClass:[NSString class]] + ? turn[@"input_transcript"] : @""; + if (transcript.length > 0) { + lastTranscript = transcript; + OPRecordVoiceTurn(transcript, taskId); + OPIslandUpdate(@{@"transcript": transcript, @"mode": @"realtime", + @"accent": @"yellow", @"task_id": taskId ?: @""}); + } + NSString *finalText = [turn[@"final_text"] isKindOfClass:[NSString class]] + ? turn[@"final_text"] : @""; + NSArray *calls = [turn[@"function_calls"] isKindOfClass:[NSArray class]] + ? turn[@"function_calls"] : @[]; + + if (calls.count == 0) { + if (finalText.length > 0) { + OPIslandPublishAssistantMessage(finalText, taskId); + } + // No tool call: agent spoke or is waiting. Re-arm mic for next turn. + pthread_mutex_lock(&stream.mutex); + stream.bargeInArmed = YES; + pthread_mutex_unlock(&stream.mutex); + continue; + } + + for (id value in calls) { + if (![value isKindOfClass:[NSDictionary class]]) continue; + NSDictionary *call = value; + if (OPVoiceCancelRequested || OPTaskCancellationRequested(taskId, &cancelReason)) { + cancelled = YES; + stopReason = @"cancelled"; + break; + } + NSDictionary *decision = OPRealtimeDecisionFromCall(call); + NSDictionary *guardrail = nil; + decision = OPModelDecisionByApplyingGuardrails(decision, lastTranscript, step, &guardrail); + NSString *tool = [decision[@"tool"] isKindOfClass:[NSString class]] + ? decision[@"tool"] : @""; + if (![OPModelToolNames() containsObject:tool]) { + toolErrors += 1; + NSError *outErr = nil; + OPRealtimeSendFunctionOutput(socket, call[@"call_id"], + OPError([NSString stringWithFormat:@"unknown_model_tool:%@", tool]), + timeoutMs, &outErr); + continue; + } + OPIslandPublishToolStep(taskId, tool, @"tool_running", step, maxSteps); + { + NSString *msg = [decision[@"assistant_message"] isKindOfClass:[NSString class]] + ? decision[@"assistant_message"] : @""; + if (msg.length > 0) OPIslandPublishAssistantMessage(msg, taskId); + } + if (screen.count == 0 && OPModelToolDrivesUI(tool)) { + screen = OPModelCompactScreenForLoop(OPGetScreen(@{ + @"task_id": taskId ?: @"", + @"include_screenshot": @YES, + @"include_activity": @YES, + @"include_ui_tree": @YES, + @"compact_trajectory": @YES, + @"reason": @"realtime stream pre-action observation" + })); + } + NSDictionary *toolResult = OPModelExecuteDecision(decision, taskId, approved); + NSString *state = [toolResult[@"state"] isKindOfClass:[NSString class]] + ? toolResult[@"state"] : @""; + NSError *outputError = nil; + OPRealtimeSendFunctionOutput(socket, call[@"call_id"], toolResult ?: @{}, + timeoutMs, &outputError); + OPRealtimeSendScreenFollowupIfUseful(socket, tool, toolResult ?: @{}, + timeoutMs, &outputError); + if ([tool isEqualToString:@"finish_task"] || [state isEqualToString:@"task.finished"]) { + terminal = YES; + succeeded = YES; + stopReason = @"finish_task"; + break; + } + if ([tool isEqualToString:@"fail_task"] || [state isEqualToString:@"task.failed"]) { + terminal = YES; + stopReason = @"fail_task"; + break; + } + } + if (terminal || cancelled) { + break; + } + // Post-action: re-arm barge-in and keep the loop alive for follow-ups. + pthread_mutex_lock(&stream.mutex); + stream.bargeInArmed = YES; + pthread_mutex_unlock(&stream.mutex); + if (stream.bargeInDetected) { + OPRecordTrajectory(taskId, @"realtime_barge_in", @{@"step": @(step)}); + [socket sendEvent:@{@"type": @"response.cancel"} timeoutMs:timeoutMs error:&error]; + pthread_mutex_lock(&stream.mutex); + stream.bargeInDetected = 0; + stream.bargeInStartMs = 0; + pthread_mutex_unlock(&stream.mutex); + } + } + + OPRealtimeStreamStop(&stream); + [socket close]; + pthread_mutex_destroy(&stream.mutex); + + if ([stopReason isEqualToString:@"unknown"]) { + stopReason = cancelled ? @"cancelled" : (stepsUsed >= maxSteps ? @"step_limit" : @"model_stopped"); + } + long long durationMs = OPNowMs() - startedMs; + NSDictionary *summary = @{ + @"status": succeeded ? @"task.finished" : (cancelled ? @"task.cancelled" : @"task.failed"), + @"task_id": taskId ?: @"", + @"runner": @"model", + @"model_provider": mode ?: @"openai_realtime2", + @"model_runtime": @"openai_realtime_streaming", + @"model": model ?: @"", + @"steps_used": @(stepsUsed), + @"tool_errors": @(toolErrors), + @"duration_ms": @(durationMs), + @"stop_reason": stopReason ?: @"unknown", + @"last_transcript": lastTranscript ?: @"", + @"trajectory": OPTrajectoryPath(taskId ?: @""), + @"source": @"openphone.agentd" + }; + OPUpdateTask(taskId, succeeded ? @"completed" : (cancelled ? @"stopped" : @"failed"), @{ + @"model_loop_summary": summary, + @"completed_at": @(OPNowMs()) + }); + OPRecordTrajectory(taskId, @"realtime_stream_finished", summary); + if (succeeded) { + OPIslandReset(@"success", @"Done", @"green"); + } else if (cancelled) { + OPIslandReset(@"idle", @"Cancelled", @"cyan"); + } else { + OPIslandReset(@"error", @"Stopped", @"red"); + } + OPVoiceSetLast(succeeded ? @"voice.realtime_finished" : @"voice.realtime_stopped", + lastTranscript, succeeded ? @"" : stopReason, mode ?: @"openai_realtime2", NO); + return summary; +} + +static void *OPAsyncVoiceTriggerMain(void *context) { + @autoreleasepool { + NSDictionary *request = (__bridge_transfer NSDictionary *)context; + NSString *voiceTaskId = [NSString stringWithFormat:@"ios-voice-%lld-%d", + OPNowMs(), getpid()]; + NSString *source = OPStringFromRequest(request, + @"source", @"openphone_agentd_voice"); + OPVoiceSetLast(@"voice.recording", @"", @"", @"", YES); + OPIslandReset(@"listening", @"Listening", @"red"); + OPIslandUpdate(@{@"task_id": voiceTaskId ?: @""}); + OPRecordContextEvent(@"voice_trigger_started", source, voiceTaskId, + @"volume voice trigger", @"daemon microphone capture started", @{ + @"microphone_owner": @"openphone-agentd", + @"runtime_authority": @"phone_local" + }); + OPRecordAudit(@"voice_trigger_started", voiceTaskId, @"background.run", + @"allow_yolo", request ?: @{}, @"daemon microphone capture"); + + // Realtime-2 streaming path: hand the live mic to the OpenAI Realtime + // WebSocket (server-VAD end-of-turn), skipping record-then-transcribe. + // Opt out per-request with stream:false for the legacy capture path. + NSDictionary *voiceModelStatus = OPModelStatusDictionary(); + NSString *voiceMode = [voiceModelStatus[@"mode"] isKindOfClass:[NSString class]] + ? voiceModelStatus[@"mode"] : @""; + if ([voiceMode isEqualToString:@"openai_realtime2"] && + OPBoolFromRequest(request ?: @{}, @"stream", YES)) { + NSDictionary *streamSummary = OPRunStreamingRealtimeVoice(voiceTaskId, request ?: @{}); + OPRecordAudit(@"voice_trigger_finished", voiceTaskId, @"background.run", + [streamSummary[@"status"] isEqualToString:@"task.finished"] ? @"completed" : @"stopped", + request ?: @{}, [streamSummary[@"stop_reason"] isKindOfClass:[NSString class]] + ? streamSummary[@"stop_reason"] : @"streaming"); + OPRecordContextEvent(@"voice_trigger_finished", source, voiceTaskId, + @"realtime streaming voice", @"streaming voice loop finished", + streamSummary ?: @{}); + return NULL; + } + + NSDictionary *captureMetadata = nil; + NSString *captureError = nil; + NSData *wavData = OPVoiceRecordWAV(request ?: @{}, &captureMetadata, &captureError); + NSString *audioPath = [OPVoicePath() stringByAppendingPathComponent:@"last-command.wav"]; + NSURL *audioURL = [NSURL fileURLWithPath:audioPath]; + if (wavData.length > 0) { + [wavData writeToFile:audioPath atomically:YES]; + chmod(audioPath.UTF8String, 0600); + } + NSString *captureStopReason = [captureMetadata[@"stop_reason"] isKindOfClass:[NSString class]] + ? captureMetadata[@"stop_reason"] : @""; + if ([captureStopReason isEqualToString:@"cancelled"]) { + OPVoiceSetLast(@"voice.cancelled", @"", @"cancelled", @"audio_queue", NO); + OPIslandReset(@"idle", @"Cancelled", @"cyan"); + OPLog(@"voice trigger cancelled during capture"); + return NULL; + } + if (!wavData || wavData.length == 0) { + NSString *reason = captureError ?: @"voice_capture_empty"; + OPVoiceSetLast(@"voice.capture_failed", @"", reason, @"audio_queue", NO); + OPIslandReset(@"error", @"Microphone failed", @"red"); + NSDictionary *result = @{ + @"status": @"error", + @"reason": reason, + @"task_id": voiceTaskId, + @"capture": captureMetadata ?: @{}, + @"source": @"openphone.agentd" + }; + OPRecordContextEvent(@"voice_trigger_failed", source, voiceTaskId, + @"voice capture", reason, result); + OPRecordAudit(@"voice_trigger_failed", voiceTaskId, @"background.run", + @"failed", request ?: @{}, reason); + OPRecordTrajectory(voiceTaskId, @"voice_trigger_failed", result); + OPLog(@"voice trigger capture failed reason=%@", reason); + return NULL; + } + + OPVoiceSetLast(@"voice.transcribing", @"", @"", @"", YES); + OPIslandUpdate(@{@"mode": @"transcribing", + @"subtitle": @"Transcribing", + @"accent": @"blue"}); + NSDictionary *transcription = OPVoiceTranscribe(wavData, audioURL, request ?: @{}); + if (![transcription[@"status"] isEqualToString:@"ok"]) { + NSString *reason = [transcription[@"reason"] isKindOfClass:[NSString class]] + ? transcription[@"reason"] : @"voice_transcription_failed"; + OPVoiceSetLast(@"voice.transcription_failed", @"", reason, + transcription[@"provider"] ?: @"unknown", NO); + OPIslandReset(@"error", @"Transcription failed", @"red"); + NSMutableDictionary *result = [@{ + @"status": @"error", + @"reason": reason, + @"task_id": voiceTaskId, + @"capture": captureMetadata ?: @{}, + @"transcription": transcription ?: @{}, + @"source": @"openphone.agentd" + } mutableCopy]; + OPRecordContextEvent(@"voice_trigger_failed", source, voiceTaskId, + @"voice transcription", reason, result); + OPRecordAudit(@"voice_trigger_failed", voiceTaskId, @"background.run", + @"failed", request ?: @{}, reason); + OPRecordTrajectory(voiceTaskId, @"voice_trigger_failed", result); + OPLog(@"voice trigger transcription failed reason=%@", reason); + return NULL; + } + + NSString *transcript = [transcription[@"transcript"] isKindOfClass:[NSString class]] + ? transcription[@"transcript"] : @""; + transcript = [transcript stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSString *provider = [transcription[@"provider"] isKindOfClass:[NSString class]] + ? transcription[@"provider"] : @"unknown"; + if (transcript.length == 0) { + OPVoiceSetLast(@"voice.empty_transcript", @"", @"empty_transcript", + provider, NO); + OPIslandReset(@"error", @"No speech heard", @"orange"); + OPRecordContextEvent(@"voice_trigger_empty", source, voiceTaskId, + @"voice transcription", @"empty transcript", @{ + @"capture": captureMetadata ?: @{}, + @"provider": provider ?: @"unknown" + }); + OPLog(@"voice trigger empty transcript provider=%@", provider); + return NULL; + } + + OPVoiceSetLast(@"voice.agent_starting", transcript, @"", provider, YES); + // Prepend recent conversation context if a follow-up turn arrives within 10s. + NSArray *recentTurns = OPRecentVoiceTurnsSnapshot(10 * 1000); + NSString *effectiveGoal = transcript; + if (recentTurns.count > 0) { + NSMutableString *contextGoal = [NSMutableString string]; + [contextGoal appendString:@"Follow-up to recent voice turn(s):\n"]; + for (NSDictionary *t in recentTurns) { + [contextGoal appendFormat:@"- \"%@\"\n", t[@"transcript"] ?: @""]; + } + [contextGoal appendFormat:@"\nCurrent request: %@", transcript]; + effectiveGoal = contextGoal; + OPLog(@"voice follow-up context turns=%lu", (unsigned long)recentTurns.count); + } + OPRecordVoiceTurn(transcript, voiceTaskId); + OPIslandUpdate(@{@"mode": @"thinking", + @"subtitle": @"Thinking", + @"transcript": transcript ?: @"", + @"goal": transcript ?: @"", + @"accent": @"blue"}); + NSDictionary *hardwareRequest = @{ + @"command": @"hardware_trigger", + @"trigger": OPStringFromRequest(request, @"trigger", @"volume_up_down_combo"), + @"source": @"openphone_agentd_voice", + @"reason": @"daemon microphone transcript", + @"goal": effectiveGoal, + @"mode": OPStringFromRequest(request, @"mode", @"auto"), + @"run_task": @YES, + @"create_background_job": @NO, + @"run_background_jobs": @NO, + @"dedupe": @NO, + @"max_steps": @(OPLongLongFromRequest(request, @"max_steps", 25, 1, 25)), + @"max_duration_ms": @(OPLongLongFromRequest(request, + @"max_duration_ms", 600000, 1000, 600000)), + @"trigger_input": @"daemon_microphone_transcript", + @"voice_task_id": voiceTaskId, + @"voice_provider": provider, + @"voice_capture": captureMetadata ?: @{} + }; + NSDictionary *agentStart = OPHardwareTrigger(hardwareRequest); + BOOL started = [agentStart[@"model_loop_status"] isEqualToString:@"started_async"]; + NSString *finalState = started ? @"voice.agent_started" : @"voice.agent_not_started"; + OPVoiceSetLast(finalState, transcript, started ? @"" : + (agentStart[@"model_loop_status"] ?: agentStart[@"state"] ?: @"agent_not_started"), + provider, NO); + NSDictionary *result = @{ + @"status": @"ok", + @"state": finalState, + @"task_id": voiceTaskId, + @"transcript": transcript, + @"provider": provider, + @"capture": captureMetadata ?: @{}, + @"transcription": transcription ?: @{}, + @"agent_start": agentStart ?: @{}, + @"source": @"openphone.agentd" + }; + OPRecordContextEvent(@"voice_trigger_finished", source, voiceTaskId, + transcript, finalState, result); + OPRecordAudit(@"voice_trigger_finished", voiceTaskId, @"background.run", + started ? @"allow_yolo" : @"failed", hardwareRequest, finalState); + OPRecordTrajectory(voiceTaskId, @"voice_trigger_finished", result); + OPLog(@"voice trigger finished state=%@ provider=%@ transcript_chars=%lu", + finalState, provider, (unsigned long)transcript.length); + } + return NULL; +} + +static NSDictionary *OPVoiceTrigger(NSDictionary *request) { + pthread_mutex_lock(&OPVoiceTriggerMutex); + if (OPVoiceTriggerRunning) { + NSDictionary *result = @{ + @"status": @"ok", + @"state": @"voice.already_running", + @"runtime_authority": @"phone_local", + @"microphone_owner": @"openphone-agentd", + @"last_started_at_ms": @(OPVoiceTriggerLastStartedMs), + @"source": @"openphone.agentd" + }; + pthread_mutex_unlock(&OPVoiceTriggerMutex); + return result; + } + pthread_mutex_unlock(&OPVoiceTriggerMutex); + + NSString *provider = [OPStringFromRequest(request, + @"transcription_provider", @"auto") lowercaseString]; + BOOL explicitAppleDebug = [provider isEqualToString:@"apple"] || + [provider isEqualToString:@"apple_speech"] || + OPBoolFromRequest(request, @"apple_speech_fallback", NO) || + OPBoolFromRequest(request, @"allow_record_without_credential", NO); + NSString *credentialSource = nil; + NSString *credential = explicitAppleDebug ? @"" : OPVoiceCredentialValue(&credentialSource); + if (!explicitAppleDebug && credential.length == 0) { + pthread_mutex_lock(&OPVoiceTriggerMutex); + OPVoiceTriggerLastStartedMs = OPNowMs(); + pthread_mutex_unlock(&OPVoiceTriggerMutex); + OPVoiceSetLast(@"voice.credential_missing", @"", + @"openai_voice_credential_missing", @"openai_transcription", NO); + OPIslandReset(@"error", @"Need OpenAI voice key", @"orange"); + NSDictionary *result = @{ + @"status": @"error", + @"state": @"voice.credential_missing", + @"reason": @"openai_voice_credential_missing", + @"runtime_authority": @"phone_local", + @"microphone_owner": @"openphone-agentd", + @"default_transcription_provider": @"openai_transcription", + @"voice_credential_file": OPVoiceCredentialPath(), + @"source": @"openphone.agentd" + }; + OPRecordContextEvent(@"voice_trigger_suppressed", @"openphone_agentd_voice", + [NSString stringWithFormat:@"ios-voice-%lld-%d", OPNowMs(), getpid()], + @"voice credential", @"openai_voice_credential_missing", result); + OPLog(@"voice trigger suppressed reason=openai_voice_credential_missing"); + return result; + } + + pthread_mutex_lock(&OPVoiceTriggerMutex); + OPVoiceTriggerRunning = YES; + OPVoiceTriggerLastStartedMs = OPNowMs(); + OPVoiceTriggerLastState = @"voice.starting"; + OPVoiceTriggerLastError = @""; + OPVoiceCancelRequested = 0; + pthread_mutex_unlock(&OPVoiceTriggerMutex); + + NSMutableDictionary *ownedRequest = [request mutableCopy] ?: [NSMutableDictionary dictionary]; + if (!ownedRequest[@"mode"]) { + ownedRequest[@"mode"] = @"auto"; + } + if (!ownedRequest[@"max_steps"]) { + ownedRequest[@"max_steps"] = @25; + } + if (!ownedRequest[@"max_duration_ms"]) { + ownedRequest[@"max_duration_ms"] = @600000; + } + pthread_t thread; + int rc = pthread_create(&thread, NULL, OPAsyncVoiceTriggerMain, + (__bridge_retained void *)[ownedRequest copy]); + if (rc != 0) { + OPVoiceSetLast(@"voice.start_failed", @"", + [NSString stringWithFormat:@"pthread_create_failed:%d", rc], + @"", NO); + return @{ + @"status": @"error", + @"reason": [NSString stringWithFormat:@"pthread_create_failed:%d", rc], + @"source": @"openphone.agentd" + }; + } + pthread_detach(thread); + return @{ + @"status": @"ok", + @"state": @"voice.started_async", + @"runtime_authority": @"phone_local", + @"microphone_owner": @"openphone-agentd", + @"max_steps": ownedRequest[@"max_steps"] ?: @25, + @"max_duration_ms": ownedRequest[@"max_duration_ms"] ?: @600000, + @"source": @"openphone.agentd" + }; +} + +static NSDictionary *OPVoiceTranscribeFile(NSDictionary *request) { + NSString *path = OPStringFromRequest(request, @"path", + OPStringFromRequest(request, @"audio_path", + [OPVoicePath() stringByAppendingPathComponent:@"last-command.wav"])); + if (path.length == 0) { + return OPError(@"voice_audio_path_missing"); + } + NSData *data = [NSData dataWithContentsOfFile:path]; + if (!data || data.length == 0) { + return OPError(@"voice_audio_file_missing_or_empty"); + } + NSURL *url = [NSURL fileURLWithPath:path]; + NSDictionary *result = OPVoiceTranscribe(data, url, request ?: @{}); + NSMutableDictionary *withMetadata = [result mutableCopy] ?: [NSMutableDictionary dictionary]; + withMetadata[@"audio_path"] = path; + withMetadata[@"audio_bytes"] = @(data.length); + return withMetadata; +} + +static pthread_t OPVolumeTriggerThread; +static volatile int OPVolumeTriggerThreadStarted = 0; +static volatile int OPVolumeTriggerCallbackRegistered = 0; +static volatile int OPVolumeTriggerActivationAttempted = 0; +static volatile int OPVolumeTriggerActivated = 0; +static volatile long long OPVolumeTriggerEventsSeen = 0; +static volatile long long OPVolumeTriggerKeyboardEventsSeen = 0; +static volatile long long OPVolumeTriggerCount = 0; +static volatile long long OPVolumeTriggerLastEventMs = 0; +static volatile long long OPVolumeTriggerLastTriggerMs = 0; +static volatile uint32_t OPVolumeTriggerLastUsagePage = 0; +static volatile uint32_t OPVolumeTriggerLastUsage = 0; +static volatile int OPVolumeTriggerLastDown = 0; +static CFAbsoluteTime OPVolumeTriggerLastUpSeconds = 0; +static CFAbsoluteTime OPVolumeTriggerLastDownSeconds = 0; +static CFAbsoluteTime OPVolumeTriggerLastFireSeconds = 0; +static NSString *OPVolumeTriggerStartError = nil; +static NSString *OPVolumeTriggerActivationError = nil; +static NSString *OPVolumeTriggerLastRoute = nil; +static NSString *OPVolumeTriggerLastFireState = nil; +static NSString *OPVolumeTriggerLastFireTaskId = nil; + +static void OPVolumeTriggerFire(void) { + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + @autoreleasepool { + NSDictionary *preferences = OPVolumeTriggerPreferences(); + BOOL daemonVoice = OPBoolFromRequest(preferences, @"DaemonVoiceAgent", YES); + NSDictionary *result = nil; + if (daemonVoice) { + OPVolumeTriggerLastRoute = @"daemon_voice_agent"; + result = OPVoiceTrigger(@{ + @"command": @"voice_trigger", + @"trigger": @"volume_up_down_combo", + @"source": @"agentd_hid_listener", + @"reason": @"phone-local HID volume combo", + @"mode": @"auto", + @"max_steps": @25, + @"max_duration_ms": @600000 + }); + OPVolumeTriggerLastFireState = result[@"state"] ?: result[@"reason"] ?: @""; + OPVolumeTriggerLastFireTaskId = result[@"task_id"] ?: @""; + } else { + OPVolumeTriggerLastRoute = @"direct_hardware_trigger"; + NSMutableDictionary *request = [@{ + @"command": @"hardware_trigger", + @"trigger": @"volume_up_down_combo", + @"source": @"agentd_hid_listener", + @"reason": @"phone-local HID volume combo", + @"goal": OPVolumeTriggerGoalFromPreferences(preferences), + @"mode": @"auto", + @"run_task": @(OPBoolFromRequest(preferences, @"RunTask", YES)), + @"create_background_job": @(OPBoolFromRequest(preferences, @"CreateBackgroundJob", NO)), + @"run_background_jobs": @(OPBoolFromRequest(preferences, @"RunBackgroundJobs", NO)), + @"cooldown_ms": @(OPVolumeTriggerCooldownMs(preferences)), + @"max_steps": @5, + @"max_duration_ms": @120000, + @"trigger_input": @"daemon_hid_listener" + } mutableCopy]; + result = OPHardwareTrigger(request); + OPVolumeTriggerLastFireState = result[@"model_loop_status"] ?: result[@"state"] ?: @""; + OPVolumeTriggerLastFireTaskId = result[@"agent_task_id"] ?: result[@"task_id"] ?: @""; + } + OPLog(@"volume trigger fired route=%@ result_status=%@ state=%@ task_id=%@", + OPVolumeTriggerLastRoute ?: @"", + result[@"status"] ?: @"unknown", + OPVolumeTriggerLastFireState ?: @"", + OPVolumeTriggerLastFireTaskId ?: @""); + } + }); +} + +static void OPVolumeTriggerRecordButton(BOOL volumeUp) { + NSDictionary *preferences = OPVolumeTriggerPreferences(); + if (!OPBoolFromRequest(preferences, @"Enabled", YES)) { + OPLog(@"volume trigger button ignored provider=agentd_hid_listener reason=disabled"); + return; + } + CFAbsoluteTime now = CFAbsoluteTimeGetCurrent(); + if (volumeUp) { + OPVolumeTriggerLastUpSeconds = now; + } else { + OPVolumeTriggerLastDownSeconds = now; + } + + BOOL combo = fabs(OPVolumeTriggerLastUpSeconds - OPVolumeTriggerLastDownSeconds) <= + ((double)OPVolumeTriggerWindowMs(preferences) / 1000.0); + BOOL cooledDown = (now - OPVolumeTriggerLastFireSeconds) >= + ((double)OPVolumeTriggerCooldownMs(preferences) / 1000.0); + if (combo && cooledDown) { + OPVolumeTriggerLastFireSeconds = now; + OPVolumeTriggerLastTriggerMs = OPNowMs(); + OPVolumeTriggerCount++; + OPLog(@"volume trigger combo detected provider=agentd_hid_listener"); + OPVolumeTriggerFire(); + } +} + +static void OPVolumeTriggerEventCallback(void *target, void *refcon, + OPHIDEventQueueRef queue, OPHIDEventRef event) { + (void)target; + (void)refcon; + (void)queue; + if (!event || !OPHIDEventGetType || !OPHIDEventGetIntegerValue) { + return; + } + OPVolumeTriggerEventsSeen++; + uint32_t type = OPHIDEventGetType(event); + if (type != OPIOHIDEventTypeKeyboard) { + return; + } + OPVolumeTriggerKeyboardEventsSeen++; + uint32_t usagePage = (uint32_t)OPHIDEventGetIntegerValue(event, + OPIOHIDEventFieldKeyboardUsagePage); + uint32_t usage = (uint32_t)OPHIDEventGetIntegerValue(event, + OPIOHIDEventFieldKeyboardUsage); + int down = OPHIDEventGetIntegerValue(event, OPIOHIDEventFieldKeyboardDown); + OPVolumeTriggerLastEventMs = OPNowMs(); + OPVolumeTriggerLastUsagePage = usagePage; + OPVolumeTriggerLastUsage = usage; + OPVolumeTriggerLastDown = down; + if (!down || usagePage != OPIOHIDUsagePageConsumer) { + return; + } + if (usage == OPIOHIDUsageConsumerVolumeIncrement) { + OPVolumeTriggerRecordButton(YES); + } else if (usage == OPIOHIDUsageConsumerVolumeDecrement) { + OPVolumeTriggerRecordButton(NO); + } +} + +static void *OPVolumeTriggerThreadMain(void *unused) { + (void)unused; + @autoreleasepool { + OPEnsureHIDInputLoaded(); + if (!OPHIDCreateSimpleClient && !OPHIDCreateClient) { + OPVolumeTriggerStartError = @"missing_hid_client_create"; + OPLog(@"volume trigger listener unavailable: %@", OPVolumeTriggerStartError); + return NULL; + } + if (!OPHIDRegisterEventCallback || !OPHIDScheduleWithRunLoop || + !OPHIDEventGetType || !OPHIDEventGetIntegerValue) { + OPVolumeTriggerStartError = @"missing_hid_callback_symbols"; + OPLog(@"volume trigger listener unavailable: %@", OPVolumeTriggerStartError); + return NULL; + } + OPHIDEventSystemClientRef client = OPHIDCreateSimpleClient + ? OPHIDCreateSimpleClient(kCFAllocatorDefault) + : OPHIDCreateClient(kCFAllocatorDefault); + if (!client) { + OPVolumeTriggerStartError = @"client_create_failed"; + OPLog(@"volume trigger listener unavailable: %@", OPVolumeTriggerStartError); + return NULL; + } + OPHIDRegisterEventCallback(client, OPVolumeTriggerEventCallback, NULL, NULL); + OPHIDScheduleWithRunLoop(client, CFRunLoopGetCurrent(), kCFRunLoopDefaultMode); + OPVolumeTriggerActivationAttempted = 0; + OPVolumeTriggerActivated = 0; + OPVolumeTriggerActivationError = @"skipped_runloop_client_avoids_ios_assert"; + OPVolumeTriggerCallbackRegistered = 1; + OPLog(@"volume trigger listener started provider=IOHIDEventSystemClient activated=%d reason=%@", + OPVolumeTriggerActivated != 0, OPVolumeTriggerActivationError ?: @""); + CFRunLoopRun(); + } + return NULL; +} + +static void OPStartVolumeTriggerListener(void) { + if (OPVolumeTriggerThreadStarted) { + return; + } + OPVolumeTriggerThreadStarted = 1; + int rc = pthread_create(&OPVolumeTriggerThread, NULL, OPVolumeTriggerThreadMain, NULL); + if (rc != 0) { + OPVolumeTriggerThreadStarted = 0; + OPVolumeTriggerStartError = [NSString stringWithFormat:@"pthread_create_failed:%d", rc]; + OPLog(@"volume trigger listener failed: %@", OPVolumeTriggerStartError); + return; + } + pthread_detach(OPVolumeTriggerThread); +} + +static NSDictionary *OPVolumeTriggerStatus(void) { + BOOL symbolsAvailable = OPHIDRegisterEventCallback && OPHIDScheduleWithRunLoop + && OPHIDEventGetType && OPHIDEventGetIntegerValue; + NSDictionary *springBoardFallback = OPSpringBoardTriggerStatus(); + NSDictionary *preferences = OPVolumeTriggerPreferences(); + return @{ + @"status": OPVolumeTriggerCallbackRegistered + ? @"implemented_experimental" + : (OPVolumeTriggerThreadStarted ? @"starting_or_unavailable" : @"not_started"), + @"provider": @"IOKit.IOHIDEventSystemClient.event_callback", + @"thread_started": @(OPVolumeTriggerThreadStarted != 0), + @"callback_registered": @(OPVolumeTriggerCallbackRegistered != 0), + @"activation_attempted": @(OPVolumeTriggerActivationAttempted != 0), + @"activated": @(OPVolumeTriggerActivated != 0), + @"activation_error": OPVolumeTriggerActivationError ?: @"", + @"symbols_available": @(symbolsAvailable), + @"events_seen": @((long long)OPVolumeTriggerEventsSeen), + @"keyboard_events_seen": @((long long)OPVolumeTriggerKeyboardEventsSeen), + @"trigger_count": @((long long)OPVolumeTriggerCount), + @"last_event_ms": @((long long)OPVolumeTriggerLastEventMs), + @"last_trigger_ms": @((long long)OPVolumeTriggerLastTriggerMs), + @"last_usage_page": @((uint32_t)OPVolumeTriggerLastUsagePage), + @"last_usage": @((uint32_t)OPVolumeTriggerLastUsage), + @"last_down": @(OPVolumeTriggerLastDown != 0), + @"start_error": OPVolumeTriggerStartError ?: @"", + @"last_route": OPVolumeTriggerLastRoute ?: @"", + @"last_fire_state": OPVolumeTriggerLastFireState ?: @"", + @"last_fire_task_id": OPVolumeTriggerLastFireTaskId ?: @"", + @"preferences": @{ + @"enabled": @(OPBoolFromRequest(preferences, @"Enabled", YES)), + @"daemon_voice_agent": @(OPBoolFromRequest(preferences, @"DaemonVoiceAgent", YES)), + @"run_task": @(OPBoolFromRequest(preferences, @"RunTask", YES)), + @"create_background_job": @(OPBoolFromRequest(preferences, @"CreateBackgroundJob", NO)), + @"run_background_jobs": @(OPBoolFromRequest(preferences, @"RunBackgroundJobs", NO)), + @"window_ms": @(OPVolumeTriggerWindowMs(preferences)), + @"cooldown_ms": @(OPVolumeTriggerCooldownMs(preferences)), + @"trigger_goal_present": @(OPVolumeTriggerGoalFromPreferences(preferences).length > 0) + }, + @"springboard_tweak": @"packaged_fallback_requires_tweak_injection", + @"springboard_fallback": springBoardFallback ?: @{} + }; +} + +static NSDictionary *OPHandleRequest(NSDictionary *request, NSDate *startedAt) { + NSString *command = [request[@"command"] isKindOfClass:[NSString class]] ? request[@"command"] : nil; + if (!command) { + command = [request[@"method"] isKindOfClass:[NSString class]] ? request[@"method"] : @"health"; + } + if (OPProtectedDataHelperRole() && !OPProtectedDataHelperCommandAllowed(command)) { + return OPError(@"protected_data_helper_command_denied"); + } + if ([command isEqualToString:@"health"]) { + return OPHealth(startedAt); + } + if ([command isEqualToString:@"jetsam_priority_set"] || + [command isEqualToString:@"openphone.jetsam.priority_set"]) { + return OPJetsamPrioritySet(request); + } + if ([command isEqualToString:@"model_status"] || + [command isEqualToString:@"openphone.model.status"]) { + return OPModelStatus(request); + } + if ([command isEqualToString:@"model_configure"] || + [command isEqualToString:@"openphone.model.configure"]) { + return OPModelConfigure(request); + } + if ([command isEqualToString:@"agent_status"] || + [command isEqualToString:@"openphone.agent.status"]) { + return OPAgentStatus(request); + } + if ([command isEqualToString:@"agent_control"] || + [command isEqualToString:@"openphone.agent.control"]) { + return OPAgentControl(request); + } + if ([command isEqualToString:@"agent_pause"]) { + NSMutableDictionary *controlRequest = [request mutableCopy] ?: [NSMutableDictionary dictionary]; + controlRequest[@"action"] = @"pause"; + return OPAgentControl(controlRequest); + } + if ([command isEqualToString:@"agent_resume"]) { + NSMutableDictionary *controlRequest = [request mutableCopy] ?: [NSMutableDictionary dictionary]; + controlRequest[@"action"] = @"resume"; + return OPAgentControl(controlRequest); + } + if ([command isEqualToString:@"start_task"]) { + return OPStartTask(request); + } + if ([command isEqualToString:@"stop_task"]) { + return OPStopTask(request); + } + if ([command isEqualToString:@"finish_task"] || + [command isEqualToString:@"openphone.task.finish"]) { + return OPFinishTask(request); + } + if ([command isEqualToString:@"fail_task"] || + [command isEqualToString:@"openphone.task.fail"]) { + return OPFailTask(request); + } + if ([command isEqualToString:@"get_task"]) { + return OPGetTask(request); + } + if ([command isEqualToString:@"list_tasks"]) { + return OPListTasks(request); + } + if ([command isEqualToString:@"task_repair_stale_active"] || + [command isEqualToString:@"openphone.tasks.repair_stale_active"]) { + return OPRepairStaleActiveTasks(request); + } + if ([command isEqualToString:@"get_audit"]) { + return OPGetAudit(request); + } + if ([command isEqualToString:@"get_trajectory"]) { + return OPGetTrajectory(request); + } + if ([command isEqualToString:@"memory_save"] || + [command isEqualToString:@"openphone.memory.save"]) { + return OPMemorySave(request); + } + if ([command isEqualToString:@"memory_search"] || + [command isEqualToString:@"openphone.memory.search"]) { + return OPMemorySearch(request); + } + if ([command isEqualToString:@"memory_update"] || + [command isEqualToString:@"openphone.memory.update"]) { + return OPMemoryUpdate(request); + } + if ([command isEqualToString:@"memory_delete"] || + [command isEqualToString:@"openphone.memory.delete"]) { + return OPMemoryDelete(request); + } + if ([command isEqualToString:@"memory_merge"] || + [command isEqualToString:@"openphone.memory.merge"]) { + return OPMemoryMerge(request); + } + if ([command isEqualToString:@"context_search"] || + [command isEqualToString:@"openphone.context.search"]) { + return OPContextSearch(request); + } + if ([command isEqualToString:@"clipboard_read"] || + [command isEqualToString:@"openphone.clipboard.read"]) { + return OPClipboardRead(request); + } + if ([command isEqualToString:@"clipboard_write"] || + [command isEqualToString:@"openphone.clipboard.write"]) { + return OPClipboardWrite(request); + } + if ([command isEqualToString:@"contacts_search"] || + [command isEqualToString:@"openphone.contacts.search"]) { + return OPContactsSearch(request); + } + if ([command isEqualToString:@"calendar_search"] || + [command isEqualToString:@"calendar_events_search"] || + [command isEqualToString:@"openphone.calendar.search"] || + [command isEqualToString:@"openphone.calendar.events.search"]) { + return OPCalendarSearch(request); + } + if ([command isEqualToString:@"calls_search"] || + [command isEqualToString:@"call_history_search"] || + [command isEqualToString:@"openphone.calls.search"] || + [command isEqualToString:@"openphone.call_history.search"]) { + return OPCallsSearch(request); + } + if ([command isEqualToString:@"messages_search"] || + [command isEqualToString:@"message_search"] || + [command isEqualToString:@"sms_search"] || + [command isEqualToString:@"openphone.messages.search"] || + [command isEqualToString:@"openphone.sms.search"]) { + return OPMessagesSearch(request); + } + if ([command isEqualToString:@"commitment_create"] || + [command isEqualToString:@"openphone.commitments.create"] || + [command isEqualToString:@"openphone.commitment.create"]) { + return OPCommitmentCreate(request); + } + if ([command isEqualToString:@"commitment_search"] || + [command isEqualToString:@"openphone.commitments.search"] || + [command isEqualToString:@"openphone.commitment.search"]) { + return OPCommitmentSearch(request); + } + if ([command isEqualToString:@"commitment_update_status"] || + [command isEqualToString:@"openphone.commitments.update_status"] || + [command isEqualToString:@"openphone.commitment.update_status"]) { + return OPCommitmentUpdateStatus(request); + } + if ([command isEqualToString:@"commitment_run_due"] || + [command isEqualToString:@"openphone.commitments.run_due"] || + [command isEqualToString:@"openphone.commitment.run_due"]) { + NSUInteger limit = OPLimitFromRequest(request, 5, 25); + if (limit == 0) { + limit = 5; + } + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *source = OPStringFromRequest(request, @"source", @"commitment_scheduler_command"); + id deliveryRequest = OPJSONObjectFromRequest(request, @"delivery", nil); + return OPCommitmentMaterializeDue(limit, OPNowMs(), taskId, source, + [deliveryRequest isKindOfClass:[NSDictionary class]] ? deliveryRequest : nil); + } + if ([command isEqualToString:@"watcher_create"] || + [command isEqualToString:@"openphone.watchers.create"]) { + return OPWatcherCreate(request); + } + if ([command isEqualToString:@"watcher_list"] || + [command isEqualToString:@"openphone.watchers.list"]) { + return OPWatcherList(request); + } + if ([command isEqualToString:@"watcher_stop"] || + [command isEqualToString:@"openphone.watchers.stop"]) { + return OPWatcherStop(request); + } + if ([command isEqualToString:@"watcher_repair_stuck"] || + [command isEqualToString:@"openphone.watchers.repair_stuck"]) { + return OPWatcherRepairStuck(request); + } + if ([command isEqualToString:@"watcher_debug_mark_running"]) { + return OPWatcherDebugMarkRunning(request); + } + if ([command isEqualToString:@"watcher_run_due"] || + [command isEqualToString:@"openphone.watchers.run_due"]) { + NSUInteger limit = OPLimitFromRequest(request, 5, 25); + if (limit == 0) { + limit = 5; + } + NSString *taskId = OPStringFromRequest(request, @"task_id", @""); + NSString *source = OPStringFromRequest(request, @"source", @"watcher_scheduler_command"); + return OPWatcherMaterializeDue(limit, OPNowMs(), taskId, source); + } + if ([command isEqualToString:@"background_job_create"] || + [command isEqualToString:@"openphone.jobs.create"]) { + return OPBackgroundJobCreate(request); + } + if ([command isEqualToString:@"background_job_list"] || + [command isEqualToString:@"openphone.jobs.list"]) { + return OPBackgroundJobList(request); + } + if ([command isEqualToString:@"background_job_stop"] || + [command isEqualToString:@"openphone.jobs.stop"]) { + return OPBackgroundJobStop(request); + } + if ([command isEqualToString:@"background_job_repair_stuck"] || + [command isEqualToString:@"openphone.jobs.repair_stuck"]) { + return OPBackgroundJobRepairStuck(request); + } + if ([command isEqualToString:@"background_job_debug_mark_running"]) { + return OPBackgroundJobDebugMarkRunning(request); + } + if ([command isEqualToString:@"background_job_run_due"] || + [command isEqualToString:@"openphone.jobs.run_due"] || + [command isEqualToString:@"run_due_background_jobs"]) { + return OPBackgroundJobRunDue(request); + } + if ([command isEqualToString:@"voice_memory_promote"] || + [command isEqualToString:@"openphone.voice.promote_memory"]) { + return OPPromoteVoiceTurnsToMemory(request); + } + if ([command isEqualToString:@"notification_ingest"] || + [command isEqualToString:@"openphone.notification.ingest"]) { + return OPNotificationIngest(request); + } + if ([command isEqualToString:@"notification_list"] || + [command isEqualToString:@"openphone.notification.list"]) { + return OPNotificationList(request); + } + if ([command isEqualToString:@"list_apps"]) { + return OPListApps(request); + } + if ([command isEqualToString:@"app_ui_publish"] || + [command isEqualToString:@"openphone.app_ui.publish"]) { + return OPAppUIPublish(request); + } + if ([command isEqualToString:@"get_screen"]) { + return OPGetScreen(request); + } + if ([command isEqualToString:@"execute_action"]) { + return OPExecuteAction(request); + } + if ([command isEqualToString:@"run_task"]) { + return OPRunTask(request); + } + if ([command isEqualToString:@"voice_trigger"] || + [command isEqualToString:@"openphone.trigger.voice"] || + [command isEqualToString:@"openphone.voice.trigger"]) { + return OPVoiceTrigger(request); + } + if ([command isEqualToString:@"voice_status"] || + [command isEqualToString:@"openphone.voice.status"]) { + return OPVoiceStatus(request); + } + if ([command isEqualToString:@"voice_transcribe_file"] || + [command isEqualToString:@"openphone.voice.transcribe_file"]) { + return OPVoiceTranscribeFile(request); + } + if ([command isEqualToString:@"voice_confirm"] || + [command isEqualToString:@"voice_deny"] || + [command isEqualToString:@"openphone.voice.confirm"] || + [command isEqualToString:@"openphone.voice.deny"]) { + BOOL approve = [command containsString:@"confirm"]; + NSDictionary *payload = @{ + @"decision": approve ? @"approve" : @"deny", + @"received_at_ms": @(OPNowMs()), + @"source": OPStringFromRequest(request, @"source", @"island_chip") + }; + NSString *dir = [OPStorePath() stringByAppendingPathComponent:@"springboard"]; + [[NSFileManager defaultManager] createDirectoryAtPath:dir + withIntermediateDirectories:YES + attributes:nil error:nil]; + NSString *path = [dir stringByAppendingPathComponent:@"confirmation-response.json"]; + OPWriteJSONFile(path, payload); + return @{ + @"status": @"ok", + @"decision": approve ? @"approve" : @"deny", + @"source": @"openphone.agentd" + }; + } + if ([command isEqualToString:@"voice_cancel"] || + [command isEqualToString:@"openphone.voice.cancel"] || + [command isEqualToString:@"cancel_active"]) { + pthread_mutex_lock(&OPVoiceTriggerMutex); + BOOL voiceRunning = OPVoiceTriggerRunning; + OPVoiceCancelRequested = 1; + pthread_mutex_unlock(&OPVoiceTriggerMutex); + + pthread_mutex_lock(&OPIslandMutex); + OPIslandEnsureState(); + NSString *activeTaskId = [OPIslandState[@"task_id"] isKindOfClass:[NSString class]] + ? [OPIslandState[@"task_id"] copy] : @""; + NSString *mode = [OPIslandState[@"mode"] isKindOfClass:[NSString class]] + ? [OPIslandState[@"mode"] copy] : @"idle"; + pthread_mutex_unlock(&OPIslandMutex); + + BOOL activeModelTask = activeTaskId.length > 0 && + ([mode isEqualToString:@"thinking"] || + [mode isEqualToString:@"action"]); + if (activeModelTask) { + OPStopTask(@{ + @"task_id": activeTaskId, + @"reason": OPStringFromRequest(request, @"reason", @"user_cancelled") + }); + } + if (voiceRunning || activeModelTask) { + OPIslandReset(@"idle", @"Cancelled", @"cyan"); + } + return @{ + @"status": @"ok", + @"state": (voiceRunning || activeModelTask) ? + @"voice.cancelling" : @"voice.cancel_noop", + @"voice_was_running": @(voiceRunning), + @"model_task_cancelled": @(activeModelTask), + @"active_task_id": activeTaskId, + @"source": @"openphone.agentd" + }; + } + if ([command isEqualToString:@"hardware_trigger"] || + [command isEqualToString:@"openphone.trigger.hardware"] || + [command isEqualToString:@"openphone.hardware.trigger"]) { + return OPHardwareTrigger(request); + } + return OPError([NSString stringWithFormat:@"unknown_command:%@", command]); +} + +static NSDictionary *OPParseRequest(NSData *data) { + if (data.length == 0) { + return @{@"command": @"health"}; + } + id object = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + if ([object isKindOfClass:[NSDictionary class]]) { + return object; + } + return @{@"command": @"health"}; +} + +static NSData *OPReadClient(int clientFd) { + NSMutableData *data = [NSMutableData data]; + char buffer[4096]; + while (data.length < (256 * 1024)) { + ssize_t count = read(clientFd, buffer, sizeof(buffer)); + if (count > 0) { + [data appendBytes:buffer length:(NSUInteger)count]; + if (memchr(buffer, '\n', (size_t)count) != NULL) { + break; + } + continue; + } + break; + } + return data; +} + +static int OPCreateServerSocket(void) { + NSString *socketPath = OPSocketPath(); + unlink(socketPath.UTF8String); + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + OPLog(@"socket failed: %s", strerror(errno)); + return -1; + } + + struct sockaddr_un address; + memset(&address, 0, sizeof(address)); + address.sun_family = AF_UNIX; + strlcpy(address.sun_path, socketPath.UTF8String, sizeof(address.sun_path)); + + if (bind(fd, (struct sockaddr *)&address, sizeof(address)) != 0) { + OPLog(@"bind failed: %s", strerror(errno)); + close(fd); + return -1; + } + OPRestrictSocketToMobile(socketPath); + if (listen(fd, 16) != 0) { + OPLog(@"listen failed: %s", strerror(errno)); + close(fd); + return -1; + } + return fd; +} + +static int OPCreateAppUIIntakeSocket(void) { + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { + OPAppUIIntakeStartError = [NSString stringWithFormat:@"socket_failed:%s", strerror(errno)]; + OPLog(@"app ui intake socket failed: %s", strerror(errno)); + return -1; + } + int yes = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + + struct sockaddr_in address; + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(27631); + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + if (bind(fd, (struct sockaddr *)&address, sizeof(address)) != 0) { + OPAppUIIntakeStartError = [NSString stringWithFormat:@"bind_failed:%s", strerror(errno)]; + OPLog(@"app ui intake bind failed: %s", strerror(errno)); + close(fd); + return -1; + } + if (listen(fd, 8) != 0) { + OPAppUIIntakeStartError = [NSString stringWithFormat:@"listen_failed:%s", strerror(errno)]; + OPLog(@"app ui intake listen failed: %s", strerror(errno)); + close(fd); + return -1; + } + return fd; +} + +static void *OPAppUIIntakeThreadMain(void *unused) { + (void)unused; + @autoreleasepool { + int fd = OPCreateAppUIIntakeSocket(); + if (fd < 0) { + return NULL; + } + OPAppUIIntakeFd = fd; + OPAppUIIntakeReady = 1; + OPLog(@"app ui intake started tcp=127.0.0.1:27631"); + while (OPRunning) { + struct sockaddr_in peer; + socklen_t peerLen = sizeof(peer); + int clientFd = accept(fd, (struct sockaddr *)&peer, &peerLen); + if (clientFd < 0) { + if (errno == EINTR || !OPRunning) { + continue; + } + OPLog(@"app ui intake accept failed: %s", strerror(errno)); + continue; + } + BOOL loopback = peer.sin_family == AF_INET && + ntohl(peer.sin_addr.s_addr) == INADDR_LOOPBACK; + struct timeval timeout; + timeout.tv_sec = 2; + timeout.tv_usec = 0; + setsockopt(clientFd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + @autoreleasepool { + NSDictionary *response = nil; + if (!loopback) { + response = OPError(@"app_ui_intake_non_loopback_peer_rejected"); + } else { + NSData *requestData = OPReadClient(clientFd); + NSDictionary *request = OPParseRequest(requestData); + NSString *command = [request[@"command"] isKindOfClass:[NSString class]] + ? request[@"command"] : @""; + if ([command isEqualToString:@"app_ui_publish"] || + [command isEqualToString:@"openphone.app_ui.publish"] || + [request[@"schema"] isEqualToString:@"openphone.app_ui_state.v1"]) { + NSMutableDictionary *publishRequest = [request mutableCopy] ?: [NSMutableDictionary dictionary]; + publishRequest[@"transport"] = @"tcp_loopback"; + response = OPAppUIPublish(publishRequest); + } else if ([command isEqualToString:@"app_input_poll"] || + [command isEqualToString:@"openphone.app_input.poll"]) { + response = OPAppInputPoll(request); + } else if ([command isEqualToString:@"app_input_complete"] || + [command isEqualToString:@"openphone.app_input.complete"]) { + response = OPAppInputComplete(request); + } else { + response = OPError(@"app_ui_intake_command_rejected"); + } + } + NSData *responseData = OPJSONData(response); + OPWriteAll(clientFd, responseData); + } + close(clientFd); + } + close(fd); + OPAppUIIntakeFd = -1; + OPAppUIIntakeReady = 0; + } + return NULL; +} + +static void OPStartAppUIIntakeServer(void) { + if (OPAppUIIntakeThreadStarted) { + return; + } + OPAppUIIntakeThreadStarted = 1; + pthread_t thread; + int rc = pthread_create(&thread, NULL, OPAppUIIntakeThreadMain, NULL); + if (rc != 0) { + OPAppUIIntakeThreadStarted = 0; + OPAppUIIntakeStartError = [NSString stringWithFormat:@"pthread_create_failed:%d", rc]; + OPLog(@"app ui intake thread failed: %@", OPAppUIIntakeStartError); + return; + } + pthread_detach(thread); +} + +// Raise the jetsam band + task limit for `pid`. Returns the band actually +// applied (0 if every attempt was rejected) and writes the last errno/rc out. +// Runs in whatever uid the caller has: as mobile it usually gets EPERM; the +// setuid-root protected-data helper can call this successfully. +static int32_t OPApplyJetsamPriority(int32_t pid, int32_t requestedBand, + int32_t limitMB, int *rcOut, int *errnoOut) { + memorystatus_priority_properties_t props; + props.priority = requestedBand > 0 ? requestedBand : JETSAM_PRIORITY_AUDIO_AND_ACCESSORY; + props.user_data = 0; + int rc = memorystatus_control(MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES, + pid, 0, &props, sizeof(props)); + if (rc != 0 && props.priority != JETSAM_PRIORITY_FOREGROUND) { + // Fall back to plain foreground band (100). + props.priority = JETSAM_PRIORITY_FOREGROUND; + rc = memorystatus_control(MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES, + pid, 0, &props, sizeof(props)); + } + if (rcOut) *rcOut = rc; + if (errnoOut) *errnoOut = rc != 0 ? errno : 0; + if (limitMB > 0) { + int32_t mb = limitMB; + (void)memorystatus_control(MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT, + pid, 0, &mb, sizeof(mb)); + } + return rc == 0 ? props.priority : 0; +} + +// Helper-role command: raise the requesting agentd's jetsam band from root. +static NSDictionary *OPJetsamPrioritySet(NSDictionary *request) { + int32_t pid = (int32_t)OPLongLongFromRequest(request, @"pid", 0, 0, INT32_MAX); + if (pid <= 0) { + return OPError(@"jetsam_priority_pid_missing"); + } + int32_t band = (int32_t)OPLongLongFromRequest(request, @"band", + JETSAM_PRIORITY_AUDIO_AND_ACCESSORY, 0, JETSAM_PRIORITY_MAX); + int32_t limitMB = (int32_t)OPLongLongFromRequest(request, @"limit_mb", 256, 0, 4096); + int rc = 0, err = 0; + int32_t applied = OPApplyJetsamPriority(pid, band, limitMB, &rc, &err); + return @{ + @"status": applied > 0 ? @"ok" : @"error", + @"band": @(applied), + @"requested_band": @(band), + @"limit_mb": @(limitMB), + @"rc": @(rc), + @"errno": @(err), + @"pid": @(pid), + @"euid": @(geteuid()), + @"source": @"openphone.agentd" + }; +} + +static void OPRaiseJetsamPriority(void) { + // Long model tasks temporarily hold screenshots + prompt text + HTTP body — + // 256 MB is conservative headroom, still below iPhone's per-process cap. + int rc = 0, err = 0; + int32_t band = OPApplyJetsamPriority(getpid(), + JETSAM_PRIORITY_AUDIO_AND_ACCESSORY, 256, &rc, &err); + if (band > 0) { + OPLog(@"jetsam priority raised band=%d rc=%d limit_mb=256 via=self", (int)band, rc); + return; + } + OPLog(@"jetsam priority set failed rc=%d errno=%d; delegating to root helper", rc, err); + // mobile-uid daemon got EPERM. Route the bump through the setuid-root + // protected-data helper, which can set our band from root. + NSDictionary *helper = OPProtectedDataHelperRequest(@{ + @"command": @"jetsam_priority_set", + @"pid": @(getpid()), + @"band": @(JETSAM_PRIORITY_AUDIO_AND_ACCESSORY), + @"limit_mb": @256 + }); + if ([helper[@"status"] isEqualToString:@"ok"]) { + OPLog(@"jetsam priority raised band=%@ via=root_helper", helper[@"band"] ?: @0); + } else { + OPLog(@"jetsam priority root-helper delegation failed: %@", + helper[@"reason"] ?: helper[@"errno"] ?: @"unavailable"); + } +} + +int main(int argc, char **argv) { + (void)argc; + (void)argv; + @autoreleasepool { + NSDate *startedAt = [NSDate date]; + OPProcessStartMs = (long long)([startedAt timeIntervalSince1970] * 1000.0); + signal(SIGTERM, OPHandleSignal); + signal(SIGINT, OPHandleSignal); + signal(SIGPIPE, SIG_IGN); + + OPRaiseJetsamPriority(); + OPEnsureDirectories(); + OPServerFd = OPCreateServerSocket(); + if (OPServerFd < 0) { + return 1; + } + OPLog(@"openphone-agentd %@ started socket=%@", OPAgentVersion, OPSocketPath()); + if (!OPProtectedDataHelperRole() && !OPEnvFlagEnabled("OPENPHONE_DISABLE_PROTECTED_DATA_HELPER")) { + BOOL helperReady = OPProtectedDataHelperSocketConnectable(); + OPLog(@"protected data helper %@ socket=%@ error=%@", + helperReady ? @"ready" : @"not_ready", + OPProtectedDataHelperSocketPath(), + OPProtectedDataHelperLastSpawnError ?: @""); + } + if (!OPProtectedDataHelperRole() && !OPEnvFlagEnabled("OPENPHONE_AGENTD_DISABLE_TASK_REPAIR")) { + // Resume any task that was active within the last 60s (likely + // interrupted by a jetsam kill mid-run) by re-enqueueing it as a + // fresh async run with the same goal. Older tasks still fail. + NSUInteger resumedCount = 0; + long long nowMs = OPNowMs(); + NSArray *taskFiles = [[NSFileManager defaultManager] + contentsOfDirectoryAtPath:OPTasksPath() error:nil] ?: @[]; + NSMutableArray *resumeIds = [NSMutableArray array]; + for (NSString *file in taskFiles) { + if (![file.pathExtension isEqualToString:@"json"]) continue; + NSDictionary *task = OPReadJSONFile( + [OPTasksPath() stringByAppendingPathComponent:file]); + if (![task isKindOfClass:[NSDictionary class]]) continue; + NSString *status = [task[@"status"] isKindOfClass:[NSString class]] + ? task[@"status"] : @""; + if (![status isEqualToString:@"active"]) continue; + long long updatedAt = [task[@"updated_at"] respondsToSelector:@selector(longLongValue)] + ? [task[@"updated_at"] longLongValue] : 0; + long long createdAt = [task[@"created_at"] respondsToSelector:@selector(longLongValue)] + ? [task[@"created_at"] longLongValue] : 0; + long long activityAt = updatedAt > 0 ? updatedAt : createdAt; + long long ageMs = MAX(0, nowMs - activityAt); + NSString *goal = [task[@"goal"] isKindOfClass:[NSString class]] + ? task[@"goal"] : @""; + NSString *taskId = [task[@"task_id"] isKindOfClass:[NSString class]] + ? task[@"task_id"] : @""; + if (ageMs <= 60000 && goal.length > 0 && taskId.length > 0) { + // Mark original task as resumed so the fail-repair sweep + // below skips it (status != "active"), and enqueue fresh + // work with the same goal. + OPUpdateTask(taskId, @"resumed_after_restart", @{ + @"resumed_at_ms": @(nowMs), + @"stop_reason": @"resumed_after_daemon_restart", + @"result": @{ + @"status": @"ok", + @"state": @"task.resumed", + @"reason": @"resumed_after_daemon_restart", + @"task_id": taskId, + @"source": @"openphone.agentd" + } + }); + OPRecordTrajectory(taskId, @"task_resumed_after_restart", @{ + @"task_id": taskId, + @"age_ms": @(ageMs), + @"goal": goal + }); + OPStartAsyncRunTask(@{ + @"command": @"run_task", + @"goal": goal, + @"mode": @"auto", + @"source": @"daemon_startup_resume", + @"reason": @"resumed after daemon restart", + @"trigger": @"resume", + @"trigger_input": @"resume_after_restart", + @"max_steps": @25, + @"max_duration_ms": @600000, + @"run_task": @YES + }); + OPIslandUpdate(@{ + @"mode": @"thinking", + @"subtitle": @"Resuming", + @"goal": goal, + @"accent": @"blue" + }); + [resumeIds addObject:taskId]; + resumedCount++; + } + } + if (resumedCount > 0) { + OPLog(@"task startup resumed count=%lu ids=%@", + (unsigned long)resumedCount, + [resumeIds componentsJoinedByString:@","]); + } + + NSDictionary *taskRepair = OPRepairStaleActiveTasks(@{ + @"source": @"daemon_startup", + @"stale_after_ms": @0, + @"limit": @50, + @"reason": @"daemon startup recovered active tasks from a previous process" + }); + long long taskRepairCount = [taskRepair[@"repaired_count"] respondsToSelector:@selector(longLongValue)] + ? [taskRepair[@"repaired_count"] longLongValue] : 0; + if (taskRepairCount > 0) { + OPLog(@"task startup recovery repaired_count=%lld", taskRepairCount); + } + } + if (!OPEnvFlagEnabled("OPENPHONE_AGENTD_DISABLE_VOLUME_TRIGGER")) { + OPStartVolumeTriggerListener(); + } + if (!OPEnvFlagEnabled("OPENPHONE_AGENTD_DISABLE_APP_UI_INTAKE")) { + OPStartAppUIIntakeServer(); + } + if (!OPEnvFlagEnabled("OPENPHONE_AGENTD_DISABLE_BACKGROUND_SCHEDULER")) { + OPStartBackgroundJobScheduler(); + } + + while (OPRunning) { + int clientFd = accept(OPServerFd, NULL, NULL); + if (clientFd < 0) { + if (errno == EINTR || !OPRunning) { + continue; + } + OPLog(@"accept failed: %s", strerror(errno)); + continue; + } + + struct timeval timeout; + timeout.tv_sec = 2; + timeout.tv_usec = 0; + setsockopt(clientFd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + + @autoreleasepool { + NSData *requestData = OPReadClient(clientFd); + NSDictionary *request = OPParseRequest(requestData); + NSDictionary *response = nil; + @try { + response = OPHandleRequest(request, startedAt); + } @catch (NSException *exception) { + NSString *command = [request[@"command"] isKindOfClass:[NSString class]] + ? request[@"command"] : @"unknown"; + OPLog(@"request exception command=%@ exception=%@ reason=%@", + command, exception.name ?: @"NSException", exception.reason ?: @""); + response = OPError([NSString stringWithFormat:@"request_exception:%@", + exception.name ?: @"NSException"]); + } + NSData *responseData = OPJSONData(response); + if (!OPWriteAll(clientFd, responseData)) { + OPLog(@"client write failed errno=%d", errno); + } + } + close(clientFd); + } + + if (OPServerFd >= 0) { + close(OPServerFd); + OPServerFd = -1; + } + if (OPAppUIIntakeFd >= 0) { + close(OPAppUIIntakeFd); + OPAppUIIntakeFd = -1; + } + unlink(OPSocketPath().UTF8String); + OPLog(@"openphone-agentd stopped"); + } + return 0; +} diff --git a/ios/agentd/src/prefs/OpenPhoneAgentPrefsRootListController.m b/ios/agentd/src/prefs/OpenPhoneAgentPrefsRootListController.m new file mode 100644 index 0000000..85db4ea --- /dev/null +++ b/ios/agentd/src/prefs/OpenPhoneAgentPrefsRootListController.m @@ -0,0 +1,561 @@ +#import +#import +#import + +#import +#import +#import +#import +#import +#import +#import + +static NSString *const OPAgentPrefsSocketPath = @"/var/mobile/Library/OpenPhone/run/agentd.sock"; + +static BOOL OPAgentPrefsWriteAll(int fd, NSData *data) { + const uint8_t *bytes = (const uint8_t *)data.bytes; + NSUInteger remaining = data.length; + while (remaining > 0) { + ssize_t written = write(fd, bytes, remaining); + if (written > 0) { + bytes += written; + remaining -= (NSUInteger)written; + continue; + } + if (written < 0 && errno == EINTR) { + continue; + } + return NO; + } + return YES; +} + +static NSString *OPAgentPrefsString(id value) { + if ([value isKindOfClass:[NSString class]]) { + return value; + } + if ([value isKindOfClass:[NSNumber class]]) { + return [(NSNumber *)value stringValue]; + } + return @""; +} + +static NSString *OPAgentPrefsBoolString(id value) { + if (![value respondsToSelector:@selector(boolValue)]) { + return @"unknown"; + } + return [value boolValue] ? @"on" : @"off"; +} + +static NSNumber *OPAgentPrefsBoolNumber(id value, BOOL fallback) { + if ([value respondsToSelector:@selector(boolValue)]) { + return @([value boolValue]); + } + return @(fallback); +} + +static id OPAgentPrefsNestedValue(NSDictionary *dictionary, NSArray *keys) { + id current = dictionary; + for (NSString *key in keys) { + if (![current isKindOfClass:[NSDictionary class]]) { + return nil; + } + current = [(NSDictionary *)current objectForKey:key]; + if (!current || current == (id)kCFNull) { + return nil; + } + } + return current; +} + +static NSDictionary *OPAgentPrefsSendRequest(NSDictionary *request, NSError **errorOut) { + NSData *requestData = [NSJSONSerialization dataWithJSONObject:request ?: @{} + options:0 error:errorOut]; + if (requestData.length == 0) { + return nil; + } + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + if (errorOut) { + *errorOut = [NSError errorWithDomain:NSPOSIXErrorDomain code:errno userInfo:nil]; + } + return nil; + } + + struct sockaddr_un address; + memset(&address, 0, sizeof(address)); + address.sun_family = AF_UNIX; + strlcpy(address.sun_path, OPAgentPrefsSocketPath.UTF8String, sizeof(address.sun_path)); + + if (connect(fd, (struct sockaddr *)&address, sizeof(address)) != 0) { + if (errorOut) { + *errorOut = [NSError errorWithDomain:NSPOSIXErrorDomain code:errno userInfo:@{ + NSLocalizedDescriptionKey: [NSString stringWithFormat:@"connect %@ failed: %s", + OPAgentPrefsSocketPath, strerror(errno)] + }]; + } + close(fd); + return nil; + } + + NSMutableData *lineData = [requestData mutableCopy]; + const char newline = '\n'; + [lineData appendBytes:&newline length:1]; + if (!OPAgentPrefsWriteAll(fd, lineData)) { + if (errorOut) { + *errorOut = [NSError errorWithDomain:NSPOSIXErrorDomain code:errno userInfo:nil]; + } + close(fd); + return nil; + } + shutdown(fd, SHUT_WR); + + NSMutableData *responseData = [NSMutableData data]; + char buffer[4096]; + while (true) { + ssize_t count = read(fd, buffer, sizeof(buffer)); + if (count > 0) { + [responseData appendBytes:buffer length:(NSUInteger)count]; + continue; + } + if (count < 0 && errno == EINTR) { + continue; + } + break; + } + close(fd); + + if (responseData.length == 0) { + if (errorOut) { + *errorOut = [NSError errorWithDomain:@"OpenPhoneAgentPrefs" code:1 userInfo:@{ + NSLocalizedDescriptionKey: @"empty agentd response" + }]; + } + return nil; + } + id object = [NSJSONSerialization JSONObjectWithData:responseData options:0 error:errorOut]; + return [object isKindOfClass:[NSDictionary class]] ? object : nil; +} + +@interface OpenPhoneAgentPrefsRootListController : PSListController +@property (nonatomic, retain) NSDictionary *latestStatus; +@property (nonatomic, copy) NSString *lastError; +@end + +@implementation OpenPhoneAgentPrefsRootListController + +- (instancetype)init { + self = [super init]; + if (self) { + signal(SIGPIPE, SIG_IGN); + } + return self; +} + +- (void)viewDidLoad { + [super viewDidLoad]; + self.title = @"OpenPhone Agent"; +} + +- (NSArray *)specifiers { + if (!_specifiers) { + [self reloadAgentStatus]; + _specifiers = [[self buildSpecifiers] mutableCopy]; + } + return _specifiers; +} + +- (void)reloadAgentStatus { + NSError *error = nil; + NSDictionary *status = OPAgentPrefsSendRequest(@{@"command": @"agent_status", @"limit": @5}, &error); + if ([status isKindOfClass:[NSDictionary class]] && [status[@"status"] isEqualToString:@"ok"]) { + self.latestStatus = status; + self.lastError = @""; + return; + } + self.latestStatus = status ?: @{}; + self.lastError = error.localizedDescription ?: OPAgentPrefsString(status[@"reason"]); + if (self.lastError.length == 0) { + self.lastError = @"agent_status failed"; + } +} + +- (NSMutableArray *)buildSpecifiers { + NSMutableArray *items = [NSMutableArray array]; + + PSSpecifier *runtimeGroup = [PSSpecifier groupSpecifierWithName:@"Runtime"]; + [runtimeGroup setProperty:@"Phone-local daemon status. This pane talks to openphone-agentd over the on-device Unix socket." forKey:PSFooterTextGroupKey]; + [items addObject:runtimeGroup]; + + [items addObject:[self valueSpecifierNamed:@"Status" value:[self statusSummary]]]; + [items addObject:[self valueSpecifierNamed:@"State" value:OPAgentPrefsString(self.latestStatus[@"state"])]]; + [items addObject:[self valueSpecifierNamed:@"Policy" value:[self policySummary]]]; + [items addObject:[self valueSpecifierNamed:@"Model" value:[self modelSummary]]]; + + PSSpecifier *policyGroup = [PSSpecifier groupSpecifierWithName:@"Agent Policy"]; + [policyGroup setProperty:@"These controls update openphone-agentd policy on the phone. They do not edit Mac-side config." forKey:PSFooterTextGroupKey]; + [items addObject:policyGroup]; + [items addObject:[self switchSpecifierNamed:@"Hardware Triggers" key:@"hardware_triggers_enabled"]]; + [items addObject:[self switchSpecifierNamed:@"YOLO Execution" key:@"yolo_enabled"]]; + [items addObject:[self autonomyModeSpecifier]]; + + PSSpecifier *modelGroup = [PSSpecifier groupSpecifierWithName:@"Model"]; + [modelGroup setProperty:@"Model provider and capture tuning. Provider credentials are external and never shown here." forKey:PSFooterTextGroupKey]; + [items addObject:modelGroup]; + [items addObject:[self modelProviderSpecifier]]; + [items addObject:[self valueSpecifierNamed:@"Model" value:[self nestedString:@[@"model", @"model"] fallback:@"unset"]]]; + [items addObject:[self valueSpecifierNamed:@"Screenshot Max Edge" value:[self screenshotDimensionSummary]]]; + + PSSpecifier *triggerGroup = [PSSpecifier groupSpecifierWithName:@"Volume Trigger"]; + [items addObject:triggerGroup]; + [items addObject:[self valueSpecifierNamed:@"SpringBoard Hook" value:[self triggerSummary]]]; + [items addObject:[self valueSpecifierNamed:@"Button Events" value:[self triggerCounterSummary]]]; + [items addObject:[self valueSpecifierNamed:@"Last Route" value:[self nestedString:@[@"triggers", @"volume_combo", @"springboard_fallback", @"last_trigger_route"] fallback:@"none"]]]; + + PSSpecifier *taskGroup = [PSSpecifier groupSpecifierWithName:@"Latest Task"]; + [items addObject:taskGroup]; + [items addObject:[self valueSpecifierNamed:@"Task" value:[self latestTaskSummary]]]; + [items addObject:[self valueSpecifierNamed:@"Current Task" value:[self currentTaskSummary]]]; + [items addObject:[self valueSpecifierNamed:@"Stop Reason" value:[self nestedString:@[@"latest_task", @"stop_reason"] fallback:@"none"]]]; + + PSSpecifier *controlGroup = [PSSpecifier groupSpecifierWithName:@"Controls"]; + [items addObject:controlGroup]; + [items addObject:[self buttonSpecifierNamed:@"Refresh" action:@selector(refreshTapped)]]; + [items addObject:[self buttonSpecifierNamed:@"Pause YOLO Triggers" action:@selector(pauseTapped)]]; + [items addObject:[self buttonSpecifierNamed:@"Resume YOLO Triggers" action:@selector(resumeTapped)]]; + [items addObject:[self buttonSpecifierNamed:@"Stop Current Task" action:@selector(stopCurrentTaskTapped)]]; + [items addObject:[self buttonSpecifierNamed:@"Smaller Screenshots" action:@selector(shrinkScreenshotTapped)]]; + [items addObject:[self buttonSpecifierNamed:@"Larger Screenshots" action:@selector(growScreenshotTapped)]]; + + if (self.lastError.length > 0) { + PSSpecifier *errorGroup = [PSSpecifier groupSpecifierWithName:@"Error"]; + [items addObject:errorGroup]; + [items addObject:[self valueSpecifierNamed:@"Daemon" value:self.lastError]]; + } + return items; +} + +- (PSSpecifier *)switchSpecifierNamed:(NSString *)name key:(NSString *)key { + PSSpecifier *specifier = [PSSpecifier preferenceSpecifierNamed:name + target:self + set:@selector(setPreferenceValue:specifier:) + get:@selector(readPreferenceValue:) + detail:nil + cell:PSSwitchCell + edit:nil]; + [specifier setProperty:key ?: @"" forKey:PSKeyNameKey]; + [specifier setProperty:key ?: @"" forKey:PSIDKey]; + return specifier; +} + +- (PSSpecifier *)valueSpecifierNamed:(NSString *)name value:(NSString *)value { + PSSpecifier *specifier = [PSSpecifier preferenceSpecifierNamed:name + target:self set:nil get:nil detail:nil cell:PSTitleValueCell edit:nil]; + [specifier setProperty:value.length > 0 ? value : @"unknown" forKey:PSValueKey]; + [specifier setProperty:@YES forKey:PSCopyableCellKey]; + return specifier; +} + +- (PSSpecifier *)buttonSpecifierNamed:(NSString *)name action:(SEL)action { + PSSpecifier *specifier = [PSSpecifier preferenceSpecifierNamed:name + target:self set:nil get:nil detail:nil cell:PSButtonCell edit:nil]; + specifier->action = action; + specifier.buttonAction = action; + [specifier setProperty:NSStringFromSelector(action) forKey:PSButtonActionKey]; + return specifier; +} + +- (PSSpecifier *)autonomyModeSpecifier { + PSSpecifier *specifier = [PSSpecifier preferenceSpecifierNamed:@"Autonomy Mode" + target:self + set:@selector(setPreferenceValue:specifier:) + get:@selector(readPreferenceValue:) + detail:[PSListItemsController class] + cell:PSLinkListCell + edit:nil]; + [specifier setProperty:@"autonomy_mode" forKey:PSKeyNameKey]; + [specifier setProperty:@"autonomy_mode" forKey:PSIDKey]; + [specifier setProperty:@[@"yolo", @"reviewed", @"dry_run"] forKey:@"values"]; + [specifier setProperty:@[@"YOLO (execute)", @"Reviewed (approve each UI action)", + @"Dry Run (refuse UI mutations)"] forKey:@"titles"]; + return specifier; +} + +- (PSSpecifier *)modelProviderSpecifier { + PSSpecifier *specifier = [PSSpecifier preferenceSpecifierNamed:@"Provider" + target:self + set:@selector(setPreferenceValue:specifier:) + get:@selector(readPreferenceValue:) + detail:[PSListItemsController class] + cell:PSLinkListCell + edit:nil]; + [specifier setProperty:@"model_mode" forKey:PSKeyNameKey]; + [specifier setProperty:@"model_mode" forKey:PSIDKey]; + [specifier setProperty:@[@"bedrock_converse", @"openai_realtime", @"broker"] forKey:@"values"]; + [specifier setProperty:@[@"Bedrock", @"OpenAI Realtime", @"Broker"] forKey:@"titles"]; + return specifier; +} + +- (id)readPreferenceValue:(PSSpecifier *)specifier { + NSString *key = OPAgentPrefsString([specifier propertyForKey:PSKeyNameKey]); + if ([key isEqualToString:@"hardware_triggers_enabled"]) { + return OPAgentPrefsBoolNumber(OPAgentPrefsNestedValue(self.latestStatus, + @[@"control", @"hardware_triggers_enabled"]), YES); + } + if ([key isEqualToString:@"yolo_enabled"]) { + return OPAgentPrefsBoolNumber(OPAgentPrefsNestedValue(self.latestStatus, + @[@"control", @"yolo_enabled"]), YES); + } + if ([key isEqualToString:@"autonomy_mode"]) { + NSString *mode = OPAgentPrefsString(OPAgentPrefsNestedValue(self.latestStatus, + @[@"control", @"autonomy_mode"])); + return mode.length > 0 ? mode : @"yolo"; + } + if ([key isEqualToString:@"model_mode"]) { + NSString *mode = OPAgentPrefsString(OPAgentPrefsNestedValue(self.latestStatus, + @[@"model", @"mode"])); + return mode.length > 0 ? mode : @"broker"; + } + return @NO; +} + +- (void)setPreferenceValue:(id)value specifier:(PSSpecifier *)specifier { + NSString *key = OPAgentPrefsString([specifier propertyForKey:PSKeyNameKey]); + if (key.length == 0) { + return; + } + if ([key isEqualToString:@"autonomy_mode"]) { + [self applyAutonomyMode:OPAgentPrefsString(value)]; + return; + } + if ([key isEqualToString:@"model_mode"]) { + [self applyModelMode:OPAgentPrefsString(value)]; + return; + } + BOOL enabled = [value respondsToSelector:@selector(boolValue)] ? [value boolValue] : NO; + NSError *error = nil; + NSDictionary *result = OPAgentPrefsSendRequest(@{ + @"command": @"agent_control", + key: @(enabled), + @"reason": [NSString stringWithFormat:@"OpenPhone Settings %@ %@", + key, enabled ? @"enabled" : @"disabled"], + @"source": @"OpenPhoneAgentPrefs" + }, &error); + if (![result isKindOfClass:[NSDictionary class]] || ![result[@"status"] isEqualToString:@"ok"]) { + NSString *message = error.localizedDescription ?: OPAgentPrefsString(result[@"reason"]); + [self showMessage:message.length > 0 ? message : @"agent_control failed" title:@"OpenPhone Agent"]; + } + [self refreshTapped]; +} + +- (void)applyAutonomyMode:(NSString *)mode { + if (mode.length == 0) { + return; + } + NSError *error = nil; + NSDictionary *result = OPAgentPrefsSendRequest(@{ + @"command": @"agent_control", + @"autonomy_mode": mode, + @"reason": [NSString stringWithFormat:@"OpenPhone Settings autonomy_mode %@", mode], + @"source": @"OpenPhoneAgentPrefs" + }, &error); + if (![result isKindOfClass:[NSDictionary class]] || ![result[@"status"] isEqualToString:@"ok"]) { + NSString *message = error.localizedDescription ?: OPAgentPrefsString(result[@"reason"]); + [self showMessage:message.length > 0 ? message : @"agent_control failed" title:@"OpenPhone Agent"]; + } + [self refreshTapped]; +} + +- (void)applyModelMode:(NSString *)mode { + if (mode.length == 0) { + return; + } + NSError *error = nil; + NSDictionary *result = OPAgentPrefsSendRequest(@{ + @"command": @"model_configure", + @"mode": mode, + @"reason": [NSString stringWithFormat:@"OpenPhone Settings model provider %@", mode], + @"source": @"OpenPhoneAgentPrefs" + }, &error); + if (![result isKindOfClass:[NSDictionary class]] || ![result[@"status"] isEqualToString:@"ok"]) { + NSString *message = error.localizedDescription ?: OPAgentPrefsString(result[@"reason"]); + [self showMessage:message.length > 0 ? message : @"model_configure failed" title:@"OpenPhone Agent"]; + } + [self refreshTapped]; +} + +- (NSString *)nestedString:(NSArray *)keys fallback:(NSString *)fallback { + NSString *value = OPAgentPrefsString(OPAgentPrefsNestedValue(self.latestStatus, keys)); + return value.length > 0 ? value : (fallback ?: @""); +} + +- (NSString *)statusSummary { + if (self.lastError.length > 0) { + return @"unavailable"; + } + return [self nestedString:@[@"user_facing", @"summary"] fallback:@"unknown"]; +} + +- (NSString *)policySummary { + NSString *policy = [self nestedString:@[@"control", @"trigger_policy"] fallback:@"unknown"]; + NSString *hardware = OPAgentPrefsBoolString(OPAgentPrefsNestedValue(self.latestStatus, @[@"control", @"hardware_triggers_enabled"])); + NSString *yolo = OPAgentPrefsBoolString(OPAgentPrefsNestedValue(self.latestStatus, @[@"control", @"yolo_enabled"])); + return [NSString stringWithFormat:@"%@, hardware %@, yolo %@", policy, hardware, yolo]; +} + +- (NSString *)modelSummary { + NSString *mode = [self nestedString:@[@"model", @"mode"] fallback:@"unknown"]; + NSString *status = [self nestedString:@[@"model", @"status"] fallback:@"unknown"]; + return [NSString stringWithFormat:@"%@ / %@", mode, status]; +} + +- (long long)currentScreenshotDimension { + id value = OPAgentPrefsNestedValue(self.latestStatus, @[@"model", @"screenshot_max_dimension_px"]); + if ([value respondsToSelector:@selector(longLongValue)]) { + return [value longLongValue]; + } + return 1024; +} + +- (NSString *)screenshotDimensionSummary { + long long dimension = [self currentScreenshotDimension]; + NSString *quality = [self nestedString:@[@"model", @"screenshot_jpeg_quality_x100"] fallback:@"60"]; + if (dimension <= 0) { + return [NSString stringWithFormat:@"full res, q%@", quality]; + } + return [NSString stringWithFormat:@"%lld px, q%@", dimension, quality]; +} + +- (NSString *)triggerSummary { + NSString *status = [self nestedString:@[@"triggers", @"volume_combo", @"springboard_fallback", @"status"] fallback:@"unknown"]; + id hooked = OPAgentPrefsNestedValue(self.latestStatus, @[@"triggers", @"volume_combo", @"springboard_fallback", @"hooks", @"volume_hooked"]); + id total = OPAgentPrefsNestedValue(self.latestStatus, @[@"triggers", @"volume_combo", @"springboard_fallback", @"hooks", @"volume_total"]); + if (hooked || total) { + return [NSString stringWithFormat:@"%@, %@/%@ volume hooks", status, + OPAgentPrefsString(hooked), OPAgentPrefsString(total)]; + } + return status; +} + +- (NSString *)triggerCounterSummary { + NSString *buttons = [self nestedString:@[@"triggers", @"volume_combo", @"springboard_fallback", @"button_events_seen"] fallback:@"0"]; + NSString *combos = [self nestedString:@[@"triggers", @"volume_combo", @"springboard_fallback", @"combo_events_seen"] fallback:@"0"]; + return [NSString stringWithFormat:@"buttons %@, combos %@", buttons, combos]; +} + +- (NSString *)latestTaskSummary { + NSString *taskId = [self nestedString:@[@"latest_task", @"task_id"] fallback:@"none"]; + NSString *status = [self nestedString:@[@"latest_task", @"status"] fallback:@"unknown"]; + NSString *tool = [self nestedString:@[@"latest_task", @"model_loop_tool"] fallback:@""]; + if (tool.length > 0) { + return [NSString stringWithFormat:@"%@ / %@ / %@", taskId, status, tool]; + } + return [NSString stringWithFormat:@"%@ / %@", taskId, status]; +} + +- (NSString *)currentTaskSummary { + NSString *taskId = [self nestedString:@[@"current_task", @"task_id"] fallback:@""]; + if (taskId.length == 0 || [taskId isEqualToString:@"none"]) { + return @"none"; + } + NSString *status = [self nestedString:@[@"current_task", @"status"] fallback:@"unknown"]; + NSString *step = [self nestedString:@[@"current_task", @"current_step"] fallback:@"0"]; + return [NSString stringWithFormat:@"%@ / %@ / step %@", taskId, status, step]; +} + +- (void)refreshTapped { + [self reloadAgentStatus]; + self.specifiers = [[self buildSpecifiers] mutableCopy]; + [self reloadSpecifiers]; +} + +- (void)pauseTapped { + [self sendControlAction:@"pause" reason:@"OpenPhone Settings pause"]; +} + +- (void)resumeTapped { + [self sendControlAction:@"resume" reason:@"OpenPhone Settings resume"]; +} + +- (void)stopCurrentTaskTapped { + NSString *taskId = [self nestedString:@[@"current_task", @"task_id"] fallback:@""]; + if (taskId.length == 0) { + [self showMessage:@"No active task is currently running." title:@"OpenPhone Agent"]; + return; + } + NSError *error = nil; + NSDictionary *result = OPAgentPrefsSendRequest(@{ + @"command": @"stop_task", + @"task_id": taskId, + @"reason": @"OpenPhone Settings stop current task" + }, &error); + if (![result isKindOfClass:[NSDictionary class]] || ![result[@"status"] isEqualToString:@"ok"]) { + NSString *message = error.localizedDescription ?: OPAgentPrefsString(result[@"reason"]); + [self showMessage:message.length > 0 ? message : @"stop_task failed" title:@"OpenPhone Agent"]; + return; + } + [self refreshTapped]; +} + +- (void)shrinkScreenshotTapped { + [self adjustScreenshotDimensionByFactor:0.5]; +} + +- (void)growScreenshotTapped { + [self adjustScreenshotDimensionByFactor:2.0]; +} + +- (void)adjustScreenshotDimensionByFactor:(double)factor { + long long current = [self currentScreenshotDimension]; + if (current <= 0) { + current = 1024; + } + long long next = (long long)llround((double)current * factor); + if (next < 256) { + next = 256; + } + if (next > 4096) { + next = 4096; + } + NSString *currentMode = [self nestedString:@[@"model", @"mode"] fallback:@"broker"]; + NSError *error = nil; + NSDictionary *result = OPAgentPrefsSendRequest(@{ + @"command": @"model_configure", + @"mode": currentMode, + @"screenshot_max_dimension_px": @(next), + @"reason": [NSString stringWithFormat:@"OpenPhone Settings screenshot_max_dimension_px %lld", next], + @"source": @"OpenPhoneAgentPrefs" + }, &error); + if (![result isKindOfClass:[NSDictionary class]] || ![result[@"status"] isEqualToString:@"ok"]) { + NSString *message = error.localizedDescription ?: OPAgentPrefsString(result[@"reason"]); + [self showMessage:message.length > 0 ? message : @"model_configure failed" title:@"OpenPhone Agent"]; + return; + } + [self refreshTapped]; +} + +- (void)sendControlAction:(NSString *)action reason:(NSString *)reason { + NSError *error = nil; + NSDictionary *result = OPAgentPrefsSendRequest(@{ + @"command": @"agent_control", + @"action": action ?: @"status", + @"reason": reason ?: @"OpenPhone Settings", + @"source": @"OpenPhoneAgentPrefs" + }, &error); + if (![result isKindOfClass:[NSDictionary class]] || ![result[@"status"] isEqualToString:@"ok"]) { + NSString *message = error.localizedDescription ?: OPAgentPrefsString(result[@"reason"]); + [self showMessage:message.length > 0 ? message : @"agent_control failed" title:@"OpenPhone Agent"]; + return; + } + [self refreshTapped]; +} + +- (void)showMessage:(NSString *)message title:(NSString *)title { + UIAlertController *alert = [UIAlertController alertControllerWithTitle:title ?: @"OpenPhone Agent" + message:message ?: @"" + preferredStyle:UIAlertControllerStyleAlert]; + [alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:nil]]; + [self presentViewController:alert animated:YES completion:nil]; +} + +@end diff --git a/ios/agentd/src/screencap_helper.m b/ios/agentd/src/screencap_helper.m new file mode 100644 index 0000000..b8e8075 --- /dev/null +++ b/ios/agentd/src/screencap_helper.m @@ -0,0 +1,360 @@ +#import + +#import +#import +#import +#import +#import +#import +#import +#import +#import + +typedef int (*OPIOMFBGetMainDisplayFunc)(void **connection); +typedef int (*OPIOMFBOpenFunc)(uint32_t service, mach_port_t owningTask, + uint32_t type, void **connection); +typedef int (*OPIOMFBGetLayerDefaultSurfaceFunc)(void *connection, int layer, void **surface); +typedef CFMutableDictionaryRef (*OPIOServiceMatchingFunc)(const char *name); +typedef uint32_t (*OPIOServiceGetMatchingServiceFunc)(uint32_t mainPort, + CFDictionaryRef matching); +typedef int (*OPIOObjectReleaseFunc)(uint32_t object); +typedef int (*OPIOSurfaceLockFunc)(void *surface, uint32_t options, uint32_t *seed); +typedef int (*OPIOSurfaceUnlockFunc)(void *surface, uint32_t options, uint32_t *seed); +typedef void *(*OPIOSurfaceGetBaseAddressFunc)(void *surface); +typedef size_t (*OPIOSurfaceGetWidthFunc)(void *surface); +typedef size_t (*OPIOSurfaceGetHeightFunc)(void *surface); +typedef size_t (*OPIOSurfaceGetBytesPerRowFunc)(void *surface); +typedef uint32_t (*OPIOSurfaceGetPixelFormatFunc)(void *surface); + +static NSString *const OPDefaultStorePath = @"/var/mobile/Library/OpenPhone"; + +static NSString *OPNowFilename(void) { + long long ms = (long long)([[NSDate date] timeIntervalSince1970] * 1000.0); + return [NSString stringWithFormat:@"screen-%lld-%d.png", ms, getpid()]; +} + +static NSString *OPSHA256Hex(NSData *data) { + unsigned char digest[CC_SHA256_DIGEST_LENGTH]; + CC_SHA256(data.bytes, (CC_LONG)data.length, digest); + NSMutableString *hex = [NSMutableString stringWithCapacity:CC_SHA256_DIGEST_LENGTH * 2]; + for (NSUInteger i = 0; i < CC_SHA256_DIGEST_LENGTH; i++) { + [hex appendFormat:@"%02x", digest[i]]; + } + return hex; +} + +static void OPPrintJSON(NSDictionary *object) { + NSData *data = [NSJSONSerialization dataWithJSONObject:object ?: @{} + options:0 + error:nil]; + if (!data) { + data = [@"{\"status\":\"error\",\"reason\":\"json_encode_failed\"}" + dataUsingEncoding:NSUTF8StringEncoding]; + } + fwrite(data.bytes, 1, data.length, stdout); + fputc('\n', stdout); +} + +static NSString *OPOutputPath(int argc, const char *argv[]) { + if (argc >= 2 && argv[1] && argv[1][0] != '\0') { + return [NSString stringWithUTF8String:argv[1]]; + } + NSString *directory = [[OPDefaultStorePath stringByAppendingPathComponent:@"screenshots"] copy]; + [[NSFileManager defaultManager] createDirectoryAtPath:directory + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + return [directory stringByAppendingPathComponent:OPNowFilename()]; +} + +// Downscale so the longest edge is at most maxDimension points. Returns a new +// CGImage the caller must release, or NULL if no scaling was needed. +static CGImageRef OPCreateScaledImage(CGImageRef source, size_t maxDimension, + size_t *scaledWidthOut, size_t *scaledHeightOut) { + if (!source || maxDimension == 0) { + return NULL; + } + size_t width = CGImageGetWidth(source); + size_t height = CGImageGetHeight(source); + size_t longest = MAX(width, height); + if (longest <= maxDimension) { + return NULL; + } + double scale = (double)maxDimension / (double)longest; + size_t scaledWidth = MAX((size_t)1, (size_t)llround((double)width * scale)); + size_t scaledHeight = MAX((size_t)1, (size_t)llround((double)height * scale)); + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate(NULL, scaledWidth, scaledHeight, 8, + 0, colorSpace, kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big); + if (colorSpace) { + CGColorSpaceRelease(colorSpace); + } + if (!context) { + return NULL; + } + CGContextSetInterpolationQuality(context, kCGInterpolationMedium); + CGContextDrawImage(context, CGRectMake(0, 0, scaledWidth, scaledHeight), source); + CGImageRef scaled = CGBitmapContextCreateImage(context); + CGContextRelease(context); + if (scaled) { + if (scaledWidthOut) *scaledWidthOut = scaledWidth; + if (scaledHeightOut) *scaledHeightOut = scaledHeight; + } + return scaled; +} + +static NSDictionary *OPUnavailable(NSString *reason, NSDictionary *extra) { + NSMutableDictionary *result = [@{ + @"status": @"unavailable", + @"provider": @"IOMobileFramebuffer.IOSurface", + @"reason": reason ?: @"unknown" + } mutableCopy]; + [result addEntriesFromDictionary:extra ?: @{}]; + return result; +} + +static NSDictionary *OPCapture(NSString *path, size_t maxDimension, double jpegQuality) { + void *iomfb = dlopen("/System/Library/PrivateFrameworks/IOMobileFramebuffer.framework/IOMobileFramebuffer", + RTLD_LAZY); + void *iosurface = dlopen("/System/Library/Frameworks/IOSurface.framework/IOSurface", + RTLD_LAZY); + if (!iosurface) { + iosurface = dlopen("/System/Library/PrivateFrameworks/IOSurface.framework/IOSurface", + RTLD_LAZY); + } + if (!iomfb || !iosurface) { + return OPUnavailable(@"framework_unavailable", @{ + @"iomobileframebuffer_loaded": @(iomfb != NULL), + @"iosurface_loaded": @(iosurface != NULL) + }); + } + + OPIOMFBGetMainDisplayFunc getMainDisplay = + (OPIOMFBGetMainDisplayFunc)dlsym(iomfb, "IOMobileFramebufferGetMainDisplay"); + OPIOMFBOpenFunc openFramebuffer = + (OPIOMFBOpenFunc)dlsym(iomfb, "IOMobileFramebufferOpen"); + OPIOMFBGetLayerDefaultSurfaceFunc getLayerDefaultSurface = + (OPIOMFBGetLayerDefaultSurfaceFunc)dlsym(iomfb, "IOMobileFramebufferGetLayerDefaultSurface"); + OPIOSurfaceLockFunc surfaceLock = + (OPIOSurfaceLockFunc)dlsym(iosurface, "IOSurfaceLock"); + OPIOSurfaceUnlockFunc surfaceUnlock = + (OPIOSurfaceUnlockFunc)dlsym(iosurface, "IOSurfaceUnlock"); + OPIOSurfaceGetBaseAddressFunc surfaceBaseAddress = + (OPIOSurfaceGetBaseAddressFunc)dlsym(iosurface, "IOSurfaceGetBaseAddress"); + OPIOSurfaceGetWidthFunc surfaceWidth = + (OPIOSurfaceGetWidthFunc)dlsym(iosurface, "IOSurfaceGetWidth"); + OPIOSurfaceGetHeightFunc surfaceHeight = + (OPIOSurfaceGetHeightFunc)dlsym(iosurface, "IOSurfaceGetHeight"); + OPIOSurfaceGetBytesPerRowFunc surfaceBytesPerRow = + (OPIOSurfaceGetBytesPerRowFunc)dlsym(iosurface, "IOSurfaceGetBytesPerRow"); + OPIOSurfaceGetPixelFormatFunc surfacePixelFormat = + (OPIOSurfaceGetPixelFormatFunc)dlsym(iosurface, "IOSurfaceGetPixelFormat"); + + NSDictionary *symbols = @{ + @"IOMobileFramebufferGetMainDisplay": @(getMainDisplay != NULL), + @"IOMobileFramebufferOpen": @(openFramebuffer != NULL), + @"IOMobileFramebufferGetLayerDefaultSurface": @(getLayerDefaultSurface != NULL), + @"IOSurfaceLock": @(surfaceLock != NULL), + @"IOSurfaceUnlock": @(surfaceUnlock != NULL), + @"IOSurfaceGetBaseAddress": @(surfaceBaseAddress != NULL), + @"IOSurfaceGetWidth": @(surfaceWidth != NULL), + @"IOSurfaceGetHeight": @(surfaceHeight != NULL), + @"IOSurfaceGetBytesPerRow": @(surfaceBytesPerRow != NULL), + @"IOSurfaceGetPixelFormat": @(surfacePixelFormat != NULL) + }; + if (!getMainDisplay || !getLayerDefaultSurface || !surfaceLock || !surfaceUnlock + || !surfaceBaseAddress || !surfaceWidth || !surfaceHeight || !surfaceBytesPerRow) { + return OPUnavailable(@"required_symbol_unavailable", @{@"symbols": symbols}); + } + + void *connection = NULL; + int mainResult = getMainDisplay(&connection); + NSMutableDictionary *acquisition = [@{ + @"main_display_kern_return": @(mainResult), + @"main_display_ok": @(mainResult == 0 && connection != NULL) + } mutableCopy]; + if (mainResult != 0 || !connection) { + void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_LAZY); + if (!iokit) { + iokit = dlopen("/System/Library/PrivateFrameworks/IOKit.framework/IOKit", RTLD_LAZY); + } + OPIOServiceMatchingFunc serviceMatching = iokit + ? (OPIOServiceMatchingFunc)dlsym(iokit, "IOServiceMatching") : NULL; + OPIOServiceGetMatchingServiceFunc getMatchingService = iokit + ? (OPIOServiceGetMatchingServiceFunc)dlsym(iokit, "IOServiceGetMatchingService") : NULL; + OPIOObjectReleaseFunc objectRelease = iokit + ? (OPIOObjectReleaseFunc)dlsym(iokit, "IOObjectRelease") : NULL; + acquisition[@"iokit_loaded"] = @(iokit != NULL); + acquisition[@"IOServiceMatching"] = @(serviceMatching != NULL); + acquisition[@"IOServiceGetMatchingService"] = @(getMatchingService != NULL); + acquisition[@"IOObjectRelease"] = @(objectRelease != NULL); + acquisition[@"open_symbol"] = @(openFramebuffer != NULL); + if (openFramebuffer && serviceMatching && getMatchingService) { + CFMutableDictionaryRef matching = serviceMatching("IOMobileFramebuffer"); + uint32_t service = matching ? getMatchingService(0, matching) : 0; + acquisition[@"service"] = @(service); + if (service != 0) { + int openResult = openFramebuffer(service, mach_task_self(), 0, &connection); + acquisition[@"open_kern_return"] = @(openResult); + acquisition[@"open_ok"] = @(openResult == 0 && connection != NULL); + if (objectRelease) { + objectRelease(service); + } + } + } + } + if (!connection) { + return OPUnavailable(@"display_connection_unavailable", @{ + @"kern_return": @(mainResult), + @"acquisition": acquisition, + @"symbols": symbols + }); + } + + void *surface = NULL; + int surfaceResult = getLayerDefaultSurface(connection, 0, &surface); + if (surfaceResult != 0 || !surface) { + return OPUnavailable(@"default_surface_unavailable", @{ + @"kern_return": @(surfaceResult), + @"symbols": symbols + }); + } + + const uint32_t readOnly = 1; + uint32_t seed = 0; + int lockResult = surfaceLock(surface, readOnly, &seed); + if (lockResult != 0) { + return OPUnavailable(@"surface_lock_failed", @{ + @"kern_return": @(lockResult), + @"symbols": symbols + }); + } + + void *base = surfaceBaseAddress(surface); + size_t width = surfaceWidth(surface); + size_t height = surfaceHeight(surface); + size_t bytesPerRow = surfaceBytesPerRow(surface); + uint32_t pixelFormat = surfacePixelFormat ? surfacePixelFormat(surface) : 0; + if (!base || width == 0 || height == 0 || bytesPerRow == 0) { + surfaceUnlock(surface, readOnly, &seed); + return OPUnavailable(@"surface_geometry_unavailable", @{ + @"width": @(width), + @"height": @(height), + @"bytes_per_row": @(bytesPerRow), + @"symbols": symbols + }); + } + + NSMutableData *pixels = [NSMutableData dataWithLength:bytesPerRow * height]; + memcpy(pixels.mutableBytes, base, pixels.length); + surfaceUnlock(surface, readOnly, &seed); + + NSString *directory = [path stringByDeletingLastPathComponent]; + [[NSFileManager defaultManager] createDirectoryAtPath:directory + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + + CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)pixels); + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGBitmapInfo bitmapInfo = kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst; + CGImageRef image = CGImageCreate(width, height, 8, 32, bytesPerRow, colorSpace, + bitmapInfo, provider, NULL, false, kCGRenderingIntentDefault); + if (colorSpace) { + CGColorSpaceRelease(colorSpace); + } + if (provider) { + CGDataProviderRelease(provider); + } + if (!image) { + return OPUnavailable(@"cgimage_create_failed", @{ + @"width": @(width), + @"height": @(height), + @"bytes_per_row": @(bytesPerRow), + @"pixel_format": @(pixelFormat), + @"symbols": symbols + }); + } + + // Optionally downscale so the longest edge is <= maxDimension. Cuts the + // encoded size (and downstream base64/model memory) ~4x on a 1290px-wide + // phone screen scaled to 1024px, more when combined with JPEG. + size_t encodedWidth = width; + size_t encodedHeight = height; + CGImageRef scaled = OPCreateScaledImage(image, maxDimension, &encodedWidth, &encodedHeight); + CGImageRef encodeImage = scaled ? scaled : image; + + NSString *lowerPath = [path lowercaseString]; + BOOL wantJPEG = [lowerPath hasSuffix:@".jpg"] || [lowerPath hasSuffix:@".jpeg"]; + CFStringRef utType = wantJPEG ? CFSTR("public.jpeg") : CFSTR("public.png"); + + NSURL *url = [NSURL fileURLWithPath:path]; + CGImageDestinationRef destination = CGImageDestinationCreateWithURL( + (__bridge CFURLRef)url, utType, 1, NULL); + if (!destination) { + if (scaled) CGImageRelease(scaled); + CGImageRelease(image); + return OPUnavailable(@"image_destination_failed", @{ + @"path": path ?: @"", + @"symbols": symbols + }); + } + NSDictionary *destProperties = wantJPEG ? @{ + (__bridge NSString *)kCGImageDestinationLossyCompressionQuality: + @(MAX(0.1, MIN(1.0, jpegQuality))) + } : @{}; + CGImageDestinationAddImage(destination, encodeImage, + (__bridge CFDictionaryRef)destProperties); + BOOL finalized = CGImageDestinationFinalize(destination); + CFRelease(destination); + if (scaled) CGImageRelease(scaled); + CGImageRelease(image); + if (!finalized) { + return OPUnavailable(@"image_write_failed", @{ + @"path": path ?: @"", + @"symbols": symbols + }); + } + + NSData *encoded = [NSData dataWithContentsOfFile:path]; + NSDictionary *attributes = [[NSFileManager defaultManager] attributesOfItemAtPath:path error:nil] ?: @{}; + return @{ + @"status": @"ok", + @"provider": @"IOMobileFramebuffer.IOSurface", + @"path": path ?: @"", + @"format": wantJPEG ? @"jpeg" : @"png", + @"width": @(encodedWidth), + @"height": @(encodedHeight), + @"source_width": @(width), + @"source_height": @(height), + @"max_dimension": @(maxDimension), + @"jpeg_quality": wantJPEG ? @(MAX(0.1, MIN(1.0, jpegQuality))) : @0, + @"bytes_per_row": @(bytesPerRow), + @"pixel_format": @(pixelFormat), + @"bytes": attributes[NSFileSize] ?: @(encoded.length), + @"sha256": encoded ? OPSHA256Hex(encoded) : @"", + @"symbols": symbols + }; +} + +int main(int argc, const char *argv[]) { + @autoreleasepool { + NSString *path = OPOutputPath(argc, argv); + // argv[2] = max longest-edge dimension in px (0 disables scaling). + // argv[3] = JPEG quality 0.1-1.0 (only used when path ends .jpg/.jpeg). + size_t maxDimension = (argc >= 3 && argv[2]) ? (size_t)strtoul(argv[2], NULL, 10) : 0; + double jpegQuality = (argc >= 4 && argv[3]) ? strtod(argv[3], NULL) : 0.6; + if (jpegQuality <= 0.0) { + jpegQuality = 0.6; + } + @try { + OPPrintJSON(OPCapture(path, maxDimension, jpegQuality)); + } @catch (NSException *exception) { + OPPrintJSON(OPUnavailable(@"exception", @{ + @"name": exception.name ?: @"", + @"detail": exception.reason ?: @"" + })); + } + } + return 0; +} diff --git a/ios/agentd/src/volume_trigger.xm b/ios/agentd/src/volume_trigger.xm new file mode 100644 index 0000000..3732e1c --- /dev/null +++ b/ios/agentd/src/volume_trigger.xm @@ -0,0 +1,7002 @@ +#import +#import +#import +#import + +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import + +static NSString *const OPVTPreferencesId = @"com.openphone.volumetrigger"; +static NSString *const OPVTPreferencesPath = @"/var/mobile/Library/Preferences/com.openphone.volumetrigger.plist"; +static NSString *const OPVTDefaultTriggerGoal = @"Use the current phone context to handle this hardware volume trigger as an immediate OpenPhone agent turn. Inspect the visible state, act only when the next useful phone action is clear, otherwise finish with a concise status."; +static NSString *const OPVTLegacyTriggerGoal = @"Handle the OpenPhone hardware volume trigger using the current phone context."; +static const char *OPVTSocketPath = "/var/mobile/Library/OpenPhone/run/agentd.sock"; +static const char *OPVTLogPath = "/var/mobile/Library/OpenPhone/openphone-volume-trigger.log"; +static NSString *const OPVTStorePath = @"/var/mobile/Library/OpenPhone"; +static NSString *const OPVTSpringBoardStateDir = @"/var/mobile/Library/OpenPhone/springboard"; +static NSString *const OPVTSpringBoardStatePath = @"/var/mobile/Library/OpenPhone/springboard/state.json"; +static NSString *const OPVTScreenshotsDir = @"/var/mobile/Library/OpenPhone/screenshots"; +static NSString *const OPVTScreenshotRequestPath = @"/var/mobile/Library/OpenPhone/springboard/screenshot-request.json"; +static NSString *const OPVTScreenshotResponsePath = @"/var/mobile/Library/OpenPhone/springboard/screenshot-response.json"; +static NSString *const OPVTInputRequestPath = @"/var/mobile/Library/OpenPhone/springboard/input-request.json"; +static NSString *const OPVTInputResponsePath = @"/var/mobile/Library/OpenPhone/springboard/input-response.json"; +static NSString *const OPVTClipboardRequestPath = @"/var/mobile/Library/OpenPhone/springboard/clipboard-request.json"; +static NSString *const OPVTClipboardResponsePath = @"/var/mobile/Library/OpenPhone/springboard/clipboard-response.json"; +static NSString *const OPVTPromptRequestPath = @"/var/mobile/Library/OpenPhone/springboard/prompt-request.json"; +static NSString *const OPVTPromptResponsePath = @"/var/mobile/Library/OpenPhone/springboard/prompt-response.json"; +static NSString *const OPVTTriggerStatusPath = @"/var/mobile/Library/OpenPhone/springboard/trigger-status.json"; + +static CFAbsoluteTime OPVTLastUp = 0; +static CFAbsoluteTime OPVTLastDown = 0; +static CFAbsoluteTime OPVTLastTrigger = 0; +static CFAbsoluteTime OPVTLastUpLog = 0; +static CFAbsoluteTime OPVTLastDownLog = 0; +static long long OPVTButtonEventCount = 0; +static long long OPVTComboEventCount = 0; +static long long OPVTLastButtonEventMs = 0; +static long long OPVTLastComboEventMs = 0; +static NSString *OPVTLastButtonEventName = nil; +static NSString *OPVTLastButtonEventSource = nil; +static NSString *OPVTLastTriggerRoute = nil; +static UIWindow *OPVTOverlayWindow = nil; +static UIWindow *OPVTPromptWindow = nil; +static NSString *OPVTIslandCurrentMode = @"idle"; +static void OPVTIslandApplyState(NSDictionary *state); +static BOOL OPVTPromptVisible = NO; +static id OPVTVolumeNotificationObserverObject = nil; +static BOOL OPVTVolumeNotificationInstalled = NO; +static long long OPVTVolumeNotificationEventCount = 0; +static long long OPVTLastVolumeNotificationMs = 0; +static double OPVTLastSystemVolume = -1.0; +static NSString *OPVTLastVolumeNotificationReason = nil; +static NSString *OPVTLastVolumeNotificationCategory = nil; +static NSString *OPVTLastVolumeNotificationDirection = nil; +static int OPVTStatePublishSuccessCount = 0; +static int OPVTStatePublishFailureCount = 0; +static pthread_mutex_t OPVTForegroundCacheLock = PTHREAD_MUTEX_INITIALIZER; +static NSString *OPVTLastForegroundBundleId = nil; +static NSString *OPVTLastForegroundSource = nil; +static long long OPVTLastForegroundAtMs = 0; + +typedef struct { + const char *className; + const char *selectorName; + BOOL volumeUp; + IMP original; + BOOL hooked; +} OPVTHookSpec; + +typedef struct { + const char *className; + const char *selectorName; + BOOL volumeUp; + IMP original; + BOOL hooked; +} OPVTFixedArgHookSpec; + +typedef struct { + const char *className; + const char *selectorName; + IMP original; + BOOL hooked; +} OPVTBoolArgHookSpec; + +typedef struct { + const char *className; + const char *selectorName; + BOOL secondArgumentMeansDown; + IMP original; + BOOL hooked; +} OPVTBoolPairHookSpec; + +typedef struct { + const char *className; + const char *selectorName; + IMP original; + BOOL hooked; +} OPVTButtonObjectVoidHookSpec; + +typedef struct { + const char *className; + const char *selectorName; + IMP original; + BOOL hooked; +} OPVTButtonObjectBoolHookSpec; + +typedef struct { + const char *className; + const char *selectorName; + IMP original; + BOOL hooked; +} OPVTButtonRawHIDVoidHookSpec; + +typedef struct { + const char *className; + const char *selectorName; + IMP original; + BOOL hooked; +} OPVTForegroundNoArgObjectHookSpec; + +typedef struct { + const char *className; + const char *selectorName; + IMP original; + BOOL hooked; +} OPVTForegroundObjectArgVoidHookSpec; + +typedef struct { + const char *className; + const char *selectorName; + IMP original; + BOOL hooked; +} OPVTForegroundObjectArgObjectHookSpec; + +typedef struct { + const char *className; + const char *selectorName; + IMP original; + BOOL hooked; +} OPVTForegroundNoArgBoolHookSpec; + +static void OPVTVolumeNoArgReplacement(id self, SEL _cmd); +static void OPVTVolumeFixedArgReplacement(id self, SEL _cmd, uintptr_t arg); +static void OPVTVolumeBoolArgReplacement(id self, SEL _cmd, BOOL increase); +static void OPVTVolumeBoolPairReplacement(id self, SEL _cmd, BOOL increase, uintptr_t second); +static void OPVTButtonObjectVoidReplacement(id self, SEL _cmd, id event); +static BOOL OPVTButtonObjectBoolReplacement(id self, SEL _cmd, id event); +static void OPVTButtonRawHIDVoidReplacement(id self, SEL _cmd, void *event); +static id OPVTForegroundNoArgObjectReplacement(id self, SEL _cmd); +static void OPVTForegroundObjectArgVoidReplacement(id self, SEL _cmd, id arg); +static id OPVTForegroundObjectArgObjectReplacement(id self, SEL _cmd, id arg); +static BOOL OPVTForegroundNoArgBoolReplacement(id self, SEL _cmd); +static BOOL OPVTShouldLogHookMiss(NSString *phase); +static void OPVTRecordButtonFromSource(BOOL volumeUp, NSString *source); +static void OPVTPresentTriggerPrompt(void); +static void OPVTCallDaemonVoiceAgent(void); +static void OPVTCallAgentWithGoal(NSString *requestedGoal, BOOL userProvidedGoal); +static void OPVTPublishTriggerStatus(NSString *eventName, NSDictionary *extra); +static void OPVTInstallVolumeNotificationObserver(void); +static void OPVTHandleSystemVolumeNotification(NSNotification *notification); +static BOOL OPVTMethodReturnTypeStartsWith(Method method, char expected); +static BOOL OPVTMethodReturnTypeLooksBool(Method method); +static BOOL OPVTMethodArgumentTypeLooksObject(Method method, unsigned int index); + +@interface OPVTVolumeNotificationObserver : NSObject +- (void)openphoneSystemVolumeDidChange:(NSNotification *)notification; +@end + +static OPVTHookSpec OPVTHookSpecs[] = { + {"SBVolumeControl", "increaseVolume", YES, NULL, NO}, + {"SBVolumeControl", "decreaseVolume", NO, NULL, NO}, + {"VolumeControl", "increaseVolume", YES, NULL, NO}, + {"VolumeControl", "decreaseVolume", NO, NULL, NO}, + {"SBMediaController", "increaseVolume", YES, NULL, NO}, + {"SBMediaController", "decreaseVolume", NO, NULL, NO}, + {"MPVolumeController", "increaseVolume", YES, NULL, NO}, + {"MPVolumeController", "decreaseVolume", NO, NULL, NO}, + {"SBVolumeHardwareButtonActions", "volumeIncreasePressUp", YES, NULL, NO}, + {"SBVolumeHardwareButtonActions", "volumeDecreasePressUp", NO, NULL, NO}, + {"SBVolumeHardwareButtonActions", "_handleVolumeIncreaseUp", YES, NULL, NO}, + {"SBVolumeHardwareButtonActions", "_handleVolumeDecreaseUp", NO, NULL, NO}, + {"SBSystemApertureController", "handleVolumeUpButtonPress", YES, NULL, NO}, + {"SBSystemApertureController", "handleVolumeDownButtonPress", NO, NULL, NO}, + {"SBSystemApertureSceneElement", "handleVolumeUpButtonPress", YES, NULL, NO}, + {"SBSystemApertureSceneElement", "handleVolumeDownButtonPress", NO, NULL, NO}, + {"SBTransientOverlayScenePresenter", "handleVolumeUpButtonPress", YES, NULL, NO}, + {"SBTransientOverlayScenePresenter", "handleVolumeDownButtonPress", NO, NULL, NO}, + {"SBDashBoardLockScreenEnvironment", "handleVolumeUpButtonPress", YES, NULL, NO}, + {"SBDashBoardLockScreenEnvironment", "handleVolumeDownButtonPress", NO, NULL, NO}, + {"SBBannerManager", "handleVolumeUpButtonPress", YES, NULL, NO}, + {"SBBannerManager", "handleVolumeDownButtonPress", NO, NULL, NO}, +}; + +static OPVTFixedArgHookSpec OPVTFixedArgHookSpecs[] = { + {"SBVolumeHardwareButtonActions", "volumeIncreasePressDownWithModifiers:", YES, NULL, NO}, + {"SBVolumeHardwareButtonActions", "volumeDecreasePressDownWithModifiers:", NO, NULL, NO}, +}; + +static OPVTBoolArgHookSpec OPVTBoolArgHookSpecs[] = { + {"SBVolumeHardwareButtonActions", "_handleVolumeButtonUpForIncrease:", NULL, NO}, + {"SBVolumeHardwareButtonActions", "_sendVolumeButtonDownToLegacyRegisteredClientsForIncrease:", NULL, NO}, + {"SBVolumeHardwareButtonActions", "_sendVolumeButtonDownToSBUIControllerForIncrease:", NULL, NO}, + {"SBVolumeHardwareButtonActions", "_sendVolumeButtonDownToSpringBoardInternalUIForIncrease:", NULL, NO}, +}; + +static OPVTBoolPairHookSpec OPVTBoolPairHookSpecs[] = { + {"SBVolumeHardwareButtonActions", "_handleVolumeButtonDownForIncrease:modifiers:", NO, NULL, NO}, + {"SBVolumeHardwareButtonActions", "_sendVolumeButtonToSBUIControllerForIncrease:down:", YES, NULL, NO}, +}; + +static OPVTButtonObjectVoidHookSpec OPVTButtonObjectVoidHookSpecs[] = { + {"SBNCSoundController", "_hardwareButtonPressed:", NULL, NO}, + {"SBCameraHardwareButtonStudyLogger", "logButtonEvent:", NULL, NO}, +}; + +static OPVTButtonObjectBoolHookSpec OPVTButtonObjectBoolHookSpecs[] = { + {"SBHomeScreenOverlayController", "wouldHandleButtonEvent:", NULL, NO}, + {"SBLiftToWakeManager", "wouldHandleButtonEvent:", NULL, NO}, + {"SBDashBoardBiometricUnlockController", "wouldHandleButtonEvent:", NULL, NO}, + {"SBRemoteTransientOverlaySession", "hasPendingButtonEvents:", NULL, NO}, +}; + +static OPVTButtonRawHIDVoidHookSpec OPVTButtonRawHIDVoidHookSpecs[] = { + {"SBCameraHardwareButtonStudyLogger", "logButtonEvent:", NULL, NO}, +}; + +static OPVTForegroundNoArgObjectHookSpec OPVTForegroundNoArgObjectHookSpecs[] = { + {"AXSpringBoardServerSideAppManager", "_activeApplicationBundleIdentifiers", NULL, NO}, + {"AXSpringBoardServerSideAppManager", "sceneLayoutState", NULL, NO}, + {"AXSpringBoardServer", "focusedOccludedAppScenes", NULL, NO}, +}; + +static OPVTForegroundObjectArgVoidHookSpec OPVTForegroundObjectArgVoidHookSpecs[] = { + {"AXSpringBoardServerHelper", "updateFrontMostApplicationWithServerInstance:", NULL, NO}, + {"AXSpringBoardServerHelper", "launchApplication:", NULL, NO}, + {"AXSpringBoardServerSideAppManager", "launchApplication:", NULL, NO}, +}; + +static OPVTForegroundObjectArgObjectHookSpec OPVTForegroundObjectArgObjectHookSpecs[] = { + {"AXSpringBoardServerHelper", "frontmostAppProcessWithServerInstance:", NULL, NO}, +}; + +static OPVTForegroundNoArgBoolHookSpec OPVTForegroundNoArgBoolHookSpecs[] = { + {"SBApplicationProcessState", "isForeground", NULL, NO}, +}; + +static void OPVTLog(NSString *format, ...) { + va_list args; + va_start(args, format); + NSString *message = [[NSString alloc] initWithFormat:format arguments:args]; + va_end(args); + + NSDateFormatter *formatter = [[NSDateFormatter alloc] init]; + formatter.locale = [NSLocale localeWithLocaleIdentifier:@"en_US_POSIX"]; + formatter.dateFormat = @"yyyy-MM-dd'T'HH:mm:ssZZZZZ"; + NSString *line = [NSString stringWithFormat:@"%@ %@\n", + [formatter stringFromDate:[NSDate date]], message ?: @""]; + FILE *file = fopen(OPVTLogPath, "a"); + if (file) { + fputs(line.UTF8String, file); + fclose(file); + } + NSLog(@"[OpenPhoneVolumeTrigger] %@", message ?: @""); +} + +static NSDictionary *OPVTPreferences(void) { + CFPreferencesAppSynchronize((CFStringRef)OPVTPreferencesId); + NSDictionary *preferences = (__bridge_transfer NSDictionary *)CFPreferencesCopyMultiple( + NULL, (CFStringRef)OPVTPreferencesId, kCFPreferencesCurrentUser, kCFPreferencesAnyHost); + NSDictionary *filePreferences = [NSDictionary dictionaryWithContentsOfFile:OPVTPreferencesPath]; + if (filePreferences.count == 0) { + return preferences ?: @{}; + } + NSMutableDictionary *merged = [preferences mutableCopy] ?: [NSMutableDictionary dictionary]; + [merged addEntriesFromDictionary:filePreferences]; + return merged; +} + +static BOOL OPVTBoolPreference(NSString *key, BOOL defaultValue) { + id value = OPVTPreferences()[key]; + return value ? [value boolValue] : defaultValue; +} + +static NSTimeInterval OPVTMillisecondsPreference(NSString *key, + double defaultMs, double minMs, double maxMs) { + id value = OPVTPreferences()[key]; + double ms = value ? [value doubleValue] : defaultMs; + if (ms < minMs) { + ms = minMs; + } + if (ms > maxMs) { + ms = maxMs; + } + return ms / 1000.0; +} + +static NSString *OPVTStringPreference(NSString *key, NSString *defaultValue) { + id value = OPVTPreferences()[key]; + if ([value isKindOfClass:[NSString class]] && [value length] > 0) { + return value; + } + return defaultValue ?: @""; +} + +static BOOL OPVTEnabled(void) { + return OPVTBoolPreference(@"Enabled", YES); +} + +static BOOL OPVTPromptForGoalEnabled(void) { + return OPVTBoolPreference(@"PromptForGoal", YES); +} + +static BOOL OPVTDaemonVoiceAgentEnabled(void) { + return OPVTBoolPreference(@"DaemonVoiceAgent", YES); +} + +static NSString *OPVTTrimmedString(NSString *value) { + if (![value isKindOfClass:[NSString class]]) { + return @""; + } + return [value stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] ?: @""; +} + +static long long OPVTNowMs(void) { + return (long long)([[NSDate date] timeIntervalSince1970] * 1000.0); +} + +static NSDictionary *OPVTReadJSONDictionary(NSString *path) { + NSData *data = [NSData dataWithContentsOfFile:path ?: @""]; + if (data.length == 0) { + return nil; + } + id object = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + return [object isKindOfClass:[NSDictionary class]] ? object : nil; +} + +static BOOL OPVTWriteJSONDictionary(NSString *path, NSDictionary *object) { + NSData *data = [NSJSONSerialization dataWithJSONObject:object ?: @{} + options:0 + error:nil]; + if (data.length == 0 || path.length == 0) { + return NO; + } + BOOL wrote = [data writeToFile:path atomically:YES]; + if (wrote) { + chmod(path.UTF8String, 0644); + } + return wrote; +} + +static void OPVTAddHookStatus(NSMutableArray *methods, + NSString *kind, const char *className, const char *selectorName, BOOL hooked) { + if (!hooked) { + return; + } + [methods addObject:@{ + @"kind": kind ?: @"unknown", + @"class": className ? [NSString stringWithUTF8String:className] : @"", + @"selector": selectorName ? [NSString stringWithUTF8String:selectorName] : @"" + }]; +} + +static NSDictionary *OPVTHookStatus(void) { + NSMutableArray *hookedMethods = [NSMutableArray array]; + NSUInteger volumeTotal = 0; + NSUInteger volumeHooked = 0; + NSUInteger foregroundTotal = 0; + NSUInteger foregroundHooked = 0; + + size_t count = sizeof(OPVTHookSpecs) / sizeof(OPVTHookSpecs[0]); + volumeTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTHookSpec *spec = &OPVTHookSpecs[index]; + if (spec->hooked) { + volumeHooked++; + } + OPVTAddHookStatus(hookedMethods, @"volume_no_arg", + spec->className, spec->selectorName, spec->hooked); + } + + count = sizeof(OPVTFixedArgHookSpecs) / sizeof(OPVTFixedArgHookSpecs[0]); + volumeTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTFixedArgHookSpec *spec = &OPVTFixedArgHookSpecs[index]; + if (spec->hooked) { + volumeHooked++; + } + OPVTAddHookStatus(hookedMethods, @"volume_fixed_arg", + spec->className, spec->selectorName, spec->hooked); + } + + count = sizeof(OPVTBoolArgHookSpecs) / sizeof(OPVTBoolArgHookSpecs[0]); + volumeTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTBoolArgHookSpec *spec = &OPVTBoolArgHookSpecs[index]; + if (spec->hooked) { + volumeHooked++; + } + OPVTAddHookStatus(hookedMethods, @"volume_bool_arg", + spec->className, spec->selectorName, spec->hooked); + } + + count = sizeof(OPVTBoolPairHookSpecs) / sizeof(OPVTBoolPairHookSpecs[0]); + volumeTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTBoolPairHookSpec *spec = &OPVTBoolPairHookSpecs[index]; + if (spec->hooked) { + volumeHooked++; + } + OPVTAddHookStatus(hookedMethods, @"volume_bool_pair", + spec->className, spec->selectorName, spec->hooked); + } + + count = sizeof(OPVTButtonObjectVoidHookSpecs) / sizeof(OPVTButtonObjectVoidHookSpecs[0]); + volumeTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTButtonObjectVoidHookSpec *spec = &OPVTButtonObjectVoidHookSpecs[index]; + if (spec->hooked) { + volumeHooked++; + } + OPVTAddHookStatus(hookedMethods, @"button_object_void", + spec->className, spec->selectorName, spec->hooked); + } + + count = sizeof(OPVTButtonObjectBoolHookSpecs) / sizeof(OPVTButtonObjectBoolHookSpecs[0]); + volumeTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTButtonObjectBoolHookSpec *spec = &OPVTButtonObjectBoolHookSpecs[index]; + if (spec->hooked) { + volumeHooked++; + } + OPVTAddHookStatus(hookedMethods, @"button_object_bool", + spec->className, spec->selectorName, spec->hooked); + } + + count = sizeof(OPVTButtonRawHIDVoidHookSpecs) / sizeof(OPVTButtonRawHIDVoidHookSpecs[0]); + volumeTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTButtonRawHIDVoidHookSpec *spec = &OPVTButtonRawHIDVoidHookSpecs[index]; + if (spec->hooked) { + volumeHooked++; + } + OPVTAddHookStatus(hookedMethods, @"button_raw_hid_void", + spec->className, spec->selectorName, spec->hooked); + } + + count = sizeof(OPVTForegroundNoArgObjectHookSpecs) / sizeof(OPVTForegroundNoArgObjectHookSpecs[0]); + foregroundTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTForegroundNoArgObjectHookSpec *spec = &OPVTForegroundNoArgObjectHookSpecs[index]; + if (spec->hooked) { + foregroundHooked++; + } + OPVTAddHookStatus(hookedMethods, @"foreground_no_arg_object", + spec->className, spec->selectorName, spec->hooked); + } + + count = sizeof(OPVTForegroundObjectArgVoidHookSpecs) / sizeof(OPVTForegroundObjectArgVoidHookSpecs[0]); + foregroundTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTForegroundObjectArgVoidHookSpec *spec = &OPVTForegroundObjectArgVoidHookSpecs[index]; + if (spec->hooked) { + foregroundHooked++; + } + OPVTAddHookStatus(hookedMethods, @"foreground_object_arg_void", + spec->className, spec->selectorName, spec->hooked); + } + + count = sizeof(OPVTForegroundObjectArgObjectHookSpecs) / sizeof(OPVTForegroundObjectArgObjectHookSpecs[0]); + foregroundTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTForegroundObjectArgObjectHookSpec *spec = &OPVTForegroundObjectArgObjectHookSpecs[index]; + if (spec->hooked) { + foregroundHooked++; + } + OPVTAddHookStatus(hookedMethods, @"foreground_object_arg_object", + spec->className, spec->selectorName, spec->hooked); + } + + count = sizeof(OPVTForegroundNoArgBoolHookSpecs) / sizeof(OPVTForegroundNoArgBoolHookSpecs[0]); + foregroundTotal += count; + for (size_t index = 0; index < count; index++) { + OPVTForegroundNoArgBoolHookSpec *spec = &OPVTForegroundNoArgBoolHookSpecs[index]; + if (spec->hooked) { + foregroundHooked++; + } + OPVTAddHookStatus(hookedMethods, @"foreground_no_arg_bool", + spec->className, spec->selectorName, spec->hooked); + } + + return @{ + @"volume_total": @(volumeTotal), + @"volume_hooked": @(volumeHooked), + @"foreground_total": @(foregroundTotal), + @"foreground_hooked": @(foregroundHooked), + @"any_volume_hooked": @(volumeHooked > 0), + @"hooked_methods": hookedMethods + }; +} + +static NSDictionary *OPVTVolumeNotificationStatus(void) { + NSMutableDictionary *status = [@{ + @"installed": @(OPVTVolumeNotificationInstalled), + @"events_seen": @(OPVTVolumeNotificationEventCount), + @"last_event_ms": @(OPVTLastVolumeNotificationMs), + @"last_volume": @(OPVTLastSystemVolume >= 0.0 ? OPVTLastSystemVolume : -1.0), + @"last_reason": OPVTLastVolumeNotificationReason ?: @"", + @"last_category": OPVTLastVolumeNotificationCategory ?: @"", + @"last_direction": OPVTLastVolumeNotificationDirection ?: @"", + @"seeded": @(OPVTLastSystemVolume >= 0.0) + } mutableCopy]; + return status; +} + +static NSDictionary *OPVTTriggerPreferenceStatus(void) { + NSDictionary *preferences = OPVTPreferences(); + NSString *goal = OPVTStringPreference(@"TriggerGoal", @""); + return @{ + @"enabled": @(OPVTBoolPreference(@"Enabled", YES)), + @"prompt_for_goal": @(OPVTBoolPreference(@"PromptForGoal", YES)), + @"daemon_voice_agent": @(OPVTBoolPreference(@"DaemonVoiceAgent", YES)), + @"run_task": @(OPVTBoolPreference(@"RunTask", YES)), + @"create_background_job": @(OPVTBoolPreference(@"CreateBackgroundJob", NO)), + @"run_background_jobs": @(OPVTBoolPreference(@"RunBackgroundJobs", NO)), + @"window_ms": preferences[@"WindowMs"] ?: @1200, + @"cooldown_ms": preferences[@"CooldownMs"] ?: @10000, + @"trigger_goal_present": @(goal.length > 0), + @"trigger_goal_length": @(goal.length) + }; +} + +static void OPVTPublishTriggerStatus(NSString *eventName, NSDictionary *extra) { + NSMutableDictionary *status = [@{ + @"schema": @"openphone.springboard_trigger_status.v1", + @"timestamp_ms": @(OPVTNowMs()), + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardVolumeHooks", + @"status": OPVTEnabled() ? @"enabled" : @"disabled", + @"event": eventName ?: @"status", + @"preferences": OPVTTriggerPreferenceStatus(), + @"hooks": OPVTHookStatus(), + @"volume_notification": OPVTVolumeNotificationStatus(), + @"button_events_seen": @(OPVTButtonEventCount), + @"combo_events_seen": @(OPVTComboEventCount), + @"last_button_event_ms": @(OPVTLastButtonEventMs), + @"last_button_event": OPVTLastButtonEventName ?: @"", + @"last_button_event_source": OPVTLastButtonEventSource ?: @"", + @"last_combo_event_ms": @(OPVTLastComboEventMs), + @"last_trigger_route": OPVTLastTriggerRoute ?: @"", + @"source": @"springboard" + } mutableCopy]; + if (extra.count > 0) { + status[@"event_detail"] = extra; + } + [[NSFileManager defaultManager] createDirectoryAtPath:OPVTStorePath + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + [[NSFileManager defaultManager] createDirectoryAtPath:OPVTSpringBoardStateDir + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + if (OPVTWriteJSONDictionary(OPVTTriggerStatusPath, status)) { + chmod(OPVTTriggerStatusPath.UTF8String, 0644); + } +} + +static double OPVTDoubleValue(id value, double defaultValue) { + if ([value respondsToSelector:@selector(doubleValue)]) { + double parsed = [value doubleValue]; + return isfinite(parsed) ? parsed : defaultValue; + } + return defaultValue; +} + +static long long OPVTLongLongValue(id value, long long defaultValue) { + if ([value respondsToSelector:@selector(longLongValue)]) { + return [value longLongValue]; + } + return defaultValue; +} + +static BOOL OPVTBoolValue(id value, BOOL defaultValue) { + if ([value respondsToSelector:@selector(boolValue)]) { + return [value boolValue]; + } + if ([value isKindOfClass:[NSString class]]) { + NSString *lower = [(NSString *)value lowercaseString]; + if ([lower isEqualToString:@"true"] || [lower isEqualToString:@"yes"] || + [lower isEqualToString:@"1"]) { + return YES; + } + if ([lower isEqualToString:@"false"] || [lower isEqualToString:@"no"] || + [lower isEqualToString:@"0"]) { + return NO; + } + } + return defaultValue; +} + +static id OPVTInvokeObjectNoArg(id target, NSString *selectorName) { + if (!target || selectorName.length == 0) { + return nil; + } + SEL selector = NSSelectorFromString(selectorName); + if (![target respondsToSelector:selector]) { + return nil; + } + NSMethodSignature *signature = [target methodSignatureForSelector:selector]; + if (!signature || signature.numberOfArguments != 2) { + return nil; + } + const char *returnType = signature.methodReturnType; + if (!returnType || (returnType[0] != '@' && returnType[0] != '#')) { + return nil; + } + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.target = target; + invocation.selector = selector; + @try { + [invocation invoke]; + __unsafe_unretained id value = nil; + [invocation getReturnValue:&value]; + return value; + } @catch (__unused NSException *exception) { + return nil; + } +} + +static id OPVTInvokeObjectOneArg(id target, NSString *selectorName, id argument) { + if (!target || selectorName.length == 0) { + return nil; + } + SEL selector = NSSelectorFromString(selectorName); + if (![target respondsToSelector:selector]) { + return nil; + } + NSMethodSignature *signature = [target methodSignatureForSelector:selector]; + if (!signature || signature.numberOfArguments != 3) { + return nil; + } + const char *returnType = signature.methodReturnType; + if (!returnType || (returnType[0] != '@' && returnType[0] != '#')) { + return nil; + } + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.target = target; + invocation.selector = selector; + id retainedArgument = argument; + [invocation setArgument:&retainedArgument atIndex:2]; + @try { + [invocation invoke]; + __unsafe_unretained id value = nil; + [invocation getReturnValue:&value]; + return value; + } @catch (__unused NSException *exception) { + return nil; + } +} + +static BOOL OPVTInvokeBoolNoArg(id target, NSString *selectorName, BOOL *outValue) { + if (!target || selectorName.length == 0 || !outValue) { + return NO; + } + SEL selector = NSSelectorFromString(selectorName); + if (![target respondsToSelector:selector]) { + return NO; + } + NSMethodSignature *signature = [target methodSignatureForSelector:selector]; + if (!signature || signature.numberOfArguments != 2) { + return NO; + } + const char *returnType = signature.methodReturnType; + if (!returnType || (returnType[0] != 'B' && returnType[0] != 'c' && returnType[0] != 'C')) { + return NO; + } + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.target = target; + invocation.selector = selector; + @try { + [invocation invoke]; + BOOL value = NO; + [invocation getReturnValue:&value]; + *outValue = value; + return YES; + } @catch (__unused NSException *exception) { + return NO; + } +} + +static BOOL OPVTMethodTypeLooksBool(const char *type) { + return type && (type[0] == 'B' || type[0] == 'c' || type[0] == 'C'); +} + +static BOOL OPVTInvokeVoidOrBoolBoolBool(id target, + NSString *selectorName, + BOOL first, + BOOL second, + NSString **outError) { + if (outError) { + *outError = nil; + } + if (!target || selectorName.length == 0) { + if (outError) { + *outError = @"target_missing"; + } + return NO; + } + SEL selector = NSSelectorFromString(selectorName); + if (![target respondsToSelector:selector]) { + if (outError) { + *outError = @"selector_missing"; + } + return NO; + } + NSMethodSignature *signature = [target methodSignatureForSelector:selector]; + if (!signature || signature.numberOfArguments != 4) { + if (outError) { + *outError = @"signature_mismatch"; + } + return NO; + } + const char *returnType = signature.methodReturnType; + BOOL returnsBool = OPVTMethodTypeLooksBool(returnType); + if (!returnType || (returnType[0] != 'v' && !returnsBool)) { + if (outError) { + *outError = @"return_type_mismatch"; + } + return NO; + } + const char *arg2 = [signature getArgumentTypeAtIndex:2]; + const char *arg3 = [signature getArgumentTypeAtIndex:3]; + if (!OPVTMethodTypeLooksBool(arg2) || !OPVTMethodTypeLooksBool(arg3)) { + if (outError) { + *outError = @"argument_type_mismatch"; + } + return NO; + } + + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.target = target; + invocation.selector = selector; + BOOL firstValue = first; + BOOL secondValue = second; + [invocation setArgument:&firstValue atIndex:2]; + [invocation setArgument:&secondValue atIndex:3]; + @try { + [invocation invoke]; + if (returnsBool) { + BOOL value = NO; + [invocation getReturnValue:&value]; + if (!value) { + if (outError) { + *outError = @"method_returned_false"; + } + return NO; + } + } + return YES; + } @catch (NSException *exception) { + if (outError) { + *outError = exception.name ?: @"exception"; + } + return NO; + } +} + +static BOOL OPVTInvokeVoidOrBoolObjectBoolObject(id target, + NSString *selectorName, + id object, + BOOL flag, + id trailingObject, + NSString **outError) { + if (outError) { + *outError = nil; + } + if (!target || selectorName.length == 0) { + if (outError) { + *outError = @"target_missing"; + } + return NO; + } + SEL selector = NSSelectorFromString(selectorName); + if (![target respondsToSelector:selector]) { + if (outError) { + *outError = @"selector_missing"; + } + return NO; + } + NSMethodSignature *signature = [target methodSignatureForSelector:selector]; + if (!signature || signature.numberOfArguments != 5) { + if (outError) { + *outError = @"signature_mismatch"; + } + return NO; + } + const char *returnType = signature.methodReturnType; + BOOL returnsBool = OPVTMethodTypeLooksBool(returnType); + if (!returnType || (returnType[0] != 'v' && !returnsBool)) { + if (outError) { + *outError = @"return_type_mismatch"; + } + return NO; + } + const char *arg2 = [signature getArgumentTypeAtIndex:2]; + const char *arg3 = [signature getArgumentTypeAtIndex:3]; + const char *arg4 = [signature getArgumentTypeAtIndex:4]; + if (!arg2 || arg2[0] != '@' || !OPVTMethodTypeLooksBool(arg3) || + !arg4 || arg4[0] != '@') { + if (outError) { + *outError = @"argument_type_mismatch"; + } + return NO; + } + + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.target = target; + invocation.selector = selector; + id retainedObject = object; + BOOL flagValue = flag; + id retainedTrailingObject = trailingObject; + [invocation setArgument:&retainedObject atIndex:2]; + [invocation setArgument:&flagValue atIndex:3]; + [invocation setArgument:&retainedTrailingObject atIndex:4]; + @try { + [invocation invoke]; + if (returnsBool) { + BOOL value = NO; + [invocation getReturnValue:&value]; + if (!value) { + if (outError) { + *outError = @"method_returned_false"; + } + return NO; + } + } + return YES; + } @catch (NSException *exception) { + if (outError) { + *outError = exception.name ?: @"exception"; + } + return NO; + } +} + +static BOOL OPVTInvokeBoolObjectBoolBoolObject(id target, + NSString *selectorName, + id object, + BOOL firstFlag, + BOOL secondFlag, + id trailingObject, + NSString **outError) { + if (outError) { + *outError = nil; + } + if (!target || selectorName.length == 0) { + if (outError) { + *outError = @"target_missing"; + } + return NO; + } + SEL selector = NSSelectorFromString(selectorName); + if (![target respondsToSelector:selector]) { + if (outError) { + *outError = @"selector_missing"; + } + return NO; + } + NSMethodSignature *signature = [target methodSignatureForSelector:selector]; + if (!signature || signature.numberOfArguments != 6 || + !OPVTMethodTypeLooksBool(signature.methodReturnType)) { + if (outError) { + *outError = @"signature_mismatch"; + } + return NO; + } + const char *arg2 = [signature getArgumentTypeAtIndex:2]; + const char *arg3 = [signature getArgumentTypeAtIndex:3]; + const char *arg4 = [signature getArgumentTypeAtIndex:4]; + const char *arg5 = [signature getArgumentTypeAtIndex:5]; + if (!arg2 || arg2[0] != '@' || !OPVTMethodTypeLooksBool(arg3) || + !OPVTMethodTypeLooksBool(arg4) || !arg5 || arg5[0] != '@') { + if (outError) { + *outError = @"argument_type_mismatch"; + } + return NO; + } + + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.target = target; + invocation.selector = selector; + id retainedObject = object; + BOOL firstValue = firstFlag; + BOOL secondValue = secondFlag; + id retainedTrailingObject = trailingObject; + [invocation setArgument:&retainedObject atIndex:2]; + [invocation setArgument:&firstValue atIndex:3]; + [invocation setArgument:&secondValue atIndex:4]; + [invocation setArgument:&retainedTrailingObject atIndex:5]; + @try { + [invocation invoke]; + BOOL value = NO; + [invocation getReturnValue:&value]; + if (!value) { + if (outError) { + *outError = @"method_returned_false"; + } + return NO; + } + return YES; + } @catch (NSException *exception) { + if (outError) { + *outError = exception.name ?: @"exception"; + } + return NO; + } +} + +static BOOL OPVTInvokeBoolIntegerObject(id target, + NSString *selectorName, + int integer, + id object, + NSString **outError) { + if (outError) { + *outError = nil; + } + if (!target || selectorName.length == 0) { + if (outError) { + *outError = @"target_missing"; + } + return NO; + } + SEL selector = NSSelectorFromString(selectorName); + if (![target respondsToSelector:selector]) { + if (outError) { + *outError = @"selector_missing"; + } + return NO; + } + NSMethodSignature *signature = [target methodSignatureForSelector:selector]; + if (!signature || signature.numberOfArguments != 4 || + !OPVTMethodTypeLooksBool(signature.methodReturnType)) { + if (outError) { + *outError = @"signature_mismatch"; + } + return NO; + } + const char *arg2 = [signature getArgumentTypeAtIndex:2]; + const char *arg3 = [signature getArgumentTypeAtIndex:3]; + if (!arg2 || (arg2[0] != 'i' && arg2[0] != 'q' && arg2[0] != 'Q') || + !arg3 || arg3[0] != '@') { + if (outError) { + *outError = @"argument_type_mismatch"; + } + return NO; + } + + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.target = target; + invocation.selector = selector; + int integerValue = integer; + long long longValue = integer; + unsigned long long unsignedValue = (unsigned long long)MAX(integer, 0); + id retainedObject = object; + if (arg2[0] == 'q') { + [invocation setArgument:&longValue atIndex:2]; + } else if (arg2[0] == 'Q') { + [invocation setArgument:&unsignedValue atIndex:2]; + } else { + [invocation setArgument:&integerValue atIndex:2]; + } + [invocation setArgument:&retainedObject atIndex:3]; + @try { + [invocation invoke]; + BOOL value = NO; + [invocation getReturnValue:&value]; + if (!value) { + if (outError) { + *outError = @"method_returned_false"; + } + return NO; + } + return YES; + } @catch (NSException *exception) { + if (outError) { + *outError = exception.name ?: @"exception"; + } + return NO; + } +} + +static BOOL OPVTInvokeIntegerNoArg(id target, NSString *selectorName, long long *outValue) { + if (!target || selectorName.length == 0 || !outValue) { + return NO; + } + SEL selector = NSSelectorFromString(selectorName); + if (![target respondsToSelector:selector]) { + return NO; + } + NSMethodSignature *signature = [target methodSignatureForSelector:selector]; + if (!signature || signature.numberOfArguments != 2) { + return NO; + } + const char *returnType = signature.methodReturnType; + if (!returnType) { + return NO; + } + NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:signature]; + invocation.target = target; + invocation.selector = selector; + @try { + [invocation invoke]; + if (returnType[0] == 'q' || returnType[0] == 'l' || returnType[0] == 'i' || + returnType[0] == 's' || returnType[0] == 'c') { + long long value = 0; + [invocation getReturnValue:&value]; + *outValue = value; + return YES; + } + if (returnType[0] == 'Q' || returnType[0] == 'L' || returnType[0] == 'I' || + returnType[0] == 'S' || returnType[0] == 'C') { + unsigned long long value = 0; + [invocation getReturnValue:&value]; + *outValue = (long long)value; + return YES; + } + } @catch (__unused NSException *exception) { + return NO; + } + return NO; +} + +static BOOL OPVTBundleIdentifierLooksValid(NSString *value) { + if (![value isKindOfClass:[NSString class]] || value.length < 3 || value.length > 200) { + return NO; + } + if (![value containsString:@"."]) { + return NO; + } + NSArray *prefixes = @[@"com.", @"org.", @"net.", @"io.", @"app."]; + BOOL hasKnownPrefix = NO; + for (NSString *prefix in prefixes) { + if ([value hasPrefix:prefix]) { + hasKnownPrefix = YES; + break; + } + } + if (!hasKnownPrefix) { + return NO; + } + NSCharacterSet *allowed = [NSCharacterSet characterSetWithCharactersInString: + @"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_"]; + for (NSUInteger i = 0; i < value.length; i++) { + if (![allowed characterIsMember:[value characterAtIndex:i]]) { + return NO; + } + } + return YES; +} + +static NSString *OPVTBundleIdentifierFromSceneIdentifier(NSString *value) { + if (![value isKindOfClass:[NSString class]] || ![value hasPrefix:@"sceneID:"]) { + return nil; + } + NSString *identifier = [value substringFromIndex:@"sceneID:".length]; + NSRange paren = [identifier rangeOfString:@"("]; + if (paren.location != NSNotFound) { + identifier = [identifier substringToIndex:paren.location]; + } + if ([identifier hasSuffix:@"-default"]) { + identifier = [identifier substringToIndex:identifier.length - @"-default".length]; + } else if (identifier.length > 37) { + NSString *suffix = [identifier substringFromIndex:identifier.length - 37]; + NSRegularExpression *uuidSuffix = [NSRegularExpression regularExpressionWithPattern: + @"^-[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$" + options:0 error:nil]; + if ([uuidSuffix numberOfMatchesInString:suffix + options:0 + range:NSMakeRange(0, suffix.length)] > 0) { + identifier = [identifier substringToIndex:identifier.length - 37]; + } + } + return OPVTBundleIdentifierLooksValid(identifier) ? identifier : nil; +} + +static NSString *OPVTStringFromObject(id value) { + if ([value isKindOfClass:[NSString class]]) { + return value; + } + if ([value isKindOfClass:[NSNumber class]]) { + return [value stringValue]; + } + return nil; +} + +static NSString *OPVTBundleIdentifierFromObject(id object, NSUInteger depth) { + if (!object || depth > 4) { + return nil; + } + NSArray *stringSelectors = @[ + @"bundleIdentifier", + @"displayIdentifier", + @"bundleID", + @"applicationIdentifier", + @"applicationBundleIdentifier", + @"clientProcessBundleIdentifier", + @"containingBundleIdentifier", + @"identifier" + ]; + for (NSString *selector in stringSelectors) { + NSString *value = OPVTStringFromObject(OPVTInvokeObjectNoArg(object, selector)); + NSString *sceneBundle = OPVTBundleIdentifierFromSceneIdentifier(value); + if (sceneBundle.length > 0) { + return sceneBundle; + } + if (OPVTBundleIdentifierLooksValid(value)) { + return value; + } + } + NSArray *objectSelectors = @[ + @"application", + @"bundle", + @"process", + @"clientProcess", + @"clientHandle", + @"scene", + @"settings", + @"displayIdentity", + @"layoutItem", + @"entity" + ]; + for (NSString *selector in objectSelectors) { + id nested = OPVTInvokeObjectNoArg(object, selector); + NSString *value = OPVTBundleIdentifierFromObject(nested, depth + 1); + if (value.length > 0) { + return value; + } + } + return nil; +} + +static NSTimeInterval OPVTWindowSeconds(void) { + return OPVTMillisecondsPreference(@"WindowMs", 1200.0, 50.0, 3000.0); +} + +static NSTimeInterval OPVTCooldownSeconds(void) { + return OPVTMillisecondsPreference(@"CooldownMs", 10000.0, 250.0, 60000.0); +} + +static void OPVTPlayHapticSuccess(void) { + AudioServicesPlaySystemSound(1519); +} + +static void OPVTPlayHapticFailure(void) { + AudioServicesPlaySystemSound(1521); +} + +static UIApplication *OPVTSafeSharedApplication(void) { + @try { + return [UIApplication sharedApplication]; + } @catch (NSException *exception) { + OPVTLog(@"springboard state UIApplication unavailable name=%@ reason=%@", + exception.name ?: @"", exception.reason ?: @""); + return nil; + } +} + +static UIScreen *OPVTSafeMainScreen(void) { + @try { + return [UIScreen mainScreen]; + } @catch (NSException *exception) { + int count = __sync_add_and_fetch(&OPVTStatePublishFailureCount, 1); + if (count <= 3 || count % 30 == 0) { + OPVTLog(@"springboard state UIScreen unavailable count=%d name=%@ reason=%@", + count, exception.name ?: @"", exception.reason ?: @""); + } + return nil; + } +} + +static UIWindowScene *OPVTActiveWindowScene(void) { + if (@available(iOS 13.0, *)) { + UIApplication *application = OPVTSafeSharedApplication(); + UIScreen *mainScreen = OPVTSafeMainScreen(); + NSSet *scenes = application.connectedScenes; + UIWindowScene *fallback = nil; + for (UIScene *scene in scenes) { + if (![scene isKindOfClass:[UIWindowScene class]]) { + continue; + } + UIWindowScene *windowScene = (UIWindowScene *)scene; + if (mainScreen && windowScene.screen != mainScreen) { + continue; + } + if (!fallback) { + fallback = windowScene; + } + if (scene.activationState == UISceneActivationStateForegroundActive || + scene.activationState == UISceneActivationStateForegroundInactive) { + return windowScene; + } + } + return fallback; + } + return nil; +} + +static NSArray *OPVTArrayFromCollection(id collection) { + if (!collection) { + return @[]; + } + if ([collection isKindOfClass:[NSArray class]]) { + return collection; + } + if ([collection isKindOfClass:[NSSet class]]) { + return [(NSSet *)collection allObjects]; + } + if ([collection isKindOfClass:[NSDictionary class]]) { + return [(NSDictionary *)collection allValues]; + } + id allObjects = OPVTInvokeObjectNoArg(collection, @"allObjects"); + if ([allObjects isKindOfClass:[NSArray class]]) { + return allObjects; + } + id enumerator = OPVTInvokeObjectNoArg(collection, @"objectEnumerator"); + if (!enumerator || ![enumerator respondsToSelector:@selector(nextObject)]) { + return @[]; + } + NSMutableArray *objects = [NSMutableArray array]; + for (NSUInteger i = 0; i < 64; i++) { + id object = [enumerator nextObject]; + if (!object) { + break; + } + [objects addObject:object]; + } + return objects; +} + +static void OPVTRecordForegroundBundleId(NSString *bundleId, NSString *source) { + if (!OPVTBundleIdentifierLooksValid(bundleId) || + [bundleId isEqualToString:@"com.apple.springboard"]) { + return; + } + long long now = OPVTNowMs(); + pthread_mutex_lock(&OPVTForegroundCacheLock); + BOOL changed = ![OPVTLastForegroundBundleId isEqualToString:bundleId]; + OPVTLastForegroundBundleId = [bundleId copy]; + OPVTLastForegroundSource = [source ?: @"runtime" copy]; + OPVTLastForegroundAtMs = now; + pthread_mutex_unlock(&OPVTForegroundCacheLock); + if (changed) { + OPVTLog(@"foreground cache updated bundle=%@ source=%@", + bundleId ?: @"", source ?: @"runtime"); + } +} + +static void OPVTRecordForegroundObject(id object, NSString *source) { + if (!object) { + return; + } + NSArray *objects = OPVTArrayFromCollection(object); + if (objects.count == 0) { + objects = @[object]; + } + for (id candidate in objects) { + NSString *bundleId = OPVTBundleIdentifierFromObject(candidate, 0); + NSString *stringValue = OPVTStringFromObject(candidate); + if (bundleId.length == 0 && OPVTBundleIdentifierLooksValid(stringValue)) { + bundleId = stringValue; + } + if (bundleId.length > 0) { + OPVTRecordForegroundBundleId(bundleId, source); + return; + } + } +} + +static NSDictionary *OPVTForegroundCacheSnapshot(void) { + pthread_mutex_lock(&OPVTForegroundCacheLock); + NSString *bundleId = [OPVTLastForegroundBundleId copy]; + NSString *source = [OPVTLastForegroundSource copy]; + long long timestamp = OPVTLastForegroundAtMs; + pthread_mutex_unlock(&OPVTForegroundCacheLock); + + if (bundleId.length == 0 || timestamp <= 0) { + return @{ + @"status": @"empty", + @"provider": @"OpenPhoneVolumeTrigger.ForegroundRuntimeCache" + }; + } + long long age = MAX(0, OPVTNowMs() - timestamp); + return @{ + @"status": age <= 30000 ? @"ok" : @"stale", + @"provider": @"OpenPhoneVolumeTrigger.ForegroundRuntimeCache", + @"bundle_id": bundleId, + @"source": source ?: @"runtime", + @"timestamp_ms": @(timestamp), + @"age_ms": @(age) + }; +} + +static NSNumber *OPVTScreenInteger(CGFloat value) { + return @((NSInteger)llround((double)value)); +} + +static NSArray *OPVTBoundsArray(CGRect rect) { + return @[ + OPVTScreenInteger(rect.origin.x), + OPVTScreenInteger(rect.origin.y), + OPVTScreenInteger(rect.size.width), + OPVTScreenInteger(rect.size.height) + ]; +} + +static NSString *OPVTAccessibilityString(id value) { + if ([value isKindOfClass:[NSString class]]) { + NSString *string = [(NSString *)value stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + return string.length > 0 ? string : nil; + } + if ([value isKindOfClass:[NSNumber class]]) { + return [(NSNumber *)value stringValue]; + } + return nil; +} + +static NSString *OPVTViewLabel(UIView *view) { + NSString *label = OPVTAccessibilityString(view.accessibilityLabel); + if (label.length > 0) { + return label; + } + if ([view isKindOfClass:[UIButton class]]) { + label = OPVTAccessibilityString([(UIButton *)view currentTitle]); + if (label.length > 0) { + return label; + } + } + if ([view isKindOfClass:[UILabel class]]) { + label = OPVTAccessibilityString([(UILabel *)view text]); + if (label.length > 0) { + return label; + } + } + if ([view isKindOfClass:[UITextField class]]) { + UITextField *field = (UITextField *)view; + label = OPVTAccessibilityString(field.placeholder); + if (label.length > 0) { + return label; + } + } + return OPVTAccessibilityString(view.accessibilityIdentifier); +} + +static NSString *OPVTViewVisibleText(UIView *view) { + NSString *label = OPVTAccessibilityString(view.accessibilityLabel); + if (label.length > 0) { + return label; + } + if ([view isKindOfClass:[UIButton class]]) { + label = OPVTAccessibilityString([(UIButton *)view currentTitle]); + if (label.length > 0) { + return label; + } + } + if ([view isKindOfClass:[UILabel class]]) { + label = OPVTAccessibilityString([(UILabel *)view text]); + if (label.length > 0) { + return label; + } + } + if ([view isKindOfClass:[UITextField class]]) { + return OPVTAccessibilityString([(UITextField *)view placeholder]); + } + return nil; +} + +static NSString *OPVTViewKind(UIView *view) { + UIAccessibilityTraits traits = view.accessibilityTraits; + if ((traits & UIAccessibilityTraitButton) == UIAccessibilityTraitButton) { + return @"button"; + } + if ((traits & UIAccessibilityTraitLink) == UIAccessibilityTraitLink) { + return @"link"; + } + if ((traits & UIAccessibilityTraitKeyboardKey) == UIAccessibilityTraitKeyboardKey) { + return @"keyboard_key"; + } + if ((traits & UIAccessibilityTraitSearchField) == UIAccessibilityTraitSearchField) { + return @"search_field"; + } + if ([view isKindOfClass:[UIButton class]]) { + return @"button"; + } + if ([view isKindOfClass:[UITextField class]]) { + return @"text_field"; + } + if ([view isKindOfClass:[UILabel class]]) { + return @"text"; + } + return view.userInteractionEnabled ? @"view" : @"text"; +} + +static NSNumber *OPVTWindowLevelNumber(UIWindow *window) { + double level = (double)window.windowLevel; + if (!isfinite(level)) { + return @0; + } + if (level > 100000.0) { + level = 100000.0; + } else if (level < -100000.0) { + level = -100000.0; + } + return @((NSInteger)llround(level)); +} + +static NSArray *OPVTApplicationWindows(UIApplication *application) { + if (!application) { + return @[]; + } + NSMutableArray *windows = [NSMutableArray array]; + NSMutableSet *seen = [NSMutableSet set]; + @try { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + for (UIWindow *window in application.windows ?: @[]) { + NSValue *key = [NSValue valueWithNonretainedObject:window]; + if (![seen containsObject:key]) { + [seen addObject:key]; + [windows addObject:window]; + } + } +#pragma clang diagnostic pop + if (@available(iOS 13.0, *)) { + for (UIScene *scene in application.connectedScenes) { + if (![scene isKindOfClass:[UIWindowScene class]]) { + continue; + } + for (UIWindow *window in ((UIWindowScene *)scene).windows ?: @[]) { + NSValue *key = [NSValue valueWithNonretainedObject:window]; + if (![seen containsObject:key]) { + [seen addObject:key]; + [windows addObject:window]; + } + } + } + } + } @catch (NSException *exception) { + OPVTLog(@"window enumeration exception name=%@ reason=%@", + exception.name ?: @"", exception.reason ?: @""); + } + return windows; +} + +static void OPVTCollectViewTree(UIView *view, + UIWindow *window, + NSInteger windowIndex, + NSUInteger depth, + NSUInteger *elementIndex, + NSMutableArray *interactiveElements, + NSMutableArray *visibleText) { + if (!view || depth > 8 || interactiveElements.count >= 80) { + return; + } + if (view.hidden || view.alpha < 0.02) { + return; + } + + CGRect bounds = CGRectZero; + @try { + bounds = [view convertRect:view.bounds toView:window]; + } @catch (__unused NSException *exception) { + bounds = CGRectZero; + } + if (CGRectIsEmpty(bounds) || bounds.size.width < 1.0 || bounds.size.height < 1.0) { + return; + } + + NSString *visibleLabel = OPVTViewVisibleText(view); + if (visibleLabel.length > 0 && visibleText.count < 80 && + ![visibleText containsObject:visibleLabel]) { + [visibleText addObject:visibleLabel]; + } + + NSString *label = OPVTViewLabel(view); + BOOL accessible = view.isAccessibilityElement || view.userInteractionEnabled || + [view isKindOfClass:[UIButton class]] || [view isKindOfClass:[UITextField class]]; + if (accessible && (label.length > 0 || view.accessibilityIdentifier.length > 0)) { + NSString *elementId = [NSString stringWithFormat:@"springboard-%ld-%lu", + (long)windowIndex, (unsigned long)(*elementIndex)]; + (*elementIndex)++; + NSMutableDictionary *element = [@{ + @"id": elementId, + @"kind": OPVTViewKind(view), + @"class": NSStringFromClass([view class]) ?: @"", + @"label": label ?: @"", + @"bounds": OPVTBoundsArray(bounds), + @"enabled": @(view.userInteractionEnabled), + @"focused": @(view.isFirstResponder), + @"window_id": @(windowIndex), + @"sensitive": @NO, + @"risk_hint": @"springboard_only" + } mutableCopy]; + NSString *identifier = OPVTAccessibilityString(view.accessibilityIdentifier); + if (identifier.length > 0) { + element[@"view_id"] = identifier; + } + [interactiveElements addObject:element]; + if (interactiveElements.count >= 80) { + return; + } + } + + NSArray *subviews = view.subviews; + for (UIView *subview in subviews) { + OPVTCollectViewTree(subview, window, windowIndex, depth + 1, + elementIndex, interactiveElements, visibleText); + if (interactiveElements.count >= 80) { + return; + } + } +} + +static NSDictionary *OPVTUITreeSnapshot(void) { + NSMutableDictionary *snapshot = [@{ + @"status": @"unavailable", + @"provider": @"SpringBoard.UIKitAccessibility" + } mutableCopy]; + UIApplication *application = OPVTSafeSharedApplication(); + if (!application) { + snapshot[@"reason"] = @"application_unavailable"; + return snapshot; + } + + @try { + NSArray *windows = OPVTApplicationWindows(application); + + NSMutableArray *windowSnapshots = [NSMutableArray array]; + NSMutableArray *interactiveElements = [NSMutableArray array]; + NSMutableArray *visibleText = [NSMutableArray array]; + NSInteger windowIndex = 0; + NSUInteger elementIndex = 0; + for (UIWindow *window in windows) { + if (!window || window.hidden || window.alpha < 0.02) { + windowIndex++; + continue; + } + CGRect bounds = window.bounds; + [windowSnapshots addObject:@{ + @"id": @(windowIndex), + @"type": OPVTWindowLevelNumber(window), + @"focused": @(window.isKeyWindow), + @"active": @(!window.hidden && window.alpha >= 0.02), + @"bounds": OPVTBoundsArray(bounds) + }]; + OPVTCollectViewTree(window, window, windowIndex, 0, + &elementIndex, interactiveElements, visibleText); + windowIndex++; + if (windowSnapshots.count >= 16 || interactiveElements.count >= 80) { + break; + } + } + + snapshot[@"status"] = @"ok"; + snapshot[@"window_count"] = @(windowSnapshots.count); + snapshot[@"element_count"] = @(interactiveElements.count); + snapshot[@"text_count"] = @(visibleText.count); + snapshot[@"windows"] = windowSnapshots; + snapshot[@"interactive_elements"] = interactiveElements; + snapshot[@"visible_text"] = visibleText; + snapshot[@"scope"] = @"springboard_only"; + return snapshot; + } @catch (NSException *exception) { + snapshot[@"reason"] = @"exception"; + snapshot[@"exception_name"] = exception.name ?: @""; + return snapshot; + } +} + +static NSDictionary *OPVTScreenshotBridgeStatus(void) { + return @{ + @"status": @"ready", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardScreenshot", + @"request_path": OPVTScreenshotRequestPath, + @"response_path": OPVTScreenshotResponsePath, + @"storage": OPVTScreenshotsDir, + @"scope": @"springboard_windows" + }; +} + +static BOOL OPVTScreenshotPathAllowed(NSString *path) { + if (![path isKindOfClass:[NSString class]] || path.length == 0) { + return NO; + } + NSString *standardPath = [path stringByStandardizingPath]; + NSString *standardDir = [OPVTScreenshotsDir stringByStandardizingPath]; + NSString *prefix = [standardDir stringByAppendingString:@"/"]; + NSString *ext = [standardPath.pathExtension lowercaseString]; + return [standardPath hasPrefix:prefix] && + ([ext isEqualToString:@"png"] || [ext isEqualToString:@"jpg"] + || [ext isEqualToString:@"jpeg"]); +} + +static NSDictionary *OPVTScreenshotResponse(NSString *status, + NSString *reason, + NSString *requestId, + NSString *path, + NSDictionary *extra) { + NSMutableDictionary *response = [@{ + @"schema": @"openphone.springboard_screenshot_response.v1", + @"status": status ?: @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardScreenshot", + @"request_id": requestId ?: @"", + @"timestamp_ms": @(OPVTNowMs()), + @"path": path ?: @"", + @"bytes_returned": @"never", + @"storage": OPVTScreenshotsDir, + @"scope": @"springboard_windows", + @"source": @"springboard" + } mutableCopy]; + if (reason.length > 0) { + response[@"reason"] = reason; + } + [response addEntriesFromDictionary:extra ?: @{}]; + return response; +} + +static NSDictionary *OPVTCaptureSpringBoardScreenshot(NSDictionary *request) { + NSString *requestId = [request[@"request_id"] isKindOfClass:[NSString class]] + ? request[@"request_id"] : @""; + NSString *path = [request[@"path"] isKindOfClass:[NSString class]] + ? request[@"path"] : @""; + if (requestId.length == 0) { + return OPVTScreenshotResponse(@"unavailable", @"missing_request_id", requestId, path, nil); + } + if (!OPVTScreenshotPathAllowed(path)) { + return OPVTScreenshotResponse(@"unavailable", @"path_not_allowed", requestId, path, nil); + } + + UIApplication *application = OPVTSafeSharedApplication(); + if (!application) { + return OPVTScreenshotResponse(@"unavailable", @"application_unavailable", requestId, path, nil); + } + UIScreen *screen = OPVTSafeMainScreen(); + if (!screen) { + return OPVTScreenshotResponse(@"unavailable", @"main_screen_unavailable", requestId, path, nil); + } + + CGRect bounds = CGRectZero; + CGFloat scale = 0.0; + @try { + bounds = screen.bounds; + scale = screen.scale; + } @catch (NSException *exception) { + return OPVTScreenshotResponse(@"unavailable", @"screen_metrics_exception", + requestId, path, @{@"exception_name": exception.name ?: @""}); + } + if (CGRectIsEmpty(bounds) || bounds.size.width < 1.0 || bounds.size.height < 1.0) { + return OPVTScreenshotResponse(@"unavailable", @"screen_bounds_unavailable", + requestId, path, nil); + } + if (scale <= 0.0 || !isfinite(scale)) { + scale = 1.0; + } + + // Cap the rendered longest edge to max_dimension_px so downstream base64 / + // model memory stays small. Rendering at a reduced scale is cheaper than + // rendering full then downscaling. 0 keeps native scale. + long long maxDimension = 1024; + if ([request[@"max_dimension_px"] respondsToSelector:@selector(longLongValue)]) { + maxDimension = [request[@"max_dimension_px"] longLongValue]; + } + CGFloat renderScale = scale; + if (maxDimension > 0) { + CGFloat nativeLongest = MAX(bounds.size.width, bounds.size.height) * scale; + if (nativeLongest > (CGFloat)maxDimension) { + renderScale = scale * ((CGFloat)maxDimension / nativeLongest); + if (renderScale <= 0.0 || !isfinite(renderScale)) { + renderScale = scale; + } + } + } + long long jpegQualityX100 = 60; + if ([request[@"jpeg_quality_x100"] respondsToSelector:@selector(longLongValue)]) { + jpegQualityX100 = [request[@"jpeg_quality_x100"] longLongValue]; + } + jpegQualityX100 = MAX(10, MIN(100, jpegQualityX100)); + NSString *pathExt = [[path pathExtension] lowercaseString]; + BOOL wantJPEG = [pathExt isEqualToString:@"jpg"] || [pathExt isEqualToString:@"jpeg"]; + + NSArray *windows = [OPVTApplicationWindows(application) + sortedArrayUsingComparator:^NSComparisonResult(UIWindow *left, UIWindow *right) { + CGFloat leftLevel = left.windowLevel; + CGFloat rightLevel = right.windowLevel; + if (leftLevel < rightLevel) { + return NSOrderedAscending; + } + if (leftLevel > rightLevel) { + return NSOrderedDescending; + } + return NSOrderedSame; + }]; + + [[NSFileManager defaultManager] createDirectoryAtPath:OPVTScreenshotsDir + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + + UIGraphicsBeginImageContextWithOptions(bounds.size, YES, renderScale); + CGContextRef context = UIGraphicsGetCurrentContext(); + if (!context) { + UIGraphicsEndImageContext(); + return OPVTScreenshotResponse(@"unavailable", @"graphics_context_unavailable", + requestId, path, nil); + } + [[UIColor blackColor] setFill]; + UIRectFill(bounds); + + NSUInteger drawnCount = 0; + NSUInteger visibleCount = 0; + for (UIWindow *window in windows) { + if (!window || window.hidden || window.alpha < 0.02) { + continue; + } + CGRect frame = window.frame; + if (CGRectIsEmpty(frame)) { + frame = window.bounds; + } + if (CGRectIsEmpty(frame) || frame.size.width < 1.0 || frame.size.height < 1.0) { + continue; + } + visibleCount++; + BOOL drawn = NO; + @try { + drawn = [window drawViewHierarchyInRect:frame afterScreenUpdates:NO]; + } @catch (NSException *exception) { + OPVTLog(@"screenshot drawViewHierarchy exception window=%@ name=%@", + NSStringFromClass([window class]) ?: @"", exception.name ?: @""); + drawn = NO; + } + if (!drawn) { + BOOL savedState = NO; + @try { + CGContextSaveGState(context); + savedState = YES; + CGContextTranslateCTM(context, frame.origin.x, frame.origin.y); + [window.layer renderInContext:context]; + CGContextRestoreGState(context); + savedState = NO; + drawn = YES; + } @catch (NSException *exception) { + OPVTLog(@"screenshot layer render exception window=%@ name=%@", + NSStringFromClass([window class]) ?: @"", exception.name ?: @""); + if (savedState) { + CGContextRestoreGState(context); + } + } + } + if (drawn) { + drawnCount++; + } + if (visibleCount >= 24) { + break; + } + } + + UIImage *image = UIGraphicsGetImageFromCurrentImageContext(); + UIGraphicsEndImageContext(); + if (!image) { + return OPVTScreenshotResponse(@"unavailable", @"image_create_failed", + requestId, path, @{@"visible_window_count": @(visibleCount)}); + } + NSData *encoded = wantJPEG + ? UIImageJPEGRepresentation(image, (CGFloat)jpegQualityX100 / 100.0) + : UIImagePNGRepresentation(image); + if (encoded.length == 0) { + return OPVTScreenshotResponse(@"unavailable", + wantJPEG ? @"jpeg_encode_failed" : @"png_encode_failed", + requestId, path, @{@"visible_window_count": @(visibleCount)}); + } + BOOL wrote = [encoded writeToFile:path atomically:YES]; + if (!wrote) { + return OPVTScreenshotResponse(@"unavailable", @"image_write_failed", + requestId, path, @{@"bytes": @(encoded.length)}); + } + chmod(path.UTF8String, 0644); + + return OPVTScreenshotResponse(@"ok", nil, requestId, path, @{ + @"format": wantJPEG ? @"jpeg" : @"png", + @"width": @((NSInteger)llround((double)(bounds.size.width * renderScale))), + @"height": @((NSInteger)llround((double)(bounds.size.height * renderScale))), + @"point_width": @((double)bounds.size.width), + @"point_height": @((double)bounds.size.height), + @"scale": @((double)renderScale), + @"native_scale": @((double)scale), + @"max_dimension_px": @(maxDimension), + @"jpeg_quality_x100": wantJPEG ? @(jpegQualityX100) : @0, + @"bytes": @(encoded.length), + @"window_count": @(windows.count), + @"visible_window_count": @(visibleCount), + @"drawn_window_count": @(drawnCount) + }); +} + +static NSDictionary *OPVTInputBridgeStatus(void) { + return @{ + @"status": @"ready", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardInput", + @"request_path": OPVTInputRequestPath, + @"response_path": OPVTInputResponsePath, + @"scope": @"springboard_windows", + @"actions": @[@"tap", @"tap_element", @"long_press", @"show_passcode", @"unlock_with_passcode"], + @"strategy": @"uikit_accessibility_activation" + }; +} + +static NSDictionary *OPVTClipboardBridgeStatus(void) { + return @{ + @"status": @"ready", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardClipboard", + @"request_path": OPVTClipboardRequestPath, + @"response_path": OPVTClipboardResponsePath, + @"scope": @"springboard_uipasteboard", + @"actions": @[@"read", @"write"], + @"strategy": @"uikit_uipasteboard" + }; +} + +static NSDictionary *OPVTPromptBridgeStatus(void) { + return @{ + @"status": @"ready", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardPrompt", + @"request_path": OPVTPromptRequestPath, + @"response_path": OPVTPromptResponsePath, + @"scope": @"springboard_prompt", + @"actions": @[@"present", @"run_goal"], + @"strategy": @"uikit_alert_agent_handoff" + }; +} + +static NSDictionary *OPVTPromptResponse(NSString *status, + NSString *reason, + NSString *requestId, + NSString *operation, + NSDictionary *extra) { + NSMutableDictionary *response = [@{ + @"schema": @"openphone.springboard_prompt_response.v1", + @"status": status ?: @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardPrompt", + @"request_id": requestId ?: @"", + @"operation": operation ?: @"", + @"timestamp_ms": @(OPVTNowMs()), + @"source": @"springboard", + @"strategy": @"uikit_alert_agent_handoff" + } mutableCopy]; + if (reason.length > 0) { + response[@"reason"] = reason; + } + [response addEntriesFromDictionary:extra ?: @{}]; + return response; +} + +static NSDictionary *OPVTPerformPromptRequest(NSDictionary *request) { + NSString *requestId = [request[@"request_id"] isKindOfClass:[NSString class]] + ? request[@"request_id"] : @""; + NSString *operationValue = [request[@"operation"] isKindOfClass:[NSString class]] + ? request[@"operation"] : @"present"; + NSString *operation = [operationValue lowercaseString]; + if (requestId.length == 0) { + return OPVTPromptResponse(@"unavailable", @"missing_request_id", requestId, operation, nil); + } + if ([operation isEqualToString:@"present"]) { + OPVTPresentTriggerPrompt(); + return OPVTPromptResponse(@"ok", nil, requestId, operation, @{ + @"prompt_requested": @YES + }); + } + if ([operation isEqualToString:@"run_goal"]) { + NSString *goal = OPVTTrimmedString([request[@"goal"] isKindOfClass:[NSString class]] + ? request[@"goal"] : @""); + if (goal.length == 0) { + return OPVTPromptResponse(@"unavailable", @"missing_goal", requestId, operation, nil); + } + OPVTCallAgentWithGoal(goal, YES); + return OPVTPromptResponse(@"ok", nil, requestId, operation, @{ + @"goal_length": @(goal.length) + }); + } + return OPVTPromptResponse(@"unavailable", @"unsupported_operation", + requestId, operation, nil); +} + +static NSDictionary *OPVTClipboardResponse(NSString *status, + NSString *reason, + NSString *requestId, + NSString *operation, + NSDictionary *extra) { + NSMutableDictionary *response = [@{ + @"schema": @"openphone.springboard_clipboard_response.v1", + @"status": status ?: @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardClipboard", + @"request_id": requestId ?: @"", + @"operation": operation ?: @"", + @"timestamp_ms": @(OPVTNowMs()), + @"source": @"springboard", + @"strategy": @"uikit_uipasteboard" + } mutableCopy]; + if (reason.length > 0) { + response[@"reason"] = reason; + } + [response addEntriesFromDictionary:extra ?: @{}]; + return response; +} + +static NSDictionary *OPVTPerformClipboardRequest(NSDictionary *request) { + NSString *requestId = [request[@"request_id"] isKindOfClass:[NSString class]] + ? request[@"request_id"] : @""; + NSString *operationValue = [request[@"operation"] isKindOfClass:[NSString class]] + ? request[@"operation"] : @"read"; + NSString *operation = [operationValue lowercaseString]; + if (requestId.length == 0) { + return OPVTClipboardResponse(@"unavailable", @"missing_request_id", requestId, operation, nil); + } + @try { + UIPasteboard *pasteboard = [UIPasteboard generalPasteboard]; + if (!pasteboard) { + return OPVTClipboardResponse(@"unavailable", @"pasteboard_unavailable", + requestId, operation, nil); + } + if ([operation isEqualToString:@"read"]) { + NSString *text = pasteboard.string ?: @""; + return OPVTClipboardResponse(@"ok", nil, requestId, operation, @{ + @"text": text, + @"text_length": @(text.length), + @"system_clipboard": @YES + }); + } + if ([operation isEqualToString:@"write"]) { + NSString *text = [request[@"text"] isKindOfClass:[NSString class]] + ? request[@"text"] : @""; + pasteboard.string = text ?: @""; + NSString *after = pasteboard.string ?: @""; + NSString *expected = text ?: @""; + BOOL verified = [after isEqualToString:expected]; + if (!verified) { + return OPVTClipboardResponse(@"unavailable", @"pasteboard_write_not_verified", + requestId, operation, @{ + @"text_length": @(text.length), + @"after_text_length": @(after.length), + @"system_clipboard": @YES + }); + } + return OPVTClipboardResponse(@"ok", nil, requestId, operation, @{ + @"text_length": @(text.length), + @"verified": @YES, + @"system_clipboard": @YES + }); + } + return OPVTClipboardResponse(@"unavailable", @"unsupported_operation", + requestId, operation, nil); + } @catch (NSException *exception) { + return OPVTClipboardResponse(@"unavailable", @"exception", requestId, operation, @{ + @"exception_name": exception.name ?: @"", + @"exception_reason": exception.reason ?: @"" + }); + } +} + +static NSDictionary *OPVTInputResponse(NSString *status, + NSString *reason, + NSString *requestId, + NSString *actionType, + NSDictionary *extra) { + NSMutableDictionary *response = [@{ + @"schema": @"openphone.springboard_input_response.v1", + @"status": status ?: @"unavailable", + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardInput", + @"request_id": requestId ?: @"", + @"action_type": actionType ?: @"", + @"timestamp_ms": @(OPVTNowMs()), + @"source": @"springboard", + @"strategy": @"uikit_accessibility_activation" + } mutableCopy]; + if (reason.length > 0) { + response[@"reason"] = reason; + } + [response addEntriesFromDictionary:extra ?: @{}]; + return response; +} + +static BOOL OPVTNameContainsAny(const char *name, NSArray *needles) { + if (!name) { + return NO; + } + NSString *value = [NSString stringWithUTF8String:name]; + for (NSString *needle in needles) { + if ([value rangeOfString:needle options:NSCaseInsensitiveSearch].location != NSNotFound) { + return YES; + } + } + return NO; +} + +static NSString *OPVTSafeStringFromCString(const char *value) { + if (!value || value[0] == '\0') { + return @""; + } + NSString *string = [NSString stringWithUTF8String:value]; + return string ?: @""; +} + +static BOOL OPVTInputDiagnosticMethodNameLooksRelevant(const char *name) { + return OPVTNameContainsAny(name, @[ + @"activate", @"action", @"tap", @"touch", @"press", @"gesture", @"swipe", + @"unlock", @"lock", @"dismiss", @"present", @"cover", @"dashboard", + @"passcode", @"authenticate", @"authenticated", @"home", @"reveal", + @"scroll", @"pan", @"grabber", @"transition", @"settle" + ]); +} + +static NSDictionary *OPVTMethodDiagnostic(Method method) { + if (!method) { + return @{}; + } + SEL selector = method_getName(method); + char returnType[96] = {0}; + char arg2[96] = {0}; + char arg3[96] = {0}; + method_getReturnType(method, returnType, sizeof(returnType)); + if (method_getNumberOfArguments(method) > 2) { + method_getArgumentType(method, 2, arg2, sizeof(arg2)); + } + if (method_getNumberOfArguments(method) > 3) { + method_getArgumentType(method, 3, arg3, sizeof(arg3)); + } + NSString *selectorName = selector ? NSStringFromSelector(selector) : @""; + selectorName = selectorName ?: @""; + NSString *returnTypeString = OPVTSafeStringFromCString(returnType); + NSString *arg2String = OPVTSafeStringFromCString(arg2); + NSString *arg3String = OPVTSafeStringFromCString(arg3); + NSMutableDictionary *info = [@{ + @"selector": selectorName, + @"args": @(method_getNumberOfArguments(method)), + @"return": returnTypeString + } mutableCopy]; + if (arg2String.length > 0) { + info[@"arg2"] = arg2String; + } + if (arg3String.length > 0) { + info[@"arg3"] = arg3String; + } + return info; +} + +static NSArray *OPVTMethodDiagnosticsForClass(Class cls, + NSUInteger maxMethods, + BOOL requireRelevantMethodName) { + if (!cls || maxMethods == 0) { + return @[]; + } + unsigned int methodCount = 0; + Method *methods = class_copyMethodList(cls, &methodCount); + NSMutableArray *result = [NSMutableArray array]; + for (unsigned int index = 0; methods && index < methodCount && result.count < maxMethods; index++) { + SEL selector = method_getName(methods[index]); + const char *name = sel_getName(selector); + if (requireRelevantMethodName && !OPVTInputDiagnosticMethodNameLooksRelevant(name)) { + continue; + } + [result addObject:OPVTMethodDiagnostic(methods[index])]; + } + if (methods) { + free(methods); + } + return result; +} + +static NSArray *OPVTTargetInputDiagnostics(id target) { + if (!target) { + return @[]; + } + NSMutableArray *classes = [NSMutableArray array]; + Class cls = [target class]; + NSUInteger depth = 0; + while (cls && depth < 8 && classes.count < 8) { + NSArray *methods = OPVTMethodDiagnosticsForClass(cls, 60, YES); + [classes addObject:@{ + @"class": NSStringFromClass(cls) ?: @"", + @"method_count": @(methods.count), + @"methods": methods + }]; + cls = class_getSuperclass(cls); + depth++; + } + return classes; +} + +static NSDictionary *OPVTClassInputDiagnosticEntry(Class cls, BOOL targeted) { + if (!cls) { + return @{}; + } + NSArray *methods = OPVTMethodDiagnosticsForClass(cls, targeted ? 100 : 20, YES); + NSArray *classMethods = OPVTMethodDiagnosticsForClass(object_getClass(cls), + targeted ? 100 : 20, + !targeted); + NSMutableDictionary *entry = [@{ + @"class": NSStringFromClass(cls) ?: @"", + @"methods": methods + } mutableCopy]; + if (classMethods.count > 0) { + entry[@"class_methods"] = classMethods; + } + + NSMutableArray *singletons = [NSMutableArray array]; + for (NSString *selectorName in @[@"sharedInstance", @"sharedManager", + @"sharedController", @"sharedApplication", @"mainWorkspace", + @"sharedWorkspace"]) { + id singleton = OPVTInvokeObjectNoArg(cls, selectorName); + if (singleton) { + [singletons addObject:@{ + @"selector": selectorName, + @"class": NSStringFromClass([singleton class]) ?: @"", + @"methods": OPVTMethodDiagnosticsForClass([singleton class], + targeted ? 100 : 20, + YES) + }]; + } + } + if (singletons.count > 0) { + entry[@"singletons"] = singletons; + NSDictionary *first = singletons.firstObject; + entry[@"singleton"] = first[@"selector"] ?: @""; + entry[@"singleton_class"] = first[@"class"] ?: @""; + } + return entry; +} + +static NSArray *OPVTTargetedGlobalInputDiagnostics(void) { + NSArray *classNames = @[ + @"SpringBoard", + @"SBUIController", + @"SBMainWorkspace", + @"SBWorkspaceController", + @"SBLockScreenManager", + @"SBLockScreenController", + @"SBLockScreenViewController", + @"SBLegacyLockScreenEnvironment", + @"SBDashBoardViewController", + @"SBDashBoardLockScreenEnvironment", + @"SBDashBoardHomeAffordanceController", + @"SBCoverSheetPresentationManager", + @"SBCoverSheetPrimarySlidingViewController", + @"SBCoverSheetViewController", + @"SBCoverSheetWindow", + @"CSCoverSheetViewController", + @"CSCombinedListViewController", + @"CSPasscodeViewController", + @"CSPasscodeBackgroundViewController", + @"SBPlatterHomeGestureContext", + @"SBHomeGestureManager", + @"SBSystemGestureManager", + @"SBSystemGestureStateAggregator", + @"SBFluidSwitcherGestureManager", + @"SBBacklightController", + @"SBLockHardwareButton", + @"SBLockScreenUnlockRequest", + @"SBLockScreenBiometricAuthenticationCoordinator", + @"SBAuthenticationFeedback", + @"SBUIPasscodeLockViewBase", + @"SBUIPasscodeLockViewWithKeypad", + @"FBSystemService", + @"BKSDisplayServices" + ]; + + NSMutableArray *result = [NSMutableArray array]; + for (NSString *className in classNames) { + Class cls = NSClassFromString(className); + if (!cls) { + [result addObject:@{@"class": className, @"status": @"absent"}]; + continue; + } + NSMutableDictionary *entry = [OPVTClassInputDiagnosticEntry(cls, YES) mutableCopy]; + entry[@"status"] = @"present"; + [result addObject:entry]; + } + return result; +} + +static BOOL OPVTTargetLooksLikeSwipeUpToUnlock(NSDictionary *targetSummary) { + NSString *label = [targetSummary[@"label"] isKindOfClass:[NSString class]] + ? targetSummary[@"label"] : @""; + if ([label rangeOfString:@"swipe up to unlock" options:NSCaseInsensitiveSearch].location != NSNotFound) { + return YES; + } + return [label rangeOfString:@"unlock" options:NSCaseInsensitiveSearch].location != NSNotFound && + [label rangeOfString:@"swipe" options:NSCaseInsensitiveSearch].location != NSNotFound; +} + +static NSMutableDictionary *OPVTAddLockScreenFallbackAttempt(NSMutableArray *attempts, + NSString *targetName, + id target, + NSString *selectorName, + NSString *status, + NSString *reason) { + NSMutableDictionary *attempt = [@{ + @"target": targetName ?: @"", + @"target_class": target ? (NSStringFromClass([target class]) ?: @"") : @"", + @"selector": selectorName ?: @"", + @"status": status ?: @"unavailable" + } mutableCopy]; + if (reason.length > 0) { + attempt[@"reason"] = reason; + } + [attempts addObject:attempt]; + return attempt; +} + +static BOOL OPVTTryLockScreenPasscodeSelector(id target, + NSString *targetName, + NSString *selectorName, + BOOL requireVisibilityCheck, + NSMutableArray *attempts, + NSMutableDictionary *extra) { + NSString *error = nil; + BOOL ok = OPVTInvokeVoidOrBoolBoolBool(target, selectorName, YES, YES, &error); + NSMutableDictionary *attempt = OPVTAddLockScreenFallbackAttempt(attempts, targetName, target, selectorName, + ok ? @"ok" : @"unavailable", error); + BOOL visible = NO; + if (ok && OPVTInvokeBoolNoArg(target, @"isPasscodeLockVisible", &visible)) { + attempt[@"passcode_visible"] = @(visible); + if (requireVisibilityCheck && !visible) { + attempt[@"status"] = @"unavailable"; + attempt[@"reason"] = @"passcode_not_visible_after_invocation"; + ok = NO; + } + } else if (ok && requireVisibilityCheck) { + attempt[@"status"] = @"unavailable"; + attempt[@"reason"] = @"visibility_check_unavailable"; + ok = NO; + } + if (ok) { + extra[@"activation_method"] = [NSString stringWithFormat:@"%@[%@]", + targetName ?: @"lock_screen_target", selectorName ?: @""]; + return YES; + } + return NO; +} + +static BOOL OPVTShowLockScreenPasscode(NSMutableDictionary *extra) { + NSMutableArray *attempts = [NSMutableArray array]; + Class managerClass = NSClassFromString(@"SBLockScreenManager"); + id manager = OPVTInvokeObjectNoArg(managerClass, @"sharedInstance"); + if (!manager) { + OPVTAddLockScreenFallbackAttempt(attempts, @"SBLockScreenManager.sharedInstance", + nil, @"sharedInstance", @"unavailable", @"target_missing"); + } else { + id coverSheetViewController = OPVTInvokeObjectNoArg(manager, @"coverSheetViewController"); + if (OPVTTryLockScreenPasscodeSelector(coverSheetViewController, @"coverSheetViewController", + @"setPasscodeLockVisible:animated:", YES, attempts, extra)) { + extra[@"lockscreen_fallback"] = @{@"status": @"ok", @"attempts": attempts}; + return YES; + } + + id environment = OPVTInvokeObjectNoArg(manager, @"lockScreenEnvironment"); + if (OPVTTryLockScreenPasscodeSelector(environment, @"lockScreenEnvironment", + @"setPasscodeLockVisible:animated:", YES, attempts, extra)) { + extra[@"lockscreen_fallback"] = @{@"status": @"ok", @"attempts": attempts}; + return YES; + } + + if (OPVTTryLockScreenPasscodeSelector(manager, @"SBLockScreenManager", + @"setPasscodeVisible:animated:", NO, attempts, extra)) { + extra[@"lockscreen_fallback"] = @{@"status": @"ok", @"attempts": attempts}; + return YES; + } + if (OPVTTryLockScreenPasscodeSelector(manager, @"SBLockScreenManager", + @"_setPasscodeVisible:animated:", NO, attempts, extra)) { + extra[@"lockscreen_fallback"] = @{@"status": @"ok", @"attempts": attempts}; + return YES; + } + } + + Class presentationManagerClass = NSClassFromString(@"SBCoverSheetPresentationManager"); + id presentationManager = OPVTInvokeObjectNoArg(presentationManagerClass, @"sharedInstance"); + id coverSheetViewController = OPVTInvokeObjectNoArg(presentationManager, @"coverSheetViewController"); + if (OPVTTryLockScreenPasscodeSelector(coverSheetViewController, + @"SBCoverSheetPresentationManager.coverSheetViewController", + @"setPasscodeLockVisible:animated:", YES, attempts, extra)) { + extra[@"lockscreen_fallback"] = @{@"status": @"ok", @"attempts": attempts}; + return YES; + } + + extra[@"lockscreen_fallback"] = @{@"status": @"unavailable", @"attempts": attempts}; + return NO; +} + +static BOOL OPVTSpringBoardLockedState(BOOL *outLocked) { + if (!outLocked) { + return NO; + } + id springBoard = OPVTInvokeObjectNoArg(NSClassFromString(@"SpringBoard"), @"sharedApplication"); + if (!springBoard) { + springBoard = UIApplication.sharedApplication; + } + if (OPVTInvokeBoolNoArg(springBoard, @"isLocked", outLocked)) { + return YES; + } + return OPVTInvokeBoolNoArg(springBoard, @"_accessibilityIsSystemLocked", outLocked); +} + +static BOOL OPVTFinalizePasscodeUnlock(NSMutableDictionary *extra, + NSMutableArray *attempts, + NSString *activationMethod) { + usleep(800000); + BOOL locked = YES; + BOOL hasLockedState = OPVTSpringBoardLockedState(&locked); + NSMutableDictionary *unlock = [@{@"attempts": attempts ?: @[]} mutableCopy]; + if (hasLockedState) { + unlock[@"locked_after_attempt"] = @(locked); + } else { + unlock[@"lock_verification"] = @"unavailable"; + } + if (hasLockedState && !locked) { + unlock[@"status"] = @"ok"; + extra[@"activation_method"] = activationMethod ?: @"lockscreen_unlock"; + extra[@"lockscreen_unlock"] = unlock; + return YES; + } + unlock[@"status"] = @"unavailable"; + unlock[@"reason"] = hasLockedState ? @"unlock_not_verified" : @"lock_verification_unavailable"; + extra[@"lockscreen_unlock"] = unlock; + return NO; +} + +static BOOL OPVTUnlockWithPasscode(NSString *passcode, NSMutableDictionary *extra) { + NSMutableArray *attempts = [NSMutableArray array]; + if (passcode.length == 0) { + extra[@"lockscreen_unlock"] = @{ + @"status": @"unavailable", + @"reason": @"missing_passcode" + }; + return NO; + } + + Class managerClass = NSClassFromString(@"SBLockScreenManager"); + id manager = OPVTInvokeObjectNoArg(managerClass, @"sharedInstance"); + if (!manager) { + OPVTAddLockScreenFallbackAttempt(attempts, @"SBLockScreenManager.sharedInstance", + nil, @"sharedInstance", @"unavailable", @"target_missing"); + extra[@"lockscreen_unlock"] = @{@"status": @"unavailable", @"attempts": attempts}; + return NO; + } + + NSString *error = nil; + BOOL ok = OPVTInvokeBoolObjectBoolBoolObject(manager, + @"_attemptUnlockWithPasscode:mesa:finishUIUnlock:completion:", + passcode, + NO, + YES, + nil, + &error); + OPVTAddLockScreenFallbackAttempt(attempts, @"SBLockScreenManager", manager, + @"_attemptUnlockWithPasscode:mesa:finishUIUnlock:completion:", + ok ? @"ok" : @"unavailable", error); + if (ok) { + NSString *finishError = nil; + BOOL finishOK = OPVTInvokeBoolIntegerObject(manager, + @"unlockUIFromSource:withOptions:", 0, nil, &finishError); + OPVTAddLockScreenFallbackAttempt(attempts, @"SBLockScreenManager", manager, + @"unlockUIFromSource:withOptions:", + finishOK ? @"ok" : @"unavailable", finishError); + if (!finishOK) { + finishError = nil; + finishOK = OPVTInvokeBoolIntegerObject(manager, + @"_finishUIUnlockFromSource:withOptions:", 0, nil, &finishError); + OPVTAddLockScreenFallbackAttempt(attempts, @"SBLockScreenManager", manager, + @"_finishUIUnlockFromSource:withOptions:", + finishOK ? @"ok" : @"unavailable", finishError); + } + if (finishOK) { + return OPVTFinalizePasscodeUnlock(extra, attempts, + @"SBLockScreenManager[_attemptUnlockWithPasscode + finishUIUnlock]"); + } + } + + error = nil; + ok = OPVTInvokeVoidOrBoolObjectBoolObject(manager, + @"attemptUnlockWithPasscode:finishUIUnlock:completion:", + passcode, + YES, + nil, + &error); + OPVTAddLockScreenFallbackAttempt(attempts, @"SBLockScreenManager", manager, + @"attemptUnlockWithPasscode:finishUIUnlock:completion:", + ok ? @"ok" : @"unavailable", error); + if (ok) { + return OPVTFinalizePasscodeUnlock(extra, attempts, + @"SBLockScreenManager[attemptUnlockWithPasscode:finishUIUnlock:completion:]"); + } + + extra[@"lockscreen_unlock"] = @{@"status": @"unavailable", @"attempts": attempts}; + return NO; +} + +static NSDictionary *OPVTElementSummaryForView(UIView *view, + UIWindow *window, + NSInteger windowIndex, + NSString *elementId) { + if (!view) { + return @{}; + } + CGRect bounds = CGRectZero; + @try { + bounds = [view convertRect:view.bounds toView:window ?: view.window]; + } @catch (__unused NSException *exception) { + bounds = CGRectZero; + } + NSMutableDictionary *summary = [@{ + @"id": elementId ?: @"", + @"kind": OPVTViewKind(view), + @"class": NSStringFromClass([view class]) ?: @"", + @"label": OPVTViewLabel(view) ?: @"", + @"bounds": OPVTBoundsArray(bounds), + @"enabled": @(view.userInteractionEnabled), + @"focused": @(view.isFirstResponder), + @"window_id": @(windowIndex), + @"sensitive": @NO, + @"risk_hint": @"springboard_only" + } mutableCopy]; + NSString *identifier = OPVTAccessibilityString(view.accessibilityIdentifier); + if (identifier.length > 0) { + summary[@"view_id"] = identifier; + } + return summary; +} + +static BOOL OPVTElementMatches(NSString *requestedId, NSString *elementId, NSString *viewId) { + if (requestedId.length == 0) { + return NO; + } + return [requestedId isEqualToString:elementId ?: @""] || + (viewId.length > 0 && [requestedId isEqualToString:viewId]); +} + +static UIView *OPVTFindInteractiveElementInView(UIView *view, + UIWindow *window, + NSInteger windowIndex, + NSUInteger depth, + NSUInteger *elementIndex, + NSString *requestedId, + NSDictionary **outSummary) { + if (!view || depth > 8 || requestedId.length == 0) { + return nil; + } + if (view.hidden || view.alpha < 0.02) { + return nil; + } + CGRect bounds = CGRectZero; + @try { + bounds = [view convertRect:view.bounds toView:window]; + } @catch (__unused NSException *exception) { + bounds = CGRectZero; + } + if (CGRectIsEmpty(bounds) || bounds.size.width < 1.0 || bounds.size.height < 1.0) { + return nil; + } + + NSString *label = OPVTViewLabel(view); + NSString *identifier = OPVTAccessibilityString(view.accessibilityIdentifier); + BOOL accessible = view.isAccessibilityElement || view.userInteractionEnabled || + [view isKindOfClass:[UIButton class]] || [view isKindOfClass:[UITextField class]]; + if (accessible && (label.length > 0 || identifier.length > 0)) { + NSString *elementId = [NSString stringWithFormat:@"springboard-%ld-%lu", + (long)windowIndex, (unsigned long)(*elementIndex)]; + (*elementIndex)++; + if (OPVTElementMatches(requestedId, elementId, identifier)) { + if (outSummary) { + *outSummary = OPVTElementSummaryForView(view, window, windowIndex, elementId); + } + return view; + } + } + + for (UIView *subview in view.subviews ?: @[]) { + UIView *match = OPVTFindInteractiveElementInView(subview, window, windowIndex, + depth + 1, elementIndex, requestedId, outSummary); + if (match) { + return match; + } + } + return nil; +} + +static UIView *OPVTFindInteractiveElement(NSString *requestedId, NSDictionary **outSummary) { + UIApplication *application = OPVTSafeSharedApplication(); + if (!application || requestedId.length == 0) { + return nil; + } + NSArray *windows = OPVTApplicationWindows(application); + NSInteger windowIndex = 0; + NSUInteger elementIndex = 0; + for (UIWindow *window in windows) { + if (!window || window.hidden || window.alpha < 0.02) { + windowIndex++; + continue; + } + UIView *match = OPVTFindInteractiveElementInView(window, window, windowIndex, 0, + &elementIndex, requestedId, outSummary); + if (match) { + return match; + } + windowIndex++; + } + return nil; +} + +static CGPoint OPVTNormalizeInputPoint(double x, double y, NSString **outCoordinateSpace) { + NSString *coordinateSpace = @"points"; + UIScreen *screen = OPVTSafeMainScreen(); + if (screen) { + @try { + CGRect bounds = screen.bounds; + CGFloat scale = screen.scale; + BOOL looksLikePixels = scale > 0.5 && + (x > bounds.size.width * 1.2 || y > bounds.size.height * 1.2) && + x <= bounds.size.width * scale * 1.2 && + y <= bounds.size.height * scale * 1.2; + if (looksLikePixels) { + x = x / scale; + y = y / scale; + coordinateSpace = @"pixels_scaled_to_points"; + } + } @catch (__unused NSException *exception) { + } + } + if (outCoordinateSpace) { + *outCoordinateSpace = coordinateSpace; + } + return CGPointMake((CGFloat)x, (CGFloat)y); +} + +static UIView *OPVTHitTestViewAtPoint(CGPoint point, + UIWindow **outWindow, + NSInteger *outWindowIndex, + NSDictionary **outSummary) { + UIApplication *application = OPVTSafeSharedApplication(); + if (!application) { + return nil; + } + NSArray *windows = [OPVTApplicationWindows(application) + sortedArrayUsingComparator:^NSComparisonResult(UIWindow *left, UIWindow *right) { + CGFloat leftLevel = left.windowLevel; + CGFloat rightLevel = right.windowLevel; + if (leftLevel < rightLevel) { + return NSOrderedDescending; + } + if (leftLevel > rightLevel) { + return NSOrderedAscending; + } + return NSOrderedSame; + }]; + + NSInteger windowIndex = 0; + for (UIWindow *window in windows) { + if (!window || window == OPVTOverlayWindow || window.hidden || window.alpha < 0.02 || + !window.userInteractionEnabled) { + windowIndex++; + continue; + } + CGPoint localPoint = CGPointZero; + @try { + localPoint = [window convertPoint:point fromWindow:nil]; + if (!CGRectContainsPoint(window.bounds, localPoint)) { + windowIndex++; + continue; + } + UIView *hitView = [window hitTest:localPoint withEvent:nil]; + if (hitView) { + if (outWindow) { + *outWindow = window; + } + if (outWindowIndex) { + *outWindowIndex = windowIndex; + } + if (outSummary) { + *outSummary = OPVTElementSummaryForView(hitView, window, windowIndex, @""); + } + return hitView; + } + } @catch (NSException *exception) { + OPVTLog(@"input hitTest exception window=%@ name=%@", + NSStringFromClass([window class]) ?: @"", exception.name ?: @""); + } + windowIndex++; + } + return nil; +} + +static NSDictionary *OPVTActivateView(UIView *view, + NSDictionary *targetSummary, + NSString *requestId, + NSString *actionType, + CGPoint point, + NSString *coordinateSpace, + long long durationMs, + BOOL includeDiagnostics) { + if (!view) { + return OPVTInputResponse(@"unavailable", @"target_view_missing", requestId, actionType, nil); + } + + NSMutableDictionary *extra = [@{ + @"target_found": @YES, + @"target": targetSummary ?: @{}, + @"point": @{@"x": @((double)point.x), @"y": @((double)point.y)}, + @"coordinate_space": coordinateSpace ?: @"points" + } mutableCopy]; + if (durationMs > 0) { + extra[@"duration_ms"] = @(durationMs); + } + if (includeDiagnostics) { + extra[@"diagnostics"] = @{ + @"target_methods": OPVTTargetInputDiagnostics(view), + @"targeted_runtime_candidates": OPVTTargetedGlobalInputDiagnostics() + }; + } + + @try { + BOOL activated = [view accessibilityActivate]; + extra[@"accessibility_activate_attempted"] = @YES; + extra[@"accessibility_activate_result"] = @(activated); + if (activated) { + extra[@"activation_method"] = @"accessibilityActivate"; + return OPVTInputResponse(@"ok", nil, requestId, actionType, extra); + } + } @catch (NSException *exception) { + extra[@"accessibility_activate_exception"] = exception.name ?: @""; + } + + if ([view isKindOfClass:[UIControl class]]) { + UIControl *control = (UIControl *)view; + if (!control.enabled) { + extra[@"control_enabled"] = @NO; + return OPVTInputResponse(@"unavailable", @"control_disabled", requestId, actionType, extra); + } + @try { + if ([actionType isEqualToString:@"long_press"]) { + [control sendActionsForControlEvents:UIControlEventTouchDown]; + long long boundedDurationMs = MAX(100LL, MIN(durationMs > 0 ? durationMs : 700LL, 1500LL)); + usleep((useconds_t)(boundedDurationMs * 1000)); + extra[@"bounded_duration_ms"] = @(boundedDurationMs); + } + [control sendActionsForControlEvents:UIControlEventPrimaryActionTriggered]; + [control sendActionsForControlEvents:UIControlEventTouchUpInside]; + extra[@"activation_method"] = @"UIControlActions"; + return OPVTInputResponse(@"ok", nil, requestId, actionType, extra); + } @catch (NSException *exception) { + extra[@"control_action_exception"] = exception.name ?: @""; + } + } + + if (OPVTTargetLooksLikeSwipeUpToUnlock(targetSummary) && + OPVTShowLockScreenPasscode(extra)) { + return OPVTInputResponse(@"ok", nil, requestId, actionType, extra); + } + + return OPVTInputResponse(@"unavailable", @"no_activation_method", requestId, actionType, extra); +} + +static NSDictionary *OPVTPerformSpringBoardInput(NSDictionary *request) { + NSString *requestId = [request[@"request_id"] isKindOfClass:[NSString class]] + ? request[@"request_id"] : @""; + NSDictionary *action = [request[@"action"] isKindOfClass:[NSDictionary class]] + ? request[@"action"] : @{}; + NSString *actionType = [action[@"type"] isKindOfClass:[NSString class]] + ? action[@"type"] : @""; + if (requestId.length == 0) { + return OPVTInputResponse(@"unavailable", @"missing_request_id", requestId, actionType, nil); + } + if (![actionType isEqualToString:@"tap"] && + ![actionType isEqualToString:@"tap_element"] && + ![actionType isEqualToString:@"long_press"] && + ![actionType isEqualToString:@"show_passcode"] && + ![actionType isEqualToString:@"unlock_with_passcode"]) { + return OPVTInputResponse(@"unavailable", @"unsupported_action_type", requestId, actionType, nil); + } + BOOL includeDiagnostics = OPVTBoolValue(action[@"diagnostics"], NO); + if ([actionType isEqualToString:@"show_passcode"]) { + NSMutableDictionary *extra = [@{ + @"target_found": @NO, + @"coordinate_space": @"springboard_lock_screen" + } mutableCopy]; + if (OPVTShowLockScreenPasscode(extra)) { + return OPVTInputResponse(@"ok", nil, requestId, actionType, extra); + } + return OPVTInputResponse(@"unavailable", @"passcode_visibility_unavailable", + requestId, actionType, extra); + } + if ([actionType isEqualToString:@"unlock_with_passcode"]) { + NSString *passcode = [action[@"passcode"] isKindOfClass:[NSString class]] + ? action[@"passcode"] : @""; + NSMutableDictionary *extra = [@{ + @"target_found": @NO, + @"coordinate_space": @"springboard_lock_screen" + } mutableCopy]; + if (passcode.length == 0) { + return OPVTInputResponse(@"unavailable", @"missing_passcode", + requestId, actionType, extra); + } + if (OPVTUnlockWithPasscode(passcode, extra)) { + return OPVTInputResponse(@"ok", nil, requestId, actionType, extra); + } + return OPVTInputResponse(@"unavailable", @"unlock_invocation_failed", + requestId, actionType, extra); + } + + NSString *coordinateSpace = @"points"; + CGPoint point = CGPointZero; + BOOL hasPoint = [action[@"x"] respondsToSelector:@selector(doubleValue)] && + [action[@"y"] respondsToSelector:@selector(doubleValue)]; + if (hasPoint) { + point = OPVTNormalizeInputPoint(OPVTDoubleValue(action[@"x"], 0.0), + OPVTDoubleValue(action[@"y"], 0.0), &coordinateSpace); + } + long long durationMs = OPVTLongLongValue(action[@"duration_ms"], + [actionType isEqualToString:@"long_press"] ? 700 : 80); + + NSDictionary *targetSummary = nil; + UIView *target = nil; + if ([actionType isEqualToString:@"tap_element"]) { + NSString *elementId = [action[@"element_id"] isKindOfClass:[NSString class]] + ? action[@"element_id"] : @""; + if (elementId.length == 0 && [action[@"view_id"] isKindOfClass:[NSString class]]) { + elementId = action[@"view_id"]; + } + if (elementId.length == 0) { + return OPVTInputResponse(@"unavailable", @"missing_element_id", requestId, actionType, nil); + } + target = OPVTFindInteractiveElement(elementId, &targetSummary); + if (!target) { + return OPVTInputResponse(@"unavailable", @"element_not_found", requestId, actionType, @{ + @"element_id": elementId + }); + } + if (!hasPoint) { + NSArray *bounds = targetSummary[@"bounds"]; + if ([bounds isKindOfClass:[NSArray class]] && bounds.count >= 4) { + double bx = OPVTDoubleValue(bounds[0], 0.0); + double by = OPVTDoubleValue(bounds[1], 0.0); + double bw = OPVTDoubleValue(bounds[2], 0.0); + double bh = OPVTDoubleValue(bounds[3], 0.0); + point = CGPointMake((CGFloat)(bx + bw / 2.0), (CGFloat)(by + bh / 2.0)); + coordinateSpace = @"ui_tree.bounds_center"; + } + } + } else { + if (!hasPoint) { + return OPVTInputResponse(@"unavailable", @"missing_coordinates", requestId, actionType, nil); + } + UIWindow *window = nil; + NSInteger windowIndex = -1; + target = OPVTHitTestViewAtPoint(point, &window, &windowIndex, &targetSummary); + if (!target) { + return OPVTInputResponse(@"unavailable", @"hit_test_miss", requestId, actionType, @{ + @"point": @{@"x": @((double)point.x), @"y": @((double)point.y)}, + @"coordinate_space": coordinateSpace + }); + } + } + + return OPVTActivateView(target, targetSummary, requestId, actionType, + point, coordinateSpace, durationMs, includeDiagnostics); +} + +static NSDictionary *OPVTSceneSnapshot(id scene, NSString *provider) { + if (!scene) { + return nil; + } + NSString *identifier = nil; + for (NSString *selector in @[@"identifier", @"sceneIdentifier", @"persistenceIdentifier"]) { + identifier = OPVTStringFromObject(OPVTInvokeObjectNoArg(scene, selector)); + if (identifier.length > 0) { + break; + } + } + NSString *bundleId = OPVTBundleIdentifierFromObject(scene, 0); + NSMutableDictionary *snapshot = [@{ + @"class": NSStringFromClass([scene class]) ?: @"", + @"provider": provider ?: @"unknown" + } mutableCopy]; + if (identifier.length > 0) { + snapshot[@"identifier"] = identifier; + } + if (bundleId.length > 0) { + snapshot[@"bundle_id"] = bundleId; + } + BOOL boolValue = NO; + for (NSString *selector in @[ + @"isForeground", + @"isForegroundActive", + @"isActive", + @"isUIApplicationScene", + @"isVisible" + ]) { + if (OPVTInvokeBoolNoArg(scene, selector, &boolValue)) { + snapshot[selector] = @(boolValue); + } + } + long long integerValue = 0; + for (NSString *selector in @[@"activationState", @"interfaceOrientation"]) { + if (OPVTInvokeIntegerNoArg(scene, selector, &integerValue)) { + snapshot[selector] = @(integerValue); + } + } + return snapshot; +} + +static NSArray *OPVTFrontBoardSceneSnapshots(void) { + NSMutableArray *snapshots = [NSMutableArray array]; + Class managerClass = objc_getClass("FBSceneManager"); + id manager = OPVTInvokeObjectNoArg(managerClass, @"sharedInstance"); + if (!manager) { + manager = OPVTInvokeObjectNoArg(managerClass, @"sharedManager"); + } + for (NSString *selector in @[@"scenes", @"allScenes", @"mutableScenes"]) { + NSArray *scenes = OPVTArrayFromCollection(OPVTInvokeObjectNoArg(manager, selector)); + for (id scene in scenes) { + NSDictionary *snapshot = OPVTSceneSnapshot(scene, [NSString stringWithFormat:@"FBSceneManager.%@", selector]); + if (snapshot) { + [snapshots addObject:snapshot]; + } + if (snapshots.count >= 32) { + return snapshots; + } + } + if (snapshots.count > 0) { + break; + } + } + return snapshots; +} + +static NSDictionary *OPVTForegroundCandidateSnapshot(id object, NSString *provider) { + if (!object) { + return nil; + } + NSString *bundleId = OPVTBundleIdentifierFromObject(object, 0); + NSString *stringValue = OPVTStringFromObject(object); + if (bundleId.length == 0 && OPVTBundleIdentifierLooksValid(stringValue)) { + bundleId = stringValue; + } + NSMutableDictionary *snapshot = [@{ + @"class": NSStringFromClass([object class]) ?: @"", + @"provider": provider ?: @"unknown" + } mutableCopy]; + if (bundleId.length > 0) { + snapshot[@"bundle_id"] = bundleId; + } + if (stringValue.length > 0 && stringValue.length < 160) { + snapshot[@"string_value"] = stringValue; + } + BOOL boolValue = NO; + for (NSString *selector in @[ + @"isForeground", + @"isForegroundActive", + @"isActive", + @"isVisible" + ]) { + if (OPVTInvokeBoolNoArg(object, selector, &boolValue)) { + snapshot[selector] = @(boolValue); + } + } + long long integerValue = 0; + for (NSString *selector in @[@"activationState", @"pid", @"processIdentifier"]) { + if (OPVTInvokeIntegerNoArg(object, selector, &integerValue)) { + snapshot[selector] = @(integerValue); + } + } + return snapshot.count > 2 ? snapshot : nil; +} + +static void OPVTAddForegroundCandidatesForTarget(NSMutableArray *candidates, + id target, NSString *providerPrefix) { + if (!target || candidates.count >= 32) { + return; + } + NSArray *selectors = @[ + @"foregroundApplication", + @"frontmostApplication", + @"frontMostApplication", + @"activeApplication", + @"currentApplication", + @"displayedApplication", + @"focusedApplication", + @"topApplication", + @"application", + @"_accessibilityFrontMostApplication", + @"accessibilityFrontMostApplication", + @"_accessibilityFocusedApplication", + @"mainDisplayLayoutState", + @"layoutState" + ]; + for (NSString *selector in selectors) { + id value = OPVTInvokeObjectNoArg(target, selector); + NSArray *values = OPVTArrayFromCollection(value); + if (values.count == 0 && value) { + values = @[value]; + } + for (id object in values) { + NSDictionary *snapshot = OPVTForegroundCandidateSnapshot( + object, [NSString stringWithFormat:@"%@.%@", providerPrefix ?: @"unknown", selector]); + if (snapshot) { + [candidates addObject:snapshot]; + } + if (candidates.count >= 32) { + return; + } + } + } +} + +static NSArray *OPVTForegroundCandidateSnapshots(void) { + NSMutableArray *candidates = [NSMutableArray array]; + UIApplication *application = OPVTSafeSharedApplication(); + OPVTAddForegroundCandidatesForTarget(candidates, application, @"UIApplication"); + + NSArray *classNames = @[ + @"SpringBoard", + @"SBMainWorkspace", + @"SBWorkspace", + @"SBUIController", + @"SBMainSwitcherController", + @"SBAppSwitcherModel", + @"SBApplicationController", + @"FBSceneManager" + ]; + NSArray *singletonSelectors = @[ + @"sharedInstance", + @"sharedManager", + @"sharedApplication", + @"sharedController" + ]; + for (NSString *className in classNames) { + Class cls = objc_getClass(className.UTF8String); + if (!cls) { + continue; + } + for (NSString *singletonSelector in singletonSelectors) { + id singleton = OPVTInvokeObjectNoArg(cls, singletonSelector); + if (!singleton) { + continue; + } + OPVTAddForegroundCandidatesForTarget( + candidates, + singleton, + [NSString stringWithFormat:@"%@.%@", className, singletonSelector]); + if (candidates.count >= 32) { + return candidates; + } + } + } + return candidates; +} + +static NSDictionary *OPVTProviderDiagnosticForTarget(id target, NSString *provider) { + NSMutableDictionary *diagnostic = [@{ + @"provider": provider ?: @"unknown", + @"available": @(target != nil) + } mutableCopy]; + if (!target) { + return diagnostic; + } + diagnostic[@"class"] = NSStringFromClass([target class]) ?: @""; + NSArray *selectors = @[ + @"scenes", + @"allScenes", + @"mutableScenes", + @"connectedScenes", + @"foregroundScenes", + @"applicationScenes", + @"runningApplications", + @"allApplications", + @"applications", + @"displayedApplications", + @"mainDisplayLayouts", + @"recentDisplayItems", + @"items" + ]; + NSMutableDictionary *counts = [NSMutableDictionary dictionary]; + for (NSString *selector in selectors) { + SEL sel = NSSelectorFromString(selector); + if (![target respondsToSelector:sel]) { + continue; + } + id value = OPVTInvokeObjectNoArg(target, selector); + NSArray *items = OPVTArrayFromCollection(value); + counts[selector] = @(items.count); + } + if (counts.count > 0) { + diagnostic[@"collection_counts"] = counts; + } + return diagnostic; +} + +static NSArray *OPVTProviderDiagnostics(void) { + NSMutableArray *diagnostics = [NSMutableArray array]; + UIApplication *application = OPVTSafeSharedApplication(); + [diagnostics addObject:OPVTProviderDiagnosticForTarget(application, @"UIApplication.sharedApplication")]; + + NSArray *classNames = @[ + @"FBSceneManager", + @"SBMainWorkspace", + @"SBWorkspace", + @"SBUIController", + @"SBMainSwitcherController", + @"SBAppSwitcherModel", + @"SBApplicationController", + @"SpringBoard" + ]; + NSArray *singletonSelectors = @[@"sharedInstance", @"sharedManager", @"sharedApplication", @"sharedController"]; + for (NSString *className in classNames) { + Class cls = objc_getClass(className.UTF8String); + if (!cls) { + [diagnostics addObject:@{ + @"provider": className, + @"available": @NO, + @"reason": @"class_missing" + }]; + continue; + } + BOOL addedSingleton = NO; + for (NSString *singletonSelector in singletonSelectors) { + id singleton = OPVTInvokeObjectNoArg(cls, singletonSelector); + if (!singleton) { + continue; + } + [diagnostics addObject:OPVTProviderDiagnosticForTarget( + singleton, + [NSString stringWithFormat:@"%@.%@", className, singletonSelector])]; + addedSingleton = YES; + } + if (!addedSingleton) { + [diagnostics addObject:@{ + @"provider": className, + @"available": @YES, + @"reason": @"singleton_unavailable" + }]; + } + } + return diagnostics; +} + +static NSString *OPVTOrientationName(NSInteger orientation) { + switch (orientation) { + case UIInterfaceOrientationPortrait: + return @"portrait"; + case UIInterfaceOrientationPortraitUpsideDown: + return @"portrait_upside_down"; + case UIInterfaceOrientationLandscapeLeft: + return @"landscape_left"; + case UIInterfaceOrientationLandscapeRight: + return @"landscape_right"; + default: + return @"unknown"; + } +} + +static NSDictionary *OPVTDisplaySnapshot(void) { + NSMutableDictionary *display = [@{ + @"status": @"unavailable", + @"provider": @"UIKit.SpringBoard" + } mutableCopy]; + UIScreen *screen = OPVTSafeMainScreen(); + if (!screen) { + display[@"reason"] = @"main_screen_unavailable"; + return display; + } + CGRect bounds = CGRectZero; + CGFloat scale = 0.0; + @try { + bounds = screen.bounds; + scale = screen.scale; + } @catch (NSException *exception) { + display[@"reason"] = @"screen_metrics_exception"; + display[@"exception_name"] = exception.name ?: @""; + return display; + } + NSInteger orientation = UIInterfaceOrientationUnknown; + if (@available(iOS 13.0, *)) { + UIWindowScene *scene = OPVTActiveWindowScene(); + orientation = scene ? scene.interfaceOrientation : UIInterfaceOrientationUnknown; + } else { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + UIApplication *application = OPVTSafeSharedApplication(); + orientation = application ? application.statusBarOrientation : UIInterfaceOrientationUnknown; +#pragma clang diagnostic pop + } + display[@"status"] = @"available"; + display[@"bounds_width"] = @((double)bounds.size.width); + display[@"bounds_height"] = @((double)bounds.size.height); + display[@"scale"] = @((double)scale); + display[@"pixel_width"] = @((double)(bounds.size.width * scale)); + display[@"pixel_height"] = @((double)(bounds.size.height * scale)); + display[@"orientation"] = @(orientation); + display[@"orientation_name"] = OPVTOrientationName(orientation); + return display; +} + +static BOOL OPVTWriteSpringBoardState(NSDictionary *state) { + NSData *data = [NSJSONSerialization dataWithJSONObject:state + options:0 + error:nil]; + if (data.length == 0) { + OPVTLog(@"springboard state encode failed"); + return NO; + } + BOOL wrote = [data writeToFile:OPVTSpringBoardStatePath atomically:YES]; + if (wrote) { + chmod(OPVTSpringBoardStatePath.UTF8String, 0644); + int count = __sync_add_and_fetch(&OPVTStatePublishSuccessCount, 1); + if (count <= 3 || count % 30 == 0) { + NSDictionary *display = [state[@"display"] isKindOfClass:[NSDictionary class]] + ? state[@"display"] : @{}; + OPVTLog(@"springboard state published count=%d foreground=%@ scenes=%@ display=%@", + count, + state[@"foreground_app"] ?: @"", + state[@"scene_count"] ?: @0, + display[@"status"] ?: @"unknown"); + } + return YES; + } + OPVTLog(@"springboard state write failed path=%@", OPVTSpringBoardStatePath); + return NO; +} + +static void OPVTPublishSpringBoardState(void) { + @autoreleasepool { + [[NSFileManager defaultManager] createDirectoryAtPath:OPVTStorePath + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + [[NSFileManager defaultManager] createDirectoryAtPath:OPVTSpringBoardStateDir + withIntermediateDirectories:YES + attributes:@{NSFilePosixPermissions: @0755} + error:nil]; + + @try { + NSArray *sceneSnapshots = OPVTFrontBoardSceneSnapshots(); + BOOL includeForegroundDiagnostics = OPVTBoolPreference(@"ForegroundDiagnosticsEnabled", NO); + NSArray *foregroundCandidates = includeForegroundDiagnostics + ? OPVTForegroundCandidateSnapshots() : @[]; + NSArray *providerDiagnostics = includeForegroundDiagnostics + ? OPVTProviderDiagnostics() : @[]; + NSDictionary *foregroundCache = OPVTForegroundCacheSnapshot(); + NSDictionary *uiTree = OPVTUITreeSnapshot(); + NSString *foreground = @""; + NSString *foregroundSource = @"unavailable"; + NSUInteger activeSceneCount = 0; + for (NSDictionary *scene in sceneSnapshots) { + NSString *bundleId = [scene[@"bundle_id"] isKindOfClass:[NSString class]] + ? scene[@"bundle_id"] : @""; + if (bundleId.length == 0 || [bundleId isEqualToString:@"com.apple.springboard"]) { + continue; + } + BOOL active = [scene[@"isForegroundActive"] boolValue] || + [scene[@"isForeground"] boolValue] || + [scene[@"isActive"] boolValue] || + ([scene[@"activationState"] respondsToSelector:@selector(integerValue)] && + [scene[@"activationState"] integerValue] <= 1); + if (active) { + activeSceneCount++; + } + if (foreground.length == 0 && active) { + foreground = bundleId; + foregroundSource = scene[@"provider"] ?: @"frontboard_scene_active"; + } + } + if (foreground.length == 0) { + for (NSDictionary *scene in sceneSnapshots) { + NSString *bundleId = [scene[@"bundle_id"] isKindOfClass:[NSString class]] + ? scene[@"bundle_id"] : @""; + if (bundleId.length > 0 && ![bundleId isEqualToString:@"com.apple.springboard"]) { + foreground = bundleId; + foregroundSource = scene[@"provider"] ?: @"frontboard_scene_first"; + break; + } + } + } + if (foreground.length == 0) { + for (NSDictionary *candidate in foregroundCandidates) { + NSString *bundleId = [candidate[@"bundle_id"] isKindOfClass:[NSString class]] + ? candidate[@"bundle_id"] : @""; + if (bundleId.length > 0 && ![bundleId isEqualToString:@"com.apple.springboard"]) { + foreground = bundleId; + foregroundSource = candidate[@"provider"] ?: @"foreground_candidate"; + break; + } + } + } + if (foreground.length == 0 && [foregroundCache[@"status"] isEqualToString:@"ok"]) { + NSString *bundleId = [foregroundCache[@"bundle_id"] isKindOfClass:[NSString class]] + ? foregroundCache[@"bundle_id"] : @""; + if (bundleId.length > 0 && ![bundleId isEqualToString:@"com.apple.springboard"]) { + foreground = bundleId; + foregroundSource = [NSString stringWithFormat:@"foreground_cache.%@", + foregroundCache[@"source"] ?: @"runtime"]; + } + } + + NSDictionary *state = @{ + @"schema": @"openphone.springboard_state.v1", + @"timestamp_ms": @(OPVTNowMs()), + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardState", + @"foreground_app": foreground ?: @"", + @"foreground_source": foregroundSource ?: @"unavailable", + @"active_scene_count": @(activeSceneCount), + @"scene_count": @(sceneSnapshots.count), + @"scenes": sceneSnapshots, + @"foreground_candidates": foregroundCandidates, + @"foreground_cache": foregroundCache, + @"provider_diagnostics": providerDiagnostics, + @"ui_tree": uiTree, + @"screenshot_bridge": OPVTScreenshotBridgeStatus(), + @"input_bridge": OPVTInputBridgeStatus(), + @"clipboard_bridge": OPVTClipboardBridgeStatus(), + @"prompt_bridge": OPVTPromptBridgeStatus(), + @"display": OPVTDisplaySnapshot(), + @"source": @"springboard" + }; + OPVTWriteSpringBoardState(state); + } @catch (NSException *exception) { + OPVTLog(@"springboard state publish exception name=%@ reason=%@", + exception.name ?: @"", exception.reason ?: @""); + NSDictionary *fallbackState = @{ + @"schema": @"openphone.springboard_state.v1", + @"timestamp_ms": @(OPVTNowMs()), + @"provider": @"OpenPhoneVolumeTrigger.SpringBoardState", + @"foreground_app": @"", + @"foreground_source": @"unavailable", + @"active_scene_count": @0, + @"scene_count": @0, + @"scenes": @[], + @"foreground_cache": OPVTForegroundCacheSnapshot(), + @"ui_tree": OPVTUITreeSnapshot(), + @"screenshot_bridge": OPVTScreenshotBridgeStatus(), + @"input_bridge": OPVTInputBridgeStatus(), + @"clipboard_bridge": OPVTClipboardBridgeStatus(), + @"prompt_bridge": OPVTPromptBridgeStatus(), + @"display": @{ + @"status": @"unavailable", + @"provider": @"UIKit.SpringBoard", + @"reason": @"publisher_exception", + @"exception_name": exception.name ?: @"" + }, + @"source": @"springboard", + @"publish_error": @"exception" + }; + OPVTWriteSpringBoardState(fallbackState); + } + } +} + +static void OPVTPublishSpringBoardStateOnMain(void) { + if ([NSThread isMainThread]) { + OPVTPublishSpringBoardState(); + } else { + dispatch_async(dispatch_get_main_queue(), ^{ + OPVTPublishSpringBoardState(); + }); + } +} + +static void OPVTShowOverlay(NSString *title, NSString *subtitle, UIColor *color) { + dispatch_async(dispatch_get_main_queue(), ^{ + UIScreen *screen = OPVTSafeMainScreen(); + if (!screen) { + OPVTLog(@"overlay skipped reason=main_screen_unavailable"); + return; + } + CGRect bounds = screen.bounds; + UIWindowScene *windowScene = OPVTActiveWindowScene(); + if (!OPVTOverlayWindow) { + if (@available(iOS 13.0, *)) { + if (windowScene) { + OPVTOverlayWindow = [[UIWindow alloc] initWithWindowScene:windowScene]; + OPVTOverlayWindow.frame = bounds; + } else { + OPVTOverlayWindow = [[UIWindow alloc] initWithFrame:bounds]; + } + } else { + OPVTOverlayWindow = [[UIWindow alloc] initWithFrame:bounds]; + } + OPVTOverlayWindow.windowLevel = UIWindowLevelAlert + 2000; + OPVTOverlayWindow.userInteractionEnabled = NO; + OPVTOverlayWindow.backgroundColor = [UIColor clearColor]; + UIViewController *controller = [[UIViewController alloc] init]; + controller.view.backgroundColor = [UIColor clearColor]; + OPVTOverlayWindow.rootViewController = controller; + } else if (@available(iOS 13.0, *)) { + if (windowScene && OPVTOverlayWindow.windowScene != windowScene) { + OPVTOverlayWindow.windowScene = windowScene; + } + } + OPVTOverlayWindow.frame = bounds; + UIView *root = OPVTOverlayWindow.rootViewController.view; + [root.subviews makeObjectsPerformSelector:@selector(removeFromSuperview)]; + + CGFloat width = MIN(bounds.size.width - 32.0, 330.0); + UIView *panel = [[UIView alloc] initWithFrame:CGRectMake((bounds.size.width - width) / 2.0, 58.0, width, 74.0)]; + panel.backgroundColor = [[UIColor blackColor] colorWithAlphaComponent:0.82]; + panel.layer.cornerRadius = 14.0; + panel.layer.masksToBounds = YES; + + UIView *stripe = [[UIView alloc] initWithFrame:CGRectMake(0, 0, 5.0, 74.0)]; + stripe.backgroundColor = color ?: [UIColor systemGreenColor]; + [panel addSubview:stripe]; + + UILabel *titleLabel = [[UILabel alloc] initWithFrame:CGRectMake(18.0, 11.0, width - 30.0, 26.0)]; + titleLabel.text = title ?: @"OpenPhone"; + titleLabel.textColor = [UIColor whiteColor]; + titleLabel.font = [UIFont boldSystemFontOfSize:17.0]; + [panel addSubview:titleLabel]; + + UILabel *subtitleLabel = [[UILabel alloc] initWithFrame:CGRectMake(18.0, 38.0, width - 30.0, 24.0)]; + subtitleLabel.text = subtitle ?: @"Triggered"; + subtitleLabel.textColor = [[UIColor whiteColor] colorWithAlphaComponent:0.86]; + subtitleLabel.font = [UIFont systemFontOfSize:13.0 weight:UIFontWeightMedium]; + subtitleLabel.adjustsFontSizeToFitWidth = YES; + subtitleLabel.minimumScaleFactor = 0.72; + [panel addSubview:subtitleLabel]; + + panel.alpha = 0.0; + [root addSubview:panel]; + OPVTOverlayWindow.hidden = NO; + OPVTOverlayWindow.alpha = 1.0; + OPVTLog(@"overlay shown title=%@ subtitle=%@ scene=%d", + title ?: @"", subtitle ?: @"", windowScene != nil); + + [UIView animateWithDuration:0.16 animations:^{ + panel.alpha = 1.0; + } completion:^(__unused BOOL finished) { + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(1.35 * NSEC_PER_SEC)), + dispatch_get_main_queue(), ^{ + [UIView animateWithDuration:0.18 animations:^{ + panel.alpha = 0.0; + } completion:^(__unused BOOL done) { + [panel removeFromSuperview]; + OPVTOverlayWindow.hidden = YES; + }]; + }); + }]; + }); +} + +static BOOL OPVTWriteAll(int fd, NSData *data, int *errorOut) { + const uint8_t *bytes = (const uint8_t *)data.bytes; + NSUInteger remaining = data.length; + while (remaining > 0) { + ssize_t written = write(fd, bytes, remaining); + if (written > 0) { + bytes += written; + remaining -= (NSUInteger)written; + continue; + } + if (written < 0 && errno == EINTR) { + continue; + } + if (errorOut) { + *errorOut = errno; + } + return NO; + } + return YES; +} + +static NSDictionary *OPVTAgentRequestOnce(NSData *data) { + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + return @{@"status": @"error", @"reason": @"socket_failed", @"errno": @(errno)}; + } +#ifdef SO_NOSIGPIPE + int noSigPipe = 1; + setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &noSigPipe, sizeof(noSigPipe)); +#endif + + struct sockaddr_un address; + memset(&address, 0, sizeof(address)); + address.sun_family = AF_UNIX; + strlcpy(address.sun_path, OPVTSocketPath, sizeof(address.sun_path)); + if (connect(fd, (struct sockaddr *)&address, sizeof(address)) != 0) { + int savedErrno = errno; + close(fd); + return @{@"status": @"error", @"reason": @"connect_failed", @"errno": @(savedErrno)}; + } + + NSMutableData *line = [data mutableCopy]; + const char newline = '\n'; + [line appendBytes:&newline length:1]; + int writeErrno = 0; + if (!OPVTWriteAll(fd, line, &writeErrno)) { + close(fd); + return @{@"status": @"error", @"reason": @"write_failed", @"errno": @(writeErrno)}; + } + shutdown(fd, SHUT_WR); + + NSMutableData *response = [NSMutableData data]; + char buffer[4096]; + while (response.length < 65536) { + ssize_t count = read(fd, buffer, sizeof(buffer)); + if (count > 0) { + [response appendBytes:buffer length:(NSUInteger)count]; + continue; + } + break; + } + close(fd); + + id object = [NSJSONSerialization JSONObjectWithData:response options:0 error:nil]; + if ([object isKindOfClass:[NSDictionary class]]) { + return object; + } + return @{@"status": @"error", @"reason": @"json_decode_failed", @"bytes": @(response.length)}; +} + +static BOOL OPVTShouldRetryAgentResponse(NSDictionary *response) { + if (![response[@"reason"] isEqualToString:@"connect_failed"]) { + return NO; + } + NSInteger code = [response[@"errno"] integerValue]; + return code == ECONNREFUSED || code == ENOENT || code == ETIMEDOUT || code == EAGAIN; +} + +static NSDictionary *OPVTAgentRequest(NSDictionary *request) { + NSData *data = [NSJSONSerialization dataWithJSONObject:request ?: @{} options:0 error:nil]; + if (!data) { + return @{@"status": @"error", @"reason": @"json_encode_failed"}; + } + + NSDictionary *response = nil; + const int attempts = 25; + for (int attempt = 1; attempt <= attempts; attempt++) { + response = OPVTAgentRequestOnce(data); + if (!OPVTShouldRetryAgentResponse(response)) { + if (attempt > 1) { + OPVTLog(@"agent request recovered attempt=%d response=%@", attempt, response); + } + return response; + } + OPVTLog(@"agent connect retry attempt=%d/%d response=%@", + attempt, attempts, response ?: @{}); + usleep(400000); + } + + NSMutableDictionary *finalResponse = [response mutableCopy] ?: [NSMutableDictionary dictionary]; + finalResponse[@"attempts"] = @(attempts); + return finalResponse; +} + +static void OPVTCallAgentWithGoal(NSString *requestedGoal, BOOL userProvidedGoal) { + OPVTPublishSpringBoardStateOnMain(); + NSString *goal = OPVTTrimmedString(requestedGoal); + if (goal.length == 0) { + goal = OPVTStringPreference(@"TriggerGoal", OPVTDefaultTriggerGoal); + } + if (goal.length == 0 || [goal isEqualToString:OPVTLegacyTriggerGoal]) { + goal = OPVTDefaultTriggerGoal; + } + BOOL runTask = OPVTBoolPreference(@"RunTask", YES); + BOOL createBackgroundJob = OPVTBoolPreference(@"CreateBackgroundJob", NO); + BOOL runBackgroundJobs = OPVTBoolPreference(@"RunBackgroundJobs", NO); + OPVTLastTriggerRoute = userProvidedGoal ? @"springboard_prompt_agent" : @"direct_agent"; + OPVTPublishTriggerStatus(@"agent_request", @{ + @"route": OPVTLastTriggerRoute ?: @"", + @"goal_length": @(goal.length), + @"user_provided_goal": @(userProvidedGoal) + }); + + NSDictionary *request = @{ + @"command": @"hardware_trigger", + @"trigger": @"volume_up_down_combo", + @"source": userProvidedGoal ? @"springboard_prompt" : @"springboard", + @"reason": userProvidedGoal ? @"hardware volume combo user prompt" : @"hardware volume combo", + @"goal": goal, + @"mode": @"auto", + @"run_task": @(runTask), + @"create_background_job": @(createBackgroundJob), + @"run_background_jobs": @(runBackgroundJobs), + @"trigger_input": userProvidedGoal ? @"springboard_prompt" : @"preference_or_default" + }; + + OPVTPlayHapticSuccess(); + OPVTShowOverlay(@"OpenPhone", + userProvidedGoal ? @"Agent request submitted" : @"Volume trigger detected", + [UIColor systemGreenColor]); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + NSDictionary *response = OPVTAgentRequest(request); + BOOL ok = [response[@"status"] isEqualToString:@"ok"]; + NSString *state = [response[@"state"] isKindOfClass:[NSString class]] + ? response[@"state"] : @""; + NSString *modelLoopStatus = [response[@"model_loop_status"] isKindOfClass:[NSString class]] + ? response[@"model_loop_status"] : @""; + NSString *detail = ok ? @"Agent trigger recorded locally" : @"Agent trigger failed"; + UIColor *overlayColor = ok ? [UIColor systemGreenColor] : [UIColor systemRedColor]; + if ([state isEqualToString:@"trigger.paused"] || + [state isEqualToString:@"trigger.disabled"] || + [state isEqualToString:@"trigger.yolo_disabled"]) { + detail = @"Agent trigger paused"; + overlayColor = [UIColor systemOrangeColor]; + } else if ([state isEqualToString:@"trigger.ignored_duplicate"]) { + detail = @"Duplicate trigger ignored"; + overlayColor = [UIColor systemOrangeColor]; + } else if ([modelLoopStatus isEqualToString:@"started_async"]) { + detail = @"Agent loop started"; + } else if ([modelLoopStatus isEqualToString:@"provider_not_ready"]) { + detail = @"Model provider not ready"; + overlayColor = [UIColor systemOrangeColor]; + } else if ([modelLoopStatus isEqualToString:@"start_failed"]) { + detail = @"Agent loop failed"; + overlayColor = [UIColor systemRedColor]; + } + OPVTLog(@"agent response ok=%d response=%@", ok, response); + OPVTPublishTriggerStatus(@"agent_response", @{ + @"route": OPVTLastTriggerRoute ?: @"", + @"ok": @(ok), + @"state": state ?: @"", + @"model_loop_status": modelLoopStatus ?: @"", + @"agent_task_id": response[@"agent_task_id"] ?: @"", + @"task_id": response[@"task_id"] ?: @"" + }); + if (ok) { + OPVTPlayHapticSuccess(); + } else { + OPVTPlayHapticFailure(); + } + OPVTShowOverlay(@"OpenPhone", detail, overlayColor); + }); +} + +static void OPVTCallAgent(void) { + OPVTCallAgentWithGoal(nil, NO); +} + +static NSString *OPVTVoiceOverlayDetail(NSDictionary *response) { + NSString *state = [response[@"state"] isKindOfClass:[NSString class]] + ? response[@"state"] : @""; + NSString *reason = [response[@"reason"] isKindOfClass:[NSString class]] + ? response[@"reason"] : @""; + NSString *lastError = [response[@"last_error"] isKindOfClass:[NSString class]] + ? response[@"last_error"] : @""; + NSString *error = lastError.length > 0 ? lastError : reason; + if ([state isEqualToString:@"voice.credential_missing"] || + [error isEqualToString:@"openai_voice_credential_missing"]) { + return @"Need OpenAI voice key"; + } + if ([state isEqualToString:@"voice.already_running"]) { + return @"Already listening"; + } + if ([state isEqualToString:@"voice.agent_started"]) { + return @"Agent loop started"; + } + if ([state isEqualToString:@"voice.transcription_failed"]) { + return @"Voice transcription failed"; + } + if ([state isEqualToString:@"voice.empty_transcript"]) { + return @"No speech heard"; + } + if ([state isEqualToString:@"voice.capture_failed"]) { + return @"Microphone failed"; + } + if ([state isEqualToString:@"voice.agent_not_started"]) { + return @"Agent loop failed"; + } + if ([state isEqualToString:@"voice.started_async"] || + [state isEqualToString:@"voice.listening_or_transcribing"] || + [state isEqualToString:@"voice.recording"] || + [state isEqualToString:@"voice.transcribing"]) { + return @"Listening"; + } + BOOL ok = [response[@"status"] isEqualToString:@"ok"]; + return ok ? @"Voice trigger recorded" : @"Voice agent failed"; +} + +// (Legacy toast-based helpers removed. The island observer now renders live +// status directly from the daemon's island-status.json file.) + +static void OPVTCallDaemonVoiceAgent(void) { + OPVTPublishSpringBoardStateOnMain(); + OPVTLastTriggerRoute = @"daemon_voice_agent"; + OPVTPublishTriggerStatus(@"voice_request", @{ + @"route": OPVTLastTriggerRoute ?: @"" + }); + NSDictionary *request = @{ + @"command": @"voice_trigger", + @"trigger": @"volume_up_down_combo", + @"source": @"springboard_volume", + @"reason": @"hardware volume combo voice trigger", + @"mode": @"auto", + @"max_steps": @25, + @"max_duration_ms": @600000 + }; + + OPVTPlayHapticSuccess(); + OPVTIslandApplyState(@{ + @"mode": @"listening", + @"subtitle": @"Listening", + @"accent": @"red" + }); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + NSDictionary *response = OPVTAgentRequest(request); + BOOL ok = [response[@"status"] isEqualToString:@"ok"]; + NSString *state = [response[@"state"] isKindOfClass:[NSString class]] + ? response[@"state"] : @""; + OPVTLog(@"daemon voice response ok=%d response=%@", ok, response); + OPVTPublishTriggerStatus(@"voice_response", @{ + @"route": OPVTLastTriggerRoute ?: @"", + @"ok": @(ok), + @"state": state ?: @"", + @"reason": response[@"reason"] ?: @"" + }); + // Only paint the island directly on early errors that the daemon + // reports synchronously (credential missing, already running, etc.). + // For the happy path the daemon will drive the island itself. + if (!ok) { + OPVTPlayHapticFailure(); + NSString *detail = OPVTVoiceOverlayDetail(response) ?: @"Voice trigger failed"; + NSString *accent = @"red"; + if ([state isEqualToString:@"voice.credential_missing"] || + [state isEqualToString:@"voice.already_running"] || + [state isEqualToString:@"voice.empty_transcript"]) { + accent = @"orange"; + } + OPVTIslandApplyState(@{ + @"mode": @"error", + @"subtitle": detail ?: @"", + @"accent": accent + }); + } + if (!ok && OPVTPromptForGoalEnabled() && + ![state isEqualToString:@"voice.credential_missing"]) { + OPVTPresentTriggerPrompt(); + } + }); +} + +static void OPVTHidePromptWindow(void) { + OPVTPromptVisible = NO; + if (!OPVTPromptWindow) { + return; + } + [OPVTPromptWindow resignKeyWindow]; + OPVTPromptWindow.hidden = YES; +} + +static void OPVTPresentTriggerPrompt(void) { + dispatch_async(dispatch_get_main_queue(), ^{ + if (OPVTPromptVisible) { + OPVTPlayHapticFailure(); + OPVTShowOverlay(@"OpenPhone", @"Prompt already open", [UIColor systemOrangeColor]); + return; + } + UIScreen *screen = OPVTSafeMainScreen(); + if (!screen) { + OPVTLog(@"prompt skipped reason=main_screen_unavailable"); + OPVTCallAgent(); + return; + } + + OPVTPromptVisible = YES; + CGRect bounds = screen.bounds; + UIWindowScene *windowScene = OPVTActiveWindowScene(); + if (!OPVTPromptWindow) { + if (@available(iOS 13.0, *)) { + if (windowScene) { + OPVTPromptWindow = [[UIWindow alloc] initWithWindowScene:windowScene]; + OPVTPromptWindow.frame = bounds; + } else { + OPVTPromptWindow = [[UIWindow alloc] initWithFrame:bounds]; + } + } else { + OPVTPromptWindow = [[UIWindow alloc] initWithFrame:bounds]; + } + OPVTPromptWindow.windowLevel = UIWindowLevelAlert + 2500; + OPVTPromptWindow.backgroundColor = [UIColor clearColor]; + OPVTPromptWindow.userInteractionEnabled = YES; + UIViewController *controller = [[UIViewController alloc] init]; + controller.view.backgroundColor = [UIColor clearColor]; + OPVTPromptWindow.rootViewController = controller; + } else if (@available(iOS 13.0, *)) { + if (windowScene && OPVTPromptWindow.windowScene != windowScene) { + OPVTPromptWindow.windowScene = windowScene; + } + } + OPVTPromptWindow.frame = bounds; + [OPVTPromptWindow makeKeyAndVisible]; + + UIAlertController *alert = [UIAlertController alertControllerWithTitle:@"OpenPhone" + message:@"What should I do?" + preferredStyle:UIAlertControllerStyleAlert]; + [alert addTextFieldWithConfigurationHandler:^(UITextField *textField) { + textField.placeholder = @"Speak or type a task"; + textField.clearButtonMode = UITextFieldViewModeWhileEditing; + textField.returnKeyType = UIReturnKeyGo; + textField.autocapitalizationType = UITextAutocapitalizationTypeSentences; + }]; + + UIAlertAction *cancelAction = [UIAlertAction actionWithTitle:@"Cancel" + style:UIAlertActionStyleCancel + handler:^(__unused UIAlertAction *action) { + OPVTLog(@"prompt cancelled"); + OPVTHidePromptWindow(); + OPVTShowOverlay(@"OpenPhone", @"Agent trigger cancelled", [UIColor systemOrangeColor]); + }]; + UIAlertAction *runAction = [UIAlertAction actionWithTitle:@"Run" + style:UIAlertActionStyleDefault + handler:^(__unused UIAlertAction *action) { + NSString *goal = OPVTTrimmedString(alert.textFields.firstObject.text); + OPVTLog(@"prompt submitted chars=%lu", (unsigned long)goal.length); + OPVTHidePromptWindow(); + if (goal.length == 0) { + OPVTPlayHapticFailure(); + OPVTShowOverlay(@"OpenPhone", @"No task entered", [UIColor systemOrangeColor]); + return; + } + OPVTCallAgentWithGoal(goal, YES); + }]; + [alert addAction:cancelAction]; + [alert addAction:runAction]; + alert.preferredAction = runAction; + + UIViewController *presenter = OPVTPromptWindow.rootViewController; + [presenter presentViewController:alert animated:YES completion:^{ + [alert.textFields.firstObject becomeFirstResponder]; + }]; + OPVTPlayHapticSuccess(); + OPVTLog(@"prompt shown scene=%d", windowScene != nil); + }); +} + +typedef CFTypeRef OPVTHIDEventRef; +typedef uint32_t (*OPVTIOHIDEventGetTypeFunc)(OPVTHIDEventRef event); +typedef int (*OPVTIOHIDEventGetIntegerValueFunc)(OPVTHIDEventRef event, uint32_t field); + +static const uint32_t OPVTIOHIDEventTypeKeyboard = 3; +static const uint32_t OPVTIOHIDEventFieldKeyboardUsagePage = (OPVTIOHIDEventTypeKeyboard << 16); +static const uint32_t OPVTIOHIDEventFieldKeyboardUsage = (OPVTIOHIDEventTypeKeyboard << 16) + 1; +static const uint32_t OPVTIOHIDEventFieldKeyboardDown = (OPVTIOHIDEventTypeKeyboard << 16) + 2; +static const uint32_t OPVTIOHIDUsagePageConsumer = 0x0c; +static const uint32_t OPVTIOHIDUsageConsumerVolumeIncrement = 0xe9; +static const uint32_t OPVTIOHIDUsageConsumerVolumeDecrement = 0xea; + +static BOOL OPVTRawHIDSymbolLoadAttempted = NO; +static BOOL OPVTRawHIDSymbolsAvailable = NO; +static OPVTIOHIDEventGetTypeFunc OPVTHIDEventGetType = NULL; +static OPVTIOHIDEventGetIntegerValueFunc OPVTHIDEventGetIntegerValue = NULL; + +static void OPVTEnsureRawHIDSymbols(void) { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + OPVTRawHIDSymbolLoadAttempted = YES; + NSArray *frameworks = @[ + @"/System/Library/Frameworks/IOKit.framework/IOKit", + @"/var/jb/System/Library/Frameworks/IOKit.framework/IOKit" + ]; + void *handle = NULL; + for (NSString *framework in frameworks) { + handle = dlopen(framework.UTF8String, RTLD_LAZY); + if (handle) { + break; + } + } + if (!handle) { + OPVTLog(@"raw HID symbols unavailable reason=dlopen_failed"); + return; + } + OPVTHIDEventGetType = (OPVTIOHIDEventGetTypeFunc)dlsym(handle, "IOHIDEventGetType"); + OPVTHIDEventGetIntegerValue = (OPVTIOHIDEventGetIntegerValueFunc)dlsym( + handle, "IOHIDEventGetIntegerValue"); + OPVTRawHIDSymbolsAvailable = OPVTHIDEventGetType && OPVTHIDEventGetIntegerValue; + OPVTLog(@"raw HID symbols loaded available=%d", OPVTRawHIDSymbolsAvailable); + }); +} + +static NSString *OPVTLimitedString(NSString *value, NSUInteger maxLength) { + if (![value isKindOfClass:[NSString class]]) { + return @""; + } + if (value.length <= maxLength) { + return value; + } + return [[value substringToIndex:maxLength] stringByAppendingString:@"..."]; +} + +static NSString *OPVTObjectClassName(id object) { + return object ? [NSString stringWithUTF8String:class_getName([object class]) ?: "unknown"] : @""; +} + +static Method OPVTEventMethod(id object, SEL selector) { + if (!object || !selector) { + return NULL; + } + Class cls = [object class]; + while (cls) { + Method method = class_getInstanceMethod(cls, selector); + if (method) { + return method; + } + cls = class_getSuperclass(cls); + } + return NULL; +} + +static BOOL OPVTEventReturnTypeLooksBool(Method method) { + if (!method) { + return NO; + } + char returnType[64] = {0}; + method_getReturnType(method, returnType, sizeof(returnType)); + return returnType[0] == 'B' || returnType[0] == 'c' || returnType[0] == 'C'; +} + +static BOOL OPVTEventReturnTypeLooksInteger(Method method) { + if (!method) { + return NO; + } + char returnType[64] = {0}; + method_getReturnType(method, returnType, sizeof(returnType)); + return returnType[0] == 'c' || returnType[0] == 'C' || + returnType[0] == 's' || returnType[0] == 'S' || + returnType[0] == 'i' || returnType[0] == 'I' || + returnType[0] == 'l' || returnType[0] == 'L' || + returnType[0] == 'q' || returnType[0] == 'Q' || + returnType[0] == 'B'; +} + +static BOOL OPVTEventInvokeBool(id object, NSString *selectorName, BOOL *okOut) { + if (okOut) { + *okOut = NO; + } + SEL selector = NSSelectorFromString(selectorName ?: @""); + if (!object || !selector || ![object respondsToSelector:selector]) { + return NO; + } + if (!OPVTEventReturnTypeLooksBool(OPVTEventMethod(object, selector))) { + return NO; + } + BOOL result = NO; + @try { + BOOL (*sendBool)(id, SEL) = (BOOL (*)(id, SEL))objc_msgSend; + result = sendBool(object, selector); + if (okOut) { + *okOut = YES; + } + } @catch (__unused NSException *exception) { + result = NO; + } + return result; +} + +static long long OPVTEventInvokeInteger(id object, NSString *selectorName, BOOL *okOut) { + if (okOut) { + *okOut = NO; + } + SEL selector = NSSelectorFromString(selectorName ?: @""); + if (!object || !selector || ![object respondsToSelector:selector]) { + return 0; + } + if (!OPVTEventReturnTypeLooksInteger(OPVTEventMethod(object, selector))) { + return 0; + } + long long result = 0; + @try { + long long (*sendInteger)(id, SEL) = (long long (*)(id, SEL))objc_msgSend; + result = sendInteger(object, selector); + if (okOut) { + *okOut = YES; + } + } @catch (__unused NSException *exception) { + result = 0; + } + return result; +} + +static NSString *OPVTEventDescription(id object) { + if (!object) { + return @""; + } + NSString *description = @""; + @try { + description = [object description] ?: @""; + } @catch (__unused NSException *exception) { + description = @""; + } + return OPVTLimitedString(description, 220); +} + +static NSDictionary *OPVTButtonEventSummary(id event, BOOL *volumeKnownOut, BOOL *volumeUpOut); + +static NSDictionary *OPVTCollectionButtonEventSummary(id event, BOOL *volumeKnownOut, BOOL *volumeUpOut) { + if (![event conformsToProtocol:@protocol(NSFastEnumeration)] || + [event isKindOfClass:[NSString class]] || + [event isKindOfClass:[NSDictionary class]]) { + return nil; + } + NSMutableArray *items = [NSMutableArray array]; + BOOL known = NO; + BOOL up = NO; + NSUInteger index = 0; + for (id item in event) { + if (index >= 6) { + break; + } + BOOL itemKnown = NO; + BOOL itemUp = NO; + NSDictionary *summary = OPVTButtonEventSummary(item, &itemKnown, &itemUp); + if (summary) { + [items addObject:summary]; + } + if (itemKnown && !known) { + known = YES; + up = itemUp; + } + index++; + } + if (items.count == 0) { + return nil; + } + if (volumeKnownOut) { + *volumeKnownOut = known; + } + if (volumeUpOut) { + *volumeUpOut = up; + } + return @{ + @"class": OPVTObjectClassName(event), + @"collection_count_sampled": @(items.count), + @"items": items + }; +} + +static NSDictionary *OPVTButtonEventSummary(id event, BOOL *volumeKnownOut, BOOL *volumeUpOut) { + if (volumeKnownOut) { + *volumeKnownOut = NO; + } + if (volumeUpOut) { + *volumeUpOut = NO; + } + if (!event) { + return @{}; + } + + BOOL collectionKnown = NO; + BOOL collectionUp = NO; + NSDictionary *collectionSummary = OPVTCollectionButtonEventSummary(event, &collectionKnown, &collectionUp); + if (collectionSummary) { + if (volumeKnownOut) { + *volumeKnownOut = collectionKnown; + } + if (volumeUpOut) { + *volumeUpOut = collectionUp; + } + return collectionSummary; + } + + NSMutableDictionary *summary = [NSMutableDictionary dictionary]; + NSString *className = OPVTObjectClassName(event); + NSString *description = OPVTEventDescription(event); + summary[@"class"] = className ?: @""; + if (description.length > 0) { + summary[@"description"] = description; + } + + NSMutableDictionary *selectors = [NSMutableDictionary dictionary]; + NSArray *upBoolSelectors = @[ + @"isVolumeUp", + @"isVolumeIncrease", + @"isVolumeIncrement", + @"volumeUp", + @"volumeIncrease" + ]; + NSArray *downBoolSelectors = @[ + @"isVolumeDown", + @"isVolumeDecrease", + @"isVolumeDecrement", + @"volumeDown", + @"volumeDecrease" + ]; + for (NSString *selectorName in upBoolSelectors) { + BOOL ok = NO; + BOOL value = OPVTEventInvokeBool(event, selectorName, &ok); + if (ok) { + selectors[selectorName] = @(value); + if (value) { + if (volumeKnownOut) { + *volumeKnownOut = YES; + } + if (volumeUpOut) { + *volumeUpOut = YES; + } + } + } + } + for (NSString *selectorName in downBoolSelectors) { + BOOL ok = NO; + BOOL value = OPVTEventInvokeBool(event, selectorName, &ok); + if (ok) { + selectors[selectorName] = @(value); + if (value) { + if (volumeKnownOut) { + *volumeKnownOut = YES; + } + if (volumeUpOut) { + *volumeUpOut = NO; + } + } + } + } + + NSArray *integerSelectors = @[ + @"usage", + @"hidUsage", + @"_usage", + @"buttonUsage", + @"usagePage", + @"hidUsagePage", + @"_usagePage", + @"button", + @"buttonType", + @"type" + ]; + for (NSString *selectorName in integerSelectors) { + BOOL ok = NO; + long long value = OPVTEventInvokeInteger(event, selectorName, &ok); + if (ok) { + selectors[selectorName] = @(value); + if (value == 0xE9) { + if (volumeKnownOut) { + *volumeKnownOut = YES; + } + if (volumeUpOut) { + *volumeUpOut = YES; + } + } else if (value == 0xEA) { + if (volumeKnownOut) { + *volumeKnownOut = YES; + } + if (volumeUpOut) { + *volumeUpOut = NO; + } + } + } + } + if (selectors.count > 0) { + summary[@"selectors"] = selectors; + } + + NSString *haystack = [[NSString stringWithFormat:@"%@ %@", className ?: @"", description ?: @""] + lowercaseString]; + BOOL textSaysUp = [haystack containsString:@"volumeup"] || + [haystack containsString:@"volume up"] || + [haystack containsString:@"volumeincrease"] || + [haystack containsString:@"volume increase"] || + [haystack containsString:@"volumeincrement"] || + [haystack containsString:@"volume increment"]; + BOOL textSaysDown = [haystack containsString:@"volumedown"] || + [haystack containsString:@"volume down"] || + [haystack containsString:@"volumedecrease"] || + [haystack containsString:@"volume decrease"] || + [haystack containsString:@"volumedecrement"] || + [haystack containsString:@"volume decrement"]; + if (textSaysUp || textSaysDown) { + if (volumeKnownOut) { + *volumeKnownOut = YES; + } + if (volumeUpOut) { + *volumeUpOut = textSaysUp && !textSaysDown; + } + summary[@"text_classified_volume"] = textSaysUp ? @"up" : @"down"; + } + return summary; +} + +static void OPVTRecordButtonObjectFromSource(id event, NSString *source) { + BOOL volumeKnown = NO; + BOOL volumeUp = NO; + NSDictionary *summary = OPVTButtonEventSummary(event, &volumeKnown, &volumeUp); + OPVTPublishTriggerStatus(volumeKnown ? @"button_event_object_volume" : + @"button_event_object_unclassified", @{ + @"source": source ?: @"", + @"volume_known": @(volumeKnown), + @"volume_up": @(volumeUp), + @"event": summary ?: @{} + }); + OPVTLog(@"button object source=%@ volume_known=%d volume_up=%d event=%@", + source ?: @"", volumeKnown, volumeUp, summary ?: @{}); + if (volumeKnown) { + NSString *recordSource = [NSString stringWithFormat:@"%@.%@", + source ?: @"button_object", volumeUp ? @"volume_up" : @"volume_down"]; + OPVTRecordButtonFromSource(volumeUp, recordSource); + } +} + +static NSDictionary *OPVTRawHIDEventSummary(void *event, BOOL *volumeKnownOut, BOOL *volumeUpOut) { + if (volumeKnownOut) { + *volumeKnownOut = NO; + } + if (volumeUpOut) { + *volumeUpOut = NO; + } + + NSMutableDictionary *summary = [NSMutableDictionary dictionary]; + summary[@"pointer"] = [NSString stringWithFormat:@"%p", event]; + OPVTEnsureRawHIDSymbols(); + summary[@"symbols_attempted"] = @(OPVTRawHIDSymbolLoadAttempted); + summary[@"symbols_available"] = @(OPVTRawHIDSymbolsAvailable); + if (!event) { + summary[@"reason"] = @"null_event"; + return summary; + } + if (!OPVTRawHIDSymbolsAvailable || !OPVTHIDEventGetType || !OPVTHIDEventGetIntegerValue) { + summary[@"reason"] = @"missing_hid_symbols"; + return summary; + } + + OPVTHIDEventRef hidEvent = (OPVTHIDEventRef)event; + uint32_t type = OPVTHIDEventGetType(hidEvent); + summary[@"type"] = @(type); + if (type != OPVTIOHIDEventTypeKeyboard) { + summary[@"reason"] = @"not_keyboard_event"; + return summary; + } + + uint32_t usagePage = (uint32_t)OPVTHIDEventGetIntegerValue(hidEvent, + OPVTIOHIDEventFieldKeyboardUsagePage); + uint32_t usage = (uint32_t)OPVTHIDEventGetIntegerValue(hidEvent, + OPVTIOHIDEventFieldKeyboardUsage); + int down = OPVTHIDEventGetIntegerValue(hidEvent, + OPVTIOHIDEventFieldKeyboardDown); + summary[@"usage_page"] = @(usagePage); + summary[@"usage"] = @(usage); + summary[@"down"] = @(down != 0); + + if (usagePage == OPVTIOHIDUsagePageConsumer && + usage == OPVTIOHIDUsageConsumerVolumeIncrement) { + if (volumeKnownOut) { + *volumeKnownOut = YES; + } + if (volumeUpOut) { + *volumeUpOut = YES; + } + summary[@"classification"] = @"volume_up"; + } else if (usagePage == OPVTIOHIDUsagePageConsumer && + usage == OPVTIOHIDUsageConsumerVolumeDecrement) { + if (volumeKnownOut) { + *volumeKnownOut = YES; + } + if (volumeUpOut) { + *volumeUpOut = NO; + } + summary[@"classification"] = @"volume_down"; + } else { + summary[@"reason"] = @"not_volume_consumer_usage"; + } + return summary; +} + +static void OPVTRecordRawHIDButtonFromSource(void *event, NSString *source) { + BOOL volumeKnown = NO; + BOOL volumeUp = NO; + NSDictionary *summary = OPVTRawHIDEventSummary(event, &volumeKnown, &volumeUp); + OPVTPublishTriggerStatus(volumeKnown ? @"button_event_raw_hid_volume" : + @"button_event_raw_hid_unclassified", @{ + @"source": source ?: @"", + @"volume_known": @(volumeKnown), + @"volume_up": @(volumeUp), + @"event": summary ?: @{} + }); + OPVTLog(@"raw HID button source=%@ volume_known=%d volume_up=%d event=%@", + source ?: @"", volumeKnown, volumeUp, summary ?: @{}); + if (volumeKnown) { + NSString *recordSource = [NSString stringWithFormat:@"%@.%@", + source ?: @"raw_hid", volumeUp ? @"volume_up" : @"volume_down"]; + OPVTRecordButtonFromSource(volumeUp, recordSource); + } +} + +static NSString *OPVTNotificationString(NSDictionary *userInfo, NSString *key) { + id value = [userInfo isKindOfClass:[NSDictionary class]] ? userInfo[key] : nil; + if ([value isKindOfClass:[NSString class]]) { + return value; + } + if ([value respondsToSelector:@selector(stringValue)]) { + return [value stringValue] ?: @""; + } + return @""; +} + +static NSString *OPVTVolumeDirectionFromNotification(NSDictionary *userInfo) { + NSArray *keys = @[ + @"AVSystemController_AudioVolumeChangeDirectionNotificationParameter", + @"AVSystemController_AudioVolumeDirectionNotificationParameter", + @"AVSystemController_AudioVolumeChangeDirection" + ]; + for (NSString *key in keys) { + NSString *value = OPVTNotificationString(userInfo, key).lowercaseString; + if (value.length == 0) { + continue; + } + if ([value containsString:@"up"] || [value containsString:@"increase"] || + [value containsString:@"increment"]) { + return @"up"; + } + if ([value containsString:@"down"] || [value containsString:@"decrease"] || + [value containsString:@"decrement"]) { + return @"down"; + } + } + return @""; +} + +static BOOL OPVTSeedSystemVolumeFromAVSystemController(void) { + Class cls = NSClassFromString(@"AVSystemController"); + SEL sharedSelector = NSSelectorFromString(@"sharedAVSystemController"); + if (!cls || ![cls respondsToSelector:sharedSelector]) { + return NO; + } + id controller = nil; + @try { + id (*sendShared)(Class, SEL) = (id (*)(Class, SEL))objc_msgSend; + controller = sendShared(cls, sharedSelector); + } @catch (__unused NSException *exception) { + controller = nil; + } + SEL getVolumeSelector = NSSelectorFromString(@"getVolume:forCategory:"); + if (!controller || ![controller respondsToSelector:getVolumeSelector]) { + return NO; + } + float volume = -1.0f; + BOOL ok = NO; + @try { + BOOL (*sendGetVolume)(id, SEL, float *, id) = + (BOOL (*)(id, SEL, float *, id))objc_msgSend; + ok = sendGetVolume(controller, getVolumeSelector, &volume, @"Audio/Video"); + } @catch (__unused NSException *exception) { + ok = NO; + } + if (!ok || !isfinite(volume) || volume < 0.0f || volume > 1.0f) { + return NO; + } + OPVTLastSystemVolume = volume; + return YES; +} + +static void OPVTInstallVolumeNotificationObserver(void) { + if (OPVTVolumeNotificationInstalled || OPVTVolumeNotificationObserverObject) { + return; + } + dispatch_async(dispatch_get_main_queue(), ^{ + if (OPVTVolumeNotificationInstalled || OPVTVolumeNotificationObserverObject) { + return; + } + BOOL seeded = OPVTSeedSystemVolumeFromAVSystemController(); + OPVTVolumeNotificationObserverObject = [OPVTVolumeNotificationObserver new]; + [[NSNotificationCenter defaultCenter] addObserver:OPVTVolumeNotificationObserverObject + selector:@selector(openphoneSystemVolumeDidChange:) + name:@"AVSystemController_SystemVolumeDidChangeNotification" + object:nil]; + OPVTVolumeNotificationInstalled = YES; + OPVTLog(@"volume notification observer installed seeded=%d volume=%0.3f", + seeded, OPVTLastSystemVolume); + OPVTPublishTriggerStatus(@"volume_notification_observer", @{ + @"installed": @YES, + @"seeded": @(seeded), + @"volume": @(OPVTLastSystemVolume >= 0.0 ? OPVTLastSystemVolume : -1.0) + }); + }); +} + +static void OPVTHandleSystemVolumeNotification(NSNotification *notification) { + NSDictionary *userInfo = [notification.userInfo isKindOfClass:[NSDictionary class]] + ? notification.userInfo : @{}; + double volume = OPVTDoubleValue(userInfo[@"AVSystemController_AudioVolumeNotificationParameter"], -1.0); + BOOL hasVolume = isfinite(volume) && volume >= 0.0 && volume <= 1.0; + double previousVolume = OPVTLastSystemVolume; + BOOL hadPreviousVolume = previousVolume >= 0.0; + NSString *reason = OPVTNotificationString(userInfo, + @"AVSystemController_AudioVolumeChangeReasonNotificationParameter"); + NSString *category = OPVTNotificationString(userInfo, + @"AVSystemController_AudioCategoryNotificationParameter"); + NSString *notificationDirection = OPVTVolumeDirectionFromNotification(userInfo); + + OPVTVolumeNotificationEventCount++; + OPVTLastVolumeNotificationMs = OPVTNowMs(); + OPVTLastVolumeNotificationReason = [reason copy] ?: @""; + OPVTLastVolumeNotificationCategory = [category copy] ?: @""; + OPVTLastVolumeNotificationDirection = notificationDirection.length > 0 + ? [notificationDirection copy] : @""; + if (hasVolume) { + OPVTLastSystemVolume = volume; + } + + double delta = (hasVolume && hadPreviousVolume) ? (volume - previousVolume) : 0.0; + BOOL directionKnown = NO; + BOOL volumeUp = NO; + if (fabs(delta) > 0.0005) { + directionKnown = YES; + volumeUp = delta > 0.0; + OPVTLastVolumeNotificationDirection = volumeUp ? @"up" : @"down"; + } else if ([notificationDirection isEqualToString:@"up"] || + [notificationDirection isEqualToString:@"down"]) { + directionKnown = YES; + volumeUp = [notificationDirection isEqualToString:@"up"]; + } + + NSDictionary *detail = @{ + @"events_seen": @(OPVTVolumeNotificationEventCount), + @"has_volume": @(hasVolume), + @"volume": @(hasVolume ? volume : -1.0), + @"previous_volume": @(hadPreviousVolume ? previousVolume : -1.0), + @"delta": @(delta), + @"direction_known": @(directionKnown), + @"volume_up": @(volumeUp), + @"reason": reason ?: @"", + @"category": category ?: @"" + }; + OPVTPublishTriggerStatus(directionKnown ? @"volume_notification_button" : + @"volume_notification_unclassified", detail); + OPVTLog(@"volume notification direction_known=%d up=%d volume=%0.3f previous=%0.3f reason=%@ category=%@", + directionKnown, volumeUp, hasVolume ? volume : -1.0, + hadPreviousVolume ? previousVolume : -1.0, reason ?: @"", category ?: @""); + if (directionKnown) { + OPVTRecordButtonFromSource(volumeUp, @"notification.AVSystemController.volume"); + } +} + +static void OPVTRecordButtonFromSource(BOOL volumeUp, NSString *source) { + if (!OPVTEnabled()) { + OPVTPublishTriggerStatus(@"button_ignored_disabled", @{ + @"volume_up": @(volumeUp), + @"source": source ?: @"" + }); + return; + } + + CFAbsoluteTime now = CFAbsoluteTimeGetCurrent(); + OPVTButtonEventCount++; + OPVTLastButtonEventMs = OPVTNowMs(); + OPVTLastButtonEventName = volumeUp ? @"volume_up" : @"volume_down"; + OPVTLastButtonEventSource = [source isKindOfClass:[NSString class]] && source.length > 0 + ? [source copy] : @"unknown"; + OPVTPublishTriggerStatus(@"button_event", @{ + @"button": OPVTLastButtonEventName ?: @"", + @"button_event_count": @(OPVTButtonEventCount), + @"source": OPVTLastButtonEventSource ?: @"" + }); + CFAbsoluteTime *lastLog = volumeUp ? &OPVTLastUpLog : &OPVTLastDownLog; + if ((now - *lastLog) >= 0.08) { + *lastLog = now; + OPVTLog(@"button event=%@ source=%@", + volumeUp ? @"volume_up" : @"volume_down", + OPVTLastButtonEventSource ?: @""); + } + + if (volumeUp) { + OPVTLastUp = now; + } else { + OPVTLastDown = now; + } + + BOOL combo = fabs(OPVTLastUp - OPVTLastDown) <= OPVTWindowSeconds(); + BOOL cooledDown = (now - OPVTLastTrigger) >= OPVTCooldownSeconds(); + if (combo && cooledDown) { + OPVTLastTrigger = now; + OPVTComboEventCount++; + OPVTLastComboEventMs = OPVTNowMs(); + + // If the island is in an active state, treat the combo as a cancel. + BOOL activeIslandMode = [OPVTIslandCurrentMode isEqualToString:@"listening"] || + [OPVTIslandCurrentMode isEqualToString:@"realtime"] || + [OPVTIslandCurrentMode isEqualToString:@"transcribing"] || + [OPVTIslandCurrentMode isEqualToString:@"thinking"] || + [OPVTIslandCurrentMode isEqualToString:@"action"]; + if (activeIslandMode) { + OPVTLog(@"volume combo -> cancel (island mode=%@)", + OPVTIslandCurrentMode ?: @""); + OPVTPublishTriggerStatus(@"volume_combo_cancel", @{ + @"mode": OPVTIslandCurrentMode ?: @"" + }); + OPVTPlayHapticFailure(); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + OPVTAgentRequest(@{@"command": @"voice_cancel", + @"reason": @"user_double_combo"}); + }); + return; + } + + NSString *route = OPVTDaemonVoiceAgentEnabled() + ? @"daemon_voice_agent" + : (OPVTPromptForGoalEnabled() ? @"springboard_prompt" : @"direct_agent"); + OPVTLastTriggerRoute = route; + OPVTPublishTriggerStatus(@"volume_combo", @{ + @"route": route ?: @"", + @"combo_event_count": @(OPVTComboEventCount), + @"source": OPVTLastButtonEventSource ?: @"", + @"last_up_delta_s": @(now - OPVTLastUp), + @"last_down_delta_s": @(now - OPVTLastDown) + }); + OPVTLog(@"volume combo detected up=%0.3f down=%0.3f source=%@", + OPVTLastUp, OPVTLastDown, OPVTLastButtonEventSource ?: @""); + if (OPVTDaemonVoiceAgentEnabled()) { + OPVTCallDaemonVoiceAgent(); + } else if (OPVTPromptForGoalEnabled()) { + OPVTPresentTriggerPrompt(); + } else { + OPVTCallAgent(); + } + } +} + +static OPVTHookSpec *OPVTHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTHookSpecs) / sizeof(OPVTHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTHookSpec *spec = &OPVTHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + if (!sel_isEqual(selector, specSelector)) { + continue; + } + Class cls = objc_getClass(spec->className); + if (cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static void OPVTVolumeNoArgReplacement(id self, SEL _cmd) { + OPVTHookSpec *spec = OPVTHookSpecFor(self, _cmd); + const char *selectorName = sel_getName(_cmd); + BOOL volumeUp = selectorName && strstr(selectorName, "increase") != NULL; + NSString *source = [NSString stringWithFormat:@"runtime.%s.%s", + class_getName([self class]) ?: "unknown", selectorName ?: "unknown"]; + if (spec) { + volumeUp = spec->volumeUp; + source = [NSString stringWithFormat:@"runtime.%s.%s", + spec->className ?: "unknown", spec->selectorName ?: "unknown"]; + } + + OPVTRecordButtonFromSource(volumeUp, source); + + if (spec && spec->original && spec->original != (IMP)OPVTVolumeNoArgReplacement) { + ((void (*)(id, SEL))spec->original)(self, _cmd); + } +} + +static OPVTFixedArgHookSpec *OPVTFixedArgHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTFixedArgHookSpecs) / sizeof(OPVTFixedArgHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTFixedArgHookSpec *spec = &OPVTFixedArgHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + Class cls = objc_getClass(spec->className); + if (sel_isEqual(selector, specSelector) && cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static OPVTBoolArgHookSpec *OPVTBoolArgHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTBoolArgHookSpecs) / sizeof(OPVTBoolArgHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTBoolArgHookSpec *spec = &OPVTBoolArgHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + Class cls = objc_getClass(spec->className); + if (sel_isEqual(selector, specSelector) && cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static OPVTBoolPairHookSpec *OPVTBoolPairHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTBoolPairHookSpecs) / sizeof(OPVTBoolPairHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTBoolPairHookSpec *spec = &OPVTBoolPairHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + Class cls = objc_getClass(spec->className); + if (sel_isEqual(selector, specSelector) && cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static OPVTButtonObjectVoidHookSpec *OPVTButtonObjectVoidHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTButtonObjectVoidHookSpecs) / sizeof(OPVTButtonObjectVoidHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTButtonObjectVoidHookSpec *spec = &OPVTButtonObjectVoidHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + Class cls = objc_getClass(spec->className); + if (sel_isEqual(selector, specSelector) && cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static OPVTButtonObjectBoolHookSpec *OPVTButtonObjectBoolHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTButtonObjectBoolHookSpecs) / sizeof(OPVTButtonObjectBoolHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTButtonObjectBoolHookSpec *spec = &OPVTButtonObjectBoolHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + Class cls = objc_getClass(spec->className); + if (sel_isEqual(selector, specSelector) && cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static OPVTButtonRawHIDVoidHookSpec *OPVTButtonRawHIDVoidHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTButtonRawHIDVoidHookSpecs) / sizeof(OPVTButtonRawHIDVoidHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTButtonRawHIDVoidHookSpec *spec = &OPVTButtonRawHIDVoidHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + Class cls = objc_getClass(spec->className); + if (sel_isEqual(selector, specSelector) && cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static NSString *OPVTForegroundSource(const char *className, const char *selectorName) { + return [NSString stringWithFormat:@"runtime.%s.%s", + className ?: "unknown", selectorName ?: "unknown"]; +} + +static OPVTForegroundNoArgObjectHookSpec *OPVTForegroundNoArgObjectHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTForegroundNoArgObjectHookSpecs) / sizeof(OPVTForegroundNoArgObjectHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTForegroundNoArgObjectHookSpec *spec = &OPVTForegroundNoArgObjectHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + Class cls = objc_getClass(spec->className); + if (sel_isEqual(selector, specSelector) && cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static OPVTForegroundObjectArgVoidHookSpec *OPVTForegroundObjectArgVoidHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTForegroundObjectArgVoidHookSpecs) / sizeof(OPVTForegroundObjectArgVoidHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTForegroundObjectArgVoidHookSpec *spec = &OPVTForegroundObjectArgVoidHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + Class cls = objc_getClass(spec->className); + if (sel_isEqual(selector, specSelector) && cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static OPVTForegroundObjectArgObjectHookSpec *OPVTForegroundObjectArgObjectHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTForegroundObjectArgObjectHookSpecs) / sizeof(OPVTForegroundObjectArgObjectHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTForegroundObjectArgObjectHookSpec *spec = &OPVTForegroundObjectArgObjectHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + Class cls = objc_getClass(spec->className); + if (sel_isEqual(selector, specSelector) && cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static OPVTForegroundNoArgBoolHookSpec *OPVTForegroundNoArgBoolHookSpecFor(id self, SEL selector) { + size_t count = sizeof(OPVTForegroundNoArgBoolHookSpecs) / sizeof(OPVTForegroundNoArgBoolHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTForegroundNoArgBoolHookSpec *spec = &OPVTForegroundNoArgBoolHookSpecs[index]; + if (!spec->original) { + continue; + } + SEL specSelector = sel_registerName(spec->selectorName); + Class cls = objc_getClass(spec->className); + if (sel_isEqual(selector, specSelector) && cls && [self isKindOfClass:cls]) { + return spec; + } + } + return NULL; +} + +static void OPVTVolumeFixedArgReplacement(id self, SEL _cmd, uintptr_t arg) { + OPVTFixedArgHookSpec *spec = OPVTFixedArgHookSpecFor(self, _cmd); + if (spec) { + OPVTRecordButtonFromSource(spec->volumeUp, + [NSString stringWithFormat:@"runtime.%s.%s", + spec->className ?: "unknown", spec->selectorName ?: "unknown"]); + if (spec->original && spec->original != (IMP)OPVTVolumeFixedArgReplacement) { + ((void (*)(id, SEL, uintptr_t))spec->original)(self, _cmd, arg); + } + } +} + +static void OPVTVolumeBoolArgReplacement(id self, SEL _cmd, BOOL increase) { + OPVTBoolArgHookSpec *spec = OPVTBoolArgHookSpecFor(self, _cmd); + NSString *source = spec + ? [NSString stringWithFormat:@"runtime.%s.%s", + spec->className ?: "unknown", spec->selectorName ?: "unknown"] + : [NSString stringWithFormat:@"runtime.%s.%s", + class_getName([self class]) ?: "unknown", sel_getName(_cmd) ?: "unknown"]; + OPVTRecordButtonFromSource(increase, source); + if (spec && spec->original && spec->original != (IMP)OPVTVolumeBoolArgReplacement) { + ((void (*)(id, SEL, BOOL))spec->original)(self, _cmd, increase); + } +} + +static void OPVTVolumeBoolPairReplacement(id self, SEL _cmd, BOOL increase, uintptr_t second) { + OPVTBoolPairHookSpec *spec = OPVTBoolPairHookSpecFor(self, _cmd); + BOOL shouldRecord = YES; + if (spec && spec->secondArgumentMeansDown) { + shouldRecord = (second != 0); + } + if (shouldRecord) { + NSString *source = spec + ? [NSString stringWithFormat:@"runtime.%s.%s", + spec->className ?: "unknown", spec->selectorName ?: "unknown"] + : [NSString stringWithFormat:@"runtime.%s.%s", + class_getName([self class]) ?: "unknown", sel_getName(_cmd) ?: "unknown"]; + OPVTRecordButtonFromSource(increase, source); + } + if (spec && spec->original && spec->original != (IMP)OPVTVolumeBoolPairReplacement) { + ((void (*)(id, SEL, BOOL, uintptr_t))spec->original)(self, _cmd, increase, second); + } +} + +static void OPVTButtonObjectVoidReplacement(id self, SEL _cmd, id event) { + OPVTButtonObjectVoidHookSpec *spec = OPVTButtonObjectVoidHookSpecFor(self, _cmd); + NSString *source = spec + ? [NSString stringWithFormat:@"runtime.%s.%s", + spec->className ?: "unknown", spec->selectorName ?: "unknown"] + : [NSString stringWithFormat:@"runtime.%s.%s", + class_getName([self class]) ?: "unknown", sel_getName(_cmd) ?: "unknown"]; + if (spec && spec->original && spec->original != (IMP)OPVTButtonObjectVoidReplacement) { + ((void (*)(id, SEL, id))spec->original)(self, _cmd, event); + } + OPVTRecordButtonObjectFromSource(event, source); +} + +static BOOL OPVTButtonObjectBoolReplacement(id self, SEL _cmd, id event) { + OPVTButtonObjectBoolHookSpec *spec = OPVTButtonObjectBoolHookSpecFor(self, _cmd); + NSString *source = spec + ? [NSString stringWithFormat:@"runtime.%s.%s", + spec->className ?: "unknown", spec->selectorName ?: "unknown"] + : [NSString stringWithFormat:@"runtime.%s.%s", + class_getName([self class]) ?: "unknown", sel_getName(_cmd) ?: "unknown"]; + BOOL result = NO; + if (spec && spec->original && spec->original != (IMP)OPVTButtonObjectBoolReplacement) { + result = ((BOOL (*)(id, SEL, id))spec->original)(self, _cmd, event); + } + OPVTRecordButtonObjectFromSource(event, source); + return result; +} + +static void OPVTButtonRawHIDVoidReplacement(id self, SEL _cmd, void *event) { + OPVTButtonRawHIDVoidHookSpec *spec = OPVTButtonRawHIDVoidHookSpecFor(self, _cmd); + NSString *source = spec + ? [NSString stringWithFormat:@"runtime.%s.%s", + spec->className ?: "unknown", spec->selectorName ?: "unknown"] + : [NSString stringWithFormat:@"runtime.%s.%s", + class_getName([self class]) ?: "unknown", sel_getName(_cmd) ?: "unknown"]; + OPVTRecordRawHIDButtonFromSource(event, source); + if (spec && spec->original && spec->original != (IMP)OPVTButtonRawHIDVoidReplacement) { + ((void (*)(id, SEL, void *))spec->original)(self, _cmd, event); + } +} + +static id OPVTForegroundNoArgObjectReplacement(id self, SEL _cmd) { + OPVTForegroundNoArgObjectHookSpec *spec = OPVTForegroundNoArgObjectHookSpecFor(self, _cmd); + id result = nil; + if (spec && spec->original && spec->original != (IMP)OPVTForegroundNoArgObjectReplacement) { + result = ((id (*)(id, SEL))spec->original)(self, _cmd); + OPVTRecordForegroundObject(result, OPVTForegroundSource(spec->className, spec->selectorName)); + } + return result; +} + +static void OPVTForegroundObjectArgVoidReplacement(id self, SEL _cmd, id arg) { + OPVTForegroundObjectArgVoidHookSpec *spec = OPVTForegroundObjectArgVoidHookSpecFor(self, _cmd); + if (spec && spec->original && spec->original != (IMP)OPVTForegroundObjectArgVoidReplacement) { + ((void (*)(id, SEL, id))spec->original)(self, _cmd, arg); + NSString *source = OPVTForegroundSource(spec->className, spec->selectorName); + OPVTRecordForegroundObject(arg, source); + if (strcmp(spec->selectorName, "updateFrontMostApplicationWithServerInstance:") == 0) { + id foreground = OPVTInvokeObjectOneArg(self, @"frontmostAppProcessWithServerInstance:", arg); + OPVTRecordForegroundObject(foreground, [source stringByAppendingString:@".frontmost"]); + } + } +} + +static id OPVTForegroundObjectArgObjectReplacement(id self, SEL _cmd, id arg) { + OPVTForegroundObjectArgObjectHookSpec *spec = OPVTForegroundObjectArgObjectHookSpecFor(self, _cmd); + id result = nil; + if (spec && spec->original && spec->original != (IMP)OPVTForegroundObjectArgObjectReplacement) { + result = ((id (*)(id, SEL, id))spec->original)(self, _cmd, arg); + OPVTRecordForegroundObject(result, OPVTForegroundSource(spec->className, spec->selectorName)); + } + return result; +} + +static BOOL OPVTForegroundNoArgBoolReplacement(id self, SEL _cmd) { + OPVTForegroundNoArgBoolHookSpec *spec = OPVTForegroundNoArgBoolHookSpecFor(self, _cmd); + BOOL result = NO; + if (spec && spec->original && spec->original != (IMP)OPVTForegroundNoArgBoolReplacement) { + result = ((BOOL (*)(id, SEL))spec->original)(self, _cmd); + if (result) { + OPVTRecordForegroundObject(self, OPVTForegroundSource(spec->className, spec->selectorName)); + } + } + return result; +} + +static void OPVTTryHookSpec(OPVTHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"runtime hook missing class=%s phase=%@", spec->className, phase ?: @""); + } + return; + } + + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"runtime hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + + unsigned int argumentCount = method_getNumberOfArguments(method); + if (argumentCount != 2) { + OPVTLog(@"runtime hook skipped method=%s[%s] args=%u phase=%@", + spec->className, spec->selectorName, argumentCount, phase ?: @""); + return; + } + + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTVolumeNoArgReplacement) { + spec->hooked = YES; + return; + } + + spec->original = method_setImplementation(method, (IMP)OPVTVolumeNoArgReplacement); + spec->hooked = YES; + OPVTLog(@"runtime hook installed method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); +} + +static void OPVTTryHookFixedArgSpec(OPVTFixedArgHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"runtime hook missing class=%s phase=%@", spec->className, phase ?: @""); + } + return; + } + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"runtime hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + unsigned int argumentCount = method_getNumberOfArguments(method); + if (argumentCount != 3) { + OPVTLog(@"runtime hook skipped method=%s[%s] args=%u phase=%@", + spec->className, spec->selectorName, argumentCount, phase ?: @""); + return; + } + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTVolumeFixedArgReplacement) { + spec->hooked = YES; + return; + } + spec->original = method_setImplementation(method, (IMP)OPVTVolumeFixedArgReplacement); + spec->hooked = YES; + OPVTLog(@"runtime hook installed method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); +} + +static void OPVTTryHookBoolArgSpec(OPVTBoolArgHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"runtime hook missing class=%s phase=%@", spec->className, phase ?: @""); + } + return; + } + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"runtime hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + unsigned int argumentCount = method_getNumberOfArguments(method); + if (argumentCount != 3) { + OPVTLog(@"runtime hook skipped method=%s[%s] args=%u phase=%@", + spec->className, spec->selectorName, argumentCount, phase ?: @""); + return; + } + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTVolumeBoolArgReplacement) { + spec->hooked = YES; + return; + } + spec->original = method_setImplementation(method, (IMP)OPVTVolumeBoolArgReplacement); + spec->hooked = YES; + OPVTLog(@"runtime hook installed method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); +} + +static void OPVTTryHookBoolPairSpec(OPVTBoolPairHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"runtime hook missing class=%s phase=%@", spec->className, phase ?: @""); + } + return; + } + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"runtime hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + unsigned int argumentCount = method_getNumberOfArguments(method); + if (argumentCount != 4) { + OPVTLog(@"runtime hook skipped method=%s[%s] args=%u phase=%@", + spec->className, spec->selectorName, argumentCount, phase ?: @""); + return; + } + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTVolumeBoolPairReplacement) { + spec->hooked = YES; + return; + } + spec->original = method_setImplementation(method, (IMP)OPVTVolumeBoolPairReplacement); + spec->hooked = YES; + OPVTLog(@"runtime hook installed method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); +} + +static void OPVTTryButtonObjectVoidHookSpec(OPVTButtonObjectVoidHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"button object hook missing class=%s phase=%@", + spec->className, phase ?: @""); + } + return; + } + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"button object hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + unsigned int argumentCount = method_getNumberOfArguments(method); + if (argumentCount != 3 || !OPVTMethodReturnTypeStartsWith(method, 'v') || + !OPVTMethodArgumentTypeLooksObject(method, 2)) { + char returnType[64] = {0}; + char argumentType[64] = {0}; + method_getReturnType(method, returnType, sizeof(returnType)); + method_getArgumentType(method, 2, argumentType, sizeof(argumentType)); + OPVTLog(@"button object hook skipped method=%s[%s] args=%u return=%s arg2=%s phase=%@", + spec->className, spec->selectorName, argumentCount, + returnType[0] ? returnType : "", + argumentType[0] ? argumentType : "", phase ?: @""); + return; + } + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTButtonObjectVoidReplacement) { + spec->hooked = YES; + return; + } + spec->original = method_setImplementation(method, (IMP)OPVTButtonObjectVoidReplacement); + spec->hooked = YES; + OPVTLog(@"button object hook installed method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); +} + +static void OPVTTryButtonObjectBoolHookSpec(OPVTButtonObjectBoolHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"button object hook missing class=%s phase=%@", + spec->className, phase ?: @""); + } + return; + } + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"button object hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + unsigned int argumentCount = method_getNumberOfArguments(method); + if (argumentCount != 3 || !OPVTMethodReturnTypeLooksBool(method) || + !OPVTMethodArgumentTypeLooksObject(method, 2)) { + char returnType[64] = {0}; + char argumentType[64] = {0}; + method_getReturnType(method, returnType, sizeof(returnType)); + method_getArgumentType(method, 2, argumentType, sizeof(argumentType)); + OPVTLog(@"button object hook skipped method=%s[%s] args=%u return=%s arg2=%s phase=%@", + spec->className, spec->selectorName, argumentCount, + returnType[0] ? returnType : "", + argumentType[0] ? argumentType : "", phase ?: @""); + return; + } + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTButtonObjectBoolReplacement) { + spec->hooked = YES; + return; + } + spec->original = method_setImplementation(method, (IMP)OPVTButtonObjectBoolReplacement); + spec->hooked = YES; + OPVTLog(@"button object hook installed method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); +} + +static void OPVTTryButtonRawHIDVoidHookSpec(OPVTButtonRawHIDVoidHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"raw HID hook missing class=%s phase=%@", + spec->className, phase ?: @""); + } + return; + } + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"raw HID hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + unsigned int argumentCount = method_getNumberOfArguments(method); + char returnType[64] = {0}; + char argumentType[64] = {0}; + method_getReturnType(method, returnType, sizeof(returnType)); + method_getArgumentType(method, 2, argumentType, sizeof(argumentType)); + if (argumentCount != 3 || returnType[0] != 'v' || argumentType[0] != '^') { + OPVTLog(@"raw HID hook skipped method=%s[%s] args=%u return=%s arg2=%s phase=%@", + spec->className, spec->selectorName, argumentCount, + returnType[0] ? returnType : "", + argumentType[0] ? argumentType : "", phase ?: @""); + return; + } + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTButtonRawHIDVoidReplacement) { + spec->hooked = YES; + return; + } + spec->original = method_setImplementation(method, (IMP)OPVTButtonRawHIDVoidReplacement); + spec->hooked = YES; + OPVTLog(@"raw HID hook installed method=%s[%s] arg2=%s phase=%@", + spec->className, spec->selectorName, + argumentType[0] ? argumentType : "", phase ?: @""); +} + +static BOOL OPVTMethodReturnTypeStartsWith(Method method, char expected) { + char returnType[64] = {0}; + method_getReturnType(method, returnType, sizeof(returnType)); + return returnType[0] == expected; +} + +static BOOL OPVTMethodReturnTypeLooksBool(Method method) { + char returnType[64] = {0}; + method_getReturnType(method, returnType, sizeof(returnType)); + return returnType[0] == 'B' || returnType[0] == 'c' || returnType[0] == 'C'; +} + +static BOOL OPVTMethodArgumentTypeLooksObject(Method method, unsigned int index) { + char argumentType[64] = {0}; + method_getArgumentType(method, index, argumentType, sizeof(argumentType)); + return argumentType[0] == '@'; +} + +static void OPVTTryForegroundNoArgObjectHookSpec(OPVTForegroundNoArgObjectHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"foreground hook missing class=%s phase=%@", spec->className, phase ?: @""); + } + return; + } + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"foreground hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + unsigned int argumentCount = method_getNumberOfArguments(method); + if (argumentCount != 2 || !OPVTMethodReturnTypeStartsWith(method, '@')) { + OPVTLog(@"foreground hook skipped method=%s[%s] args=%u phase=%@", + spec->className, spec->selectorName, argumentCount, phase ?: @""); + return; + } + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTForegroundNoArgObjectReplacement) { + spec->hooked = YES; + return; + } + spec->original = method_setImplementation(method, (IMP)OPVTForegroundNoArgObjectReplacement); + spec->hooked = YES; + OPVTLog(@"foreground hook installed method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); +} + +static void OPVTTryForegroundObjectArgVoidHookSpec(OPVTForegroundObjectArgVoidHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"foreground hook missing class=%s phase=%@", spec->className, phase ?: @""); + } + return; + } + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"foreground hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + unsigned int argumentCount = method_getNumberOfArguments(method); + if (argumentCount != 3 || !OPVTMethodReturnTypeStartsWith(method, 'v') || + !OPVTMethodArgumentTypeLooksObject(method, 2)) { + OPVTLog(@"foreground hook skipped method=%s[%s] args=%u phase=%@", + spec->className, spec->selectorName, argumentCount, phase ?: @""); + return; + } + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTForegroundObjectArgVoidReplacement) { + spec->hooked = YES; + return; + } + spec->original = method_setImplementation(method, (IMP)OPVTForegroundObjectArgVoidReplacement); + spec->hooked = YES; + OPVTLog(@"foreground hook installed method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); +} + +static void OPVTTryForegroundObjectArgObjectHookSpec(OPVTForegroundObjectArgObjectHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"foreground hook missing class=%s phase=%@", spec->className, phase ?: @""); + } + return; + } + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"foreground hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + unsigned int argumentCount = method_getNumberOfArguments(method); + if (argumentCount != 3 || !OPVTMethodReturnTypeStartsWith(method, '@') || + !OPVTMethodArgumentTypeLooksObject(method, 2)) { + OPVTLog(@"foreground hook skipped method=%s[%s] args=%u phase=%@", + spec->className, spec->selectorName, argumentCount, phase ?: @""); + return; + } + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTForegroundObjectArgObjectReplacement) { + spec->hooked = YES; + return; + } + spec->original = method_setImplementation(method, (IMP)OPVTForegroundObjectArgObjectReplacement); + spec->hooked = YES; + OPVTLog(@"foreground hook installed method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); +} + +static void OPVTTryForegroundNoArgBoolHookSpec(OPVTForegroundNoArgBoolHookSpec *spec, NSString *phase) { + if (!spec || spec->hooked) { + return; + } + Class cls = objc_getClass(spec->className); + if (!cls) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"foreground hook missing class=%s phase=%@", spec->className, phase ?: @""); + } + return; + } + SEL selector = sel_registerName(spec->selectorName); + Method method = class_getInstanceMethod(cls, selector); + if (!method) { + if (OPVTShouldLogHookMiss(phase)) { + OPVTLog(@"foreground hook missing method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); + } + return; + } + unsigned int argumentCount = method_getNumberOfArguments(method); + if (argumentCount != 2 || !OPVTMethodReturnTypeLooksBool(method)) { + OPVTLog(@"foreground hook skipped method=%s[%s] args=%u phase=%@", + spec->className, spec->selectorName, argumentCount, phase ?: @""); + return; + } + IMP current = method_getImplementation(method); + if (current == (IMP)OPVTForegroundNoArgBoolReplacement) { + spec->hooked = YES; + return; + } + spec->original = method_setImplementation(method, (IMP)OPVTForegroundNoArgBoolReplacement); + spec->hooked = YES; + OPVTLog(@"foreground hook installed method=%s[%s] phase=%@", + spec->className, spec->selectorName, phase ?: @""); +} + +static void OPVTTryForegroundRuntimeHooks(NSString *phase) { + size_t count = sizeof(OPVTForegroundNoArgObjectHookSpecs) / sizeof(OPVTForegroundNoArgObjectHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryForegroundNoArgObjectHookSpec(&OPVTForegroundNoArgObjectHookSpecs[index], phase); + } + count = sizeof(OPVTForegroundObjectArgVoidHookSpecs) / sizeof(OPVTForegroundObjectArgVoidHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryForegroundObjectArgVoidHookSpec(&OPVTForegroundObjectArgVoidHookSpecs[index], phase); + } + count = sizeof(OPVTForegroundObjectArgObjectHookSpecs) / sizeof(OPVTForegroundObjectArgObjectHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryForegroundObjectArgObjectHookSpec(&OPVTForegroundObjectArgObjectHookSpecs[index], phase); + } + count = sizeof(OPVTForegroundNoArgBoolHookSpecs) / sizeof(OPVTForegroundNoArgBoolHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryForegroundNoArgBoolHookSpec(&OPVTForegroundNoArgBoolHookSpecs[index], phase); + } +} + +static BOOL OPVTHaveAnyRuntimeHook(void) { + size_t count = sizeof(OPVTHookSpecs) / sizeof(OPVTHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + if (OPVTHookSpecs[index].hooked) { + return YES; + } + } + count = sizeof(OPVTFixedArgHookSpecs) / sizeof(OPVTFixedArgHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + if (OPVTFixedArgHookSpecs[index].hooked) { + return YES; + } + } + count = sizeof(OPVTBoolArgHookSpecs) / sizeof(OPVTBoolArgHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + if (OPVTBoolArgHookSpecs[index].hooked) { + return YES; + } + } + count = sizeof(OPVTBoolPairHookSpecs) / sizeof(OPVTBoolPairHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + if (OPVTBoolPairHookSpecs[index].hooked) { + return YES; + } + } + count = sizeof(OPVTButtonObjectVoidHookSpecs) / sizeof(OPVTButtonObjectVoidHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + if (OPVTButtonObjectVoidHookSpecs[index].hooked) { + return YES; + } + } + count = sizeof(OPVTButtonObjectBoolHookSpecs) / sizeof(OPVTButtonObjectBoolHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + if (OPVTButtonObjectBoolHookSpecs[index].hooked) { + return YES; + } + } + count = sizeof(OPVTButtonRawHIDVoidHookSpecs) / sizeof(OPVTButtonRawHIDVoidHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + if (OPVTButtonRawHIDVoidHookSpecs[index].hooked) { + return YES; + } + } + return NO; +} + +static BOOL OPVTShouldLogHookMiss(NSString *phase) { + if (![phase hasPrefix:@"delayed-"]) { + return YES; + } + return [phase isEqualToString:@"delayed-1"] || + [phase isEqualToString:@"delayed-5"] || + [phase isEqualToString:@"delayed-20"]; +} + +static void OPVTTryRuntimeHooks(NSString *phase) { + size_t count = sizeof(OPVTHookSpecs) / sizeof(OPVTHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryHookSpec(&OPVTHookSpecs[index], phase); + } + count = sizeof(OPVTFixedArgHookSpecs) / sizeof(OPVTFixedArgHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryHookFixedArgSpec(&OPVTFixedArgHookSpecs[index], phase); + } + count = sizeof(OPVTBoolArgHookSpecs) / sizeof(OPVTBoolArgHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryHookBoolArgSpec(&OPVTBoolArgHookSpecs[index], phase); + } + count = sizeof(OPVTBoolPairHookSpecs) / sizeof(OPVTBoolPairHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryHookBoolPairSpec(&OPVTBoolPairHookSpecs[index], phase); + } + count = sizeof(OPVTButtonObjectVoidHookSpecs) / sizeof(OPVTButtonObjectVoidHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryButtonObjectVoidHookSpec(&OPVTButtonObjectVoidHookSpecs[index], phase); + } + count = sizeof(OPVTButtonObjectBoolHookSpecs) / sizeof(OPVTButtonObjectBoolHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryButtonObjectBoolHookSpec(&OPVTButtonObjectBoolHookSpecs[index], phase); + } + count = sizeof(OPVTButtonRawHIDVoidHookSpecs) / sizeof(OPVTButtonRawHIDVoidHookSpecs[0]); + for (size_t index = 0; index < count; index++) { + OPVTTryButtonRawHIDVoidHookSpec(&OPVTButtonRawHIDVoidHookSpecs[index], phase); + } + OPVTTryForegroundRuntimeHooks(phase); + OPVTPublishTriggerStatus(@"hook_scan", @{@"phase": phase ?: @""}); +} + +static BOOL OPVTNameLooksVolumeRelated(const char *name) { + return name && (strstr(name, "Volume") || + strstr(name, "volume") || + strstr(name, "Button") || + strstr(name, "button")); +} + +static BOOL OPVTNameLooksForegroundClassRelated(const char *name) { + if (!name) { + return NO; + } + return strstr(name, "SpringBoard") || + strstr(name, "SBMainWorkspace") || + strstr(name, "SBWorkspace") || + strstr(name, "SBApplication") || + strstr(name, "SBApp") || + strstr(name, "SBScene") || + strstr(name, "SBDisplay") || + strstr(name, "SBMainDisplay") || + strstr(name, "SBMainSwitcher") || + strstr(name, "SBHomeScreen") || + strstr(name, "SBIconController") || + strstr(name, "FBScene") || + strstr(name, "FBWorkspace") || + strstr(name, "FBSScene"); +} + +static BOOL OPVTNameLooksForegroundMethodRelated(const char *name) { + if (!name) { + return NO; + } + return strstr(name, "foreground") || + strstr(name, "Foreground") || + strstr(name, "frontmost") || + strstr(name, "Frontmost") || + strstr(name, "frontMost") || + strstr(name, "active") || + strstr(name, "Active") || + strstr(name, "activate") || + strstr(name, "Activate") || + strstr(name, "deactivate") || + strstr(name, "Deactivate") || + strstr(name, "application") || + strstr(name, "Application") || + strstr(name, "bundle") || + strstr(name, "Bundle") || + strstr(name, "scene") || + strstr(name, "Scene") || + strstr(name, "display") || + strstr(name, "Display") || + strstr(name, "layout") || + strstr(name, "Layout") || + strstr(name, "switcher") || + strstr(name, "Switcher") || + strstr(name, "launch") || + strstr(name, "Launch"); +} + +static void OPVTLogVolumeRuntimeSnapshot(NSString *phase) { + int classCount = objc_getClassList(NULL, 0); + if (classCount <= 0) { + OPVTLog(@"runtime snapshot empty phase=%@", phase ?: @""); + return; + } + + Class *classes = (Class *)calloc((size_t)classCount, sizeof(Class)); + if (!classes) { + OPVTLog(@"runtime snapshot allocation failed count=%d phase=%@", classCount, phase ?: @""); + return; + } + + classCount = objc_getClassList(classes, classCount); + int logged = 0; + const int maxLines = 180; + for (int index = 0; index < classCount && logged < maxLines; index++) { + Class cls = classes[index]; + const char *className = class_getName(cls); + BOOL classMatches = OPVTNameLooksVolumeRelated(className); + unsigned int methodCount = 0; + Method *methods = class_copyMethodList(cls, &methodCount); + for (unsigned int methodIndex = 0; methods && methodIndex < methodCount && logged < maxLines; methodIndex++) { + SEL selector = method_getName(methods[methodIndex]); + const char *selectorName = sel_getName(selector); + if (!classMatches && !OPVTNameLooksVolumeRelated(selectorName)) { + continue; + } + if (!strstr(selectorName ?: "", "Volume") && + !strstr(selectorName ?: "", "volume") && + !strstr(selectorName ?: "", "increase") && + !strstr(selectorName ?: "", "decrease") && + !strstr(selectorName ?: "", "button") && + !strstr(selectorName ?: "", "Button")) { + continue; + } + char returnType[64] = {0}; + char argumentType[64] = {0}; + method_getReturnType(methods[methodIndex], returnType, sizeof(returnType)); + if (method_getNumberOfArguments(methods[methodIndex]) > 2) { + method_getArgumentType(methods[methodIndex], 2, argumentType, sizeof(argumentType)); + } + OPVTLog(@"runtime volume candidate phase=%@ class=%s method=%s args=%u return=%s arg2=%s", + phase ?: @"", className ?: "(null)", selectorName ?: "(null)", + method_getNumberOfArguments(methods[methodIndex]), + returnType[0] ? returnType : "", + argumentType[0] ? argumentType : ""); + logged++; + } + if (methods) { + free(methods); + } + } + free(classes); + OPVTLog(@"runtime snapshot complete phase=%@ lines=%d", phase ?: @"", logged); +} + +static void OPVTLogForegroundRuntimeSnapshot(NSString *phase) { + int classCount = objc_getClassList(NULL, 0); + if (classCount <= 0) { + OPVTLog(@"foreground runtime snapshot empty phase=%@", phase ?: @""); + return; + } + + Class *classes = (Class *)calloc((size_t)classCount, sizeof(Class)); + if (!classes) { + OPVTLog(@"foreground runtime snapshot allocation failed count=%d phase=%@", classCount, phase ?: @""); + return; + } + + classCount = objc_getClassList(classes, classCount); + int logged = 0; + const int maxLines = 260; + for (int pass = 0; pass < 2 && logged < maxLines; pass++) { + BOOL requireClassMatch = pass == 0; + for (int index = 0; index < classCount && logged < maxLines; index++) { + Class cls = classes[index]; + const char *className = class_getName(cls); + BOOL classMatches = OPVTNameLooksForegroundClassRelated(className); + if (requireClassMatch && !classMatches) { + continue; + } + if (!requireClassMatch && classMatches) { + continue; + } + unsigned int methodCount = 0; + Method *methods = class_copyMethodList(cls, &methodCount); + for (unsigned int methodIndex = 0; methods && methodIndex < methodCount && logged < maxLines; methodIndex++) { + Method method = methods[methodIndex]; + SEL selector = method_getName(method); + const char *selectorName = sel_getName(selector); + if (!OPVTNameLooksForegroundMethodRelated(selectorName)) { + continue; + } + char returnType[64] = {0}; + char argumentType[64] = {0}; + method_getReturnType(method, returnType, sizeof(returnType)); + if (method_getNumberOfArguments(method) > 2) { + method_getArgumentType(method, 2, argumentType, sizeof(argumentType)); + } + OPVTLog(@"foreground runtime candidate phase=%@ pass=%d class=%s method=%s args=%u return=%s arg2=%s", + phase ?: @"", pass + 1, className ?: "(null)", selectorName ?: "(null)", + method_getNumberOfArguments(method), + returnType[0] ? returnType : "", + argumentType[0] ? argumentType : ""); + logged++; + } + if (methods) { + free(methods); + } + } + } + free(classes); + OPVTLog(@"foreground runtime snapshot complete phase=%@ lines=%d", phase ?: @"", logged); +} + +static void *OPVTDelayedHookThread(void *unused) { + (void)unused; + @autoreleasepool { + for (int attempt = 1; attempt <= 20; attempt++) { + sleep(1); + NSString *phase = [NSString stringWithFormat:@"delayed-%d", attempt]; + OPVTTryRuntimeHooks(phase); + if (!OPVTHaveAnyRuntimeHook() && + (attempt == 1 || attempt == 5 || attempt == 20)) { + OPVTLogVolumeRuntimeSnapshot(phase); + } + if (OPVTBoolPreference(@"ForegroundRuntimeSnapshotEnabled", NO) && + (attempt == 5 || attempt == 20)) { + OPVTLogForegroundRuntimeSnapshot(phase); + } + } + } + return NULL; +} + +static void *OPVTSpringBoardStateThread(void *unused) { + (void)unused; + for (int attempt = 0; attempt < 5; attempt++) { + @autoreleasepool { + OPVTPublishSpringBoardStateOnMain(); + OPVTPublishTriggerStatus(@"heartbeat", @{@"phase": @"startup"}); + } + sleep(1); + } + while (true) { + @autoreleasepool { + OPVTPublishSpringBoardStateOnMain(); + OPVTPublishTriggerStatus(@"heartbeat", @{@"phase": @"steady"}); + } + sleep(2); + } + return NULL; +} + +static void *OPVTScreenshotRequestThread(void *unused) { + (void)unused; + __strong NSString *lastRequestId = nil; + while (true) { + @autoreleasepool { + NSDictionary *request = OPVTReadJSONDictionary(OPVTScreenshotRequestPath); + NSString *requestId = [request[@"request_id"] isKindOfClass:[NSString class]] + ? request[@"request_id"] : @""; + if (requestId.length > 0 && ![requestId isEqualToString:lastRequestId]) { + lastRequestId = [requestId copy]; + long long requestTimestamp = [request[@"timestamp_ms"] respondsToSelector:@selector(longLongValue)] + ? [request[@"timestamp_ms"] longLongValue] : 0; + long long requestAge = requestTimestamp > 0 ? MAX(0, OPVTNowMs() - requestTimestamp) : 0; + NSDictionary *existingResponse = OPVTReadJSONDictionary(OPVTScreenshotResponsePath); + NSString *existingResponseId = [existingResponse[@"request_id"] isKindOfClass:[NSString class]] + ? existingResponse[@"request_id"] : @""; + if ([existingResponseId isEqualToString:requestId]) { + continue; + } + if (requestTimestamp <= 0 || requestAge > 10000) { + OPVTLog(@"screenshot request ignored stale request=%@ age_ms=%lld", + requestId, requestAge); + continue; + } + __block NSDictionary *response = nil; + dispatch_sync(dispatch_get_main_queue(), ^{ + response = OPVTCaptureSpringBoardScreenshot(request); + }); + if (!OPVTWriteJSONDictionary(OPVTScreenshotResponsePath, response ?: @{})) { + OPVTLog(@"screenshot response write failed request=%@", requestId); + } else { + OPVTLog(@"screenshot response status=%@ request=%@ path=%@", + response[@"status"] ?: @"unknown", + requestId, + response[@"path"] ?: @""); + } + } + } + usleep(250000); + } + return NULL; +} + +static void *OPVTInputRequestThread(void *unused) { + (void)unused; + __strong NSString *lastRequestId = nil; + while (true) { + @autoreleasepool { + NSDictionary *request = OPVTReadJSONDictionary(OPVTInputRequestPath); + NSString *requestId = [request[@"request_id"] isKindOfClass:[NSString class]] + ? request[@"request_id"] : @""; + if (requestId.length > 0 && ![requestId isEqualToString:lastRequestId]) { + lastRequestId = [requestId copy]; + long long requestTimestamp = OPVTLongLongValue(request[@"timestamp_ms"], 0); + long long requestAge = requestTimestamp > 0 ? MAX(0, OPVTNowMs() - requestTimestamp) : 0; + NSDictionary *existingResponse = OPVTReadJSONDictionary(OPVTInputResponsePath); + NSString *existingResponseId = [existingResponse[@"request_id"] isKindOfClass:[NSString class]] + ? existingResponse[@"request_id"] : @""; + if ([existingResponseId isEqualToString:requestId]) { + continue; + } + if (requestTimestamp <= 0 || requestAge > 10000) { + OPVTLog(@"input request ignored stale request=%@ age_ms=%lld", + requestId, requestAge); + [[NSFileManager defaultManager] removeItemAtPath:OPVTInputRequestPath error:nil]; + continue; + } + __block NSDictionary *response = nil; + dispatch_sync(dispatch_get_main_queue(), ^{ + response = OPVTPerformSpringBoardInput(request); + }); + if (!OPVTWriteJSONDictionary(OPVTInputResponsePath, response ?: @{})) { + OPVTLog(@"input response write failed request=%@", requestId); + } else { + OPVTLog(@"input response status=%@ request=%@ action=%@ reason=%@", + response[@"status"] ?: @"unknown", + requestId, + response[@"action_type"] ?: @"", + response[@"reason"] ?: @""); + } + [[NSFileManager defaultManager] removeItemAtPath:OPVTInputRequestPath error:nil]; + } + } + usleep(250000); + } + return NULL; +} + +static void *OPVTClipboardRequestThread(void *unused) { + (void)unused; + __strong NSString *lastRequestId = nil; + while (true) { + @autoreleasepool { + NSDictionary *request = OPVTReadJSONDictionary(OPVTClipboardRequestPath); + NSString *requestId = [request[@"request_id"] isKindOfClass:[NSString class]] + ? request[@"request_id"] : @""; + if (requestId.length > 0 && ![requestId isEqualToString:lastRequestId]) { + lastRequestId = [requestId copy]; + long long requestTimestamp = OPVTLongLongValue(request[@"timestamp_ms"], 0); + long long requestAge = requestTimestamp > 0 ? MAX(0, OPVTNowMs() - requestTimestamp) : 0; + NSDictionary *existingResponse = OPVTReadJSONDictionary(OPVTClipboardResponsePath); + NSString *existingResponseId = [existingResponse[@"request_id"] isKindOfClass:[NSString class]] + ? existingResponse[@"request_id"] : @""; + if ([existingResponseId isEqualToString:requestId]) { + continue; + } + if (requestTimestamp <= 0 || requestAge > 10000) { + OPVTLog(@"clipboard request ignored stale request=%@ age_ms=%lld", + requestId, requestAge); + [[NSFileManager defaultManager] removeItemAtPath:OPVTClipboardRequestPath error:nil]; + continue; + } + __block NSDictionary *response = nil; + dispatch_sync(dispatch_get_main_queue(), ^{ + response = OPVTPerformClipboardRequest(request); + }); + if (!OPVTWriteJSONDictionary(OPVTClipboardResponsePath, response ?: @{})) { + OPVTLog(@"clipboard response write failed request=%@", requestId); + } else { + OPVTLog(@"clipboard response status=%@ request=%@ operation=%@ reason=%@", + response[@"status"] ?: @"unknown", + requestId, + response[@"operation"] ?: @"", + response[@"reason"] ?: @""); + } + [[NSFileManager defaultManager] removeItemAtPath:OPVTClipboardRequestPath error:nil]; + } + } + usleep(250000); + } + return NULL; +} + +static void *OPVTPromptRequestThread(void *unused) { + (void)unused; + __strong NSString *lastRequestId = nil; + while (true) { + @autoreleasepool { + NSDictionary *request = OPVTReadJSONDictionary(OPVTPromptRequestPath); + NSString *requestId = [request[@"request_id"] isKindOfClass:[NSString class]] + ? request[@"request_id"] : @""; + if (requestId.length > 0 && ![requestId isEqualToString:lastRequestId]) { + lastRequestId = [requestId copy]; + long long requestTimestamp = OPVTLongLongValue(request[@"timestamp_ms"], 0); + long long requestAge = requestTimestamp > 0 ? MAX(0, OPVTNowMs() - requestTimestamp) : 0; + NSDictionary *existingResponse = OPVTReadJSONDictionary(OPVTPromptResponsePath); + NSString *existingResponseId = [existingResponse[@"request_id"] isKindOfClass:[NSString class]] + ? existingResponse[@"request_id"] : @""; + if ([existingResponseId isEqualToString:requestId]) { + continue; + } + if (requestTimestamp <= 0 || requestAge > 10000) { + OPVTLog(@"prompt request ignored stale request=%@ age_ms=%lld", + requestId, requestAge); + [[NSFileManager defaultManager] removeItemAtPath:OPVTPromptRequestPath error:nil]; + continue; + } + __block NSDictionary *response = nil; + dispatch_sync(dispatch_get_main_queue(), ^{ + response = OPVTPerformPromptRequest(request); + }); + if (!OPVTWriteJSONDictionary(OPVTPromptResponsePath, response ?: @{})) { + OPVTLog(@"prompt response write failed request=%@", requestId); + } else { + OPVTLog(@"prompt response status=%@ request=%@ operation=%@ reason=%@", + response[@"status"] ?: @"unknown", + requestId, + response[@"operation"] ?: @"", + response[@"reason"] ?: @""); + } + [[NSFileManager defaultManager] removeItemAtPath:OPVTPromptRequestPath error:nil]; + } + } + usleep(250000); + } + return NULL; +} + +@implementation OPVTVolumeNotificationObserver + +- (void)openphoneSystemVolumeDidChange:(NSNotification *)notification { + OPVTHandleSystemVolumeNotification(notification); +} + +@end + +%hook SBVolumeControl + +- (void)increaseVolume { + OPVTRecordButtonFromSource(YES, @"logos.SBVolumeControl.increaseVolume"); + %orig; +} + +- (void)decreaseVolume { + OPVTRecordButtonFromSource(NO, @"logos.SBVolumeControl.decreaseVolume"); + %orig; +} + +%end + +%hook VolumeControl + +- (void)increaseVolume { + OPVTRecordButtonFromSource(YES, @"logos.VolumeControl.increaseVolume"); + %orig; +} + +- (void)decreaseVolume { + OPVTRecordButtonFromSource(NO, @"logos.VolumeControl.decreaseVolume"); + %orig; +} + +%end + +// --------------------------------------------------------------------------- +// Notification provider. Hook NCNotificationDispatcher (SpringBoard's incoming +// banner pipeline) and forward each request to the daemon's notification_ingest +// command over the local Unix socket. The daemon keeps a bounded, redacted log +// and fires any active `notification`-source watchers. KVC-based field access +// keeps this resilient to private-API shape changes across iOS versions. +// --------------------------------------------------------------------------- + +@interface NCNotificationRequest : NSObject +@end + +static NSString *OPVTNotifKVCString(id object, NSString *key) { + if (!object || key.length == 0) { + return @""; + } + @try { + id value = [object valueForKey:key]; + if ([value isKindOfClass:[NSString class]]) { + return value; + } + } @catch (__unused NSException *e) { + } + return @""; +} + +static void OPVTForwardNotification(id request) { + if (!request) { + return; + } + NSString *bundleId = OPVTNotifKVCString(request, @"sectionIdentifier"); + if (bundleId.length == 0) { + bundleId = OPVTNotifKVCString(request, @"bundleIdentifier"); + } + // Never forward our own island-driven signals or empty bundles. + if (bundleId.length == 0 || [bundleId hasPrefix:@"com.openphone"]) { + return; + } + id content = nil; + @try { + content = [request valueForKey:@"content"]; + } @catch (__unused NSException *e) { + } + NSString *title = OPVTNotifKVCString(content, @"title"); + NSString *subtitle = OPVTNotifKVCString(content, @"subtitle"); + NSString *body = OPVTNotifKVCString(content, @"message"); + if (body.length == 0) { + body = OPVTNotifKVCString(content, @"body"); + } + NSString *notifId = OPVTNotifKVCString(request, @"notificationIdentifier"); + NSString *threadId = OPVTNotifKVCString(request, @"threadIdentifier"); + if (title.length == 0 && body.length == 0 && subtitle.length == 0) { + return; + } + NSDictionary *payload = @{ + @"command": @"notification_ingest", + @"bundle_id": bundleId, + @"title": title ?: @"", + @"subtitle": subtitle ?: @"", + @"body": body ?: @"", + @"notification_id": notifId ?: @"", + @"thread_id": threadId ?: @"", + @"source": @"springboard_nc_dispatcher", + @"reason": @"incoming notification banner" + }; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + OPVTAgentRequest(payload); + }); +} + +%hook NCNotificationDispatcher + +- (void)postNotificationRequest:(id)request { + %orig; + @try { + OPVTForwardNotification(request); + } @catch (__unused NSException *e) { + } +} + +- (void)postNotificationRequest:(id)request forCoalescedNotifications:(id)coalesced { + %orig; + @try { + OPVTForwardNotification(request); + } @catch (__unused NSException *e) { + } +} + +%end + +// --------------------------------------------------------------------------- +// OpenPhone Island: persistent Dynamic-Island-style overlay driven by the +// daemon's live status file (island-status.json). Renders idle/listening/ +// transcribing/thinking/action/success/error states with a live subtitle, +// tool label, and transcript. Mirrors the Android voice-agent island. +// --------------------------------------------------------------------------- + +static NSString *const OPVTIslandStatusPath = + @"/var/mobile/Library/OpenPhone/springboard/island-status.json"; +static const char *const OPVTIslandNotification = "com.openphone.island.status"; + +@class OPVTIslandGestureBridge; +static void OPVTIslandCollapse(void); +static void OPVTIslandExpand(void); +static void OPVTIslandAnimateToLevel(NSInteger level); +static void OPVTIslandAttachGestures(void); +static void OPVTIslandStartTicker(void); + +// Custom passthrough window: only forward touches to hits within the pill so +// the rest of the screen (SpringBoard, apps) is not blocked by the overlay. +@interface OPVTPassthroughWindow : UIWindow +@end +@implementation OPVTPassthroughWindow +- (UIView *)hitTest:(CGPoint)point withEvent:(UIEvent *)event { + UIView *hit = [super hitTest:point withEvent:event]; + if (hit == self || hit == self.rootViewController.view) { + return nil; // Pass through + } + return hit; +} +@end + +static UIWindow *OPVTIslandWindow = nil; +static UIView *OPVTIslandPill = nil; +static UIView *OPVTIslandExpanded = nil; +static UILabel *OPVTIslandTitleLabel = nil; +static UILabel *OPVTIslandSubtitleLabel = nil; +static UILabel *OPVTIslandExpandedLine1 = nil; +static UILabel *OPVTIslandExpandedLine2 = nil; +static UIView *OPVTIslandStatusDot = nil; +static CAShapeLayer *OPVTIslandGlowLayer = nil; +static CAGradientLayer *OPVTIslandGradientGlowLayer = nil; +static CAShapeLayer *OPVTIslandGradientMask = nil; +static CADisplayLink *OPVTIslandTicker = nil; +static CGFloat OPVTIslandGradientShift = 0.0; +static NSInteger OPVTIslandThinkingTick = 0; +static UIButton *OPVTIslandApproveBtn = nil; +static UIButton *OPVTIslandDenyBtn = nil; +static UIButton *OPVTIslandCancelChip = nil; +static UIView *OPVTIslandDragCatcher = nil; +static UIScrollView *OPVTIslandChatScroll = nil; +static UIView *OPVTIslandChatStack = nil; +static UIView *OPVTIslandTabBar = nil; +static UIButton *OPVTIslandTabChat = nil; +static UIButton *OPVTIslandTabRuns = nil; +static UIButton *OPVTIslandTabWatchers = nil; +static NSString *OPVTIslandActiveTab = @"chat"; +// Expansion level: 0 = compact DI shape, 1 = medium panel, 2 = large full-screen panel. +static NSInteger OPVTIslandExpansionLevel = 0; +static NSArray *OPVTIslandChatTurns = nil; +static int OPVTIslandChatNotifyToken = 0; +static int OPVTIslandHideNotifyToken = 0; +static int OPVTIslandShowNotifyToken = 0; +static BOOL OPVTIslandCaptureObserverInstalled = NO; +static BOOL OPVTIslandChatObserverInstalled = NO; +static UIView *OPVTIslandUserBubble = nil; +static UILabel *OPVTIslandUserBubbleLabel = nil; +static UIView *OPVTIslandAssistantBubble = nil; +static UILabel *OPVTIslandAssistantBubbleLabel = nil; +static unsigned long long OPVTIslandLastSequence = 0; +static long long OPVTIslandLastAppliedMs = 0; +static NSDictionary *OPVTIslandCurrentState = nil; +static int OPVTIslandStatusNotifyToken = 0; +static BOOL OPVTIslandStatusNotifyRegistered = NO; + +static UIColor *OPVTIslandAccentColor(NSString *accent) { + NSString *key = [accent isKindOfClass:[NSString class]] ? accent.lowercaseString : @"cyan"; + if ([key isEqualToString:@"red"]) { + return [UIColor colorWithRed:1.0 green:0.42 blue:0.42 alpha:1.0]; + } + if ([key isEqualToString:@"green"]) { + return [UIColor colorWithRed:0.125 green:0.89 blue:0.416 alpha:1.0]; + } + if ([key isEqualToString:@"blue"]) { + return [UIColor colorWithRed:0.604 green:0.722 blue:1.0 alpha:1.0]; + } + if ([key isEqualToString:@"orange"] || [key isEqualToString:@"yellow"]) { + return [UIColor colorWithRed:1.0 green:0.82 blue:0.42 alpha:1.0]; + } + return [UIColor colorWithRed:0.447 green:0.878 blue:0.769 alpha:1.0]; // cyan +} + +static BOOL OPVTIslandDeviceHasDynamicIsland(void) { + // iPhone 14 Pro / 14 Pro Max, 15/16 Pro line: safeAreaTop ~59pt while + // status bar is 54pt on non-DI devices. Simple heuristic works reliably. + if (@available(iOS 13.0, *)) { + UIWindowScene *scene = OPVTActiveWindowScene(); + UIWindow *keyWindow = scene.windows.firstObject; + if (keyWindow) { + return keyWindow.safeAreaInsets.top >= 55.0; + } + } + return NO; +} + +// level: 0 = compact (DI shape), 1 = medium panel, 2 = large full-screen panel. +static CGRect OPVTIslandPillFrameForLevel(CGRect bounds, NSInteger level) { + CGFloat topBase = OPVTIslandDeviceHasDynamicIsland() ? 11.0 : 12.0; + if (!OPVTIslandDeviceHasDynamicIsland()) { + if (@available(iOS 13.0, *)) { + UIWindowScene *scene = OPVTActiveWindowScene(); + UIWindow *keyWindow = scene.windows.firstObject; + if (keyWindow && keyWindow.safeAreaInsets.top > 20.0) { + topBase = MAX(topBase, keyWindow.safeAreaInsets.top - 34.0); + } + } + } + if (level >= 2) { + // Large: full width minus small margin, ~75% of screen height. + CGFloat pillWidth = bounds.size.width - 16.0; + CGFloat pillHeight = bounds.size.height * 0.75; + CGFloat originX = (bounds.size.width - pillWidth) / 2.0; + return CGRectMake(originX, topBase, pillWidth, pillHeight); + } + if (level == 1) { + // Medium panel. + CGFloat pillWidth = OPVTIslandDeviceHasDynamicIsland() + ? MIN(bounds.size.width - 16.0, 380.0) + : MIN(bounds.size.width - 20.0, 360.0); + CGFloat pillHeight = OPVTIslandDeviceHasDynamicIsland() ? 210.0 : 200.0; + CGFloat originX = (bounds.size.width - pillWidth) / 2.0; + return CGRectMake(originX, topBase, pillWidth, pillHeight); + } + // Compact: match Dynamic Island exactly on DI devices, plain pill elsewhere. + if (OPVTIslandDeviceHasDynamicIsland()) { + CGFloat pillWidth = 122.0; + CGFloat pillHeight = 37.0; + CGFloat originX = (bounds.size.width - pillWidth) / 2.0; + return CGRectMake(originX, topBase, pillWidth, pillHeight); + } + CGFloat pillWidth = MIN(bounds.size.width - 20.0, 260.0); + CGFloat pillHeight = 44.0; + CGFloat originX = (bounds.size.width - pillWidth) / 2.0; + return CGRectMake(originX, topBase, pillWidth, pillHeight); +} + +// Legacy shim used elsewhere in the file — maps BOOL to level 0/1. +static CGRect OPVTIslandPillFrame(CGRect bounds, BOOL expanded) { + return OPVTIslandPillFrameForLevel(bounds, expanded ? MAX(1, OPVTIslandExpansionLevel) : 0); +} + +static void OPVTIslandEnsureWindow(void) { + if (OPVTIslandWindow) { + return; + } + dispatch_assert_queue(dispatch_get_main_queue()); + UIScreen *screen = OPVTSafeMainScreen(); + if (!screen) { + return; + } + CGRect bounds = screen.bounds; + if (@available(iOS 13.0, *)) { + UIWindowScene *scene = OPVTActiveWindowScene(); + if (scene) { + OPVTIslandWindow = [[OPVTPassthroughWindow alloc] initWithWindowScene:scene]; + OPVTIslandWindow.frame = bounds; + } else { + OPVTIslandWindow = [[OPVTPassthroughWindow alloc] initWithFrame:bounds]; + } + } else { + OPVTIslandWindow = [[OPVTPassthroughWindow alloc] initWithFrame:bounds]; + } + OPVTIslandWindow.windowLevel = UIWindowLevelAlert + 2500; + OPVTIslandWindow.userInteractionEnabled = YES; + OPVTIslandWindow.backgroundColor = [UIColor clearColor]; + // Always visible so the user can tap into chat history at any time. Idle + // opacity is set low in OPVTIslandApplyState. + OPVTIslandWindow.hidden = NO; + UIViewController *controller = [[UIViewController alloc] init]; + controller.view.backgroundColor = [UIColor clearColor]; + controller.view.userInteractionEnabled = YES; + OPVTIslandWindow.rootViewController = controller; + + UIView *root = controller.view; + + // Pill container. + OPVTIslandPill = [[UIView alloc] initWithFrame:OPVTIslandPillFrame(bounds, NO)]; + OPVTIslandPill.backgroundColor = [UIColor colorWithWhite:0.0 alpha:0.94]; + // Match the Dynamic Island's exact corner curvature — its height is ~37pt + // and it's a full pill (radius = height/2). We keep that ratio on expand + // too so the shape stays consistent. + OPVTIslandPill.layer.cornerRadius = 18.5; + OPVTIslandPill.layer.masksToBounds = NO; + OPVTIslandPill.userInteractionEnabled = YES; + + // Animated rainbow gradient glow — matches Android's PointerOverlayController + // GlowBorderView. Two layers: a soft outer shadow to bloom the light, and a + // masked gradient stroke that shifts horizontally over time. + OPVTIslandGlowLayer = [CAShapeLayer layer]; + OPVTIslandGlowLayer.fillColor = [UIColor clearColor].CGColor; + OPVTIslandGlowLayer.strokeColor = OPVTIslandAccentColor(@"cyan").CGColor; + OPVTIslandGlowLayer.lineWidth = 2.4; + OPVTIslandGlowLayer.shadowColor = OPVTIslandAccentColor(@"cyan").CGColor; + OPVTIslandGlowLayer.shadowRadius = 18.0; + OPVTIslandGlowLayer.shadowOpacity = 0.85; + OPVTIslandGlowLayer.shadowOffset = CGSizeZero; + [OPVTIslandPill.layer addSublayer:OPVTIslandGlowLayer]; + + OPVTIslandGradientGlowLayer = [CAGradientLayer layer]; + OPVTIslandGradientGlowLayer.colors = @[ + (id)[UIColor colorWithRed:0.365 green:0.863 blue:1.0 alpha:1.0].CGColor, + (id)[UIColor colorWithRed:0.580 green:0.424 blue:1.0 alpha:1.0].CGColor, + (id)[UIColor colorWithRed:1.0 green:0.337 blue:0.659 alpha:1.0].CGColor, + (id)[UIColor colorWithRed:1.0 green:0.800 blue:0.424 alpha:1.0].CGColor, + (id)[UIColor colorWithRed:0.365 green:0.863 blue:1.0 alpha:1.0].CGColor + ]; + OPVTIslandGradientGlowLayer.startPoint = CGPointMake(0.0, 0.5); + OPVTIslandGradientGlowLayer.endPoint = CGPointMake(1.0, 0.5); + OPVTIslandGradientGlowLayer.opacity = 0.9; + [OPVTIslandPill.layer addSublayer:OPVTIslandGradientGlowLayer]; + + OPVTIslandGradientMask = [CAShapeLayer layer]; + OPVTIslandGradientMask.fillColor = [UIColor clearColor].CGColor; + OPVTIslandGradientMask.strokeColor = [UIColor blackColor].CGColor; + OPVTIslandGradientMask.lineWidth = 2.4; + OPVTIslandGradientGlowLayer.mask = OPVTIslandGradientMask; + + // Left status dot (with pulse). Slightly larger + rounded so a pulse feels + // premium rather than a bug. + OPVTIslandStatusDot = [[UIView alloc] initWithFrame:CGRectMake(14, 14, 14, 14)]; + OPVTIslandStatusDot.backgroundColor = OPVTIslandAccentColor(@"cyan"); + OPVTIslandStatusDot.layer.cornerRadius = 7.0; + OPVTIslandStatusDot.layer.shadowColor = OPVTIslandAccentColor(@"cyan").CGColor; + OPVTIslandStatusDot.layer.shadowRadius = 6.0; + OPVTIslandStatusDot.layer.shadowOpacity = 0.9; + OPVTIslandStatusDot.layer.shadowOffset = CGSizeZero; + [OPVTIslandPill addSubview:OPVTIslandStatusDot]; + + OPVTIslandTitleLabel = [[UILabel alloc] initWithFrame:CGRectMake(34, 6, 200, 16)]; + OPVTIslandTitleLabel.text = @"OpenPhone"; + OPVTIslandTitleLabel.textColor = [UIColor colorWithRed:0.957 green:0.969 blue:0.973 alpha:1.0]; + OPVTIslandTitleLabel.font = [UIFont systemFontOfSize:11.0 weight:UIFontWeightSemibold]; + [OPVTIslandPill addSubview:OPVTIslandTitleLabel]; + + OPVTIslandSubtitleLabel = [[UILabel alloc] initWithFrame:CGRectMake(34, 20, 200, 16)]; + OPVTIslandSubtitleLabel.text = @"Idle"; + OPVTIslandSubtitleLabel.textColor = [UIColor colorWithRed:0.682 green:0.722 blue:0.749 alpha:1.0]; + OPVTIslandSubtitleLabel.font = [UIFont systemFontOfSize:13.0 weight:UIFontWeightMedium]; + OPVTIslandSubtitleLabel.adjustsFontSizeToFitWidth = YES; + OPVTIslandSubtitleLabel.minimumScaleFactor = 0.72; + [OPVTIslandPill addSubview:OPVTIslandSubtitleLabel]; + + // Small × cancel chip on the right, shown only during active states. + OPVTIslandCancelChip = [UIButton buttonWithType:UIButtonTypeCustom]; + OPVTIslandCancelChip.backgroundColor = [UIColor colorWithWhite:1.0 alpha:0.18]; + [OPVTIslandCancelChip setTitle:@"×" forState:UIControlStateNormal]; + [OPVTIslandCancelChip setTitleColor:[UIColor whiteColor] forState:UIControlStateNormal]; + OPVTIslandCancelChip.titleLabel.font = [UIFont systemFontOfSize:16.0 weight:UIFontWeightSemibold]; + OPVTIslandCancelChip.layer.cornerRadius = 11.0; + OPVTIslandCancelChip.hidden = YES; + // Target wired later in OPVTIslandAttachGestures once the bridge class + // is fully defined; before that, referencing @selector via a forward + // declaration is not allowed. + [OPVTIslandPill addSubview:OPVTIslandCancelChip]; + + [root addSubview:OPVTIslandPill]; + + // Invisible drag catcher below the pill so pan gestures work reliably + // even when the pill is only 37pt tall (Dynamic Island geometry). + OPVTIslandDragCatcher = [[UIView alloc] initWithFrame:CGRectZero]; + OPVTIslandDragCatcher.backgroundColor = [UIColor clearColor]; + OPVTIslandDragCatcher.userInteractionEnabled = YES; + [root addSubview:OPVTIslandDragCatcher]; + + OPVTIslandAttachGestures(); + OPVTIslandStartTicker(); +} + +@interface OPVTIslandGestureBridge : NSObject ++ (instancetype)shared; +- (void)handleTap:(UITapGestureRecognizer *)tap; +- (void)handleHold:(UILongPressGestureRecognizer *)hold; +- (void)handlePan:(UIPanGestureRecognizer *)pan; +- (void)tick:(CADisplayLink *)link; +- (void)thinkingDotsTick:(NSTimer *)timer; +- (void)cancelChipTapped:(id)sender; +@end + +@implementation OPVTIslandGestureBridge ++ (instancetype)shared { + static OPVTIslandGestureBridge *instance; + static dispatch_once_t once; + dispatch_once(&once, ^{ + instance = [[OPVTIslandGestureBridge alloc] init]; + }); + return instance; +} +- (void)handleTap:(UITapGestureRecognizer *)tap { + (void)tap; + // Tap toggles expansion. Never hides the pill entirely — it's always + // present so the user can tap into chat history at any time. Fresh + // volume trigger is the only way to start a new voice turn. + if (OPVTIslandExpanded && !OPVTIslandExpanded.hidden) { + OPVTIslandCollapse(); + } else { + OPVTIslandExpand(); + } +} +- (void)handleHold:(UILongPressGestureRecognizer *)hold { + if (hold.state != UIGestureRecognizerStateBegan) { + return; + } + OPVTLog(@"island long-press -> voice_cancel"); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + OPVTAgentRequest(@{@"command": @"voice_cancel", @"reason": @"user_long_press"}); + }); +} +- (void)cancelChipTapped:(id)sender { + (void)sender; + OPVTLog(@"island cancel chip tapped -> voice_cancel"); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + OPVTAgentRequest(@{@"command": @"voice_cancel", @"reason": @"user_cancel_chip"}); + }); +} +- (void)tabChat:(id)sender { (void)sender; OPVTIslandActiveTab = @"chat"; OPVTIslandApplyState(OPVTIslandCurrentState ?: @{}); } +- (void)tabRuns:(id)sender { (void)sender; OPVTIslandActiveTab = @"runs"; OPVTIslandApplyState(OPVTIslandCurrentState ?: @{}); } +- (void)tabWatchers:(id)sender { (void)sender; OPVTIslandActiveTab = @"watchers"; OPVTIslandApplyState(OPVTIslandCurrentState ?: @{}); } +- (void)approve:(id)sender { + (void)sender; + OPVTLog(@"island approve chip -> voice_confirm"); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + OPVTAgentRequest(@{@"command": @"voice_confirm", @"source": @"island_chip"}); + }); +} +- (void)deny:(id)sender { + (void)sender; + OPVTLog(@"island deny chip -> voice_deny"); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + OPVTAgentRequest(@{@"command": @"voice_deny", @"source": @"island_chip"}); + }); +} +- (void)handlePan:(UIPanGestureRecognizer *)pan { + if (!OPVTIslandPill) return; + // Skip pan events that start inside the chat scroll view — those should + // scroll history, not resize the pill. + CGPoint location = [pan locationInView:OPVTIslandPill]; + if (OPVTIslandChatScroll && !OPVTIslandChatScroll.hidden && + OPVTIslandExpanded && !OPVTIslandExpanded.hidden) { + CGRect scrollFrame = [OPVTIslandChatScroll.superview convertRect:OPVTIslandChatScroll.frame + toView:OPVTIslandPill]; + if (CGRectContainsPoint(scrollFrame, location)) { + pan.enabled = NO; pan.enabled = YES; + return; + } + } + CGPoint translation = [pan translationInView:OPVTIslandPill]; + if (pan.state == UIGestureRecognizerStateEnded || + pan.state == UIGestureRecognizerStateChanged) { + // Down drag: escalate level (0 → 1 → 2). Up drag: de-escalate. + if (translation.y > 24.0) { + NSInteger next = MIN(2, OPVTIslandExpansionLevel + 1); + if (next != OPVTIslandExpansionLevel) { + OPVTIslandAnimateToLevel(next); + [pan setTranslation:CGPointZero inView:OPVTIslandPill]; + } + } else if (translation.y < -24.0) { + NSInteger next = MAX(0, OPVTIslandExpansionLevel - 1); + if (next != OPVTIslandExpansionLevel) { + OPVTIslandAnimateToLevel(next); + [pan setTranslation:CGPointZero inView:OPVTIslandPill]; + } + } + } +} +- (void)tick:(CADisplayLink *)link { + (void)link; + if (!OPVTIslandWindow || OPVTIslandWindow.hidden) { + return; + } + // Only gradient shift runs at framerate. Everything else (pulse, dots) + // is driven by CABasicAnimation / repeating timers so we don't fight + // daemon-driven subtitle updates. + OPVTIslandGradientShift += 0.006; + if (OPVTIslandGradientShift > 1.0) OPVTIslandGradientShift -= 1.0; + if (OPVTIslandGradientGlowLayer) { + OPVTIslandGradientGlowLayer.startPoint = + CGPointMake(-1.0 + OPVTIslandGradientShift, 0.5); + OPVTIslandGradientGlowLayer.endPoint = + CGPointMake(2.0 + OPVTIslandGradientShift, 0.5); + } +} +- (void)thinkingDotsTick:(NSTimer *)timer { + (void)timer; + if (!OPVTIslandWindow || OPVTIslandWindow.hidden) return; + if (![OPVTIslandCurrentMode isEqualToString:@"thinking"] && + ![OPVTIslandCurrentMode isEqualToString:@"transcribing"]) return; + NSString *subtitleRaw = [OPVTIslandCurrentState[@"subtitle"] + isKindOfClass:[NSString class]] + ? OPVTIslandCurrentState[@"subtitle"] : @""; + NSString *lower = subtitleRaw.lowercaseString ?: @""; + BOOL generic = [lower isEqualToString:@"thinking"] || + [lower isEqualToString:@"transcribing"] || subtitleRaw.length == 0; + if (!generic) return; + OPVTIslandThinkingTick += 1; + NSString *base = [OPVTIslandCurrentMode isEqualToString:@"thinking"] + ? @"Thinking" : @"Transcribing"; + NSString *dots = @[@".", @"..", @"..."][OPVTIslandThinkingTick % 3]; + OPVTIslandSubtitleLabel.text = [base stringByAppendingString:dots]; +} +@end + +static void OPVTIslandAttachGestures(void) { + if (!OPVTIslandPill) { + return; + } + UITapGestureRecognizer *tap = [[UITapGestureRecognizer alloc] + initWithTarget:[OPVTIslandGestureBridge shared] + action:@selector(handleTap:)]; + tap.cancelsTouchesInView = YES; + [OPVTIslandPill addGestureRecognizer:tap]; + + UILongPressGestureRecognizer *hold = [[UILongPressGestureRecognizer alloc] + initWithTarget:[OPVTIslandGestureBridge shared] + action:@selector(handleHold:)]; + hold.minimumPressDuration = 0.55; + [OPVTIslandPill addGestureRecognizer:hold]; + + UIPanGestureRecognizer *pan = [[UIPanGestureRecognizer alloc] + initWithTarget:[OPVTIslandGestureBridge shared] + action:@selector(handlePan:)]; + [OPVTIslandPill addGestureRecognizer:pan]; + + if (OPVTIslandCancelChip) { + [OPVTIslandCancelChip addTarget:[OPVTIslandGestureBridge shared] + action:@selector(cancelChipTapped:) + forControlEvents:UIControlEventTouchUpInside]; + } + + // Pan gesture on the invisible drag catcher too so downward drags off + // the bottom edge of the tiny compact pill still expand. + if (OPVTIslandDragCatcher) { + UIPanGestureRecognizer *panCatch = [[UIPanGestureRecognizer alloc] + initWithTarget:[OPVTIslandGestureBridge shared] + action:@selector(handlePan:)]; + [OPVTIslandDragCatcher addGestureRecognizer:panCatch]; + + UITapGestureRecognizer *tapCatch = [[UITapGestureRecognizer alloc] + initWithTarget:[OPVTIslandGestureBridge shared] + action:@selector(handleTap:)]; + [OPVTIslandDragCatcher addGestureRecognizer:tapCatch]; + } +} + +static void OPVTIslandStartTicker(void) { + if (OPVTIslandTicker) return; + OPVTIslandTicker = [CADisplayLink + displayLinkWithTarget:[OPVTIslandGestureBridge shared] + selector:@selector(tick:)]; + [OPVTIslandTicker addToRunLoop:[NSRunLoop mainRunLoop] + forMode:NSRunLoopCommonModes]; + // 3Hz timer for "thinking..." dots — no need to run at 60fps for text. + [NSTimer scheduledTimerWithTimeInterval:0.42 + target:[OPVTIslandGestureBridge shared] + selector:@selector(thinkingDotsTick:) + userInfo:nil + repeats:YES]; +} + +static void OPVTIslandLayoutInterior(void) { + if (!OPVTIslandPill) return; + CGRect b = OPVTIslandPill.bounds; + BOOL compact = b.size.height < 55.0; + BOOL activeMode = [OPVTIslandCurrentMode isEqualToString:@"listening"] || + [OPVTIslandCurrentMode isEqualToString:@"realtime"] || + [OPVTIslandCurrentMode isEqualToString:@"transcribing"] || + [OPVTIslandCurrentMode isEqualToString:@"thinking"] || + [OPVTIslandCurrentMode isEqualToString:@"action"]; + // Show cancel chip only while active. Never during idle/terminal. + if (OPVTIslandCancelChip) OPVTIslandCancelChip.hidden = !activeMode; + CGFloat rightReserve = activeMode ? 30.0 : 0.0; + if (compact) { + OPVTIslandStatusDot.frame = CGRectMake(11, (b.size.height - 10) / 2.0, 10, 10); + OPVTIslandStatusDot.layer.cornerRadius = 5.0; + OPVTIslandTitleLabel.hidden = YES; + OPVTIslandSubtitleLabel.frame = CGRectMake(28, 0, b.size.width - 40 - rightReserve, b.size.height); + OPVTIslandSubtitleLabel.font = [UIFont systemFontOfSize:12.0 weight:UIFontWeightSemibold]; + OPVTIslandSubtitleLabel.textAlignment = NSTextAlignmentLeft; + if (OPVTIslandCancelChip) { + OPVTIslandCancelChip.frame = CGRectMake(b.size.width - 30, (b.size.height - 22) / 2.0, 22, 22); + } + } else { + OPVTIslandStatusDot.frame = CGRectMake(14, 14, 14, 14); + OPVTIslandStatusDot.layer.cornerRadius = 7.0; + OPVTIslandTitleLabel.hidden = NO; + OPVTIslandTitleLabel.frame = CGRectMake(34, 6, b.size.width - 48 - rightReserve, 16); + OPVTIslandSubtitleLabel.frame = CGRectMake(34, 20, b.size.width - 48 - rightReserve, 16); + OPVTIslandSubtitleLabel.font = [UIFont systemFontOfSize:13.0 weight:UIFontWeightMedium]; + OPVTIslandSubtitleLabel.textAlignment = NSTextAlignmentLeft; + if (OPVTIslandCancelChip) { + OPVTIslandCancelChip.frame = CGRectMake(b.size.width - 32, 10, 22, 22); + } + } +} + +static void OPVTIslandUpdateGlowPath(void) { + if (!OPVTIslandGlowLayer || !OPVTIslandPill) { + return; + } + CGRect bounds = OPVTIslandPill.bounds; + // Keep the rounded-pill feel: radius = 22 when expanded (matches iOS + // system smart-stack corner), 18.5 when compact (matches hardware DI). + CGFloat radius = bounds.size.height > 60 ? 22.0 : 18.5; + OPVTIslandPill.layer.cornerRadius = radius; + OPVTIslandLayoutInterior(); + UIBezierPath *path = [UIBezierPath bezierPathWithRoundedRect:bounds + cornerRadius:radius]; + OPVTIslandGlowLayer.path = path.CGPath; + OPVTIslandGlowLayer.frame = bounds; + if (OPVTIslandGradientGlowLayer) { + OPVTIslandGradientGlowLayer.frame = bounds; + } + if (OPVTIslandGradientMask) { + OPVTIslandGradientMask.path = path.CGPath; + OPVTIslandGradientMask.frame = bounds; + } +} + +static void OPVTIslandLayoutBubbles(CGFloat availableWidth); +static void OPVTIslandRenderChat(CGFloat availableWidth); +static void OPVTIslandRenderStubTab(NSString *label, CGFloat availableWidth); + +static UIButton *OPVTIslandMakeTabButton(NSString *title, SEL action) { + UIButton *b = [UIButton buttonWithType:UIButtonTypeSystem]; + [b setTitle:title forState:UIControlStateNormal]; + [b setTitleColor:[UIColor colorWithRed:0.957 green:0.969 blue:0.973 alpha:1.0] + forState:UIControlStateNormal]; + b.titleLabel.font = [UIFont systemFontOfSize:11.0 weight:UIFontWeightSemibold]; + b.backgroundColor = [UIColor clearColor]; + b.layer.cornerRadius = 10.0; + [b addTarget:[OPVTIslandGestureBridge shared] action:action + forControlEvents:UIControlEventTouchUpInside]; + return b; +} + +static void OPVTIslandEnsureExpandedViews(CGFloat availableWidth) { + if (OPVTIslandExpanded) { + // Recompute expanded frame so scroll view width tracks pill width. + CGFloat expandedH = OPVTIslandPill ? OPVTIslandPill.bounds.size.height - 50 : 78; + expandedH = MAX(expandedH, 40); + OPVTIslandExpanded.frame = CGRectMake(14, 46, availableWidth, expandedH); + if (OPVTIslandTabBar) OPVTIslandTabBar.frame = CGRectMake(0, 0, availableWidth, 22); + if (OPVTIslandChatScroll) { + OPVTIslandChatScroll.frame = CGRectMake(0, 26, availableWidth, expandedH - 26); + } + return; + } + OPVTIslandExpanded = [[UIView alloc] initWithFrame:CGRectMake(14, 46, availableWidth, 78)]; + OPVTIslandExpanded.backgroundColor = [UIColor clearColor]; + OPVTIslandExpanded.hidden = YES; + + // Tab bar at the top of the expanded panel. + OPVTIslandTabBar = [[UIView alloc] initWithFrame:CGRectMake(0, 0, availableWidth, 22)]; + OPVTIslandTabBar.backgroundColor = [UIColor clearColor]; + [OPVTIslandExpanded addSubview:OPVTIslandTabBar]; + + OPVTIslandTabChat = OPVTIslandMakeTabButton(@"Chat", @selector(tabChat:)); + OPVTIslandTabRuns = OPVTIslandMakeTabButton(@"Runs", @selector(tabRuns:)); + OPVTIslandTabWatchers = OPVTIslandMakeTabButton(@"Watchers", @selector(tabWatchers:)); + OPVTIslandTabChat.frame = CGRectMake(0, 0, 60, 22); + OPVTIslandTabRuns.frame = CGRectMake(64, 0, 60, 22); + OPVTIslandTabWatchers.frame = CGRectMake(128, 0, 80, 22); + [OPVTIslandTabBar addSubview:OPVTIslandTabChat]; + [OPVTIslandTabBar addSubview:OPVTIslandTabRuns]; + [OPVTIslandTabBar addSubview:OPVTIslandTabWatchers]; + + // Scrolling chat area below the tab bar. + OPVTIslandChatScroll = [[UIScrollView alloc] initWithFrame:CGRectMake(0, 26, availableWidth, 52)]; + OPVTIslandChatScroll.backgroundColor = [UIColor clearColor]; + OPVTIslandChatScroll.showsVerticalScrollIndicator = YES; + OPVTIslandChatScroll.alwaysBounceVertical = YES; + OPVTIslandChatScroll.clipsToBounds = YES; + [OPVTIslandExpanded addSubview:OPVTIslandChatScroll]; + + OPVTIslandChatStack = [[UIView alloc] initWithFrame:CGRectMake(0, 0, availableWidth, 0)]; + OPVTIslandChatStack.backgroundColor = [UIColor clearColor]; + [OPVTIslandChatScroll addSubview:OPVTIslandChatStack]; + + // Kept as legacy placeholder views so lingering references in ApplyState + // don't crash. They stay hidden — the scroll stack renders everything. + OPVTIslandUserBubble = [[UIView alloc] init]; OPVTIslandUserBubble.hidden = YES; + OPVTIslandUserBubbleLabel = [[UILabel alloc] init]; OPVTIslandUserBubbleLabel.hidden = YES; + OPVTIslandAssistantBubble = [[UIView alloc] init]; OPVTIslandAssistantBubble.hidden = YES; + OPVTIslandAssistantBubbleLabel = [[UILabel alloc] init]; OPVTIslandAssistantBubbleLabel.hidden = YES; + + // Approve / Deny chips for needs_review mode. + OPVTIslandApproveBtn = [UIButton buttonWithType:UIButtonTypeSystem]; + OPVTIslandApproveBtn.backgroundColor = [UIColor colorWithRed:0.125 green:0.89 blue:0.416 alpha:1.0]; + OPVTIslandApproveBtn.tintColor = [UIColor colorWithRed:0.063 green:0.078 blue:0.094 alpha:1.0]; + [OPVTIslandApproveBtn setTitle:@"✓ Approve" forState:UIControlStateNormal]; + [OPVTIslandApproveBtn setTitleColor:[UIColor colorWithRed:0.063 green:0.078 blue:0.094 alpha:1.0] + forState:UIControlStateNormal]; + OPVTIslandApproveBtn.titleLabel.font = [UIFont systemFontOfSize:12.0 weight:UIFontWeightSemibold]; + OPVTIslandApproveBtn.layer.cornerRadius = 12.0; + OPVTIslandApproveBtn.hidden = YES; + [OPVTIslandApproveBtn addTarget:[OPVTIslandGestureBridge shared] + action:@selector(approve:) + forControlEvents:UIControlEventTouchUpInside]; + [OPVTIslandExpanded addSubview:OPVTIslandApproveBtn]; + + OPVTIslandDenyBtn = [UIButton buttonWithType:UIButtonTypeSystem]; + OPVTIslandDenyBtn.backgroundColor = [UIColor colorWithWhite:1.0 alpha:0.2]; + [OPVTIslandDenyBtn setTitle:@"× Deny" forState:UIControlStateNormal]; + [OPVTIslandDenyBtn setTitleColor:[UIColor colorWithRed:0.957 green:0.969 blue:0.973 alpha:1.0] + forState:UIControlStateNormal]; + OPVTIslandDenyBtn.titleLabel.font = [UIFont systemFontOfSize:12.0 weight:UIFontWeightSemibold]; + OPVTIslandDenyBtn.layer.cornerRadius = 12.0; + OPVTIslandDenyBtn.hidden = YES; + [OPVTIslandDenyBtn addTarget:[OPVTIslandGestureBridge shared] + action:@selector(deny:) + forControlEvents:UIControlEventTouchUpInside]; + [OPVTIslandExpanded addSubview:OPVTIslandDenyBtn]; + + // Keep the old plain labels as invisible fallbacks so existing references + // in ApplyState don't crash. They stay hidden. + OPVTIslandExpandedLine1 = [[UILabel alloc] init]; + OPVTIslandExpandedLine1.hidden = YES; + [OPVTIslandExpanded addSubview:OPVTIslandExpandedLine1]; + OPVTIslandExpandedLine2 = [[UILabel alloc] init]; + OPVTIslandExpandedLine2.hidden = YES; + [OPVTIslandExpanded addSubview:OPVTIslandExpandedLine2]; + + [OPVTIslandPill addSubview:OPVTIslandExpanded]; + OPVTIslandLayoutBubbles(availableWidth); +} + +static void OPVTIslandLayoutBubbles(CGFloat availableWidth) { (void)availableWidth; /* legacy no-op */ } + +// Render Chat tab: rebuild scroll stack from chat-history.json plus the +// current in-flight turn (from island-status transcript/reply). +static void OPVTIslandRenderChat(CGFloat availableWidth) { + if (!OPVTIslandChatStack || !OPVTIslandChatScroll) return; + for (UIView *v in [OPVTIslandChatStack.subviews copy]) { + [v removeFromSuperview]; + } + CGFloat maxBubbleWidth = MIN(availableWidth - 40.0, availableWidth * 0.72); + UIFont *font = [UIFont systemFontOfSize:12.0 weight:UIFontWeightMedium]; + UIColor *userBg = [UIColor colorWithRed:0.122 green:0.368 blue:1.0 alpha:1.0]; + UIColor *assistantBg = [UIColor colorWithRed:0.125 green:0.153 blue:0.176 alpha:1.0]; + UIColor *lightText = [UIColor colorWithRed:0.957 green:0.969 blue:0.973 alpha:1.0]; + CGFloat y = 0; + + // Merge chat history with current in-flight turn. + NSMutableArray *turns = [NSMutableArray arrayWithArray:OPVTIslandChatTurns ?: @[]]; + NSString *liveTranscript = [OPVTIslandCurrentState[@"transcript"] isKindOfClass:[NSString class]] + ? OPVTIslandCurrentState[@"transcript"] : @""; + NSString *liveReply = [OPVTIslandCurrentState[@"reply"] isKindOfClass:[NSString class]] + ? OPVTIslandCurrentState[@"reply"] : @""; + NSString *liveTaskId = [OPVTIslandCurrentState[@"task_id"] isKindOfClass:[NSString class]] + ? OPVTIslandCurrentState[@"task_id"] : @""; + if (liveTranscript.length > 0) { + BOOL alreadyInHistory = NO; + for (NSDictionary *t in turns.reverseObjectEnumerator) { + if ([t[@"role"] isEqualToString:@"user"] && + [t[@"text"] isEqualToString:liveTranscript]) { + alreadyInHistory = YES; break; + } + } + if (!alreadyInHistory) { + [turns addObject:@{@"role": @"user", @"text": liveTranscript, @"task_id": liveTaskId ?: @""}]; + } + } + if (liveReply.length > 0) { + BOOL alreadyInHistory = NO; + for (NSDictionary *t in turns.reverseObjectEnumerator) { + if ([t[@"role"] isEqualToString:@"assistant"] && + [t[@"text"] isEqualToString:liveReply]) { + alreadyInHistory = YES; break; + } + } + if (!alreadyInHistory) { + [turns addObject:@{@"role": @"assistant", @"text": liveReply}]; + } + } + + for (NSDictionary *turn in turns) { + if (![turn isKindOfClass:[NSDictionary class]]) continue; + NSString *role = [turn[@"role"] isKindOfClass:[NSString class]] ? turn[@"role"] : @"user"; + NSString *text = [turn[@"text"] isKindOfClass:[NSString class]] ? turn[@"text"] : @""; + if (text.length == 0) continue; + BOOL isUser = [role isEqualToString:@"user"]; + CGSize sz = [text boundingRectWithSize:CGSizeMake(maxBubbleWidth - 20.0, 400.0) + options:NSStringDrawingUsesLineFragmentOrigin + attributes:@{NSFontAttributeName: font} + context:nil].size; + CGFloat w = ceil(sz.width) + 20.0; + CGFloat h = MAX(28.0, ceil(sz.height) + 12.0); + UIView *bubble = [[UIView alloc] initWithFrame:CGRectMake( + isUser ? (availableWidth - w) : 0, y, w, h)]; + bubble.backgroundColor = isUser ? userBg : assistantBg; + bubble.layer.cornerRadius = 12.0; + bubble.layer.masksToBounds = YES; + if (!isUser) { + bubble.layer.borderWidth = 1.0 / [UIScreen mainScreen].scale; + bubble.layer.borderColor = [UIColor colorWithWhite:1.0 alpha:0.15].CGColor; + } + UILabel *label = [[UILabel alloc] initWithFrame:CGRectMake(10, 6, w - 20, h - 12)]; + label.font = font; + label.textColor = isUser ? [UIColor whiteColor] : lightText; + label.numberOfLines = 0; + label.text = text; + [bubble addSubview:label]; + [OPVTIslandChatStack addSubview:bubble]; + y += h + 6; + } + OPVTIslandChatStack.frame = CGRectMake(0, 0, availableWidth, MAX(y, 0)); + OPVTIslandChatScroll.contentSize = CGSizeMake(availableWidth, y); + // Scroll to bottom to show newest turn. + if (y > OPVTIslandChatScroll.bounds.size.height) { + CGPoint bottom = CGPointMake(0, y - OPVTIslandChatScroll.bounds.size.height); + [OPVTIslandChatScroll setContentOffset:bottom animated:YES]; + } +} + +static void OPVTIslandRenderStubTab(NSString *label, CGFloat availableWidth) { + if (!OPVTIslandChatStack || !OPVTIslandChatScroll) return; + for (UIView *v in [OPVTIslandChatStack.subviews copy]) { + [v removeFromSuperview]; + } + UILabel *msg = [[UILabel alloc] initWithFrame:CGRectMake(0, 8, availableWidth, 24)]; + msg.text = label; + msg.textAlignment = NSTextAlignmentCenter; + msg.textColor = [UIColor colorWithRed:0.682 green:0.722 blue:0.749 alpha:1.0]; + msg.font = [UIFont systemFontOfSize:12.0 weight:UIFontWeightRegular]; + [OPVTIslandChatStack addSubview:msg]; + OPVTIslandChatStack.frame = CGRectMake(0, 0, availableWidth, 32); + OPVTIslandChatScroll.contentSize = CGSizeMake(availableWidth, 32); + [OPVTIslandChatScroll setContentOffset:CGPointZero animated:NO]; +} + +static void OPVTIslandRepositionDragCatcher(CGRect pillFrame, NSInteger level) { + if (!OPVTIslandDragCatcher) return; + // On compact, place a 60pt tall transparent catcher below the pill so + // the user can start a downward drag well past the tiny pill's edge. + // On expanded levels the pill has plenty of vertical area itself so we + // still keep a small catcher below for symmetry. + CGFloat catcherHeight = level == 0 ? 60.0 : 20.0; + CGFloat catcherWidth = MAX(pillFrame.size.width + 40.0, 200.0); + CGFloat catcherX = pillFrame.origin.x + (pillFrame.size.width - catcherWidth) / 2.0; + CGFloat catcherY = pillFrame.origin.y + pillFrame.size.height; + OPVTIslandDragCatcher.frame = CGRectMake(catcherX, catcherY, catcherWidth, catcherHeight); +} + +static void OPVTIslandAnimateToLevel(NSInteger level) { + if (!OPVTIslandWindow || !OPVTIslandPill) return; + NSInteger clamped = MAX(0, MIN(2, level)); + NSInteger prev = OPVTIslandExpansionLevel; + OPVTIslandExpansionLevel = clamped; + CGRect bounds = OPVTIslandWindow.bounds; + CGRect target = OPVTIslandPillFrameForLevel(bounds, clamped); + BOOL wasCollapsed = prev == 0; + BOOL nowCollapsed = clamped == 0; + + if (nowCollapsed) { + [UIView animateWithDuration:0.16 delay:0 + options:UIViewAnimationOptionCurveEaseIn + animations:^{ + if (OPVTIslandExpanded) OPVTIslandExpanded.alpha = 0.0; + } completion:^(BOOL f) { + (void)f; + if (OPVTIslandExpanded) OPVTIslandExpanded.hidden = YES; + [UIView animateWithDuration:0.34 delay:0 + usingSpringWithDamping:0.80 + initialSpringVelocity:0.4 + options:UIViewAnimationOptionCurveEaseOut + animations:^{ + OPVTIslandPill.frame = target; + OPVTIslandUpdateGlowPath(); + } completion:nil]; + }]; + return; + } + OPVTIslandEnsureExpandedViews(target.size.width - 28); + if (wasCollapsed) { + OPVTIslandExpanded.alpha = 0.0; + OPVTIslandExpanded.hidden = NO; + } + [UIView animateWithDuration:0.42 delay:0 + usingSpringWithDamping:0.75 + initialSpringVelocity:0.5 + options:UIViewAnimationOptionCurveEaseOut + animations:^{ + OPVTIslandPill.frame = target; + OPVTIslandUpdateGlowPath(); + OPVTIslandRepositionDragCatcher(target, clamped); + } completion:nil]; + [UIView animateWithDuration:0.24 delay:wasCollapsed ? 0.18 : 0.0 + options:UIViewAnimationOptionCurveEaseOut + animations:^{ + OPVTIslandExpanded.alpha = 1.0; + } completion:nil]; + // Re-render expanded contents with the new width. + OPVTIslandApplyState(OPVTIslandCurrentState ?: @{}); +} + +static void OPVTIslandExpand(void) { + dispatch_async(dispatch_get_main_queue(), ^{ + NSInteger next = OPVTIslandExpansionLevel == 0 ? 1 + : (OPVTIslandExpansionLevel == 1 ? 2 : 2); + OPVTIslandAnimateToLevel(next); + }); +} + +static void OPVTIslandCollapse(void) { + dispatch_async(dispatch_get_main_queue(), ^{ + NSInteger next = OPVTIslandExpansionLevel > 1 ? 1 : 0; + OPVTIslandAnimateToLevel(next); + }); +} + +static void OPVTIslandApplyState(NSDictionary *state) { + if (![state isKindOfClass:[NSDictionary class]]) { + return; + } + NSString *mode = [state[@"mode"] isKindOfClass:[NSString class]] + ? state[@"mode"] : @"idle"; + NSString *subtitle = [state[@"subtitle"] isKindOfClass:[NSString class]] + ? state[@"subtitle"] : @""; + NSString *transcript = [state[@"transcript"] isKindOfClass:[NSString class]] + ? state[@"transcript"] : @""; + NSString *tool = [state[@"tool"] isKindOfClass:[NSString class]] + ? state[@"tool"] : @""; + NSString *goal = [state[@"goal"] isKindOfClass:[NSString class]] + ? state[@"goal"] : @""; + NSString *accent = [state[@"accent"] isKindOfClass:[NSString class]] + ? state[@"accent"] : @"cyan"; + + long long step = [state[@"step"] isKindOfClass:[NSNumber class]] + ? [state[@"step"] longLongValue] : 0; + long long maxSteps = [state[@"max_steps"] isKindOfClass:[NSNumber class]] + ? [state[@"max_steps"] longLongValue] : 0; + // reply is rendered by OPVTIslandRenderChat via OPVTIslandCurrentState. + (void)state; + + dispatch_async(dispatch_get_main_queue(), ^{ + OPVTIslandEnsureWindow(); + if (!OPVTIslandWindow) { + return; + } + // Persistent overlay: always visible so user can tap into chat history. + // Idle/hidden modes just get low opacity — never fully hidden. + OPVTIslandWindow.hidden = NO; + BOOL isIdle = ![mode isKindOfClass:[NSString class]] || + mode.length == 0 || + [mode isEqualToString:@"hidden"] || + [mode isEqualToString:@"idle"]; + OPVTIslandPill.alpha = isIdle ? 0.55 : 1.0; + // Fire haptic on state transitions to terminal (Android-parity feel). + BOOL modeChanged = ![OPVTIslandCurrentMode isEqualToString:mode]; + if (modeChanged && [mode isEqualToString:@"success"]) { + OPVTPlayHapticSuccess(); + } else if (modeChanged && [mode isEqualToString:@"error"]) { + OPVTPlayHapticFailure(); + } else if (modeChanged && [mode isEqualToString:@"needs_review"]) { + OPVTPlayHapticFailure(); // Attention-grabbing double-tap. + } + // Toggle the CAAnimation-based dot pulse when entering/leaving + // listening. Way smoother than frame-by-frame transform math. + if (modeChanged && OPVTIslandStatusDot) { + [OPVTIslandStatusDot.layer removeAnimationForKey:@"pulse"]; + if ([mode isEqualToString:@"listening"]) { + CABasicAnimation *pulse = [CABasicAnimation animationWithKeyPath:@"transform.scale"]; + pulse.fromValue = @(1.0); + pulse.toValue = @(1.35); + pulse.duration = 0.6; + pulse.autoreverses = YES; + pulse.repeatCount = HUGE_VALF; + pulse.timingFunction = [CAMediaTimingFunction functionWithName:kCAMediaTimingFunctionEaseInEaseOut]; + [OPVTIslandStatusDot.layer addAnimation:pulse forKey:@"pulse"]; + } + } + OPVTIslandCurrentMode = mode ?: @"idle"; + UIColor *accentColor = OPVTIslandAccentColor(accent); + OPVTIslandStatusDot.backgroundColor = accentColor; + OPVTIslandGlowLayer.strokeColor = accentColor.CGColor; + OPVTIslandGlowLayer.shadowColor = accentColor.CGColor; + + NSString *title = @"OpenPhone"; + if ([mode isEqualToString:@"listening"]) title = @"Listening"; + else if ([mode isEqualToString:@"realtime"]) title = @"Listening"; + else if ([mode isEqualToString:@"transcribing"]) title = @"Transcribing"; + else if ([mode isEqualToString:@"thinking"]) title = @"Thinking"; + else if ([mode isEqualToString:@"action"]) title = @"Acting"; + else if ([mode isEqualToString:@"success"]) title = @"Done"; + else if ([mode isEqualToString:@"error"]) title = @"Error"; + + // Show step counter next to title once we're past capture. + if (step > 0) { + title = [NSString stringWithFormat:@"%@ · %lld/%lld", + title, step, maxSteps > 0 ? maxSteps : (long long)25]; + } + OPVTIslandTitleLabel.text = title; + OPVTIslandSubtitleLabel.text = subtitle.length > 0 ? subtitle : @""; + + // Auto-expand when we have any meaningful content to show. The pill + // stays expanded across states until it collapses on terminal. + NSString *primaryText = goal.length > 0 ? goal : transcript; + BOOL hasContent = primaryText.length > 0 || + tool.length > 0 || + [mode isEqualToString:@"success"] || + [mode isEqualToString:@"error"] || + [mode isEqualToString:@"needs_review"]; + BOOL isTerminal = [mode isEqualToString:@"success"] || + [mode isEqualToString:@"error"]; + BOOL shouldExpand = hasContent && !isTerminal; + if (shouldExpand) { + OPVTIslandPill.frame = OPVTIslandPillFrame(OPVTIslandWindow.bounds, YES); + OPVTIslandEnsureExpandedViews(OPVTIslandPill.bounds.size.width - 28); + OPVTIslandExpanded.hidden = NO; + } else if (isTerminal && primaryText.length > 0) { + // Keep expanded on terminal so user can read summary. + OPVTIslandPill.frame = OPVTIslandPillFrame(OPVTIslandWindow.bounds, YES); + OPVTIslandEnsureExpandedViews(OPVTIslandPill.bounds.size.width - 28); + OPVTIslandExpanded.hidden = NO; + } else { + OPVTIslandPill.frame = OPVTIslandPillFrame(OPVTIslandWindow.bounds, NO); + if (OPVTIslandExpanded) { + OPVTIslandExpanded.hidden = YES; + } + } + OPVTIslandUpdateGlowPath(); + + if (OPVTIslandExpanded && !OPVTIslandExpanded.hidden) { + CGFloat availW = OPVTIslandPill.bounds.size.width - 28; + // Highlight active tab. + UIColor *activeBg = [UIColor colorWithWhite:1.0 alpha:0.18]; + OPVTIslandTabChat.backgroundColor = [OPVTIslandActiveTab isEqualToString:@"chat"] ? activeBg : [UIColor clearColor]; + OPVTIslandTabRuns.backgroundColor = [OPVTIslandActiveTab isEqualToString:@"runs"] ? activeBg : [UIColor clearColor]; + OPVTIslandTabWatchers.backgroundColor = [OPVTIslandActiveTab isEqualToString:@"watchers"] ? activeBg : [UIColor clearColor]; + if ([OPVTIslandActiveTab isEqualToString:@"chat"]) { + OPVTIslandRenderChat(availW); + } else if ([OPVTIslandActiveTab isEqualToString:@"runs"]) { + OPVTIslandRenderStubTab(@"Runs — no active tasks", availW); + } else { + OPVTIslandRenderStubTab(@"Watchers — no active watchers", availW); + } + + BOOL showChips = [mode isEqualToString:@"needs_review"]; + if (OPVTIslandApproveBtn && OPVTIslandDenyBtn) { + OPVTIslandApproveBtn.hidden = !showChips; + OPVTIslandDenyBtn.hidden = !showChips; + if (showChips) { + CGFloat w = OPVTIslandExpanded.bounds.size.width; + CGFloat y = OPVTIslandExpanded.bounds.size.height - 34; + OPVTIslandApproveBtn.frame = CGRectMake(w - 108, y, 100, 30); + OPVTIslandDenyBtn.frame = CGRectMake(w - 216, y, 100, 30); + if (!OPVTIslandApproveBtn.superview) { + [OPVTIslandExpanded addSubview:OPVTIslandApproveBtn]; + [OPVTIslandExpanded addSubview:OPVTIslandDenyBtn]; + } + } + } + } + + [OPVTIslandWindow.rootViewController.view setNeedsLayout]; + OPVTIslandRepositionDragCatcher(OPVTIslandPill.frame, OPVTIslandExpansionLevel); + // Terminal states: keep the pill visible with the assistant message + // as the subtitle so the user can actually read the answer. Shrink + // the expanded panel back to compact after a short beat so it's + // out of the way, but never hide entirely. Tap or a fresh trigger + // is the only dismissal. + if (isTerminal) { + NSString *terminalMode = [mode copy]; + unsigned long long snapshotSeq = OPVTIslandLastSequence; + double delay = [mode isEqualToString:@"success"] ? 2.2 : 4.0; + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(delay * NSEC_PER_SEC)), + dispatch_get_main_queue(), ^{ + if ([OPVTIslandCurrentMode isEqualToString:terminalMode] && + OPVTIslandExpansionLevel > 0) { + OPVTIslandAnimateToLevel(0); + } + }); + // After 20s, fade the answer to idle so the persistent pill sits + // at low opacity — Android auto-collapses at 7s but iOS is more + // conservative because there's no gesture-away like Android has. + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(20.0 * NSEC_PER_SEC)), + dispatch_get_main_queue(), ^{ + if ([OPVTIslandCurrentMode isEqualToString:terminalMode] && + OPVTIslandLastSequence == snapshotSeq) { + OPVTIslandApplyState(@{ + @"mode": @"idle", + @"subtitle": @"OpenPhone", + @"accent": @"cyan" + }); + } + }); + } + }); +} + +static void OPVTIslandRefreshFromDisk(void) { + NSData *data = [NSData dataWithContentsOfFile:OPVTIslandStatusPath]; + if (data.length == 0) { + return; + } + id object = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + if (![object isKindOfClass:[NSDictionary class]]) { + return; + } + NSDictionary *state = object; + unsigned long long sequence = [state[@"sequence"] isKindOfClass:[NSNumber class]] + ? [state[@"sequence"] unsignedLongLongValue] : 0; + if (sequence != 0 && sequence == OPVTIslandLastSequence) { + return; + } + OPVTIslandLastSequence = sequence; + OPVTIslandCurrentState = state; + OPVTIslandLastAppliedMs = (long long)(CFAbsoluteTimeGetCurrent() * 1000.0); + OPVTIslandApplyState(state); +} + +static void OPVTIslandNotificationCallback(int token) { + (void)token; + OPVTIslandRefreshFromDisk(); +} + +static void OPVTIslandLoadChatHistory(void) { + NSData *data = [NSData dataWithContentsOfFile:@"/var/mobile/Library/OpenPhone/springboard/chat-history.json"]; + if (!data) { OPVTIslandChatTurns = @[]; return; } + id obj = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + if (![obj isKindOfClass:[NSDictionary class]]) { OPVTIslandChatTurns = @[]; return; } + NSArray *turns = obj[@"turns"]; + OPVTIslandChatTurns = [turns isKindOfClass:[NSArray class]] ? turns : @[]; +} + +// Daemon posts these notifications around every screen capture so our +// island overlay does not appear in the screenshot — otherwise the model +// sees its own pill covering the app UI and loops "trying" to finish. +static void OPVTStartIslandCaptureObserver(void) { + if (OPVTIslandCaptureObserverInstalled) return; + uint32_t rc1 = notify_register_dispatch("com.openphone.island.hide-for-capture", + &OPVTIslandHideNotifyToken, dispatch_get_main_queue(), ^(int t) { + (void)t; + if (OPVTIslandPill) OPVTIslandPill.hidden = YES; + if (OPVTIslandDragCatcher) OPVTIslandDragCatcher.hidden = YES; + }); + uint32_t rc2 = notify_register_dispatch("com.openphone.island.show-after-capture", + &OPVTIslandShowNotifyToken, dispatch_get_main_queue(), ^(int t) { + (void)t; + if (OPVTIslandPill) OPVTIslandPill.hidden = NO; + if (OPVTIslandDragCatcher) OPVTIslandDragCatcher.hidden = NO; + }); + if (rc1 == NOTIFY_STATUS_OK && rc2 == NOTIFY_STATUS_OK) { + OPVTIslandCaptureObserverInstalled = YES; + } +} + +static void OPVTStartIslandChatObserver(void) { + if (OPVTIslandChatObserverInstalled) return; + uint32_t rc = notify_register_dispatch("com.openphone.island.chat", + &OPVTIslandChatNotifyToken, dispatch_get_main_queue(), ^(int t) { + (void)t; + OPVTIslandLoadChatHistory(); + if (OPVTIslandExpanded && !OPVTIslandExpanded.hidden && + [OPVTIslandActiveTab isEqualToString:@"chat"]) { + OPVTIslandApplyState(OPVTIslandCurrentState ?: @{}); + } + }); + if (rc == NOTIFY_STATUS_OK) OPVTIslandChatObserverInstalled = YES; + OPVTIslandLoadChatHistory(); +} + +static void OPVTStartIslandStatusObserver(void) { + if (OPVTIslandStatusNotifyRegistered) { + return; + } + uint32_t rc = notify_register_dispatch(OPVTIslandNotification, + &OPVTIslandStatusNotifyToken, dispatch_get_main_queue(), ^(int token) { + OPVTIslandNotificationCallback(token); + }); + if (rc == NOTIFY_STATUS_OK) { + OPVTIslandStatusNotifyRegistered = YES; + OPVTLog(@"island status notification observer installed"); + } else { + OPVTLog(@"island status notification observer failed rc=%u", rc); + } + // Initial paint from any existing snapshot. If none, still paint an idle + // pill so the persistent overlay is visible from boot. + dispatch_async(dispatch_get_main_queue(), ^{ + OPVTIslandRefreshFromDisk(); + if (!OPVTIslandCurrentState || OPVTIslandCurrentState.count == 0) { + OPVTIslandApplyState(@{ + @"mode": @"idle", + @"subtitle": @"OpenPhone", + @"accent": @"cyan", + @"sequence": @(0) + }); + } + }); +} + +%ctor { + BOOL enabled = OPVTEnabled(); + OPVTLog(@"loaded local daemon socket=%s enabled=%d prompt_for_goal=%d", + OPVTSocketPath, enabled, OPVTPromptForGoalEnabled()); + OPVTPublishTriggerStatus(@"loaded", @{@"enabled": @(enabled)}); + if (!enabled) { + OPVTLog(@"disabled by preference; skipping SpringBoard hooks"); + return; + } + OPVTPublishSpringBoardStateOnMain(); + OPVTStartIslandStatusObserver(); + OPVTStartIslandChatObserver(); + OPVTStartIslandCaptureObserver(); + OPVTInstallVolumeNotificationObserver(); + OPVTTryRuntimeHooks(@"ctor"); + if (OPVTBoolPreference(@"RuntimeSnapshotEnabled", NO)) { + OPVTLogVolumeRuntimeSnapshot(@"ctor"); + } + pthread_t thread; + int rc = pthread_create(&thread, NULL, OPVTDelayedHookThread, NULL); + if (rc == 0) { + pthread_detach(thread); + } else { + OPVTLog(@"delayed hook thread failed rc=%d", rc); + } + pthread_t stateThread; + rc = pthread_create(&stateThread, NULL, OPVTSpringBoardStateThread, NULL); + if (rc == 0) { + pthread_detach(stateThread); + } else { + OPVTLog(@"springboard state thread failed rc=%d", rc); + } + pthread_t screenshotThread; + rc = pthread_create(&screenshotThread, NULL, OPVTScreenshotRequestThread, NULL); + if (rc == 0) { + pthread_detach(screenshotThread); + } else { + OPVTLog(@"screenshot request thread failed rc=%d", rc); + } + pthread_t inputThread; + rc = pthread_create(&inputThread, NULL, OPVTInputRequestThread, NULL); + if (rc == 0) { + pthread_detach(inputThread); + } else { + OPVTLog(@"input request thread failed rc=%d", rc); + } + pthread_t clipboardThread; + rc = pthread_create(&clipboardThread, NULL, OPVTClipboardRequestThread, NULL); + if (rc == 0) { + pthread_detach(clipboardThread); + } else { + OPVTLog(@"clipboard request thread failed rc=%d", rc); + } + pthread_t promptThread; + rc = pthread_create(&promptThread, NULL, OPVTPromptRequestThread, NULL); + if (rc == 0) { + pthread_detach(promptThread); + } else { + OPVTLog(@"prompt request thread failed rc=%d", rc); + } +} diff --git a/ios/contracts/README.md b/ios/contracts/README.md new file mode 100644 index 0000000..4925c53 --- /dev/null +++ b/ios/contracts/README.md @@ -0,0 +1,23 @@ +# OpenPhone Contracts + +This directory mirrors the Android OpenPhone schemas and runtime protocol files +that iOS must target. + +The files under `schemas/` and `protocol/` are copied from the Android +OpenPhone repo so iOS can keep command names, capabilities, screen context, +actions, audit events, trajectories, and jobs aligned while the implementation +is still private. + +Current iOS status: + +- `openphone-agentd` exposes the first local daemon boundary. +- The daemon defaults to `yolo`. +- The daemon currently serves health, task, screen, input, app, memory, context, + commitment, watcher-store, background-job-store, audit, and trajectory + commands. +- Real screenshot pixels, UI tree, notifications, messages, calls, calendar, + contacts, watcher scheduling, background job execution, and commitment + trigger execution still need implementation. + +When contracts change in Android, update this mirror deliberately and note any +iOS compatibility work in `IOS_PLAN.md`. diff --git a/ios/contracts/protocol/openphone-capabilities.json b/ios/contracts/protocol/openphone-capabilities.json new file mode 100644 index 0000000..312fad9 --- /dev/null +++ b/ios/contracts/protocol/openphone-capabilities.json @@ -0,0 +1,30 @@ +{ + "version": 1, + "capabilities": [ + { "id": "screen.read.visible", "risk": "medium" }, + { "id": "apps.read", "risk": "low" }, + { "id": "apps.launch", "risk": "medium" }, + { "id": "input.perform", "risk": "high" }, + { "id": "network.use", "risk": "medium" }, + { "id": "notifications.read", "risk": "medium" }, + { "id": "notifications.act", "risk": "high" }, + { "id": "contacts.read", "risk": "medium" }, + { "id": "calendar.read", "risk": "medium" }, + { "id": "calendar.write", "risk": "high" }, + { "id": "calendar.delete", "risk": "high" }, + { "id": "messages.read", "risk": "high" }, + { "id": "messages.draft", "risk": "high" }, + { "id": "messages.send", "risk": "critical" }, + { "id": "calls.read", "risk": "medium" }, + { "id": "calls.place", "risk": "critical" }, + { "id": "clipboard.read", "risk": "medium" }, + { "id": "clipboard.write", "risk": "high" }, + { "id": "share.content", "risk": "high" }, + { "id": "tasks.observe", "risk": "low" }, + { "id": "background.run", "risk": "high" }, + { "id": "memory.read", "risk": "medium" }, + { "id": "memory.write", "risk": "high" }, + { "id": "watchers.read", "risk": "medium" }, + { "id": "watchers.write", "risk": "high" } + ] +} diff --git a/ios/contracts/protocol/openphone-commands.json b/ios/contracts/protocol/openphone-commands.json new file mode 100644 index 0000000..cf7b51c --- /dev/null +++ b/ios/contracts/protocol/openphone-commands.json @@ -0,0 +1,2124 @@ +{ + "version": 1, + "commands": [ + { + "name": "openphone.device.status", + "aliases": [ + "device.status", + "device.info" + ], + "android_tool": "phone_context", + "description": "Gather a call context card from call log, contacts, messages, and calendar, including missed calls not yet returned.", + "category": "device", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional person, business, trip, or confirmation query." + }, + "number": { + "type": "string", + "description": "Optional phone number." + }, + "limit": { + "type": "integer", + "description": "Maximum number of rows per source." + }, + "reason": { + "type": "string", + "description": "Why phone context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "calls.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.apps.search", + "aliases": [ + "device.apps" + ], + "android_tool": "apps_search", + "description": "Search launchable installed Android apps by label, package, or activity.", + "category": "apps", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional app label, package, or activity query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of apps to return." + }, + "include_system": { + "type": "boolean", + "description": "Include system apps." + }, + "reason": { + "type": "string", + "description": "Why installed app context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "apps.read", + "confirmation": "none", + "default_exposure": "safe_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.screen.get", + "aliases": [ + "canvas.snapshot" + ], + "android_tool": "get_screen", + "description": "Read the current phone screen, optional screenshot artifact, foreground metadata, and recent app context.", + "category": "screen", + "input_schema": { + "type": "object", + "properties": { + "include_screenshot": { + "type": "boolean", + "description": "Request a phone-local screenshot artifact and return metadata/path, not image bytes." + }, + "include_activity": { + "type": "boolean", + "description": "Include foreground app/activity." + }, + "include_ui_tree": { + "type": "boolean", + "description": "Include accessibility UI tree." + }, + "reason": { + "type": "string", + "description": "Why this observation is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "screen.read.visible", + "confirmation": "none", + "default_exposure": "safe_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.screen.understand_local", + "aliases": [ + "openphone.local.screen_understanding" + ], + "android_tool": "local_screen_understanding", + "description": "Run bounded local screen understanding on the phone without returning screenshot bytes.", + "category": "screen", + "input_schema": { + "type": "object", + "properties": { + "max_visible_text": { + "type": "integer", + "description": "Maximum visible text entries to return, capped by the phone." + }, + "max_interactive_elements": { + "type": "integer", + "description": "Maximum interactive elements to return, capped by the phone." + }, + "timeout_ms": { + "type": "integer", + "description": "Requested compute timeout in milliseconds, capped by the phone." + }, + "reason": { + "type": "string", + "description": "Why local screen understanding is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "screen.read.visible", + "confirmation": "none", + "default_exposure": "safe_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.notifications.list", + "aliases": [ + "notifications.list" + ], + "android_tool": "notifications_list", + "description": "List recent indexed Android notifications.", + "category": "notifications", + "input_schema": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "description": "Maximum number of notifications to return." + }, + "reason": { + "type": "string", + "description": "Why notification context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "notifications.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.notifications.search", + "aliases": [ + "notifications.search" + ], + "android_tool": "notifications_search", + "description": "Search recent indexed Android notifications.", + "category": "notifications", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of notifications to return." + }, + "reason": { + "type": "string", + "description": "Why notification context is needed." + } + }, + "required": [ + "query", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "notifications.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.notifications.open", + "aliases": [ + "notifications.open" + ], + "android_tool": "notifications_open", + "description": "Open an active Android notification through its content intent.", + "category": "notifications", + "input_schema": { + "type": "object", + "properties": { + "notification_key": { + "type": "string", + "description": "Optional notification key." + }, + "package": { + "type": "string", + "description": "Optional source package." + }, + "query": { + "type": "string", + "description": "Optional title/text/package search query." + }, + "reason": { + "type": "string", + "description": "Why this notification should open." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "notifications.act", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.contacts.search", + "aliases": [ + "contacts.search" + ], + "android_tool": "contacts_search", + "description": "Search Android contacts by name, phone, email, or organization.", + "category": "contacts", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional name, phone, or email query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of contacts to return." + }, + "include_details": { + "type": "boolean", + "description": "Include phone and email details." + }, + "reason": { + "type": "string", + "description": "Why contact context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "contacts.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.calendar.search", + "aliases": [ + "calendar.events" + ], + "android_tool": "calendar_search", + "description": "Search Android calendar events in a bounded time window.", + "category": "calendar", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional search query." + }, + "start_at": { + "type": "integer", + "description": "Unix time in milliseconds, or 0." + }, + "end_at": { + "type": "integer", + "description": "Unix time in milliseconds, or 0." + }, + "limit": { + "type": "integer", + "description": "Maximum number of events to return." + }, + "reason": { + "type": "string", + "description": "Why calendar context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "calendar.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.calendar.add", + "aliases": [ + "calendar.add" + ], + "android_tool": "calendar_create_event", + "description": "Create an Android calendar event in the user's writable calendar.", + "category": "calendar", + "input_schema": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Event title." + }, + "description": { + "type": "string", + "description": "Optional event details." + }, + "location": { + "type": "string", + "description": "Optional location." + }, + "start_at": { + "type": "integer", + "description": "Unix time in milliseconds." + }, + "end_at": { + "type": "integer", + "description": "Unix time in milliseconds, or 0." + }, + "duration_minutes": { + "type": "integer", + "description": "Duration if end_at is omitted." + }, + "all_day": { + "type": "boolean", + "description": "Whether this is an all-day event." + }, + "calendar_id": { + "type": "integer", + "description": "Optional target calendar id." + }, + "reason": { + "type": "string", + "description": "Why this event should be created." + } + }, + "required": [ + "title", + "start_at", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "calendar.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.calendar.update", + "aliases": [ + "calendar.update" + ], + "android_tool": "calendar_update_event", + "description": "Update fields of an existing Android calendar event (title, time, location, description). Recurring events update the whole series.", + "category": "calendar", + "input_schema": { + "type": "object", + "properties": { + "event_id": { + "type": "integer", + "description": "Calendar event id from calendar_search." + }, + "title": { + "type": "string", + "description": "New event title." + }, + "description": { + "type": "string", + "description": "New event details." + }, + "location": { + "type": "string", + "description": "New location." + }, + "start_at": { + "type": "integer", + "description": "New start, Unix time in milliseconds." + }, + "end_at": { + "type": "integer", + "description": "New end, Unix time in milliseconds." + }, + "duration_minutes": { + "type": "integer", + "description": "New duration if end_at is omitted." + }, + "all_day": { + "type": "boolean", + "description": "Whether this is an all-day event." + }, + "reason": { + "type": "string", + "description": "Why this event should be changed." + } + }, + "required": [ + "event_id", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "calendar.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.calendar.delete", + "aliases": [ + "calendar.delete" + ], + "android_tool": "calendar_delete_event", + "description": "Delete an Android calendar event permanently. Recurring events delete the whole series.", + "category": "calendar", + "input_schema": { + "type": "object", + "properties": { + "event_id": { + "type": "integer", + "description": "Calendar event id from calendar_search." + }, + "reason": { + "type": "string", + "description": "Why this event should be deleted." + } + }, + "required": [ + "event_id", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "high", + "capability": "calendar.delete", + "confirmation": "always", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.messages.search", + "aliases": [ + "sms.search" + ], + "android_tool": "messages_search", + "description": "Search Android SMS message history by sender, body, or address.", + "category": "messages", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional sender, body, or address query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of messages to return." + }, + "thread_id": { + "type": "integer", + "description": "Optional SMS thread id, or 0." + }, + "reason": { + "type": "string", + "description": "Why message context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "messages.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.messages.draft", + "aliases": [ + "sms.draft" + ], + "android_tool": "messages_draft", + "description": "Prepare a structured SMS draft for review without sending it.", + "category": "messages", + "input_schema": { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "Phone number if known." + }, + "body": { + "type": "string", + "description": "Draft message body." + }, + "contact_query": { + "type": "string", + "description": "Contact name if recipient should be resolved." + }, + "reason": { + "type": "string", + "description": "Why this draft is needed." + } + }, + "required": [ + "body", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "messages.draft", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.messages.send", + "aliases": [ + "sms.send" + ], + "android_tool": "messages_send", + "description": "Send an SMS message to a phone number.", + "category": "messages", + "input_schema": { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "Phone number." + }, + "body": { + "type": "string", + "description": "Message body." + }, + "reason": { + "type": "string", + "description": "Why this SMS should be sent." + } + }, + "required": [ + "to", + "body", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "high", + "capability": "messages.send", + "confirmation": "always", + "default_exposure": "dangerous", + "mcp": false, + "remote_runtime": true + }, + { + "name": "openphone.calls.search", + "aliases": [ + "callLog.search" + ], + "android_tool": "calls_search", + "description": "Search Android call log entries by number, contact name (joined from contacts when the log has none), type, or date range.", + "category": "calls", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional number or cached contact name." + }, + "type": { + "type": "string", + "description": "Optional call type filter: missed, incoming, outgoing, voicemail, rejected, or blocked." + }, + "limit": { + "type": "integer", + "description": "Maximum number of calls to return." + }, + "since": { + "type": "integer", + "description": "Only calls at or after this Unix time in milliseconds, or 0." + }, + "until": { + "type": "integer", + "description": "Only calls at or before this Unix time in milliseconds, or 0." + }, + "reason": { + "type": "string", + "description": "Why call history context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "calls.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.calls.place", + "aliases": [ + "calls.place" + ], + "android_tool": "calls_place", + "description": "Place a phone call to a number.", + "category": "calls", + "input_schema": { + "type": "object", + "properties": { + "number": { + "type": "string", + "description": "Phone number if known." + }, + "contact_query": { + "type": "string", + "description": "Contact name if number should be resolved." + }, + "reason": { + "type": "string", + "description": "Why this call should be placed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "high", + "capability": "calls.place", + "confirmation": "always", + "default_exposure": "dangerous", + "mcp": false, + "remote_runtime": true + }, + { + "name": "openphone.context.search", + "android_tool": "context_search", + "description": "Search durable OpenPhone context events indexed on the phone, including assistant/device events that should provide continuity for future agent turns.", + "category": "context", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of context events to return." + }, + "reason": { + "type": "string", + "description": "Why continuity context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "memory.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.commitments.search", + "android_tool": "commitment_search", + "description": "List or search durable OpenPhone commitments: reminders, follow-ups, due tasks, promises, and open loops the phone should remember. Use query=\"\" to list recent commitments; use a specific query only for a specific topic.", + "category": "commitments", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Free-text search over commitment titles and descriptions. Use empty string for list everything." + }, + "limit": { + "type": "integer", + "description": "Maximum number of commitments to return." + }, + "reason": { + "type": "string", + "description": "Why this commitment context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "commitments.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.commitments.create", + "android_tool": "commitment_create", + "description": "Create a durable OpenPhone commitment for an open loop the phone should remember. Use watcher_create instead when the trigger depends on an external event such as a reply, call, notification, web/page change, or semantic condition.", + "category": "commitments", + "input_schema": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Short open-loop title." + }, + "description": { + "type": "string", + "description": "Commitment details." + }, + "trigger_type": { + "type": "string", + "description": "manual or time." + }, + "trigger_spec": { + "type": "object", + "description": "Structured trigger details." + }, + "due_at": { + "type": "integer", + "description": "Unix time in milliseconds, or 0." + }, + "expires_at": { + "type": "integer", + "description": "Unix time in milliseconds, or 0." + }, + "confidence": { + "type": "number", + "description": "Confidence from 0 to 1." + }, + "reason": { + "type": "string", + "description": "Why this commitment should exist." + } + }, + "required": [ + "title", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "commitments.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.commitments.update_status", + "android_tool": "commitment_update_status", + "description": "Update a durable OpenPhone commitment status.", + "category": "commitments", + "input_schema": { + "type": "object", + "properties": { + "commitment_id": { + "description": "Commitment id.", + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "status": { + "type": "string", + "description": "completed, dismissed, snoozed, or active." + }, + "reason": { + "type": "string", + "description": "Why the status should change." + } + }, + "required": [ + "commitment_id", + "status", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "commitments.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.memory.search", + "android_tool": "memory_search", + "description": "Search durable OpenPhone memories: user preferences, facts, standing instructions, and long-lived personal context. Use before answering questions that depend on remembered user preferences or identity.", + "category": "memory", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of memories to return." + }, + "reason": { + "type": "string", + "description": "Why this memory is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "memory.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.memory.save", + "android_tool": "memory_save", + "description": "Save explicit durable user memory: stable preferences, facts, or standing instructions that should affect future phone behavior. Do not use for one-off reminders or future conditions; use commitments or watchers for those.", + "category": "memory", + "input_schema": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "fact, preference, or standing_instruction." + }, + "subject": { + "type": "string", + "description": "Memory subject, usually user." + }, + "text": { + "type": "string", + "description": "Durable memory text." + }, + "confidence": { + "type": "number", + "description": "Confidence from 0 to 1." + }, + "reason": { + "type": "string", + "description": "Why this should become memory." + } + }, + "required": [ + "text", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "memory.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.memory.update", + "android_tool": "memory_update", + "description": "Update an existing durable memory when the user corrects, refines, or replaces remembered information.", + "category": "memory", + "input_schema": { + "type": "object", + "properties": { + "memory_id": { + "type": "string", + "description": "Memory identifier to update." + }, + "text": { + "type": "string", + "description": "Replacement memory text." + }, + "type": { + "type": "string", + "description": "Optional replacement type." + }, + "subject": { + "type": "string", + "description": "Optional replacement subject." + }, + "confidence": { + "type": "number", + "description": "Optional confidence from 0 to 1." + }, + "reason": { + "type": "string", + "description": "Why this memory should be updated." + } + }, + "required": [ + "memory_id", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "memory.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.memory.delete", + "android_tool": "memory_delete", + "description": "Delete an incorrect, obsolete, or user-requested durable memory.", + "category": "memory", + "input_schema": { + "type": "object", + "properties": { + "memory_id": { + "type": "string", + "description": "Memory identifier to delete." + }, + "reason": { + "type": "string", + "description": "Why this memory should be deleted." + } + }, + "required": [ + "memory_id", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "memory.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.memory.merge", + "android_tool": "memory_merge", + "description": "Merge two overlapping durable memories into one updated target memory and remove the source memory.", + "category": "memory", + "input_schema": { + "type": "object", + "properties": { + "target_memory_id": { + "type": "string", + "description": "Memory to keep and update." + }, + "source_memory_id": { + "type": "string", + "description": "Memory to merge into the target and delete." + }, + "text": { + "type": "string", + "description": "Optional merged memory text. If omitted, target and source text are concatenated." + }, + "reason": { + "type": "string", + "description": "Why these memories should be merged." + } + }, + "required": [ + "target_memory_id", + "source_memory_id", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "memory.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.watchers.list", + "android_tool": "watcher_list", + "description": "List active OpenPhone watchers: external-condition monitors that may alert later. Use when the user asks what OpenPhone is watching or what conditions are being monitored. Do not use for queued background agent jobs; use background_job_list for those.", + "category": "watchers", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional search query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of watchers to return." + }, + "reason": { + "type": "string", + "description": "Why watcher context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "watchers.read", + "confirmation": "none", + "default_exposure": "private_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.watchers.create", + "android_tool": "watcher_create", + "description": "Create a durable OpenPhone watcher: a future trigger plus an optional reaction. Use for proactive monitoring of message replies/no-reply deadlines, calls/no-call deadlines, notification matches, web/page changes, semantic page conditions, and requests like tell me when, remind me if, watch this, let me know, or make sure I do not forget. If the user asks the phone to do something when the trigger happens, store that reaction in delivery using delivery.tool plus delivery.arguments, or delivery.prompt for a background agent job. Tool arguments may reference trigger data with templates such as {{event.number}}, {{event.address}}, {{event.title}}, {{event.text}}, {{event.url}}, {{watcher.id}}, and {{watcher.title}}. For requests to monitor incoming SMS/messages and check, research, classify, summarize, or otherwise reason about each message, create a message watcher with match_any=true and recurring=true, then use delivery.prompt so the fired watcher starts a background agent job; the prompt should include {{event.text}} and may tell the agent to use browser_search or browser_fetch_page for internet context before notifying the user. Do not use watcher_create for explicit run-as-soon-as-possible queued agent work; use background_job_create for that.", + "category": "watchers", + "input_schema": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "Generic source such as web, notification, message, call, time, or screen." + }, + "evaluator": { + "type": "string", + "description": "hash_change, text_contains, or semantic_match." + }, + "query": { + "type": "string", + "description": "Condition text for text or semantic evaluators." + }, + "url": { + "type": "string", + "description": "Web URL when source is web." + }, + "type": { + "type": "string", + "description": "Compatibility type: time, notification, web_change, message_reply, or call_back." + }, + "address": { + "type": "string", + "description": "Phone number or sender to watch when source is message." + }, + "number": { + "type": "string", + "description": "Phone number to watch when source is call. Omit and set match_any=true for every caller." + }, + "match_any": { + "type": "boolean", + "description": "For call watchers, true means match any phone number. Use for requests like every incoming call or any caller." + }, + "direction": { + "type": "string", + "description": "For call watchers: any (default), incoming, or outgoing." + }, + "thread_id": { + "type": "integer", + "description": "SMS thread id to watch when source is message." + }, + "deadline_at": { + "type": "integer", + "description": "Unix milliseconds deadline for message or call watchers; with notify_on no_reply or no_call, fires if nothing arrived by then." + }, + "notify_on": { + "type": "string", + "description": "For message watchers: reply (default) or no_reply. For call watchers: call (default; alert when a call with the number appears) or no_call (alert only if the deadline passes without one)." + }, + "title": { + "type": "string", + "description": "Short watcher title." + }, + "condition": { + "type": "object", + "description": "Structured condition." + }, + "schedule": { + "type": "object", + "description": "Schedule, including next_run_at or interval_ms." + }, + "delivery": { + "type": "object", + "description": "Delivery or reaction details. Use {\"mode\":\"notification\"} for alerts, {\"tool\":\"model_tool_name\",\"arguments\":{...}} for a stored tool reaction, or {\"mode\":\"background_job\",\"prompt\":\"...\"} for an observe-first background agent turn. For SMS research/classification/evaluation watchers, use delivery.prompt with {{event.text}} and browser_search/browser_fetch_page instructions rather than hardcoding the check in the watcher. Set notify=false or mode=silent to avoid a notification after the reaction." + }, + "tool": { + "type": "string", + "description": "Shortcut for delivery.tool: the model-facing tool to run when the watcher fires." + }, + "arguments": { + "type": "object", + "description": "Shortcut for delivery.arguments. Values may use event templates such as {{event.number}}." + }, + "prompt": { + "type": "string", + "description": "Shortcut for delivery.prompt when the watcher should enqueue a background agent job after the trigger." + }, + "recurring": { + "type": "boolean", + "description": "True when the watcher should continue after matching, such as every matching call or notification." + }, + "sms_body": { + "type": "string", + "description": "Legacy shortcut only; prefer delivery.tool with delivery.arguments for new watcher reactions." + }, + "next_run_at": { + "type": "integer", + "description": "Unix time in milliseconds, or 0." + }, + "interval_ms": { + "type": "integer", + "description": "Polling interval in milliseconds if relevant." + }, + "reason": { + "type": "string", + "description": "Why this watcher should exist." + } + }, + "required": [ + "title", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "watchers.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.watchers.stop", + "android_tool": "watcher_stop", + "description": "Stop active OpenPhone watchers. Use watcher_id when the user names a specific watcher id. Use scope=all or all=true when the user asks to turn off, disable, clear, remove, delete, cancel, stop, or kill all watchers. Use query when the user describes a watcher by title, source, or condition instead of giving an id.", + "category": "watchers", + "input_schema": { + "type": "object", + "properties": { + "watcher_id": { + "type": "integer", + "description": "Specific watcher id. Omit when stopping all watchers or a query-matched set." + }, + "scope": { + "type": "string", + "description": "Use all to stop every active watcher." + }, + "all": { + "type": "boolean", + "description": "True to stop every active watcher." + }, + "query": { + "type": "string", + "description": "Optional search text to stop matching active watchers when the user described the watcher but did not provide an id." + }, + "reason": { + "type": "string", + "description": "Why this watcher should stop." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "watchers.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.jobs.list", + "android_tool": "background_job_list", + "description": "List durable OpenPhone background agent jobs, including active, running, completed, failed, and stopped jobs. Use when the user asks what OpenPhone is working on in the background or what background jobs are active.", + "category": "background", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional search query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of jobs to return." + }, + "reason": { + "type": "string", + "description": "Why background job context is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "tasks.observe", + "confirmation": "none", + "default_exposure": "safe_read", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.jobs.create", + "android_tool": "background_job_create", + "description": "Create a durable OpenPhone background agent job: a queued agent turn that can run after the current conversation. Use for explicit requests like create a background job, do this in the background, run this as soon as possible, keep working and notify me, periodically summarize, check later by running an agent task, or run a non-urgent retrieval task. Do not use watcher_create for these queued agent jobs. Background jobs may observe and summarize; state-changing tools require foreground review.", + "category": "background", + "input_schema": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Short user-visible job title." + }, + "prompt": { + "type": "string", + "description": "The agent turn to run when the job wakes." + }, + "type": { + "type": "string", + "description": "agent_turn, system_event, or heartbeat. Default agent_turn." + }, + "schedule": { + "type": "object", + "description": "Schedule fields such as next_run_at or interval_ms." + }, + "run_at": { + "type": "integer", + "description": "Unix milliseconds when the job should first run. Omit or 0 to run soon." + }, + "next_run_at": { + "type": "integer", + "description": "Unix milliseconds when the job should next run." + }, + "interval_ms": { + "type": "integer", + "description": "Optional recurrence interval in milliseconds." + }, + "session_target": { + "type": "string", + "description": "main, isolated, current, or session:." + }, + "delivery": { + "type": "object", + "description": "Delivery mode. Use {\"mode\":\"notification\"}, {\"mode\":\"silent\"}, or {\"mode\":\"none\"}. If the user gives exact notification wording, include it as notification_text, text, or message." + }, + "notification_text": { + "type": "string", + "description": "Optional exact notification body to show when the background job completes. Use this when the user says to notify them with an exact phrase." + }, + "payload": { + "type": "object", + "description": "Optional structured payload for future runtime integrations." + }, + "reason": { + "type": "string", + "description": "Why this background job should exist." + } + }, + "required": [ + "title", + "prompt", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "background.run", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.jobs.stop", + "android_tool": "background_job_stop", + "description": "Stop a durable OpenPhone background agent job.", + "category": "background", + "input_schema": { + "type": "object", + "properties": { + "job_id": { + "type": "integer", + "description": "Background job id." + }, + "reason": { + "type": "string", + "description": "Why this background job should stop." + } + }, + "required": [ + "job_id", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "background.run", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.app.open", + "android_tool": "open_app", + "description": "Open an installed Android app. This tool completes only simple goals whose whole intent is to open, launch, or show the app. If the user wants something done inside the app, such as playing media, searching, selecting content, changing settings, ordering, booking, posting, sending, or any other in-app workflow, use the autonomous multi-step agent: open the app as one step, then keep observing and acting until the requested result is visibly complete or blocked.", + "category": "apps", + "input_schema": { + "type": "object", + "properties": { + "package_or_label": { + "type": "string", + "description": "Package name or app label." + }, + "label": { + "type": "string", + "description": "Optional explicit app label when package_or_label is a package name." + }, + "reason": { + "type": "string", + "description": "Why this app is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "apps.launch", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.url.open", + "android_tool": "open_url", + "description": "Open a URL directly in the browser.", + "category": "apps", + "input_schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "URL to open." + }, + "reason": { + "type": "string", + "description": "Why this URL is needed." + } + }, + "required": [ + "url", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "network.use", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.ui.tap", + "android_tool": "tap", + "description": "Tap absolute screen coordinates.", + "category": "ui", + "input_schema": { + "type": "object", + "properties": { + "x": { + "type": "number", + "description": "X coordinate." + }, + "y": { + "type": "number", + "description": "Y coordinate." + }, + "reason": { + "type": "string", + "description": "Why this coordinate should be tapped." + } + }, + "required": [ + "x", + "y", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "input.perform", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.ui.tap_element", + "android_tool": "tap_element", + "description": "Tap an enabled interactive element by screen element id.", + "category": "ui", + "input_schema": { + "type": "object", + "properties": { + "element_id": { + "type": "string", + "description": "Element id from interactive_elements." + }, + "reason": { + "type": "string", + "description": "Why this element should be tapped." + } + }, + "required": [ + "element_id", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "input.perform", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.ui.long_press", + "android_tool": "long_press", + "description": "Long press absolute screen coordinates.", + "category": "ui", + "input_schema": { + "type": "object", + "properties": { + "x": { + "type": "number", + "description": "X coordinate." + }, + "y": { + "type": "number", + "description": "Y coordinate." + }, + "duration_ms": { + "type": "integer", + "description": "Press duration." + }, + "reason": { + "type": "string", + "description": "Why this coordinate should be long-pressed." + } + }, + "required": [ + "x", + "y", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "input.perform", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.ui.long_press_element", + "android_tool": "long_press_element", + "description": "Long press an enabled interactive element by screen element id.", + "category": "ui", + "input_schema": { + "type": "object", + "properties": { + "element_id": { + "type": "string", + "description": "Element id from interactive_elements." + }, + "duration_ms": { + "type": "integer", + "description": "Press duration." + }, + "reason": { + "type": "string", + "description": "Why this element should be long-pressed." + } + }, + "required": [ + "element_id", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "input.perform", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.ui.swipe", + "android_tool": "swipe", + "description": "Swipe from one coordinate to another.", + "category": "ui", + "input_schema": { + "type": "object", + "properties": { + "start_x": { + "type": "number", + "description": "Start X." + }, + "start_y": { + "type": "number", + "description": "Start Y." + }, + "end_x": { + "type": "number", + "description": "End X." + }, + "end_y": { + "type": "number", + "description": "End Y." + }, + "reason": { + "type": "string", + "description": "Why this swipe is needed." + } + }, + "required": [ + "start_x", + "start_y", + "end_x", + "end_y", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "input.perform", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.ui.type_text", + "android_tool": "type_text", + "description": "Type text into the focused field.", + "category": "ui", + "input_schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "Text to type." + }, + "reason": { + "type": "string", + "description": "Why this text is needed." + } + }, + "required": [ + "text", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "input.perform", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.input.press_key", + "android_tool": "press_key", + "description": "Press a supported Android navigation or IME key.", + "category": "input", + "input_schema": { + "type": "object", + "properties": { + "key": { + "type": "string", + "description": "back, home, recents, enter, search, or go." + }, + "reason": { + "type": "string", + "description": "Why this key is needed." + } + }, + "required": [ + "key", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "input.perform", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.clipboard.set", + "android_tool": "set_clipboard", + "description": "Set clipboard text.", + "category": "clipboard", + "input_schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "Clipboard text." + }, + "reason": { + "type": "string", + "description": "Why clipboard is needed." + } + }, + "required": [ + "text", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "clipboard.write", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.clipboard.paste", + "android_tool": "paste", + "description": "Paste clipboard into the focused field.", + "category": "clipboard", + "input_schema": { + "type": "object", + "properties": { + "reason": { + "type": "string", + "description": "Why paste is needed." + } + }, + "required": [ + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "clipboard.read", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.share.text", + "android_tool": "share_text", + "description": "Share text through Android's mediated share flow.", + "category": "share", + "input_schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "Text to share." + }, + "chooser_title": { + "type": "string", + "description": "Optional chooser title." + }, + "reason": { + "type": "string", + "description": "Why sharing is needed." + } + }, + "required": [ + "text", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "high", + "capability": "share.content", + "confirmation": "always", + "default_exposure": "dangerous", + "mcp": true, + "remote_runtime": true + }, + { + "name": "openphone.model.status", + "android_tool": "model_status", + "description": "Return phone-local model-loop configuration status without exposing provider credential values.", + "category": "model", + "input_schema": { + "type": "object", + "properties": { + "reason": { + "type": "string", + "description": "Why model-loop status is needed." + } + } + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "tasks.observe", + "confirmation": "none", + "default_exposure": "safe_read", + "mcp": false, + "remote_runtime": false + }, + { + "name": "openphone.model.configure", + "android_tool": "model_configure", + "description": "Configure phone-local model-loop metadata such as mode, endpoint URL, model name, and limits. Provider credential values are external-only and rejected inline.", + "category": "model", + "input_schema": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether model mode may be selected by run_task auto mode." + }, + "mode": { + "type": "string", + "description": "broker or direct_dev. Broker is the default." + }, + "endpoint_url": { + "type": "string", + "description": "Broker or direct development endpoint URL." + }, + "model": { + "type": "string", + "description": "Model identifier to request from the endpoint." + }, + "timeout_ms": { + "type": "integer", + "description": "Per-request timeout in milliseconds." + }, + "max_steps": { + "type": "integer", + "description": "Default model-loop step limit." + }, + "max_duration_ms": { + "type": "integer", + "description": "Default model-loop duration limit." + }, + "credential_required": { + "type": "boolean", + "description": "Whether model mode requires an external credential file or environment credential before status becomes ready. Defaults to true; use false only for trusted local broker fixtures." + }, + "reason": { + "type": "string", + "description": "Why model-loop configuration is changing." + } + }, + "required": [ + "mode", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "tasks.observe", + "confirmation": "ask_before_action", + "default_exposure": "debug_only", + "mcp": false, + "remote_runtime": false + }, + { + "name": "openphone.task.finish", + "android_tool": "finish_task", + "description": "Mark the current OpenPhone task finished with a concise summary. Intended for the daemon-owned model loop.", + "category": "tasks", + "input_schema": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "description": "Task id to finish." + }, + "summary": { + "type": "string", + "description": "Concise completion summary." + } + }, + "required": [ + "task_id", + "summary" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "tasks.observe", + "confirmation": "none", + "default_exposure": "model_tool", + "mcp": false, + "remote_runtime": false + }, + { + "name": "openphone.task.fail", + "android_tool": "fail_task", + "description": "Mark the current OpenPhone task failed with a concise reason. Intended for the daemon-owned model loop.", + "category": "tasks", + "input_schema": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "description": "Task id to fail." + }, + "reason": { + "type": "string", + "description": "Concise failure reason." + } + }, + "required": [ + "task_id", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "low", + "capability": "tasks.observe", + "confirmation": "none", + "default_exposure": "model_tool", + "mcp": false, + "remote_runtime": false + }, + { + "name": "openphone.task.run", + "android_tool": "run_task", + "description": "Start a phone-owned OpenPhone task loop. Supports deterministic, model, and auto modes; model mode executes one parsed openphone.model_decision.v1 decision per step through registered daemon tools, repairs prose/fenced-code wrapped decision JSON, and cooperatively honors stop/cancel state on adopted task ids.", + "category": "tasks", + "input_schema": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "description": "Optional existing task id to adopt. The daemon checks this task for stop/cancel state before and between model steps." + }, + "goal": { + "type": "string", + "description": "Task goal." + }, + "mode": { + "type": "string", + "description": "deterministic, model, or auto." + }, + "max_steps": { + "type": "integer", + "description": "Maximum loop steps." + }, + "max_duration_ms": { + "type": "integer", + "description": "Maximum duration in milliseconds." + }, + "reason": { + "type": "string", + "description": "Why this task should run." + } + }, + "required": [ + "goal", + "reason" + ] + }, + "output_schema": { + "type": "object", + "additionalProperties": true + }, + "risk": "medium", + "capability": "tasks.observe", + "confirmation": "ask_before_action", + "default_exposure": "debug_only", + "mcp": false, + "remote_runtime": false + } + ] +} diff --git a/ios/contracts/protocol/openphone-events.json b/ios/contracts/protocol/openphone-events.json new file mode 100644 index 0000000..ad8c7ab --- /dev/null +++ b/ios/contracts/protocol/openphone-events.json @@ -0,0 +1,30 @@ +{ + "version": 1, + "events": [ + { "name": "runtime.presence.online", "direction": "phone_to_runtime" }, + { "name": "runtime.presence.offline", "direction": "phone_to_runtime" }, + { "name": "runtime.attention.requested", "direction": "phone_to_runtime" }, + { "name": "runtime.attention.accepted", "direction": "runtime_to_phone" }, + { "name": "runtime.attention.failed", "direction": "runtime_to_phone" }, + { "name": "runtime.message.delta", "direction": "runtime_to_phone" }, + { "name": "runtime.message.final", "direction": "runtime_to_phone" }, + { "name": "runtime.tool.requested", "direction": "runtime_to_phone" }, + { "name": "runtime.tool.result", "direction": "phone_to_runtime" }, + { "name": "runtime.confirmation.required", "direction": "phone_to_runtime" }, + { "name": "runtime.confirmation.resolved", "direction": "phone_to_runtime" }, + { "name": "runtime.session.updated", "direction": "bidirectional" } + ], + "transport_mappings": [ + { + "runtime": "openclaw", + "transport": "node.event", + "events": [ + { "protocol": "runtime.presence.online", "wire": "openphone.presence.online" }, + { "protocol": "runtime.presence.offline", "wire": "openphone.presence.offline" }, + { "protocol": "runtime.confirmation.required", "wire": "openphone.confirmation.required" }, + { "protocol": "runtime.confirmation.resolved", "wire": "openphone.confirmation.resolved" }, + { "protocol": "runtime.presence.online", "wire": "node.presence.alive" } + ] + } + ] +} diff --git a/ios/contracts/protocol/openphone-ios-tools.json b/ios/contracts/protocol/openphone-ios-tools.json new file mode 100644 index 0000000..c224153 --- /dev/null +++ b/ios/contracts/protocol/openphone-ios-tools.json @@ -0,0 +1,368 @@ +{ + "version": 1, + "platform": "ios", + "runtime_authority": "openphone-agentd", + "default_autonomy_mode": "yolo", + "tools": [ + { + "tool": "get_screen", + "command": "openphone.screen.get", + "capability": "screen.read.visible", + "status": "metadata_only", + "provider": "openphone-agentd.screen_metadata" + }, + { + "tool": "list_tasks", + "command": "openphone.task.list", + "capability": "tasks.observe", + "status": "implemented", + "provider": "openphone-agentd" + }, + { + "tool": "get_task", + "command": "openphone.task.get", + "capability": "tasks.observe", + "status": "implemented", + "provider": "openphone-agentd" + }, + { + "tool": "get_audit", + "command": "openphone.audit.get", + "capability": "tasks.observe", + "status": "implemented", + "provider": "openphone-agentd" + }, + { + "tool": "get_trajectory", + "command": "openphone.trajectory.get", + "capability": "tasks.observe", + "status": "implemented", + "provider": "openphone-agentd" + }, + { + "tool": "memory_search", + "command": "openphone.memory.search", + "capability": "memory.read", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "memory_save", + "command": "openphone.memory.save", + "capability": "memory.write", + "status": "implemented", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "memory_update", + "command": "openphone.memory.update", + "capability": "memory.write", + "status": "implemented", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "memory_delete", + "command": "openphone.memory.delete", + "capability": "memory.write", + "status": "implemented", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "memory_merge", + "command": "openphone.memory.merge", + "capability": "memory.write", + "status": "implemented", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "context_search", + "command": "openphone.context.search", + "capability": "memory.read", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "clipboard_read", + "command": "openphone.clipboard.read", + "capability": "clipboard.read", + "status": "implemented_partial", + "provider": "OpenPhoneVolumeTrigger.SpringBoardClipboard", + "notes": "Reads bounded text from the phone clipboard through a SpringBoard-context UIPasteboard bridge, returns text to the caller/model, records context/audit metadata, and falls back to a daemon-local file only when the SpringBoard bridge is unavailable in local host smoke." + }, + { + "tool": "clipboard_write", + "command": "openphone.clipboard.write", + "capability": "clipboard.write", + "status": "implemented_partial", + "provider": "OpenPhoneVolumeTrigger.SpringBoardClipboard", + "notes": "Writes text to the phone clipboard through a SpringBoard-context UIPasteboard bridge and indexes a bounded preview plus length/hash metadata into the context store; write responses do not echo the raw text." + }, + { + "tool": "contacts_search", + "command": "openphone.contacts.search", + "capability": "contacts.read", + "status": "implemented_partial", + "provider": "AddressBook.sqlite", + "notes": "Read-only bounded search over the phone AddressBook SQLite database with local smoke fixture fallback; context/audit store counts and query hashes rather than raw contact values." + }, + { + "tool": "calendar_search", + "command": "openphone.calendar.search", + "capability": "calendar.read", + "status": "implemented_partial", + "provider": "Calendar.sqlitedb", + "notes": "Read-only bounded search over the phone Calendar SQLite database with local smoke fixture fallback; context/audit store counts, query hashes, and time ranges rather than raw event values." + }, + { + "tool": "calls_search", + "command": "openphone.calls.search", + "capability": "calls.read", + "status": "implemented_partial", + "provider": "CallHistory.storedata", + "notes": "Read-only bounded search over the phone call-history SQLite database with local smoke fixture fallback; context/audit store counts, query hashes, and time ranges rather than raw call values." + }, + { + "tool": "messages_search", + "command": "openphone.messages.search", + "capability": "messages.read", + "status": "implemented_partial", + "provider": "SMS.sqlite", + "notes": "Read-only bounded search over the phone SMS/iMessage SQLite database with local smoke fixture fallback; context/audit store counts, query hashes, and time ranges rather than raw message values." + }, + { + "tool": "commitment_create", + "command": "openphone.commitments.create", + "capability": "commitments.write", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite", + "notes": "Durable commitment store with a local due-time bridge. Active commitments with due_at_ms can be claimed and materialized into scheduler-enabled background jobs through commitment_run_due/background_job_run_due." + }, + { + "tool": "commitment_search", + "command": "openphone.commitments.search", + "capability": "commitments.read", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "commitment_update_status", + "command": "openphone.commitments.update_status", + "capability": "commitments.write", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "commitment_run_due", + "command": "openphone.commitments.run_due", + "capability": "commitments.write", + "status": "implemented_partial", + "provider": "openphone-agentd.scheduler", + "notes": "Claims due active commitments as running, creates audited scheduler-enabled commitment_due background jobs, and marks successfully materialized commitments as triggered. It does not execute generated jobs directly; background_job_run_due or the daemon scheduler runs them." + }, + { + "tool": "watcher_create", + "command": "openphone.watchers.create", + "capability": "watchers.write", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite", + "notes": "Durable store plus local timer watcher bridge. Active time/timer/deadline watchers with next_run_at_ms create scheduler-enabled background jobs through watcher_run_due/background_job_run_due. Notification/message/call/web watchers remain durable-only until providers exist." + }, + { + "tool": "watcher_list", + "command": "openphone.watchers.list", + "capability": "watchers.read", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "watcher_stop", + "command": "openphone.watchers.stop", + "capability": "watchers.write", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "watcher_repair_stuck", + "command": "openphone.watchers.repair_stuck", + "capability": "watchers.write", + "status": "implemented_partial", + "provider": "openphone-agentd.scheduler", + "notes": "Requeues stale running local timer watchers with metadata.stuck_repair and makes them immediately due again. watcher_run_due also performs this repair before claiming due watchers." + }, + { + "tool": "watcher_run_due", + "command": "openphone.watchers.run_due", + "capability": "watchers.write", + "status": "implemented_partial", + "provider": "openphone-agentd.scheduler", + "notes": "Repairs stale running local timer watchers, claims due local timer watchers as running, and materializes them into audited scheduler-enabled background jobs. It does not execute the generated jobs directly; background_job_run_due or the daemon scheduler runs them." + }, + { + "tool": "background_job_create", + "command": "openphone.jobs.create", + "capability": "background.run", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite", + "notes": "Creates durable queued jobs. Due jobs run through phone-local run_task with the requested mode or auto model selection. Jobs with interval_ms or recurring=true requeue after successful runs unless recurring is explicitly false." + }, + { + "tool": "background_job_list", + "command": "openphone.jobs.list", + "capability": "tasks.observe", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "background_job_stop", + "command": "openphone.jobs.stop", + "capability": "background.run", + "status": "implemented_partial", + "provider": "openphone-agentd.sqlite" + }, + { + "tool": "background_job_repair_stuck", + "command": "openphone.jobs.repair_stuck", + "capability": "background.run", + "status": "implemented_partial", + "provider": "openphone-agentd.scheduler", + "notes": "Requeues stale scheduler-enabled running jobs with stuck_repair metadata. background_job_run_due runs this repair before claiming due work." + }, + { + "tool": "background_job_run_due", + "command": "openphone.jobs.run_due", + "capability": "background.run", + "status": "implemented_partial", + "provider": "openphone-agentd.scheduler", + "notes": "Repairs stale scheduler-enabled running jobs, materializes due commitments and due timer watchers, claims due queued jobs, runs each job through phone-local run_task with requested mode/auto policy, and stores the run result back on the job. Successful recurring jobs requeue by interval; failed recurring jobs requeue with bounded exponential backoff." + }, + { + "tool": "hardware_trigger", + "command": "openphone.trigger.hardware", + "capability": "background.run", + "status": "implemented_partial", + "provider": "openphone-agentd", + "notes": "Daemon command records/audits the trigger and, when run_task=true with model provider ready, starts an async phone-local run_task loop in auto/model mode. It can still create or run background jobs when explicitly requested. Daemon HID listener registers but avoids unsafe activation on the current phone; the SpringBoard fallback tweak requires a tweak loader active in the userspace boot." + }, + { + "tool": "local_screen_understanding", + "command": "openphone.screen.understand_local", + "capability": "screen.read.visible", + "status": "not_started", + "provider": "ios.providers.screen" + }, + { + "tool": "open_app", + "command": "openphone.apps.open", + "capability": "apps.launch", + "status": "implemented", + "provider": "openphone-agentd.uiopen" + }, + { + "tool": "list_apps", + "command": "openphone.apps.search", + "capability": "apps.read", + "status": "implemented", + "provider": "openphone-agentd" + }, + { + "tool": "open_url", + "command": "openphone.url.open", + "capability": "network.use", + "status": "implemented", + "provider": "openphone-agentd.uiopen" + }, + { + "tool": "tap", + "command": "openphone.ui.tap", + "capability": "input.perform", + "status": "implemented", + "provider": "openphone-agentd.iohid" + }, + { + "tool": "tap_element", + "command": "openphone.ui.tap_element", + "capability": "input.perform", + "status": "implemented_partial", + "provider": "openphone-agentd.iohid" + }, + { + "tool": "long_press", + "command": "openphone.ui.long_press", + "capability": "input.perform", + "status": "implemented", + "provider": "openphone-agentd.iohid" + }, + { + "tool": "swipe", + "command": "openphone.ui.swipe", + "capability": "input.perform", + "status": "implemented", + "provider": "openphone-agentd.iohid" + }, + { + "tool": "type_text", + "command": "openphone.ui.type_text", + "capability": "input.perform", + "status": "implemented", + "provider": "openphone-agentd.iohid" + }, + { + "tool": "home", + "command": "openphone.input.press_key", + "capability": "input.perform", + "status": "implemented", + "provider": "openphone-agentd.graphicsservices" + }, + { + "tool": "wake_and_home", + "command": "openphone.input.wake_and_home", + "capability": "input.perform", + "status": "implemented", + "provider": "openphone-agentd.graphicsservices" + }, + { + "tool": "wait", + "command": "openphone.task.wait", + "capability": "tasks.observe", + "status": "implemented", + "provider": "openphone-agentd" + }, + { + "tool": "model_status", + "command": "openphone.model.status", + "capability": "tasks.observe", + "status": "implemented", + "provider": "openphone-agentd.config" + }, + { + "tool": "model_configure", + "command": "openphone.model.configure", + "capability": "tasks.observe", + "status": "implemented_partial", + "provider": "openphone-agentd.config", + "notes": "Writes non-credential model metadata only. Provider credential values are external-only and rejected inline. Supports broker, direct_dev, bedrock_converse, openai_realtime, and openai_realtime2. credential_required may be false for trusted local broker fixtures or trusted external brokers that hold credentials outside the phone; direct OpenAI Realtime modes always require an external credential." + }, + { + "tool": "finish_task", + "command": "openphone.task.finish", + "capability": "tasks.observe", + "status": "implemented", + "provider": "openphone-agentd" + }, + { + "tool": "fail_task", + "command": "openphone.task.fail", + "capability": "tasks.observe", + "status": "implemented", + "provider": "openphone-agentd" + }, + { + "tool": "run_task", + "command": "openphone.task.run", + "capability": "tasks.observe", + "status": "implemented_partial", + "provider": "openphone-agentd", + "notes": "Supports deterministic, model, and auto modes. Model mode has daemon-local parser/executor/trajectory support, conservative parser repair for prose/fenced-code wrapped decision JSON, fixture-provider smoke coverage, localhost broker HTTP smoke coverage, direct Bedrock Converse execution, and direct OpenAI Realtime WebSocket execution for openai_realtime/openai_realtime2. Realtime modes keep one session open per task, expose daemon tools as function tools, and execute function outputs locally through openphone-agentd." + } + ] +} diff --git a/ios/contracts/protocol/openphone-runtime.schema.json b/ios/contracts/protocol/openphone-runtime.schema.json new file mode 100644 index 0000000..a84c933 --- /dev/null +++ b/ios/contracts/protocol/openphone-runtime.schema.json @@ -0,0 +1,195 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/runtime/openphone-runtime.schema.json", + "title": "OpenPhone Runtime Agent Protocol", + "type": "object", + "additionalProperties": false, + "properties": { + "version": { + "const": 1 + }, + "attention_request": { + "$ref": "#/$defs/attentionRequest" + }, + "tool_request": { + "$ref": "#/$defs/toolRequest" + }, + "tool_result": { + "$ref": "#/$defs/toolResult" + }, + "runtime_event": { + "$ref": "#/$defs/runtimeEvent" + }, + "confirmation": { + "$ref": "#/$defs/confirmation" + } + }, + "required": [ + "version" + ], + "$defs": { + "attentionRequest": { + "type": "object", + "additionalProperties": true, + "required": [ + "runtime", + "phone_session_id", + "attention_id", + "source", + "text", + "autonomy" + ], + "properties": { + "runtime": { + "type": "string" + }, + "phone_session_id": { + "type": "string" + }, + "runtime_session_id": { + "type": "string" + }, + "attention_id": { + "type": "string" + }, + "source": { + "type": "string" + }, + "text": { + "type": "string" + }, + "autonomy": { + "enum": [ + "observe_only", + "ask_before_action", + "trusted_actions" + ] + }, + "include_screen": { + "type": "boolean" + }, + "context": { + "type": "object" + } + } + }, + "toolRequest": { + "type": "object", + "additionalProperties": true, + "required": [ + "request_id", + "runtime", + "runtime_session_id", + "tool", + "params" + ], + "properties": { + "request_id": { + "type": "string" + }, + "runtime": { + "type": "string" + }, + "runtime_session_id": { + "type": "string" + }, + "phone_session_id": { + "type": "string" + }, + "tool": { + "type": "string" + }, + "params": { + "type": "object" + }, + "reason": { + "type": "string" + }, + "autonomy": { + "type": "string" + }, + "idempotency_key": { + "type": "string" + }, + "timeout_ms": { + "type": "integer", + "minimum": 1 + } + } + }, + "toolResult": { + "type": "object", + "additionalProperties": true, + "required": [ + "request_id", + "status", + "requires_confirmation" + ], + "properties": { + "request_id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "result": { + "type": "object" + }, + "requires_confirmation": { + "type": "boolean" + }, + "error": { + "type": "object" + } + } + }, + "runtimeEvent": { + "type": "object", + "additionalProperties": true, + "required": [ + "event", + "payload" + ], + "properties": { + "event": { + "type": "string" + }, + "payload": { + "type": "object" + } + } + }, + "confirmation": { + "type": "object", + "additionalProperties": true, + "required": [ + "confirmation_id", + "request_id", + "runtime", + "tool", + "timeout_ms" + ], + "properties": { + "confirmation_id": { + "type": "string" + }, + "request_id": { + "type": "string" + }, + "runtime": { + "type": "string" + }, + "tool": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "timeout_ms": { + "type": "integer", + "minimum": 1 + } + } + } + } +} diff --git a/ios/contracts/schemas/action-registry.schema.json b/ios/contracts/schemas/action-registry.schema.json new file mode 100644 index 0000000..fc82f55 --- /dev/null +++ b/ios/contracts/schemas/action-registry.schema.json @@ -0,0 +1,88 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/action-registry.schema.json", + "title": "OpenPhone Action Registry", + "type": "object", + "required": ["version", "actions"], + "properties": { + "version": { + "const": 1 + }, + "actions": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "name", + "description", + "model_tool", + "kind", + "input_schema_json", + "output_schema_json", + "risk_class", + "required_capabilities", + "authorization_policy", + "allowed_callers", + "executor_service", + "audit_event_type" + ], + "properties": { + "name": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*(\\.[a-z][a-z0-9_]*)+$" + }, + "description": { + "type": "string", + "minLength": 1 + }, + "model_tool": { + "type": "string", + "minLength": 1 + }, + "kind": { + "enum": ["observe", "action", "control"] + }, + "input_schema_json": { + "type": "object" + }, + "output_schema_json": { + "type": "object" + }, + "risk_class": { + "enum": ["low", "medium", "high", "critical"] + }, + "required_capabilities": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "authorization_policy": { + "enum": ["task_grant", "confirm", "explicit_confirm"] + }, + "allowed_callers": { + "type": "array", + "minItems": 1, + "items": { + "enum": ["model", "assistant_ui", "system", "watcher"] + }, + "uniqueItems": true + }, + "executor_service": { + "type": "string", + "minLength": 1 + }, + "audit_event_type": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/ios/contracts/schemas/action-request.schema.json b/ios/contracts/schemas/action-request.schema.json new file mode 100644 index 0000000..06af37f --- /dev/null +++ b/ios/contracts/schemas/action-request.schema.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/action-request.schema.json", + "title": "OpenPhone Action Request", + "type": "object", + "required": ["type"], + "properties": { + "type": { + "enum": [ + "tap", + "long_press", + "scroll", + "type_text", + "back", + "home", + "recents", + "open_app", + "open_url", + "launch_intent", + "notification_action", + "copy", + "paste", + "share" + ] + }, + "capability": { + "type": "string" + }, + "target": { + "type": "object", + "properties": { + "package": { + "type": "string" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + }, + "start_x": { + "type": "number" + }, + "start_y": { + "type": "number" + }, + "end_x": { + "type": "number" + }, + "end_y": { + "type": "number" + }, + "dx": { + "type": "number" + }, + "dy": { + "type": "number" + } + }, + "additionalProperties": false + }, + "text": { + "type": "string" + }, + "url": { + "type": "string" + }, + "mime_type": { + "type": "string" + }, + "subject": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "additionalProperties": false +} diff --git a/ios/contracts/schemas/action-result.schema.json b/ios/contracts/schemas/action-result.schema.json new file mode 100644 index 0000000..6561c16 --- /dev/null +++ b/ios/contracts/schemas/action-result.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/action-result.schema.json", + "title": "OpenPhone Action Result", + "type": "object", + "required": ["state", "capability", "source"], + "properties": { + "state": { + "type": "string" + }, + "task_id": { + "type": ["string", "null"] + }, + "caller_uid": { + "type": "integer" + }, + "capability": { + "type": "string" + }, + "detail": { + "type": ["string", "null"] + }, + "input": { + "type": ["string", "null"] + }, + "pending_action_id": { + "type": ["string", "null"] + }, + "source": { + "type": "string" + } + }, + "additionalProperties": false +} diff --git a/ios/contracts/schemas/agent-job.schema.json b/ios/contracts/schemas/agent-job.schema.json new file mode 100644 index 0000000..9a7ffc5 --- /dev/null +++ b/ios/contracts/schemas/agent-job.schema.json @@ -0,0 +1,60 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/agent-job.schema.json", + "title": "OpenPhone Agent Job", + "type": "object", + "required": [ + "id", + "type", + "title", + "prompt", + "payload_json", + "schedule_json", + "session_target", + "delivery_json", + "status", + "created_at", + "updated_at", + "next_run_at", + "running_at", + "last_run_at", + "last_result", + "failure_count", + "failure_alert_at" + ], + "properties": { + "id": { "type": "integer", "minimum": 1 }, + "type": { + "enum": ["agent_turn", "system_event", "heartbeat"] + }, + "title": { "type": "string", "minLength": 1 }, + "prompt": { "type": "string" }, + "payload_json": { "type": "object" }, + "schedule_json": { "type": "object" }, + "session_target": { "type": "string" }, + "delivery_json": { + "type": "object", + "properties": { + "mode": { + "enum": ["notification", "silent", "none"] + }, + "notification_text": { + "type": "string" + } + }, + "additionalProperties": true + }, + "status": { + "enum": ["active", "running", "completed", "failed", "stopped"] + }, + "created_at": { "type": "integer", "minimum": 0 }, + "updated_at": { "type": "integer", "minimum": 0 }, + "next_run_at": { "type": "integer", "minimum": 0 }, + "running_at": { "type": "integer", "minimum": 0 }, + "last_run_at": { "type": "integer", "minimum": 0 }, + "last_result": { "type": "string" }, + "failure_count": { "type": "integer", "minimum": 0 }, + "failure_alert_at": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": false +} diff --git a/ios/contracts/schemas/audit-event.schema.json b/ios/contracts/schemas/audit-event.schema.json new file mode 100644 index 0000000..65d48a8 --- /dev/null +++ b/ios/contracts/schemas/audit-event.schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/audit-event.schema.json", + "title": "OpenPhone Audit Event", + "type": "object", + "required": ["schema", "timestamp_ms", "event_type", "previous_hash", "event_hash"], + "properties": { + "schema": { + "type": "string", + "const": "openphone.audit_event.v1" + }, + "timestamp_ms": { + "type": "integer" + }, + "timestamp_elapsed_ms": { + "type": "integer" + }, + "task_id": { + "type": ["string", "null"] + }, + "event_type": { + "enum": [ + "task_started", + "task_stopped", + "task_finished", + "task_failed", + "screen_context_read", + "screen_capture", + "screen_capture_rejected", + "screen_watch_started", + "screen_watch_rejected", + "apps_listed", + "capability_evaluated", + "action_rejected", + "action_confirmed", + "action_executed", + "ui_tree_snapshot_submitted" + ] + }, + "capability": { + "type": ["string", "null"] + }, + "decision": { + "type": ["string", "null"] + }, + "caller_uid": { + "type": "integer" + }, + "detail": { + "type": ["string", "null"] + }, + "input": { + "type": ["object", "array", "string", "number", "boolean", "null"] + }, + "input_recorded": { + "type": "boolean" + }, + "source": { + "type": "string" + }, + "previous_hash": { + "type": "string", + "pattern": "^$|^[a-f0-9]{64}$" + }, + "event_hash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "additionalProperties": false +} diff --git a/ios/contracts/schemas/screen-context.schema.json b/ios/contracts/schemas/screen-context.schema.json new file mode 100644 index 0000000..c5b84d4 --- /dev/null +++ b/ios/contracts/schemas/screen-context.schema.json @@ -0,0 +1,273 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/screen-context.schema.json", + "title": "OpenPhone Screen Context", + "type": "object", + "required": ["interactive_elements", "risk_flags"], + "properties": { + "source": { + "type": "string" + }, + "timestamp_ms": { + "type": "integer" + }, + "foreground_app": { + "type": "string" + }, + "foreground_package": { + "type": "string" + }, + "foreground_source": { + "type": "string" + }, + "running_apps": { + "type": "array", + "items": { + "type": "object", + "required": ["pid", "bundle_id", "display_name", "app_path", "executable_path"], + "properties": { + "pid": { + "type": "integer" + }, + "bundle_id": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "app_path": { + "type": "string" + }, + "executable_path": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "display": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "pixel_width": { + "type": "number" + }, + "pixel_height": { + "type": "number" + }, + "point_width": { + "type": "number" + }, + "point_height": { + "type": "number" + }, + "screen_width": { + "type": "number" + }, + "screen_height": { + "type": "number" + }, + "scale": { + "type": "number" + }, + "orientation": { + "type": "integer" + }, + "orientation_name": { + "type": "string" + } + }, + "additionalProperties": false + }, + "lock": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "locked": { + "type": "boolean" + }, + "passcode_enabled": { + "type": "boolean" + } + }, + "additionalProperties": false + }, + "root_packages": { + "type": "array", + "items": { + "type": "string" + } + }, + "activity": { + "type": "string" + }, + "visible_text": { + "type": "array", + "items": { + "type": "string" + } + }, + "ui_tree": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "window_count": { + "type": "integer" + }, + "element_count": { + "type": "integer" + }, + "text_count": { + "type": "integer" + }, + "visible_text": { + "type": "array", + "items": { + "type": "string" + } + }, + "interactive_elements": { + "type": "array" + }, + "windows": { + "type": "array" + } + }, + "additionalProperties": true + }, + "visible_activities": { + "type": "array", + "items": { + "type": "object", + "required": ["focused", "task_id", "user_id", "package", "activity"], + "properties": { + "focused": { + "type": "boolean" + }, + "task_id": { + "type": "integer" + }, + "user_id": { + "type": "integer" + }, + "package": { + "type": ["string", "null"] + }, + "activity": { + "type": ["string", "null"] + } + }, + "additionalProperties": false + } + }, + "interactive_elements": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "kind", "label"], + "properties": { + "id": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "label": { + "type": "string" + }, + "bounds": { + "type": "array", + "items": { + "type": "integer" + }, + "minItems": 4, + "maxItems": 4 + }, + "enabled": { + "type": "boolean" + }, + "focused": { + "type": "boolean" + }, + "view_id": { + "type": "string" + }, + "window_id": { + "type": "integer" + }, + "sensitive": { + "type": "boolean" + }, + "risk_hint": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "windows": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "type", "focused", "active", "bounds"], + "properties": { + "id": { + "type": "integer" + }, + "type": { + "type": "integer" + }, + "focused": { + "type": "boolean" + }, + "active": { + "type": "boolean" + }, + "bounds": { + "type": "array", + "items": { + "type": "integer" + }, + "minItems": 4, + "maxItems": 4 + } + }, + "additionalProperties": false + } + }, + "notifications": { + "type": "array" + }, + "risk_flags": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false +} diff --git a/ios/contracts/schemas/trajectory-event.schema.json b/ios/contracts/schemas/trajectory-event.schema.json new file mode 100644 index 0000000..42de9a5 --- /dev/null +++ b/ios/contracts/schemas/trajectory-event.schema.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/trajectory-event.schema.json", + "title": "OpenPhone Trajectory Event", + "type": "object", + "required": ["schema", "timestamp_ms", "event", "payload"], + "properties": { + "schema": { + "type": "string", + "const": "openphone.trajectory_event.v1" + }, + "index": { + "type": "integer", + "minimum": 0 + }, + "timestamp_ms": { + "type": "integer" + }, + "event": { + "enum": [ + "task_started", + "task_stopped", + "screen_observed", + "tool_call", + "tool_result", + "model_decision", + "task_finished", + "task_failed", + "agent_result" + ] + }, + "payload": { + "type": "object" + }, + "source": { + "type": "string" + } + }, + "additionalProperties": false +} diff --git a/ios/shared/contracts/README.md b/ios/shared/contracts/README.md new file mode 100644 index 0000000..b5f6dcb --- /dev/null +++ b/ios/shared/contracts/README.md @@ -0,0 +1,37 @@ +# shared/contracts + +Platform-neutral contracts that both the iOS and Android OpenPhone agents +consume. This directory is the single source of truth for the pieces that must +stay identical across platforms, extracted here to lower the eventual +main-repo merge cost (see `IOS_PLAN.md` Tier 3 / Gate F). + +This is purely additive. It does not change any runtime behavior; the iOS +daemon still embeds these definitions in `ios/agentd/src/main.m`. When the +daemon's tool surface, decision schema, or capability list changes, update the +matching file here in the same change. + +## Files + +| File | Source of truth | Consumed by | +|---|---|---| +| `model-tools.json` | `OPModelToolNames()` / `OPModelToolCapability()` / `OPModelToolDrivesUI()` in `ios/agentd/src/main.m` | Model/Realtime tool registration on both platforms | +| `model-decision.v3.schema.json` | `OPModelParseDecision` in `ios/agentd/src/main.m` | Model decision parser/validator on both platforms | +| `capabilities.json` | Canonical capability + risk list (mirrors `contracts/protocol/openphone-capabilities.json`) | Policy/consent gating on both platforms | + +## Relationship to `contracts/` + +`contracts/` is the incoming Android mirror (schemas + protocol snapshots) used +to keep iOS aligned during private development. `shared/contracts/` is the +outgoing, deduplicated set both platforms are meant to build against after the +merge. As the merge proceeds, `contracts/` collapses into this directory. + +## Consumption check + +Both platforms can consume these as plain JSON: + +- **iOS:** the daemon already hard-codes the same values in `main.m`; a build + or test step can diff the embedded list against `model-tools.json` to catch + drift. +- **Android:** load `model-tools.json` to register the tool surface and + `capabilities.json` to drive consent, then validate model output against + `model-decision.v3.schema.json`. diff --git a/ios/shared/contracts/capabilities.json b/ios/shared/contracts/capabilities.json new file mode 100644 index 0000000..312fad9 --- /dev/null +++ b/ios/shared/contracts/capabilities.json @@ -0,0 +1,30 @@ +{ + "version": 1, + "capabilities": [ + { "id": "screen.read.visible", "risk": "medium" }, + { "id": "apps.read", "risk": "low" }, + { "id": "apps.launch", "risk": "medium" }, + { "id": "input.perform", "risk": "high" }, + { "id": "network.use", "risk": "medium" }, + { "id": "notifications.read", "risk": "medium" }, + { "id": "notifications.act", "risk": "high" }, + { "id": "contacts.read", "risk": "medium" }, + { "id": "calendar.read", "risk": "medium" }, + { "id": "calendar.write", "risk": "high" }, + { "id": "calendar.delete", "risk": "high" }, + { "id": "messages.read", "risk": "high" }, + { "id": "messages.draft", "risk": "high" }, + { "id": "messages.send", "risk": "critical" }, + { "id": "calls.read", "risk": "medium" }, + { "id": "calls.place", "risk": "critical" }, + { "id": "clipboard.read", "risk": "medium" }, + { "id": "clipboard.write", "risk": "high" }, + { "id": "share.content", "risk": "high" }, + { "id": "tasks.observe", "risk": "low" }, + { "id": "background.run", "risk": "high" }, + { "id": "memory.read", "risk": "medium" }, + { "id": "memory.write", "risk": "high" }, + { "id": "watchers.read", "risk": "medium" }, + { "id": "watchers.write", "risk": "high" } + ] +} diff --git a/ios/shared/contracts/model-decision.v3.schema.json b/ios/shared/contracts/model-decision.v3.schema.json new file mode 100644 index 0000000..3e4724a --- /dev/null +++ b/ios/shared/contracts/model-decision.v3.schema.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/shared/contracts/model-decision.v3.schema.json", + "title": "OpenPhone Model Decision v3 (router)", + "description": "One decision emitted per model step. v3 is the router shape: it carries mode/reply and does not require a top-level tool field (see main.m: v3 short-circuits tool validation). Legacy v1/v2 single-tool decisions are still accepted by the daemon parser.", + "type": "object", + "required": ["schema", "mode"], + "properties": { + "schema": { + "type": "string", + "enum": [ + "openphone.model_decision.v3", + "openphone.model_decision.v2", + "openphone.model_decision.v1" + ] + }, + "mode": { + "type": "string", + "enum": ["answer", "act", "stop"], + "description": "answer = user-facing reply only; act = run proposed_actions; stop = terminate the loop." + }, + "reply": { + "type": "string", + "description": "User-facing text. Present for answer/stop; optional for act." + }, + "task_goal": { + "type": "string", + "description": "Only set when the request is multi-step." + }, + "proposed_actions": { + "type": "array", + "description": "For mode=act. Each action names a tool from model-tools.json and its arguments.", + "items": { + "type": "object", + "required": ["tool"], + "properties": { + "tool": { "type": "string" }, + "arguments": { "type": "object" }, + "expected_visible_change": { + "type": "string", + "description": "Required when the tool drives UI (see model-tools.json drives_ui)." + } + } + } + }, + "reason": { + "type": "string", + "description": "Private, brief rationale. Not shown to the user." + }, + "confidence": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0 + } + } +} diff --git a/ios/shared/contracts/model-tools.json b/ios/shared/contracts/model-tools.json new file mode 100644 index 0000000..44df298 --- /dev/null +++ b/ios/shared/contracts/model-tools.json @@ -0,0 +1,34 @@ +{ + "version": 1, + "description": "Canonical model-callable tool registry shared by iOS and Android. iOS source of truth is OPModelToolNames() / OPModelToolCapability() / OPModelToolDrivesUI() in ios/agentd/src/main.m. Keep this file in sync when the daemon tool surface changes.", + "decision_schema": "openphone.model_decision.v3", + "tools": [ + { "tool": "get_screen", "capability": "screen.read.visible", "drives_ui": false, "arguments": { "max_dimension_px": "integer?", "include_screenshot": "boolean?" } }, + { "tool": "tap", "capability": "input.perform", "drives_ui": true, "arguments": { "x": "number", "y": "number", "expected_visible_change": "string" } }, + { "tool": "tap_element", "capability": "input.perform", "drives_ui": true, "arguments": { "element_id": "string", "expected_visible_change": "string" } }, + { "tool": "long_press", "capability": "input.perform", "drives_ui": true, "arguments": { "x": "number", "y": "number", "duration_ms": "integer?", "expected_visible_change": "string" } }, + { "tool": "swipe", "capability": "input.perform", "drives_ui": true, "arguments": { "from_x": "number", "from_y": "number", "to_x": "number", "to_y": "number", "expected_visible_change": "string" } }, + { "tool": "type_text", "capability": "input.perform", "drives_ui": true, "arguments": { "text": "string", "element_id": "string?", "expected_visible_change": "string" } }, + { "tool": "open_app", "capability": "apps.launch", "drives_ui": true, "arguments": { "bundle_id": "string", "expected_visible_change": "string" } }, + { "tool": "open_url", "capability": "network.use", "drives_ui": true, "arguments": { "url": "string", "expected_visible_change": "string" } }, + { "tool": "home", "capability": "input.perform", "drives_ui": true, "arguments": { "expected_visible_change": "string" } }, + { "tool": "wake_and_home", "capability": "input.perform", "drives_ui": true, "arguments": { "expected_visible_change": "string" } }, + { "tool": "wait", "capability": "tasks.observe", "drives_ui": false, "arguments": { "seconds": "number?" } }, + { "tool": "clipboard_read", "capability": "clipboard.read", "drives_ui": false, "arguments": {} }, + { "tool": "clipboard_write", "capability": "clipboard.write", "drives_ui": false, "arguments": { "text": "string" } }, + { "tool": "contacts_search", "capability": "contacts.read", "drives_ui": false, "arguments": { "query": "string", "allow_empty_query": "boolean?", "limit": "integer?" } }, + { "tool": "calendar_search", "capability": "calendar.read", "drives_ui": false, "arguments": { "query": "string", "allow_empty_query": "boolean?", "limit": "integer?", "start_at_ms": "integer?", "end_at_ms": "integer?" } }, + { "tool": "calls_search", "capability": "calls.read", "drives_ui": false, "arguments": { "query": "string", "allow_empty_query": "boolean?", "limit": "integer?", "start_at_ms": "integer?", "end_at_ms": "integer?" } }, + { "tool": "messages_search", "capability": "messages.read", "drives_ui": false, "arguments": { "query": "string", "allow_empty_query": "boolean?", "limit": "integer?" } }, + { "tool": "memory_save", "capability": "memory.write", "drives_ui": false, "arguments": { "text": "string", "type": "string?", "metadata": "object?" } }, + { "tool": "memory_search", "capability": "memory.read", "drives_ui": false, "arguments": { "query": "string", "limit": "integer?" } }, + { "tool": "context_search", "capability": "memory.read", "drives_ui": false, "arguments": { "query": "string", "limit": "integer?" } }, + { "tool": "finish_task", "capability": "tasks.observe", "drives_ui": false, "arguments": { "reply": "string?" } }, + { "tool": "fail_task", "capability": "tasks.observe", "drives_ui": false, "arguments": { "reason": "string?" } } + ], + "notes": { + "expected_visible_change": "Required for any tool where drives_ui is true. The daemon rejects a UI-driving decision without it (missing_expected_visible_change).", + "search_query": "contacts/calendar/calls/messages searches require a non-empty query OR allow_empty_query:true.", + "capability_fallback": "Any model tool not explicitly mapped defaults to capability input.perform (see OPModelToolCapability)." + } +} diff --git a/ios/tools/mac/agentd/README.md b/ios/tools/mac/agentd/README.md new file mode 100644 index 0000000..0257ae3 --- /dev/null +++ b/ios/tools/mac/agentd/README.md @@ -0,0 +1,213 @@ +# Agentd Mac Tools + +These scripts install, restart, and inspect `openphone-agentd` on an owned +iPhone with a rootless runtime prefix at `/var/jb`. + +They are development and recovery tools. They do not run the OpenPhone agent on +the Mac. + +Local host smoke test before packaging or device install: + +```sh +tools/mac/agentd/smoke-agentd-local.sh +``` + +The smoke test compiles `openphone-agentd` and `openphone-agentctl` for macOS, +runs the daemon against a temporary store, verifies `health`, exercises memory, +context, commitments, watchers, timer watcher materialization, stale watcher +repair, background-job creation, `background_job_run_due`, stale background-job +repair, the deferred hardware-trigger command path, audit/trajectory export, and one +deterministic `run_task` request. It also runs a fixture-backed model-mode +`run_task` through the daemon's model decision parser, +registered-tool executor, terminal `finish_task`, parser repair for +prose/fenced-code wrapped decision JSON, and trajectory export. It then +configures a local localhost model broker fixture and verifies broker-backed +model mode through the daemon HTTP provider path without any credential. A local +deterministic `task.failed` result is expected for Safari because `uiopen` only +exists on the device under the rootless prefix. + +Validate an agentd store pulled from the phone or produced by the smoke test: + +```sh +tools/mac/agentd/validate-agentd-store.py /var/mobile/Library/OpenPhone +``` + +The validator parses task, audit, and trajectory files and verifies the audit +hash chain. + +Environment: + +```sh +export OPENPHONE_IOS_HOST= +export OPENPHONE_IOS_SSH_PORT=22 +export OPENPHONE_IOS_USER=mobile +export OPENPHONE_IOS_PASSWORD= +``` + +When more than one iPhone is connected, do not start a generic USB tunnel. +Pin `iproxy` to the intended device UDID: + +```sh +export OPENPHONE_IOS_HOST=127.0.0.1 +export OPENPHONE_IOS_SSH_PORT=2222 +export OPENPHONE_IOS_UDID= +export OPENPHONE_IOS_EXPECTED_DEVICE_NAME= +export OPENPHONE_IOS_EXPECTED_PRODUCT_TYPE=iPhone15,3 +``` + +`iPhone15,3` is Apple's hardware identifier for the iPhone 14 Pro Max. The +scripts treat it only as an identity guard; it does not mean the separate +iPhone 15 named `` should be used. + +Build and install the rootless package: + +```sh +(cd ios/agentd && make package) +tools/mac/agentd/install-agentd-package.sh +``` + +`install-agentd-package.sh` copies the latest package from `ios/agentd/packages`, +runs `dpkg -i` with `sudo` on the phone, reloads the launchd plist, and prints +daemon health. Set `OPENPHONE_AGENTD_DEB=/path/to/package.deb` to install a +specific package. Set `OPENPHONE_IOS_START_IPROXY=1` with +`OPENPHONE_IOS_UDID=` to let the installer start and clean up a temporary +pinned USB tunnel. It refuses unpinned automated tunnels by default. + +Install loose binaries after building `ios/agentd`: + +```sh +tools/mac/agentd/install-agentd.sh +``` + +Restart: + +```sh +tools/mac/agentd/restart-agentd.sh +``` + +Tail daemon log: + +```sh +tools/mac/agentd/tail-agentd-log.sh +``` + +Run an on-device validation pass: + +```sh +tools/mac/agentd/validate-on-device.sh --mode collect-only +tools/mac/agentd/validate-on-device.sh --mode baseline +tools/mac/agentd/validate-on-device.sh --mode full +OPENPHONE_VALIDATE_INCLUDE_STORES=1 tools/mac/agentd/validate-on-device.sh --mode collect-only +OPENPHONE_VALIDATE_INCLUDE_APP_UI=1 tools/mac/agentd/validate-on-device.sh --mode collect-only +OPENPHONE_VALIDATE_INCLUDE_TRIGGER_DIAGNOSTICS=1 \ + OPENPHONE_VALIDATE_TRIGGER_WAIT_SECONDS=20 \ + tools/mac/agentd/validate-on-device.sh --mode collect-only +``` + +Set `OPENPHONE_VALIDATE_START_IPROXY=1` with `OPENPHONE_IOS_UDID=` to +let validation start a temporary pinned tunnel. If a listener already exists on +the local SSH port, validation fails unless +`OPENPHONE_VALIDATE_ALLOW_EXISTING_IPROXY=1` is set after manually confirming +that listener targets the intended iPhone. + +The validator writes a structured report under +`artifacts/validation//report.json`. `collect-only` does not build or +install anything. `baseline` runs local smoke, builds and installs the package, +then collects health, SpringBoard crash state, safe-mode markers, screen state, +and logs. `full` adds safe store/task queries. Screenshots are opt-in with +`OPENPHONE_VALIDATE_INCLUDE_SCREENSHOT=1`; when enabled, the validator pulls the +phone-local PNG and samples decoded pixels so blank captures fail the screenshot +gate. Optional gates stop early if a safe-mode marker or new SpringBoard crash is +detected. Set `OPENPHONE_VALIDATE_INCLUDE_STORES=1` with `collect-only` to run +read-only task, audit, memory, context, background-job, commitment, and watcher +queries without rebuilding or reinstalling the package. Store validation checks +both command status and required response shape, including expected arrays, +counts, and key identifiers. It also selects the latest safe task id from +`list_tasks`, then validates `get_task` and `get_trajectory` response shape. + +`OPENPHONE_VALIDATE_INCLUDE_TRIGGER_DIAGNOSTICS=1` is the physical trigger gate: +unlock the phone first, start the validator, then press Volume Up followed by +Volume Down during the wait window. The report compares before/after +SpringBoard button, combo, and passive volume-notification counters, links the +latest task/trajectory, and fails with explicit reasons such as +`no_physical_volume_event_observed`, `volume_combo_not_observed`, or +`combo_observed_without_new_agent_task`. Press only after the validator has +entered the wait window; pressing during setup can still start a real task, but +the before/after delta gate will correctly fail because the event happened +before the baseline snapshot. + +Set `OPENPHONE_VALIDATE_INCLUDE_PROVIDER_ATTEMPTS=1` to add a no-dispatch +input-provider sample: the validator starts a tiny validation task, runs a +missing-coordinate tap that cannot dispatch HID or activate SpringBoard, and +checks the resulting provider-attempt and trajectory schema. +Set `OPENPHONE_VALIDATE_INCLUDE_MEMORY_LIFECYCLE=1` to add a contained memory +lifecycle sample that saves, updates, merges, deletes, and searches validation +memories on the phone. +Set `OPENPHONE_VALIDATE_INCLUDE_MODEL_LOOP=1` to add a fixture-backed model-loop +sample that runs `run_task mode=model`, executes parsed +`openphone.model_decision.v1` decisions through daemon tools, finishes with +`finish_task`, verifies a stopped adopted task exits as `task.cancelled` before +executing a model decision, verifies parser repair for prose/fenced-code wrapped +decision JSON, and validates the model trajectory shape without network access. +Set `OPENPHONE_VALIDATE_INCLUDE_WATCHER_TIMER=1` to add a due timer watcher +sample. The validator creates a local `time` watcher, materializes it through +`watcher_run_due`, confirms a scheduler-enabled background job was created and +listed, checks that the watcher reached `fired`, and then stops the watcher. +Set `OPENPHONE_VALIDATE_INCLUDE_WATCHER_REPAIR=1` to add a stale watcher repair +sample. The validator creates a future local timer watcher, marks it +validation-only stale `running`, repairs it through `watcher_repair_stuck`, +verifies `metadata.stuck_repair`, fires it through `watcher_run_due`, optionally +drains the generated background job, and stops the fixture watcher. +Set `OPENPHONE_VALIDATE_INCLUDE_UNLOCKED_FOREGROUND=1` to add foreground source +validation. The validator requires the phone to be physically unlocked, launches +Safari through the daemon, waits for the SpringBoard state publisher, verifies +that `get_screen.context.foreground_app` is `com.apple.mobilesafari` with +`foreground_source=springboard_state`, rejects recent-layout inference, sends +Home, and captures cleanup screen state. If the phone is locked, the gate reports +`blocked_locked`; set `OPENPHONE_VALIDATE_REQUIRE_UNLOCKED=1` to make that gate +required. +Set `OPENPHONE_VALIDATE_INCLUDE_APP_UI=1` to add app-process UI validation. The +validator requires the phone to be physically unlocked, relaunches Safari and +Settings, waits for app tweak publication through daemon TCP intake, and +requires `get_screen.context.ui_tree_source=app_process` plus nonempty UI trees +for both apps. +Set `OPENPHONE_VALIDATE_INCLUDE_JOB_REPAIR=1` to add a stale background-job +repair sample. The validator creates a future scheduler-enabled job, marks that +job as validation-only stale `running`, repairs it through +`background_job_repair_stuck`, verifies the row is requeued with +`payload.stuck_repair`, lets the scheduler race/run it if due, and cleans up the +fixture job. +Set `OPENPHONE_VALIDATE_INCLUDE_RESTART_RECOVERY=1` to add daemon restart +recovery validation. The validator creates durable future watcher/job fixtures, +marks both validation-only stale `running`, kills `openphone-agentd`, waits for +launchd to restart it and for repair/materialization scheduler ticks to run, +then validates that stale watcher/job rows were repaired after restart. Override +the launchd restart timeout with +`OPENPHONE_VALIDATE_RESTART_RECOVERY_START_TIMEOUT_SECONDS`; the default is +`60`. Override the restarted-PID stability window with +`OPENPHONE_VALIDATE_RESTART_RECOVERY_STABLE_SECONDS`; the default is `12`. +Override the post-restart scheduler wait with +`OPENPHONE_VALIDATE_RESTART_RECOVERY_WAIT_SECONDS`; the default is `36`. +Set `OPENPHONE_VALIDATE_INCLUDE_PROVIDER_MODEL=1` to add a provider-backed +broker sample. The validator starts `bedrock-model-broker.py` on the Mac, opens +an SSH reverse tunnel so the phone can call `127.0.0.1:`, configures +`openphone-agentd` to that broker, runs `run_task mode=model`, validates +Bedrock-provider metadata in the phone trajectory, and resets model config. The +real Bedrock path requires `AWS_BEARER_TOKEN_BEDROCK` or +`OPENPHONE_BEDROCK_BEARER_TOKEN`; set `OPENPHONE_BEDROCK_MODEL` and +`OPENPHONE_BEDROCK_REGION` when needed. For transport tests without a provider +credential, set `OPENPHONE_BEDROCK_RUNTIME_URL` to a local mock Bedrock Runtime +endpoint. +Every report also runs an artifact hygiene gate over generated text/JSON +artifacts; it reports only detector names and locations, skips binary +screenshots/packages, and fails with exit code `90` if a password, passcode, +token, API key, private key, or bearer-token-shaped leak is found. The +provider-model gate fails with exit code `100` when the broker, reverse tunnel, +phone model config, task result, trajectory metadata, or reset step fails. + +If Theos outputs binaries to a non-default location, set: + +```sh +export OPENPHONE_AGENTD_BIN=/path/to/openphone-agentd +export OPENPHONE_AGENTCTL_BIN=/path/to/openphone-agentctl +``` diff --git a/ios/tools/mac/agentd/bedrock-model-broker.py b/ios/tools/mac/agentd/bedrock-model-broker.py new file mode 100755 index 0000000..21ab507 --- /dev/null +++ b/ios/tools/mac/agentd/bedrock-model-broker.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +import argparse +import http.server +import json +import os +import pathlib +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + + +DEFAULT_MODEL = "anthropic.claude-haiku-4-5-20251001-v1:0" + + +def env_first(*names, default=""): + for name in names: + value = os.environ.get(name) + if value: + return value + return default + + +def env_int(name, default): + value = os.environ.get(name) or str(default) + return int(value) + + +def env_float(name, default): + value = os.environ.get(name) or str(default) + return float(value) + + +def json_response(handler, status, payload): + encoded = json.dumps(payload, separators=(",", ":")).encode("utf-8") + handler.send_response(status) + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(encoded))) + handler.end_headers() + handler.wfile.write(encoded) + + +def compact_json(value, limit=12000): + text = json.dumps(value, separators=(",", ":"), sort_keys=True) + if len(text) > limit: + return text[:limit] + "..." + return text + + +def bedrock_converse(token, region, model, request_body, timeout, runtime_url): + endpoint = runtime_url.rstrip("/") if runtime_url else f"https://bedrock-runtime.{region}.amazonaws.com" + model_path = urllib.parse.quote(model, safe="") + url = f"{endpoint}/model/{model_path}/converse" + goal = request_body.get("goal") or "" + context = request_body.get("context") if isinstance(request_body.get("context"), dict) else {} + tools = request_body.get("tools") if isinstance(request_body.get("tools"), list) else [] + instructions = request_body.get("instructions") or "Return exactly one JSON object." + prompt = ( + "You are the model decision broker for OpenPhone-iOS. The iPhone daemon is the " + "runtime authority and will execute only a registered tool from your decision.\n\n" + f"Goal:\n{goal}\n\n" + f"Registered tools:\n{json.dumps(tools, separators=(',', ':'))}\n\n" + f"Context JSON:\n{compact_json(context)}\n\n" + "Return exactly one JSON object with this schema and no prose outside the JSON:\n" + '{"schema":"openphone.model_decision.v1","thought":"short rationale",' + '"tool":"finish_task","arguments":{"summary":"short result"},' + '"expected_visible_change":"none","confidence":0.0}\n\n' + "For this provider-backed validation request, choose finish_task unless the context " + "explicitly proves the task should fail. Do not request UI-driving tools." + ) + body = { + "system": [{"text": instructions}], + "messages": [{"role": "user", "content": [{"text": prompt}]}], + "inferenceConfig": { + "maxTokens": env_int("OPENPHONE_BEDROCK_MAX_TOKENS", 800), + "temperature": env_float("OPENPHONE_BEDROCK_TEMPERATURE", 0), + }, + } + encoded = json.dumps(body).encode("utf-8") + req = urllib.request.Request( + url, + data=encoded, + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + }, + ) + started = time.time() + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + response_body = response.read() + status = response.status + except urllib.error.HTTPError as exc: + detail = exc.read(2048).decode("utf-8", errors="replace") + return { + "ok": False, + "http_status": exc.code, + "reason": "bedrock_http_error", + "detail": detail[:500], + "latency_ms": int((time.time() - started) * 1000), + } + except Exception as exc: + return { + "ok": False, + "http_status": 0, + "reason": "bedrock_request_failed", + "detail": str(exc)[:500], + "latency_ms": int((time.time() - started) * 1000), + } + try: + parsed = json.loads(response_body.decode("utf-8")) + except Exception as exc: + return { + "ok": False, + "http_status": status, + "reason": "bedrock_response_not_json", + "detail": str(exc)[:500], + "latency_ms": int((time.time() - started) * 1000), + } + blocks = (((parsed.get("output") or {}).get("message") or {}).get("content") or []) + text_parts = [block.get("text") for block in blocks if isinstance(block, dict) and isinstance(block.get("text"), str)] + decision_text = "\n".join(text_parts).strip() + if not decision_text: + return { + "ok": False, + "http_status": status, + "reason": "bedrock_empty_text", + "latency_ms": int((time.time() - started) * 1000), + } + return { + "ok": True, + "http_status": status, + "decision_json": decision_text, + "usage": parsed.get("usage") if isinstance(parsed.get("usage"), dict) else {}, + "latency_ms": int((time.time() - started) * 1000), + } + + +def main(): + parser = argparse.ArgumentParser(description="OpenPhone provider-backed Bedrock model broker") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=0) + parser.add_argument("--port-file", required=True) + args = parser.parse_args() + + token = env_first("AWS_BEARER_TOKEN_BEDROCK", "OPENPHONE_BEDROCK_BEARER_TOKEN") + region = env_first("OPENPHONE_BEDROCK_REGION", "AWS_REGION", "AWS_DEFAULT_REGION", default="us-east-1") + model = env_first("OPENPHONE_BEDROCK_MODEL", default=DEFAULT_MODEL) + timeout = env_float("OPENPHONE_BEDROCK_TIMEOUT_SECONDS", 60) + runtime_url = env_first("OPENPHONE_BEDROCK_RUNTIME_URL") + if not token: + print("missing AWS_BEARER_TOKEN_BEDROCK or OPENPHONE_BEDROCK_BEARER_TOKEN", file=sys.stderr) + return 2 + + class Handler(http.server.BaseHTTPRequestHandler): + counter = 0 + + def log_message(self, fmt, *items): + return + + def do_POST(self): + if self.path != "/decision": + json_response(self, 404, {"status": "error", "reason": "not_found"}) + return + length = int(self.headers.get("Content-Length", "0") or "0") + body = self.rfile.read(length) + try: + request_body = json.loads(body.decode("utf-8")) + except Exception: + json_response(self, 400, {"status": "error", "reason": "request_not_json"}) + return + Handler.counter += 1 + result = bedrock_converse(token, region, model, request_body, timeout, runtime_url) + log_record = { + "event": "provider_broker_request", + "request_count": Handler.counter, + "ok": result.get("ok"), + "http_status": result.get("http_status"), + "reason": result.get("reason", ""), + "latency_ms": result.get("latency_ms"), + "model": model, + } + print(json.dumps(log_record, separators=(",", ":")), flush=True) + if not result.get("ok"): + json_response(self, 502, { + "status": "error", + "reason": result.get("reason", "provider_error"), + "http_status": result.get("http_status", 0), + "metadata": { + "provider": "bedrock_converse", + "model": model, + "latency_ms": result.get("latency_ms", 0), + }, + }) + return + json_response(self, 200, { + "schema": "openphone.model_response.v1", + "decision_json": result["decision_json"], + "metadata": { + "provider": "bedrock_converse", + "provider_backed": True, + "model": model, + "region": region, + "request_count": Handler.counter, + "latency_ms": result.get("latency_ms", 0), + }, + "usage": result.get("usage", {}), + }) + + server = http.server.ThreadingHTTPServer((args.host, args.port), Handler) + port_file = pathlib.Path(args.port_file) + port_file.write_text(str(server.server_port), encoding="utf-8") + print(json.dumps({ + "event": "provider_broker_started", + "host": args.host, + "port": server.server_port, + "provider": "bedrock_converse", + "model": model, + "region": region, + }, separators=(",", ":")), flush=True) + server.serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ios/tools/mac/agentd/install-agentd-package.sh b/ios/tools/mac/agentd/install-agentd-package.sh new file mode 100755 index 0000000..997ee68 --- /dev/null +++ b/ios/tools/mac/agentd/install-agentd-package.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +host="${OPENPHONE_IOS_HOST:-127.0.0.1}" +port="${OPENPHONE_IOS_SSH_PORT:-22}" +user="${OPENPHONE_IOS_USER:-mobile}" +password="${OPENPHONE_IOS_PASSWORD:-}" +known_hosts="${OPENPHONE_IOS_KNOWN_HOSTS:-/tmp/openphone-ios-known-hosts}" +remote_deb="${OPENPHONE_AGENTD_REMOTE_DEB:-/var/mobile/openphone-agentd.deb}" +package="${OPENPHONE_AGENTD_DEB:-}" +target_udid="${OPENPHONE_IOS_UDID:-}" +started_iproxy_pid="" + +redact_udid() { + local value="$1" + if [[ -z "$value" ]]; then + printf '%s\n' "" + elif [[ "${#value}" -le 8 ]]; then + printf '%s\n' "..." + else + printf '%s\n' "...${value: -8}" + fi +} + +cleanup() { + if [[ -n "$started_iproxy_pid" ]]; then + kill "$started_iproxy_pid" >/dev/null 2>&1 || true + wait "$started_iproxy_pid" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +start_iproxy_if_requested() { + if [[ "${OPENPHONE_IOS_START_IPROXY:-0}" != "1" ]]; then + return + fi + if ! command -v iproxy >/dev/null 2>&1; then + echo "missing required command: iproxy" >&2 + exit 30 + fi + if [[ -z "$target_udid" && "${OPENPHONE_IOS_ALLOW_UNPINNED_IPROXY:-0}" != "1" ]]; then + echo "refusing to start or reuse iproxy without OPENPHONE_IOS_UDID; set OPENPHONE_IOS_ALLOW_UNPINNED_IPROXY=1 only for a confirmed single-device setup" >&2 + exit 30 + fi + if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then + if [[ "${OPENPHONE_IOS_ALLOW_EXISTING_IPROXY:-0}" != "1" ]]; then + echo "local port $port already has a listener; stop it or set OPENPHONE_IOS_ALLOW_EXISTING_IPROXY=1 after confirming it targets the intended iPhone" >&2 + exit 30 + fi + echo "Using existing listener on local port $port by explicit override" + return + fi + local device_port="${OPENPHONE_IOS_IPROXY_DEVICE_PORT:-22}" + local -a iproxy_args=() + if [[ -n "$target_udid" ]]; then + iproxy_args=(-u "$target_udid") + echo "Starting temporary pinned iproxy $port:$device_port for UDID $(redact_udid "$target_udid")" + else + echo "Starting temporary unpinned iproxy $port:$device_port" + fi + iproxy "${iproxy_args[@]}" "$port:$device_port" >/tmp/openphone-install-agentd-iproxy.log 2>&1 & + started_iproxy_pid="$!" + sleep 2 + if ! lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then + echo "iproxy did not start; see /tmp/openphone-install-agentd-iproxy.log" >&2 + exit 30 + fi +} + +if [[ -z "$package" ]]; then + package="$(ls -t "$repo_root"/agentd/packages/com.openphone.agentd_*_iphoneos-arm64.deb 2>/dev/null | head -n 1 || true)" +fi + +if [[ -z "$package" || ! -f "$package" ]]; then + echo "openphone-agentd package not found. Run: (cd ios/agentd && make package)" >&2 + exit 1 +fi + +ssh_target="$user@$host" + +start_iproxy_if_requested + +expect_scp() { + OPENPHONE_IOS_PASSWORD="$password" expect -f - -- "$package" "$ssh_target:$remote_deb" "$port" "$known_hosts" <<'EXPECT' +set timeout 120 +set local_path [lindex $argv 0] +set remote_path [lindex $argv 1] +set port [lindex $argv 2] +set known_hosts [lindex $argv 3] +spawn scp -P $port -o StrictHostKeyChecking=no -o UserKnownHostsFile=$known_hosts $local_path $remote_path +expect { + -nocase -re "yes/no" { send "yes\r"; exp_continue } + -nocase -re "password.*:" { send "$env(OPENPHONE_IOS_PASSWORD)\r"; exp_continue } + eof {} + timeout { exit 124 } +} +catch wait result +exit [lindex $result 3] +EXPECT +} + +expect_ssh_tty() { + local remote_cmd="$1" + OPENPHONE_IOS_PASSWORD="$password" expect -f - -- "$port" "$known_hosts" "$ssh_target" "$remote_cmd" <<'EXPECT' +set timeout 120 +set port [lindex $argv 0] +set known_hosts [lindex $argv 1] +set target [lindex $argv 2] +set remote_cmd [lindex $argv 3] +spawn ssh -tt -p $port -o StrictHostKeyChecking=no -o UserKnownHostsFile=$known_hosts $target $remote_cmd +expect { + -nocase -re "yes/no" { send "yes\r"; exp_continue } + -nocase -re "password.*:" { send "$env(OPENPHONE_IOS_PASSWORD)\r"; exp_continue } + eof {} + timeout { exit 124 } +} +catch wait result +exit [lindex $result 3] +EXPECT +} + +plain_scp() { + scp -P "$port" -o StrictHostKeyChecking=no -o UserKnownHostsFile="$known_hosts" \ + "$package" "$ssh_target:$remote_deb" +} + +plain_ssh_tty() { + ssh -tt -p "$port" -o StrictHostKeyChecking=no -o UserKnownHostsFile="$known_hosts" \ + "$ssh_target" "$1" +} + +install_cmd=" +set -eu +sudo -v +sudo dpkg -i '$remote_deb' +sudo killall Preferences >/dev/null 2>&1 || true +sudo launchctl bootout system/com.openphone.protecteddatahelper >/dev/null 2>&1 || true +sudo launchctl unload /var/jb/Library/LaunchDaemons/com.openphone.protected-data-helper.plist >/dev/null 2>&1 || true +sudo launchctl remove com.openphone.protected-data-helper >/dev/null 2>&1 || true +sudo rm -f /var/jb/Library/LaunchDaemons/com.openphone.protected-data-helper.plist +sudo killall openphone-protected-data-helper >/dev/null 2>&1 || true +sudo rm -f /var/mobile/Library/OpenPhone/protected-data-helper/run/agentd.sock +sudo mkdir -p /var/mobile/Library/OpenPhone/protected-data-helper +sudo launchctl bootstrap system /var/jb/Library/LaunchDaemons/com.openphone.protecteddatahelper.plist >/dev/null 2>&1 || true +sudo launchctl kickstart -k system/com.openphone.protecteddatahelper >/dev/null 2>&1 || true +sudo launchctl unload /var/jb/Library/LaunchDaemons/com.openphone.agentd.plist >/dev/null 2>&1 || true +sudo launchctl load -w /var/jb/Library/LaunchDaemons/com.openphone.agentd.plist +sleep 2 +/var/jb/usr/local/bin/openphone-agentctl +" + +echo "Installing $package to $ssh_target:$remote_deb" +if [[ -n "$password" ]]; then + expect_scp + expect_ssh_tty "$install_cmd" +else + plain_scp + plain_ssh_tty "$install_cmd" +fi diff --git a/ios/tools/mac/agentd/install-agentd.sh b/ios/tools/mac/agentd/install-agentd.sh new file mode 100755 index 0000000..fb7f47d --- /dev/null +++ b/ios/tools/mac/agentd/install-agentd.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +host="${OPENPHONE_IOS_HOST:-127.0.0.1}" +port="${OPENPHONE_IOS_SSH_PORT:-22}" +user="${OPENPHONE_IOS_USER:-mobile}" + +agentd_bin="${OPENPHONE_AGENTD_BIN:-}" +agentctl_bin="${OPENPHONE_AGENTCTL_BIN:-}" + +if [[ -z "$agentd_bin" ]]; then + for candidate in \ + "$repo_root/agentd/.theos/obj/debug/openphone-agentd" \ + "$repo_root/agentd/.theos/obj/openphone-agentd"; do + if [[ -x "$candidate" ]]; then + agentd_bin="$candidate" + break + fi + done +fi + +if [[ -z "$agentctl_bin" ]]; then + for candidate in \ + "$repo_root/agentd/.theos/obj/debug/openphone-agentctl" \ + "$repo_root/agentd/.theos/obj/openphone-agentctl"; do + if [[ -x "$candidate" ]]; then + agentctl_bin="$candidate" + break + fi + done +fi + +if [[ -z "$agentd_bin" || ! -x "$agentd_bin" ]]; then + echo "openphone-agentd binary not found. Build ios/agentd first or set OPENPHONE_AGENTD_BIN." >&2 + exit 1 +fi + +if [[ -z "$agentctl_bin" || ! -x "$agentctl_bin" ]]; then + echo "openphone-agentctl binary not found. Build ios/agentd first or set OPENPHONE_AGENTCTL_BIN." >&2 + exit 1 +fi + +ssh_target="$user@$host" +ssh_base=(ssh -p "$port" "$ssh_target") +scp_base=(scp -P "$port") + +"${ssh_base[@]}" 'mkdir -p /tmp/openphone-agentd-upload' +"${scp_base[@]}" "$agentd_bin" "$ssh_target:/tmp/openphone-agentd-upload/openphone-agentd" +"${scp_base[@]}" "$agentctl_bin" "$ssh_target:/tmp/openphone-agentd-upload/openphone-agentctl" +"${scp_base[@]}" "$repo_root/agentd/launchd/com.openphone.agentd.plist" \ + "$ssh_target:/tmp/openphone-agentd-upload/com.openphone.agentd.plist" + +"${ssh_base[@]}" ' +set -eu +mkdir -p /var/mobile/Library/OpenPhone +if [ -d /var/jb/usr/local/bin ] && [ -w /var/jb/usr/local/bin ]; then + install -m 0755 /tmp/openphone-agentd-upload/openphone-agentd /var/jb/usr/local/bin/openphone-agentd + install -m 0755 /tmp/openphone-agentd-upload/openphone-agentctl /var/jb/usr/local/bin/openphone-agentctl +else + sudo mkdir -p /var/jb/usr/local/bin + sudo install -m 0755 /tmp/openphone-agentd-upload/openphone-agentd /var/jb/usr/local/bin/openphone-agentd + sudo install -m 0755 /tmp/openphone-agentd-upload/openphone-agentctl /var/jb/usr/local/bin/openphone-agentctl +fi + +if [ -d /var/jb/Library/LaunchDaemons ] && [ -w /var/jb/Library/LaunchDaemons ]; then + install -m 0644 /tmp/openphone-agentd-upload/com.openphone.agentd.plist /var/jb/Library/LaunchDaemons/com.openphone.agentd.plist +else + sudo mkdir -p /var/jb/Library/LaunchDaemons + sudo install -m 0644 /tmp/openphone-agentd-upload/com.openphone.agentd.plist /var/jb/Library/LaunchDaemons/com.openphone.agentd.plist +fi + +launchctl unload /var/jb/Library/LaunchDaemons/com.openphone.agentd.plist >/dev/null 2>&1 || true +launchctl load -w /var/jb/Library/LaunchDaemons/com.openphone.agentd.plist +sleep 1 +/var/jb/usr/local/bin/openphone-agentctl +' diff --git a/ios/tools/mac/agentd/pull-crashes.sh b/ios/tools/mac/agentd/pull-crashes.sh new file mode 100755 index 0000000..022a11e --- /dev/null +++ b/ios/tools/mac/agentd/pull-crashes.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Pull openphone-agentd crash logs + matching JetsamEvent files from the phone +# into artifacts/crashes// for triage. +# +# Uses the same env vars as the other agentd tools: +# OPENPHONE_IOS_HOST (default 127.0.0.1) +# OPENPHONE_IOS_SSH_PORT (default 2224) +# OPENPHONE_IOS_USER (default mobile) +# OPENPHONE_IOS_PASSWORD (required for expect-based auth) +# OPENPHONE_IOS_KNOWN_HOSTS (default /tmp/openphone_ios14_known_hosts) + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +host="${OPENPHONE_IOS_HOST:-127.0.0.1}" +port="${OPENPHONE_IOS_SSH_PORT:-2224}" +user="${OPENPHONE_IOS_USER:-mobile}" +password="${OPENPHONE_IOS_PASSWORD:-}" +known_hosts="${OPENPHONE_IOS_KNOWN_HOSTS:-/tmp/openphone_ios14_known_hosts}" + +if [[ -z "$password" ]]; then + echo "OPENPHONE_IOS_PASSWORD not set" >&2 + exit 1 +fi + +stamp="$(date +%Y%m%d-%H%M%S)" +out_dir="$repo_root/artifacts/crashes/$stamp" +mkdir -p "$out_dir" + +echo "Pulling crash reports into $out_dir ..." + +run_remote() { + OPENPHONE_IOS_PASSWORD="$password" expect <&1 +set timeout 30 +log_user 1 +spawn -noecho ssh -p $port -o ConnectTimeout=8 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=$known_hosts $user@$host "$1" +expect { -nocase -re "assword.*:" { send "\$env(OPENPHONE_IOS_PASSWORD)\r"; exp_continue } eof } +EOF +} + +# List crash files. +run_remote 'ls -t /var/mobile/Library/Logs/CrashReporter/ 2>/dev/null | grep -iE "openphone-agentd|JetsamEvent" | head -40' \ + > "$out_dir/file-list.txt" + +# Pull each file via scp. +while IFS= read -r file; do + [[ -z "$file" ]] && continue + [[ "$file" =~ ^\( ]] && continue + echo "Fetching $file" + OPENPHONE_IOS_PASSWORD="$password" expect <&1 >/dev/null || true +set timeout 60 +log_user 0 +spawn -noecho scp -P $port -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=$known_hosts $user@$host:/var/mobile/Library/Logs/CrashReporter/$file $out_dir/ +expect { -nocase -re "assword.*:" { send "\$env(OPENPHONE_IOS_PASSWORD)\r"; exp_continue } eof } +EOF +done < "$out_dir/file-list.txt" + +echo "Done. Files:" +ls -la "$out_dir" | head diff --git a/ios/tools/mac/agentd/restart-agentd.sh b/ios/tools/mac/agentd/restart-agentd.sh new file mode 100755 index 0000000..d31a227 --- /dev/null +++ b/ios/tools/mac/agentd/restart-agentd.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +host="${OPENPHONE_IOS_HOST:-127.0.0.1}" +port="${OPENPHONE_IOS_SSH_PORT:-22}" +user="${OPENPHONE_IOS_USER:-mobile}" + +ssh -p "$port" "$user@$host" ' +set -eu +plist=/var/jb/Library/LaunchDaemons/com.openphone.agentd.plist +launchctl unload "$plist" >/dev/null 2>&1 || true +launchctl load -w "$plist" +sleep 1 +/var/jb/usr/local/bin/openphone-agentctl +' diff --git a/ios/tools/mac/agentd/smoke-agentd-local.sh b/ios/tools/mac/agentd/smoke-agentd-local.sh new file mode 100755 index 0000000..4774bca --- /dev/null +++ b/ios/tools/mac/agentd/smoke-agentd-local.sh @@ -0,0 +1,1903 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +work="${OPENPHONE_AGENTD_SMOKE_DIR:-/tmp/openphone-agentd-smoke}" +broker_pid="" +pid="" + +rm -rf "$work" +mkdir -p "$work" + +python3 - <<'PY' "$work/model-broker-port.txt" >"$work/model-broker.log" 2>&1 & +import http.server +import json +import pathlib +import sys + +port_path = pathlib.Path(sys.argv[1]) + +class Handler(http.server.BaseHTTPRequestHandler): + counter = 0 + + def log_message(self, fmt, *args): + return + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0") or "0") + body = self.rfile.read(length) + try: + request = json.loads(body.decode("utf-8")) + except Exception: + request = {} + Handler.counter += 1 + if Handler.counter == 1: + decision = { + "schema": "openphone.model_decision.v1", + "thought": "broker fixture pauses before finishing", + "tool": "wait", + "arguments": {"duration_ms": 10}, + "expected_visible_change": "none", + "confidence": 0.9, + } + else: + decision = { + "schema": "openphone.model_decision.v1", + "thought": "broker fixture finished", + "tool": "finish_task", + "arguments": {"summary": "Broker fixture model loop completed."}, + "expected_visible_change": "none", + "confidence": 0.95, + } + response = { + "schema": "openphone.model_response.v1", + "decision": decision, + "metadata": { + "fixture": True, + "step": request.get("step"), + "request_schema": request.get("schema"), + }, + "usage": {"requests": Handler.counter}, + } + encoded = json.dumps(response).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + +server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) +port_path.write_text(str(server.server_port), encoding="utf-8") +server.serve_forever() +PY +broker_pid=$! +for _ in $(seq 1 50); do + if [[ -s "$work/model-broker-port.txt" ]]; then + break + fi + sleep 0.1 +done +if [[ ! -s "$work/model-broker-port.txt" ]]; then + echo "model broker fixture did not start" >&2 + exit 1 +fi +broker_port="$(cat "$work/model-broker-port.txt")" + +cleanup() { + if [[ -n "$pid" ]]; then + kill "$pid" >/dev/null 2>&1 || true + fi + if [[ -n "$broker_pid" ]]; then + kill "$broker_pid" >/dev/null 2>&1 || true + fi + if [[ -n "$pid" ]]; then + wait "$pid" >/dev/null 2>&1 || true + fi + if [[ -n "$broker_pid" ]]; then + wait "$broker_pid" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +xcrun clang -fobjc-arc -framework Foundation -framework AVFoundation \ + -framework AudioToolbox -framework Speech \ + -lcompression -lsqlite3 \ + "$repo_root/agentd/src/main.m" \ + -o "$work/openphone-agentd" +xcrun clang -fobjc-arc -framework Foundation \ + "$repo_root/agentd/src/agentctl.m" \ + -o "$work/openphone-agentctl" + +mkdir -p "$work/store/config" +python3 - <<'PY' "$work/store/config/contacts-fixture.json" +import json +import pathlib +import sys + +fixture = { + "schema": "openphone.contacts_fixture.v1", + "contacts": [ + { + "contact_id": "fixture-ada", + "display_name": "Ada Lovelace", + "given_name": "Ada", + "family_name": "Lovelace", + "organization": "Analytical Engines", + "phone_numbers": ["+1555010101"], + "emails": ["ada@example.test"], + }, + { + "contact_id": "fixture-grace", + "display_name": "Grace Hopper", + "emails": ["grace@example.test"], + }, + ], +} +pathlib.Path(sys.argv[1]).write_text(json.dumps(fixture), encoding="utf-8") +PY +python3 - <<'PY' "$work/store/config/calendar-fixture.json" +import json +import pathlib +import sys + +fixture = { + "schema": "openphone.calendar_fixture.v1", + "events": [ + { + "event_id": "fixture-calendar-standup", + "title": "OpenPhone Standup", + "calendar_title": "Engineering", + "location": "Room 101", + "notes": "Discuss iOS agent progress and provider validation.", + "start_at_ms": 1893456000000, + "end_at_ms": 1893457800000, + "all_day": False, + }, + { + "event_id": "fixture-calendar-review", + "title": "Roadmap Review", + "calendar_title": "Product", + "start_at_ms": 1893542400000, + "end_at_ms": 1893546000000, + }, + ], +} +pathlib.Path(sys.argv[1]).write_text(json.dumps(fixture), encoding="utf-8") +PY +python3 - <<'PY' "$work/store/config/calls-fixture.json" +import json +import pathlib +import sys + +fixture = { + "schema": "openphone.calls_fixture.v1", + "calls": [ + { + "call_id": "fixture-call-openphone", + "address": "+15550101234", + "display_name": "OpenPhone Test Call", + "service": "Phone", + "direction": "incoming", + "answered": True, + "duration_seconds": 120, + "start_at_ms": 1893459600000, + "call_type": "phone", + }, + { + "call_id": "fixture-call-facetime", + "address": "facetime@example.test", + "display_name": "FaceTime Fixture", + "service": "FaceTime", + "direction": "outgoing", + "answered": False, + "duration_seconds": 0, + "start_at_ms": 1893546000000, + "call_type": "facetime", + }, + ], +} +pathlib.Path(sys.argv[1]).write_text(json.dumps(fixture), encoding="utf-8") +PY +python3 - <<'PY' "$work/store/config/messages-fixture.json" +import json +import pathlib +import sys + +fixture = { + "schema": "openphone.messages_fixture.v1", + "messages": [ + { + "message_id": "fixture-message-openphone", + "guid": "fixture-message-openphone-guid", + "handle": "+15550105555", + "service": "iMessage", + "direction": "incoming", + "text": "OpenPhone message fixture says hello from local smoke.", + "subject": "", + "sent_at_ms": 1893463200000, + "read": True, + "delivered": True, + }, + { + "message_id": "fixture-message-roadmap", + "handle": "roadmap@example.test", + "service": "SMS", + "direction": "outgoing", + "text": "Roadmap fixture message for provider validation.", + "sent_at_ms": 1893549600000, + }, + ], +} +pathlib.Path(sys.argv[1]).write_text(json.dumps(fixture), encoding="utf-8") +PY + +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentd" \ + >"$work/stdout.log" 2>"$work/stderr.log" & +pid=$! + +sleep 1 + +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + >"$work/health.json" +mkdir -p "$work/store/springboard" +python3 - <<'PY' "$work/store/springboard/state.json" +import json +import pathlib +import sys +import time + +path = pathlib.Path(sys.argv[1]) +path.write_text(json.dumps({ + "schema": "openphone.springboard_state.v1", + "timestamp_ms": int(time.time() * 1000), + "provider": "OpenPhoneVolumeTrigger.SpringBoardState", + "foreground_app": "com.apple.mobilesafari", + "foreground_source": "smoke_fixture", + "active_scene_count": 1, + "scene_count": 1, + "scenes": [{ + "provider": "smoke_fixture", + "bundle_id": "com.apple.mobilesafari", + "identifier": "sceneID:com.apple.mobilesafari-default", + "isForegroundActive": True + }], + "display": { + "orientation": 1, + "orientation_name": "portrait" + }, + "ui_tree": { + "status": "ok", + "provider": "SpringBoard.UIKitAccessibility", + "scope": "springboard_only", + "window_count": 1, + "element_count": 1, + "text_count": 1, + "visible_text": ["Safari"], + "interactive_elements": [{ + "id": "springboard-0-0", + "kind": "button", + "label": "Safari", + "bounds": [12, 34, 56, 78], + "enabled": True, + "focused": False, + "window_id": 0, + "sensitive": False, + "risk_hint": "springboard_only" + }], + "windows": [{ + "id": 0, + "type": 0, + "focused": True, + "active": True, + "bounds": [0, 0, 430, 932] + }] + }, + "source": "smoke" +})) +PY +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_screen >"$work/get_screen_springboard_state.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + tap_element springboard-0-0 >"$work/tap_element.json" +app_ui_publish_request="$(python3 - <<'PY' +import json, time +state = { + "schema": "openphone.app_ui_state.v1", + "status": "ok", + "provider": "OpenPhoneAppIntrospector.UIKitAccessibility", + "bundle_id": "com.apple.mobilesafari", + "process_name": "MobileSafari", + "pid": 4242, + "timestamp_ms": int(time.time() * 1000), + "application_state": 0, + "application_state_name": "active", + "ui_tree": { + "status": "ok", + "provider": "OpenPhoneAppIntrospector.UIKitAccessibility", + "scope": "app_process", + "bundle_id": "com.apple.mobilesafari", + "window_count": 1, + "element_count": 2, + "text_count": 2, + "visible_text": ["Example Domain", "Address"], + "interactive_elements": [{ + "id": "app-com.apple.mobilesafari-0-0", + "kind": "text_field", + "label": "Address", + "bounds": [16, 50, 300, 42], + "enabled": True, + "focused": False, + "window_id": 0, + "source_bundle_id": "com.apple.mobilesafari", + "scope": "app_process", + "sensitive": False, + "risk_hint": "app_process" + }, { + "id": "app-com.apple.mobilesafari-0-1", + "kind": "button", + "label": "Share", + "bounds": [360, 50, 44, 42], + "enabled": True, + "focused": False, + "window_id": 0, + "source_bundle_id": "com.apple.mobilesafari", + "scope": "app_process", + "sensitive": False, + "risk_hint": "app_process" + }], + "windows": [{ + "id": 0, + "type": 0, + "focused": True, + "active": True, + "bounds": [0, 0, 430, 932] + }] + }, + "source": "smoke" +} +print(json.dumps({ + "command": "app_ui_publish", + "transport": "local_smoke_agentctl", + "state": state +})) +PY +)" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$app_ui_publish_request" >"$work/app_ui_publish.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_screen >"$work/get_screen_app_ui_state.json" +python3 - <<'PY' "$work" >"$work/app_input_type_text_bridge.log" 2>&1 & +import json +import pathlib +import socket +import sys +import time + +work = pathlib.Path(sys.argv[1]) +text = "SmokeText" + +def intake_request(payload): + encoded = json.dumps(payload).encode("utf-8") + b"\n" + with socket.create_connection(("127.0.0.1", 27631), timeout=2) as sock: + sock.sendall(encoded) + sock.shutdown(socket.SHUT_WR) + chunks = [] + while True: + chunk = sock.recv(65536) + if not chunk: + break + chunks.append(chunk) + if b"\n" in chunk: + break + return json.loads(b"".join(chunks).decode("utf-8")) + +request = None +deadline = time.time() + 8 +while time.time() < deadline: + poll = intake_request({ + "command": "app_input_poll", + "bundle_id": "com.apple.mobilesafari", + "scope": "app_process", + }) + if poll.get("status") == "ok": + request = poll["request"] + break + time.sleep(0.05) + +if request is None: + raise SystemExit("timed out waiting for app input request") + +response = { + "status": "ok", + "provider": "OpenPhoneAppIntrospector.AppInput", + "source": "app_process", + "action_type": "type_text", + "activation_method": "text_input_insert", + "target_class": "UISearchBarTextField", + "text_length": len(text), + "before_text_length": 0, + "after_text_length": len(text), +} +complete = intake_request({ + "command": "app_input_complete", + "request_id": request["request_id"], + "bundle_id": "com.apple.mobilesafari", + "response": response, +}) +if complete.get("status") != "ok": + raise SystemExit(f"app input complete failed: {complete}") +PY +app_input_bridge_pid=$! +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"execute_action","action":{"type":"type_text","element_id":"app-com.apple.mobilesafari-0-0","text":"SmokeText","reason":"local smoke app-process verified text input"}}' \ + >"$work/app_input_type_text.json" +wait "$app_input_bridge_pid" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + memory_save "user prefers concise OpenPhone status updates" preference user >"$work/memory_save.json" +memory_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["memory"]["memory_id"])' "$work/memory_save.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + memory_update "$memory_id" "user prefers concise OpenPhone validation details" preference user \ + >"$work/memory_update.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + memory_save "user prefers terse OpenPhone summaries" preference user >"$work/memory_save_merge_source.json" +merge_source_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["memory"]["memory_id"])' "$work/memory_save_merge_source.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + memory_merge "$memory_id" "$merge_source_id" "user prefers concise OpenPhone validation details and terse summaries" \ + >"$work/memory_merge.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + memory_save "temporary OpenPhone memory delete fixture" fact user >"$work/memory_save_delete_fixture.json" +delete_memory_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["memory"]["memory_id"])' "$work/memory_save_delete_fixture.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + memory_delete "$delete_memory_id" "agentctl memory_delete smoke cleanup" >"$work/memory_delete.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"clipboard_write","text":"OpenPhone clipboard smoke value","reason":"local smoke clipboard write"}' \ + >"$work/clipboard_write.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"clipboard_read","max_chars":200,"reason":"local smoke clipboard read"}' \ + >"$work/clipboard_read.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"contacts_search","query":"Ada","limit":5,"reason":"local smoke contacts search"}' \ + >"$work/contacts_search.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"calendar_search","query":"Standup","limit":5,"reason":"local smoke calendar search"}' \ + >"$work/calendar_search.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"calls_search","query":"OpenPhone","limit":5,"reason":"local smoke call-history search"}' \ + >"$work/calls_search.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"messages_search","query":"OpenPhone","limit":5,"reason":"local smoke messages search"}' \ + >"$work/messages_search.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$(python3 - <<'PY' +import json +# Over-length title/body carrying a unique marker whose *tail* must never be +# retained verbatim, proving the notification provider only keeps bounded +# previews (title<=200, body<=1000) and no full payload leaks to any store. +tail_marker = "NOTIFYPIItail_+15559998888_leak@example.test" +title = "OpenPhone Notification " + ("A" * 250) + tail_marker +body = "OpenPhone body preview head. " + ("B" * 1200) + tail_marker +print(json.dumps({ + "command": "notification_ingest", + "bundle_id": "com.openphone.smoke", + "title": title, + "subtitle": "smoke subtitle", + "body": body, + "notification_id": "smoke-notif-1", + "thread_id": "smoke-thread-1", +})) +PY +)" >"$work/notification_ingest.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"notification_list","limit":10}' >"$work/notification_list.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + context_search notification 5 >"$work/context_search_notification.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + memory_search terse 5 >"$work/memory_search.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + context_search concise 5 >"$work/context_search.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + context_search clipboard 5 >"$work/context_search_clipboard.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + context_search contacts 5 >"$work/context_search_contacts.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + context_search calendar 5 >"$work/context_search_calendar.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + context_search calls 5 >"$work/context_search_calls.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + context_search messages 5 >"$work/context_search_messages.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + commitment_create "follow up about OpenPhone iOS smoke" >"$work/commitment_create.json" +commitment_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["commitment_id"])' "$work/commitment_create.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + commitment_search OpenPhone 5 >"$work/commitment_search.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + commitment_update_status "$commitment_id" completed >"$work/commitment_update_status.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$(python3 - <<'PY' +import json, time +print(json.dumps({ + "command": "commitment_create", + "title": "due commitment smoke", + "description": "summarize due commitment smoke", + "trigger_type": "time", + "trigger_spec": { + "prompt": "summarize due commitment smoke", + "delivery": {"mode": "notification"} + }, + "due_at": int(time.time() * 1000) - 1000, + "reason": "local smoke due commitment" +})) +PY +)" >"$work/commitment_due_create.json" +commitment_due_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["commitment_id"])' "$work/commitment_due_create.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"commitment_run_due","limit":5,"source":"local_smoke_commitment_scheduler"}' \ + >"$work/commitment_run_due.json" +commitment_due_job_id="$(python3 - <<'PY' "$work/commitment_run_due.json" "$commitment_due_id" +import json, sys +data = json.load(open(sys.argv[1])) +target = f"ios-commitment-{sys.argv[2]}" +for entry in data.get("commitments", []): + if entry.get("commitment_id") == target: + print(entry["job_id"]) + break +else: + raise SystemExit(f"commitment job not found for {target}: {data}") +PY +)" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "{\"command\":\"background_job_run_due\",\"job_id\":\"$commitment_due_job_id\",\"limit\":1,\"max_steps\":1,\"max_duration_ms\":15000,\"mode\":\"deterministic\",\"source\":\"local_smoke_commitment_scheduler_job\",\"materialize_commitments\":false,\"materialize_watchers\":false,\"repair_stuck\":false}" \ + >"$work/commitment_due_job_run.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + commitment_search "due commitment smoke" 5 >"$work/commitment_due_after.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$(python3 - <<'PY' +import json, time +print(json.dumps({ + "command": "watcher_create", + "title": "watch smoke condition", + "source": "time", + "type": "time", + "next_run_at": int(time.time() * 1000) - 1000, + "prompt": "summarize watcher smoke condition", + "reason": "agentctl watcher scheduler smoke" +})) +PY +)" >"$work/watcher_create.json" +watcher_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["watcher_id"])' "$work/watcher_create.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + watcher_list smoke 5 >"$work/watcher_list.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"watcher_run_due","limit":5,"source":"local_smoke"}' >"$work/watcher_materialize_due.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_run_due 5 1 15000 >"$work/watcher_run_due.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + watcher_list smoke 5 >"$work/watcher_after_run.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + watcher_stop "$watcher_id" >"$work/watcher_stop.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$(python3 - <<'PY' +import json, time +print(json.dumps({ + "command": "watcher_create", + "title": "recurring watcher smoke", + "source": "time", + "type": "time", + "next_run_at": int(time.time() * 1000) - 1000, + "interval_ms": 5, + "recurring": True, + "prompt": "summarize recurring watcher smoke", + "reason": "local smoke recurring watcher" +})) +PY +)" >"$work/watcher_recurring_create.json" +recurring_watcher_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["watcher_id"])' "$work/watcher_recurring_create.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"watcher_run_due","limit":5,"source":"local_smoke_recurring_watcher_first"}' \ + >"$work/watcher_recurring_first.json" +sleep 0.05 +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"watcher_run_due","limit":5,"source":"local_smoke_recurring_watcher_second"}' \ + >"$work/watcher_recurring_second.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + watcher_list "recurring watcher smoke" 5 >"$work/watcher_recurring_after.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + watcher_stop "$recurring_watcher_id" >"$work/watcher_recurring_stop.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$(python3 - <<'PY' +import json, time +print(json.dumps({ + "command": "watcher_create", + "title": "watcher stuck repair smoke", + "source": "time", + "type": "time", + "next_run_at": int(time.time() * 1000) + 600000, + "prompt": "summarize watcher stuck repair fixture", + "reason": "local smoke watcher stuck repair fixture" +})) +PY +)" >"$work/watcher_repair_create.json" +repair_watcher_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["watcher_id"])' "$work/watcher_repair_create.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "{\"command\":\"watcher_debug_mark_running\",\"watcher_id\":$repair_watcher_id,\"validation\":true,\"age_ms\":600000,\"source\":\"local_smoke_watcher_stuck_repair\"}" \ + >"$work/watcher_debug_mark_running.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"watcher_repair_stuck","limit":5,"stale_after_ms":1000,"source":"local_smoke_watcher_stuck_repair"}' \ + >"$work/watcher_repair_stuck.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"watcher_run_due","limit":5,"source":"local_smoke_watcher_repair"}' \ + >"$work/watcher_repair_run_due.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + watcher_list "watcher stuck repair smoke" 5 >"$work/watcher_repair_after_run.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + watcher_stop "$repair_watcher_id" >"$work/watcher_repair_stop.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_create "smoke background job" "summarize durable stores" >"$work/background_job_create.json" +job_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["job_id"])' "$work/background_job_create.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_run_due 5 1 15000 >"$work/background_job_run_due.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_list durable 5 >"$work/background_job_list.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_stop "$job_id" >"$work/background_job_stop.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$(python3 - <<'PY' +import json, time +print(json.dumps({ + "command": "background_job_create", + "title": "recurring background job", + "prompt": "summarize recurring background job", + "next_run_at": int(time.time() * 1000) - 1000, + "interval_ms": 50, + "recurring": True, + "reason": "local smoke recurring background job" +})) +PY +)" >"$work/background_job_recurring_create.json" +recurring_job_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["job_id"])' "$work/background_job_recurring_create.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "{\"command\":\"background_job_run_due\",\"job_id\":$recurring_job_id,\"limit\":1,\"max_steps\":1,\"max_duration_ms\":15000,\"mode\":\"deterministic\",\"source\":\"local_smoke_recurring_job\"}" \ + >"$work/background_job_recurring_run_due.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_list "recurring background job" 5 >"$work/background_job_recurring_list.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_stop "$recurring_job_id" >"$work/background_job_recurring_stop.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$(python3 - <<'PY' +import json, time +print(json.dumps({ + "command": "background_job_create", + "title": "nonrecurring interval background job", + "prompt": "summarize nonrecurring interval background job", + "next_run_at": int(time.time() * 1000) - 1000, + "interval_ms": 50, + "recurring": False, + "reason": "local smoke nonrecurring interval background job" +})) +PY +)" >"$work/background_job_nonrecurring_interval_create.json" +nonrecurring_interval_job_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["job_id"])' "$work/background_job_nonrecurring_interval_create.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "{\"command\":\"background_job_run_due\",\"job_id\":$nonrecurring_interval_job_id,\"limit\":1,\"max_steps\":1,\"max_duration_ms\":15000,\"mode\":\"deterministic\",\"source\":\"local_smoke_nonrecurring_interval_job\"}" \ + >"$work/background_job_nonrecurring_interval_run_due.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_list "nonrecurring interval background job" 5 >"$work/background_job_nonrecurring_interval_list.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$(python3 - <<'PY' +import json, time +print(json.dumps({ + "command": "background_job_create", + "title": "recurring backoff background job", + "prompt": "model provider should be unavailable for this smoke job", + "next_run_at": int(time.time() * 1000) - 1000, + "interval_ms": 50, + "recurring": True, + "reason": "local smoke recurring background job backoff" +})) +PY +)" >"$work/background_job_backoff_create.json" +backoff_job_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["job_id"])' "$work/background_job_backoff_create.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "{\"command\":\"background_job_run_due\",\"job_id\":$backoff_job_id,\"limit\":1,\"max_steps\":1,\"max_duration_ms\":15000,\"mode\":\"model\",\"source\":\"local_smoke_recurring_job_backoff\"}" \ + >"$work/background_job_backoff_run_due.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_list "recurring backoff background job" 5 >"$work/background_job_backoff_list.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_stop "$backoff_job_id" >"$work/background_job_backoff_stop.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$(python3 - <<'PY' +import json, time +print(json.dumps({ + "command": "background_job_create", + "title": "stuck repair background job", + "prompt": "summarize stuck repair fixture", + "next_run_at": int(time.time() * 1000) + 600000, + "reason": "local smoke stuck repair fixture" +})) +PY +)" >"$work/background_job_repair_create.json" +repair_job_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["job_id"])' "$work/background_job_repair_create.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "{\"command\":\"background_job_debug_mark_running\",\"job_id\":$repair_job_id,\"validation\":true,\"age_ms\":600000,\"source\":\"local_smoke_stuck_repair\"}" \ + >"$work/background_job_debug_mark_running.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"background_job_repair_stuck","limit":5,"stale_after_ms":1000,"source":"local_smoke_stuck_repair"}' \ + >"$work/background_job_repair_stuck.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_run_due 5 1 15000 >"$work/background_job_repair_run_due.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_list "stuck repair" 10 >"$work/background_job_repair_list.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + background_job_stop "$repair_job_id" >"$work/background_job_repair_stop.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + agent_status >"$work/agent_status_initial.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + agent_control pause "local smoke pause" >"$work/agent_control_pause.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + agent_status >"$work/agent_status_paused.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"hardware_trigger","trigger":"volume_up_down_combo","source":"local_smoke","reason":"local smoke paused hardware trigger","run_task":true,"create_background_job":false,"dedupe":false}' \ + >"$work/hardware_trigger_paused.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + agent_control resume >"$work/agent_control_resume.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + agent_status >"$work/agent_status_resumed.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"agent_control","hardware_triggers_enabled":false,"reason":"local smoke disable hardware triggers","source":"local_smoke"}' \ + >"$work/agent_control_disable_hardware.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"hardware_trigger","trigger":"volume_up_down_combo","source":"local_smoke","reason":"local smoke disabled hardware trigger","run_task":true,"create_background_job":false,"dedupe":false}' \ + >"$work/hardware_trigger_disabled.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"agent_control","hardware_triggers_enabled":true,"reason":"local smoke re-enable hardware triggers","source":"local_smoke"}' \ + >"$work/agent_control_enable_hardware.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"agent_control","yolo_enabled":false,"reason":"local smoke disable yolo","source":"local_smoke"}' \ + >"$work/agent_control_disable_yolo.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"hardware_trigger","trigger":"volume_up_down_combo","source":"local_smoke","reason":"local smoke yolo-disabled hardware trigger","run_task":true,"create_background_job":false,"dedupe":false}' \ + >"$work/hardware_trigger_yolo_disabled.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"agent_control","yolo_enabled":true,"reason":"local smoke re-enable yolo","source":"local_smoke"}' \ + >"$work/agent_control_enable_yolo.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + hardware_trigger volume_up_down_combo "local smoke hardware trigger" >"$work/hardware_trigger.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"execute_action","task_id":"smoke-redaction-task","action":{"type":"unlock_with_passcode","passcode":"000000"}}' \ + >"$work/unlock_with_passcode.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + run_task "open Safari" >"$work/run_task.json" +task_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["task_id"])' "$work/run_task.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + model_status >"$work/model_status.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"run_task","goal":"fixture model smoke","mode":"model","max_steps":3,"max_duration_ms":30000,"model_decisions":[{"schema":"openphone.model_decision.v1","thought":"pause before finishing","tool":"wait","arguments":{"duration_ms":10},"expected_visible_change":"none","confidence":0.9},{"schema":"openphone.model_decision.v1","thought":"fixture task is complete","tool":"finish_task","arguments":{"summary":"Fixture model loop completed."},"expected_visible_change":"none","confidence":0.95}]}' \ + >"$work/run_task_model.json" +model_task_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["task_id"])' "$work/run_task_model.json")" +fresh_app_ui_publish_request="$(APP_UI_PUBLISH_REQUEST="$app_ui_publish_request" python3 - <<'PY' +import json +import os +import time + +payload = json.loads(os.environ["APP_UI_PUBLISH_REQUEST"]) +payload["state"]["timestamp_ms"] = int(time.time() * 1000) +print(json.dumps(payload)) +PY +)" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "$fresh_app_ui_publish_request" >"$work/app_ui_publish_model_refresh.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"run_task","goal":"fixture model unverified tap smoke","mode":"model","max_steps":3,"max_duration_ms":30000,"model_decisions":[{"schema":"openphone.model_decision.v1","thought":"tap existing synthetic app element without changing state","tool":"tap_element","arguments":{"element_id":"app-com.apple.mobilesafari-0-0"},"expected_visible_change":"Safari address field is focused","confidence":0.8},{"schema":"openphone.model_decision.v1","thought":"finish after unverified dispatch sample","tool":"finish_task","arguments":{"summary":"Unverified dispatch sample completed."},"expected_visible_change":"none","confidence":0.95}]}' \ + >"$work/run_task_model_unverified.json" +model_unverified_task_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["task_id"])' "$work/run_task_model_unverified.json")" +python3 - <<'PY' "$work/store" >"$work/model-visible-mutator.log" 2>&1 & +import json +import pathlib +import socket +import sys +import time + +store = pathlib.Path(sys.argv[1]) +springboard_path = store / "springboard" / "state.json" +time.sleep(0.25) +springboard_path.write_text(json.dumps({ + "schema": "openphone.springboard_state.v1", + "timestamp_ms": int(time.time() * 1000), + "provider": "OpenPhoneVolumeTrigger.SpringBoardState", + "foreground_app": "com.apple.mobilesafari", + "foreground_source": "smoke_fixture_after_action", + "active_scene_count": 1, + "scene_count": 1, + "scenes": [{ + "provider": "smoke_fixture_after_action", + "bundle_id": "com.apple.mobilesafari", + "identifier": "sceneID:com.apple.mobilesafari-default", + "isForegroundActive": True + }], + "display": { + "orientation": 1, + "orientation_name": "portrait" + }, + "ui_tree": { + "status": "ok", + "provider": "SpringBoard.UIKitAccessibility", + "scope": "springboard_only", + "window_count": 1, + "element_count": 2, + "text_count": 2, + "visible_text": ["Safari", "Visible verifier changed"], + "interactive_elements": [{ + "id": "springboard-0-0", + "kind": "button", + "label": "Safari", + "bounds": [12, 34, 56, 78], + "enabled": True, + "focused": False, + "window_id": 0, + "sensitive": False, + "risk_hint": "springboard_only" + }, { + "id": "springboard-0-1", + "kind": "button", + "label": "Changed", + "bounds": [82, 34, 56, 78], + "enabled": True, + "focused": False, + "window_id": 0, + "sensitive": False, + "risk_hint": "springboard_only" + }], + "windows": [{ + "id": 0, + "type": 0, + "focused": True, + "active": True, + "bounds": [0, 0, 430, 932] + }] + }, + "source": "smoke" +}), encoding="utf-8") +state = { + "schema": "openphone.app_ui_state.v1", + "status": "ok", + "provider": "OpenPhoneAppIntrospector.UIKitAccessibility", + "bundle_id": "com.apple.mobilesafari", + "process_name": "MobileSafari", + "pid": 4242, + "timestamp_ms": int(time.time() * 1000), + "application_state": 0, + "application_state_name": "active", + "ui_tree": { + "status": "ok", + "provider": "OpenPhoneAppIntrospector.UIKitAccessibility", + "scope": "app_process", + "bundle_id": "com.apple.mobilesafari", + "window_count": 1, + "element_count": 2, + "text_count": 2, + "visible_text": ["Example Domain", "Visible verifier changed"], + "interactive_elements": [{ + "id": "app-com.apple.mobilesafari-0-0", + "kind": "text_field", + "label": "Address focused", + "bounds": [16, 50, 300, 42], + "enabled": True, + "focused": True, + "window_id": 0, + "source_bundle_id": "com.apple.mobilesafari", + "scope": "app_process", + "sensitive": False, + "risk_hint": "app_process" + }, { + "id": "app-com.apple.mobilesafari-0-1", + "kind": "button", + "label": "Done", + "bounds": [360, 50, 44, 42], + "enabled": True, + "focused": False, + "window_id": 0, + "source_bundle_id": "com.apple.mobilesafari", + "scope": "app_process", + "sensitive": False, + "risk_hint": "app_process" + }], + "windows": [{ + "id": 0, + "type": 0, + "focused": True, + "active": True, + "bounds": [0, 0, 430, 932] + }] + }, + "source": "smoke" +} +payload = json.dumps({ + "command": "app_ui_publish", + "transport": "local_smoke_tcp", + "state": state +}).encode("utf-8") + b"\n" +with socket.create_connection(("127.0.0.1", 27631), timeout=2) as sock: + sock.sendall(payload) + response = sock.recv(4096) + print(response.decode("utf-8", errors="replace")) +PY +visible_mutator_pid=$! +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"run_task","goal":"fixture model visible verifier smoke","mode":"model","max_steps":3,"max_duration_ms":30000,"model_decisions":[{"schema":"openphone.model_decision.v1","thought":"long press while synthetic screen changes","tool":"long_press","arguments":{"x":40,"y":73,"duration_ms":900},"expected_visible_change":"visible text changes to include Visible verifier changed","confidence":0.9},{"schema":"openphone.model_decision.v1","thought":"finish after verified visible change","tool":"finish_task","arguments":{"summary":"Visible verifier model loop completed."},"expected_visible_change":"none","confidence":0.95}]}' \ + >"$work/run_task_model_visible.json" +wait "$visible_mutator_pid" +model_visible_task_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["task_id"])' "$work/run_task_model_visible.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"start_task","goal":"cancelled model smoke"}' \ + >"$work/start_task_model_cancel.json" +model_cancel_task_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["task_id"])' "$work/start_task_model_cancel.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + stop_task "$model_cancel_task_id" "local smoke cancellation" \ + >"$work/stop_task_model_cancel.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "{\"command\":\"run_task\",\"task_id\":\"$model_cancel_task_id\",\"goal\":\"cancelled model smoke\",\"mode\":\"model\",\"max_steps\":3,\"max_duration_ms\":30000,\"model_decisions\":[{\"schema\":\"openphone.model_decision.v1\",\"thought\":\"should not execute after cancellation\",\"tool\":\"finish_task\",\"arguments\":{\"summary\":\"Should not finish.\"},\"expected_visible_change\":\"none\",\"confidence\":0.95}]}" \ + >"$work/run_task_model_cancelled.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"start_task","goal":"stale active task repair smoke"}' \ + >"$work/start_task_stale_repair.json" +stale_repair_task_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["task_id"])' "$work/start_task_stale_repair.json")" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"task_repair_stale_active","limit":5,"stale_after_ms":0,"source":"local_smoke_task_repair","reason":"local smoke stale active task repair"}' \ + >"$work/task_repair_stale_active.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_task "$stale_repair_task_id" >"$work/get_task_stale_repaired.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_trajectory "$stale_repair_task_id" 50 >"$work/get_task_stale_repaired_trajectory.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"run_task","goal":"fixture model parser repair smoke","mode":"model","max_steps":2,"max_duration_ms":30000,"model_decisions":["Here is the decision:\n```json\n{\"schema\":\"openphone.model_decision.v1\",\"thought\":\"wrapped finish\",\"tool\":\"finish_task\",\"arguments\":{\"summary\":\"Parser repair model loop completed.\"},\"expected_visible_change\":\"none\",\"confidence\":0.96}\n```\n"]}' \ + >"$work/run_task_model_repaired.json" +model_repaired_task_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["task_id"])' "$work/run_task_model_repaired.json")" +broker_endpoint="http://127.0.0.1:${broker_port}/decision" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + "{\"command\":\"model_configure\",\"mode\":\"broker\",\"endpoint_url\":\"$broker_endpoint\",\"model\":\"local-smoke-broker\",\"enabled\":true,\"credential_required\":false,\"timeout_ms\":30000,\"max_steps\":3,\"max_duration_ms\":30000,\"reason\":\"local smoke broker fixture\"}" \ + >"$work/model_configure.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + model_status >"$work/model_status_configured.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"run_task","goal":"broker model smoke","mode":"model","max_steps":3,"max_duration_ms":30000}' \ + >"$work/run_task_model_broker.json" +broker_model_task_id="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["task_id"])' "$work/run_task_model_broker.json")" +mkdir -p "$work/store/config" +printf '{"credential":"local-smoke-openai-realtime-value"}\n' >"$work/store/config/model-credential.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + '{"command":"model_configure","mode":"openai_realtime2","enabled":true,"credential_required":true,"max_steps":40,"max_duration_ms":600000,"reason":"local smoke realtime2 config"}' \ + >"$work/model_configure_realtime2.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + model_status >"$work/model_status_realtime2.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + list_tasks 10 >"$work/list_tasks.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_task "$task_id" >"$work/get_task.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_trajectory "$task_id" 20 >"$work/get_trajectory.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_trajectory "$model_task_id" 50 >"$work/get_model_trajectory.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_trajectory "$model_unverified_task_id" 50 >"$work/get_model_unverified_trajectory.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_trajectory "$model_visible_task_id" 50 >"$work/get_model_visible_trajectory.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_trajectory "$model_cancel_task_id" 50 >"$work/get_model_cancel_trajectory.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_trajectory "$model_repaired_task_id" 50 >"$work/get_model_repaired_trajectory.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_trajectory "$broker_model_task_id" 80 >"$work/get_model_broker_trajectory.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_trajectory smoke-redaction-task 20 >"$work/get_redaction_trajectory.json" +OPENPHONE_AGENTD_STORE="$work/store" "$work/openphone-agentctl" \ + get_audit 100 >"$work/get_audit.json" +"$repo_root/tools/mac/agentd/validate-agentd-store.py" \ + "$work/store" --require-task-artifacts >"$work/validate_store.json" + +OPENPHONE_AGENTD_SMOKE_DIR="$work" python3 - <<'PY' +import json +import os +import pathlib + +base = pathlib.Path(os.environ['OPENPHONE_AGENTD_SMOKE_DIR']) +health = json.loads(base.joinpath('health.json').read_text()) +get_screen_springboard = json.loads(base.joinpath('get_screen_springboard_state.json').read_text()) +app_ui_publish = json.loads(base.joinpath('app_ui_publish.json').read_text()) +get_screen_app_ui = json.loads(base.joinpath('get_screen_app_ui_state.json').read_text()) +tap_element = json.loads(base.joinpath('tap_element.json').read_text()) +app_input_type_text = json.loads(base.joinpath('app_input_type_text.json').read_text()) +memory_save = json.loads(base.joinpath('memory_save.json').read_text()) +memory_update = json.loads(base.joinpath('memory_update.json').read_text()) +memory_save_merge_source = json.loads(base.joinpath('memory_save_merge_source.json').read_text()) +memory_merge = json.loads(base.joinpath('memory_merge.json').read_text()) +memory_save_delete_fixture = json.loads(base.joinpath('memory_save_delete_fixture.json').read_text()) +memory_delete = json.loads(base.joinpath('memory_delete.json').read_text()) +memory_search = json.loads(base.joinpath('memory_search.json').read_text()) +context_search = json.loads(base.joinpath('context_search.json').read_text()) +clipboard_write = json.loads(base.joinpath('clipboard_write.json').read_text()) +clipboard_read = json.loads(base.joinpath('clipboard_read.json').read_text()) +context_search_clipboard = json.loads(base.joinpath('context_search_clipboard.json').read_text()) +contacts_search = json.loads(base.joinpath('contacts_search.json').read_text()) +context_search_contacts = json.loads(base.joinpath('context_search_contacts.json').read_text()) +calendar_search = json.loads(base.joinpath('calendar_search.json').read_text()) +context_search_calendar = json.loads(base.joinpath('context_search_calendar.json').read_text()) +calls_search = json.loads(base.joinpath('calls_search.json').read_text()) +context_search_calls = json.loads(base.joinpath('context_search_calls.json').read_text()) +messages_search = json.loads(base.joinpath('messages_search.json').read_text()) +context_search_messages = json.loads(base.joinpath('context_search_messages.json').read_text()) +notification_ingest = json.loads(base.joinpath('notification_ingest.json').read_text()) +notification_list = json.loads(base.joinpath('notification_list.json').read_text()) +context_search_notification = json.loads(base.joinpath('context_search_notification.json').read_text()) +commitment_create = json.loads(base.joinpath('commitment_create.json').read_text()) +commitment_search = json.loads(base.joinpath('commitment_search.json').read_text()) +commitment_update = json.loads(base.joinpath('commitment_update_status.json').read_text()) +commitment_due_create = json.loads(base.joinpath('commitment_due_create.json').read_text()) +commitment_run_due = json.loads(base.joinpath('commitment_run_due.json').read_text()) +commitment_due_job_run = json.loads(base.joinpath('commitment_due_job_run.json').read_text()) +commitment_due_after = json.loads(base.joinpath('commitment_due_after.json').read_text()) +watcher_create = json.loads(base.joinpath('watcher_create.json').read_text()) +watcher_list = json.loads(base.joinpath('watcher_list.json').read_text()) +watcher_materialize_due = json.loads(base.joinpath('watcher_materialize_due.json').read_text()) +watcher_run_due = json.loads(base.joinpath('watcher_run_due.json').read_text()) +watcher_after_run = json.loads(base.joinpath('watcher_after_run.json').read_text()) +watcher_stop = json.loads(base.joinpath('watcher_stop.json').read_text()) +watcher_repair_create = json.loads(base.joinpath('watcher_repair_create.json').read_text()) +watcher_debug_mark_running = json.loads(base.joinpath('watcher_debug_mark_running.json').read_text()) +watcher_repair_stuck = json.loads(base.joinpath('watcher_repair_stuck.json').read_text()) +watcher_repair_run_due = json.loads(base.joinpath('watcher_repair_run_due.json').read_text()) +watcher_repair_after_run = json.loads(base.joinpath('watcher_repair_after_run.json').read_text()) +watcher_repair_stop = json.loads(base.joinpath('watcher_repair_stop.json').read_text()) +watcher_recurring_create = json.loads(base.joinpath('watcher_recurring_create.json').read_text()) +watcher_recurring_first = json.loads(base.joinpath('watcher_recurring_first.json').read_text()) +watcher_recurring_second = json.loads(base.joinpath('watcher_recurring_second.json').read_text()) +watcher_recurring_after = json.loads(base.joinpath('watcher_recurring_after.json').read_text()) +watcher_recurring_stop = json.loads(base.joinpath('watcher_recurring_stop.json').read_text()) +background_job_create = json.loads(base.joinpath('background_job_create.json').read_text()) +background_job_run_due = json.loads(base.joinpath('background_job_run_due.json').read_text()) +background_job_list = json.loads(base.joinpath('background_job_list.json').read_text()) +background_job_stop = json.loads(base.joinpath('background_job_stop.json').read_text()) +background_job_recurring_create = json.loads(base.joinpath('background_job_recurring_create.json').read_text()) +background_job_recurring_run_due = json.loads(base.joinpath('background_job_recurring_run_due.json').read_text()) +background_job_recurring_list = json.loads(base.joinpath('background_job_recurring_list.json').read_text()) +background_job_recurring_stop = json.loads(base.joinpath('background_job_recurring_stop.json').read_text()) +background_job_nonrecurring_interval_create = json.loads(base.joinpath('background_job_nonrecurring_interval_create.json').read_text()) +background_job_nonrecurring_interval_run_due = json.loads(base.joinpath('background_job_nonrecurring_interval_run_due.json').read_text()) +background_job_nonrecurring_interval_list = json.loads(base.joinpath('background_job_nonrecurring_interval_list.json').read_text()) +background_job_backoff_create = json.loads(base.joinpath('background_job_backoff_create.json').read_text()) +background_job_backoff_run_due = json.loads(base.joinpath('background_job_backoff_run_due.json').read_text()) +background_job_backoff_list = json.loads(base.joinpath('background_job_backoff_list.json').read_text()) +background_job_backoff_stop = json.loads(base.joinpath('background_job_backoff_stop.json').read_text()) +background_job_repair_create = json.loads(base.joinpath('background_job_repair_create.json').read_text()) +background_job_debug_mark_running = json.loads(base.joinpath('background_job_debug_mark_running.json').read_text()) +background_job_repair_stuck = json.loads(base.joinpath('background_job_repair_stuck.json').read_text()) +background_job_repair_run_due = json.loads(base.joinpath('background_job_repair_run_due.json').read_text()) +background_job_repair_list = json.loads(base.joinpath('background_job_repair_list.json').read_text()) +background_job_repair_stop = json.loads(base.joinpath('background_job_repair_stop.json').read_text()) +agent_status_initial = json.loads(base.joinpath('agent_status_initial.json').read_text()) +agent_control_pause = json.loads(base.joinpath('agent_control_pause.json').read_text()) +agent_status_paused = json.loads(base.joinpath('agent_status_paused.json').read_text()) +hardware_trigger_paused = json.loads(base.joinpath('hardware_trigger_paused.json').read_text()) +agent_control_resume = json.loads(base.joinpath('agent_control_resume.json').read_text()) +agent_status_resumed = json.loads(base.joinpath('agent_status_resumed.json').read_text()) +agent_control_disable_hardware = json.loads(base.joinpath('agent_control_disable_hardware.json').read_text()) +hardware_trigger_disabled = json.loads(base.joinpath('hardware_trigger_disabled.json').read_text()) +agent_control_enable_hardware = json.loads(base.joinpath('agent_control_enable_hardware.json').read_text()) +agent_control_disable_yolo = json.loads(base.joinpath('agent_control_disable_yolo.json').read_text()) +hardware_trigger_yolo_disabled = json.loads(base.joinpath('hardware_trigger_yolo_disabled.json').read_text()) +agent_control_enable_yolo = json.loads(base.joinpath('agent_control_enable_yolo.json').read_text()) +hardware_trigger = json.loads(base.joinpath('hardware_trigger.json').read_text()) +unlock_with_passcode = json.loads(base.joinpath('unlock_with_passcode.json').read_text()) +run = json.loads(base.joinpath('run_task.json').read_text()) +model_status = json.loads(base.joinpath('model_status.json').read_text()) +model_run = json.loads(base.joinpath('run_task_model.json').read_text()) +model_unverified_run = json.loads(base.joinpath('run_task_model_unverified.json').read_text()) +model_visible_run = json.loads(base.joinpath('run_task_model_visible.json').read_text()) +start_model_cancel = json.loads(base.joinpath('start_task_model_cancel.json').read_text()) +stop_model_cancel = json.loads(base.joinpath('stop_task_model_cancel.json').read_text()) +model_cancel_run = json.loads(base.joinpath('run_task_model_cancelled.json').read_text()) +start_stale_repair = json.loads(base.joinpath('start_task_stale_repair.json').read_text()) +task_repair_stale_active = json.loads(base.joinpath('task_repair_stale_active.json').read_text()) +get_task_stale_repaired = json.loads(base.joinpath('get_task_stale_repaired.json').read_text()) +get_task_stale_repaired_trajectory = json.loads(base.joinpath('get_task_stale_repaired_trajectory.json').read_text()) +model_repaired_run = json.loads(base.joinpath('run_task_model_repaired.json').read_text()) +model_configure = json.loads(base.joinpath('model_configure.json').read_text()) +model_status_configured = json.loads(base.joinpath('model_status_configured.json').read_text()) +model_broker_run = json.loads(base.joinpath('run_task_model_broker.json').read_text()) +model_configure_realtime2 = json.loads(base.joinpath('model_configure_realtime2.json').read_text()) +model_status_realtime2 = json.loads(base.joinpath('model_status_realtime2.json').read_text()) +list_tasks = json.loads(base.joinpath('list_tasks.json').read_text()) +get_task = json.loads(base.joinpath('get_task.json').read_text()) +get_trajectory = json.loads(base.joinpath('get_trajectory.json').read_text()) +get_model_trajectory = json.loads(base.joinpath('get_model_trajectory.json').read_text()) +get_model_unverified_trajectory = json.loads(base.joinpath('get_model_unverified_trajectory.json').read_text()) +get_model_visible_trajectory = json.loads(base.joinpath('get_model_visible_trajectory.json').read_text()) +get_model_cancel_trajectory = json.loads(base.joinpath('get_model_cancel_trajectory.json').read_text()) +get_model_repaired_trajectory = json.loads(base.joinpath('get_model_repaired_trajectory.json').read_text()) +get_model_broker_trajectory = json.loads(base.joinpath('get_model_broker_trajectory.json').read_text()) +get_redaction_trajectory = json.loads(base.joinpath('get_redaction_trajectory.json').read_text()) +get_audit = json.loads(base.joinpath('get_audit.json').read_text()) +validate_store = json.loads(base.joinpath('validate_store.json').read_text()) + +def assert_no_sensitive_keys(value, path='root'): + markers = ( + 'password', + 'passcode', + 'token', + 'secret', + 'authorization', + 'api_key', + 'apikey', + 'private_key', + ) + if isinstance(value, dict): + for key, item in value.items(): + normalized = str(key).lower() + assert not any(marker in normalized for marker in markers), f'sensitive key exported at {path}.{key}' + assert_no_sensitive_keys(item, f'{path}.{key}') + elif isinstance(value, list): + for index, item in enumerate(value): + assert_no_sensitive_keys(item, f'{path}[{index}]') + +def assert_provider_attempt_shape(result, expected_user_facing_status): + assert result['provider_attempts'], result + assert result['verification']['status'] in ('verified', 'unverified', 'failed'), result + assert result['verification']['source'], result + assert result['verification']['reason'], result + assert result['user_facing_status'] == expected_user_facing_status, result + for index, attempt in enumerate(result['provider_attempts']): + prefix = f'provider_attempts[{index}]' + assert attempt['provider'], {prefix: attempt} + assert attempt['scope'], {prefix: attempt} + assert attempt['action_type'], {prefix: attempt} + assert attempt['status'] in ('ok', 'unavailable', 'not_attempted'), {prefix: attempt} + assert attempt['verification']['status'] in ('verified', 'unverified', 'failed'), {prefix: attempt} + assert attempt['verification']['source'], {prefix: attempt} + assert attempt['verification']['reason'], {prefix: attempt} + +assert health['status'] == 'ok', health +assert health['autonomy_mode'] == 'yolo', health +assert health['agent']['state'] == 'running', health +assert health['agent']['trigger_policy'] == 'allow_yolo', health +assert health['providers']['model']['status'] == 'disabled', health +assert health['providers']['model']['mode'] == 'broker', health +assert get_screen_springboard['context']['foreground_app'] == 'com.apple.mobilesafari', get_screen_springboard +assert get_screen_springboard['context']['foreground_source'] == 'springboard_state', get_screen_springboard +assert get_screen_springboard['context']['springboard_state']['status'] == 'ok', get_screen_springboard +assert get_screen_springboard['context']['display']['orientation_name'] == 'portrait', get_screen_springboard +assert get_screen_springboard['context']['ui_tree']['status'] == 'ok', get_screen_springboard +assert get_screen_springboard['context']['visible_text'] == ['Safari'], get_screen_springboard +assert get_screen_springboard['context']['interactive_elements'][0]['id'] == 'springboard-0-0', get_screen_springboard +assert get_screen_springboard['context']['windows'][0]['bounds'] == [0, 0, 430, 932], get_screen_springboard +assert 'ui_tree_springboard_only' in get_screen_springboard['context']['risk_flags'], get_screen_springboard +assert app_ui_publish['status'] == 'ok', app_ui_publish +assert app_ui_publish['bundle_id'] == 'com.apple.mobilesafari', app_ui_publish +assert app_ui_publish['ui_tree_status'] == 'ok', app_ui_publish +assert get_screen_app_ui['context']['foreground_app'] == 'com.apple.mobilesafari', get_screen_app_ui +assert get_screen_app_ui['context']['foreground_source'] == 'springboard_state', get_screen_app_ui +assert get_screen_app_ui['context']['springboard_state']['status'] == 'ok', get_screen_app_ui +assert get_screen_app_ui['context']['app_ui_state']['status'] == 'ok', get_screen_app_ui +assert get_screen_app_ui['context']['app_ui_state']['bundle_id'] == 'com.apple.mobilesafari', get_screen_app_ui +assert get_screen_app_ui['context']['app_ui_state']['received_transport'] == 'local_smoke_agentctl', get_screen_app_ui +assert get_screen_app_ui['context']['ui_tree_source'] == 'app_process', get_screen_app_ui +assert get_screen_app_ui['context']['ui_tree']['scope'] == 'app_process', get_screen_app_ui +assert get_screen_app_ui['context']['ui_tree']['provider'] == 'OpenPhoneAppIntrospector.UIKitAccessibility', get_screen_app_ui +assert get_screen_app_ui['context']['visible_text'] == ['Example Domain', 'Address'], get_screen_app_ui +assert get_screen_app_ui['context']['interactive_elements'][0]['id'] == 'app-com.apple.mobilesafari-0-0', get_screen_app_ui +assert get_screen_app_ui['context']['interactive_elements'][0]['scope'] == 'app_process', get_screen_app_ui +assert 'ui_tree_app_process' in get_screen_app_ui['context']['risk_flags'], get_screen_app_ui +assert 'ui_tree_springboard_only' not in get_screen_app_ui['context']['risk_flags'], get_screen_app_ui +assert tap_element['state'] in ('action.executed', 'action.denied.input_failed'), tap_element +assert tap_element['detail'] == 'tap_element:springboard-0-0:40.0,73.0', tap_element +assert tap_element['coordinate_source'] == 'ui_tree.bounds_center', tap_element +assert tap_element['target']['id'] == 'springboard-0-0', tap_element +assert tap_element['target']['label'] == 'Safari', tap_element +assert tap_element['target']['bounds'] == [12, 34, 56, 78], tap_element +assert_provider_attempt_shape(tap_element, 'dispatch_unverified') +assert tap_element['verification']['status'] == 'unverified', tap_element +assert any(attempt['scope'] == 'daemon_hid' for attempt in tap_element['provider_attempts']), tap_element +assert app_input_type_text['state'] == 'action.executed', app_input_type_text +assert app_input_type_text['detail'] == 'type_text:9:app_process', app_input_type_text +assert_provider_attempt_shape(app_input_type_text, 'verified') +assert app_input_type_text['verification']['status'] == 'verified', app_input_type_text +assert app_input_type_text['verification']['source'] == 'app_process_text_state', app_input_type_text +assert app_input_type_text['provider_attempts'][0]['activation_method'] == 'text_input_insert', app_input_type_text +assert app_input_type_text['provider_attempts'][0]['before_text_length'] == 0, app_input_type_text +assert app_input_type_text['provider_attempts'][0]['after_text_length'] == 9, app_input_type_text +assert memory_save['status'] == 'ok', memory_save +assert memory_save['memory']['text'] == 'user prefers concise OpenPhone status updates', memory_save +assert memory_update['status'] == 'ok', memory_update +assert memory_update['memory_id'] == memory_save['memory']['memory_id'], memory_update +assert memory_update['memory']['text'] == 'user prefers concise OpenPhone validation details', memory_update +assert memory_update['previous_memory']['text'] == 'user prefers concise OpenPhone status updates', memory_update +assert memory_save_merge_source['status'] == 'ok', memory_save_merge_source +assert memory_merge['status'] == 'ok', memory_merge +assert memory_merge['memory_id'] == memory_save['memory']['memory_id'], memory_merge +assert memory_merge['merged_from'] == memory_save_merge_source['memory']['memory_id'], memory_merge +assert 'terse summaries' in memory_merge['memory']['text'], memory_merge +assert memory_merge['deleted_memory']['memory_id'] == memory_save_merge_source['memory']['memory_id'], memory_merge +assert memory_save_delete_fixture['status'] == 'ok', memory_save_delete_fixture +assert memory_delete['status'] == 'ok', memory_delete +assert memory_delete['deleted'] is True, memory_delete +assert memory_delete['memory_id'] == memory_save_delete_fixture['memory']['memory_id'], memory_delete +assert memory_search['status'] == 'ok', memory_search +assert memory_search['count'] >= 1, memory_search +assert any('terse summaries' in item['text'] for item in memory_search['memories']), memory_search +assert not any(item['memory_id'] == memory_save_merge_source['memory']['memory_id'] for item in memory_search['memories']), memory_search +assert not any(item['memory_id'] == memory_save_delete_fixture['memory']['memory_id'] for item in memory_search['memories']), memory_search +assert clipboard_write['status'] == 'ok', clipboard_write +assert clipboard_write['tool'] == 'clipboard_write', clipboard_write +assert clipboard_write['text_length'] == len('OpenPhone clipboard smoke value'), clipboard_write +assert clipboard_write['text_sha256'], clipboard_write +assert clipboard_read['status'] == 'ok', clipboard_read +assert clipboard_read['tool'] == 'clipboard_read', clipboard_read +assert clipboard_read['text'] == 'OpenPhone clipboard smoke value', clipboard_read +assert clipboard_read['text_length'] == len('OpenPhone clipboard smoke value'), clipboard_read +assert clipboard_read['text_sha256'] == clipboard_write['text_sha256'], clipboard_read +assert context_search_clipboard['status'] == 'ok', context_search_clipboard +assert context_search_clipboard['count'] >= 1, context_search_clipboard +assert any(item['type'] in ('clipboard_read', 'clipboard_written') for item in context_search_clipboard['events']), context_search_clipboard +assert contacts_search['status'] == 'ok', contacts_search +assert contacts_search['tool'] == 'contacts_search', contacts_search +assert contacts_search['provider'] == 'openphone.contacts_fixture_file', contacts_search +assert contacts_search['count'] == 1, contacts_search +assert contacts_search['query_length'] == 3, contacts_search +assert contacts_search['query_sha256'], contacts_search +assert contacts_search['contacts'][0]['display_name'] == 'Ada Lovelace', contacts_search +assert contacts_search['contacts'][0]['phone_numbers'] == ['+1555010101'], contacts_search +assert contacts_search['contacts'][0]['emails'] == ['ada@example.test'], contacts_search +assert context_search_contacts['status'] == 'ok', context_search_contacts +assert context_search_contacts['count'] >= 1, context_search_contacts +assert any(item['type'] == 'contacts_searched' for item in context_search_contacts['events']), context_search_contacts +assert not any('Ada Lovelace' in item['body'] for item in context_search_contacts['events']), context_search_contacts +assert calendar_search['status'] == 'ok', calendar_search +assert calendar_search['tool'] == 'calendar_search', calendar_search +assert calendar_search['provider'] == 'openphone.calendar_fixture_file', calendar_search +assert calendar_search['count'] == 1, calendar_search +assert calendar_search['query_length'] == len('Standup'), calendar_search +assert calendar_search['query_sha256'], calendar_search +assert calendar_search['events'][0]['title'] == 'OpenPhone Standup', calendar_search +assert calendar_search['events'][0]['calendar_title'] == 'Engineering', calendar_search +assert calendar_search['events'][0]['start_at_ms'] == 1893456000000, calendar_search +assert context_search_calendar['status'] == 'ok', context_search_calendar +assert context_search_calendar['count'] >= 1, context_search_calendar +assert any(item['type'] == 'calendar_searched' for item in context_search_calendar['events']), context_search_calendar +assert not any('OpenPhone Standup' in item['body'] for item in context_search_calendar['events']), context_search_calendar +assert calls_search['status'] == 'ok', calls_search +assert calls_search['tool'] == 'calls_search', calls_search +assert calls_search['provider'] == 'openphone.calls_fixture_file', calls_search +assert calls_search['count'] == 1, calls_search +assert calls_search['query_length'] == len('OpenPhone'), calls_search +assert calls_search['query_sha256'], calls_search +assert calls_search['calls'][0]['display_name'] == 'OpenPhone Test Call', calls_search +assert calls_search['calls'][0]['address'] == '+15550101234', calls_search +assert calls_search['calls'][0]['direction'] == 'incoming', calls_search +assert calls_search['calls'][0]['duration_seconds'] == 120, calls_search +assert calls_search['calls'][0]['start_at_ms'] == 1893459600000, calls_search +assert context_search_calls['status'] == 'ok', context_search_calls +assert context_search_calls['count'] >= 1, context_search_calls +assert any(item['type'] == 'calls_searched' for item in context_search_calls['events']), context_search_calls +assert not any('OpenPhone Test Call' in item['body'] for item in context_search_calls['events']), context_search_calls +assert not any('+15550101234' in item['body'] for item in context_search_calls['events']), context_search_calls +assert messages_search['status'] == 'ok', messages_search +assert messages_search['tool'] == 'messages_search', messages_search +assert messages_search['provider'] == 'openphone.messages_fixture_file', messages_search +assert messages_search['count'] == 1, messages_search +assert messages_search['query_length'] == len('OpenPhone'), messages_search +assert messages_search['query_sha256'], messages_search +assert messages_search['messages'][0]['handle'] == '+15550105555', messages_search +assert messages_search['messages'][0]['service'] == 'iMessage', messages_search +assert messages_search['messages'][0]['direction'] == 'incoming', messages_search +assert messages_search['messages'][0]['text_preview'] == 'OpenPhone message fixture says hello from local smoke.', messages_search +assert messages_search['messages'][0]['text_sha256'], messages_search +assert messages_search['messages'][0]['sent_at_ms'] == 1893463200000, messages_search +assert context_search_messages['status'] == 'ok', context_search_messages +assert context_search_messages['count'] >= 1, context_search_messages +assert any(item['type'] == 'messages_searched' for item in context_search_messages['events']), context_search_messages +assert not any('OpenPhone message fixture' in item['body'] for item in context_search_messages['events']), context_search_messages +assert not any('+15550105555' in item['body'] for item in context_search_messages['events']), context_search_messages +# Notification provider redaction: bounded previews only (title<=200, body<=1000), +# no full payload / tail PII in the stored log or the derived context event. +notification_tail = 'NOTIFYPIItail_+15559998888_leak@example.test' +assert notification_ingest['status'] == 'ok', notification_ingest +assert notification_ingest['bundle_id'] == 'com.openphone.smoke', notification_ingest +assert notification_ingest['stored_count'] >= 1, notification_ingest +assert notification_list['status'] == 'ok', notification_list +assert notification_list['count'] >= 1, notification_list +smoke_notifs = [n for n in notification_list['notifications'] if n.get('notification_id') == 'smoke-notif-1'] +assert smoke_notifs, notification_list +notif = smoke_notifs[0] +assert notif['bundle_id'] == 'com.openphone.smoke', notif +assert len(notif['title']) <= 201, notif # 200 chars + single ellipsis +assert len(notif['body']) <= 1001, notif +assert notification_tail not in notif['title'], notif +assert notification_tail not in notif['body'], notif +assert notif['body'].startswith('OpenPhone body preview head.'), notif +assert context_search_notification['status'] == 'ok', context_search_notification +assert context_search_notification['count'] >= 1, context_search_notification +assert any(item['type'] == 'notification_received' for item in context_search_notification['events']), context_search_notification +assert not any(notification_tail in item['body'] for item in context_search_notification['events']), context_search_notification +assert not any(notification_tail in item.get('title', '') for item in context_search_notification['events']), context_search_notification +assert context_search['status'] == 'ok', context_search +assert context_search['count'] >= 1, context_search +assert any('concise' in item['body'] for item in context_search['events']), context_search +assert commitment_create['status'] == 'ok', commitment_create +assert commitment_create['commitment']['title'] == 'follow up about OpenPhone iOS smoke', commitment_create +assert commitment_search['status'] == 'ok', commitment_search +assert commitment_search['count'] >= 1, commitment_search +assert commitment_update['status'] == 'ok', commitment_update +assert commitment_update['commitment']['status'] == 'completed', commitment_update +assert commitment_due_create['status'] == 'ok', commitment_due_create +assert commitment_due_create['commitment']['title'] == 'due commitment smoke', commitment_due_create +assert commitment_due_create['commitment']['status'] == 'active', commitment_due_create +assert commitment_due_create['commitment']['trigger_type'] == 'time', commitment_due_create +commitment_due_public_id = commitment_due_create['commitment']['commitment_id'] +assert commitment_run_due['status'] == 'ok', commitment_run_due +assert commitment_run_due['scheduler_status'] == 'implemented_time_bridge', commitment_run_due +assert commitment_run_due['triggered_count'] >= 1, commitment_run_due +assert commitment_run_due['job_count'] >= 1, commitment_run_due +commitment_due_entries = [ + item for item in commitment_run_due['commitments'] + if item.get('commitment_id') == commitment_due_public_id +] +assert commitment_due_entries and commitment_due_entries[0]['status'] == 'background_job_queued', commitment_run_due +commitment_due_entry = commitment_due_entries[0] +assert commitment_due_entry['commitment']['status'] == 'triggered', commitment_due_entry +assert commitment_due_entry['commitment']['evidence']['scheduler']['last_trigger_status'] == 'background_job_queued', commitment_due_entry +assert commitment_due_entry['commitment']['evidence']['scheduler']['last_job_id'] == commitment_due_entry['job_id'], commitment_due_entry +assert commitment_due_entry['job']['type'] == 'commitment_due', commitment_due_entry +assert commitment_due_entry['job']['payload']['commitment_id'] == commitment_due_public_id, commitment_due_entry +assert commitment_due_job_run['status'] == 'ok', commitment_due_job_run +assert commitment_due_job_run['commitment_scheduler_status'] == 'implemented_time_bridge', commitment_due_job_run +commitment_due_job_entries = [ + item for item in commitment_due_job_run['jobs'] + if item.get('job_id') == commitment_due_entry['job_id'] +] +assert commitment_due_job_entries and commitment_due_job_entries[0]['run_task']['status'] == 'task.finished', commitment_due_job_run +assert commitment_due_after['status'] == 'ok', commitment_due_after +commitment_due_after_entries = [ + item for item in commitment_due_after['commitments'] + if item.get('commitment_id') == commitment_due_public_id +] +assert commitment_due_after_entries and commitment_due_after_entries[0]['status'] == 'triggered', commitment_due_after +assert watcher_create['status'] == 'ok', watcher_create +assert watcher_create['watcher']['title'] == 'watch smoke condition', watcher_create +assert watcher_create['scheduler_status'] == 'implemented_timer_bridge', watcher_create +assert watcher_create['fires_locally'] is True, watcher_create +assert watcher_create['watcher']['scheduler_status'] == 'implemented_timer_bridge', watcher_create +assert watcher_create['watcher']['fires_locally'] is True, watcher_create +assert watcher_list['status'] == 'ok', watcher_list +assert watcher_list['count'] >= 1, watcher_list +assert watcher_list['scheduler_status'] == 'implemented_timer_bridge', watcher_list +assert watcher_materialize_due['status'] == 'ok', watcher_materialize_due +assert watcher_materialize_due['scheduler_status'] == 'implemented_timer_bridge', watcher_materialize_due +assert watcher_materialize_due['fired_count'] >= 1, watcher_materialize_due +assert watcher_materialize_due['job_count'] >= 1, watcher_materialize_due +assert watcher_materialize_due['watchers'][0]['status'] == 'background_job_queued', watcher_materialize_due +assert watcher_run_due['status'] == 'ok', watcher_run_due +assert watcher_run_due['watcher_scheduler_status'] == 'implemented_timer_bridge', watcher_run_due +assert watcher_run_due['ran_count'] >= 1, watcher_run_due +assert watcher_run_due['jobs'][0]['run_task']['status'] in ('task.finished', 'task.failed'), watcher_run_due +assert watcher_after_run['status'] == 'ok', watcher_after_run +assert watcher_after_run['watchers'][0]['status'] == 'fired', watcher_after_run +assert watcher_stop['status'] == 'ok', watcher_stop +assert watcher_stop['stopped_count'] == 1, watcher_stop +assert watcher_stop['watchers'][0]['status'] == 'stopped', watcher_stop +assert watcher_recurring_create['status'] == 'ok', watcher_recurring_create +recurring_watcher_public_id = watcher_recurring_create['watcher']['watcher_id'] +assert watcher_recurring_create['watcher']['recurring'] == True, watcher_recurring_create +assert watcher_recurring_create['watcher']['interval_ms'] == 5, watcher_recurring_create +assert watcher_recurring_first['status'] == 'ok', watcher_recurring_first +assert watcher_recurring_first['scheduler_status'] == 'implemented_timer_bridge', watcher_recurring_first +recurring_first_entries = [item for item in watcher_recurring_first['watchers'] if item.get('watcher_id') == recurring_watcher_public_id] +assert recurring_first_entries and recurring_first_entries[0]['status'] == 'background_job_queued', watcher_recurring_first +assert recurring_first_entries[0]['watcher']['status'] == 'active', watcher_recurring_first +assert recurring_first_entries[0]['watcher']['metadata']['recurring'] == True, watcher_recurring_first +assert recurring_first_entries[0]['watcher']['metadata']['last_fire_status'] == 'background_job_queued', watcher_recurring_first +assert watcher_recurring_second['status'] == 'ok', watcher_recurring_second +recurring_second_entries = [item for item in watcher_recurring_second['watchers'] if item.get('watcher_id') == recurring_watcher_public_id] +assert recurring_second_entries and recurring_second_entries[0]['status'] == 'background_job_queued', watcher_recurring_second +assert recurring_second_entries[0]['watcher']['status'] == 'active', watcher_recurring_second +assert recurring_second_entries[0]['watcher']['schedule']['last_job_id'] != recurring_first_entries[0]['watcher']['schedule']['last_job_id'], watcher_recurring_second +assert watcher_recurring_after['status'] == 'ok', watcher_recurring_after +recurring_after_entries = [item for item in watcher_recurring_after['watchers'] if item.get('watcher_id') == recurring_watcher_public_id] +assert recurring_after_entries and recurring_after_entries[0]['status'] == 'active', watcher_recurring_after +assert recurring_after_entries[0]['metadata']['recurring'] == True, watcher_recurring_after +assert watcher_recurring_stop['status'] == 'ok', watcher_recurring_stop +assert watcher_recurring_stop['stopped_count'] == 1, watcher_recurring_stop +assert watcher_repair_create['status'] == 'ok', watcher_repair_create +repair_watcher_public_id = watcher_repair_create['watcher']['watcher_id'] +assert watcher_debug_mark_running['status'] == 'ok', watcher_debug_mark_running +assert watcher_debug_mark_running['watcher']['status'] == 'running', watcher_debug_mark_running +assert watcher_repair_stuck['status'] == 'ok', watcher_repair_stuck +assert watcher_repair_stuck['repair_policy'] == 'requeue_stale_running', watcher_repair_stuck +assert watcher_repair_stuck['repaired_count'] >= 1, watcher_repair_stuck +watcher_repair_entries = [item for item in watcher_repair_stuck['watchers'] if item.get('watcher_id') == repair_watcher_public_id] +assert watcher_repair_entries and watcher_repair_entries[0]['status'] == 'requeued', watcher_repair_stuck +assert watcher_repair_entries[0]['watcher']['status'] == 'active', watcher_repair_stuck +assert watcher_repair_entries[0]['watcher']['metadata']['stuck_repair']['repair_action'] == 'requeued', watcher_repair_stuck +assert watcher_repair_run_due['status'] == 'ok', watcher_repair_run_due +assert watcher_repair_run_due['scheduler_status'] == 'implemented_timer_bridge', watcher_repair_run_due +assert watcher_repair_run_due['fired_count'] >= 1, watcher_repair_run_due +repair_due_entries = [item for item in watcher_repair_run_due['watchers'] if item.get('watcher_id') == repair_watcher_public_id] +assert repair_due_entries and repair_due_entries[0]['status'] == 'background_job_queued', watcher_repair_run_due +assert watcher_repair_after_run['status'] == 'ok', watcher_repair_after_run +repair_after_entries = [item for item in watcher_repair_after_run['watchers'] if item.get('watcher_id') == repair_watcher_public_id] +assert repair_after_entries and repair_after_entries[0]['status'] == 'fired', watcher_repair_after_run +assert repair_after_entries[0]['metadata']['stuck_repair']['repair_action'] == 'requeued', watcher_repair_after_run +assert watcher_repair_stop['status'] == 'ok', watcher_repair_stop +assert background_job_create['status'] == 'ok', background_job_create +assert background_job_create['job']['title'] == 'smoke background job', background_job_create +assert background_job_create['scheduler_status'] == 'implemented_agent_loop', background_job_create +assert background_job_create['runs_locally'] is True, background_job_create +assert background_job_run_due['status'] == 'ok', background_job_run_due +assert background_job_run_due['scheduler_status'] == 'implemented_agent_loop', background_job_run_due +assert background_job_run_due['runner'] == 'deterministic', background_job_run_due +assert background_job_run_due['ran_count'] >= 1, background_job_run_due +assert background_job_run_due['jobs'][0]['run_task']['status'] in ('task.finished', 'task.failed'), background_job_run_due +assert background_job_run_due['jobs'][0]['run_task']['task_id'], background_job_run_due +assert background_job_list['status'] == 'ok', background_job_list +assert background_job_list['count'] >= 1, background_job_list +assert background_job_stop['status'] == 'ok', background_job_stop +assert background_job_stop['job']['status'] == 'stopped', background_job_stop +assert background_job_recurring_create['status'] == 'ok', background_job_recurring_create +recurring_job_public_id = background_job_recurring_create['job']['job_id'] +assert background_job_recurring_create['job']['status'] == 'queued', background_job_recurring_create +assert background_job_recurring_create['job']['schedule']['recurring'] == True, background_job_recurring_create +assert background_job_recurring_create['job']['interval_ms'] == 50, background_job_recurring_create +assert background_job_recurring_run_due['status'] == 'ok', background_job_recurring_run_due +recurring_job_entries = [item for item in background_job_recurring_run_due['jobs'] if item.get('job_id') == recurring_job_public_id] +assert recurring_job_entries and recurring_job_entries[0]['status'] == 'queued', background_job_recurring_run_due +assert recurring_job_entries[0]['run_task']['status'] == 'task.finished', background_job_recurring_run_due +recurring_job = recurring_job_entries[0]['job'] +assert recurring_job['status'] == 'queued', recurring_job +assert recurring_job['schedule']['run_policy'] == 'recurring_interval', recurring_job +assert recurring_job['schedule']['retry_backoff_ms'] == 0, recurring_job +assert recurring_job['payload']['scheduler']['run_policy'] == 'recurring_interval', recurring_job +assert recurring_job['payload']['scheduler']['failure_count'] == 0, recurring_job +assert recurring_job['next_run_at_ms'] > recurring_job['payload']['scheduler']['last_run_at_ms'], recurring_job +assert background_job_recurring_list['status'] == 'ok', background_job_recurring_list +recurring_list_entries = [item for item in background_job_recurring_list['jobs'] if item.get('job_id') == recurring_job_public_id] +assert recurring_list_entries and recurring_list_entries[0]['status'] == 'queued', background_job_recurring_list +assert background_job_recurring_stop['status'] == 'ok', background_job_recurring_stop +assert background_job_recurring_stop['job']['status'] == 'stopped', background_job_recurring_stop +assert background_job_nonrecurring_interval_create['status'] == 'ok', background_job_nonrecurring_interval_create +nonrecurring_interval_job_public_id = background_job_nonrecurring_interval_create['job']['job_id'] +assert background_job_nonrecurring_interval_create['job']['schedule']['recurring'] == False, background_job_nonrecurring_interval_create +assert background_job_nonrecurring_interval_create['job']['interval_ms'] == 50, background_job_nonrecurring_interval_create +assert background_job_nonrecurring_interval_run_due['status'] == 'ok', background_job_nonrecurring_interval_run_due +nonrecurring_interval_entries = [ + item for item in background_job_nonrecurring_interval_run_due['jobs'] + if item.get('job_id') == nonrecurring_interval_job_public_id +] +assert nonrecurring_interval_entries and nonrecurring_interval_entries[0]['status'] == 'completed', background_job_nonrecurring_interval_run_due +assert nonrecurring_interval_entries[0]['run_task']['status'] == 'task.finished', background_job_nonrecurring_interval_run_due +nonrecurring_interval_job = nonrecurring_interval_entries[0]['job'] +assert nonrecurring_interval_job['status'] == 'completed', nonrecurring_interval_job +assert nonrecurring_interval_job['schedule']['recurring'] == False, nonrecurring_interval_job +assert nonrecurring_interval_job['schedule']['run_policy'] == 'terminal', nonrecurring_interval_job +assert nonrecurring_interval_job['next_run_at_ms'] == 0, nonrecurring_interval_job +assert background_job_nonrecurring_interval_list['status'] == 'ok', background_job_nonrecurring_interval_list +nonrecurring_interval_list_entries = [ + item for item in background_job_nonrecurring_interval_list['jobs'] + if item.get('job_id') == nonrecurring_interval_job_public_id +] +assert nonrecurring_interval_list_entries and nonrecurring_interval_list_entries[0]['status'] == 'completed', background_job_nonrecurring_interval_list +assert background_job_backoff_create['status'] == 'ok', background_job_backoff_create +backoff_job_public_id = background_job_backoff_create['job']['job_id'] +assert background_job_backoff_create['job']['schedule']['recurring'] == True, background_job_backoff_create +assert background_job_backoff_run_due['status'] == 'ok', background_job_backoff_run_due +backoff_entries = [item for item in background_job_backoff_run_due['jobs'] if item.get('job_id') == backoff_job_public_id] +assert backoff_entries and backoff_entries[0]['status'] == 'queued', background_job_backoff_run_due +assert backoff_entries[0]['run_task']['status'] == 'error', background_job_backoff_run_due +assert backoff_entries[0]['run_task']['reason'] == 'model_provider_not_configured', background_job_backoff_run_due +backoff_job = backoff_entries[0]['job'] +assert backoff_job['status'] == 'queued', backoff_job +assert backoff_job['schedule']['run_policy'] == 'recurring_failure_backoff', backoff_job +assert backoff_job['schedule']['failure_count'] == 1, backoff_job +assert backoff_job['schedule']['retry_backoff_ms'] == 30000, backoff_job +assert backoff_job['payload']['scheduler']['run_policy'] == 'recurring_failure_backoff', backoff_job +assert backoff_job['payload']['scheduler']['failure_count'] == 1, backoff_job +assert backoff_job['next_run_at_ms'] >= backoff_job['payload']['scheduler']['last_run_at_ms'] + 30000, backoff_job +assert background_job_backoff_list['status'] == 'ok', background_job_backoff_list +backoff_list_entries = [item for item in background_job_backoff_list['jobs'] if item.get('job_id') == backoff_job_public_id] +assert backoff_list_entries and backoff_list_entries[0]['status'] == 'queued', background_job_backoff_list +assert background_job_backoff_stop['status'] == 'ok', background_job_backoff_stop +assert background_job_backoff_stop['job']['status'] == 'stopped', background_job_backoff_stop +assert background_job_repair_create['status'] == 'ok', background_job_repair_create +repair_job_public_id = background_job_repair_create['job']['job_id'] +assert background_job_debug_mark_running['status'] == 'ok', background_job_debug_mark_running +assert background_job_debug_mark_running['job']['status'] == 'running', background_job_debug_mark_running +assert background_job_repair_stuck['status'] == 'ok', background_job_repair_stuck +assert background_job_repair_stuck['repair_policy'] == 'requeue_stale_running', background_job_repair_stuck +assert background_job_repair_stuck['repaired_count'] >= 1, background_job_repair_stuck +repair_entries = [item for item in background_job_repair_stuck['jobs'] if item.get('job_id') == repair_job_public_id] +assert repair_entries and repair_entries[0]['status'] == 'requeued', background_job_repair_stuck +assert repair_entries[0]['job']['status'] == 'queued', background_job_repair_stuck +assert repair_entries[0]['job']['payload']['stuck_repair']['repair_action'] == 'requeued', background_job_repair_stuck +assert background_job_repair_run_due['status'] == 'ok', background_job_repair_run_due +assert background_job_repair_run_due['scheduler_status'] == 'implemented_agent_loop', background_job_repair_run_due +assert background_job_repair_list['status'] == 'ok', background_job_repair_list +repair_list_entries = [item for item in background_job_repair_list['jobs'] if item.get('job_id') == repair_job_public_id] +assert repair_list_entries, background_job_repair_list +assert repair_list_entries[0]['status'] in ('queued', 'completed', 'failed'), background_job_repair_list +assert repair_list_entries[0]['payload']['stuck_repair']['repair_action'] == 'requeued', background_job_repair_list +assert background_job_repair_stop['status'] == 'ok', background_job_repair_stop +assert agent_status_initial['status'] == 'ok', agent_status_initial +assert agent_status_initial['schema'] == 'openphone.agent_status.v1', agent_status_initial +assert agent_status_initial['control']['state'] == 'running', agent_status_initial +assert agent_status_initial['control']['trigger_policy'] == 'allow_yolo', agent_status_initial +assert agent_control_pause['status'] == 'ok', agent_control_pause +assert agent_control_pause['changed'] is True, agent_control_pause +assert agent_control_pause['control']['paused'] is True, agent_control_pause +assert agent_control_pause['control']['state'] == 'paused', agent_control_pause +assert agent_control_pause['control']['pause_reason'] == 'local smoke pause', agent_control_pause +assert agent_status_paused['state'] == 'paused', agent_status_paused +assert agent_status_paused['control']['trigger_policy'] == 'paused', agent_status_paused +assert hardware_trigger_paused['status'] == 'ok', hardware_trigger_paused +assert hardware_trigger_paused['state'] == 'trigger.paused', hardware_trigger_paused +assert hardware_trigger_paused['model_loop_status'] == 'paused', hardware_trigger_paused +assert hardware_trigger_paused['control']['paused'] is True, hardware_trigger_paused +assert 'agent_task_id' not in hardware_trigger_paused, hardware_trigger_paused +assert 'background_job' not in hardware_trigger_paused, hardware_trigger_paused +assert agent_control_resume['status'] == 'ok', agent_control_resume +assert agent_control_resume['changed'] is True, agent_control_resume +assert agent_control_resume['control']['paused'] is False, agent_control_resume +assert agent_control_resume['control']['trigger_policy'] == 'allow_yolo', agent_control_resume +assert agent_status_resumed['control']['state'] == 'running', agent_status_resumed +assert agent_status_resumed['control']['paused'] is False, agent_status_resumed +assert agent_control_disable_hardware['status'] == 'ok', agent_control_disable_hardware +assert agent_control_disable_hardware['changed'] is True, agent_control_disable_hardware +assert agent_control_disable_hardware['control']['hardware_triggers_enabled'] is False, agent_control_disable_hardware +assert agent_control_disable_hardware['control']['trigger_policy'] == 'disabled', agent_control_disable_hardware +assert hardware_trigger_disabled['status'] == 'ok', hardware_trigger_disabled +assert hardware_trigger_disabled['state'] == 'trigger.disabled', hardware_trigger_disabled +assert hardware_trigger_disabled['control']['hardware_triggers_enabled'] is False, hardware_trigger_disabled +assert 'agent_task_id' not in hardware_trigger_disabled, hardware_trigger_disabled +assert 'background_job' not in hardware_trigger_disabled, hardware_trigger_disabled +assert agent_control_enable_hardware['status'] == 'ok', agent_control_enable_hardware +assert agent_control_enable_hardware['control']['hardware_triggers_enabled'] is True, agent_control_enable_hardware +assert agent_control_enable_hardware['control']['trigger_policy'] == 'allow_yolo', agent_control_enable_hardware +assert agent_control_disable_yolo['status'] == 'ok', agent_control_disable_yolo +assert agent_control_disable_yolo['changed'] is True, agent_control_disable_yolo +assert agent_control_disable_yolo['control']['yolo_enabled'] is False, agent_control_disable_yolo +assert agent_control_disable_yolo['control']['trigger_policy'] == 'manual_only', agent_control_disable_yolo +assert hardware_trigger_yolo_disabled['status'] == 'ok', hardware_trigger_yolo_disabled +assert hardware_trigger_yolo_disabled['state'] == 'trigger.yolo_disabled', hardware_trigger_yolo_disabled +assert hardware_trigger_yolo_disabled['control']['yolo_enabled'] is False, hardware_trigger_yolo_disabled +assert 'agent_task_id' not in hardware_trigger_yolo_disabled, hardware_trigger_yolo_disabled +assert 'background_job' not in hardware_trigger_yolo_disabled, hardware_trigger_yolo_disabled +assert agent_control_enable_yolo['status'] == 'ok', agent_control_enable_yolo +assert agent_control_enable_yolo['control']['yolo_enabled'] is True, agent_control_enable_yolo +assert agent_control_enable_yolo['control']['trigger_policy'] == 'allow_yolo', agent_control_enable_yolo +assert hardware_trigger['status'] == 'ok', hardware_trigger +assert hardware_trigger['trigger'] == 'volume_up_down_combo', hardware_trigger +assert hardware_trigger['runtime_authority'] == 'phone_local', hardware_trigger +assert hardware_trigger['model_loop_status'] == 'not_started', hardware_trigger +assert hardware_trigger['background_job']['status'] == 'ok', hardware_trigger +assert hardware_trigger['background_job']['job']['status'] == 'queued', hardware_trigger +assert hardware_trigger['scheduler']['status'] == 'ok', hardware_trigger +assert hardware_trigger['scheduler']['scheduler_status'] == 'implemented_agent_loop', hardware_trigger +assert hardware_trigger['scheduler']['run_jobs'] is False, hardware_trigger +assert hardware_trigger['scheduler']['ran_count'] == 0, hardware_trigger +assert hardware_trigger['scheduler']['target_job_id'] == hardware_trigger['background_job']['job_id'], hardware_trigger +assert unlock_with_passcode['state'] == 'action.denied.input_failed', unlock_with_passcode +assert_provider_attempt_shape(unlock_with_passcode, 'failed') +assert unlock_with_passcode['verification']['status'] == 'failed', unlock_with_passcode +assert_no_sensitive_keys(unlock_with_passcode, 'unlock_with_passcode') +assert run['status'] in ('task.finished', 'task.failed'), run +assert run['task_id'], run +assert run['runner'] == 'deterministic', run +assert run['steps_used'] == 1, run +assert run['limits']['max_steps'] == 1, run +assert run['duration_ms'] >= 0, run +assert run['stop_reason'], run +assert pathlib.Path(run['trajectory']).exists(), run +assert model_status['schema'] == 'openphone.model_status.v1', model_status +assert model_status['status'] == 'disabled', model_status +assert model_status['mode'] == 'broker', model_status +assert model_status['credential']['status'] == 'missing', model_status +assert model_run['status'] == 'task.finished', model_run +assert model_run['runner'] == 'model', model_run +assert model_run['model_provider'] == 'fixture', model_run +assert model_run['steps_used'] == 2, model_run +assert model_run['stop_reason'] == 'finish_task', model_run +assert model_run['parser_failures'] == 0, model_run +assert model_run['tool_errors'] == 0, model_run +assert model_run['unverified_ui_actions'] == 0, model_run +assert pathlib.Path(model_run['trajectory']).exists(), model_run +assert model_unverified_run['status'] == 'task.finished', model_unverified_run +assert model_unverified_run['runner'] == 'model', model_unverified_run +assert model_unverified_run['model_provider'] == 'fixture', model_unverified_run +assert model_unverified_run['steps_used'] == 2, model_unverified_run +assert model_unverified_run['stop_reason'] == 'finish_task', model_unverified_run +assert model_unverified_run['tool_errors'] == 0, model_unverified_run +assert model_unverified_run['unverified_ui_actions'] == 1, model_unverified_run +assert pathlib.Path(model_unverified_run['trajectory']).exists(), model_unverified_run +assert model_visible_run['status'] == 'task.finished', model_visible_run +assert model_visible_run['runner'] == 'model', model_visible_run +assert model_visible_run['model_provider'] == 'fixture', model_visible_run +assert model_visible_run['steps_used'] == 2, model_visible_run +assert model_visible_run['stop_reason'] == 'finish_task', model_visible_run +assert model_visible_run['tool_errors'] == 0, model_visible_run +assert model_visible_run['unverified_ui_actions'] == 0, model_visible_run +assert pathlib.Path(model_visible_run['trajectory']).exists(), model_visible_run +assert start_model_cancel['task_id'], start_model_cancel +assert stop_model_cancel['state'] == 'task.stopped', stop_model_cancel +assert stop_model_cancel['cancel_requested'] is True, stop_model_cancel +assert model_cancel_run['status'] == 'task.cancelled', model_cancel_run +assert model_cancel_run['runner'] == 'model', model_cancel_run +assert model_cancel_run['model_provider'] == 'fixture', model_cancel_run +assert model_cancel_run['steps_used'] == 0, model_cancel_run +assert model_cancel_run['stop_reason'] == 'cancelled', model_cancel_run +assert model_cancel_run['cancel_reason'] == 'local smoke cancellation', model_cancel_run +assert model_cancel_run['parser_failures'] == 0, model_cancel_run +assert model_cancel_run['tool_errors'] == 0, model_cancel_run +assert start_stale_repair['task_id'], start_stale_repair +assert task_repair_stale_active['status'] == 'ok', task_repair_stale_active +assert task_repair_stale_active['repair_policy'] == 'fail_stale_active', task_repair_stale_active +assert task_repair_stale_active['repaired_count'] >= 1, task_repair_stale_active +stale_repair_entries = [ + item for item in task_repair_stale_active['tasks'] + if item.get('task_id') == start_stale_repair['task_id'] +] +assert stale_repair_entries and stale_repair_entries[0]['status'] == 'failed', task_repair_stale_active +assert stale_repair_entries[0]['stop_reason'] == 'stale_active_repaired', task_repair_stale_active +assert get_task_stale_repaired['status'] == 'ok', get_task_stale_repaired +assert get_task_stale_repaired['task']['status'] == 'failed', get_task_stale_repaired +assert get_task_stale_repaired['task']['recovery']['repair_policy'] == 'fail_stale_active', get_task_stale_repaired +assert get_task_stale_repaired['task']['stop_reason'] == 'stale_active_repaired', get_task_stale_repaired +assert get_task_stale_repaired_trajectory['status'] == 'ok', get_task_stale_repaired_trajectory +stale_repair_events = [event['event'] for event in get_task_stale_repaired_trajectory['events']] +assert 'task_repaired' in stale_repair_events, stale_repair_events +assert 'task_failed' in stale_repair_events, stale_repair_events +assert model_repaired_run['status'] == 'task.finished', model_repaired_run +assert model_repaired_run['runner'] == 'model', model_repaired_run +assert model_repaired_run['model_provider'] == 'fixture', model_repaired_run +assert model_repaired_run['steps_used'] == 1, model_repaired_run +assert model_repaired_run['stop_reason'] == 'finish_task', model_repaired_run +assert model_repaired_run['parser_failures'] == 0, model_repaired_run +assert model_repaired_run['tool_errors'] == 0, model_repaired_run +assert pathlib.Path(model_repaired_run['trajectory']).exists(), model_repaired_run +assert model_configure['schema'] == 'openphone.model_status.v1', model_configure +assert model_configure['status'] == 'ready', model_configure +assert model_configure['credential_required'] is False, model_configure +assert model_status_configured['status'] == 'ready', model_status_configured +assert model_status_configured['mode'] == 'broker', model_status_configured +assert model_status_configured['credential']['status'] == 'missing', model_status_configured +assert model_broker_run['status'] == 'task.finished', model_broker_run +assert model_broker_run['runner'] == 'model', model_broker_run +assert model_broker_run['model_provider'] == 'broker', model_broker_run +assert model_broker_run['steps_used'] == 2, model_broker_run +assert model_broker_run['stop_reason'] == 'finish_task', model_broker_run +assert model_broker_run['parser_failures'] == 0, model_broker_run +assert model_broker_run['tool_errors'] == 0, model_broker_run +assert pathlib.Path(model_broker_run['trajectory']).exists(), model_broker_run +assert model_configure_realtime2['schema'] == 'openphone.model_status.v1', model_configure_realtime2 +assert model_configure_realtime2['status'] == 'ready', model_configure_realtime2 +assert model_configure_realtime2['mode'] == 'openai_realtime2', model_configure_realtime2 +assert model_configure_realtime2['model'] == 'gpt-realtime-2', model_configure_realtime2 +assert model_configure_realtime2['endpoint_configured'] is True, model_configure_realtime2 +assert not model_configure_realtime2['endpoint_url_configured'], model_configure_realtime2 +assert model_configure_realtime2['credential_required'] is True, model_configure_realtime2 +assert model_configure_realtime2['credential']['status'] == 'present', model_configure_realtime2 +assert model_configure_realtime2['credential']['source'] == 'credential_file', model_configure_realtime2 +assert model_configure_realtime2['max_steps'] == 40, model_configure_realtime2 +assert model_configure_realtime2['max_duration_ms'] == 600000, model_configure_realtime2 +assert model_status_realtime2['status'] == 'ready', model_status_realtime2 +assert model_status_realtime2['mode'] == 'openai_realtime2', model_status_realtime2 +assert model_status_realtime2['model'] == 'gpt-realtime-2', model_status_realtime2 +assert model_status_realtime2['credential_required'] is True, model_status_realtime2 +assert model_status_realtime2['credential']['status'] == 'present', model_status_realtime2 +assert list(base.joinpath('store/tasks').glob('*.json')), 'missing task file' +assert base.joinpath('store/audit/audit-events.jsonl').exists(), 'missing audit jsonl' +# Value-level retention scan: distinctive PII markers planted in provider +# fixtures must never appear verbatim in the persisted audit log or notification +# log. assert_no_sensitive_keys only checks key *names*; this proves the +# retention contract at the raw-bytes level for the durable stores. +provider_pii_markers = ( + '+15550101234', # calls fixture address + 'OpenPhone Test Call', # calls fixture display name + 'OpenPhone message fixture says hello', # messages fixture body + '+15550105555', # messages fixture handle + 'Discuss iOS agent progress', # calendar fixture notes + 'ada@example.test', # contacts fixture email + 'NOTIFYPIItail_+15559998888_leak@example.test', # notification tail marker +) +audit_raw = base.joinpath('store/audit/audit-events.jsonl').read_text(encoding='utf-8') +for marker in provider_pii_markers: + assert marker not in audit_raw, f'provider PII leaked verbatim into audit log: {marker}' +notif_log_path = base.joinpath('store/notifications/recent.json') +if notif_log_path.exists(): + notif_log_raw = notif_log_path.read_text(encoding='utf-8') + assert 'NOTIFYPIItail_+15559998888_leak@example.test' not in notif_log_raw, \ + 'notification tail PII leaked into stored notification log' +assert list_tasks['status'] == 'ok', list_tasks +assert list_tasks['tasks'], list_tasks +assert get_task['status'] == 'ok', get_task +assert get_task['task']['task_id'] == run['task_id'], get_task +assert get_trajectory['status'] == 'ok', get_trajectory +assert get_trajectory['events'], get_trajectory +assert get_model_trajectory['status'] == 'ok', get_model_trajectory +model_events = [event['event'] for event in get_model_trajectory['events']] +assert 'model_prompt_prepared' in model_events, model_events +assert model_events.count('model_decision') == 2, model_events +assert 'model_step_verified' in model_events, model_events +assert 'model_loop_finished' in model_events, model_events +assert get_model_unverified_trajectory['status'] == 'ok', get_model_unverified_trajectory +unverified_events = get_model_unverified_trajectory['events'] +unverified_model_events = [event['event'] for event in unverified_events] +assert 'model_step_verified' in unverified_model_events, unverified_model_events +unverified_checks = [ + event['payload']['verification'] + for event in unverified_events + if event.get('event') == 'model_step_verified' +] +assert any(check['status'] == 'unverified_dispatch_only' for check in unverified_checks), unverified_checks +assert any( + check['screen_delta']['status'] == 'unchanged' + for check in unverified_checks + if check['status'] == 'unverified_dispatch_only' +), unverified_checks +assert get_model_visible_trajectory['status'] == 'ok', get_model_visible_trajectory +visible_events = get_model_visible_trajectory['events'] +visible_model_events = [event['event'] for event in visible_events] +assert 'model_step_verified' in visible_model_events, visible_model_events +visible_checks = [ + event['payload']['verification'] + for event in visible_events + if event.get('event') == 'model_step_verified' +] +assert any(check['status'] == 'verified' for check in visible_checks), visible_checks +assert any( + check['screen_delta']['strong_signal'] is True and + ('visible_text' in check['screen_delta']['signals'] or 'ui_tree' in check['screen_delta']['signals']) + for check in visible_checks + if check['status'] == 'verified' +), visible_checks +assert get_model_cancel_trajectory['status'] == 'ok', get_model_cancel_trajectory +cancel_model_events = [event['event'] for event in get_model_cancel_trajectory['events']] +assert 'task_stopped' in cancel_model_events, cancel_model_events +assert 'model_loop_cancelled' in cancel_model_events, cancel_model_events +assert 'model_loop_finished' in cancel_model_events, cancel_model_events +assert 'model_decision' not in cancel_model_events, cancel_model_events +assert get_model_repaired_trajectory['status'] == 'ok', get_model_repaired_trajectory +repaired_model_events = [event['event'] for event in get_model_repaired_trajectory['events']] +assert 'model_prompt_prepared' in repaired_model_events, repaired_model_events +assert 'model_parse_repaired' in repaired_model_events, repaired_model_events +assert repaired_model_events.count('model_decision') == 1, repaired_model_events +assert 'model_loop_finished' in repaired_model_events, repaired_model_events +assert get_model_broker_trajectory['status'] == 'ok', get_model_broker_trajectory +broker_model_events = [event['event'] for event in get_model_broker_trajectory['events']] +assert 'model_prompt_prepared' in broker_model_events, broker_model_events +assert broker_model_events.count('model_request') == 2, broker_model_events +assert broker_model_events.count('model_response') == 2, broker_model_events +assert broker_model_events.count('model_decision') == 2, broker_model_events +assert 'model_step_verified' in broker_model_events, broker_model_events +assert 'model_loop_finished' in broker_model_events, broker_model_events +assert get_redaction_trajectory['status'] == 'ok', get_redaction_trajectory +assert get_redaction_trajectory['events'], get_redaction_trajectory +assert get_audit['status'] == 'ok', get_audit +assert get_audit['events'], get_audit +assert_no_sensitive_keys(get_audit, 'get_audit') +assert_no_sensitive_keys(get_redaction_trajectory, 'get_redaction_trajectory') +assert_no_sensitive_keys(agent_status_initial, 'agent_status_initial') +assert_no_sensitive_keys(agent_control_pause, 'agent_control_pause') +assert_no_sensitive_keys(agent_status_paused, 'agent_status_paused') +assert_no_sensitive_keys(hardware_trigger_paused, 'hardware_trigger_paused') +assert_no_sensitive_keys(agent_control_resume, 'agent_control_resume') +assert_no_sensitive_keys(agent_status_resumed, 'agent_status_resumed') +assert_no_sensitive_keys(agent_control_disable_hardware, 'agent_control_disable_hardware') +assert_no_sensitive_keys(hardware_trigger_disabled, 'hardware_trigger_disabled') +assert_no_sensitive_keys(agent_control_enable_hardware, 'agent_control_enable_hardware') +assert_no_sensitive_keys(agent_control_disable_yolo, 'agent_control_disable_yolo') +assert_no_sensitive_keys(hardware_trigger_yolo_disabled, 'hardware_trigger_yolo_disabled') +assert_no_sensitive_keys(agent_control_enable_yolo, 'agent_control_enable_yolo') +assert_no_sensitive_keys(model_status, 'model_status') +assert_no_sensitive_keys(model_configure, 'model_configure') +assert_no_sensitive_keys(model_status_configured, 'model_status_configured') +assert_no_sensitive_keys(model_configure_realtime2, 'model_configure_realtime2') +assert_no_sensitive_keys(model_status_realtime2, 'model_status_realtime2') +assert_no_sensitive_keys(get_model_trajectory, 'get_model_trajectory') +assert_no_sensitive_keys(get_model_unverified_trajectory, 'get_model_unverified_trajectory') +assert_no_sensitive_keys(get_model_visible_trajectory, 'get_model_visible_trajectory') +assert_no_sensitive_keys(get_model_cancel_trajectory, 'get_model_cancel_trajectory') +assert_no_sensitive_keys(get_task_stale_repaired_trajectory, 'get_task_stale_repaired_trajectory') +assert_no_sensitive_keys(get_model_repaired_trajectory, 'get_model_repaired_trajectory') +assert_no_sensitive_keys(get_model_broker_trajectory, 'get_model_broker_trajectory') +assert validate_store['status'] == 'ok', validate_store +assert validate_store['audit_events'] >= get_audit['count'], validate_store +assert validate_store['sqlite']['commitment_rows'] >= 1, validate_store +assert validate_store['sqlite']['watcher_rows'] >= 1, validate_store +assert validate_store['sqlite']['agent_job_rows'] >= 1, validate_store + +print(json.dumps({ + 'health_status': health['status'], + 'springboard_state_foreground': get_screen_springboard['context']['foreground_app'], + 'app_ui_tree_source': get_screen_app_ui['context']['ui_tree_source'], + 'tap_element_state': tap_element['state'], + 'tap_element_detail': tap_element['detail'], + 'tap_element_user_facing_status': tap_element['user_facing_status'], + 'app_input_type_text_user_facing_status': app_input_type_text['user_facing_status'], + 'app_input_type_text_verification': app_input_type_text['verification']['status'], + 'memory_id': memory_save['memory']['memory_id'], + 'memory_updated_id': memory_update['memory_id'], + 'memory_merged_from': memory_merge['merged_from'], + 'memory_deleted_id': memory_delete['memory_id'], + 'memory_search_count': memory_search['count'], + 'context_search_count': context_search['count'], + 'clipboard_provider': clipboard_read['provider'], + 'clipboard_system_clipboard': clipboard_read['system_clipboard'], + 'clipboard_text_length': clipboard_read['text_length'], + 'clipboard_context_events': context_search_clipboard['count'], + 'contacts_provider': contacts_search['provider'], + 'contacts_count': contacts_search['count'], + 'contacts_context_events': context_search_contacts['count'], + 'calendar_provider': calendar_search['provider'], + 'calendar_count': calendar_search['count'], + 'calendar_context_events': context_search_calendar['count'], + 'calls_provider': calls_search['provider'], + 'calls_count': calls_search['count'], + 'calls_context_events': context_search_calls['count'], + 'messages_provider': messages_search['provider'], + 'messages_count': messages_search['count'], + 'messages_context_events': context_search_messages['count'], + 'notification_stored_count': notification_ingest['stored_count'], + 'notification_context_events': context_search_notification['count'], + 'commitment_id': commitment_create['commitment']['commitment_id'], + 'commitment_due_id': commitment_due_public_id, + 'commitment_due_job_id': commitment_due_entry['job_id'], + 'watcher_id': watcher_create['watcher']['watcher_id'], + 'recurring_watcher_id': recurring_watcher_public_id, + 'recurring_watcher_second_job_id': recurring_second_entries[0]['job_id'], + 'watcher_repair_id': repair_watcher_public_id, + 'job_id': background_job_create['job']['job_id'], + 'job_scheduler_ran_count': background_job_run_due['ran_count'], + 'recurring_job_id': recurring_job_public_id, + 'recurring_job_next_run_at_ms': recurring_job['next_run_at_ms'], + 'nonrecurring_interval_job_id': nonrecurring_interval_job_public_id, + 'backoff_job_id': backoff_job_public_id, + 'backoff_retry_ms': backoff_job['schedule']['retry_backoff_ms'], + 'agent_status_state': agent_status_resumed['state'], + 'agent_trigger_policy': agent_status_resumed['control']['trigger_policy'], + 'paused_trigger_state': hardware_trigger_paused['state'], + 'disabled_trigger_state': hardware_trigger_disabled['state'], + 'yolo_disabled_trigger_state': hardware_trigger_yolo_disabled['state'], + 'trigger_task_id': hardware_trigger['task_id'], + 'trigger_scheduler_ran_count': hardware_trigger['scheduler']['ran_count'], + 'trigger_background_job_id': hardware_trigger['background_job']['job']['job_id'], + 'redaction_trajectory_events': get_redaction_trajectory['count'], + 'unlock_user_facing_status': unlock_with_passcode['user_facing_status'], + 'run_status': run['status'], + 'model_status': model_status['status'], + 'model_run_status': model_run['status'], + 'model_run_provider': model_run['model_provider'], + 'model_run_steps': model_run['steps_used'], + 'model_unverified_ui_actions': model_unverified_run['unverified_ui_actions'], + 'model_visible_ui_actions': model_visible_run['unverified_ui_actions'], + 'model_cancel_status': model_cancel_run['status'], + 'model_cancel_stop_reason': model_cancel_run['stop_reason'], + 'model_cancel_task_id': model_cancel_run['task_id'], + 'stale_task_repair_status': get_task_stale_repaired['task']['status'], + 'stale_task_repair_task_id': start_stale_repair['task_id'], + 'model_repair_status': model_repaired_run['status'], + 'model_repair_task_id': model_repaired_run['task_id'], + 'model_broker_status': model_broker_run['status'], + 'model_broker_provider': model_broker_run['model_provider'], + 'model_broker_steps': model_broker_run['steps_used'], + 'model_realtime2_status': model_status_realtime2['status'], + 'model_realtime2_mode': model_status_realtime2['mode'], + 'model_realtime2_model': model_status_realtime2['model'], + 'task_id': run['task_id'], + 'model_task_id': model_run['task_id'], + 'model_repair_trajectory_events': get_model_repaired_trajectory['count'], + 'model_broker_task_id': model_broker_run['task_id'], + 'trajectory_exists': pathlib.Path(run['trajectory']).exists(), + 'trajectory_events': get_trajectory['count'], + 'model_trajectory_events': get_model_trajectory['count'], + 'model_cancel_trajectory_events': get_model_cancel_trajectory['count'], + 'model_broker_trajectory_events': get_model_broker_trajectory['count'], + 'audit_events': get_audit['count'], + 'validated_audit_events': validate_store['audit_events'], + 'audit_bytes': base.joinpath('store/audit/audit-events.jsonl').stat().st_size, +}, indent=2)) +PY diff --git a/ios/tools/mac/agentd/tail-agentd-log.sh b/ios/tools/mac/agentd/tail-agentd-log.sh new file mode 100755 index 0000000..bc6e9ef --- /dev/null +++ b/ios/tools/mac/agentd/tail-agentd-log.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +host="${OPENPHONE_IOS_HOST:-127.0.0.1}" +port="${OPENPHONE_IOS_SSH_PORT:-22}" +user="${OPENPHONE_IOS_USER:-mobile}" +lines="${OPENPHONE_AGENTD_TAIL_LINES:-80}" +if [[ ! "$lines" =~ ^[0-9]+$ ]]; then + lines=80 +fi + +ssh -p "$port" "$user@$host" " +set -eu +mkdir -p /var/mobile/Library/OpenPhone +touch /var/mobile/Library/OpenPhone/openphone-agentd.log +tail -n $lines -f /var/mobile/Library/OpenPhone/openphone-agentd.log +" diff --git a/ios/tools/mac/agentd/validate-agentd-store.py b/ios/tools/mac/agentd/validate-agentd-store.py new file mode 100755 index 0000000..5e7137f --- /dev/null +++ b/ios/tools/mac/agentd/validate-agentd-store.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import pathlib +import sqlite3 +import sys + + +def read_json(path): + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def read_jsonl(path): + events = [] + if not path.exists(): + return events + with path.open("r", encoding="utf-8") as handle: + for line_no, line in enumerate(handle, 1): + stripped = line.strip() + if not stripped: + continue + try: + events.append(json.loads(stripped)) + except json.JSONDecodeError as exc: + raise ValueError(f"{path}:{line_no}: invalid jsonl: {exc}") from exc + return events + + +def canonical_hashes(event): + body = dict(event) + body.pop("event_hash", None) + encoded = json.dumps( + body, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + hashes = {hashlib.sha256(encoded).hexdigest()} + escaped_slash = encoded.replace(b"/", b"\\/") + if escaped_slash != encoded: + hashes.add(hashlib.sha256(escaped_slash).hexdigest()) + return hashes + + +def validate_audit(events): + previous_hash = "" + for index, event in enumerate(events): + event_hash = event.get("event_hash") + if not event_hash: + raise ValueError(f"audit event {index} missing event_hash") + if event.get("previous_hash", "") != previous_hash: + raise ValueError(f"audit event {index} previous_hash mismatch") + if event_hash not in canonical_hashes(event): + raise ValueError(f"audit event {index} event_hash mismatch") + previous_hash = event_hash + + +def inspect_sqlite_store(store): + db_path = store / "db" / "openphone.sqlite" + if not db_path.exists(): + return { + "db_exists": False, + "path": str(db_path), + } + conn = sqlite3.connect(db_path) + try: + tables = { + row[0] + for row in conn.execute("select name from sqlite_master where type in ('table', 'view')") + } + result = { + "db_exists": True, + "path": str(db_path), + "tables": sorted(tables), + "memory_rows": 0, + "context_event_rows": 0, + "commitment_rows": 0, + "watcher_rows": 0, + "agent_job_rows": 0, + "memory_fts": "memory_fts" in tables, + "context_event_fts": "context_event_fts" in tables, + "commitment_fts": "commitment_fts" in tables, + "watcher_fts": "watcher_fts" in tables, + "agent_job_fts": "agent_job_fts" in tables, + } + if "memory" in tables: + result["memory_rows"] = conn.execute("select count(*) from memory").fetchone()[0] + if "context_event" in tables: + result["context_event_rows"] = conn.execute("select count(*) from context_event").fetchone()[0] + if "commitment" in tables: + result["commitment_rows"] = conn.execute("select count(*) from commitment").fetchone()[0] + if "watcher" in tables: + result["watcher_rows"] = conn.execute("select count(*) from watcher").fetchone()[0] + if "agent_job" in tables: + result["agent_job_rows"] = conn.execute("select count(*) from agent_job").fetchone()[0] + return result + finally: + conn.close() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("store", type=pathlib.Path) + parser.add_argument("--require-task-artifacts", action="store_true") + args = parser.parse_args() + + store = args.store + tasks_dir = store / "tasks" + audit_path = store / "audit" / "audit-events.jsonl" + trajectories_dir = store / "trajectories" + + task_paths = sorted(tasks_dir.glob("*.json")) if tasks_dir.exists() else [] + tasks = [read_json(path) for path in task_paths] + audit_events = read_jsonl(audit_path) + validate_audit(audit_events) + + trajectory_paths = sorted(trajectories_dir.glob("*.jsonl")) if trajectories_dir.exists() else [] + trajectory_counts = { + path.name: len(read_jsonl(path)) + for path in trajectory_paths + } + sqlite_store = inspect_sqlite_store(store) + + if args.require_task_artifacts: + if not tasks: + raise ValueError("missing task artifacts") + if not audit_events: + raise ValueError("missing audit events") + if not trajectory_paths: + raise ValueError("missing trajectory artifacts") + for task in tasks: + task_id = task.get("task_id") + if not task_id: + raise ValueError("task missing task_id") + expected = trajectories_dir / f"{task_id}.jsonl" + if not expected.exists(): + raise ValueError(f"missing trajectory for {task_id}") + + print(json.dumps({ + "status": "ok", + "store": str(store), + "tasks": len(tasks), + "audit_events": len(audit_events), + "trajectories": len(trajectory_paths), + "trajectory_events": sum(trajectory_counts.values()), + "sqlite": sqlite_store, + }, indent=2)) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"validate-agentd-store: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/ios/tools/mac/agentd/validate-on-device.sh b/ios/tools/mac/agentd/validate-on-device.sh new file mode 100755 index 0000000..9caa4b2 --- /dev/null +++ b/ios/tools/mac/agentd/validate-on-device.sh @@ -0,0 +1,5784 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +mode="baseline" +run_id="$(date '+%Y%m%dT%H%M%S%z')" + +usage() { + cat <<'EOF' +Usage: + tools/mac/agentd/validate-on-device.sh [--mode baseline|full|collect-only] [--run-id RUN_ID] + +Environment: + OPENPHONE_IOS_HOST SSH host, default 127.0.0.1 + OPENPHONE_IOS_SSH_PORT SSH port, default 22 + OPENPHONE_IOS_USER SSH user, default mobile + OPENPHONE_IOS_PASSWORD Optional development password + OPENPHONE_IOS_KNOWN_HOSTS Known-hosts file, default /tmp/openphone-ios-known-hosts + OPENPHONE_IOS_UDID Target iPhone UDID for temporary iproxy + OPENPHONE_IOS_EXPECTED_DEVICE_NAME Optional remote uname -n guard, for example + OPENPHONE_IOS_EXPECTED_PRODUCT_TYPE Optional remote uname guard, for example iPhone15,3 + OPENPHONE_AGENTD_DEB Optional explicit package path + OPENPHONE_VALIDATION_DIR Output root, default artifacts/validation + OPENPHONE_VALIDATE_START_IPROXY Set to 1 to start a temporary iproxy tunnel + OPENPHONE_VALIDATE_ALLOW_UNPINNED_IPROXY + Set to 1 only for a confirmed single-device setup + OPENPHONE_VALIDATE_ALLOW_EXISTING_IPROXY + Set to 1 to use an existing listener when START_IPROXY=1 + OPENPHONE_VALIDATE_INCLUDE_SCREENSHOT + Set to 1 to run get_screen screenshot and pull the PNG + OPENPHONE_VALIDATE_INCLUDE_UNLOCKED_FOREGROUND + Set to 1 to launch Safari and verify unlocked foreground source + OPENPHONE_VALIDATE_INCLUDE_APP_UI Set to 1 to relaunch Safari/Settings and verify app-process UI trees + OPENPHONE_VALIDATE_INCLUDE_LOCKSCREEN + Set to 1 to verify locked SpringBoard show_passcode behavior. + OPENPHONE_VALIDATE_INCLUDE_PREFS_UI Set to 1 to open and exercise the OpenPhone Settings pane. Requires unlocked phone. + OPENPHONE_VALIDATE_INCLUDE_PREFS_BACKEND + Set to 1 to verify OpenPhone Settings pane files and daemon policy controls + OPENPHONE_VALIDATE_INCLUDE_STORES Set to 1 to collect safe store/task reads + OPENPHONE_VALIDATE_INCLUDE_PROVIDER_ATTEMPTS + Set to 1 to run a no-dispatch provider-attempt shape sample + OPENPHONE_VALIDATE_INCLUDE_VISIBLE_EFFECTS + Set to 1 to verify real UI visible effects and screenshot-hash changes for Settings tap, Safari DOM text entry, and Notes body text entry + OPENPHONE_VALIDATE_INCLUDE_MEMORY_LIFECYCLE + Set to 1 to run memory save/update/merge/delete lifecycle sample + OPENPHONE_VALIDATE_INCLUDE_MODEL_LOOP + Set to 1 to run a fixture-backed model-loop task sample + OPENPHONE_VALIDATE_INCLUDE_VOICE Set to 1 to snapshot voice_status + island-status + credential presence + OPENPHONE_VALIDATE_INCLUDE_WATCHER_TIMER + Set to 1 to run a due timer watcher firing sample + OPENPHONE_VALIDATE_INCLUDE_WATCHER_REPAIR + Set to 1 to run a stale watcher repair sample + OPENPHONE_VALIDATE_INCLUDE_JOB_REPAIR + Set to 1 to run a stale background-job repair sample + OPENPHONE_VALIDATE_INCLUDE_RESTART_RECOVERY + Set to 1 to restart daemon with stale watcher/job rows + OPENPHONE_VALIDATE_RESTART_RECOVERY_WAIT_SECONDS + Seconds to wait after daemon restart before grading recovery + OPENPHONE_VALIDATE_RESTART_RECOVERY_START_TIMEOUT_SECONDS + Seconds to wait for launchd to restart daemon + OPENPHONE_VALIDATE_RESTART_RECOVERY_STABLE_SECONDS + Seconds the restarted daemon PID must remain stable + OPENPHONE_VALIDATE_INCLUDE_PROVIDER_MODEL + Set to 1 to run a provider-backed Bedrock broker model sample + OPENPHONE_VALIDATE_INCLUDE_SAFARI_DOM_MODEL + Set to 1 to run a direct Bedrock Safari DOM text-entry model sample. + Uses an existing phone credential file when present; otherwise requires a Bedrock token env var. + OPENPHONE_VALIDATE_INCLUDE_PROMPT_BRIDGE_MODEL + Set to 1 to run the SpringBoard prompt bridge into the phone-local model loop. + Requires the phone to be unlocked and the current model config to be ready. + OPENPHONE_VALIDATE_INCLUDE_TRIGGER_DIAGNOSTICS + Set to 1 to wait for a real physical Volume Up + Volume Down press and compare + before/after SpringBoard trigger counters. Requires an unlocked phone and a + human pressing the hardware buttons during the wait window. + OPENPHONE_VALIDATE_TRIGGER_WAIT_SECONDS + Seconds to wait for the physical trigger diagnostic window, default 20. + AWS_BEARER_TOKEN_BEDROCK Bedrock API key for provider-backed validation + OPENPHONE_BEDROCK_BEARER_TOKEN Alternate Bedrock API key env var + OPENPHONE_BEDROCK_MODEL Bedrock model id, default Claude Haiku 4.5 + OPENPHONE_BEDROCK_REGION Bedrock region, default us-east-1 + OPENPHONE_VALIDATE_PROVIDER_MODEL_PHONE_PORT + Phone-local reverse-forward port, default 18765 + OPENPHONE_VALIDATE_REQUIRE_UNLOCKED Set to 1 to require the unlocked foreground gate + +Modes: + baseline local preflight, package build/install, health/stability collection + full baseline plus safe store/task collection + collect-only no build/install; collect current device state only +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --mode) + mode="${2:-}" + shift 2 + ;; + --run-id) + run_id="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +case "$mode" in + baseline|full|collect-only) ;; + *) + echo "invalid mode: $mode" >&2 + exit 2 + ;; +esac + +if [[ ! "$run_id" =~ ^[A-Za-z0-9_.:+-]+$ ]]; then + echo "invalid run id: $run_id" >&2 + exit 2 +fi + +host="${OPENPHONE_IOS_HOST:-127.0.0.1}" +port="${OPENPHONE_IOS_SSH_PORT:-22}" +user="${OPENPHONE_IOS_USER:-mobile}" +password="${OPENPHONE_IOS_PASSWORD:-}" +known_hosts="${OPENPHONE_IOS_KNOWN_HOSTS:-/tmp/openphone-ios-known-hosts}" +validation_root="${OPENPHONE_VALIDATION_DIR:-$repo_root/artifacts/validation}" +run_dir="$validation_root/$run_id" +remote_tmp="/tmp/openphone-validation-$run_id" +ssh_target="$user@$host" +tail_lines="${OPENPHONE_VALIDATE_TAIL_LINES:-160}" +package="${OPENPHONE_AGENTD_DEB:-}" +target_udid="${OPENPHONE_IOS_UDID:-${OPENPHONE_VALIDATE_IPROXY_UDID:-}}" +started_iproxy_pid="" +remote_tmp_created=0 +provider_broker_pid="" +provider_ssh_pid="" +provider_broker_port="" +provider_phone_port="${OPENPHONE_VALIDATE_PROVIDER_MODEL_PHONE_PORT:-18765}" +direct_bedrock_credential_touched=0 +direct_bedrock_credential_had_backup=0 +direct_bedrock_credential_temp="" +direct_bedrock_config_touched=0 +direct_bedrock_config_had_backup=0 +trigger_wait_seconds="${OPENPHONE_VALIDATE_TRIGGER_WAIT_SECONDS:-20}" + +if [[ ! "$tail_lines" =~ ^[0-9]+$ ]]; then + echo "invalid OPENPHONE_VALIDATE_TAIL_LINES: $tail_lines" >&2 + exit 2 +fi +if [[ ! "$trigger_wait_seconds" =~ ^[0-9]+$ ]]; then + echo "invalid OPENPHONE_VALIDATE_TRIGGER_WAIT_SECONDS: $trigger_wait_seconds" >&2 + exit 2 +fi + +mkdir -p "$run_dir" +summary_log="$run_dir/validate.log" +: >"$summary_log" + +log() { + printf '%s\n' "$*" | tee -a "$summary_log" +} + +fail() { + local code="$1" + shift + log "ERROR: $*" + exit "$code" +} + +redact_udid() { + local value="$1" + if [[ -z "$value" ]]; then + printf '%s\n' "" + elif [[ "${#value}" -le 8 ]]; then + printf '%s\n' "..." + else + printf '%s\n' "...${value: -8}" + fi +} + +cleanup() { + if [[ "$direct_bedrock_config_touched" == "1" ]]; then + if [[ "$direct_bedrock_config_had_backup" == "1" ]]; then + remote_exec "install -m 0600 '$remote_tmp/model-config.backup' /var/mobile/Library/OpenPhone/config/model.json" "$run_dir/cleanup.log" >/dev/null 2>&1 || true + else + remote_exec "rm -f /var/mobile/Library/OpenPhone/config/model.json" "$run_dir/cleanup.log" >/dev/null 2>&1 || true + fi + fi + if [[ "$direct_bedrock_credential_touched" == "1" ]]; then + if [[ "$direct_bedrock_credential_had_backup" == "1" ]]; then + remote_exec "install -m 0600 '$remote_tmp/model-credential.backup' /var/mobile/Library/OpenPhone/config/model-credential.json" "$run_dir/cleanup.log" >/dev/null 2>&1 || true + else + remote_exec "rm -f /var/mobile/Library/OpenPhone/config/model-credential.json" "$run_dir/cleanup.log" >/dev/null 2>&1 || true + fi + fi + if [[ -n "$direct_bedrock_credential_temp" ]]; then + rm -f "$direct_bedrock_credential_temp" >/dev/null 2>&1 || true + fi + if [[ "$remote_tmp_created" == "1" && "${OPENPHONE_VALIDATE_KEEP_REMOTE_TMP:-0}" != "1" ]]; then + remote_exec "rm -rf '$remote_tmp'" "$run_dir/cleanup.log" >/dev/null 2>&1 || true + fi + if [[ -n "$started_iproxy_pid" ]]; then + kill "$started_iproxy_pid" >/dev/null 2>&1 || true + wait "$started_iproxy_pid" >/dev/null 2>&1 || true + fi + if [[ -n "$provider_ssh_pid" ]]; then + kill "$provider_ssh_pid" >/dev/null 2>&1 || true + wait "$provider_ssh_pid" >/dev/null 2>&1 || true + fi + if [[ -n "$provider_broker_pid" ]]; then + kill "$provider_broker_pid" >/dev/null 2>&1 || true + wait "$provider_broker_pid" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + fail "${2:-10}" "missing required command: $1" + fi +} + +run_local() { + local name="$1" + local code="$2" + shift 2 + local log_path="$run_dir/$name.log" + log "Running $name" + if ! "$@" >"$log_path" 2>&1; then + fail "$code" "$name failed; see $log_path" + fi +} + +run_expect_ssh() { + local remote_cmd="$1" + local log_path="$2" + OPENPHONE_IOS_PASSWORD="$password" expect -f - -- \ + "$port" "$known_hosts" "$ssh_target" "$remote_cmd" >>"$log_path" 2>&1 <<'EXPECT' +set timeout 180 +set port [lindex $argv 0] +set known_hosts [lindex $argv 1] +set target [lindex $argv 2] +set remote_cmd [lindex $argv 3] +spawn ssh -p $port -o StrictHostKeyChecking=no -o UserKnownHostsFile=$known_hosts $target $remote_cmd +expect { + -nocase -re "yes/no" { send "yes\r"; exp_continue } + -nocase -re "password.*:" { send "$env(OPENPHONE_IOS_PASSWORD)\r"; exp_continue } + eof {} + timeout { exit 124 } +} +catch wait result +exit [lindex $result 3] +EXPECT +} + +run_expect_ssh_reverse() { + local remote_port="$1" + local local_port="$2" + local log_path="$3" + OPENPHONE_IOS_PASSWORD="$password" expect -f - -- \ + "$port" "$known_hosts" "$ssh_target" "$remote_port" "$local_port" >>"$log_path" 2>&1 <<'EXPECT' +set timeout 180 +set port [lindex $argv 0] +set known_hosts [lindex $argv 1] +set target [lindex $argv 2] +set remote_port [lindex $argv 3] +set local_port [lindex $argv 4] +spawn ssh -N -p $port -o StrictHostKeyChecking=no -o UserKnownHostsFile=$known_hosts -o ExitOnForwardFailure=yes -o ServerAliveInterval=15 -R 127.0.0.1:$remote_port:127.0.0.1:$local_port $target +expect { + -nocase -re "yes/no" { send "yes\r"; exp_continue } + -nocase -re "password.*:" { send "$env(OPENPHONE_IOS_PASSWORD)\r"; exp_continue } + eof { exit 1 } + timeout { } +} +expect eof +EXPECT +} + +start_ssh_reverse_forward() { + local remote_port="$1" + local local_port="$2" + local log_path="$run_dir/provider-model-ssh-reverse.log" + if [[ -n "$password" ]]; then + require_command expect 30 + run_expect_ssh_reverse "$remote_port" "$local_port" "$log_path" & + provider_ssh_pid="$!" + else + ssh -N -p "$port" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile="$known_hosts" \ + -o ExitOnForwardFailure=yes \ + -o ServerAliveInterval=15 \ + -R "127.0.0.1:$remote_port:127.0.0.1:$local_port" \ + "$ssh_target" >>"$log_path" 2>&1 & + provider_ssh_pid="$!" + fi + sleep 2 + if ! kill -0 "$provider_ssh_pid" >/dev/null 2>&1; then + fail 100 "provider model SSH reverse tunnel failed; see $log_path" + fi +} + +remote_exec() { + local remote_cmd="$1" + local log_path="${2:-$run_dir/ssh.log}" + if [[ -n "$password" ]]; then + require_command expect 30 + run_expect_ssh "$remote_cmd" "$log_path" + else + ssh -p "$port" -o StrictHostKeyChecking=no -o UserKnownHostsFile="$known_hosts" \ + "$ssh_target" "$remote_cmd" >>"$log_path" 2>&1 + fi +} + +run_expect_scp_from() { + local remote_path="$1" + local local_path="$2" + local log_path="$3" + OPENPHONE_IOS_PASSWORD="$password" expect -f - -- \ + "$port" "$known_hosts" "$ssh_target:$remote_path" "$local_path" >>"$log_path" 2>&1 <<'EXPECT' +set timeout 180 +set port [lindex $argv 0] +set known_hosts [lindex $argv 1] +set source [lindex $argv 2] +set dest [lindex $argv 3] +spawn scp -P $port -o StrictHostKeyChecking=no -o UserKnownHostsFile=$known_hosts $source $dest +expect { + -nocase -re "yes/no" { send "yes\r"; exp_continue } + -nocase -re "password.*:" { send "$env(OPENPHONE_IOS_PASSWORD)\r"; exp_continue } + eof {} + timeout { exit 124 } +} +catch wait result +exit [lindex $result 3] +EXPECT +} + +run_expect_scp_to() { + local local_path="$1" + local remote_path="$2" + local log_path="$3" + OPENPHONE_IOS_PASSWORD="$password" expect -f - -- \ + "$port" "$known_hosts" "$local_path" "$ssh_target:$remote_path" >>"$log_path" 2>&1 <<'EXPECT' +set timeout 180 +set port [lindex $argv 0] +set known_hosts [lindex $argv 1] +set source [lindex $argv 2] +set dest [lindex $argv 3] +spawn scp -P $port -o StrictHostKeyChecking=no -o UserKnownHostsFile=$known_hosts $source $dest +expect { + -nocase -re "yes/no" { send "yes\r"; exp_continue } + -nocase -re "password.*:" { send "$env(OPENPHONE_IOS_PASSWORD)\r"; exp_continue } + eof {} + timeout { exit 124 } +} +catch wait result +exit [lindex $result 3] +EXPECT +} + +scp_from() { + local remote_path="$1" + local local_path="$2" + local log_path="${3:-$run_dir/scp.log}" + mkdir -p "$(dirname "$local_path")" + if [[ -n "$password" ]]; then + require_command expect 30 + run_expect_scp_from "$remote_path" "$local_path" "$log_path" + else + scp -P "$port" -o StrictHostKeyChecking=no -o UserKnownHostsFile="$known_hosts" \ + "$ssh_target:$remote_path" "$local_path" >>"$log_path" 2>&1 + fi +} + +scp_to() { + local local_path="$1" + local remote_path="$2" + local log_path="${3:-$run_dir/scp.log}" + if [[ -n "$password" ]]; then + require_command expect 30 + run_expect_scp_to "$local_path" "$remote_path" "$log_path" + else + scp -P "$port" -o StrictHostKeyChecking=no -o UserKnownHostsFile="$known_hosts" \ + "$local_path" "$ssh_target:$remote_path" >>"$log_path" 2>&1 + fi +} + +remote_capture() { + local name="$1" + local command="$2" + local remote_path="$remote_tmp/$name" + local status_path="$remote_path.status" + remote_exec "mkdir -p '$remote_tmp'; ( $command ) > '$remote_path' 2> '$remote_path.stderr'; printf '%s\n' \$? > '$status_path'; exit 0" + remote_tmp_created=1 + scp_from "$remote_path" "$run_dir/$name" + scp_from "$remote_path.stderr" "$run_dir/$name.stderr" || true + scp_from "$status_path" "$run_dir/$name.status" || true +} + +start_iproxy_if_requested() { + if [[ "${OPENPHONE_VALIDATE_START_IPROXY:-0}" != "1" ]]; then + return + fi + require_command iproxy 30 + if [[ -z "$target_udid" && "${OPENPHONE_VALIDATE_ALLOW_UNPINNED_IPROXY:-0}" != "1" ]]; then + fail 30 "refusing to start or reuse iproxy without OPENPHONE_IOS_UDID; set OPENPHONE_VALIDATE_ALLOW_UNPINNED_IPROXY=1 only for a confirmed single-device setup" + fi + if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then + if [[ "${OPENPHONE_VALIDATE_ALLOW_EXISTING_IPROXY:-0}" != "1" ]]; then + fail 30 "local port $port already has a listener; stop it or set OPENPHONE_VALIDATE_ALLOW_EXISTING_IPROXY=1 after confirming it targets the intended iPhone" + fi + log "Using existing listener on local port $port by explicit override" + return + fi + local device_port="${OPENPHONE_VALIDATE_IPROXY_DEVICE_PORT:-22}" + local -a iproxy_args=() + if [[ -n "$target_udid" ]]; then + iproxy_args=(-u "$target_udid") + log "Starting temporary pinned iproxy $port:$device_port for UDID $(redact_udid "$target_udid")" + else + log "Starting temporary unpinned iproxy $port:$device_port" + fi + iproxy "${iproxy_args[@]}" "$port:$device_port" >"$run_dir/iproxy.log" 2>&1 & + started_iproxy_pid="$!" + sleep 2 + if ! lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then + fail 30 "iproxy did not start; see $run_dir/iproxy.log" + fi +} + +check_remote_target_identity() { + local expected_name="${OPENPHONE_IOS_EXPECTED_DEVICE_NAME:-}" + local expected_product="${OPENPHONE_IOS_EXPECTED_PRODUCT_TYPE:-}" + if [[ -z "$expected_name" && -z "$expected_product" ]]; then + return + fi + log "Checking remote target identity" + remote_capture "target-identity.txt" \ + 'printf "nodename=%s\n" "$(uname -n 2>/dev/null || true)"; printf "uname=%s\n" "$(uname -a 2>/dev/null || true)"' + if ! OPENPHONE_EXPECTED_DEVICE_NAME="$expected_name" \ + OPENPHONE_EXPECTED_PRODUCT_TYPE="$expected_product" \ + OPENPHONE_TARGET_IDENTITY_PATH="$run_dir/target-identity.txt" \ + python3 - <<'PY' +import os +import pathlib +import sys + +path = pathlib.Path(os.environ["OPENPHONE_TARGET_IDENTITY_PATH"]) +expected_name = os.environ.get("OPENPHONE_EXPECTED_DEVICE_NAME", "") +expected_product = os.environ.get("OPENPHONE_EXPECTED_PRODUCT_TYPE", "") +fields = {} +for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if "=" in line: + key, value = line.split("=", 1) + fields[key.strip()] = value.strip() + +errors = [] +if expected_name and fields.get("nodename") != expected_name: + errors.append(f"device_name:{fields.get('nodename', '')}") +if expected_product and expected_product not in fields.get("uname", ""): + errors.append(f"product_type_missing:{expected_product}") +if errors: + print("; ".join(errors), file=sys.stderr) + sys.exit(1) +PY + then + fail 30 "remote target identity did not match expected iPhone; see $run_dir/target-identity.txt" + fi +} + +collect_safety() { + local suffix="$1" + local crash_name="springboard-crashes.txt" + local marker_name="safe-mode-markers.txt" + if [[ -n "$suffix" ]]; then + crash_name="springboard-crashes.$suffix.txt" + marker_name="safe-mode-markers.$suffix.txt" + fi + remote_capture "$crash_name" \ + 'find /var/mobile/Library/Logs/CrashReporter /private/var/mobile/Library/Logs/CrashReporter -name SpringBoard-\*.ips -type f -print 2>/dev/null | sort | tail -20' + remote_capture "$marker_name" \ + 'for p in /private/var/mobile/.eksafemode /var/mobile/Library/Preferences/.eksafemode; do if [ -e "$p" ]; then ls -ld "$p"; fi; done' +} + +check_preinstall_safety() { + local marker_file="$run_dir/safe-mode-markers.before.txt" + if [[ -s "$marker_file" ]]; then + fail 40 "safe-mode marker present before install; see $marker_file" + fi +} + +check_stability_gate() { + local marker_file="$run_dir/safe-mode-markers.txt" + if [[ -s "$marker_file" ]]; then + generate_report >/dev/null + fail 40 "safe-mode marker present after device collection; see $marker_file" + fi + if OPENPHONE_VALIDATE_RUN_DIR="$run_dir" python3 - <<'PY' +import pathlib +import sys + +run_dir = pathlib.Path(__import__("os").environ["OPENPHONE_VALIDATE_RUN_DIR"]) + +def latest(name): + path = run_dir / name + if not path.exists(): + return "" + names = [] + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + line = line.strip() + if line: + names.append(pathlib.Path(line).name) + return max(set(names)) if names else "" + +before = latest("springboard-crashes.before.txt") +after = latest("springboard-crashes.txt") +if before and after and after > before: + print(f"new SpringBoard crash after install: {after} > {before}", file=sys.stderr) + sys.exit(40) +PY + then + return + else + local code=$? + generate_report >/dev/null + fail "$code" "SpringBoard stability gate failed; see $run_dir/report.json" + fi +} + +collect_device_state() { + log "Collecting device state" + remote_capture "health.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "get-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + collect_safety "" + remote_capture "processes.txt" \ + 'ps -A 2>/dev/null | grep "[o]penphone-agentd" || true' + remote_capture "openphone-agentd.log.tail" \ + "if [ -f /var/mobile/Library/OpenPhone/openphone-agentd.log ]; then tail -n $tail_lines /var/mobile/Library/OpenPhone/openphone-agentd.log; fi" + remote_capture "openphone-volume-trigger.log.tail" \ + "if [ -f /var/mobile/Library/OpenPhone/openphone-volume-trigger.log ]; then tail -n $tail_lines /var/mobile/Library/OpenPhone/openphone-volume-trigger.log; fi" + remote_capture "springboard-state.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/springboard/state.json ]; then cat /var/mobile/Library/OpenPhone/springboard/state.json; else printf "%s\n" "{}"; fi' + remote_capture "springboard-trigger-status.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/springboard/trigger-status.json ]; then cat /var/mobile/Library/OpenPhone/springboard/trigger-status.json; else printf "%s\n" "{}"; fi' +} + +collect_trigger_diagnostics_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_TRIGGER_DIAGNOSTICS:-0}" != "1" ]]; then + return + fi + log "Collecting physical trigger diagnostics" + remote_capture "trigger-diagnostics-before-status.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl agent_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "trigger-diagnostics-before-trigger.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/springboard/trigger-status.json ]; then cat /var/mobile/Library/OpenPhone/springboard/trigger-status.json; else printf "%s\n" "{}"; fi' + + log "Waiting ${trigger_wait_seconds}s for a real Volume Up then Volume Down press on the unlocked phone" + sleep "$trigger_wait_seconds" + + remote_capture "trigger-diagnostics-after-status.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl agent_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "trigger-diagnostics-after-trigger.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/springboard/trigger-status.json ]; then cat /var/mobile/Library/OpenPhone/springboard/trigger-status.json; else printf "%s\n" "{}"; fi' + remote_capture "trigger-diagnostics-list-tasks.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl list_tasks 10; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "trigger-diagnostics-tweak-log-tail.txt" \ + "if [ -f /var/mobile/Library/OpenPhone/openphone-volume-trigger.log ]; then tail -n $tail_lines /var/mobile/Library/OpenPhone/openphone-volume-trigger.log; fi" + + local latest_task_id + latest_task_id="$(json_field "$run_dir/trigger-diagnostics-after-status.json" "latest_task.task_id")" + if [[ -n "$latest_task_id" && "$latest_task_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "trigger-diagnostics-latest-trajectory.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_trajectory '$latest_task_id' 120; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_latest_task_id"}' >"$run_dir/trigger-diagnostics-latest-trajectory.json" + fi +} + +collect_safe_store_state() { + log "Collecting safe store/task state" + remote_capture "list-tasks.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl list_tasks 10; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local latest_task_id + latest_task_id="$(OPENPHONE_LIST_TASKS_JSON="$run_dir/list-tasks.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_LIST_TASKS_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +tasks = data.get("tasks") +if not isinstance(tasks, list) or not tasks: + sys.exit(0) + +task_id = tasks[0].get("task_id") if isinstance(tasks[0], dict) else "" +if isinstance(task_id, str) and re.match(r"^[A-Za-z0-9_.:-]+$", task_id): + print(task_id) +PY +)" + if [[ -n "$latest_task_id" ]]; then + remote_capture "get-task.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_task '$latest_task_id'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "get-trajectory.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_trajectory '$latest_task_id' 50; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"no_safe_task_id"}' >"$run_dir/get-task.json" + printf '%s\n' '{"status":"skipped","reason":"no_safe_task_id"}' >"$run_dir/get-trajectory.json" + fi + remote_capture "get-audit.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_audit 30; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "memory-search.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl memory_search OpenPhone 5; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "context-search.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl context_search OpenPhone 5; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "background-job-list.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl background_job_list durable 10; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "commitment-search.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl commitment_search OpenPhone 10; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "watcher-list.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl watcher_list OpenPhone 10; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' +} + +collect_provider_attempt_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_PROVIDER_ATTEMPTS:-0}" != "1" ]]; then + return + fi + log "Collecting provider-attempt no-dispatch sample" + remote_capture "provider-attempt-start-task.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"start_task","goal":"validator provider-attempt no-dispatch sample","approved_capabilities":["input.perform","tasks.observe"]}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local sample_task_id + sample_task_id="$(OPENPHONE_PROVIDER_ATTEMPT_START_JSON="$run_dir/provider-attempt-start-task.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_PROVIDER_ATTEMPT_START_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +task_id = data.get("task_id") +if isinstance(task_id, str) and re.match(r"^[A-Za-z0-9_.:-]+$", task_id): + print(task_id) +PY +)" + if [[ -n "$sample_task_id" ]]; then + remote_capture "provider-attempt-action.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"execute_action\",\"task_id\":\"$sample_task_id\",\"action\":{\"type\":\"tap\",\"reason\":\"validator provider-attempt missing-coordinate sample\"}}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "provider-attempt-trajectory.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_trajectory '$sample_task_id' 20; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"no_provider_attempt_sample_task_id"}' >"$run_dir/provider-attempt-action.json" + printf '%s\n' '{"status":"skipped","reason":"no_provider_attempt_sample_task_id"}' >"$run_dir/provider-attempt-trajectory.json" + fi +} + +collect_watcher_timer_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_WATCHER_TIMER:-0}" != "1" ]]; then + return + fi + log "Collecting timer watcher firing sample" + remote_capture "watcher-timer-create.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then now_ms=$(($(date +%s) * 1000 - 1000)); /var/jb/usr/local/bin/openphone-agentctl "{\"command\":\"watcher_create\",\"title\":\"validator timer watcher\",\"source\":\"time\",\"type\":\"time\",\"next_run_at\":$now_ms,\"prompt\":\"summarize validator timer watcher\",\"reason\":\"validator timer watcher gate\"}"; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "watcher-timer-run-due.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl "{\"command\":\"watcher_run_due\",\"limit\":5,\"source\":\"validator_watcher_timer_gate\",\"reason\":\"validator timer watcher gate\"}"; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "watcher-timer-job-run-due.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl background_job_run_due 5 1 15000; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "watcher-timer-job-list.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl background_job_list "validator timer watcher" 10; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "watcher-timer-after-list.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl watcher_list "validator timer watcher" 5; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local watcher_id + watcher_id="$(OPENPHONE_WATCHER_TIMER_CREATE_JSON="$run_dir/watcher-timer-create.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_WATCHER_TIMER_CREATE_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +watcher_id = data.get("watcher_id") +if isinstance(watcher_id, int): + print(watcher_id) +elif isinstance(watcher_id, str) and re.match(r"^[A-Za-z0-9_.:-]+$", watcher_id): + print(watcher_id) +PY +)" + if [[ -n "$watcher_id" ]]; then + remote_capture "watcher-timer-stop.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl watcher_stop '$watcher_id'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"no_watcher_timer_id"}' >"$run_dir/watcher-timer-stop.json" + fi +} + +collect_watcher_repair_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_WATCHER_REPAIR:-0}" != "1" ]]; then + return + fi + log "Collecting stale watcher repair sample" + remote_capture "watcher-repair-create.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then future_ms=$(($(date +%s) * 1000 + 600000)); /var/jb/usr/local/bin/openphone-agentctl "{\"command\":\"watcher_create\",\"title\":\"validator watcher stuck repair\",\"source\":\"time\",\"type\":\"time\",\"next_run_at\":$future_ms,\"prompt\":\"summarize validator watcher stuck repair\",\"reason\":\"validator watcher stuck repair gate\"}"; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local watcher_id + watcher_id="$(OPENPHONE_WATCHER_REPAIR_CREATE_JSON="$run_dir/watcher-repair-create.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_WATCHER_REPAIR_CREATE_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +watcher_id = data.get("watcher_id") +if isinstance(watcher_id, int): + print(watcher_id) +elif isinstance(watcher_id, str) and re.match(r"^[A-Za-z0-9_.:-]+$", watcher_id): + print(watcher_id) +PY +)" + if [[ -n "$watcher_id" ]]; then + remote_capture "watcher-repair-mark-running.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"watcher_debug_mark_running\",\"watcher_id\":$watcher_id,\"validation\":true,\"age_ms\":600000,\"source\":\"validator_watcher_stuck_repair_gate\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "watcher-repair-run.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"watcher_repair_stuck\",\"limit\":5,\"stale_after_ms\":1000,\"source\":\"validator_watcher_stuck_repair_gate\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "watcher-repair-run-due.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl "{\"command\":\"watcher_run_due\",\"limit\":5,\"source\":\"validator_watcher_repair_gate\",\"reason\":\"validator watcher repair gate\"}"; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "watcher-repair-job-run-due.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl background_job_run_due 5 1 15000; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "watcher-repair-after-list.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl watcher_list "validator watcher stuck repair" 5; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "watcher-repair-stop.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl watcher_stop '$watcher_id'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"no_watcher_repair_id"}' >"$run_dir/watcher-repair-mark-running.json" + printf '%s\n' '{"status":"skipped","reason":"no_watcher_repair_id"}' >"$run_dir/watcher-repair-run.json" + printf '%s\n' '{"status":"skipped","reason":"no_watcher_repair_id"}' >"$run_dir/watcher-repair-run-due.json" + printf '%s\n' '{"status":"skipped","reason":"no_watcher_repair_id"}' >"$run_dir/watcher-repair-job-run-due.json" + printf '%s\n' '{"status":"skipped","reason":"no_watcher_repair_id"}' >"$run_dir/watcher-repair-after-list.json" + printf '%s\n' '{"status":"skipped","reason":"no_watcher_repair_id"}' >"$run_dir/watcher-repair-stop.json" + fi +} + +collect_job_repair_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_JOB_REPAIR:-0}" != "1" ]]; then + return + fi + log "Collecting stale background-job repair sample" + remote_capture "job-repair-create.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then future_ms=$(($(date +%s) * 1000 + 600000)); /var/jb/usr/local/bin/openphone-agentctl "{\"command\":\"background_job_create\",\"title\":\"validator stuck repair job\",\"prompt\":\"summarize validator stuck repair job\",\"next_run_at\":$future_ms,\"reason\":\"validator stuck repair gate\"}"; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local job_id + job_id="$(OPENPHONE_JOB_REPAIR_CREATE_JSON="$run_dir/job-repair-create.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_JOB_REPAIR_CREATE_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +job_id = data.get("job_id") +if isinstance(job_id, int): + print(job_id) +elif isinstance(job_id, str) and re.match(r"^[A-Za-z0-9_.:-]+$", job_id): + print(job_id) +PY +)" + if [[ -n "$job_id" ]]; then + remote_capture "job-repair-mark-running.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"background_job_debug_mark_running\",\"job_id\":$job_id,\"validation\":true,\"age_ms\":600000,\"source\":\"validator_stuck_repair_gate\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "job-repair-run.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"background_job_repair_stuck\",\"limit\":5,\"stale_after_ms\":1000,\"source\":\"validator_stuck_repair_gate\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "job-repair-run-due.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl background_job_run_due 5 1 15000; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "job-repair-list.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl background_job_list "validator stuck repair" 10; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "job-repair-stop.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl background_job_stop '$job_id'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"no_job_repair_id"}' >"$run_dir/job-repair-mark-running.json" + printf '%s\n' '{"status":"skipped","reason":"no_job_repair_id"}' >"$run_dir/job-repair-run.json" + printf '%s\n' '{"status":"skipped","reason":"no_job_repair_id"}' >"$run_dir/job-repair-run-due.json" + printf '%s\n' '{"status":"skipped","reason":"no_job_repair_id"}' >"$run_dir/job-repair-list.json" + printf '%s\n' '{"status":"skipped","reason":"no_job_repair_id"}' >"$run_dir/job-repair-stop.json" + fi +} + +collect_restart_recovery_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_RESTART_RECOVERY:-0}" != "1" ]]; then + return + fi + local wait_seconds="${OPENPHONE_VALIDATE_RESTART_RECOVERY_WAIT_SECONDS:-36}" + local start_timeout_seconds="${OPENPHONE_VALIDATE_RESTART_RECOVERY_START_TIMEOUT_SECONDS:-60}" + local stable_seconds="${OPENPHONE_VALIDATE_RESTART_RECOVERY_STABLE_SECONDS:-12}" + log "Collecting daemon restart recovery sample" + remote_capture "restart-recovery-watcher-create.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then future_ms=$(($(date +%s) * 1000 + 600000)); /var/jb/usr/local/bin/openphone-agentctl "{\"command\":\"watcher_create\",\"title\":\"validator restart recovery watcher\",\"source\":\"time\",\"type\":\"time\",\"next_run_at\":$future_ms,\"prompt\":\"summarize validator restart recovery watcher\",\"reason\":\"validator restart recovery gate\"}"; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "restart-recovery-job-create.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then future_ms=$(($(date +%s) * 1000 + 600000)); /var/jb/usr/local/bin/openphone-agentctl "{\"command\":\"background_job_create\",\"title\":\"validator restart recovery job\",\"prompt\":\"summarize validator restart recovery job\",\"next_run_at\":$future_ms,\"reason\":\"validator restart recovery gate\"}"; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local watcher_id + watcher_id="$(OPENPHONE_RESTART_WATCHER_CREATE_JSON="$run_dir/restart-recovery-watcher-create.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_RESTART_WATCHER_CREATE_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +watcher_id = data.get("watcher_id") +if isinstance(watcher_id, int): + print(watcher_id) +elif isinstance(watcher_id, str) and re.match(r"^[A-Za-z0-9_.:-]+$", watcher_id): + print(watcher_id) +PY +)" + local job_id + job_id="$(OPENPHONE_RESTART_JOB_CREATE_JSON="$run_dir/restart-recovery-job-create.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_RESTART_JOB_CREATE_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +job_id = data.get("job_id") +if isinstance(job_id, int): + print(job_id) +elif isinstance(job_id, str) and re.match(r"^[A-Za-z0-9_.:-]+$", job_id): + print(job_id) +PY +)" + if [[ -n "$watcher_id" && -n "$job_id" ]]; then + remote_capture "restart-recovery-watcher-mark-running.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"watcher_debug_mark_running\",\"watcher_id\":$watcher_id,\"validation\":true,\"age_ms\":600000,\"source\":\"validator_restart_recovery_gate\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "restart-recovery-job-mark-running.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"background_job_debug_mark_running\",\"job_id\":$job_id,\"validation\":true,\"age_ms\":600000,\"source\":\"validator_restart_recovery_gate\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "restart-recovery-restart.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then before=\$(ps -A 2>/dev/null | grep '[o]penphone-agentd$' | sed -n '1s/^ *\\([0-9][0-9]*\\).*/\\1/p'); if [ -n \"\$before\" ]; then kill \"\$before\" >/dev/null 2>&1 || true; fi; after=''; last_seen=''; stable_elapsed=0; elapsed=0; while [ \"\$elapsed\" -lt $start_timeout_seconds ]; do current=\$(ps -A 2>/dev/null | grep '[o]penphone-agentd$' | sed -n '1s/^ *\\([0-9][0-9]*\\).*/\\1/p'); if [ -n \"\$current\" ] && [ \"\$current\" != \"\$before\" ]; then if [ \"\$current\" = \"\$last_seen\" ]; then stable_elapsed=\$((stable_elapsed + 1)); else last_seen=\"\$current\"; stable_elapsed=0; fi; if [ \"\$stable_elapsed\" -ge $stable_seconds ]; then after=\"\$current\"; break; fi; else stable_elapsed=0; fi; sleep 1; elapsed=\$((elapsed + 1)); done; post_wait_pid=''; if [ -n \"\$after\" ] && [ \"\$after\" != \"\$before\" ]; then restart_status=ok; sleep $wait_seconds; post_wait_pid=\$(ps -A 2>/dev/null | grep '[o]penphone-agentd$' | sed -n '1s/^ *\\([0-9][0-9]*\\).*/\\1/p'); else restart_status=error; fi; printf '{\"status\":\"%s\",\"before_pid\":\"%s\",\"after_pid\":\"%s\",\"post_wait_pid\":\"%s\",\"restart_elapsed_seconds\":%s,\"stable_elapsed_seconds\":%s,\"required_stable_seconds\":$stable_seconds,\"post_restart_wait_seconds\":$wait_seconds,\"start_timeout_seconds\":$start_timeout_seconds}\\n' \"\$restart_status\" \"\$before\" \"\$after\" \"\$post_wait_pid\" \"\$elapsed\" \"\$stable_elapsed\"; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "restart-recovery-after-health.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "restart-recovery-watcher-list.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl watcher_list "validator restart recovery watcher" 25; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "restart-recovery-job-list.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl background_job_list "validator restart recovery job" 25; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local generated_job_id + generated_job_id="$(OPENPHONE_RESTART_WATCHER_LIST_JSON="$run_dir/restart-recovery-watcher-list.json" OPENPHONE_RESTART_WATCHER_ROW_ID="$watcher_id" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_RESTART_WATCHER_LIST_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +watcher_row_id = os.environ.get("OPENPHONE_RESTART_WATCHER_ROW_ID", "") +for watcher in data.get("watchers", []): + if not isinstance(watcher, dict): + continue + if watcher_row_id and str(watcher.get("id", "")) != watcher_row_id: + continue + metadata = watcher.get("metadata") if isinstance(watcher.get("metadata"), dict) else {} + schedule = watcher.get("schedule") if isinstance(watcher.get("schedule"), dict) else {} + job_id = metadata.get("last_job_id") or schedule.get("last_job_id") or "" + if isinstance(job_id, int): + print(job_id) + break + if isinstance(job_id, str) and re.match(r"^[A-Za-z0-9_.:-]+$", job_id): + print(job_id) + break +PY +)" + remote_capture "restart-recovery-watcher-stop.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl watcher_stop '$watcher_id'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "restart-recovery-job-stop.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl background_job_stop '$job_id'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + if [[ -n "$generated_job_id" ]]; then + remote_capture "restart-recovery-generated-job-stop.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl background_job_stop '$generated_job_id' 'validator restart recovery generated job cleanup'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_restart_recovery_generated_job_id"}' >"$run_dir/restart-recovery-generated-job-stop.json" + fi + else + printf '%s\n' '{"status":"skipped","reason":"missing_restart_recovery_fixture_id"}' >"$run_dir/restart-recovery-watcher-mark-running.json" + printf '%s\n' '{"status":"skipped","reason":"missing_restart_recovery_fixture_id"}' >"$run_dir/restart-recovery-job-mark-running.json" + printf '%s\n' '{"status":"skipped","reason":"missing_restart_recovery_fixture_id"}' >"$run_dir/restart-recovery-restart.json" + printf '%s\n' '{"status":"skipped","reason":"missing_restart_recovery_fixture_id"}' >"$run_dir/restart-recovery-after-health.json" + printf '%s\n' '{"status":"skipped","reason":"missing_restart_recovery_fixture_id"}' >"$run_dir/restart-recovery-watcher-list.json" + printf '%s\n' '{"status":"skipped","reason":"missing_restart_recovery_fixture_id"}' >"$run_dir/restart-recovery-job-list.json" + printf '%s\n' '{"status":"skipped","reason":"missing_restart_recovery_fixture_id"}' >"$run_dir/restart-recovery-watcher-stop.json" + printf '%s\n' '{"status":"skipped","reason":"missing_restart_recovery_fixture_id"}' >"$run_dir/restart-recovery-job-stop.json" + printf '%s\n' '{"status":"skipped","reason":"missing_restart_recovery_fixture_id"}' >"$run_dir/restart-recovery-generated-job-stop.json" + fi +} + +safe_json_string() { + python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$1" +} + +json_field() { + local artifact="$1" + local expression="$2" + OPENPHONE_JSON_FIELD_FILE="$artifact" OPENPHONE_JSON_FIELD_EXPR="$expression" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_JSON_FIELD_FILE"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +value = data +for part in os.environ["OPENPHONE_JSON_FIELD_EXPR"].split("."): + if isinstance(value, dict): + value = value.get(part) + else: + sys.exit(0) +if isinstance(value, str): + print(value) +elif isinstance(value, (int, float)): + print(value) +PY +} + +collect_memory_lifecycle_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_MEMORY_LIFECYCLE:-0}" != "1" ]]; then + return + fi + log "Collecting memory lifecycle sample" + local primary_text="validator memory lifecycle primary $(date '+%Y%m%d%H%M%S')" + local updated_text="$primary_text updated merge-ready" + local source_text="$primary_text secondary merge source" + local merged_text="$primary_text merged durable result" + local delete_text="$primary_text delete fixture" + + remote_capture "memory-lifecycle-save-primary.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"memory_save\",\"type\":\"fact\",\"subject\":\"validator\",\"text\":$(safe_json_string "$primary_text"),\"reason\":\"validator memory lifecycle primary\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + local primary_id + primary_id="$(json_field "$run_dir/memory-lifecycle-save-primary.json" "memory.memory_id")" + + if [[ -n "$primary_id" ]]; then + remote_capture "memory-lifecycle-update.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"memory_update\",\"memory_id\":\"$primary_id\",\"type\":\"fact\",\"subject\":\"validator\",\"text\":$(safe_json_string "$updated_text"),\"reason\":\"validator memory lifecycle update\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_primary_memory_id"}' >"$run_dir/memory-lifecycle-update.json" + fi + + remote_capture "memory-lifecycle-save-source.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"memory_save\",\"type\":\"fact\",\"subject\":\"validator\",\"text\":$(safe_json_string "$source_text"),\"reason\":\"validator memory lifecycle merge source\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + local source_id + source_id="$(json_field "$run_dir/memory-lifecycle-save-source.json" "memory.memory_id")" + + if [[ -n "$primary_id" && -n "$source_id" ]]; then + remote_capture "memory-lifecycle-merge.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"memory_merge\",\"target_memory_id\":\"$primary_id\",\"source_memory_id\":\"$source_id\",\"text\":$(safe_json_string "$merged_text"),\"reason\":\"validator memory lifecycle merge\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_merge_memory_id"}' >"$run_dir/memory-lifecycle-merge.json" + fi + + remote_capture "memory-lifecycle-save-delete.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"memory_save\",\"type\":\"fact\",\"subject\":\"validator\",\"text\":$(safe_json_string "$delete_text"),\"reason\":\"validator memory lifecycle delete source\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + local delete_id + delete_id="$(json_field "$run_dir/memory-lifecycle-save-delete.json" "memory.memory_id")" + if [[ -n "$delete_id" ]]; then + remote_capture "memory-lifecycle-delete.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"memory_delete\",\"memory_id\":\"$delete_id\",\"reason\":\"validator memory lifecycle delete\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_delete_memory_id"}' >"$run_dir/memory-lifecycle-delete.json" + fi + + remote_capture "memory-lifecycle-search.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"memory_search\",\"query\":$(safe_json_string "$primary_text"),\"limit\":10,\"reason\":\"validator memory lifecycle search\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" +} + +collect_model_loop_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_MODEL_LOOP:-0}" != "1" ]]; then + return + fi + log "Collecting fixture model-loop sample" + remote_capture "model-status.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl model_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local goal="validator fixture model loop $(date '+%Y%m%d%H%M%S')" + remote_capture "model-loop-run.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"run_task\",\"goal\":$(safe_json_string "$goal"),\"mode\":\"model\",\"max_steps\":3,\"max_duration_ms\":30000,\"model_decisions\":[{\"schema\":\"openphone.model_decision.v1\",\"thought\":\"pause before finishing\",\"tool\":\"wait\",\"arguments\":{\"duration_ms\":10},\"expected_visible_change\":\"none\",\"confidence\":0.9},{\"schema\":\"openphone.model_decision.v1\",\"thought\":\"validation complete\",\"tool\":\"finish_task\",\"arguments\":{\"summary\":\"Fixture model loop completed.\"},\"expected_visible_change\":\"none\",\"confidence\":0.95}]}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + local model_task_id + model_task_id="$(json_field "$run_dir/model-loop-run.json" "task_id")" + if [[ -n "$model_task_id" ]]; then + remote_capture "model-loop-trajectory.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_trajectory '$model_task_id' 80; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_model_loop_task_id"}' >"$run_dir/model-loop-trajectory.json" + fi + + local repair_goal="validator parser repair model loop $(date '+%Y%m%d%H%M%S')" + local repair_decision=$'Here is the decision:\n```json\n{"schema":"openphone.model_decision.v1","thought":"wrapped validation complete","tool":"finish_task","arguments":{"summary":"Parser repair model loop completed."},"expected_visible_change":"none","confidence":0.96}\n```\n' + remote_capture "model-loop-repair-run.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"run_task\",\"goal\":$(safe_json_string "$repair_goal"),\"mode\":\"model\",\"max_steps\":2,\"max_duration_ms\":30000,\"model_decisions\":[$(safe_json_string "$repair_decision")]}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + local repair_task_id + repair_task_id="$(json_field "$run_dir/model-loop-repair-run.json" "task_id")" + if [[ -n "$repair_task_id" ]]; then + remote_capture "model-loop-repair-trajectory.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_trajectory '$repair_task_id' 80; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_model_loop_repair_task_id"}' >"$run_dir/model-loop-repair-trajectory.json" + fi + + local cancel_goal="validator cancelled model loop $(date '+%Y%m%d%H%M%S')" + remote_capture "model-loop-cancel-start.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"start_task\",\"goal\":$(safe_json_string "$cancel_goal")}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + local cancel_task_id + cancel_task_id="$(json_field "$run_dir/model-loop-cancel-start.json" "task_id")" + if [[ -n "$cancel_task_id" && "$cancel_task_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "model-loop-cancel-stop.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"stop_task\",\"task_id\":\"$cancel_task_id\",\"reason\":\"validator model loop cancellation\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "model-loop-cancel-run.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"run_task\",\"task_id\":\"$cancel_task_id\",\"goal\":$(safe_json_string "$cancel_goal"),\"mode\":\"model\",\"max_steps\":3,\"max_duration_ms\":30000,\"model_decisions\":[{\"schema\":\"openphone.model_decision.v1\",\"thought\":\"should not execute after cancellation\",\"tool\":\"finish_task\",\"arguments\":{\"summary\":\"Should not finish.\"},\"expected_visible_change\":\"none\",\"confidence\":0.95}]}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "model-loop-cancel-trajectory.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_trajectory '$cancel_task_id' 80; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_model_loop_cancel_task_id"}' >"$run_dir/model-loop-cancel-stop.json" + printf '%s\n' '{"status":"skipped","reason":"missing_model_loop_cancel_task_id"}' >"$run_dir/model-loop-cancel-run.json" + printf '%s\n' '{"status":"skipped","reason":"missing_model_loop_cancel_task_id"}' >"$run_dir/model-loop-cancel-trajectory.json" + fi +} + +collect_voice_status_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_VOICE:-0}" != "1" ]]; then + return + fi + log "Collecting voice_status + island snapshot sample" + remote_capture "voice-status.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"voice_status"}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "island-status.json" \ + 'cat /var/mobile/Library/OpenPhone/springboard/island-status.json 2>/dev/null || printf "%s\n" "{\"status\":\"absent\"}"' + remote_capture "voice-credential-file-exists.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/config/voice-credential.json ]; then printf "%s\n" "{\"status\":\"present\"}"; else printf "%s\n" "{\"status\":\"absent\"}"; fi' +} + +start_provider_broker_if_requested() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_PROVIDER_MODEL:-0}" != "1" ]]; then + return + fi + local token="${AWS_BEARER_TOKEN_BEDROCK:-${OPENPHONE_BEDROCK_BEARER_TOKEN:-}}" + if [[ -z "$token" ]]; then + fail 100 "OPENPHONE_VALIDATE_INCLUDE_PROVIDER_MODEL=1 requires AWS_BEARER_TOKEN_BEDROCK or OPENPHONE_BEDROCK_BEARER_TOKEN" + fi + require_command python3 100 + require_command ssh 100 + log "Starting provider-backed Bedrock broker" + local port_file="$run_dir/provider-model-broker-port.txt" + AWS_BEARER_TOKEN_BEDROCK="$token" \ + OPENPHONE_BEDROCK_BEARER_TOKEN="${OPENPHONE_BEDROCK_BEARER_TOKEN:-}" \ + OPENPHONE_BEDROCK_MODEL="${OPENPHONE_BEDROCK_MODEL:-}" \ + OPENPHONE_BEDROCK_REGION="${OPENPHONE_BEDROCK_REGION:-}" \ + AWS_REGION="${AWS_REGION:-}" \ + AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-}" \ + OPENPHONE_BEDROCK_RUNTIME_URL="${OPENPHONE_BEDROCK_RUNTIME_URL:-}" \ + OPENPHONE_BEDROCK_MAX_TOKENS="${OPENPHONE_BEDROCK_MAX_TOKENS:-}" \ + OPENPHONE_BEDROCK_TEMPERATURE="${OPENPHONE_BEDROCK_TEMPERATURE:-}" \ + "$repo_root/tools/mac/agentd/bedrock-model-broker.py" \ + --port-file "$port_file" >"$run_dir/provider-model-broker.log" 2>&1 & + provider_broker_pid="$!" + for _ in $(seq 1 100); do + if [[ -s "$port_file" ]]; then + break + fi + if ! kill -0 "$provider_broker_pid" >/dev/null 2>&1; then + fail 100 "provider model broker exited early; see $run_dir/provider-model-broker.log" + fi + sleep 0.1 + done + if [[ ! -s "$port_file" ]]; then + fail 100 "provider model broker did not publish a port; see $run_dir/provider-model-broker.log" + fi + provider_broker_port="$(cat "$port_file")" + log "Starting provider model reverse tunnel phone:$provider_phone_port -> mac:$provider_broker_port" + start_ssh_reverse_forward "$provider_phone_port" "$provider_broker_port" +} + +collect_provider_model_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_PROVIDER_MODEL:-0}" != "1" ]]; then + return + fi + log "Collecting provider-backed model-loop sample" + local endpoint="http://127.0.0.1:${provider_phone_port}/decision" + local model_name="${OPENPHONE_BEDROCK_MODEL:-anthropic.claude-haiku-4-5-20251001-v1:0}" + local goal="provider-backed broker validation $(date '+%Y%m%d%H%M%S')" + remote_capture "provider-model-configure.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"model_configure\",\"mode\":\"broker\",\"endpoint_url\":$(safe_json_string "$endpoint"),\"model\":$(safe_json_string "$model_name"),\"enabled\":true,\"credential_required\":false,\"timeout_ms\":60000,\"max_steps\":2,\"max_duration_ms\":60000,\"reason\":\"validator provider-backed broker sample\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + remote_capture "provider-model-status.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl model_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "provider-model-run.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"run_task\",\"goal\":$(safe_json_string "$goal"),\"mode\":\"model\",\"max_steps\":2,\"max_duration_ms\":60000}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + local provider_task_id + provider_task_id="$(json_field "$run_dir/provider-model-run.json" "task_id")" + if [[ -n "$provider_task_id" && "$provider_task_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "provider-model-trajectory.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_trajectory '$provider_task_id' 120; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_provider_model_task_id"}' >"$run_dir/provider-model-trajectory.json" + fi + remote_capture "provider-model-reset.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '\''{"command":"model_configure","mode":"broker","endpoint_url":"","model":"","enabled":false,"credential_required":true,"reason":"validator provider-backed broker cleanup"}'\''; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' +} + +install_direct_bedrock_credential() { + local token="${AWS_BEARER_TOKEN_BEDROCK:-${OPENPHONE_BEDROCK_BEARER_TOKEN:-}}" + require_command python3 100 + # shellcheck disable=SC2016 + remote_capture "safari-dom-model-backup.json" \ + 'mkdir -p /var/mobile/Library/OpenPhone/config '"$remote_tmp"'; \ + credential_backup=false; config_backup=false; \ + if [ -f /var/mobile/Library/OpenPhone/config/model-credential.json ]; then cp /var/mobile/Library/OpenPhone/config/model-credential.json '"$remote_tmp"'/model-credential.backup && credential_backup=true; fi; \ + if [ -f /var/mobile/Library/OpenPhone/config/model.json ]; then cp /var/mobile/Library/OpenPhone/config/model.json '"$remote_tmp"'/model-config.backup && config_backup=true; fi; \ + printf "{\"status\":\"ok\",\"credential_backup\":%s,\"config_backup\":%s}\n" "$credential_backup" "$config_backup"' + if OPENPHONE_BACKUP_JSON="$run_dir/safari-dom-model-backup.json" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_BACKUP_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(1) +sys.exit(0 if data.get("credential_backup") is True else 1) +PY + then + direct_bedrock_credential_had_backup=1 + fi + if OPENPHONE_BACKUP_JSON="$run_dir/safari-dom-model-backup.json" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_BACKUP_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(1) +sys.exit(0 if data.get("config_backup") is True else 1) +PY + then + direct_bedrock_config_had_backup=1 + fi + + if [[ -z "$token" ]]; then + if [[ "$direct_bedrock_credential_had_backup" == "1" ]]; then + log "Using existing phone Bedrock credential file for Safari DOM model sample" + return + fi + fail 100 "OPENPHONE_VALIDATE_INCLUDE_SAFARI_DOM_MODEL=1 requires AWS_BEARER_TOKEN_BEDROCK, OPENPHONE_BEDROCK_BEARER_TOKEN, or an existing phone credential file" + fi + + direct_bedrock_credential_temp="$(mktemp -t openphone-bedrock-credential.XXXXXX)" + chmod 0600 "$direct_bedrock_credential_temp" + OPENPHONE_VALIDATE_BEDROCK_TOKEN="$token" python3 - <<'PY' >"$direct_bedrock_credential_temp" +import json +import os + +print(json.dumps({"credential": os.environ["OPENPHONE_VALIDATE_BEDROCK_TOKEN"]})) +PY + scp_to "$direct_bedrock_credential_temp" "$remote_tmp/model-credential.upload" "$run_dir/safari-dom-model-credential-scp.log" + remote_exec "install -m 0600 '$remote_tmp/model-credential.upload' /var/mobile/Library/OpenPhone/config/model-credential.json; rm -f '$remote_tmp/model-credential.upload'" "$run_dir/safari-dom-model-credential-install.log" + direct_bedrock_credential_touched=1 + rm -f "$direct_bedrock_credential_temp" + direct_bedrock_credential_temp="" +} + +restore_direct_bedrock_state() { + # shellcheck disable=SC2016 + remote_capture "safari-dom-model-reset.json" \ + 'mkdir -p /var/mobile/Library/OpenPhone/config; \ + reset_status=ok; credential_restored=false; credential_removed=false; config_restored=false; config_removed=false; \ + if [ -f '"$remote_tmp"'/model-config.backup ]; then install -m 0600 '"$remote_tmp"'/model-config.backup /var/mobile/Library/OpenPhone/config/model.json && config_restored=true || reset_status=error; else rm -f /var/mobile/Library/OpenPhone/config/model.json && config_removed=true || reset_status=error; fi; \ + if [ -f '"$remote_tmp"'/model-credential.backup ]; then install -m 0600 '"$remote_tmp"'/model-credential.backup /var/mobile/Library/OpenPhone/config/model-credential.json && credential_restored=true || reset_status=error; else rm -f /var/mobile/Library/OpenPhone/config/model-credential.json && credential_removed=true || reset_status=error; fi; \ + printf "{\"status\":\"%s\",\"credential_restored\":%s,\"credential_removed\":%s,\"config_restored\":%s,\"config_removed\":%s}\n" "$reset_status" "$credential_restored" "$credential_removed" "$config_restored" "$config_removed"' + if grep -q '"status":"ok"' "$run_dir/safari-dom-model-reset.json"; then + direct_bedrock_credential_touched=0 + direct_bedrock_config_touched=0 + fi +} + +collect_safari_dom_model_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_SAFARI_DOM_MODEL:-0}" != "1" ]]; then + return + fi + log "Collecting direct Bedrock Safari DOM model-loop sample" + + remote_capture "safari-dom-model-before-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local locked_state + locked_state="$(OPENPHONE_SAFARI_DOM_BEFORE_JSON="$run_dir/safari-dom-model-before-screen.json" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_SAFARI_DOM_BEFORE_JSON"], "r", encoding="utf-8")) +except Exception: + print("unknown") + sys.exit(0) + +lock = data.get("context", {}).get("lock", {}) if isinstance(data, dict) else {} +locked = lock.get("locked") +if locked is True: + print("true") +elif locked is False: + print("false") +else: + print("unknown") +PY +)" + if [[ "$locked_state" != "false" ]]; then + for name in safari-dom-model-open-url safari-dom-model-pre-screen safari-dom-model-configure safari-dom-model-status safari-dom-model-run safari-dom-model-trajectory safari-dom-model-after-screen safari-dom-model-safari-state; do + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/$name.json" + done + printf '%s\n' "" >"$run_dir/safari-dom-model-marker.txt" + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/safari-dom-model-reset.json" + return + fi + + install_direct_bedrock_credential + + local marker + marker="OPModelSafariDOM-$(date '+%H%M%S')" + printf '%s\n' "$marker" >"$run_dir/safari-dom-model-marker.txt" + local model_name="${OPENPHONE_BEDROCK_MODEL:-us.anthropic.claude-haiku-4-5-20251001-v1:0}" + local region="${OPENPHONE_BEDROCK_REGION:-${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-1}}}" + local timeout_ms="${OPENPHONE_VALIDATE_SAFARI_DOM_MODEL_TIMEOUT_MS:-120000}" + local max_steps="${OPENPHONE_VALIDATE_SAFARI_DOM_MODEL_MAX_STEPS:-4}" + local max_duration_ms="${OPENPHONE_VALIDATE_SAFARI_DOM_MODEL_MAX_DURATION_MS:-180000}" + remote_capture "safari-dom-model-open-url.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then killall MobileSafari >/dev/null 2>&1 || true; sleep 2; /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"open_url","url":"https://www.wikipedia.org/","reason":"validator Safari DOM model launch"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "safari-dom-model-pre-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 8; /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "safari-dom-model-configure.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"model_configure\",\"mode\":\"bedrock_converse\",\"endpoint_url\":\"\",\"model\":$(safe_json_string "$model_name"),\"region\":$(safe_json_string "$region"),\"enabled\":true,\"credential_required\":true,\"timeout_ms\":$timeout_ms,\"max_steps\":$max_steps,\"max_duration_ms\":$max_duration_ms,\"reason\":\"validator direct Bedrock Safari DOM sample\"}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + direct_bedrock_config_touched=1 + remote_capture "safari-dom-model-status.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl model_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local goal="On the currently open Wikipedia home page in Safari, enter the exact text $marker into the visible or focused search field. Do not submit the search. Finish only after that exact text is visible in the field." + remote_capture "safari-dom-model-run.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"run_task\",\"goal\":$(safe_json_string "$goal"),\"mode\":\"model\",\"max_steps\":$max_steps,\"max_duration_ms\":$max_duration_ms}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + local safari_dom_task_id + safari_dom_task_id="$(json_field "$run_dir/safari-dom-model-run.json" "task_id")" + if [[ -n "$safari_dom_task_id" && "$safari_dom_task_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "safari-dom-model-trajectory.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_trajectory '$safari_dom_task_id' 160; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_safari_dom_model_task_id"}' >"$run_dir/safari-dom-model-trajectory.json" + fi + remote_capture "safari-dom-model-after-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "safari-dom-model-safari-state.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/app-ui/com.apple.mobilesafari.json ]; then cat /var/mobile/Library/OpenPhone/app-ui/com.apple.mobilesafari.json; else printf "%s\n" "{\"status\":\"missing\",\"bundle_id\":\"com.apple.mobilesafari\"}"; fi' + restore_direct_bedrock_state +} + +collect_prompt_bridge_model_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_PROMPT_BRIDGE_MODEL:-0}" != "1" ]]; then + return + fi + log "Collecting SpringBoard prompt bridge model-loop sample" + + remote_capture "prompt-bridge-before-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local locked_state + locked_state="$(OPENPHONE_PROMPT_BRIDGE_BEFORE_JSON="$run_dir/prompt-bridge-before-screen.json" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_PROMPT_BRIDGE_BEFORE_JSON"], "r", encoding="utf-8")) +except Exception: + print("unknown") + sys.exit(0) + +lock = data.get("context", {}).get("lock", {}) if isinstance(data, dict) else {} +locked = lock.get("locked") +if locked is True: + print("true") +elif locked is False: + print("false") +else: + print("unknown") +PY +)" + if [[ "$locked_state" != "false" ]]; then + for name in prompt-bridge-open-url prompt-bridge-pre-screen prompt-bridge-model-status prompt-bridge-response prompt-bridge-agent-status prompt-bridge-trajectory prompt-bridge-after-screen prompt-bridge-safari-state prompt-bridge-tweak-log; do + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/$name.json" + done + printf '%s\n' "" >"$run_dir/prompt-bridge-marker.txt" + printf '%s\n' "" >"$run_dir/prompt-bridge-request-id.txt" + return + fi + + local marker request_id goal request_b64 + marker="OPPromptBridge-$(date '+%H%M%S')" + request_id="prompt-bridge-$run_id-$(date '+%s')" + goal="On the current Safari Wikipedia home page, type the exact text $marker into the visible Search field, then finish only after the text is visible. Do not submit the search." + printf '%s\n' "$marker" >"$run_dir/prompt-bridge-marker.txt" + printf '%s\n' "$request_id" >"$run_dir/prompt-bridge-request-id.txt" + + remote_capture "prompt-bridge-open-url.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then killall MobileSafari >/dev/null 2>&1 || true; sleep 2; /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"open_url","url":"https://www.wikipedia.org/","reason":"validator prompt bridge Safari launch"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prompt-bridge-pre-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 8; /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prompt-bridge-model-status.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl model_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + + request_b64="$(OPENPHONE_PROMPT_BRIDGE_REQUEST_ID="$request_id" OPENPHONE_PROMPT_BRIDGE_GOAL="$goal" python3 - <<'PY' +import base64 +import json +import os +import time + +request = { + "schema": "openphone.springboard_prompt_request.v1", + "request_id": os.environ["OPENPHONE_PROMPT_BRIDGE_REQUEST_ID"], + "operation": "run_goal", + "goal": os.environ["OPENPHONE_PROMPT_BRIDGE_GOAL"], + "timestamp_ms": int(time.time() * 1000), +} +print(base64.b64encode(json.dumps(request, separators=(",", ":")).encode("utf-8")).decode("ascii")) +PY +)" + remote_capture "prompt-bridge-response.json" \ + "mkdir -p /var/mobile/Library/OpenPhone/springboard; rm -f /var/mobile/Library/OpenPhone/springboard/prompt-response.json /var/mobile/Library/OpenPhone/springboard/prompt-request.json; printf '%s' '$request_b64' | base64 -d > /var/mobile/Library/OpenPhone/springboard/prompt-request.json; for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do if [ -f /var/mobile/Library/OpenPhone/springboard/prompt-response.json ]; then cat /var/mobile/Library/OpenPhone/springboard/prompt-response.json; exit 0; fi; sleep 1; done; printf '%s\n' '{\"status\":\"timeout\",\"reason\":\"prompt_bridge_response_timeout\"}'" + remote_capture "prompt-bridge-agent-status.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then status_file=/tmp/openphone-prompt-bridge-agent-status.json; for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24; do /var/jb/usr/local/bin/openphone-agentctl agent_status > \"\$status_file\"; if grep -q '$marker' \"\$status_file\" && grep -q '\"latest_task\":{[^}]*\"status\":\"completed\"' \"\$status_file\"; then cat \"\$status_file\"; rm -f \"\$status_file\"; exit 0; fi; sleep 5; done; cat \"\$status_file\"; rm -f \"\$status_file\"; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + + local prompt_bridge_task_id + prompt_bridge_task_id="$(json_field "$run_dir/prompt-bridge-agent-status.json" "latest_task.task_id")" + if [[ -n "$prompt_bridge_task_id" && "$prompt_bridge_task_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "prompt-bridge-trajectory.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_trajectory '$prompt_bridge_task_id' 160; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_prompt_bridge_task_id"}' >"$run_dir/prompt-bridge-trajectory.json" + fi + remote_capture "prompt-bridge-after-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prompt-bridge-safari-state.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/app-ui/com.apple.mobilesafari.json ]; then cat /var/mobile/Library/OpenPhone/app-ui/com.apple.mobilesafari.json; else printf "%s\n" "{\"status\":\"missing\",\"bundle_id\":\"com.apple.mobilesafari\"}"; fi' + remote_capture "prompt-bridge-tweak-log.txt" \ + "if [ -f /var/mobile/Library/OpenPhone/openphone-volume-trigger.log ]; then tail -n $tail_lines /var/mobile/Library/OpenPhone/openphone-volume-trigger.log; fi" +} + +collect_screenshot_if_requested() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_SCREENSHOT:-0}" != "1" ]]; then + return + fi + log "Collecting screenshot artifact" + remote_capture "get-screen-screenshot.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + OPENPHONE_SCREENSHOT_JSON="$run_dir/get-screen-screenshot.json" python3 - <<'PY' >"$run_dir/screenshot-remote-path.txt" +import json +import os +import sys + +path = os.environ["OPENPHONE_SCREENSHOT_JSON"] +try: + data = json.load(open(path, "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +def walk(value): + if isinstance(value, dict): + if value.get("status") == "ok" and isinstance(value.get("path"), str): + yield value["path"] + for item in value.values(): + yield from walk(item) + elif isinstance(value, list): + for item in value: + yield from walk(item) + +for candidate in walk(data): + if candidate.endswith(".png"): + print(candidate) + break +PY + local screenshot_remote + screenshot_remote="$(head -n 1 "$run_dir/screenshot-remote-path.txt" || true)" + if [[ -n "$screenshot_remote" ]]; then + scp_from "$screenshot_remote" "$run_dir/screenshot.png" || true + if [[ -f "$run_dir/screenshot.png" ]]; then + python3 - <<'PY' "$run_dir/screenshot.png" >"$run_dir/screenshot-sanity.json" +import binascii +import itertools +import json +import pathlib +import struct +import sys +import zlib + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + +def paeth(a, b, c): + p = a + b - c + pa = abs(p - a) + pb = abs(p - b) + pc = abs(p - c) + if pa <= pb and pa <= pc: + return a + if pb <= pc: + return b + return c + +def parse_png(path): + data = path.read_bytes() + if not data.startswith(PNG_SIGNATURE): + raise ValueError("not_png") + pos = len(PNG_SIGNATURE) + chunks = [] + idat = [] + info = {} + while pos + 8 <= len(data): + length = struct.unpack(">I", data[pos:pos + 4])[0] + kind = data[pos + 4:pos + 8] + payload = data[pos + 8:pos + 8 + length] + crc_expected = struct.unpack(">I", data[pos + 8 + length:pos + 12 + length])[0] + crc_actual = binascii.crc32(kind + payload) & 0xFFFFFFFF + if crc_actual != crc_expected: + raise ValueError(f"crc_mismatch:{kind.decode('ascii', 'replace')}") + pos += 12 + length + chunks.append(kind.decode("ascii", "replace")) + if kind == b"IHDR": + width, height, bit_depth, color_type, compression, filter_method, interlace = struct.unpack(">IIBBBBB", payload) + info.update({ + "width": width, + "height": height, + "bit_depth": bit_depth, + "color_type": color_type, + "compression": compression, + "filter_method": filter_method, + "interlace": interlace, + }) + elif kind == b"IDAT": + idat.append(payload) + elif kind == b"IEND": + break + if not info: + raise ValueError("missing_ihdr") + return data, chunks, info, b"".join(idat) + +def channel_count(color_type): + return { + 0: 1, + 2: 3, + 4: 2, + 6: 4, + }.get(color_type) + +def decode_rows(info, compressed): + width = info["width"] + height = info["height"] + bit_depth = info["bit_depth"] + color_type = info["color_type"] + if info["interlace"] != 0: + raise ValueError("unsupported_interlace") + if bit_depth not in (8, 16): + raise ValueError(f"unsupported_bit_depth:{bit_depth}") + channels = channel_count(color_type) + if channels is None: + raise ValueError(f"unsupported_color_type:{color_type}") + bytes_per_sample = bit_depth // 8 + bpp = channels * bytes_per_sample + row_len = width * bpp + raw = zlib.decompress(compressed) + expected = height * (row_len + 1) + if len(raw) < expected: + raise ValueError("truncated_idat") + rows = [] + prev = bytearray(row_len) + offset = 0 + for _ in range(height): + filter_type = raw[offset] + offset += 1 + row = bytearray(raw[offset:offset + row_len]) + offset += row_len + for i in range(row_len): + left = row[i - bpp] if i >= bpp else 0 + up = prev[i] + up_left = prev[i - bpp] if i >= bpp else 0 + if filter_type == 0: + recon = row[i] + elif filter_type == 1: + recon = (row[i] + left) & 0xFF + elif filter_type == 2: + recon = (row[i] + up) & 0xFF + elif filter_type == 3: + recon = (row[i] + ((left + up) // 2)) & 0xFF + elif filter_type == 4: + recon = (row[i] + paeth(left, up, up_left)) & 0xFF + else: + raise ValueError(f"unsupported_filter:{filter_type}") + row[i] = recon + rows.append(bytes(row)) + prev = row + return rows, channels, bytes_per_sample + +def sampled_pixels(rows, info, channels, bytes_per_sample): + width = info["width"] + height = info["height"] + bpp = channels * bytes_per_sample + xs = sorted(set(int(round(v)) for v in [i * (width - 1) / 10 for i in range(11)])) if width > 1 else [0] + ys = sorted(set(int(round(v)) for v in [i * (height - 1) / 10 for i in range(11)])) if height > 1 else [0] + for y, x in itertools.product(ys, xs): + start = x * bpp + raw = rows[y][start:start + bpp] + if bytes_per_sample == 2: + values = tuple(raw[i] for i in range(0, len(raw), 2)) + else: + values = tuple(raw) + yield values + +path = pathlib.Path(sys.argv[1]) +result = {"path": str(path), "exists": path.exists(), "status": "error"} +try: + if not path.exists(): + raise ValueError("missing_file") + data, chunks, info, compressed = parse_png(path) + rows, channels, bytes_per_sample = decode_rows(info, compressed) + samples = list(sampled_pixels(rows, info, channels, bytes_per_sample)) + color_components = [sample[:3] if len(sample) >= 3 else sample[:1] for sample in samples] + unique_colors = {component for component in color_components} + nonzero_samples = sum(1 for component in color_components if any(value != 0 for value in component)) + result.update({ + "bytes": len(data), + "format": "png", + "width": info["width"], + "height": info["height"], + "bit_depth": info["bit_depth"], + "color_type": info["color_type"], + "interlace": info["interlace"], + "chunk_counts": {kind: chunks.count(kind) for kind in sorted(set(chunks))}, + "sampled_pixels": len(samples), + "sampled_unique_colors": len(unique_colors), + "sampled_nonzero_pixels": nonzero_samples, + "nonblank": nonzero_samples > 0 and len(unique_colors) > 1, + }) + result["status"] = "ok" if result["nonblank"] else "blank_or_flat" +except Exception as exc: + result["reason"] = str(exc) + +print(json.dumps(result, indent=2)) +PY + fi + fi +} + +collect_unlocked_foreground_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_UNLOCKED_FOREGROUND:-0}" != "1" && \ + "${OPENPHONE_VALIDATE_REQUIRE_UNLOCKED:-0}" != "1" ]]; then + return + fi + log "Collecting unlocked foreground sample" + remote_capture "unlocked-foreground-before-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local locked_state + locked_state="$(OPENPHONE_UNLOCKED_FOREGROUND_BEFORE_JSON="$run_dir/unlocked-foreground-before-screen.json" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_UNLOCKED_FOREGROUND_BEFORE_JSON"], "r", encoding="utf-8")) +except Exception: + print("unknown") + sys.exit(0) + +lock = data.get("context", {}).get("lock", {}) if isinstance(data, dict) else {} +locked = lock.get("locked") +if locked is True: + print("true") +elif locked is False: + print("false") +else: + print("unknown") +PY +)" + if [[ "$locked_state" != "false" ]]; then + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/unlocked-foreground-open-safari.json" + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/unlocked-foreground-safari-screen.json" + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/unlocked-foreground-home.json" + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/unlocked-foreground-home-screen.json" + return + fi + remote_capture "unlocked-foreground-open-safari.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"open_app","bundle_id":"com.apple.mobilesafari","reason":"validator unlocked foreground Safari launch"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "unlocked-foreground-safari-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 2; /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "unlocked-foreground-home.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"home","reason":"validator unlocked foreground cleanup home"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "unlocked-foreground-home-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 1; /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' +} + +collect_app_ui_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_APP_UI:-0}" != "1" ]]; then + return + fi + log "Collecting app-process UI sample" + remote_capture "app-ui-before-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local locked_state + locked_state="$(OPENPHONE_APP_UI_BEFORE_JSON="$run_dir/app-ui-before-screen.json" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_APP_UI_BEFORE_JSON"], "r", encoding="utf-8")) +except Exception: + print("unknown") + sys.exit(0) + +lock = data.get("context", {}).get("lock", {}) if isinstance(data, dict) else {} +locked = lock.get("locked") +if locked is True: + print("true") +elif locked is False: + print("false") +else: + print("unknown") +PY +)" + if [[ "$locked_state" != "false" ]]; then + for name in app-ui-relaunch app-ui-open-safari app-ui-safari-screen app-ui-open-settings app-ui-settings-screen app-ui-health app-ui-safari-state app-ui-settings-state; do + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/$name.json" + done + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/app-ui-ls.txt" + return + fi + remote_capture "app-ui-relaunch.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then killall MobileSafari Preferences >/dev/null 2>&1 || true; sleep 2; printf "%s\n" "{\"status\":\"ok\",\"action\":\"relaunch_targets\"}"; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "app-ui-open-safari.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"open_app","bundle_id":"com.apple.mobilesafari","reason":"validator app UI Safari launch"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "app-ui-safari-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 8; /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "app-ui-open-settings.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"open_app","bundle_id":"com.apple.Preferences","reason":"validator app UI Settings launch"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "app-ui-settings-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 8; /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "app-ui-health.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "app-ui-ls.txt" \ + 'ls -la /var/mobile/Library/OpenPhone/app-ui 2>&1 || true' + remote_capture "app-ui-safari-state.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/app-ui/com.apple.mobilesafari.json ]; then cat /var/mobile/Library/OpenPhone/app-ui/com.apple.mobilesafari.json; else printf "%s\n" "{\"status\":\"missing\",\"bundle_id\":\"com.apple.mobilesafari\"}"; fi' + remote_capture "app-ui-settings-state.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/app-ui/com.apple.Preferences.json ]; then cat /var/mobile/Library/OpenPhone/app-ui/com.apple.Preferences.json; else printf "%s\n" "{\"status\":\"missing\",\"bundle_id\":\"com.apple.Preferences\"}"; fi' + remote_capture "openphone-app-introspector.log.tail" \ + "if [ -f /var/mobile/Library/OpenPhone/openphone-app-introspector.log ]; then tail -n $tail_lines /var/mobile/Library/OpenPhone/openphone-app-introspector.log; fi" +} + +collect_lockscreen_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_LOCKSCREEN:-0}" != "1" ]]; then + return + fi + log "Collecting lock-screen passcode sample" + remote_capture "lockscreen-before-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "lockscreen-show-passcode.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl show_passcode; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "lockscreen-after-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 2; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "lockscreen-status-after.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl agent_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' +} + +collect_prefs_backend_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_PREFS_BACKEND:-0}" != "1" ]]; then + return + fi + log "Collecting OpenPhone Settings backend sample" + remote_capture "prefs-backend-files.json" \ + 'bundle="/var/jb/Library/PreferenceBundles/OpenPhoneAgentPrefs.bundle/OpenPhoneAgentPrefs"; info="/var/jb/Library/PreferenceBundles/OpenPhoneAgentPrefs.bundle/Info.plist"; entry="/var/jb/Library/PreferenceLoader/Preferences/OpenPhoneAgentPrefs.plist"; printf "{\"status\":\"ok\",\"bundle_executable\":%s,\"bundle_info\":%s,\"loader_entry\":%s,\"bundle_path\":\"%s\",\"info_path\":\"%s\",\"loader_entry_path\":\"%s\"}\n" "$([ -x "$bundle" ] && printf true || printf false)" "$([ -f "$info" ] && printf true || printf false)" "$([ -f "$entry" ] && printf true || printf false)" "$bundle" "$info" "$entry"' + remote_capture "prefs-backend-status-before.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl agent_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-backend-disable-hardware.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"agent_control","hardware_triggers_enabled":false,"reason":"validator prefs backend disable hardware triggers","source":"prefs_backend_validator"}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-backend-trigger-disabled.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"hardware_trigger","trigger":"volume_up_down_combo","source":"prefs_backend_validator","reason":"validator hardware disabled suppression","run_task":true,"create_background_job":false,"dedupe":false}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-backend-enable-hardware.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"agent_control","hardware_triggers_enabled":true,"reason":"validator prefs backend restore hardware triggers","source":"prefs_backend_validator"}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-backend-disable-yolo.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"agent_control","yolo_enabled":false,"reason":"validator prefs backend disable yolo","source":"prefs_backend_validator"}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-backend-trigger-yolo-disabled.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"hardware_trigger","trigger":"volume_up_down_combo","source":"prefs_backend_validator","reason":"validator yolo disabled suppression","run_task":true,"create_background_job":false,"dedupe":false}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-backend-enable-yolo.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"agent_control","yolo_enabled":true,"reason":"validator prefs backend restore yolo","source":"prefs_backend_validator"}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-backend-status-after.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl agent_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' +} + +collect_prefs_ui_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_PREFS_UI:-0}" != "1" ]]; then + return + fi + log "Collecting OpenPhone Settings UI sample" + remote_capture "prefs-ui-before-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local locked_state + locked_state="$(OPENPHONE_PREFS_UI_BEFORE_JSON="$run_dir/prefs-ui-before-screen.json" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_PREFS_UI_BEFORE_JSON"], "r", encoding="utf-8")) +except Exception: + print("unknown") + sys.exit(0) + +lock = data.get("context", {}).get("lock", {}) if isinstance(data, dict) else {} +locked = lock.get("locked") +if locked is True: + print("true") +elif locked is False: + print("false") +else: + print("unknown") +PY +)" + if [[ "$locked_state" != "false" ]]; then + for name in prefs-ui-prepare prefs-ui-open-url prefs-ui-url-screen prefs-ui-open-settings prefs-ui-settings-screen prefs-ui-tap-row prefs-ui-pane-screen prefs-ui-disable-hardware prefs-ui-after-disable-screen prefs-ui-status-disabled prefs-ui-enable-hardware prefs-ui-after-enable-screen prefs-ui-status-enabled prefs-ui-final-restore prefs-ui-status-after; do + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/$name.json" + done + printf '%s\n' "" >"$run_dir/prefs-ui-row-element.txt" + printf '%s\n' "" >"$run_dir/prefs-ui-hardware-element.txt" + return + fi + + remote_capture "prefs-ui-prepare.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"agent_control","hardware_triggers_enabled":true,"yolo_enabled":true,"reason":"validator prefs UI prepare policy","source":"prefs_ui_validator"}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-ui-open-url.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then killall Preferences >/dev/null 2>&1 || true; sleep 2; /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"open_url","url":"prefs:root=OpenPhoneAgentPrefs","reason":"validator OpenPhone Settings URL launch"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-ui-url-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 5; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + + local pane_visible + pane_visible="$(OPENPHONE_PREFS_UI_SCREEN_JSON="$run_dir/prefs-ui-url-screen.json" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_PREFS_UI_SCREEN_JSON"], "r", encoding="utf-8")) +except Exception: + print("false") + sys.exit(0) + +tree = data.get("context", {}).get("ui_tree", {}) if isinstance(data, dict) else {} +visible = tree.get("visible_text", []) if isinstance(tree, dict) else [] +visible_set = {str(item) for item in visible} if isinstance(visible, list) else set() +required = {"OpenPhone Agent", "Hardware Triggers", "YOLO Execution"} +print("true" if required.issubset(visible_set) else "false") +PY +)" + if [[ "$pane_visible" == "true" ]]; then + printf '{"status":"skipped","reason":"pane_opened_by_url"}\n' >"$run_dir/prefs-ui-open-settings.json" + cp "$run_dir/prefs-ui-url-screen.json" "$run_dir/prefs-ui-settings-screen.json" + printf '%s\n' "" >"$run_dir/prefs-ui-row-element.txt" + printf '{"status":"skipped","reason":"pane_opened_by_url"}\n' >"$run_dir/prefs-ui-tap-row.json" + cp "$run_dir/prefs-ui-url-screen.json" "$run_dir/prefs-ui-pane-screen.json" + else + remote_capture "prefs-ui-open-settings.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then killall Preferences >/dev/null 2>&1 || true; sleep 2; /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"open_app","bundle_id":"com.apple.Preferences","reason":"validator OpenPhone Settings root launch"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-ui-settings-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 5; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local prefs_row_element_id + prefs_row_element_id="$(OPENPHONE_PREFS_UI_SETTINGS_JSON="$run_dir/prefs-ui-settings-screen.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_PREFS_UI_SETTINGS_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +elements = data.get("context", {}).get("ui_tree", {}).get("interactive_elements", []) +if not isinstance(elements, list): + sys.exit(0) +for element in elements: + if not isinstance(element, dict): + continue + label = element.get("label") + klass = element.get("class") + bounds = element.get("bounds") + element_id = element.get("id") or element.get("element_id") + wide = isinstance(bounds, list) and len(bounds) >= 3 and isinstance(bounds[2], (int, float)) and bounds[2] >= 250 + table_cell = isinstance(klass, str) and "TableCell" in klass + if label == "OpenPhone Agent" and element.get("enabled") is True and table_cell and wide and isinstance(element_id, str): + if re.match(r"^[A-Za-z0-9_.:-]+$", element_id): + print(element_id) + break +PY +)" + printf '%s\n' "$prefs_row_element_id" >"$run_dir/prefs-ui-row-element.txt" + if [[ -n "$prefs_row_element_id" && "$prefs_row_element_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "prefs-ui-tap-row.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"execute_action\",\"action\":{\"type\":\"tap_element\",\"element_id\":\"$prefs_row_element_id\",\"reason\":\"validator OpenPhone Settings pane row\"}}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '{"status":"skipped","reason":"missing_openphone_agent_row"}\n' >"$run_dir/prefs-ui-tap-row.json" + fi + remote_capture "prefs-ui-pane-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 5; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + fi + + local hardware_element_id + hardware_element_id="$(OPENPHONE_PREFS_UI_PANE_JSON="$run_dir/prefs-ui-pane-screen.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_PREFS_UI_PANE_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +elements = data.get("context", {}).get("ui_tree", {}).get("interactive_elements", []) +if not isinstance(elements, list): + sys.exit(0) + +def valid_id(value): + return isinstance(value, str) and re.match(r"^[A-Za-z0-9_.:-]+$", value) + +fallback = "" +for element in elements: + if not isinstance(element, dict): + continue + element_id = element.get("id") or element.get("element_id") + if not valid_id(element_id): + continue + label = element.get("label") + klass = element.get("class") + kind = element.get("kind") + if label == "Hardware Triggers" and ((isinstance(klass, str) and "Switch" in klass) or kind == "switch"): + print(element_id) + sys.exit(0) + if not fallback and label == "Hardware Triggers": + fallback = element_id +if fallback: + print(fallback) +PY +)" + printf '%s\n' "$hardware_element_id" >"$run_dir/prefs-ui-hardware-element.txt" + if [[ -n "$hardware_element_id" && "$hardware_element_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "prefs-ui-disable-hardware.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"execute_action\",\"action\":{\"type\":\"tap_element\",\"element_id\":\"$hardware_element_id\",\"reason\":\"validator OpenPhone Settings disable Hardware Triggers\"}}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '{"status":"skipped","reason":"missing_hardware_triggers_element"}\n' >"$run_dir/prefs-ui-disable-hardware.json" + fi + remote_capture "prefs-ui-after-disable-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 3; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-ui-status-disabled.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl agent_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + + local restore_hardware_element_id + local refreshed_hardware_element_id + restore_hardware_element_id="$hardware_element_id" + refreshed_hardware_element_id="$(OPENPHONE_PREFS_UI_PANE_JSON="$run_dir/prefs-ui-after-disable-screen.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_PREFS_UI_PANE_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +elements = data.get("context", {}).get("ui_tree", {}).get("interactive_elements", []) +if not isinstance(elements, list): + sys.exit(0) + +def valid_id(value): + return isinstance(value, str) and re.match(r"^[A-Za-z0-9_.:-]+$", value) + +fallback = "" +for element in elements: + if not isinstance(element, dict): + continue + element_id = element.get("id") or element.get("element_id") + if not valid_id(element_id): + continue + label = element.get("label") + klass = element.get("class") + kind = element.get("kind") + if label == "Hardware Triggers" and ((isinstance(klass, str) and "Switch" in klass) or kind == "switch"): + print(element_id) + sys.exit(0) + if not fallback and label == "Hardware Triggers": + fallback = element_id +if fallback: + print(fallback) +PY +)" + if [[ -n "$refreshed_hardware_element_id" && "$refreshed_hardware_element_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + restore_hardware_element_id="$refreshed_hardware_element_id" + fi + + if [[ -n "$restore_hardware_element_id" && "$restore_hardware_element_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "prefs-ui-enable-hardware.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"execute_action\",\"action\":{\"type\":\"tap_element\",\"element_id\":\"$restore_hardware_element_id\",\"reason\":\"validator OpenPhone Settings restore Hardware Triggers\"}}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '{"status":"skipped","reason":"missing_hardware_triggers_element"}\n' >"$run_dir/prefs-ui-enable-hardware.json" + fi + remote_capture "prefs-ui-after-enable-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 3; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-ui-status-enabled.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl agent_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-ui-final-restore.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"agent_control","hardware_triggers_enabled":true,"yolo_enabled":true,"reason":"validator prefs UI final restore","source":"prefs_ui_validator"}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "prefs-ui-status-after.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl agent_status; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' +} + +collect_visible_effect_sample() { + if [[ "${OPENPHONE_VALIDATE_INCLUDE_VISIBLE_EFFECTS:-0}" != "1" ]]; then + return + fi + log "Collecting visible-effect UI sample" + remote_capture "visible-effects-before-screen.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl get_screen; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local locked_state + locked_state="$(OPENPHONE_VISIBLE_EFFECTS_BEFORE_JSON="$run_dir/visible-effects-before-screen.json" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_VISIBLE_EFFECTS_BEFORE_JSON"], "r", encoding="utf-8")) +except Exception: + print("unknown") + sys.exit(0) + +lock = data.get("context", {}).get("lock", {}) if isinstance(data, dict) else {} +locked = lock.get("locked") +if locked is True: + print("true") +elif locked is False: + print("false") +else: + print("unknown") +PY +)" + if [[ "$locked_state" != "false" ]]; then + for name in visible-effects-open-settings visible-effects-settings-precheck visible-effects-settings-reset visible-effects-settings-before visible-effects-tap-settings visible-effects-settings-after visible-effects-open-safari visible-effects-safari-before visible-effects-type-safari visible-effects-safari-after visible-effects-safari-state visible-effects-open-notes visible-effects-notes-before visible-effects-type-notes visible-effects-notes-after visible-effects-notes-state; do + printf '{"status":"skipped","reason":"device_locked_or_unknown","locked_state":"%s"}\n' "$locked_state" \ + >"$run_dir/$name.json" + done + printf '%s\n' "" >"$run_dir/visible-effects-settings-scenario.txt" + printf '%s\n' "" >"$run_dir/visible-effects-settings-target-label.txt" + printf '%s\n' "" >"$run_dir/visible-effects-settings-back-element.txt" + printf '%s\n' "" >"$run_dir/visible-effects-settings-element.txt" + printf '%s\n' "" >"$run_dir/visible-effects-safari-field.txt" + printf '%s\n' "" >"$run_dir/visible-effects-safari-marker.txt" + printf '%s\n' "" >"$run_dir/visible-effects-notes-field.txt" + printf '%s\n' "" >"$run_dir/visible-effects-notes-marker.txt" + return + fi + + remote_capture "visible-effects-open-settings.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then killall Preferences >/dev/null 2>&1 || true; sleep 2; /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"open_app","bundle_id":"com.apple.Preferences","reason":"validator visible-effect Settings launch"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "visible-effects-settings-precheck.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 5; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + cp "$run_dir/visible-effects-settings-precheck.json" "$run_dir/visible-effects-settings-before.json" + printf '%s\n' "" >"$run_dir/visible-effects-settings-back-element.txt" + printf '%s\n' '{"status":"skipped","reason":"adaptive_settings_target_does_not_require_root_reset"}' >"$run_dir/visible-effects-settings-reset.json" + local settings_target + settings_target="$(OPENPHONE_VISIBLE_EFFECTS_SETTINGS_PRECHECK_JSON="$run_dir/visible-effects-settings-precheck.json" python3 - <<'PY' +import json +import os +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_VISIBLE_EFFECTS_SETTINGS_PRECHECK_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +tree = data.get("context", {}).get("ui_tree", {}) +visible = tree.get("visible_text", []) if isinstance(tree, dict) else [] +visible_set = {str(item) for item in visible} if isinstance(visible, list) else set() +if {"Text Replacement", "Keyboards"}.issubset(visible_set) and ( + "Enable Dictation" in visible_set or "One-Handed Keyboard" in visible_set +): + print("keyboard_to_keyboards\tKeyboards") +elif {"Add New Keyboard…", "English (US)", "Emoji", "Keyboards"}.issubset(visible_set): + print("keyboards_to_english\tEnglish (US)") +elif {"About", "Software Update", "Keyboard"}.issubset(visible_set): + print("general_to_keyboard\tKeyboard") +else: + print("root_to_general\tGeneral") +PY +)" + local settings_scenario="${settings_target%%$'\t'*}" + local settings_target_label="${settings_target#*$'\t'}" + if [[ -z "$settings_scenario" || "$settings_scenario" == "$settings_target" ]]; then + settings_scenario="root_to_general" + settings_target_label="General" + fi + printf '%s\n' "$settings_scenario" >"$run_dir/visible-effects-settings-scenario.txt" + printf '%s\n' "$settings_target_label" >"$run_dir/visible-effects-settings-target-label.txt" + local settings_element_id + settings_element_id="$(OPENPHONE_VISIBLE_EFFECTS_SETTINGS_BEFORE_JSON="$run_dir/visible-effects-settings-before.json" OPENPHONE_VISIBLE_EFFECTS_SETTINGS_TARGET_LABEL="$settings_target_label" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_VISIBLE_EFFECTS_SETTINGS_BEFORE_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +elements = data.get("context", {}).get("ui_tree", {}).get("interactive_elements", []) +if not isinstance(elements, list): + sys.exit(0) +target_label = os.environ.get("OPENPHONE_VISIBLE_EFFECTS_SETTINGS_TARGET_LABEL", "General") +for element in elements: + if not isinstance(element, dict): + continue + label = element.get("label") + klass = element.get("class") + bounds = element.get("bounds") + element_id = element.get("id") or element.get("element_id") + wide = isinstance(bounds, list) and len(bounds) >= 3 and isinstance(bounds[2], (int, float)) and bounds[2] >= 250 + table_cell = isinstance(klass, str) and "TableCell" in klass + if label == target_label and element.get("enabled") is True and table_cell and wide and isinstance(element_id, str): + if re.match(r"^[A-Za-z0-9_.:-]+$", element_id): + print(element_id) + break +PY +)" + printf '%s\n' "$settings_element_id" >"$run_dir/visible-effects-settings-element.txt" + if [[ -n "$settings_element_id" && "$settings_element_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "visible-effects-tap-settings.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"execute_action\",\"action\":{\"type\":\"tap_element\",\"element_id\":\"$settings_element_id\",\"reason\":\"validator visible-effect Settings $settings_target_label row\"}}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '{"status":"skipped","reason":"missing_settings_target_element","target_label":%s}\n' "$(safe_json_string "$settings_target_label")" >"$run_dir/visible-effects-tap-settings.json" + fi + remote_capture "visible-effects-settings-after.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 5; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + + local marker + marker="OPVisibleEffect-$(date '+%H%M%S')" + printf '%s\n' "$marker" >"$run_dir/visible-effects-safari-marker.txt" + remote_capture "visible-effects-open-safari.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then killall MobileSafari >/dev/null 2>&1 || true; sleep 2; /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"open_url","url":"https://www.wikipedia.org/","reason":"validator visible-effect Safari launch"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "visible-effects-safari-before.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 8; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local safari_field_id + safari_field_id="$(OPENPHONE_VISIBLE_EFFECTS_SAFARI_BEFORE_JSON="$run_dir/visible-effects-safari-before.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_VISIBLE_EFFECTS_SAFARI_BEFORE_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +elements = data.get("context", {}).get("ui_tree", {}).get("interactive_elements", []) +if not isinstance(elements, list): + sys.exit(0) +for element in elements: + if not isinstance(element, dict): + continue + element_id = element.get("id") or element.get("element_id") + if not isinstance(element_id, str) or not element_id.startswith("app-com.apple.mobilesafari-web-"): + continue + label = element.get("label") + kind = element.get("kind") + tag = element.get("tag") + input_type = element.get("input_type") + if label == "INPUT" or kind == "web_text_field" or tag == "input" or input_type == "search": + if re.match(r"^[A-Za-z0-9_.:-]+$", element_id): + print(element_id) + break +PY +)" + printf '%s\n' "$safari_field_id" >"$run_dir/visible-effects-safari-field.txt" + if [[ -n "$safari_field_id" && "$safari_field_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "visible-effects-type-safari.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"execute_action\",\"action\":{\"type\":\"type_text\",\"element_id\":\"$safari_field_id\",\"text\":$(safe_json_string "$marker"),\"reason\":\"validator visible-effect Safari DOM text entry\"}}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_safari_dom_field"}' >"$run_dir/visible-effects-type-safari.json" + fi + remote_capture "visible-effects-safari-after.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 3; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "visible-effects-safari-state.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/app-ui/com.apple.mobilesafari.json ]; then cat /var/mobile/Library/OpenPhone/app-ui/com.apple.mobilesafari.json; else printf "%s\n" "{\"status\":\"missing\",\"bundle_id\":\"com.apple.mobilesafari\"}"; fi' + + local notes_marker + notes_marker="OPVisibleNotes-$(date '+%H%M%S')" + printf '%s\n' "$notes_marker" >"$run_dir/visible-effects-notes-marker.txt" + remote_capture "visible-effects-open-notes.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '"'"'{"command":"execute_action","action":{"type":"open_app","bundle_id":"com.apple.mobilenotes","reason":"validator visible-effect Notes launch"}}'"'"'; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "visible-effects-notes-before.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 5; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + local notes_field_id + notes_field_id="$(OPENPHONE_VISIBLE_EFFECTS_NOTES_BEFORE_JSON="$run_dir/visible-effects-notes-before.json" python3 - <<'PY' +import json +import os +import re +import sys + +try: + data = json.load(open(os.environ["OPENPHONE_VISIBLE_EFFECTS_NOTES_BEFORE_JSON"], "r", encoding="utf-8")) +except Exception: + sys.exit(0) + +elements = data.get("context", {}).get("ui_tree", {}).get("interactive_elements", []) +if not isinstance(elements, list): + sys.exit(0) +for element in elements: + if not isinstance(element, dict): + continue + element_id = element.get("id") or element.get("element_id") + if not isinstance(element_id, str) or not element_id.startswith("app-com.apple.mobilenotes-"): + continue + klass = element.get("class") + kind = element.get("kind") + source_bundle = element.get("source_bundle_id") + editable = ( + kind in ("text_area", "text_field") + or (isinstance(klass, str) and ("TextView" in klass or "TextField" in klass)) + ) + if source_bundle == "com.apple.mobilenotes" and editable and element.get("enabled") is True and not element.get("sensitive"): + if re.match(r"^[A-Za-z0-9_.:-]+$", element_id): + print(element_id) + break +PY +)" + printf '%s\n' "$notes_field_id" >"$run_dir/visible-effects-notes-field.txt" + if [[ -n "$notes_field_id" && "$notes_field_id" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + remote_capture "visible-effects-type-notes.json" \ + "if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then /var/jb/usr/local/bin/openphone-agentctl '{\"command\":\"execute_action\",\"action\":{\"type\":\"type_text\",\"element_id\":\"$notes_field_id\",\"text\":$(safe_json_string "$notes_marker"),\"reason\":\"validator visible-effect Notes body text entry\"}}'; else printf '%s\n' '{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}'; fi" + else + printf '%s\n' '{"status":"skipped","reason":"missing_notes_text_field"}' >"$run_dir/visible-effects-type-notes.json" + fi + remote_capture "visible-effects-notes-after.json" \ + 'if [ -x /var/jb/usr/local/bin/openphone-agentctl ]; then sleep 3; /var/jb/usr/local/bin/openphone-agentctl get_screen screenshot; else printf "%s\n" "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}"; fi' + remote_capture "visible-effects-notes-state.json" \ + 'if [ -f /var/mobile/Library/OpenPhone/app-ui/com.apple.mobilenotes.json ]; then cat /var/mobile/Library/OpenPhone/app-ui/com.apple.mobilenotes.json; else printf "%s\n" "{\"status\":\"missing\",\"bundle_id\":\"com.apple.mobilenotes\"}"; fi' +} + +generate_report() { + OPENPHONE_VALIDATE_MODE="$mode" \ + OPENPHONE_VALIDATE_RUN_ID="$run_id" \ + OPENPHONE_VALIDATE_PACKAGE="$package" \ + OPENPHONE_VALIDATE_INSTALLED="$([[ "$mode" == "collect-only" ]] && printf false || printf true)" \ + OPENPHONE_VALIDATE_REQUIRE_UNLOCKED="${OPENPHONE_VALIDATE_REQUIRE_UNLOCKED:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_SCREENSHOT="${OPENPHONE_VALIDATE_INCLUDE_SCREENSHOT:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_UNLOCKED_FOREGROUND="${OPENPHONE_VALIDATE_INCLUDE_UNLOCKED_FOREGROUND:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_APP_UI="${OPENPHONE_VALIDATE_INCLUDE_APP_UI:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_LOCKSCREEN="${OPENPHONE_VALIDATE_INCLUDE_LOCKSCREEN:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_PREFS_UI="${OPENPHONE_VALIDATE_INCLUDE_PREFS_UI:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_PREFS_BACKEND="${OPENPHONE_VALIDATE_INCLUDE_PREFS_BACKEND:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_STORES="${OPENPHONE_VALIDATE_INCLUDE_STORES:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_PROVIDER_ATTEMPTS="${OPENPHONE_VALIDATE_INCLUDE_PROVIDER_ATTEMPTS:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_VISIBLE_EFFECTS="${OPENPHONE_VALIDATE_INCLUDE_VISIBLE_EFFECTS:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_MEMORY_LIFECYCLE="${OPENPHONE_VALIDATE_INCLUDE_MEMORY_LIFECYCLE:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_MODEL_LOOP="${OPENPHONE_VALIDATE_INCLUDE_MODEL_LOOP:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_PROVIDER_MODEL="${OPENPHONE_VALIDATE_INCLUDE_PROVIDER_MODEL:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_SAFARI_DOM_MODEL="${OPENPHONE_VALIDATE_INCLUDE_SAFARI_DOM_MODEL:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_PROMPT_BRIDGE_MODEL="${OPENPHONE_VALIDATE_INCLUDE_PROMPT_BRIDGE_MODEL:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_TRIGGER_DIAGNOSTICS="${OPENPHONE_VALIDATE_INCLUDE_TRIGGER_DIAGNOSTICS:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_WATCHER_TIMER="${OPENPHONE_VALIDATE_INCLUDE_WATCHER_TIMER:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_WATCHER_REPAIR="${OPENPHONE_VALIDATE_INCLUDE_WATCHER_REPAIR:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_JOB_REPAIR="${OPENPHONE_VALIDATE_INCLUDE_JOB_REPAIR:-0}" \ + OPENPHONE_VALIDATE_INCLUDE_RESTART_RECOVERY="${OPENPHONE_VALIDATE_INCLUDE_RESTART_RECOVERY:-0}" \ + OPENPHONE_VALIDATE_RUN_DIR="$run_dir" \ + python3 - <<'PY' +import datetime +import json +import os +import pathlib +import re +import sys + +run_dir = pathlib.Path(os.environ["OPENPHONE_VALIDATE_RUN_DIR"]) +mode = os.environ["OPENPHONE_VALIDATE_MODE"] +package_path = os.environ.get("OPENPHONE_VALIDATE_PACKAGE", "") + +def read_text(name): + path = run_dir / name + if not path.exists(): + return "" + return path.read_text(encoding="utf-8", errors="replace") + +def read_json(name): + text = read_text(name).strip() + if not text: + return {} + try: + return json.loads(text) + except Exception: + start = text.find("{") + end = text.rfind("}") + if start >= 0 and end > start: + try: + return json.loads(text[start:end + 1]) + except Exception: + return {"status": "error", "reason": "invalid_json", "artifact": name} + return {"status": "error", "reason": "invalid_json", "artifact": name} + +def latest_crash(name): + crashes = [] + for line in read_text(name).splitlines(): + line = line.strip() + if line: + crashes.append(pathlib.Path(line).name) + return max(set(crashes)) if crashes else "" + +def status_of(name): + data = read_json(name) + return data.get("status") if isinstance(data, dict) else None + +def nested_value(data, path, default=None): + value = data + for key in path: + if isinstance(value, dict): + value = value.get(key) + else: + return default + return default if value is None else value + +def int_value(value, default=0): + try: + return int(value) + except Exception: + return default + +def trigger_status_from(data): + if not isinstance(data, dict): + return {} + if data.get("schema") == "openphone.springboard_trigger_status.v1": + return data + if "hooks" in data and ( + "button_events_seen" in data + or "combo_events_seen" in data + or "volume_notification" in data + ): + return data + fallback = nested_value(data, ["triggers", "volume_combo", "springboard_fallback"], {}) + return fallback if isinstance(fallback, dict) else {} + +def trigger_snapshot(data): + status = trigger_status_from(data) + hooks = status.get("hooks") if isinstance(status.get("hooks"), dict) else {} + volume_notification = ( + status.get("volume_notification") + if isinstance(status.get("volume_notification"), dict) + else {} + ) + return { + "status": status.get("status"), + "event": status.get("event"), + "volume_hooked": int_value(hooks.get("volume_hooked")), + "volume_total": int_value(hooks.get("volume_total")), + "any_volume_hooked": bool(hooks.get("any_volume_hooked")), + "button_events_seen": int_value(status.get("button_events_seen")), + "combo_events_seen": int_value(status.get("combo_events_seen")), + "last_button_event": status.get("last_button_event") or "", + "last_button_event_source": status.get("last_button_event_source") or "", + "last_button_event_ms": int_value(status.get("last_button_event_ms")), + "last_combo_event_ms": int_value(status.get("last_combo_event_ms")), + "last_trigger_route": status.get("last_trigger_route") or "", + "volume_notification": { + "installed": bool(volume_notification.get("installed")), + "seeded": bool(volume_notification.get("seeded")), + "events_seen": int_value(volume_notification.get("events_seen")), + "last_event_ms": int_value(volume_notification.get("last_event_ms")), + "last_volume": volume_notification.get("last_volume"), + "last_direction": volume_notification.get("last_direction") or "", + "last_reason": volume_notification.get("last_reason") or "", + "last_category": volume_notification.get("last_category") or "", + }, + } + +def task_summary_from_agent_status(data): + if not isinstance(data, dict): + return {} + latest = data.get("latest_task") if isinstance(data.get("latest_task"), dict) else {} + current = data.get("current_task") if isinstance(data.get("current_task"), dict) else {} + return { + "latest_task_id": latest.get("task_id") or "", + "latest_status": latest.get("status") or "", + "latest_runner": latest.get("runner") or "", + "latest_source": latest.get("source") or "", + "latest_model_provider": latest.get("model_provider") or "", + "latest_model_loop_status": latest.get("model_loop_status") or "", + "latest_stop_reason": latest.get("stop_reason") or "", + "current_task_id": current.get("task_id") or "", + "current_status": current.get("status") or "", + "current_runner": current.get("runner") or "", + } + +def check_trigger_diagnostics_shape(): + before_trigger = read_json("trigger-diagnostics-before-trigger.json") + after_trigger = read_json("trigger-diagnostics-after-trigger.json") + before_status = read_json("trigger-diagnostics-before-status.json") + after_status = read_json("trigger-diagnostics-after-status.json") + before = trigger_snapshot(before_trigger or before_status) + after = trigger_snapshot(after_trigger or after_status) + before_task = task_summary_from_agent_status(before_status) + after_task = task_summary_from_agent_status(after_status) + button_delta = max(0, after["button_events_seen"] - before["button_events_seen"]) + combo_delta = max(0, after["combo_events_seen"] - before["combo_events_seen"]) + notification_delta = max( + 0, + after["volume_notification"]["events_seen"] + - before["volume_notification"]["events_seen"], + ) + latest_task_changed = ( + bool(after_task["latest_task_id"]) + and after_task["latest_task_id"] != before_task["latest_task_id"] + ) + current_task_started = bool(after_task["current_task_id"]) + agent_task_observed = latest_task_changed or current_task_started + latest_runner = after_task["latest_runner"] or after_task["current_runner"] + checks = { + "ok": True, + "errors": [], + "before": before, + "after": after, + "button_event_delta": button_delta, + "combo_event_delta": combo_delta, + "volume_notification_event_delta": notification_delta, + "before_task": before_task, + "after_task": after_task, + "latest_task_changed": latest_task_changed, + "current_task_started": current_task_started, + "agent_task_observed": agent_task_observed, + } + if after["status"] not in ("enabled", "disabled"): + checks["ok"] = False + checks["errors"].append(f"trigger_status:{after['status']}") + if after["status"] != "enabled": + checks["ok"] = False + checks["errors"].append("trigger_not_enabled") + if after["volume_hooked"] <= 0 and not after["volume_notification"]["installed"]: + checks["ok"] = False + checks["errors"].append("no_volume_hook_or_notification_observer") + if button_delta <= 0 and notification_delta <= 0: + checks["ok"] = False + checks["errors"].append("no_physical_volume_event_observed") + if combo_delta <= 0: + checks["ok"] = False + checks["errors"].append("volume_combo_not_observed") + if combo_delta > 0 and not agent_task_observed: + checks["ok"] = False + checks["errors"].append("combo_observed_without_new_agent_task") + if combo_delta > 0 and agent_task_observed and latest_runner and latest_runner != "model": + checks["ok"] = False + checks["errors"].append(f"trigger_task_runner:{latest_runner}") + return checks + +def check_response_shape(name, list_key=None, item_keys=(), required_keys=()): + data = read_json(name) + checks = { + "artifact": name, + "ok": True, + "status": data.get("status") if isinstance(data, dict) else None, + "errors": [], + } + if not isinstance(data, dict): + checks["ok"] = False + checks["errors"].append("not_object") + return checks + if data.get("status") != "ok": + checks["ok"] = False + checks["errors"].append("status_not_ok") + for key in required_keys: + if key not in data: + checks["ok"] = False + checks["errors"].append(f"missing:{key}") + if list_key: + items = data.get(list_key) + if not isinstance(items, list): + checks["ok"] = False + checks["errors"].append(f"{list_key}_not_list") + items = [] + count = data.get("count") + if not isinstance(count, int) or count < 0: + checks["ok"] = False + checks["errors"].append("count_not_nonnegative_int") + elif len(items) > count: + checks["ok"] = False + checks["errors"].append(f"{list_key}_longer_than_count") + checks["count"] = count if isinstance(count, int) else None + checks["items_seen"] = len(items) + if item_keys and items: + for index, item in enumerate(items[:5]): + if not isinstance(item, dict): + checks["ok"] = False + checks["errors"].append(f"{list_key}[{index}]_not_object") + continue + for key in item_keys: + if key not in item: + checks["ok"] = False + checks["errors"].append(f"{list_key}[{index}]_missing:{key}") + return checks + +def check_task_detail_shape(name): + data = read_json(name) + checks = { + "artifact": name, + "ok": True, + "status": data.get("status") if isinstance(data, dict) else None, + "errors": [], + } + if not isinstance(data, dict): + checks["ok"] = False + checks["errors"].append("not_object") + return checks + if data.get("status") != "ok": + checks["ok"] = False + checks["errors"].append("status_not_ok") + task = data.get("task") + if not isinstance(task, dict): + checks["ok"] = False + checks["errors"].append("task_not_object") + task = {} + task_id = data.get("task_id") + if not isinstance(task_id, str) or not task_id: + checks["ok"] = False + checks["errors"].append("missing_task_id") + elif task.get("task_id") != task_id: + checks["ok"] = False + checks["errors"].append("task_id_mismatch") + for key in ("state", "status", "autonomy_mode", "goal"): + if key not in task: + checks["ok"] = False + checks["errors"].append(f"task_missing:{key}") + checks["task_id"] = task_id if isinstance(task_id, str) else "" + return checks + +def check_provider_attempt_shapes(root): + checks = { + "status": "not_observed", + "attempts_seen": 0, + "modern_attempts": 0, + "legacy_attempts": 0, + "errors": [], + } + + def walk(value, path): + if isinstance(value, dict): + attempts = value.get("provider_attempts") + if isinstance(attempts, list): + for index, attempt in enumerate(attempts[:20]): + attempt_path = f"{path}.provider_attempts[{index}]" + checks["attempts_seen"] += 1 + if not isinstance(attempt, dict): + checks["errors"].append(f"{attempt_path}_not_object") + continue + modern = any(key in attempt for key in ( + "scope", + "action_type", + "verification", + "dispatch_metadata", + )) + if not modern: + checks["legacy_attempts"] += 1 + continue + checks["modern_attempts"] += 1 + for key in ("provider", "scope", "action_type", "status"): + if not isinstance(attempt.get(key), str) or not attempt.get(key): + checks["errors"].append(f"{attempt_path}_missing:{key}") + if attempt.get("status") not in ("ok", "unavailable", "not_attempted"): + checks["errors"].append(f"{attempt_path}_status") + verification = attempt.get("verification") + if not isinstance(verification, dict): + checks["errors"].append(f"{attempt_path}_verification_not_object") + else: + if verification.get("status") not in ("verified", "unverified", "failed"): + checks["errors"].append(f"{attempt_path}_verification_status") + for key in ("source", "reason"): + if not isinstance(verification.get(key), str) or not verification.get(key): + checks["errors"].append(f"{attempt_path}_verification_missing:{key}") + for key, item in value.items(): + walk(item, f"{path}.{key}" if path else str(key)) + elif isinstance(value, list): + for index, item in enumerate(value[:50]): + walk(item, f"{path}[{index}]") + + walk(root, "") + if checks["errors"]: + checks["status"] = "fail" + elif checks["modern_attempts"] > 0: + checks["status"] = "pass" + elif checks["legacy_attempts"] > 0: + checks["status"] = "legacy_only" + return checks + +def check_provider_attempt_sample_shape(name): + data = read_json(name) + checks = { + "artifact": name, + "ok": True, + "state": data.get("state") if isinstance(data, dict) else None, + "errors": [], + } + if not isinstance(data, dict): + checks["ok"] = False + checks["errors"].append("not_object") + checks["provider_attempts"] = check_provider_attempt_shapes({}) + return checks + state = data.get("state") + if not isinstance(state, str) or not state: + checks["ok"] = False + checks["errors"].append("missing_state") + attempts = check_provider_attempt_shapes(data) + checks["provider_attempts"] = attempts + if attempts["status"] != "pass": + checks["ok"] = False + checks["errors"].append(f"provider_attempts:{attempts['status']}") + verification = data.get("verification") + if not isinstance(verification, dict): + checks["ok"] = False + checks["errors"].append("verification_not_object") + elif verification.get("status") not in ("verified", "unverified", "failed"): + checks["ok"] = False + checks["errors"].append("verification_status") + if data.get("user_facing_status") not in ("verified", "dispatch_unverified", "failed"): + checks["ok"] = False + checks["errors"].append("user_facing_status") + return checks + +def check_memory_lifecycle_shape(): + artifacts = { + "save_primary": read_json("memory-lifecycle-save-primary.json"), + "update": read_json("memory-lifecycle-update.json"), + "save_source": read_json("memory-lifecycle-save-source.json"), + "merge": read_json("memory-lifecycle-merge.json"), + "save_delete": read_json("memory-lifecycle-save-delete.json"), + "delete": read_json("memory-lifecycle-delete.json"), + "search": read_json("memory-lifecycle-search.json"), + } + checks = { + "ok": True, + "errors": [], + "artifacts": {}, + } + for name, data in artifacts.items(): + status = data.get("status") if isinstance(data, dict) else None + checks["artifacts"][name] = {"status": status} + if status != "ok": + checks["ok"] = False + checks["errors"].append(f"{name}_status:{status}") + + primary_id = artifacts["save_primary"].get("memory", {}).get("memory_id") if isinstance(artifacts["save_primary"].get("memory"), dict) else "" + source_id = artifacts["save_source"].get("memory", {}).get("memory_id") if isinstance(artifacts["save_source"].get("memory"), dict) else "" + delete_id = artifacts["save_delete"].get("memory", {}).get("memory_id") if isinstance(artifacts["save_delete"].get("memory"), dict) else "" + update_memory = artifacts["update"].get("memory") if isinstance(artifacts["update"].get("memory"), dict) else {} + merge_memory = artifacts["merge"].get("memory") if isinstance(artifacts["merge"].get("memory"), dict) else {} + + if not primary_id: + checks["ok"] = False + checks["errors"].append("missing_primary_id") + if not source_id: + checks["ok"] = False + checks["errors"].append("missing_source_id") + if not delete_id: + checks["ok"] = False + checks["errors"].append("missing_delete_id") + if update_memory.get("memory_id") != primary_id: + checks["ok"] = False + checks["errors"].append("update_primary_id_mismatch") + if merge_memory.get("memory_id") != primary_id: + checks["ok"] = False + checks["errors"].append("merge_primary_id_mismatch") + if artifacts["merge"].get("merged_from") != source_id: + checks["ok"] = False + checks["errors"].append("merge_source_id_mismatch") + if artifacts["delete"].get("memory_id") != delete_id or artifacts["delete"].get("deleted") is not True: + checks["ok"] = False + checks["errors"].append("delete_result_mismatch") + + memories = artifacts["search"].get("memories") + if not isinstance(memories, list): + checks["ok"] = False + checks["errors"].append("search_memories_not_list") + memories = [] + if primary_id and not any(item.get("memory_id") == primary_id for item in memories if isinstance(item, dict)): + checks["ok"] = False + checks["errors"].append("merged_memory_not_searchable") + if source_id and any(item.get("memory_id") == source_id for item in memories if isinstance(item, dict)): + checks["ok"] = False + checks["errors"].append("source_memory_still_searchable") + if delete_id and any(item.get("memory_id") == delete_id for item in memories if isinstance(item, dict)): + checks["ok"] = False + checks["errors"].append("deleted_memory_still_searchable") + + checks["primary_memory_id"] = primary_id + checks["source_memory_id"] = source_id + checks["deleted_memory_id"] = delete_id + checks["search_count"] = artifacts["search"].get("count") + return checks + +def check_watcher_timer_shape(): + create = read_json("watcher-timer-create.json") + run_due = read_json("watcher-timer-run-due.json") + job_run_due = read_json("watcher-timer-job-run-due.json") + job_list = read_json("watcher-timer-job-list.json") + after = read_json("watcher-timer-after-list.json") + stop = read_json("watcher-timer-stop.json") + checks = { + "ok": True, + "errors": [], + "create_status": create.get("status") if isinstance(create, dict) else None, + "run_due_status": run_due.get("status") if isinstance(run_due, dict) else None, + "job_run_due_status": job_run_due.get("status") if isinstance(job_run_due, dict) else None, + "job_list_status": job_list.get("status") if isinstance(job_list, dict) else None, + "after_status": after.get("status") if isinstance(after, dict) else None, + "stop_status": stop.get("status") if isinstance(stop, dict) else None, + } + if not isinstance(create, dict) or create.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"create_status:{create.get('status') if isinstance(create, dict) else None}") + create = {} + watcher = create.get("watcher") if isinstance(create.get("watcher"), dict) else {} + public_id = watcher.get("watcher_id") if isinstance(watcher.get("watcher_id"), str) else "" + if create.get("scheduler_status") != "implemented_timer_bridge": + checks["ok"] = False + checks["errors"].append("create_scheduler_status") + if create.get("fires_locally") is not True or watcher.get("fires_locally") is not True: + checks["ok"] = False + checks["errors"].append("create_not_firing_locally") + if not public_id: + checks["ok"] = False + checks["errors"].append("missing_watcher_id") + + if not isinstance(run_due, dict) or run_due.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"run_due_status:{run_due.get('status') if isinstance(run_due, dict) else None}") + run_due = {} + if run_due.get("scheduler_status") != "implemented_timer_bridge": + checks["ok"] = False + checks["errors"].append("run_due_scheduler_status") + fired_count = run_due.get("fired_count") if isinstance(run_due.get("fired_count"), int) else 0 + job_count = run_due.get("job_count") if isinstance(run_due.get("job_count"), int) else 0 + watcher_entries = run_due.get("watchers") + if not isinstance(watcher_entries, list): + checks["ok"] = False + checks["errors"].append("watcher_entries_not_list") + watcher_entries = [] + matched_entry = {} + for entry in watcher_entries: + if not isinstance(entry, dict): + continue + if not public_id or entry.get("watcher_id") == public_id: + matched_entry = entry + break + if not matched_entry: + if fired_count > 0: + checks["ok"] = False + checks["errors"].append("watcher_entry_not_found") + elif matched_entry.get("status") != "background_job_queued": + checks["ok"] = False + checks["errors"].append(f"watcher_entry_status:{matched_entry.get('status')}") + job_id = matched_entry.get("job_id") if isinstance(matched_entry.get("job_id"), str) else "" + + if not isinstance(job_run_due, dict) or job_run_due.get("status") not in ("ok", None): + checks["ok"] = False + checks["errors"].append(f"job_run_due_status:{job_run_due.get('status') if isinstance(job_run_due, dict) else None}") + job_run_due = {} + jobs = job_run_due.get("jobs") + if not isinstance(jobs, list): + jobs = [] + ran_job = {} + for job in jobs: + if not isinstance(job, dict): + continue + if not job_id or job.get("job_id") == job_id: + ran_job = job + break + if not ran_job: + checks["job_run_race"] = True + else: + run_task = ran_job.get("run_task") if isinstance(ran_job.get("run_task"), dict) else {} + if run_task.get("status") not in ("task.finished", "task.failed"): + checks["ok"] = False + checks["errors"].append(f"run_task_status:{run_task.get('status')}") + if not isinstance(run_task.get("task_id"), str) or not run_task.get("task_id"): + checks["ok"] = False + checks["errors"].append("run_task_missing_task_id") + checks["task_id"] = run_task.get("task_id") if isinstance(run_task.get("task_id"), str) else "" + + if not isinstance(after, dict) or after.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"after_status:{after.get('status') if isinstance(after, dict) else None}") + after = {} + after_watchers = after.get("watchers") + if not isinstance(after_watchers, list): + checks["ok"] = False + checks["errors"].append("after_watchers_not_list") + after_watchers = [] + after_match = next((item for item in after_watchers + if isinstance(item, dict) and (not public_id or item.get("watcher_id") == public_id)), {}) + if not after_match: + checks["ok"] = False + checks["errors"].append("after_watcher_missing") + elif after_match.get("status") != "fired": + checks["ok"] = False + checks["errors"].append(f"after_watcher_status:{after_match.get('status')}") + else: + metadata = after_match.get("metadata") if isinstance(after_match.get("metadata"), dict) else {} + if not job_id: + job_id = metadata.get("last_job_id") if isinstance(metadata.get("last_job_id"), str) else "" + if metadata.get("last_fire_status") != "background_job_queued": + checks["ok"] = False + checks["errors"].append("after_last_fire_status") + if not job_id: + checks["ok"] = False + checks["errors"].append("missing_job_id") + + if not isinstance(job_list, dict) or job_list.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"job_list_status:{job_list.get('status') if isinstance(job_list, dict) else None}") + job_list = {} + listed_jobs = job_list.get("jobs") + if not isinstance(listed_jobs, list): + checks["ok"] = False + checks["errors"].append("job_list_jobs_not_list") + listed_jobs = [] + listed_job = next((item for item in listed_jobs + if isinstance(item, dict) and (not job_id or item.get("job_id") == job_id)), {}) + if not listed_job: + checks["ok"] = False + checks["errors"].append("generated_job_not_listed") + + if not isinstance(stop, dict) or stop.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"stop_status:{stop.get('status') if isinstance(stop, dict) else None}") + elif not isinstance(stop.get("stopped_count"), int) or stop.get("stopped_count") < 1: + checks["ok"] = False + checks["errors"].append("stop_count") + checks["watcher_id"] = public_id + checks["job_id"] = job_id + checks["watcher_fire_count"] = fired_count + checks["watcher_jobs_created"] = job_count + checks["job_status"] = listed_job.get("status") if isinstance(listed_job, dict) else "" + return checks + +def check_watcher_repair_shape(): + create = read_json("watcher-repair-create.json") + mark = read_json("watcher-repair-mark-running.json") + repair = read_json("watcher-repair-run.json") + run_due = read_json("watcher-repair-run-due.json") + job_run_due = read_json("watcher-repair-job-run-due.json") + after = read_json("watcher-repair-after-list.json") + stop = read_json("watcher-repair-stop.json") + checks = { + "ok": True, + "errors": [], + "create_status": create.get("status") if isinstance(create, dict) else None, + "mark_status": mark.get("status") if isinstance(mark, dict) else None, + "repair_status": repair.get("status") if isinstance(repair, dict) else None, + "run_due_status": run_due.get("status") if isinstance(run_due, dict) else None, + "job_run_due_status": job_run_due.get("status") if isinstance(job_run_due, dict) else None, + "after_status": after.get("status") if isinstance(after, dict) else None, + "stop_status": stop.get("status") if isinstance(stop, dict) else None, + } + if not isinstance(create, dict) or create.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"create_status:{create.get('status') if isinstance(create, dict) else None}") + create = {} + watcher = create.get("watcher") if isinstance(create.get("watcher"), dict) else {} + public_id = watcher.get("watcher_id") if isinstance(watcher.get("watcher_id"), str) else "" + if not public_id: + checks["ok"] = False + checks["errors"].append("missing_watcher_id") + if create.get("scheduler_status") != "implemented_timer_bridge": + checks["ok"] = False + checks["errors"].append("create_scheduler_status") + + if not isinstance(mark, dict) or mark.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"mark_status:{mark.get('status') if isinstance(mark, dict) else None}") + mark = {} + marked_watcher = mark.get("watcher") if isinstance(mark.get("watcher"), dict) else {} + if marked_watcher.get("status") != "running": + checks["ok"] = False + checks["errors"].append(f"mark_watcher_status:{marked_watcher.get('status')}") + fixture = marked_watcher.get("metadata", {}).get("validation_stuck_fixture") if isinstance(marked_watcher.get("metadata"), dict) else {} + if not isinstance(fixture, dict) or fixture.get("status") != "marked_running": + checks["ok"] = False + checks["errors"].append("missing_validation_fixture") + + if not isinstance(repair, dict) or repair.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"repair_status:{repair.get('status') if isinstance(repair, dict) else None}") + repair = {} + if repair.get("repair_policy") != "requeue_stale_running": + checks["ok"] = False + checks["errors"].append("repair_policy") + if not isinstance(repair.get("repaired_count"), int) or repair.get("repaired_count") < 1: + checks["ok"] = False + checks["errors"].append("repaired_count") + repair_watchers = repair.get("watchers") + if not isinstance(repair_watchers, list): + checks["ok"] = False + checks["errors"].append("repair_watchers_not_list") + repair_watchers = [] + repaired_entry = next((item for item in repair_watchers + if isinstance(item, dict) and (not public_id or item.get("watcher_id") == public_id)), {}) + if not repaired_entry: + checks["ok"] = False + checks["errors"].append("repaired_watcher_missing") + repaired_watcher = {} + else: + if repaired_entry.get("status") != "requeued": + checks["ok"] = False + checks["errors"].append(f"repaired_entry_status:{repaired_entry.get('status')}") + repaired_watcher = repaired_entry.get("watcher") if isinstance(repaired_entry.get("watcher"), dict) else {} + if repaired_watcher.get("status") != "active": + checks["ok"] = False + checks["errors"].append(f"repaired_watcher_status:{repaired_watcher.get('status')}") + stuck_repair = repaired_watcher.get("metadata", {}).get("stuck_repair") if isinstance(repaired_watcher.get("metadata"), dict) else {} + if not isinstance(stuck_repair, dict) or stuck_repair.get("repair_action") != "requeued": + checks["ok"] = False + checks["errors"].append("missing_stuck_repair_metadata") + + if not isinstance(run_due, dict) or run_due.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"run_due_status:{run_due.get('status') if isinstance(run_due, dict) else None}") + run_due = {} + if run_due.get("scheduler_status") != "implemented_timer_bridge": + checks["ok"] = False + checks["errors"].append("run_due_scheduler_status") + if not isinstance(run_due.get("fired_count"), int) or run_due.get("fired_count") < 1: + checks["ok"] = False + checks["errors"].append("run_due_fired_count") + run_due_watchers = run_due.get("watchers") + if not isinstance(run_due_watchers, list): + checks["ok"] = False + checks["errors"].append("run_due_watchers_not_list") + run_due_watchers = [] + fired_entry = next((item for item in run_due_watchers + if isinstance(item, dict) and (not public_id or item.get("watcher_id") == public_id)), {}) + if not fired_entry: + checks["ok"] = False + checks["errors"].append("fired_watcher_missing") + elif fired_entry.get("status") != "background_job_queued": + checks["ok"] = False + checks["errors"].append(f"fired_entry_status:{fired_entry.get('status')}") + + job_run_ok = isinstance(job_run_due, dict) and job_run_due.get("status") == "ok" + if not job_run_ok: + checks["job_run_race"] = True + job_run_due = {} + elif job_run_due.get("scheduler_status") != "implemented_agent_loop": + checks["ok"] = False + checks["errors"].append("job_run_due_scheduler_status") + + if not isinstance(after, dict) or after.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"after_status:{after.get('status') if isinstance(after, dict) else None}") + after = {} + after_watchers = after.get("watchers") + if not isinstance(after_watchers, list): + checks["ok"] = False + checks["errors"].append("after_watchers_not_list") + after_watchers = [] + after_match = next((item for item in after_watchers + if isinstance(item, dict) and (not public_id or item.get("watcher_id") == public_id)), {}) + if not after_match: + checks["ok"] = False + checks["errors"].append("after_watcher_missing") + else: + if after_match.get("status") not in ("fired", "active", "stopped"): + checks["ok"] = False + checks["errors"].append(f"after_watcher_status:{after_match.get('status')}") + metadata = after_match.get("metadata") if isinstance(after_match.get("metadata"), dict) else {} + listed_repair = metadata.get("stuck_repair") if isinstance(metadata.get("stuck_repair"), dict) else {} + if listed_repair.get("repair_action") != "requeued": + checks["ok"] = False + checks["errors"].append("after_missing_stuck_repair") + if metadata.get("last_fire_status") != "background_job_queued": + checks["ok"] = False + checks["errors"].append("after_last_fire_status") + + if not isinstance(stop, dict) or stop.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"stop_status:{stop.get('status') if isinstance(stop, dict) else None}") + elif not isinstance(stop.get("stopped_count"), int) or stop.get("stopped_count") < 1: + checks["ok"] = False + checks["errors"].append("stop_count") + + checks["watcher_id"] = public_id + checks["repaired_count"] = repair.get("repaired_count") if isinstance(repair, dict) else None + checks["run_due_fired_count"] = run_due.get("fired_count") if isinstance(run_due, dict) else None + checks["final_watcher_status"] = after_match.get("status") if isinstance(after_match, dict) else "" + return checks + +def check_job_repair_shape(): + create = read_json("job-repair-create.json") + mark = read_json("job-repair-mark-running.json") + repair = read_json("job-repair-run.json") + run_due = read_json("job-repair-run-due.json") + listing = read_json("job-repair-list.json") + stop = read_json("job-repair-stop.json") + checks = { + "ok": True, + "errors": [], + "create_status": create.get("status") if isinstance(create, dict) else None, + "mark_status": mark.get("status") if isinstance(mark, dict) else None, + "repair_status": repair.get("status") if isinstance(repair, dict) else None, + "run_due_status": run_due.get("status") if isinstance(run_due, dict) else None, + "list_status": listing.get("status") if isinstance(listing, dict) else None, + "stop_status": stop.get("status") if isinstance(stop, dict) else None, + } + if not isinstance(create, dict) or create.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"create_status:{create.get('status') if isinstance(create, dict) else None}") + create = {} + created_job = create.get("job") if isinstance(create.get("job"), dict) else {} + public_id = created_job.get("job_id") if isinstance(created_job.get("job_id"), str) else "" + if not public_id: + checks["ok"] = False + checks["errors"].append("missing_job_id") + + if not isinstance(mark, dict) or mark.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"mark_status:{mark.get('status') if isinstance(mark, dict) else None}") + mark = {} + marked_job = mark.get("job") if isinstance(mark.get("job"), dict) else {} + if marked_job.get("status") != "running": + checks["ok"] = False + checks["errors"].append(f"mark_job_status:{marked_job.get('status')}") + fixture = marked_job.get("payload", {}).get("validation_stuck_fixture") if isinstance(marked_job.get("payload"), dict) else {} + if not isinstance(fixture, dict) or fixture.get("status") != "marked_running": + checks["ok"] = False + checks["errors"].append("missing_validation_fixture") + + if not isinstance(repair, dict) or repair.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"repair_status:{repair.get('status') if isinstance(repair, dict) else None}") + repair = {} + if repair.get("repair_policy") != "requeue_stale_running": + checks["ok"] = False + checks["errors"].append("repair_policy") + if not isinstance(repair.get("repaired_count"), int) or repair.get("repaired_count") < 1: + checks["ok"] = False + checks["errors"].append("repaired_count") + repair_jobs = repair.get("jobs") + if not isinstance(repair_jobs, list): + checks["ok"] = False + checks["errors"].append("repair_jobs_not_list") + repair_jobs = [] + repaired_entry = next((item for item in repair_jobs + if isinstance(item, dict) and (not public_id or item.get("job_id") == public_id)), {}) + if not repaired_entry: + checks["ok"] = False + checks["errors"].append("repaired_job_missing") + repaired_job = {} + else: + if repaired_entry.get("status") != "requeued": + checks["ok"] = False + checks["errors"].append(f"repaired_entry_status:{repaired_entry.get('status')}") + repaired_job = repaired_entry.get("job") if isinstance(repaired_entry.get("job"), dict) else {} + if repaired_job.get("status") != "queued": + checks["ok"] = False + checks["errors"].append(f"repaired_job_status:{repaired_job.get('status')}") + stuck_repair = repaired_job.get("payload", {}).get("stuck_repair") if isinstance(repaired_job.get("payload"), dict) else {} + if not isinstance(stuck_repair, dict) or stuck_repair.get("repair_action") != "requeued": + checks["ok"] = False + checks["errors"].append("missing_stuck_repair_metadata") + + run_due_ok = isinstance(run_due, dict) and run_due.get("status") == "ok" + if not run_due_ok: + checks["job_run_race"] = True + run_due = {} + elif run_due.get("scheduler_status") != "implemented_agent_loop": + checks["ok"] = False + checks["errors"].append("run_due_scheduler_status") + + if not isinstance(listing, dict) or listing.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"list_status:{listing.get('status') if isinstance(listing, dict) else None}") + listing = {} + listed_jobs = listing.get("jobs") + if not isinstance(listed_jobs, list): + checks["ok"] = False + checks["errors"].append("listed_jobs_not_list") + listed_jobs = [] + listed_job = next((item for item in listed_jobs + if isinstance(item, dict) and (not public_id or item.get("job_id") == public_id)), {}) + if not listed_job: + checks["ok"] = False + checks["errors"].append("listed_repaired_job_missing") + else: + if listed_job.get("status") not in ("queued", "completed", "failed", "stopped"): + checks["ok"] = False + checks["errors"].append(f"listed_job_status:{listed_job.get('status')}") + if not run_due_ok and listed_job.get("status") not in ("completed", "failed", "stopped"): + checks["ok"] = False + checks["errors"].append("run_due_missing_without_completed_job") + listed_repair = listed_job.get("payload", {}).get("stuck_repair") if isinstance(listed_job.get("payload"), dict) else {} + if not isinstance(listed_repair, dict) or listed_repair.get("repair_action") != "requeued": + checks["ok"] = False + checks["errors"].append("listed_missing_stuck_repair") + + if not isinstance(stop, dict) or stop.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"stop_status:{stop.get('status') if isinstance(stop, dict) else None}") + elif not isinstance(stop.get("stopped_count"), int) or stop.get("stopped_count") < 1: + checks["ok"] = False + checks["errors"].append("stop_count") + + checks["job_id"] = public_id + checks["repaired_count"] = repair.get("repaired_count") if isinstance(repair, dict) else None + checks["run_due_ran_count"] = run_due.get("ran_count") if isinstance(run_due, dict) else None + checks["final_job_status"] = listed_job.get("status") if isinstance(listed_job, dict) else "" + return checks + +def check_restart_recovery_shape(): + watcher_create = read_json("restart-recovery-watcher-create.json") + job_create = read_json("restart-recovery-job-create.json") + watcher_mark = read_json("restart-recovery-watcher-mark-running.json") + job_mark = read_json("restart-recovery-job-mark-running.json") + restart = read_json("restart-recovery-restart.json") + after_health = read_json("restart-recovery-after-health.json") + watcher_list = read_json("restart-recovery-watcher-list.json") + job_list = read_json("restart-recovery-job-list.json") + watcher_stop = read_json("restart-recovery-watcher-stop.json") + job_stop = read_json("restart-recovery-job-stop.json") + generated_job_stop = read_json("restart-recovery-generated-job-stop.json") + checks = { + "ok": True, + "errors": [], + "watcher_create_status": watcher_create.get("status") if isinstance(watcher_create, dict) else None, + "job_create_status": job_create.get("status") if isinstance(job_create, dict) else None, + "watcher_mark_status": watcher_mark.get("status") if isinstance(watcher_mark, dict) else None, + "job_mark_status": job_mark.get("status") if isinstance(job_mark, dict) else None, + "restart_status": restart.get("status") if isinstance(restart, dict) else None, + "after_health_status": after_health.get("status") if isinstance(after_health, dict) else None, + "watcher_list_status": watcher_list.get("status") if isinstance(watcher_list, dict) else None, + "job_list_status": job_list.get("status") if isinstance(job_list, dict) else None, + "watcher_stop_status": watcher_stop.get("status") if isinstance(watcher_stop, dict) else None, + "job_stop_status": job_stop.get("status") if isinstance(job_stop, dict) else None, + "generated_job_stop_status": generated_job_stop.get("status") if isinstance(generated_job_stop, dict) else None, + } + if not isinstance(watcher_create, dict) or watcher_create.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"watcher_create_status:{watcher_create.get('status') if isinstance(watcher_create, dict) else None}") + watcher_create = {} + if not isinstance(job_create, dict) or job_create.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"job_create_status:{job_create.get('status') if isinstance(job_create, dict) else None}") + job_create = {} + created_watcher = watcher_create.get("watcher") if isinstance(watcher_create.get("watcher"), dict) else {} + created_job = job_create.get("job") if isinstance(job_create.get("job"), dict) else {} + watcher_public_id = created_watcher.get("watcher_id") if isinstance(created_watcher.get("watcher_id"), str) else "" + job_public_id = created_job.get("job_id") if isinstance(created_job.get("job_id"), str) else "" + if not watcher_public_id: + checks["ok"] = False + checks["errors"].append("missing_watcher_id") + if not job_public_id: + checks["ok"] = False + checks["errors"].append("missing_job_id") + + if not isinstance(watcher_mark, dict) or watcher_mark.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"watcher_mark_status:{watcher_mark.get('status') if isinstance(watcher_mark, dict) else None}") + watcher_mark = {} + marked_watcher = watcher_mark.get("watcher") if isinstance(watcher_mark.get("watcher"), dict) else {} + if marked_watcher.get("status") != "running": + checks["ok"] = False + checks["errors"].append(f"watcher_mark_state:{marked_watcher.get('status')}") + watcher_fixture = marked_watcher.get("metadata", {}).get("validation_stuck_fixture") if isinstance(marked_watcher.get("metadata"), dict) else {} + if not isinstance(watcher_fixture, dict) or watcher_fixture.get("status") != "marked_running": + checks["ok"] = False + checks["errors"].append("watcher_fixture_missing") + + if not isinstance(job_mark, dict) or job_mark.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"job_mark_status:{job_mark.get('status') if isinstance(job_mark, dict) else None}") + job_mark = {} + marked_job = job_mark.get("job") if isinstance(job_mark.get("job"), dict) else {} + if marked_job.get("status") != "running": + checks["ok"] = False + checks["errors"].append(f"job_mark_state:{marked_job.get('status')}") + job_fixture = marked_job.get("payload", {}).get("validation_stuck_fixture") if isinstance(marked_job.get("payload"), dict) else {} + if not isinstance(job_fixture, dict) or job_fixture.get("status") != "marked_running": + checks["ok"] = False + checks["errors"].append("job_fixture_missing") + + if not isinstance(restart, dict) or restart.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"restart_status:{restart.get('status') if isinstance(restart, dict) else None}") + restart = {} + before_pid = restart.get("before_pid") if isinstance(restart.get("before_pid"), str) else "" + after_pid = restart.get("after_pid") if isinstance(restart.get("after_pid"), str) else "" + post_wait_pid = restart.get("post_wait_pid") if isinstance(restart.get("post_wait_pid"), str) else "" + if not before_pid or not after_pid: + checks["ok"] = False + checks["errors"].append("restart_pid_missing") + elif before_pid == after_pid: + checks["ok"] = False + checks["errors"].append("restart_pid_unchanged") + if after_pid and post_wait_pid and after_pid != post_wait_pid: + checks["ok"] = False + checks["errors"].append("restart_pid_changed_after_wait") + + if not isinstance(after_health, dict) or after_health.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"after_health_status:{after_health.get('status') if isinstance(after_health, dict) else None}") + elif after_health.get("autonomy_mode") != "yolo": + checks["ok"] = False + checks["errors"].append("after_health_autonomy") + else: + health_pid = str(after_health.get("pid", "")) + if post_wait_pid and health_pid and health_pid != post_wait_pid: + checks["ok"] = False + checks["errors"].append("after_health_pid_mismatch") + wait_ms = restart.get("post_restart_wait_seconds", 0) + uptime_ms = after_health.get("uptime_ms", 0) + if isinstance(wait_ms, int) and isinstance(uptime_ms, int) and wait_ms > 0: + if uptime_ms < max(0, (wait_ms - 5) * 1000): + checks["ok"] = False + checks["errors"].append("after_health_uptime_short") + + if not isinstance(watcher_list, dict) or watcher_list.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"watcher_list_status:{watcher_list.get('status') if isinstance(watcher_list, dict) else None}") + watcher_list = {} + listed_watchers = watcher_list.get("watchers") + if not isinstance(listed_watchers, list): + checks["ok"] = False + checks["errors"].append("watcher_list_not_list") + listed_watchers = [] + listed_watcher = next((item for item in listed_watchers + if isinstance(item, dict) and (not watcher_public_id or item.get("watcher_id") == watcher_public_id)), {}) + generated_job_id = "" + if not listed_watcher: + checks["ok"] = False + checks["errors"].append("watcher_missing_after_restart") + else: + if listed_watcher.get("status") != "fired": + checks["ok"] = False + checks["errors"].append(f"watcher_final_status:{listed_watcher.get('status')}") + watcher_metadata = listed_watcher.get("metadata") if isinstance(listed_watcher.get("metadata"), dict) else {} + watcher_schedule = listed_watcher.get("schedule") if isinstance(listed_watcher.get("schedule"), dict) else {} + watcher_repair = watcher_metadata.get("stuck_repair") if isinstance(watcher_metadata.get("stuck_repair"), dict) else {} + if watcher_repair.get("repair_action") != "requeued": + checks["ok"] = False + checks["errors"].append("watcher_missing_stuck_repair") + if watcher_metadata.get("last_fire_status") != "background_job_queued": + checks["ok"] = False + checks["errors"].append("watcher_not_fired_after_restart") + generated_job_id = watcher_metadata.get("last_job_id") or watcher_schedule.get("last_job_id") or "" + if not generated_job_id: + checks["ok"] = False + checks["errors"].append("watcher_generated_job_missing") + + if not isinstance(job_list, dict) or job_list.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"job_list_status:{job_list.get('status') if isinstance(job_list, dict) else None}") + job_list = {} + listed_jobs = job_list.get("jobs") + if not isinstance(listed_jobs, list): + checks["ok"] = False + checks["errors"].append("job_list_not_list") + listed_jobs = [] + listed_job = next((item for item in listed_jobs + if isinstance(item, dict) and (not job_public_id or item.get("job_id") == job_public_id)), {}) + if not listed_job: + checks["ok"] = False + checks["errors"].append("job_missing_after_restart") + else: + if listed_job.get("status") not in ("queued", "completed", "failed"): + checks["ok"] = False + checks["errors"].append(f"job_final_status:{listed_job.get('status')}") + job_repair = listed_job.get("payload", {}).get("stuck_repair") if isinstance(listed_job.get("payload"), dict) else {} + if job_repair.get("repair_action") != "requeued": + checks["ok"] = False + checks["errors"].append("job_missing_stuck_repair") + + if not isinstance(watcher_stop, dict) or watcher_stop.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"watcher_stop_status:{watcher_stop.get('status') if isinstance(watcher_stop, dict) else None}") + elif not isinstance(watcher_stop.get("stopped_count"), int) or watcher_stop.get("stopped_count") < 1: + checks["ok"] = False + checks["errors"].append("watcher_stop_count") + if not isinstance(job_stop, dict) or job_stop.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"job_stop_status:{job_stop.get('status') if isinstance(job_stop, dict) else None}") + elif not isinstance(job_stop.get("stopped_count"), int) or job_stop.get("stopped_count") < 1: + checks["ok"] = False + checks["errors"].append("job_stop_count") + if generated_job_id: + if not isinstance(generated_job_stop, dict) or generated_job_stop.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"generated_job_stop_status:{generated_job_stop.get('status') if isinstance(generated_job_stop, dict) else None}") + elif not isinstance(generated_job_stop.get("stopped_count"), int) or generated_job_stop.get("stopped_count") < 1: + checks["ok"] = False + checks["errors"].append("generated_job_stop_count") + + checks["watcher_id"] = watcher_public_id + checks["job_id"] = job_public_id + checks["generated_job_id"] = generated_job_id + checks["before_pid"] = before_pid + checks["after_pid"] = after_pid + checks["post_wait_pid"] = post_wait_pid + checks["final_watcher_status"] = listed_watcher.get("status") if isinstance(listed_watcher, dict) else "" + checks["final_job_status"] = listed_job.get("status") if isinstance(listed_job, dict) else "" + return checks + +def check_model_loop_shape(): + status = read_json("model-status.json") + run = read_json("model-loop-run.json") + trajectory = read_json("model-loop-trajectory.json") + repair_run = read_json("model-loop-repair-run.json") + repair_trajectory = read_json("model-loop-repair-trajectory.json") + cancel_start = read_json("model-loop-cancel-start.json") + cancel_stop = read_json("model-loop-cancel-stop.json") + cancel_run = read_json("model-loop-cancel-run.json") + cancel_trajectory = read_json("model-loop-cancel-trajectory.json") + checks = { + "ok": True, + "errors": [], + "model_status": status.get("status") if isinstance(status, dict) else None, + "run_status": run.get("status") if isinstance(run, dict) else None, + "trajectory_status": trajectory.get("status") if isinstance(trajectory, dict) else None, + "repair_run_status": repair_run.get("status") if isinstance(repair_run, dict) else None, + "repair_trajectory_status": repair_trajectory.get("status") if isinstance(repair_trajectory, dict) else None, + "cancel_run_status": cancel_run.get("status") if isinstance(cancel_run, dict) else None, + "cancel_trajectory_status": cancel_trajectory.get("status") if isinstance(cancel_trajectory, dict) else None, + } + if not isinstance(status, dict) or status.get("schema") != "openphone.model_status.v1": + checks["ok"] = False + checks["errors"].append("model_status_schema") + if not isinstance(run, dict): + checks["ok"] = False + checks["errors"].append("run_not_object") + run = {} + if run.get("status") != "task.finished": + checks["ok"] = False + checks["errors"].append(f"run_status:{run.get('status')}") + if run.get("runner") != "model": + checks["ok"] = False + checks["errors"].append("runner_not_model") + if run.get("model_provider") != "fixture": + checks["ok"] = False + checks["errors"].append("provider_not_fixture") + if run.get("stop_reason") != "finish_task": + checks["ok"] = False + checks["errors"].append("stop_reason_not_finish_task") + if run.get("steps_used") != 2: + checks["ok"] = False + checks["errors"].append("steps_used_not_2") + if run.get("parser_failures") != 0: + checks["ok"] = False + checks["errors"].append("parser_failures") + if run.get("tool_errors") != 0: + checks["ok"] = False + checks["errors"].append("tool_errors") + task_id = run.get("task_id") + if not isinstance(task_id, str) or not task_id: + checks["ok"] = False + checks["errors"].append("missing_task_id") + if not isinstance(trajectory, dict) or trajectory.get("status") != "ok": + checks["ok"] = False + checks["errors"].append("trajectory_status") + events = [] + else: + events = trajectory.get("events") + if not isinstance(events, list): + checks["ok"] = False + checks["errors"].append("trajectory_events_not_list") + events = [] + event_names = [event.get("event") for event in events if isinstance(event, dict)] + if "model_prompt_prepared" not in event_names: + checks["ok"] = False + checks["errors"].append("missing_model_prompt_prepared") + if event_names.count("model_decision") < 2: + checks["ok"] = False + checks["errors"].append("missing_model_decisions") + if "model_step_verified" not in event_names: + checks["ok"] = False + checks["errors"].append("missing_model_step_verified") + if "model_loop_finished" not in event_names: + checks["ok"] = False + checks["errors"].append("missing_model_loop_finished") + + if not isinstance(repair_run, dict): + checks["ok"] = False + checks["errors"].append("repair_run_not_object") + repair_run = {} + if repair_run.get("status") != "task.finished": + checks["ok"] = False + checks["errors"].append(f"repair_run_status:{repair_run.get('status')}") + if repair_run.get("runner") != "model": + checks["ok"] = False + checks["errors"].append("repair_runner_not_model") + if repair_run.get("model_provider") != "fixture": + checks["ok"] = False + checks["errors"].append("repair_provider_not_fixture") + if repair_run.get("stop_reason") != "finish_task": + checks["ok"] = False + checks["errors"].append("repair_stop_reason_not_finish_task") + if repair_run.get("steps_used") != 1: + checks["ok"] = False + checks["errors"].append("repair_steps_used_not_1") + if repair_run.get("parser_failures") != 0: + checks["ok"] = False + checks["errors"].append("repair_parser_failures") + if repair_run.get("tool_errors") != 0: + checks["ok"] = False + checks["errors"].append("repair_tool_errors") + repair_task_id = repair_run.get("task_id") + if not isinstance(repair_task_id, str) or not repair_task_id: + checks["ok"] = False + checks["errors"].append("missing_repair_task_id") + if not isinstance(repair_trajectory, dict) or repair_trajectory.get("status") != "ok": + checks["ok"] = False + checks["errors"].append("repair_trajectory_status") + repair_events = [] + else: + repair_events = repair_trajectory.get("events") + if not isinstance(repair_events, list): + checks["ok"] = False + checks["errors"].append("repair_trajectory_events_not_list") + repair_events = [] + repair_event_names = [event.get("event") for event in repair_events if isinstance(event, dict)] + if "model_prompt_prepared" not in repair_event_names: + checks["ok"] = False + checks["errors"].append("repair_missing_model_prompt_prepared") + if "model_parse_repaired" not in repair_event_names: + checks["ok"] = False + checks["errors"].append("repair_missing_model_parse_repaired") + if repair_event_names.count("model_decision") != 1: + checks["ok"] = False + checks["errors"].append("repair_model_decision_count") + if "model_loop_finished" not in repair_event_names: + checks["ok"] = False + checks["errors"].append("repair_missing_model_loop_finished") + + cancel_task_id = cancel_start.get("task_id") if isinstance(cancel_start, dict) else "" + if not isinstance(cancel_task_id, str) or not cancel_task_id: + checks["ok"] = False + checks["errors"].append("missing_cancel_task_id") + if not isinstance(cancel_stop, dict) or cancel_stop.get("state") != "task.stopped": + checks["ok"] = False + checks["errors"].append("cancel_stop_state") + if isinstance(cancel_stop, dict) and cancel_stop.get("cancel_requested") is not True: + checks["ok"] = False + checks["errors"].append("cancel_requested_not_true") + if not isinstance(cancel_run, dict): + checks["ok"] = False + checks["errors"].append("cancel_run_not_object") + cancel_run = {} + if cancel_run.get("status") != "task.cancelled": + checks["ok"] = False + checks["errors"].append(f"cancel_run_status:{cancel_run.get('status')}") + if cancel_run.get("runner") != "model": + checks["ok"] = False + checks["errors"].append("cancel_runner_not_model") + if cancel_run.get("model_provider") != "fixture": + checks["ok"] = False + checks["errors"].append("cancel_provider_not_fixture") + if cancel_run.get("stop_reason") != "cancelled": + checks["ok"] = False + checks["errors"].append("cancel_stop_reason") + if cancel_run.get("steps_used") != 0: + checks["ok"] = False + checks["errors"].append("cancel_steps_used_not_0") + if cancel_run.get("parser_failures") != 0: + checks["ok"] = False + checks["errors"].append("cancel_parser_failures") + if cancel_run.get("tool_errors") != 0: + checks["ok"] = False + checks["errors"].append("cancel_tool_errors") + if not isinstance(cancel_trajectory, dict) or cancel_trajectory.get("status") != "ok": + checks["ok"] = False + checks["errors"].append("cancel_trajectory_status") + cancel_events = [] + else: + cancel_events = cancel_trajectory.get("events") + if not isinstance(cancel_events, list): + checks["ok"] = False + checks["errors"].append("cancel_trajectory_events_not_list") + cancel_events = [] + cancel_event_names = [event.get("event") for event in cancel_events if isinstance(event, dict)] + if "task_stopped" not in cancel_event_names: + checks["ok"] = False + checks["errors"].append("cancel_missing_task_stopped") + if "model_loop_cancelled" not in cancel_event_names: + checks["ok"] = False + checks["errors"].append("cancel_missing_model_loop_cancelled") + if "model_loop_finished" not in cancel_event_names: + checks["ok"] = False + checks["errors"].append("cancel_missing_model_loop_finished") + if "model_decision" in cancel_event_names: + checks["ok"] = False + checks["errors"].append("cancel_model_decision_executed") + checks["task_id"] = task_id if isinstance(task_id, str) else "" + checks["repair_task_id"] = repair_task_id if isinstance(repair_task_id, str) else "" + checks["cancel_task_id"] = cancel_task_id if isinstance(cancel_task_id, str) else "" + checks["events_seen"] = len(events) + checks["repair_events_seen"] = len(repair_events) + checks["cancel_events_seen"] = len(cancel_events) + return checks + +def check_provider_model_shape(): + configure = read_json("provider-model-configure.json") + status = read_json("provider-model-status.json") + run = read_json("provider-model-run.json") + trajectory = read_json("provider-model-trajectory.json") + reset = read_json("provider-model-reset.json") + checks = { + "ok": True, + "errors": [], + "configure_status": configure.get("status") if isinstance(configure, dict) else None, + "status_status": status.get("status") if isinstance(status, dict) else None, + "run_status": run.get("status") if isinstance(run, dict) else None, + "trajectory_status": trajectory.get("status") if isinstance(trajectory, dict) else None, + "reset_status": reset.get("status") if isinstance(reset, dict) else None, + } + if not isinstance(configure, dict) or configure.get("status") != "ready": + checks["ok"] = False + checks["errors"].append(f"configure_status:{configure.get('status') if isinstance(configure, dict) else None}") + if isinstance(configure, dict): + if configure.get("mode") != "broker": + checks["ok"] = False + checks["errors"].append("configure_mode_not_broker") + if configure.get("credential_required") is not False: + checks["ok"] = False + checks["errors"].append("configure_credential_required") + if not bool(configure.get("endpoint_configured")): + checks["ok"] = False + checks["errors"].append("configure_endpoint_not_configured") + if not isinstance(status, dict) or status.get("status") != "ready": + checks["ok"] = False + checks["errors"].append(f"model_status:{status.get('status') if isinstance(status, dict) else None}") + if not isinstance(run, dict): + checks["ok"] = False + checks["errors"].append("run_not_object") + run = {} + if run.get("status") != "task.finished": + checks["ok"] = False + checks["errors"].append(f"run_status:{run.get('status')}") + if run.get("runner") != "model": + checks["ok"] = False + checks["errors"].append("runner_not_model") + if run.get("model_provider") != "broker": + checks["ok"] = False + checks["errors"].append("provider_not_broker") + if run.get("stop_reason") != "finish_task": + checks["ok"] = False + checks["errors"].append("stop_reason_not_finish_task") + if run.get("parser_failures") != 0: + checks["ok"] = False + checks["errors"].append("parser_failures") + if run.get("tool_errors") != 0: + checks["ok"] = False + checks["errors"].append("tool_errors") + task_id = run.get("task_id") + if not isinstance(task_id, str) or not task_id: + checks["ok"] = False + checks["errors"].append("missing_task_id") + if not isinstance(trajectory, dict) or trajectory.get("status") != "ok": + checks["ok"] = False + checks["errors"].append("trajectory_status") + events = [] + else: + events = trajectory.get("events") + if not isinstance(events, list): + checks["ok"] = False + checks["errors"].append("trajectory_events_not_list") + events = [] + event_names = [event.get("event") for event in events if isinstance(event, dict)] + for required in ("model_prompt_prepared", "model_request", "model_response", "model_decision", "model_loop_finished"): + if required not in event_names: + checks["ok"] = False + checks["errors"].append(f"missing_{required}") + provider_backed = False + provider_model = "" + for event in events: + if not isinstance(event, dict) or event.get("event") != "model_response": + continue + payload = event.get("payload") + metadata = payload.get("metadata") if isinstance(payload, dict) and isinstance(payload.get("metadata"), dict) else {} + if metadata.get("provider") == "bedrock_converse" and metadata.get("provider_backed") is True: + provider_backed = True + provider_model = metadata.get("model") if isinstance(metadata.get("model"), str) else "" + break + if not provider_backed: + checks["ok"] = False + checks["errors"].append("missing_provider_backed_metadata") + if not isinstance(reset, dict) or reset.get("enabled") is not False: + checks["ok"] = False + checks["errors"].append("reset_not_disabled") + checks["task_id"] = task_id if isinstance(task_id, str) else "" + checks["events_seen"] = len(events) + checks["provider_backed"] = provider_backed + checks["provider_model"] = provider_model + return checks + +def contains_text(value, needle): + if not needle: + return False + if isinstance(value, str): + return needle in value + if isinstance(value, dict): + return any(contains_text(item, needle) for item in value.values()) + if isinstance(value, list): + return any(contains_text(item, needle) for item in value) + return False + +def web_dom_status(data): + context = data.get("context") if isinstance(data, dict) else {} + ui_tree = context.get("ui_tree") if isinstance(context, dict) else {} + web_dom = ui_tree.get("web_dom") if isinstance(ui_tree, dict) else {} + return web_dom if isinstance(web_dom, dict) else {} + +def check_safari_dom_model_shape(): + marker = read_text("safari-dom-model-marker.txt").strip() + before = read_json("safari-dom-model-before-screen.json") + open_url = read_json("safari-dom-model-open-url.json") + pre_screen = read_json("safari-dom-model-pre-screen.json") + configure = read_json("safari-dom-model-configure.json") + status = read_json("safari-dom-model-status.json") + run = read_json("safari-dom-model-run.json") + trajectory = read_json("safari-dom-model-trajectory.json") + after_screen = read_json("safari-dom-model-after-screen.json") + safari_state = read_json("safari-dom-model-safari-state.json") + reset = read_json("safari-dom-model-reset.json") + before_fields = app_ui_fields(before) + pre_fields = app_ui_fields(pre_screen) + after_fields = app_ui_fields(after_screen) + pre_web_dom = web_dom_status(pre_screen) + after_web_dom = web_dom_status(after_screen) + checks = { + "ok": True, + "blocked": False, + "errors": [], + "marker": marker, + "before_locked": before_fields["locked"], + "open_url_state": action_state(open_url), + "pre": pre_fields, + "after": after_fields, + "pre_web_dom_status": pre_web_dom.get("status"), + "pre_web_dom_element_count": pre_web_dom.get("element_count"), + "after_web_dom_status": after_web_dom.get("status"), + "after_web_dom_element_count": after_web_dom.get("element_count"), + "configure_status": configure.get("status") if isinstance(configure, dict) else None, + "model_status": status.get("status") if isinstance(status, dict) else None, + "run_status": run.get("status") if isinstance(run, dict) else None, + "trajectory_status": trajectory.get("status") if isinstance(trajectory, dict) else None, + "safari_state_status": safari_state.get("status") if isinstance(safari_state, dict) else None, + "reset_status": reset.get("status") if isinstance(reset, dict) else None, + } + if before_fields["locked"] is not False: + checks["ok"] = False + checks["blocked"] = True + checks["errors"].append("device_locked_or_unknown") + return checks + if not marker: + checks["ok"] = False + checks["errors"].append("missing_marker") + if not isinstance(open_url, dict) or action_state(open_url) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"open_url_state:{action_state(open_url)}") + if pre_fields["foreground_app"] != "com.apple.mobilesafari": + checks["ok"] = False + checks["errors"].append(f"pre_foreground_app:{pre_fields['foreground_app']}") + if pre_fields["ui_tree_source"] != "app_process": + checks["ok"] = False + checks["errors"].append(f"pre_ui_tree_source:{pre_fields['ui_tree_source']}") + if pre_web_dom.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"pre_web_dom_status:{pre_web_dom.get('status')}") + if not isinstance(pre_web_dom.get("element_count"), int) or pre_web_dom.get("element_count") <= 0: + checks["ok"] = False + checks["errors"].append(f"pre_web_dom_element_count:{pre_web_dom.get('element_count')}") + if not isinstance(configure, dict) or configure.get("status") != "ready": + checks["ok"] = False + checks["errors"].append(f"configure_status:{configure.get('status') if isinstance(configure, dict) else None}") + if isinstance(configure, dict): + if configure.get("mode") != "bedrock_converse": + checks["ok"] = False + checks["errors"].append("configure_mode_not_bedrock_converse") + if not configure.get("credential", {}).get("credential_file_present"): + checks["ok"] = False + checks["errors"].append("credential_file_not_present") + if not isinstance(status, dict) or status.get("status") != "ready": + checks["ok"] = False + checks["errors"].append(f"model_status:{status.get('status') if isinstance(status, dict) else None}") + if isinstance(status, dict) and status.get("mode") != "bedrock_converse": + checks["ok"] = False + checks["errors"].append("status_mode_not_bedrock_converse") + if not isinstance(run, dict): + checks["ok"] = False + checks["errors"].append("run_not_object") + run = {} + if run.get("status") != "task.finished": + checks["ok"] = False + checks["errors"].append(f"run_status:{run.get('status')}") + if run.get("runner") != "model": + checks["ok"] = False + checks["errors"].append("runner_not_model") + if run.get("model_provider") != "bedrock_converse": + checks["ok"] = False + checks["errors"].append(f"model_provider:{run.get('model_provider')}") + if run.get("stop_reason") != "verified_type_text_goal_complete": + checks["ok"] = False + checks["errors"].append(f"stop_reason:{run.get('stop_reason')}") + if run.get("parser_failures") != 0: + checks["ok"] = False + checks["errors"].append("parser_failures") + if run.get("tool_errors") != 0: + checks["ok"] = False + checks["errors"].append("tool_errors") + if run.get("unverified_ui_actions") != 0: + checks["ok"] = False + checks["errors"].append("unverified_ui_actions") + task_id = run.get("task_id") + if not isinstance(task_id, str) or not task_id: + checks["ok"] = False + checks["errors"].append("missing_task_id") + if not isinstance(trajectory, dict) or trajectory.get("status") != "ok": + checks["ok"] = False + checks["errors"].append("trajectory_status") + events = [] + else: + events = trajectory.get("events") + if not isinstance(events, list): + checks["ok"] = False + checks["errors"].append("trajectory_events_not_list") + events = [] + event_names = [event.get("event") for event in events if isinstance(event, dict)] + for required in ("model_prompt_prepared", "model_request", "model_response", "model_decision", "tool_call", "model_step_verified", "model_loop_finished"): + if required not in event_names: + checks["ok"] = False + checks["errors"].append(f"missing_{required}") + provider_backed = False + chose_type_text = False + typed_marker = False + dom_verified = False + webcontent_attempt = False + for event in events: + if not isinstance(event, dict): + continue + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + if event.get("event") == "model_response": + metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} + if metadata.get("provider") == "bedrock_converse" and metadata.get("provider_backed") is True: + provider_backed = True + if event.get("event") == "model_decision": + decision = payload.get("decision") if isinstance(payload.get("decision"), dict) else {} + arguments = decision.get("arguments") if isinstance(decision.get("arguments"), dict) else {} + if decision.get("tool") == "type_text": + chose_type_text = True + if marker and arguments.get("text") == marker: + typed_marker = True + if event.get("event") == "model_step_verified": + if payload.get("tool") == "type_text": + verification = payload.get("verification") if isinstance(payload.get("verification"), dict) else {} + provider_verification = verification.get("provider_verification") if isinstance(verification.get("provider_verification"), dict) else {} + if verification.get("status") == "verified" and provider_verification.get("source") == "web_content_dom_state": + dom_verified = True + tool_result = payload.get("tool_result") if isinstance(payload.get("tool_result"), dict) else {} + attempts = tool_result.get("provider_attempts") if isinstance(tool_result.get("provider_attempts"), list) else [] + for attempt in attempts: + if not isinstance(attempt, dict): + continue + if attempt.get("provider") == "OpenPhoneAppIntrospector.WebContentInput" and attempt.get("activation_method") == "webkit_dom_text_input": + webcontent_attempt = True + if not provider_backed: + checks["ok"] = False + checks["errors"].append("missing_bedrock_provider_metadata") + if not chose_type_text: + checks["ok"] = False + checks["errors"].append("missing_type_text_decision") + if not typed_marker: + checks["ok"] = False + checks["errors"].append("type_text_marker_mismatch") + if not dom_verified: + checks["ok"] = False + checks["errors"].append("missing_dom_verified_step") + if not webcontent_attempt: + checks["ok"] = False + checks["errors"].append("missing_webcontent_input_attempt") + marker_visible = contains_text(after_screen, marker) or contains_text(safari_state, marker) + if not marker_visible: + checks["ok"] = False + checks["errors"].append("marker_not_visible_after") + if after_fields["foreground_app"] != "com.apple.mobilesafari": + checks["ok"] = False + checks["errors"].append(f"after_foreground_app:{after_fields['foreground_app']}") + if not isinstance(reset, dict) or reset.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"reset_status:{reset.get('status') if isinstance(reset, dict) else None}") + checks["task_id"] = task_id if isinstance(task_id, str) else "" + checks["events_seen"] = len(events) + checks["provider_backed"] = provider_backed + checks["type_text_decision"] = chose_type_text + checks["typed_marker"] = typed_marker + checks["dom_verified"] = dom_verified + checks["webcontent_attempt"] = webcontent_attempt + checks["marker_visible_after"] = marker_visible + return checks + +def check_prompt_bridge_model_shape(): + marker = read_text("prompt-bridge-marker.txt").strip() + request_id = read_text("prompt-bridge-request-id.txt").strip() + before = read_json("prompt-bridge-before-screen.json") + open_url = read_json("prompt-bridge-open-url.json") + pre_screen = read_json("prompt-bridge-pre-screen.json") + model_status = read_json("prompt-bridge-model-status.json") + response = read_json("prompt-bridge-response.json") + agent_status = read_json("prompt-bridge-agent-status.json") + trajectory = read_json("prompt-bridge-trajectory.json") + after_screen = read_json("prompt-bridge-after-screen.json") + safari_state = read_json("prompt-bridge-safari-state.json") + tweak_log = read_text("prompt-bridge-tweak-log.txt") + before_fields = app_ui_fields(before) + pre_fields = app_ui_fields(pre_screen) + after_fields = app_ui_fields(after_screen) + pre_web_dom = web_dom_status(pre_screen) + latest_task = agent_status.get("latest_task") if isinstance(agent_status, dict) else {} + if not isinstance(latest_task, dict): + latest_task = {} + checks = { + "ok": True, + "blocked": False, + "errors": [], + "marker": marker, + "request_id": request_id, + "before_locked": before_fields["locked"], + "open_url_state": action_state(open_url), + "pre": pre_fields, + "after": after_fields, + "pre_web_dom_status": pre_web_dom.get("status"), + "pre_web_dom_element_count": pre_web_dom.get("element_count"), + "model_status": model_status.get("status") if isinstance(model_status, dict) else None, + "model_mode": model_status.get("mode") if isinstance(model_status, dict) else None, + "prompt_response_status": response.get("status") if isinstance(response, dict) else None, + "prompt_response_operation": response.get("operation") if isinstance(response, dict) else None, + "agent_state": agent_status.get("state") if isinstance(agent_status, dict) else None, + "latest_task_id": latest_task.get("task_id"), + "latest_task_status": latest_task.get("status"), + "latest_task_stop_reason": latest_task.get("stop_reason"), + "latest_task_model_provider": latest_task.get("model_provider"), + "latest_task_tool": latest_task.get("model_loop_tool"), + "trajectory_status": trajectory.get("status") if isinstance(trajectory, dict) else None, + "safari_state_status": safari_state.get("status") if isinstance(safari_state, dict) else None, + "tweak_log_mentions_request": request_id in tweak_log if request_id else False, + } + if before_fields["locked"] is not False: + checks["ok"] = False + checks["blocked"] = True + checks["errors"].append("device_locked_or_unknown") + return checks + if not marker: + checks["ok"] = False + checks["errors"].append("missing_marker") + if not request_id: + checks["ok"] = False + checks["errors"].append("missing_request_id") + if not isinstance(open_url, dict) or action_state(open_url) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"open_url_state:{action_state(open_url)}") + if pre_fields["foreground_app"] != "com.apple.mobilesafari": + checks["ok"] = False + checks["errors"].append(f"pre_foreground_app:{pre_fields['foreground_app']}") + if pre_fields["ui_tree_source"] != "app_process": + checks["ok"] = False + checks["errors"].append(f"pre_ui_tree_source:{pre_fields['ui_tree_source']}") + if pre_web_dom.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"pre_web_dom_status:{pre_web_dom.get('status')}") + if not isinstance(model_status, dict) or model_status.get("status") != "ready": + checks["ok"] = False + checks["errors"].append(f"model_status:{model_status.get('status') if isinstance(model_status, dict) else None}") + if not isinstance(response, dict) or response.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"prompt_response_status:{response.get('status') if isinstance(response, dict) else None}") + if isinstance(response, dict): + if response.get("operation") != "run_goal": + checks["ok"] = False + checks["errors"].append(f"prompt_response_operation:{response.get('operation')}") + if request_id and response.get("request_id") != request_id: + checks["ok"] = False + checks["errors"].append("prompt_response_request_id_mismatch") + if latest_task.get("status") != "completed": + checks["ok"] = False + checks["errors"].append(f"latest_task_status:{latest_task.get('status')}") + if latest_task.get("stop_reason") != "verified_type_text_goal_complete": + checks["ok"] = False + checks["errors"].append(f"latest_task_stop_reason:{latest_task.get('stop_reason')}") + if latest_task.get("model_provider") != "bedrock_converse": + checks["ok"] = False + checks["errors"].append(f"latest_task_model_provider:{latest_task.get('model_provider')}") + if latest_task.get("model_loop_tool") != "type_text": + checks["ok"] = False + checks["errors"].append(f"latest_task_tool:{latest_task.get('model_loop_tool')}") + if latest_task.get("tool_errors") != 0: + checks["ok"] = False + checks["errors"].append("latest_task_tool_errors") + task_id = latest_task.get("task_id") + if not isinstance(task_id, str) or not task_id: + checks["ok"] = False + checks["errors"].append("missing_task_id") + if not isinstance(trajectory, dict) or trajectory.get("status") != "ok": + checks["ok"] = False + checks["errors"].append("trajectory_status") + events = [] + else: + events = trajectory.get("events") + if not isinstance(events, list): + checks["ok"] = False + checks["errors"].append("trajectory_events_not_list") + events = [] + event_names = [event.get("event") for event in events if isinstance(event, dict)] + for required in ("task_started", "model_prompt_prepared", "model_request", "model_response", "model_decision", "tool_call", "model_step_verified", "model_loop_finished"): + if required not in event_names: + checks["ok"] = False + checks["errors"].append(f"missing_{required}") + provider_backed = False + chose_type_text = False + typed_marker = False + dom_verified = False + webcontent_attempt = False + for event in events: + if not isinstance(event, dict): + continue + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + if event.get("event") == "model_response": + metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} + if metadata.get("provider") == "bedrock_converse" and metadata.get("provider_backed") is True: + provider_backed = True + if event.get("event") == "model_decision": + decision = payload.get("decision") if isinstance(payload.get("decision"), dict) else {} + arguments = decision.get("arguments") if isinstance(decision.get("arguments"), dict) else {} + if decision.get("tool") == "type_text": + chose_type_text = True + if marker and arguments.get("text") == marker: + typed_marker = True + if event.get("event") == "model_step_verified" and payload.get("tool") == "type_text": + verification = payload.get("verification") if isinstance(payload.get("verification"), dict) else {} + provider_verification = verification.get("provider_verification") if isinstance(verification.get("provider_verification"), dict) else {} + if verification.get("status") == "verified" and provider_verification.get("source") == "web_content_dom_state": + dom_verified = True + tool_result = payload.get("tool_result") if isinstance(payload.get("tool_result"), dict) else {} + attempts = tool_result.get("provider_attempts") if isinstance(tool_result.get("provider_attempts"), list) else [] + for attempt in attempts: + if not isinstance(attempt, dict): + continue + if attempt.get("provider") == "OpenPhoneAppIntrospector.WebContentInput" and attempt.get("activation_method") == "webkit_dom_text_input": + webcontent_attempt = True + marker_visible = contains_text(after_screen, marker) or contains_text(safari_state, marker) + if not provider_backed: + checks["ok"] = False + checks["errors"].append("missing_bedrock_provider_metadata") + if not chose_type_text: + checks["ok"] = False + checks["errors"].append("missing_type_text_decision") + if not typed_marker: + checks["ok"] = False + checks["errors"].append("type_text_marker_mismatch") + if not dom_verified: + checks["ok"] = False + checks["errors"].append("missing_dom_verified_step") + if not webcontent_attempt: + checks["ok"] = False + checks["errors"].append("missing_webcontent_input_attempt") + if not marker_visible: + checks["ok"] = False + checks["errors"].append("marker_not_visible_after") + if after_fields["foreground_app"] != "com.apple.mobilesafari": + checks["ok"] = False + checks["errors"].append(f"after_foreground_app:{after_fields['foreground_app']}") + if request_id and request_id not in tweak_log: + checks["ok"] = False + checks["errors"].append("tweak_log_missing_request") + checks["task_id"] = task_id if isinstance(task_id, str) else "" + checks["events_seen"] = len(events) + checks["provider_backed"] = provider_backed + checks["type_text_decision"] = chose_type_text + checks["typed_marker"] = typed_marker + checks["dom_verified"] = dom_verified + checks["webcontent_attempt"] = webcontent_attempt + checks["marker_visible_after"] = marker_visible + return checks + +def check_trajectory_shape(name): + data = read_json(name) + checks = { + "artifact": name, + "ok": True, + "status": data.get("status") if isinstance(data, dict) else None, + "errors": [], + } + if not isinstance(data, dict): + checks["ok"] = False + checks["errors"].append("not_object") + return checks + if data.get("status") != "ok": + checks["ok"] = False + checks["errors"].append("status_not_ok") + if not isinstance(data.get("task_id"), str) or not data.get("task_id"): + checks["ok"] = False + checks["errors"].append("missing_task_id") + if not isinstance(data.get("trajectory_path"), str) or not data.get("trajectory_path"): + checks["ok"] = False + checks["errors"].append("missing_trajectory_path") + events = data.get("events") + if not isinstance(events, list): + checks["ok"] = False + checks["errors"].append("events_not_list") + events = [] + count = data.get("count") + if not isinstance(count, int) or count < 0: + checks["ok"] = False + checks["errors"].append("count_not_nonnegative_int") + elif len(events) > count: + checks["ok"] = False + checks["errors"].append("events_longer_than_count") + if not events: + checks["ok"] = False + checks["errors"].append("events_empty") + for index, event in enumerate(events[:10]): + if not isinstance(event, dict): + checks["ok"] = False + checks["errors"].append(f"events[{index}]_not_object") + continue + for key in ("schema", "timestamp_ms", "event", "payload", "source"): + if key not in event: + checks["ok"] = False + checks["errors"].append(f"events[{index}]_missing:{key}") + if event.get("schema") != "openphone.trajectory_event.v1": + checks["ok"] = False + checks["errors"].append(f"events[{index}]_schema") + provider_attempts = check_provider_attempt_shapes(events) + checks["provider_attempts"] = provider_attempts + if provider_attempts["status"] == "fail": + checks["ok"] = False + checks["errors"].extend(provider_attempts["errors"]) + checks["count"] = count if isinstance(count, int) else None + checks["items_seen"] = len(events) + return checks + +FORBIDDEN_SECRET_KEYS = { + "password", + "passcode", + "pin", + "token", + "access_token", + "refresh_token", + "bearer", + "bearer_token", + "authorization", + "auth_header", + "api_key", + "apikey", + "secret", + "client_secret", + "private_key", + "secret_key", + "aws_access_key_id", + "aws_secret_access_key", +} + +SAFE_SECRET_KEY_EXCEPTIONS = { + "passcode_available", + "passcode_visible", + "requires_passcode", + "has_passcode", +} + +SECRET_VALUE_PATTERNS = ( + ("private_key_pem", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")), + ("authorization_bearer", re.compile(r"\bauthorization\s*:\s*bearer\s+[A-Za-z0-9._~+/=-]{12,}", re.IGNORECASE)), + ("sensitive_env_assignment", re.compile(r"\b(AWS_BEARER_TOKEN_BEDROCK|OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENPHONE_IOS_PASSWORD)\s*=\s*['\"]?[^'\"\s]+", re.IGNORECASE)), + ("openai_api_key", re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b")), + ("bedrock_bearer_token", re.compile(r"\bABSK[A-Za-z0-9+/=]{20,}\b")), + ("aws_access_key", re.compile(r"\b(AKIA|ASIA)[A-Z0-9]{16}\b")), + ("slack_token", re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b")), + ("basic_auth_url", re.compile(r"https?://[^/\s:@]{1,128}:[^/\s:@]{1,128}@")), +) + +MAX_HYGIENE_FINDINGS = 50 +MAX_HYGIENE_FILE_BYTES = 2 * 1024 * 1024 +BINARY_ARTIFACT_SUFFIXES = { + ".deb", + ".dylib", + ".gif", + ".heic", + ".jpg", + ".jpeg", + ".png", + ".sqlite", +} + +def normalize_secret_key(key): + return re.sub(r"[^a-z0-9]+", "_", str(key).strip().lower()).strip("_") + +def is_forbidden_secret_key(key): + normalized = normalize_secret_key(key) + if normalized in SAFE_SECRET_KEY_EXCEPTIONS: + return False + if normalized in FORBIDDEN_SECRET_KEYS: + return True + return normalized.endswith(( + "_password", + "_passcode", + "_token", + "_api_key", + "_apikey", + "_secret", + "_private_key", + "_secret_key", + )) + +def add_hygiene_finding(findings, finding): + if len(findings) < MAX_HYGIENE_FINDINGS: + findings.append(finding) + +def scan_secret_string(artifact, locator, value, findings): + for detector, pattern in SECRET_VALUE_PATTERNS: + if pattern.search(value): + finding = { + "artifact": artifact, + "detector": detector, + } + if locator: + finding["location"] = locator + add_hygiene_finding(findings, finding) + +def scan_json_for_hygiene(artifact, value, path, findings): + if isinstance(value, dict): + for key, item in value.items(): + next_path = f"{path}.{key}" if path else str(key) + if is_forbidden_secret_key(key): + add_hygiene_finding(findings, { + "artifact": artifact, + "detector": "forbidden_json_key", + "location": next_path, + }) + scan_json_for_hygiene(artifact, item, next_path, findings) + elif isinstance(value, list): + for index, item in enumerate(value): + scan_json_for_hygiene(artifact, item, f"{path}[{index}]", findings) + elif isinstance(value, str): + scan_secret_string(artifact, path, value, findings) + +def scan_text_for_hygiene(artifact, text, findings): + for line_number, line in enumerate(text.splitlines(), 1): + before = len(findings) + scan_secret_string(artifact, f"line:{line_number}", line, findings) + if len(findings) > before and len(findings) >= MAX_HYGIENE_FINDINGS: + return + +def check_artifact_hygiene(): + findings = [] + checked_files = [] + skipped_files = [] + + for path in sorted(run_dir.iterdir()): + if not path.is_file(): + continue + artifact = path.name + if artifact in ("report.json", "exit-code.txt"): + continue + suffix = path.suffix.lower() + if suffix in BINARY_ARTIFACT_SUFFIXES: + skipped_files.append({"artifact": artifact, "reason": "binary_suffix"}) + continue + try: + size = path.stat().st_size + except OSError: + skipped_files.append({"artifact": artifact, "reason": "stat_failed"}) + continue + if size > MAX_HYGIENE_FILE_BYTES: + skipped_files.append({"artifact": artifact, "reason": "too_large"}) + continue + try: + raw = path.read_bytes() + except OSError: + skipped_files.append({"artifact": artifact, "reason": "read_failed"}) + continue + if b"\x00" in raw[:4096]: + skipped_files.append({"artifact": artifact, "reason": "binary_probe"}) + continue + + text = raw.decode("utf-8", errors="replace") + checked_files.append(artifact) + if suffix == ".json" and text.strip(): + try: + data = json.loads(text) + except Exception: + scan_text_for_hygiene(artifact, text, findings) + else: + scan_json_for_hygiene(artifact, data, "", findings) + else: + scan_text_for_hygiene(artifact, text, findings) + + return { + "status": "fail" if findings else "pass", + "checked_files": len(checked_files), + "skipped_files": skipped_files, + "findings": findings, + "truncated": len(findings) >= MAX_HYGIENE_FINDINGS, + } + +def action_state(data): + if not isinstance(data, dict): + return None + return data.get("state") or data.get("status") + +def screen_context(data): + return data.get("context", {}) if isinstance(data, dict) else {} + +def context_lock_state(context): + lock = context.get("lock", {}) if isinstance(context, dict) else {} + return lock.get("locked") if isinstance(lock, dict) else None + +def foreground_fields(data): + context = screen_context(data) + springboard = context.get("springboard_state", {}) if isinstance(context, dict) else {} + risk_flags = context.get("risk_flags", []) if isinstance(context, dict) else [] + return { + "foreground_app": context.get("foreground_app") if isinstance(context, dict) else None, + "foreground_source": context.get("foreground_source") if isinstance(context, dict) else None, + "springboard_state_status": springboard.get("status") if isinstance(springboard, dict) else None, + "springboard_state_foreground_app": springboard.get("foreground_app") if isinstance(springboard, dict) else None, + "risk_flags": risk_flags if isinstance(risk_flags, list) else [], + "locked": context_lock_state(context), + } + +def app_ui_fields(data): + context = screen_context(data) + app_state = context.get("app_ui_state", {}) if isinstance(context, dict) else {} + ui_tree = context.get("ui_tree", {}) if isinstance(context, dict) else {} + risk_flags = context.get("risk_flags", []) if isinstance(context, dict) else [] + return { + "status": data.get("status") if isinstance(data, dict) else None, + "locked": context_lock_state(context), + "foreground_app": context.get("foreground_app") if isinstance(context, dict) else None, + "foreground_source": context.get("foreground_source") if isinstance(context, dict) else None, + "ui_tree_source": context.get("ui_tree_source") if isinstance(context, dict) else None, + "app_ui_status": app_state.get("status") if isinstance(app_state, dict) else None, + "app_ui_bundle_id": app_state.get("bundle_id") if isinstance(app_state, dict) else None, + "ui_tree_status": ui_tree.get("status") if isinstance(ui_tree, dict) else None, + "element_count": ui_tree.get("element_count") if isinstance(ui_tree, dict) else None, + "text_count": ui_tree.get("text_count") if isinstance(ui_tree, dict) else None, + "risk_flags": risk_flags if isinstance(risk_flags, list) else [], + } + +def ui_visible_text(data): + context = screen_context(data) + ui_tree = context.get("ui_tree", {}) if isinstance(context, dict) else {} + visible = ui_tree.get("visible_text") if isinstance(ui_tree, dict) else [] + return [str(item) for item in visible] if isinstance(visible, list) else [] + +def json_contains_string(value, needle): + if not needle: + return False + if isinstance(value, str): + return needle in value + if isinstance(value, dict): + return any(json_contains_string(item, needle) for item in value.values()) + if isinstance(value, list): + return any(json_contains_string(item, needle) for item in value) + return False + +def screenshot_fields(data): + screenshot = data.get("screenshot") if isinstance(data, dict) else {} + if not isinstance(screenshot, dict): + context = screen_context(data) + screenshot = context.get("screenshot", {}) if isinstance(context, dict) else {} + if not isinstance(screenshot, dict): + screenshot = {} + return { + "status": screenshot.get("status"), + "provider": screenshot.get("provider"), + "path": screenshot.get("path"), + "sha256": screenshot.get("sha256"), + "bytes": screenshot.get("bytes"), + "width": screenshot.get("width"), + "height": screenshot.get("height"), + } + +def screenshot_hash_changed(before, after): + before_screenshot = screenshot_fields(before) + after_screenshot = screenshot_fields(after) + before_hash = before_screenshot.get("sha256") + after_hash = after_screenshot.get("sha256") + return { + "before": before_screenshot, + "after": after_screenshot, + "changed": ( + before_screenshot.get("status") == "ok" + and after_screenshot.get("status") == "ok" + and isinstance(before_hash, str) + and isinstance(after_hash, str) + and len(before_hash) >= 16 + and len(after_hash) >= 16 + and before_hash != after_hash + ), + } + +def check_unlocked_foreground_shape(): + before = read_json("unlocked-foreground-before-screen.json") + open_safari = read_json("unlocked-foreground-open-safari.json") + safari_screen = read_json("unlocked-foreground-safari-screen.json") + home = read_json("unlocked-foreground-home.json") + home_screen = read_json("unlocked-foreground-home-screen.json") + before_fields = foreground_fields(before) + safari_fields = foreground_fields(safari_screen) + home_fields = foreground_fields(home_screen) + checks = { + "ok": True, + "blocked": False, + "errors": [], + "before_locked": before_fields["locked"], + "open_safari_state": action_state(open_safari), + "safari_foreground_app": safari_fields["foreground_app"], + "safari_foreground_source": safari_fields["foreground_source"], + "safari_springboard_state_status": safari_fields["springboard_state_status"], + "safari_springboard_state_foreground_app": safari_fields["springboard_state_foreground_app"], + "home_state": action_state(home), + "home_foreground_app": home_fields["foreground_app"], + "home_foreground_source": home_fields["foreground_source"], + "home_springboard_state_status": home_fields["springboard_state_status"], + } + if before_fields["locked"] is not False: + checks["ok"] = False + checks["blocked"] = True + checks["errors"].append("device_locked_or_unknown") + return checks + if not isinstance(open_safari, dict) or open_safari.get("status") == "skipped": + checks["ok"] = False + checks["errors"].append(f"open_safari_status:{open_safari.get('status') if isinstance(open_safari, dict) else None}") + elif action_state(open_safari) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"open_safari_state:{action_state(open_safari)}") + + if not isinstance(safari_screen, dict) or safari_screen.get("status") == "skipped": + checks["ok"] = False + checks["errors"].append(f"safari_screen_status:{safari_screen.get('status') if isinstance(safari_screen, dict) else None}") + else: + if safari_fields["locked"] is not False: + checks["ok"] = False + checks["errors"].append(f"safari_locked:{safari_fields['locked']}") + if safari_fields["foreground_app"] != "com.apple.mobilesafari": + checks["ok"] = False + checks["errors"].append(f"safari_foreground_app:{safari_fields['foreground_app']}") + if safari_fields["foreground_source"] != "springboard_state": + checks["ok"] = False + checks["errors"].append(f"safari_foreground_source:{safari_fields['foreground_source']}") + if safari_fields["springboard_state_status"] != "ok": + checks["ok"] = False + checks["errors"].append(f"safari_springboard_state_status:{safari_fields['springboard_state_status']}") + if safari_fields["springboard_state_foreground_app"] != "com.apple.mobilesafari": + checks["ok"] = False + checks["errors"].append(f"safari_springboard_state_foreground_app:{safari_fields['springboard_state_foreground_app']}") + if "foreground_app_inferred" in safari_fields["risk_flags"]: + checks["ok"] = False + checks["errors"].append("safari_foreground_inferred") + + if not isinstance(home, dict) or home.get("status") == "skipped": + checks["ok"] = False + checks["errors"].append(f"home_status:{home.get('status') if isinstance(home, dict) else None}") + elif action_state(home) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"home_state:{action_state(home)}") + + if not isinstance(home_screen, dict) or home_screen.get("status") == "skipped": + checks["ok"] = False + checks["errors"].append(f"home_screen_status:{home_screen.get('status') if isinstance(home_screen, dict) else None}") + elif home_fields["locked"] is not False: + checks["ok"] = False + checks["errors"].append(f"home_locked:{home_fields['locked']}") + return checks + +def check_app_ui_shape(): + before = read_json("app-ui-before-screen.json") + relaunch = read_json("app-ui-relaunch.json") + open_safari = read_json("app-ui-open-safari.json") + safari_screen = read_json("app-ui-safari-screen.json") + open_settings = read_json("app-ui-open-settings.json") + settings_screen = read_json("app-ui-settings-screen.json") + health_after = read_json("app-ui-health.json") + safari_state = read_json("app-ui-safari-state.json") + settings_state = read_json("app-ui-settings-state.json") + before_fields = app_ui_fields(before) + safari_fields = app_ui_fields(safari_screen) + settings_fields = app_ui_fields(settings_screen) + health_screen = ((health_after.get("providers") or {}).get("screen") or {}) if isinstance(health_after, dict) else {} + intake = health_screen.get("app_ui_intake") if isinstance(health_screen, dict) else {} + checks = { + "ok": True, + "blocked": False, + "errors": [], + "before_locked": before_fields["locked"], + "relaunch_status": relaunch.get("status") if isinstance(relaunch, dict) else None, + "open_safari_state": action_state(open_safari), + "open_settings_state": action_state(open_settings), + "safari": safari_fields, + "settings": settings_fields, + "intake_status": intake.get("status") if isinstance(intake, dict) else None, + "publish_count": intake.get("publish_count") if isinstance(intake, dict) else None, + "last_bundle_id": intake.get("last_bundle_id") if isinstance(intake, dict) else None, + "safari_state_status": safari_state.get("status") if isinstance(safari_state, dict) else None, + "settings_state_status": settings_state.get("status") if isinstance(settings_state, dict) else None, + } + if before_fields["locked"] is not False: + checks["ok"] = False + checks["blocked"] = True + checks["errors"].append("device_locked_or_unknown") + return checks + if not isinstance(relaunch, dict) or relaunch.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"relaunch_status:{relaunch.get('status') if isinstance(relaunch, dict) else None}") + for label, open_result, screen_result, fields, bundle_id, state in ( + ("safari", open_safari, safari_screen, safari_fields, "com.apple.mobilesafari", safari_state), + ("settings", open_settings, settings_screen, settings_fields, "com.apple.Preferences", settings_state), + ): + if not isinstance(open_result, dict) or action_state(open_result) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"{label}_open_state:{action_state(open_result)}") + if not isinstance(screen_result, dict) or screen_result.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"{label}_screen_status:{screen_result.get('status') if isinstance(screen_result, dict) else None}") + continue + if fields["locked"] is not False: + checks["ok"] = False + checks["errors"].append(f"{label}_locked:{fields['locked']}") + if fields["foreground_app"] != bundle_id: + checks["ok"] = False + checks["errors"].append(f"{label}_foreground_app:{fields['foreground_app']}") + if fields["ui_tree_source"] != "app_process": + checks["ok"] = False + checks["errors"].append(f"{label}_ui_tree_source:{fields['ui_tree_source']}") + if fields["app_ui_status"] != "ok": + checks["ok"] = False + checks["errors"].append(f"{label}_app_ui_status:{fields['app_ui_status']}") + if fields["app_ui_bundle_id"] != bundle_id: + checks["ok"] = False + checks["errors"].append(f"{label}_app_ui_bundle:{fields['app_ui_bundle_id']}") + if fields["ui_tree_status"] != "ok": + checks["ok"] = False + checks["errors"].append(f"{label}_ui_tree_status:{fields['ui_tree_status']}") + if not isinstance(fields["element_count"], int) or fields["element_count"] <= 0: + checks["ok"] = False + checks["errors"].append(f"{label}_element_count:{fields['element_count']}") + if not isinstance(fields["text_count"], int) or fields["text_count"] <= 0: + checks["ok"] = False + checks["errors"].append(f"{label}_text_count:{fields['text_count']}") + bad_flags = {"app_ui_state_missing", "app_ui_state_stale", "app_ui_unavailable"} + present_bad_flags = sorted(flag for flag in fields["risk_flags"] if flag in bad_flags) + if present_bad_flags: + checks["ok"] = False + checks["errors"].append(f"{label}_risk_flags:{','.join(present_bad_flags)}") + if not isinstance(state, dict) or state.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"{label}_state_status:{state.get('status') if isinstance(state, dict) else None}") + elif state.get("bundle_id") != bundle_id: + checks["ok"] = False + checks["errors"].append(f"{label}_state_bundle:{state.get('bundle_id')}") + if not isinstance(intake, dict) or intake.get("status") != "ready": + checks["ok"] = False + checks["errors"].append(f"intake_status:{intake.get('status') if isinstance(intake, dict) else None}") + publish_count = intake.get("publish_count") if isinstance(intake, dict) else None + if not isinstance(publish_count, int) or publish_count < 2: + checks["ok"] = False + checks["errors"].append(f"publish_count:{publish_count}") + return checks + +def check_lockscreen_shape(): + before = read_json("lockscreen-before-screen.json") + show_passcode = read_json("lockscreen-show-passcode.json") + after = read_json("lockscreen-after-screen.json") + status_after = read_json("lockscreen-status-after.json") + before_fields = app_ui_fields(before) + after_fields = app_ui_fields(after) + before_screenshot = screenshot_fields(before) + after_screenshot = screenshot_fields(after) + provider_result = show_passcode.get("provider_result") if isinstance(show_passcode.get("provider_result"), dict) else {} + fallback = provider_result.get("lockscreen_fallback") if isinstance(provider_result.get("lockscreen_fallback"), dict) else {} + attempts = fallback.get("attempts") if isinstance(fallback.get("attempts"), list) else [] + passcode_visible = any( + isinstance(attempt, dict) and attempt.get("passcode_visible") is True + for attempt in attempts + ) + checks = { + "ok": True, + "blocked": False, + "blocker": "", + "errors": [], + "before_locked": before_fields["locked"], + "after_locked": after_fields["locked"], + "show_passcode_state": action_state(show_passcode), + "show_passcode_user_facing_status": show_passcode.get("user_facing_status") if isinstance(show_passcode, dict) else None, + "provider": provider_result.get("provider") if isinstance(provider_result, dict) else None, + "activation_method": provider_result.get("activation_method") if isinstance(provider_result, dict) else None, + "fallback_status": fallback.get("status") if isinstance(fallback, dict) else None, + "passcode_visible": passcode_visible, + "attempt_count": len(attempts), + "before_screenshot": before_screenshot, + "after_screenshot": after_screenshot, + "screenshot_hash_changed": ( + bool(before_screenshot.get("sha256")) + and bool(after_screenshot.get("sha256")) + and before_screenshot.get("sha256") != after_screenshot.get("sha256") + ), + "after_status": status_after.get("status") if isinstance(status_after, dict) else None, + "after_agent_state": status_after.get("state") if isinstance(status_after, dict) else None, + } + if before_fields["locked"] is not True: + checks["ok"] = False + checks["blocked"] = True + checks["blocker"] = "device_unlocked_or_unknown" + checks["errors"].append("device_unlocked_or_unknown") + return checks + if not isinstance(show_passcode, dict) or action_state(show_passcode) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"show_passcode_state:{action_state(show_passcode)}") + if provider_result.get("provider") != "OpenPhoneVolumeTrigger.SpringBoardInput": + checks["ok"] = False + checks["errors"].append(f"provider:{provider_result.get('provider')}") + if fallback.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"fallback_status:{fallback.get('status')}") + if not passcode_visible: + checks["ok"] = False + checks["errors"].append("passcode_not_visible") + activation_method = provider_result.get("activation_method") + if not isinstance(activation_method, str) or "PasscodeLockVisible" not in activation_method: + checks["ok"] = False + checks["errors"].append(f"activation_method:{activation_method}") + if not isinstance(after, dict) or after.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"after_screen_status:{after.get('status') if isinstance(after, dict) else None}") + if after_fields["locked"] is not True: + checks["ok"] = False + checks["errors"].append(f"after_locked:{after_fields['locked']}") + if after_screenshot.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"after_screenshot_status:{after_screenshot.get('status')}") + if after_screenshot.get("provider") != "OpenPhoneVolumeTrigger.SpringBoardScreenshot": + checks["ok"] = False + checks["errors"].append(f"after_screenshot_provider:{after_screenshot.get('provider')}") + if not after_screenshot.get("sha256"): + checks["ok"] = False + checks["errors"].append("after_screenshot_missing_sha256") + width = after_screenshot.get("width") + height = after_screenshot.get("height") + if not isinstance(width, int) or width <= 0 or not isinstance(height, int) or height <= 0: + checks["ok"] = False + checks["errors"].append(f"after_screenshot_dimensions:{width}x{height}") + if status_after.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"after_agent_status:{status_after.get('status')}") + return checks + +def check_prefs_backend_shape(): + files = read_json("prefs-backend-files.json") + before = read_json("prefs-backend-status-before.json") + disable_hardware = read_json("prefs-backend-disable-hardware.json") + trigger_disabled = read_json("prefs-backend-trigger-disabled.json") + enable_hardware = read_json("prefs-backend-enable-hardware.json") + disable_yolo = read_json("prefs-backend-disable-yolo.json") + trigger_yolo_disabled = read_json("prefs-backend-trigger-yolo-disabled.json") + enable_yolo = read_json("prefs-backend-enable-yolo.json") + after = read_json("prefs-backend-status-after.json") + + def control(data): + return data.get("control") if isinstance(data.get("control"), dict) else {} + + before_control = control(before) + after_control = control(after) + checks = { + "ok": True, + "errors": [], + "files": { + "bundle_executable": files.get("bundle_executable") if isinstance(files, dict) else None, + "bundle_info": files.get("bundle_info") if isinstance(files, dict) else None, + "loader_entry": files.get("loader_entry") if isinstance(files, dict) else None, + }, + "before": { + "status": before.get("status") if isinstance(before, dict) else None, + "trigger_policy": before_control.get("trigger_policy"), + "hardware_triggers_enabled": before_control.get("hardware_triggers_enabled"), + "yolo_enabled": before_control.get("yolo_enabled"), + }, + "disable_hardware": { + "status": disable_hardware.get("status") if isinstance(disable_hardware, dict) else None, + "hardware_triggers_enabled": control(disable_hardware).get("hardware_triggers_enabled"), + }, + "trigger_disabled_state": trigger_disabled.get("state") if isinstance(trigger_disabled, dict) else None, + "enable_hardware": { + "status": enable_hardware.get("status") if isinstance(enable_hardware, dict) else None, + "hardware_triggers_enabled": control(enable_hardware).get("hardware_triggers_enabled"), + }, + "disable_yolo": { + "status": disable_yolo.get("status") if isinstance(disable_yolo, dict) else None, + "yolo_enabled": control(disable_yolo).get("yolo_enabled"), + }, + "trigger_yolo_disabled_state": trigger_yolo_disabled.get("state") if isinstance(trigger_yolo_disabled, dict) else None, + "enable_yolo": { + "status": enable_yolo.get("status") if isinstance(enable_yolo, dict) else None, + "yolo_enabled": control(enable_yolo).get("yolo_enabled"), + }, + "after": { + "status": after.get("status") if isinstance(after, dict) else None, + "paused": after_control.get("paused"), + "trigger_policy": after_control.get("trigger_policy"), + "hardware_triggers_enabled": after_control.get("hardware_triggers_enabled"), + "yolo_enabled": after_control.get("yolo_enabled"), + }, + } + for key in ("bundle_executable", "bundle_info", "loader_entry"): + if files.get(key) is not True: + checks["ok"] = False + checks["errors"].append(f"missing_{key}") + if before.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"before_status:{before.get('status')}") + if disable_hardware.get("status") != "ok" or control(disable_hardware).get("hardware_triggers_enabled") is not False: + checks["ok"] = False + checks["errors"].append("disable_hardware_failed") + if trigger_disabled.get("status") != "ok" or trigger_disabled.get("state") != "trigger.disabled": + checks["ok"] = False + checks["errors"].append(f"trigger_disabled_state:{trigger_disabled.get('state')}") + if "agent_task_id" in trigger_disabled or "background_job" in trigger_disabled: + checks["ok"] = False + checks["errors"].append("trigger_disabled_created_work") + if enable_hardware.get("status") != "ok" or control(enable_hardware).get("hardware_triggers_enabled") is not True: + checks["ok"] = False + checks["errors"].append("enable_hardware_failed") + if disable_yolo.get("status") != "ok" or control(disable_yolo).get("yolo_enabled") is not False: + checks["ok"] = False + checks["errors"].append("disable_yolo_failed") + if trigger_yolo_disabled.get("status") != "ok" or trigger_yolo_disabled.get("state") != "trigger.yolo_disabled": + checks["ok"] = False + checks["errors"].append(f"trigger_yolo_disabled_state:{trigger_yolo_disabled.get('state')}") + if "agent_task_id" in trigger_yolo_disabled or "background_job" in trigger_yolo_disabled: + checks["ok"] = False + checks["errors"].append("trigger_yolo_disabled_created_work") + if enable_yolo.get("status") != "ok" or control(enable_yolo).get("yolo_enabled") is not True: + checks["ok"] = False + checks["errors"].append("enable_yolo_failed") + if after.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"after_status:{after.get('status')}") + if after_control.get("paused") is not False: + checks["ok"] = False + checks["errors"].append(f"after_paused:{after_control.get('paused')}") + if after_control.get("trigger_policy") != "allow_yolo": + checks["ok"] = False + checks["errors"].append(f"after_trigger_policy:{after_control.get('trigger_policy')}") + if after_control.get("hardware_triggers_enabled") is not True: + checks["ok"] = False + checks["errors"].append(f"after_hardware:{after_control.get('hardware_triggers_enabled')}") + if after_control.get("yolo_enabled") is not True: + checks["ok"] = False + checks["errors"].append(f"after_yolo:{after_control.get('yolo_enabled')}") + return checks + +def check_prefs_ui_shape(): + before = read_json("prefs-ui-before-screen.json") + prepare = read_json("prefs-ui-prepare.json") + open_url = read_json("prefs-ui-open-url.json") + url_screen = read_json("prefs-ui-url-screen.json") + open_settings = read_json("prefs-ui-open-settings.json") + settings_screen = read_json("prefs-ui-settings-screen.json") + tap_row = read_json("prefs-ui-tap-row.json") + pane_screen = read_json("prefs-ui-pane-screen.json") + disable_hardware = read_json("prefs-ui-disable-hardware.json") + after_disable_screen = read_json("prefs-ui-after-disable-screen.json") + status_disabled = read_json("prefs-ui-status-disabled.json") + enable_hardware = read_json("prefs-ui-enable-hardware.json") + after_enable_screen = read_json("prefs-ui-after-enable-screen.json") + status_enabled = read_json("prefs-ui-status-enabled.json") + final_restore = read_json("prefs-ui-final-restore.json") + status_after = read_json("prefs-ui-status-after.json") + row_element = read_text("prefs-ui-row-element.txt").strip() + hardware_element = read_text("prefs-ui-hardware-element.txt").strip() + + def control(data): + return data.get("control") if isinstance(data.get("control"), dict) else {} + + def provider(data): + return data.get("provider_result") if isinstance(data.get("provider_result"), dict) else {} + + def pane_fields(data): + fields = app_ui_fields(data) + visible = ui_visible_text(data) + return { + "foreground_app": fields["foreground_app"], + "ui_tree_source": fields["ui_tree_source"], + "app_ui_status": fields["app_ui_status"], + "element_count": fields["element_count"], + "text_count": fields["text_count"], + "visible_required": { + "OpenPhone Agent": "OpenPhone Agent" in visible, + "Hardware Triggers": "Hardware Triggers" in visible, + "YOLO Execution": "YOLO Execution" in visible, + }, + "screenshot": screenshot_fields(data), + } + + before_fields = app_ui_fields(before) + pane = pane_fields(pane_screen) + disabled_control = control(status_disabled) + enabled_control = control(status_enabled) + after_control = control(status_after) + disable_provider = provider(disable_hardware) + enable_provider = provider(enable_hardware) + checks = { + "ok": True, + "blocked": False, + "errors": [], + "before_locked": before_fields["locked"], + "prepare_status": prepare.get("status") if isinstance(prepare, dict) else None, + "open_url_state": action_state(open_url), + "url_screen_pane": pane_fields(url_screen), + "open_settings_state": action_state(open_settings), + "settings_screen": pane_fields(settings_screen), + "row_element": row_element, + "tap_row_state": action_state(tap_row), + "pane": pane, + "hardware_element": hardware_element, + "disable_hardware_state": action_state(disable_hardware), + "disable_hardware_provider": disable_provider.get("provider") if isinstance(disable_provider, dict) else None, + "disable_hardware_activation": disable_provider.get("activation_method") if isinstance(disable_provider, dict) else None, + "after_disable_screen": pane_fields(after_disable_screen), + "status_disabled": { + "status": status_disabled.get("status") if isinstance(status_disabled, dict) else None, + "hardware_triggers_enabled": disabled_control.get("hardware_triggers_enabled"), + "trigger_policy": disabled_control.get("trigger_policy"), + }, + "enable_hardware_state": action_state(enable_hardware), + "enable_hardware_provider": enable_provider.get("provider") if isinstance(enable_provider, dict) else None, + "enable_hardware_activation": enable_provider.get("activation_method") if isinstance(enable_provider, dict) else None, + "after_enable_screen": pane_fields(after_enable_screen), + "status_enabled": { + "status": status_enabled.get("status") if isinstance(status_enabled, dict) else None, + "hardware_triggers_enabled": enabled_control.get("hardware_triggers_enabled"), + "trigger_policy": enabled_control.get("trigger_policy"), + }, + "final_restore_status": final_restore.get("status") if isinstance(final_restore, dict) else None, + "after": { + "status": status_after.get("status") if isinstance(status_after, dict) else None, + "paused": after_control.get("paused"), + "trigger_policy": after_control.get("trigger_policy"), + "hardware_triggers_enabled": after_control.get("hardware_triggers_enabled"), + "yolo_enabled": after_control.get("yolo_enabled"), + }, + } + if before_fields["locked"] is not False: + checks["ok"] = False + checks["blocked"] = True + checks["errors"].append("device_locked_or_unknown") + return checks + if prepare.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"prepare_status:{prepare.get('status')}") + if not all(pane["visible_required"].values()): + checks["ok"] = False + checks["errors"].append("pane_missing_required_text") + if pane["foreground_app"] != "com.apple.Preferences": + checks["ok"] = False + checks["errors"].append(f"pane_foreground:{pane['foreground_app']}") + if pane["ui_tree_source"] != "app_process": + checks["ok"] = False + checks["errors"].append(f"pane_ui_tree_source:{pane['ui_tree_source']}") + if pane["app_ui_status"] != "ok": + checks["ok"] = False + checks["errors"].append(f"pane_app_ui_status:{pane['app_ui_status']}") + if not isinstance(pane["element_count"], int) or pane["element_count"] <= 0: + checks["ok"] = False + checks["errors"].append(f"pane_element_count:{pane['element_count']}") + if tap_row.get("status") != "skipped" and action_state(tap_row) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"tap_row_state:{action_state(tap_row)}") + if tap_row.get("status") != "skipped" and not row_element: + checks["ok"] = False + checks["errors"].append("missing_openphone_agent_row") + if not hardware_element: + checks["ok"] = False + checks["errors"].append("missing_hardware_triggers_element") + if action_state(disable_hardware) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"disable_hardware_state:{action_state(disable_hardware)}") + if isinstance(disable_provider, dict) and disable_provider.get("provider") != "OpenPhoneAppIntrospector.AppInput": + checks["ok"] = False + checks["errors"].append(f"disable_hardware_provider:{disable_provider.get('provider')}") + if disabled_control.get("hardware_triggers_enabled") is not False: + checks["ok"] = False + checks["errors"].append(f"status_disabled_hardware:{disabled_control.get('hardware_triggers_enabled')}") + if action_state(enable_hardware) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"enable_hardware_state:{action_state(enable_hardware)}") + if isinstance(enable_provider, dict) and enable_provider.get("provider") != "OpenPhoneAppIntrospector.AppInput": + checks["ok"] = False + checks["errors"].append(f"enable_hardware_provider:{enable_provider.get('provider')}") + if enabled_control.get("hardware_triggers_enabled") is not True: + checks["ok"] = False + checks["errors"].append(f"status_enabled_hardware:{enabled_control.get('hardware_triggers_enabled')}") + if final_restore.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"final_restore_status:{final_restore.get('status')}") + if status_after.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"after_status:{status_after.get('status')}") + if after_control.get("paused") is not False: + checks["ok"] = False + checks["errors"].append(f"after_paused:{after_control.get('paused')}") + if after_control.get("trigger_policy") != "allow_yolo": + checks["ok"] = False + checks["errors"].append(f"after_trigger_policy:{after_control.get('trigger_policy')}") + if after_control.get("hardware_triggers_enabled") is not True: + checks["ok"] = False + checks["errors"].append(f"after_hardware:{after_control.get('hardware_triggers_enabled')}") + if after_control.get("yolo_enabled") is not True: + checks["ok"] = False + checks["errors"].append(f"after_yolo:{after_control.get('yolo_enabled')}") + return checks + +def check_visible_effects_shape(): + before = read_json("visible-effects-before-screen.json") + open_settings = read_json("visible-effects-open-settings.json") + settings_precheck = read_json("visible-effects-settings-precheck.json") + settings_reset = read_json("visible-effects-settings-reset.json") + settings_before = read_json("visible-effects-settings-before.json") + tap_settings = read_json("visible-effects-tap-settings.json") + settings_after = read_json("visible-effects-settings-after.json") + open_safari = read_json("visible-effects-open-safari.json") + safari_before = read_json("visible-effects-safari-before.json") + type_safari = read_json("visible-effects-type-safari.json") + safari_after = read_json("visible-effects-safari-after.json") + safari_state = read_json("visible-effects-safari-state.json") + open_notes = read_json("visible-effects-open-notes.json") + notes_before = read_json("visible-effects-notes-before.json") + type_notes = read_json("visible-effects-type-notes.json") + notes_after = read_json("visible-effects-notes-after.json") + notes_state = read_json("visible-effects-notes-state.json") + marker = read_text("visible-effects-safari-marker.txt").strip() + notes_marker = read_text("visible-effects-notes-marker.txt").strip() + settings_scenario = read_text("visible-effects-settings-scenario.txt").strip() + settings_target_label = read_text("visible-effects-settings-target-label.txt").strip() + settings_back_element = read_text("visible-effects-settings-back-element.txt").strip() + settings_element = read_text("visible-effects-settings-element.txt").strip() + safari_field = read_text("visible-effects-safari-field.txt").strip() + notes_field = read_text("visible-effects-notes-field.txt").strip() + before_fields = app_ui_fields(before) + settings_before_fields = app_ui_fields(settings_before) + settings_after_fields = app_ui_fields(settings_after) + safari_before_fields = app_ui_fields(safari_before) + safari_after_fields = app_ui_fields(safari_after) + notes_before_fields = app_ui_fields(notes_before) + notes_after_fields = app_ui_fields(notes_after) + settings_before_text = ui_visible_text(settings_before) + settings_after_text = ui_visible_text(settings_after) + safari_after_text = ui_visible_text(safari_after) + notes_after_text = ui_visible_text(notes_after) + settings_screenshot = screenshot_hash_changed(settings_before, settings_after) + safari_screenshot = screenshot_hash_changed(safari_before, safari_after) + notes_screenshot = screenshot_hash_changed(notes_before, notes_after) + settings_before_has_general_page = "About" in settings_before_text and "Software Update" in settings_before_text + if not settings_scenario: + settings_scenario = "general_to_keyboard" if settings_before_has_general_page else "root_to_general" + if not settings_target_label: + settings_target_label = "Keyboard" if settings_scenario == "general_to_keyboard" else "General" + settings_before_expected = False + settings_after_expected_page = False + settings_expected_after_name = "" + if settings_scenario == "root_to_general": + settings_expected_after_name = "general_page" + settings_before_expected = "General" in settings_before_text and not settings_before_has_general_page + settings_after_expected_page = ( + "About" in settings_after_text + and "Software Update" in settings_after_text + and "General" in settings_after_text + ) + elif settings_scenario == "general_to_keyboard": + settings_expected_after_name = "keyboard_page" + settings_before_expected = settings_before_has_general_page and "Keyboard" in settings_before_text + settings_after_expected_page = ( + "Keyboards" in settings_after_text + and ( + "Text Replacement" in settings_after_text + or "One-Handed Keyboard" in settings_after_text + or "Enable Dictation" in settings_after_text + ) + ) + elif settings_scenario == "keyboard_to_keyboards": + settings_expected_after_name = "keyboards_page" + settings_before_expected = ( + "Keyboards" in settings_before_text + and "Text Replacement" in settings_before_text + and ( + "Enable Dictation" in settings_before_text + or "One-Handed Keyboard" in settings_before_text + ) + ) + settings_after_expected_page = ( + "Add New Keyboard..." in settings_after_text + or "Add New Keyboard…" in settings_after_text + or "English (US)" in settings_after_text + or "Emoji" in settings_after_text + ) + elif settings_scenario == "keyboards_to_english": + settings_expected_after_name = "english_keyboard_page" + settings_before_expected = ( + "Keyboards" in settings_before_text + and "English (US)" in settings_before_text + and "Emoji" in settings_before_text + and ("Add New Keyboard…" in settings_before_text or "Add New Keyboard..." in settings_before_text) + ) + settings_after_expected_page = ( + "English (US)" in settings_after_text + and ( + "QWERTY" in settings_after_text + or "AZERTY" in settings_after_text + or "QWERTZ" in settings_after_text + or "Software Keyboard Layout" in settings_after_text + or "Hardware Keyboard Layout" in settings_after_text + ) + ) + tap_provider = tap_settings.get("provider_result", {}) if isinstance(tap_settings, dict) else {} + type_provider = type_safari.get("provider_result", {}) if isinstance(type_safari, dict) else {} + type_verification = type_safari.get("verification", {}) if isinstance(type_safari, dict) else {} + type_user_facing_status = type_safari.get("user_facing_status") if isinstance(type_safari, dict) else None + provider_attempts = type_safari.get("provider_attempts") if isinstance(type_safari, dict) else [] + notes_provider = type_notes.get("provider_result", {}) if isinstance(type_notes, dict) else {} + notes_verification = type_notes.get("verification", {}) if isinstance(type_notes, dict) else {} + notes_user_facing_status = type_notes.get("user_facing_status") if isinstance(type_notes, dict) else None + notes_provider_attempts = type_notes.get("provider_attempts") if isinstance(type_notes, dict) else [] + webcontent_verified = False + if isinstance(provider_attempts, list): + for attempt in provider_attempts: + if not isinstance(attempt, dict): + continue + verification = attempt.get("verification", {}) + if ( + attempt.get("provider") == "OpenPhoneAppIntrospector.WebContentInput" + and attempt.get("status") == "ok" + and isinstance(verification, dict) + and verification.get("status") == "verified" + and verification.get("source") == "web_content_dom_state" + ): + webcontent_verified = True + break + notes_text_verified = False + if isinstance(notes_provider_attempts, list): + for attempt in notes_provider_attempts: + if not isinstance(attempt, dict): + continue + verification = attempt.get("verification", {}) + if ( + attempt.get("provider") == "OpenPhoneAppIntrospector.AppInput" + and attempt.get("status") == "ok" + and attempt.get("activation_method") == "text_input_insert" + and isinstance(verification, dict) + and verification.get("status") == "verified" + and verification.get("source") == "app_process_text_state" + ): + notes_text_verified = True + break + checks = { + "ok": True, + "blocked": False, + "errors": [], + "before_locked": before_fields["locked"], + "settings_scenario": settings_scenario, + "settings_target_label": settings_target_label, + "settings_element": settings_element, + "settings_back_element": settings_back_element, + "settings_open_state": action_state(open_settings), + "settings_precheck": app_ui_fields(settings_precheck), + "settings_reset_state": action_state(settings_reset), + "settings_before": settings_before_fields, + "settings_tap_state": action_state(tap_settings), + "settings_tap_provider": tap_provider.get("provider") if isinstance(tap_provider, dict) else None, + "settings_activation_method": tap_provider.get("activation_method") if isinstance(tap_provider, dict) else None, + "settings_after": settings_after_fields, + "settings_before_has_general_page": settings_before_has_general_page, + "settings_before_expected": settings_before_expected, + "settings_expected_after_name": settings_expected_after_name, + "settings_after_expected_page": settings_after_expected_page, + "settings_after_has_general_page": "About" in settings_after_text and "Software Update" in settings_after_text and "General" in settings_after_text, + "settings_screenshot_hash": settings_screenshot, + "safari_field": safari_field, + "safari_marker": marker, + "safari_open_state": action_state(open_safari), + "safari_before": safari_before_fields, + "safari_type_state": action_state(type_safari), + "safari_type_user_facing_status": type_user_facing_status, + "safari_type_provider": type_provider.get("provider") if isinstance(type_provider, dict) else None, + "safari_type_verification": type_verification, + "safari_webcontent_verified": webcontent_verified, + "safari_after": safari_after_fields, + "safari_marker_visible_after": marker in safari_after_text or json_contains_string(safari_after, marker) or json_contains_string(safari_state, marker), + "safari_screenshot_hash": safari_screenshot, + "safari_state_status": safari_state.get("status") if isinstance(safari_state, dict) else None, + "notes_field": notes_field, + "notes_marker": notes_marker, + "notes_open_state": action_state(open_notes), + "notes_before": notes_before_fields, + "notes_type_state": action_state(type_notes), + "notes_type_user_facing_status": notes_user_facing_status, + "notes_type_provider": notes_provider.get("provider") if isinstance(notes_provider, dict) else None, + "notes_activation_method": notes_provider.get("activation_method") if isinstance(notes_provider, dict) else None, + "notes_type_verification": notes_verification, + "notes_text_verified": notes_text_verified, + "notes_after": notes_after_fields, + "notes_marker_visible_after": notes_marker in notes_after_text or json_contains_string(notes_after, notes_marker) or json_contains_string(notes_state, notes_marker), + "notes_screenshot_hash": notes_screenshot, + "notes_state_status": notes_state.get("status") if isinstance(notes_state, dict) else None, + } + if before_fields["locked"] is not False: + checks["ok"] = False + checks["blocked"] = True + checks["errors"].append("device_locked_or_unknown") + return checks + if action_state(open_settings) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"settings_open_state:{action_state(open_settings)}") + if settings_back_element and action_state(settings_reset) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"settings_reset_state:{action_state(settings_reset)}") + if not settings_element: + checks["ok"] = False + checks["errors"].append(f"missing_settings_target_element:{settings_target_label}") + if settings_before_fields["foreground_app"] != "com.apple.Preferences": + checks["ok"] = False + checks["errors"].append(f"settings_before_foreground:{settings_before_fields['foreground_app']}") + if settings_before_fields["ui_tree_source"] != "app_process": + checks["ok"] = False + checks["errors"].append(f"settings_before_ui_tree_source:{settings_before_fields['ui_tree_source']}") + if settings_scenario not in ("root_to_general", "general_to_keyboard", "keyboard_to_keyboards", "keyboards_to_english"): + checks["ok"] = False + checks["errors"].append(f"settings_unknown_scenario:{settings_scenario}") + elif not settings_before_expected: + checks["ok"] = False + checks["errors"].append(f"settings_before_unexpected:{settings_scenario}") + if action_state(tap_settings) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"settings_tap_state:{action_state(tap_settings)}") + if isinstance(tap_provider, dict) and tap_provider.get("provider") != "OpenPhoneAppIntrospector.AppInput": + checks["ok"] = False + checks["errors"].append(f"settings_tap_provider:{tap_provider.get('provider')}") + if not settings_after_expected_page: + checks["ok"] = False + checks["errors"].append(f"settings_after_missing_expected_page:{settings_expected_after_name or settings_scenario}") + if not settings_screenshot["changed"]: + checks["ok"] = False + checks["errors"].append( + "settings_screenshot_hash_not_changed:" + f"{settings_screenshot['before'].get('status')}:{settings_screenshot['after'].get('status')}" + ) + if settings_after_fields["foreground_app"] != "com.apple.Preferences": + checks["ok"] = False + checks["errors"].append(f"settings_after_foreground:{settings_after_fields['foreground_app']}") + if settings_after_fields["ui_tree_source"] != "app_process": + checks["ok"] = False + checks["errors"].append(f"settings_after_ui_tree_source:{settings_after_fields['ui_tree_source']}") + + if action_state(open_safari) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"safari_open_state:{action_state(open_safari)}") + if not safari_field: + checks["ok"] = False + checks["errors"].append("missing_safari_dom_field") + if safari_before_fields["foreground_app"] != "com.apple.mobilesafari": + checks["ok"] = False + checks["errors"].append(f"safari_before_foreground:{safari_before_fields['foreground_app']}") + if safari_before_fields["ui_tree_source"] != "app_process": + checks["ok"] = False + checks["errors"].append(f"safari_before_ui_tree_source:{safari_before_fields['ui_tree_source']}") + if action_state(type_safari) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"safari_type_state:{action_state(type_safari)}") + if type_user_facing_status != "verified": + checks["ok"] = False + checks["errors"].append(f"safari_type_user_facing_status:{type_user_facing_status}") + if not ( + isinstance(type_verification, dict) + and type_verification.get("status") == "verified" + and type_verification.get("source") == "web_content_dom_state" + ): + checks["ok"] = False + checks["errors"].append(f"safari_type_verification:{type_verification.get('status') if isinstance(type_verification, dict) else None}:{type_verification.get('source') if isinstance(type_verification, dict) else None}") + if not webcontent_verified: + checks["ok"] = False + checks["errors"].append("safari_webcontent_attempt_not_verified") + if not checks["safari_marker_visible_after"]: + checks["ok"] = False + checks["errors"].append("safari_marker_not_visible_after") + if not safari_screenshot["changed"]: + checks["ok"] = False + checks["errors"].append( + "safari_screenshot_hash_not_changed:" + f"{safari_screenshot['before'].get('status')}:{safari_screenshot['after'].get('status')}" + ) + if safari_after_fields["foreground_app"] != "com.apple.mobilesafari": + checks["ok"] = False + checks["errors"].append(f"safari_after_foreground:{safari_after_fields['foreground_app']}") + if safari_after_fields["ui_tree_source"] != "app_process": + checks["ok"] = False + checks["errors"].append(f"safari_after_ui_tree_source:{safari_after_fields['ui_tree_source']}") + if isinstance(safari_state, dict) and safari_state.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"safari_state_status:{safari_state.get('status')}") + if action_state(open_notes) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"notes_open_state:{action_state(open_notes)}") + if not notes_field: + checks["ok"] = False + checks["errors"].append("missing_notes_text_field") + if notes_before_fields["foreground_app"] != "com.apple.mobilenotes": + checks["ok"] = False + checks["errors"].append(f"notes_before_foreground:{notes_before_fields['foreground_app']}") + if notes_before_fields["ui_tree_source"] != "app_process": + checks["ok"] = False + checks["errors"].append(f"notes_before_ui_tree_source:{notes_before_fields['ui_tree_source']}") + if action_state(type_notes) != "action.executed": + checks["ok"] = False + checks["errors"].append(f"notes_type_state:{action_state(type_notes)}") + if notes_user_facing_status != "verified": + checks["ok"] = False + checks["errors"].append(f"notes_type_user_facing_status:{notes_user_facing_status}") + if not ( + isinstance(notes_verification, dict) + and notes_verification.get("status") == "verified" + and notes_verification.get("source") == "app_process_text_state" + ): + checks["ok"] = False + checks["errors"].append(f"notes_type_verification:{notes_verification.get('status') if isinstance(notes_verification, dict) else None}:{notes_verification.get('source') if isinstance(notes_verification, dict) else None}") + if not notes_text_verified: + checks["ok"] = False + checks["errors"].append("notes_app_input_attempt_not_verified") + if not checks["notes_marker_visible_after"]: + checks["ok"] = False + checks["errors"].append("notes_marker_not_visible_after") + if not notes_screenshot["changed"]: + checks["ok"] = False + checks["errors"].append( + "notes_screenshot_hash_not_changed:" + f"{notes_screenshot['before'].get('status')}:{notes_screenshot['after'].get('status')}" + ) + if notes_after_fields["foreground_app"] != "com.apple.mobilenotes": + checks["ok"] = False + checks["errors"].append(f"notes_after_foreground:{notes_after_fields['foreground_app']}") + if notes_after_fields["ui_tree_source"] != "app_process": + checks["ok"] = False + checks["errors"].append(f"notes_after_ui_tree_source:{notes_after_fields['ui_tree_source']}") + if isinstance(notes_state, dict) and notes_state.get("status") != "ok": + checks["ok"] = False + checks["errors"].append(f"notes_state_status:{notes_state.get('status')}") + return checks + +health = read_json("health.json") +screen = read_json("get-screen.json") +springboard_state = read_json("springboard-state.json") +springboard_trigger_status = read_json("springboard-trigger-status.json") +markers = read_text("safe-mode-markers.txt").strip() +latest = latest_crash("springboard-crashes.txt") +previous = latest_crash("springboard-crashes.before.txt") +new_crash = bool(previous and latest and latest > previous) +process_lines = [line for line in read_text("processes.txt").splitlines() if "openphone-agentd" in line] + +providers = health.get("providers", {}) if isinstance(health, dict) else {} +screen_provider = providers.get("screen", {}) if isinstance(providers, dict) else {} +input_provider = providers.get("input", {}) if isinstance(providers, dict) else {} +bridge = input_provider.get("springboard_bridge", {}) if isinstance(input_provider, dict) else {} +trigger_provider = providers.get("triggers", {}) if isinstance(providers, dict) else {} +volume_combo_provider = trigger_provider.get("volume_combo", {}) if isinstance(trigger_provider, dict) else {} +daemon_springboard_trigger = ( + volume_combo_provider.get("springboard_fallback", {}) + if isinstance(volume_combo_provider, dict) + else {} +) +springboard_trigger_snapshot = trigger_snapshot( + springboard_trigger_status if springboard_trigger_status else daemon_springboard_trigger +) +device = health.get("device", {}) if isinstance(health, dict) else {} + +context = screen.get("context", {}) if isinstance(screen, dict) else {} +display = context.get("display", {}) if isinstance(context, dict) else {} +ui_tree = context.get("ui_tree", {}) if isinstance(context, dict) else {} +screen_springboard = context.get("springboard_state", {}) if isinstance(context, dict) else {} +lock = context.get("lock", {}) if isinstance(context, dict) else {} + +health_ok = health.get("status") == "ok" +stability_ok = not markers and not new_crash +screen_ok = ( + bool(context) + and display.get("status") in ("available", "ok") + and screen_springboard.get("status") == "ok" + and ui_tree.get("status") == "ok" +) + +if os.environ.get("OPENPHONE_VALIDATE_INCLUDE_SCREENSHOT") == "1": + screenshot = read_json("get-screen-screenshot.json") + screenshot_sanity = read_json("screenshot-sanity.json") + screenshot_ok = screenshot_sanity.get("status") == "ok" +else: + screenshot = {} + screenshot_sanity = {} + screenshot_ok = None + +unlocked_foreground_requested = ( + os.environ.get("OPENPHONE_VALIDATE_INCLUDE_UNLOCKED_FOREGROUND") == "1" + or os.environ.get("OPENPHONE_VALIDATE_REQUIRE_UNLOCKED") == "1" +) +if unlocked_foreground_requested: + unlocked_foreground_checks = check_unlocked_foreground_shape() + if unlocked_foreground_checks.get("blocked"): + unlocked_foreground = "blocked_locked" + else: + unlocked_foreground = "pass" if unlocked_foreground_checks["ok"] else "fail" +else: + unlocked_foreground_checks = {} + unlocked_foreground = "not_run" + +app_ui_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_APP_UI") == "1" +if app_ui_requested: + app_ui_checks = check_app_ui_shape() + if app_ui_checks.get("blocked"): + app_ui_gate = "blocked_locked" + else: + app_ui_gate = "pass" if app_ui_checks["ok"] else "fail" +else: + app_ui_checks = {} + app_ui_gate = "not_run" + +lockscreen_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_LOCKSCREEN") == "1" +if lockscreen_requested: + lockscreen_checks = check_lockscreen_shape() + if lockscreen_checks.get("blocked"): + blocker = lockscreen_checks.get("blocker") or "blocked" + lockscreen_gate = "blocked_unlocked" if blocker == "device_unlocked_or_unknown" else "blocked" + else: + lockscreen_gate = "pass" if lockscreen_checks["ok"] else "fail" +else: + lockscreen_checks = {} + lockscreen_gate = "not_run" + +prefs_ui_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_PREFS_UI") == "1" +if prefs_ui_requested: + prefs_ui_checks = check_prefs_ui_shape() + if prefs_ui_checks.get("blocked"): + prefs_ui_gate = "blocked_locked" + else: + prefs_ui_gate = "pass" if prefs_ui_checks["ok"] else "fail" +else: + prefs_ui_checks = {} + prefs_ui_gate = "not_run" + +prefs_backend_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_PREFS_BACKEND") == "1" +if prefs_backend_requested: + prefs_backend_checks = check_prefs_backend_shape() + prefs_backend_gate = "pass" if prefs_backend_checks["ok"] else "fail" +else: + prefs_backend_checks = {} + prefs_backend_gate = "not_run" + +stores_requested = mode == "full" or os.environ.get("OPENPHONE_VALIDATE_INCLUDE_STORES") == "1" +if stores_requested: + store_checks = { + "tasks": check_response_shape( + "list-tasks.json", + "tasks", + item_keys=("task_id", "state", "autonomy_mode"), + ), + "task_detail": check_task_detail_shape("get-task.json"), + "trajectory": check_trajectory_shape("get-trajectory.json"), + "audit": check_response_shape( + "get-audit.json", + "events", + item_keys=("schema", "event_type", "decision", "capability", "event_hash"), + required_keys=("audit_path",), + ), + "memory": check_response_shape( + "memory-search.json", + "memories", + item_keys=("memory_id", "text", "type", "subject"), + required_keys=("provider", "fts_available"), + ), + "context": check_response_shape( + "context-search.json", + "events", + item_keys=("event_id", "type", "body"), + required_keys=("provider", "fts_available"), + ), + "background_jobs": check_response_shape( + "background-job-list.json", + "jobs", + item_keys=("job_id", "status", "title"), + required_keys=("provider", "runner"), + ), + "commitments": check_response_shape( + "commitment-search.json", + "commitments", + item_keys=("commitment_id", "status", "title"), + required_keys=("provider", "fts_available"), + ), + "watchers": check_response_shape( + "watcher-list.json", + "watchers", + item_keys=("watcher_id", "status", "title"), + required_keys=("provider", "fts_available", "scheduler_status"), + ), + } + stores_gate = "pass" if all(check["ok"] for check in store_checks.values()) else "fail" +else: + store_checks = {} + stores_gate = "not_run" + +provider_attempts_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_PROVIDER_ATTEMPTS") == "1" +if provider_attempts_requested: + provider_attempt_checks = { + "action": check_provider_attempt_sample_shape("provider-attempt-action.json"), + "trajectory": check_trajectory_shape("provider-attempt-trajectory.json"), + } + provider_attempts_gate = "pass" if all(check["ok"] for check in provider_attempt_checks.values()) else "fail" +else: + provider_attempt_checks = {} + provider_attempts_gate = "not_run" + +visible_effects_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_VISIBLE_EFFECTS") == "1" +if visible_effects_requested: + visible_effects_checks = check_visible_effects_shape() + if visible_effects_checks.get("blocked"): + visible_effects_gate = "blocked_locked" + else: + visible_effects_gate = "pass" if visible_effects_checks["ok"] else "fail" +else: + visible_effects_checks = {} + visible_effects_gate = "not_run" + +memory_lifecycle_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_MEMORY_LIFECYCLE") == "1" +if memory_lifecycle_requested: + memory_lifecycle_checks = check_memory_lifecycle_shape() + memory_lifecycle_gate = "pass" if memory_lifecycle_checks["ok"] else "fail" +else: + memory_lifecycle_checks = {} + memory_lifecycle_gate = "not_run" + +watcher_timer_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_WATCHER_TIMER") == "1" +if watcher_timer_requested: + watcher_timer_checks = check_watcher_timer_shape() + watcher_timer_gate = "pass" if watcher_timer_checks["ok"] else "fail" +else: + watcher_timer_checks = {} + watcher_timer_gate = "not_run" + +watcher_repair_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_WATCHER_REPAIR") == "1" +if watcher_repair_requested: + watcher_repair_checks = check_watcher_repair_shape() + watcher_repair_gate = "pass" if watcher_repair_checks["ok"] else "fail" +else: + watcher_repair_checks = {} + watcher_repair_gate = "not_run" + +job_repair_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_JOB_REPAIR") == "1" +if job_repair_requested: + job_repair_checks = check_job_repair_shape() + job_repair_gate = "pass" if job_repair_checks["ok"] else "fail" +else: + job_repair_checks = {} + job_repair_gate = "not_run" + +restart_recovery_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_RESTART_RECOVERY") == "1" +if restart_recovery_requested: + restart_recovery_checks = check_restart_recovery_shape() + restart_recovery_gate = "pass" if restart_recovery_checks["ok"] else "fail" +else: + restart_recovery_checks = {} + restart_recovery_gate = "not_run" + +model_loop_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_MODEL_LOOP") == "1" +if model_loop_requested: + model_loop_checks = check_model_loop_shape() + model_loop_gate = "pass" if model_loop_checks["ok"] else "fail" +else: + model_loop_checks = {} + model_loop_gate = "not_run" + +provider_model_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_PROVIDER_MODEL") == "1" +if provider_model_requested: + provider_model_checks = check_provider_model_shape() + provider_model_gate = "pass" if provider_model_checks["ok"] else "fail" +else: + provider_model_checks = {} + provider_model_gate = "not_run" + +safari_dom_model_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_SAFARI_DOM_MODEL") == "1" +if safari_dom_model_requested: + safari_dom_model_checks = check_safari_dom_model_shape() + if safari_dom_model_checks.get("blocked"): + safari_dom_model_gate = "blocked_locked" + else: + safari_dom_model_gate = "pass" if safari_dom_model_checks["ok"] else "fail" +else: + safari_dom_model_checks = {} + safari_dom_model_gate = "not_run" + +prompt_bridge_model_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_PROMPT_BRIDGE_MODEL") == "1" +if prompt_bridge_model_requested: + prompt_bridge_model_checks = check_prompt_bridge_model_shape() + if prompt_bridge_model_checks.get("blocked"): + prompt_bridge_model_gate = "blocked_locked" + else: + prompt_bridge_model_gate = "pass" if prompt_bridge_model_checks["ok"] else "fail" +else: + prompt_bridge_model_checks = {} + prompt_bridge_model_gate = "not_run" + +trigger_diagnostics_requested = os.environ.get("OPENPHONE_VALIDATE_INCLUDE_TRIGGER_DIAGNOSTICS") == "1" +if trigger_diagnostics_requested: + trigger_diagnostics_checks = check_trigger_diagnostics_shape() + trigger_diagnostics_gate = "pass" if trigger_diagnostics_checks["ok"] else "fail" +else: + trigger_diagnostics_checks = {} + trigger_diagnostics_gate = "not_run" + +artifact_hygiene = check_artifact_hygiene() +artifact_hygiene_ok = artifact_hygiene.get("status") == "pass" + +package_version = "" +if package_path: + match = re.search(r"com\.openphone\.agentd_(.+?)_iphoneos-arm64\.deb$", pathlib.Path(package_path).name) + if match: + package_version = match.group(1) + +exit_code = 0 +if not health_ok: + exit_code = 30 +elif not stability_ok: + exit_code = 40 +elif not screen_ok: + exit_code = 50 +elif screenshot_ok is False: + exit_code = 50 +elif unlocked_foreground == "fail": + exit_code = 50 +elif app_ui_gate == "fail": + exit_code = 50 +elif lockscreen_gate == "fail": + exit_code = 60 +elif prefs_ui_gate == "fail": + exit_code = 50 +elif provider_attempts_gate == "fail": + exit_code = 60 +elif visible_effects_gate == "fail": + exit_code = 60 +elif prefs_backend_gate == "fail": + exit_code = 70 +elif stores_gate == "fail": + exit_code = 70 +elif memory_lifecycle_gate == "fail": + exit_code = 70 +elif watcher_timer_gate == "fail": + exit_code = 70 +elif watcher_repair_gate == "fail": + exit_code = 70 +elif job_repair_gate == "fail": + exit_code = 70 +elif restart_recovery_gate == "fail": + exit_code = 70 +elif model_loop_gate == "fail": + exit_code = 70 +elif provider_model_gate == "fail": + exit_code = 100 +elif safari_dom_model_gate == "fail": + exit_code = 100 +elif prompt_bridge_model_gate == "fail": + exit_code = 100 +elif trigger_diagnostics_gate == "fail": + exit_code = 60 +elif unlocked_foreground == "blocked_locked": + exit_code = 80 +elif app_ui_gate == "blocked_locked": + exit_code = 80 +elif lockscreen_gate in ("blocked_unlocked", "blocked"): + exit_code = 80 +elif prefs_ui_gate == "blocked_locked": + exit_code = 80 +elif visible_effects_gate == "blocked_locked": + exit_code = 80 +elif safari_dom_model_gate == "blocked_locked": + exit_code = 80 +elif prompt_bridge_model_gate == "blocked_locked": + exit_code = 80 +elif not artifact_hygiene_ok: + exit_code = 90 + +report = { + "schema": "openphone.ios_validation_report.v1", + "timestamp": datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat(), + "mode": mode, + "run_id": os.environ["OPENPHONE_VALIDATE_RUN_ID"], + "package": { + "path": package_path, + "version": package_version, + "installed": os.environ.get("OPENPHONE_VALIDATE_INSTALLED") == "true", + }, + "device": { + "product_type": "iPhone15,3" if "iPhone15,3" in device.get("uname", "") else "", + "marketing_model": "iPhone 14 Pro Max", + "ios_darwin": device.get("uname", ""), + }, + "stability": { + "safe_mode_marker_present": bool(markers), + "safe_mode_markers": markers.splitlines(), + "latest_springboard_crash": latest, + "previous_springboard_crash": previous, + "new_crash_during_run": new_crash, + }, + "daemon": { + "health_status": health.get("status"), + "autonomy_mode": health.get("autonomy_mode"), + "pid_count": len(process_lines), + "springboard_bridge": bridge.get("status"), + "springboard_state": screen_springboard.get("status") or screen_provider.get("springboard_state", {}).get("status"), + "springboard_trigger_status": springboard_trigger_snapshot["status"], + "springboard_trigger_event": springboard_trigger_snapshot["event"], + "springboard_trigger_volume_hooked": springboard_trigger_snapshot["volume_hooked"], + "springboard_trigger_volume_total": springboard_trigger_snapshot["volume_total"], + "springboard_trigger_button_events_seen": springboard_trigger_snapshot["button_events_seen"], + "springboard_trigger_combo_events_seen": springboard_trigger_snapshot["combo_events_seen"], + "springboard_trigger_last_button_event_source": springboard_trigger_snapshot["last_button_event_source"], + "springboard_trigger_last_trigger_route": springboard_trigger_snapshot["last_trigger_route"], + "springboard_trigger_volume_notification_installed": springboard_trigger_snapshot["volume_notification"]["installed"], + "springboard_trigger_volume_notification_seeded": springboard_trigger_snapshot["volume_notification"]["seeded"], + "springboard_trigger_volume_notification_events_seen": springboard_trigger_snapshot["volume_notification"]["events_seen"], + "springboard_trigger_volume_notification_last_direction": springboard_trigger_snapshot["volume_notification"]["last_direction"], + }, + "gates": { + "screen": "pass" if screen_ok else "fail", + "screenshot": "pass" if screenshot_ok is True else ("fail" if screenshot_ok is False else "not_run"), + "trigger": "not_run", + "input": "not_run", + "unlocked_foreground": unlocked_foreground, + "app_ui": app_ui_gate, + "lockscreen": lockscreen_gate, + "prefs_ui": prefs_ui_gate, + "prefs_backend": prefs_backend_gate, + "stores": stores_gate, + "provider_attempts": provider_attempts_gate, + "visible_effects": visible_effects_gate, + "memory_lifecycle": memory_lifecycle_gate, + "watcher_timer": watcher_timer_gate, + "watcher_repair": watcher_repair_gate, + "job_repair": job_repair_gate, + "restart_recovery": restart_recovery_gate, + "model_loop": model_loop_gate, + "provider_model": provider_model_gate, + "safari_dom_model": safari_dom_model_gate, + "prompt_bridge_model": prompt_bridge_model_gate, + "trigger_diagnostics": trigger_diagnostics_gate, + "artifact_hygiene": "pass" if artifact_hygiene_ok else "fail", + }, + "store_checks": store_checks, + "unlocked_foreground_checks": unlocked_foreground_checks, + "app_ui_checks": app_ui_checks, + "lockscreen_checks": lockscreen_checks, + "prefs_ui_checks": prefs_ui_checks, + "prefs_backend_checks": prefs_backend_checks, + "provider_attempt_checks": provider_attempt_checks, + "visible_effects_checks": visible_effects_checks, + "memory_lifecycle_checks": memory_lifecycle_checks, + "watcher_timer_checks": watcher_timer_checks, + "watcher_repair_checks": watcher_repair_checks, + "job_repair_checks": job_repair_checks, + "restart_recovery_checks": restart_recovery_checks, + "model_loop_checks": model_loop_checks, + "provider_model_checks": provider_model_checks, + "safari_dom_model_checks": safari_dom_model_checks, + "prompt_bridge_model_checks": prompt_bridge_model_checks, + "trigger_diagnostics_checks": trigger_diagnostics_checks, + "artifact_hygiene": artifact_hygiene, + "artifacts": { + "health_json": str(run_dir / "health.json"), + "get_screen_json": str(run_dir / "get-screen.json"), + "screenshot_json": str(run_dir / "get-screen-screenshot.json") if screenshot else "", + "screenshot_png": str(run_dir / "screenshot.png") if (run_dir / "screenshot.png").exists() else "", + "screenshot_sanity": screenshot_sanity, + "unlocked_foreground_before_screen_json": str(run_dir / "unlocked-foreground-before-screen.json") if (run_dir / "unlocked-foreground-before-screen.json").exists() else "", + "unlocked_foreground_open_safari_json": str(run_dir / "unlocked-foreground-open-safari.json") if (run_dir / "unlocked-foreground-open-safari.json").exists() else "", + "unlocked_foreground_safari_screen_json": str(run_dir / "unlocked-foreground-safari-screen.json") if (run_dir / "unlocked-foreground-safari-screen.json").exists() else "", + "unlocked_foreground_home_json": str(run_dir / "unlocked-foreground-home.json") if (run_dir / "unlocked-foreground-home.json").exists() else "", + "unlocked_foreground_home_screen_json": str(run_dir / "unlocked-foreground-home-screen.json") if (run_dir / "unlocked-foreground-home-screen.json").exists() else "", + "app_ui_before_screen_json": str(run_dir / "app-ui-before-screen.json") if (run_dir / "app-ui-before-screen.json").exists() else "", + "app_ui_relaunch_json": str(run_dir / "app-ui-relaunch.json") if (run_dir / "app-ui-relaunch.json").exists() else "", + "app_ui_open_safari_json": str(run_dir / "app-ui-open-safari.json") if (run_dir / "app-ui-open-safari.json").exists() else "", + "app_ui_safari_screen_json": str(run_dir / "app-ui-safari-screen.json") if (run_dir / "app-ui-safari-screen.json").exists() else "", + "app_ui_open_settings_json": str(run_dir / "app-ui-open-settings.json") if (run_dir / "app-ui-open-settings.json").exists() else "", + "app_ui_settings_screen_json": str(run_dir / "app-ui-settings-screen.json") if (run_dir / "app-ui-settings-screen.json").exists() else "", + "app_ui_health_json": str(run_dir / "app-ui-health.json") if (run_dir / "app-ui-health.json").exists() else "", + "app_ui_ls": str(run_dir / "app-ui-ls.txt") if (run_dir / "app-ui-ls.txt").exists() else "", + "app_ui_safari_state_json": str(run_dir / "app-ui-safari-state.json") if (run_dir / "app-ui-safari-state.json").exists() else "", + "app_ui_settings_state_json": str(run_dir / "app-ui-settings-state.json") if (run_dir / "app-ui-settings-state.json").exists() else "", + "app_introspector_log_tail": str(run_dir / "openphone-app-introspector.log.tail") if (run_dir / "openphone-app-introspector.log.tail").exists() else "", + "lockscreen_before_screen_json": str(run_dir / "lockscreen-before-screen.json") if (run_dir / "lockscreen-before-screen.json").exists() else "", + "lockscreen_show_passcode_json": str(run_dir / "lockscreen-show-passcode.json") if (run_dir / "lockscreen-show-passcode.json").exists() else "", + "lockscreen_after_screen_json": str(run_dir / "lockscreen-after-screen.json") if (run_dir / "lockscreen-after-screen.json").exists() else "", + "lockscreen_status_after_json": str(run_dir / "lockscreen-status-after.json") if (run_dir / "lockscreen-status-after.json").exists() else "", + "prefs_ui_before_screen_json": str(run_dir / "prefs-ui-before-screen.json") if (run_dir / "prefs-ui-before-screen.json").exists() else "", + "prefs_ui_prepare_json": str(run_dir / "prefs-ui-prepare.json") if (run_dir / "prefs-ui-prepare.json").exists() else "", + "prefs_ui_open_url_json": str(run_dir / "prefs-ui-open-url.json") if (run_dir / "prefs-ui-open-url.json").exists() else "", + "prefs_ui_url_screen_json": str(run_dir / "prefs-ui-url-screen.json") if (run_dir / "prefs-ui-url-screen.json").exists() else "", + "prefs_ui_open_settings_json": str(run_dir / "prefs-ui-open-settings.json") if (run_dir / "prefs-ui-open-settings.json").exists() else "", + "prefs_ui_settings_screen_json": str(run_dir / "prefs-ui-settings-screen.json") if (run_dir / "prefs-ui-settings-screen.json").exists() else "", + "prefs_ui_row_element": str(run_dir / "prefs-ui-row-element.txt") if (run_dir / "prefs-ui-row-element.txt").exists() else "", + "prefs_ui_tap_row_json": str(run_dir / "prefs-ui-tap-row.json") if (run_dir / "prefs-ui-tap-row.json").exists() else "", + "prefs_ui_pane_screen_json": str(run_dir / "prefs-ui-pane-screen.json") if (run_dir / "prefs-ui-pane-screen.json").exists() else "", + "prefs_ui_hardware_element": str(run_dir / "prefs-ui-hardware-element.txt") if (run_dir / "prefs-ui-hardware-element.txt").exists() else "", + "prefs_ui_disable_hardware_json": str(run_dir / "prefs-ui-disable-hardware.json") if (run_dir / "prefs-ui-disable-hardware.json").exists() else "", + "prefs_ui_after_disable_screen_json": str(run_dir / "prefs-ui-after-disable-screen.json") if (run_dir / "prefs-ui-after-disable-screen.json").exists() else "", + "prefs_ui_status_disabled_json": str(run_dir / "prefs-ui-status-disabled.json") if (run_dir / "prefs-ui-status-disabled.json").exists() else "", + "prefs_ui_enable_hardware_json": str(run_dir / "prefs-ui-enable-hardware.json") if (run_dir / "prefs-ui-enable-hardware.json").exists() else "", + "prefs_ui_after_enable_screen_json": str(run_dir / "prefs-ui-after-enable-screen.json") if (run_dir / "prefs-ui-after-enable-screen.json").exists() else "", + "prefs_ui_status_enabled_json": str(run_dir / "prefs-ui-status-enabled.json") if (run_dir / "prefs-ui-status-enabled.json").exists() else "", + "prefs_ui_final_restore_json": str(run_dir / "prefs-ui-final-restore.json") if (run_dir / "prefs-ui-final-restore.json").exists() else "", + "prefs_ui_status_after_json": str(run_dir / "prefs-ui-status-after.json") if (run_dir / "prefs-ui-status-after.json").exists() else "", + "prefs_backend_files_json": str(run_dir / "prefs-backend-files.json") if (run_dir / "prefs-backend-files.json").exists() else "", + "prefs_backend_status_before_json": str(run_dir / "prefs-backend-status-before.json") if (run_dir / "prefs-backend-status-before.json").exists() else "", + "prefs_backend_disable_hardware_json": str(run_dir / "prefs-backend-disable-hardware.json") if (run_dir / "prefs-backend-disable-hardware.json").exists() else "", + "prefs_backend_trigger_disabled_json": str(run_dir / "prefs-backend-trigger-disabled.json") if (run_dir / "prefs-backend-trigger-disabled.json").exists() else "", + "prefs_backend_enable_hardware_json": str(run_dir / "prefs-backend-enable-hardware.json") if (run_dir / "prefs-backend-enable-hardware.json").exists() else "", + "prefs_backend_disable_yolo_json": str(run_dir / "prefs-backend-disable-yolo.json") if (run_dir / "prefs-backend-disable-yolo.json").exists() else "", + "prefs_backend_trigger_yolo_disabled_json": str(run_dir / "prefs-backend-trigger-yolo-disabled.json") if (run_dir / "prefs-backend-trigger-yolo-disabled.json").exists() else "", + "prefs_backend_enable_yolo_json": str(run_dir / "prefs-backend-enable-yolo.json") if (run_dir / "prefs-backend-enable-yolo.json").exists() else "", + "prefs_backend_status_after_json": str(run_dir / "prefs-backend-status-after.json") if (run_dir / "prefs-backend-status-after.json").exists() else "", + "visible_effects_before_screen_json": str(run_dir / "visible-effects-before-screen.json") if (run_dir / "visible-effects-before-screen.json").exists() else "", + "visible_effects_open_settings_json": str(run_dir / "visible-effects-open-settings.json") if (run_dir / "visible-effects-open-settings.json").exists() else "", + "visible_effects_settings_precheck_json": str(run_dir / "visible-effects-settings-precheck.json") if (run_dir / "visible-effects-settings-precheck.json").exists() else "", + "visible_effects_settings_scenario": str(run_dir / "visible-effects-settings-scenario.txt") if (run_dir / "visible-effects-settings-scenario.txt").exists() else "", + "visible_effects_settings_target_label": str(run_dir / "visible-effects-settings-target-label.txt") if (run_dir / "visible-effects-settings-target-label.txt").exists() else "", + "visible_effects_settings_back_element": str(run_dir / "visible-effects-settings-back-element.txt") if (run_dir / "visible-effects-settings-back-element.txt").exists() else "", + "visible_effects_settings_reset_json": str(run_dir / "visible-effects-settings-reset.json") if (run_dir / "visible-effects-settings-reset.json").exists() else "", + "visible_effects_settings_before_json": str(run_dir / "visible-effects-settings-before.json") if (run_dir / "visible-effects-settings-before.json").exists() else "", + "visible_effects_settings_element": str(run_dir / "visible-effects-settings-element.txt") if (run_dir / "visible-effects-settings-element.txt").exists() else "", + "visible_effects_tap_settings_json": str(run_dir / "visible-effects-tap-settings.json") if (run_dir / "visible-effects-tap-settings.json").exists() else "", + "visible_effects_settings_after_json": str(run_dir / "visible-effects-settings-after.json") if (run_dir / "visible-effects-settings-after.json").exists() else "", + "visible_effects_open_safari_json": str(run_dir / "visible-effects-open-safari.json") if (run_dir / "visible-effects-open-safari.json").exists() else "", + "visible_effects_safari_before_json": str(run_dir / "visible-effects-safari-before.json") if (run_dir / "visible-effects-safari-before.json").exists() else "", + "visible_effects_safari_field": str(run_dir / "visible-effects-safari-field.txt") if (run_dir / "visible-effects-safari-field.txt").exists() else "", + "visible_effects_safari_marker": str(run_dir / "visible-effects-safari-marker.txt") if (run_dir / "visible-effects-safari-marker.txt").exists() else "", + "visible_effects_type_safari_json": str(run_dir / "visible-effects-type-safari.json") if (run_dir / "visible-effects-type-safari.json").exists() else "", + "visible_effects_safari_after_json": str(run_dir / "visible-effects-safari-after.json") if (run_dir / "visible-effects-safari-after.json").exists() else "", + "visible_effects_safari_state_json": str(run_dir / "visible-effects-safari-state.json") if (run_dir / "visible-effects-safari-state.json").exists() else "", + "visible_effects_open_notes_json": str(run_dir / "visible-effects-open-notes.json") if (run_dir / "visible-effects-open-notes.json").exists() else "", + "visible_effects_notes_before_json": str(run_dir / "visible-effects-notes-before.json") if (run_dir / "visible-effects-notes-before.json").exists() else "", + "visible_effects_notes_field": str(run_dir / "visible-effects-notes-field.txt") if (run_dir / "visible-effects-notes-field.txt").exists() else "", + "visible_effects_notes_marker": str(run_dir / "visible-effects-notes-marker.txt") if (run_dir / "visible-effects-notes-marker.txt").exists() else "", + "visible_effects_type_notes_json": str(run_dir / "visible-effects-type-notes.json") if (run_dir / "visible-effects-type-notes.json").exists() else "", + "visible_effects_notes_after_json": str(run_dir / "visible-effects-notes-after.json") if (run_dir / "visible-effects-notes-after.json").exists() else "", + "visible_effects_notes_state_json": str(run_dir / "visible-effects-notes-state.json") if (run_dir / "visible-effects-notes-state.json").exists() else "", + "springboard_crashes": str(run_dir / "springboard-crashes.txt"), + "safe_mode_markers": str(run_dir / "safe-mode-markers.txt"), + "agentd_log_tail": str(run_dir / "openphone-agentd.log.tail"), + "tweak_log_tail": str(run_dir / "openphone-volume-trigger.log.tail"), + "springboard_trigger_status_json": str(run_dir / "springboard-trigger-status.json") if (run_dir / "springboard-trigger-status.json").exists() else "", + "trigger_diagnostics_before_status_json": str(run_dir / "trigger-diagnostics-before-status.json") if (run_dir / "trigger-diagnostics-before-status.json").exists() else "", + "trigger_diagnostics_before_trigger_json": str(run_dir / "trigger-diagnostics-before-trigger.json") if (run_dir / "trigger-diagnostics-before-trigger.json").exists() else "", + "trigger_diagnostics_after_status_json": str(run_dir / "trigger-diagnostics-after-status.json") if (run_dir / "trigger-diagnostics-after-status.json").exists() else "", + "trigger_diagnostics_after_trigger_json": str(run_dir / "trigger-diagnostics-after-trigger.json") if (run_dir / "trigger-diagnostics-after-trigger.json").exists() else "", + "trigger_diagnostics_list_tasks_json": str(run_dir / "trigger-diagnostics-list-tasks.json") if (run_dir / "trigger-diagnostics-list-tasks.json").exists() else "", + "trigger_diagnostics_latest_trajectory_json": str(run_dir / "trigger-diagnostics-latest-trajectory.json") if (run_dir / "trigger-diagnostics-latest-trajectory.json").exists() else "", + "trigger_diagnostics_tweak_log_tail": str(run_dir / "trigger-diagnostics-tweak-log-tail.txt") if (run_dir / "trigger-diagnostics-tweak-log-tail.txt").exists() else "", + "list_tasks_json": str(run_dir / "list-tasks.json") if (run_dir / "list-tasks.json").exists() else "", + "get_task_json": str(run_dir / "get-task.json") if (run_dir / "get-task.json").exists() else "", + "get_trajectory_json": str(run_dir / "get-trajectory.json") if (run_dir / "get-trajectory.json").exists() else "", + "get_audit_json": str(run_dir / "get-audit.json") if (run_dir / "get-audit.json").exists() else "", + "memory_search_json": str(run_dir / "memory-search.json") if (run_dir / "memory-search.json").exists() else "", + "context_search_json": str(run_dir / "context-search.json") if (run_dir / "context-search.json").exists() else "", + "background_job_list_json": str(run_dir / "background-job-list.json") if (run_dir / "background-job-list.json").exists() else "", + "commitment_search_json": str(run_dir / "commitment-search.json") if (run_dir / "commitment-search.json").exists() else "", + "watcher_list_json": str(run_dir / "watcher-list.json") if (run_dir / "watcher-list.json").exists() else "", + "provider_attempt_start_task_json": str(run_dir / "provider-attempt-start-task.json") if (run_dir / "provider-attempt-start-task.json").exists() else "", + "provider_attempt_action_json": str(run_dir / "provider-attempt-action.json") if (run_dir / "provider-attempt-action.json").exists() else "", + "provider_attempt_trajectory_json": str(run_dir / "provider-attempt-trajectory.json") if (run_dir / "provider-attempt-trajectory.json").exists() else "", + "memory_lifecycle_save_primary_json": str(run_dir / "memory-lifecycle-save-primary.json") if (run_dir / "memory-lifecycle-save-primary.json").exists() else "", + "memory_lifecycle_update_json": str(run_dir / "memory-lifecycle-update.json") if (run_dir / "memory-lifecycle-update.json").exists() else "", + "memory_lifecycle_save_source_json": str(run_dir / "memory-lifecycle-save-source.json") if (run_dir / "memory-lifecycle-save-source.json").exists() else "", + "memory_lifecycle_merge_json": str(run_dir / "memory-lifecycle-merge.json") if (run_dir / "memory-lifecycle-merge.json").exists() else "", + "memory_lifecycle_save_delete_json": str(run_dir / "memory-lifecycle-save-delete.json") if (run_dir / "memory-lifecycle-save-delete.json").exists() else "", + "memory_lifecycle_delete_json": str(run_dir / "memory-lifecycle-delete.json") if (run_dir / "memory-lifecycle-delete.json").exists() else "", + "memory_lifecycle_search_json": str(run_dir / "memory-lifecycle-search.json") if (run_dir / "memory-lifecycle-search.json").exists() else "", + "watcher_timer_create_json": str(run_dir / "watcher-timer-create.json") if (run_dir / "watcher-timer-create.json").exists() else "", + "watcher_timer_run_due_json": str(run_dir / "watcher-timer-run-due.json") if (run_dir / "watcher-timer-run-due.json").exists() else "", + "watcher_timer_job_run_due_json": str(run_dir / "watcher-timer-job-run-due.json") if (run_dir / "watcher-timer-job-run-due.json").exists() else "", + "watcher_timer_job_list_json": str(run_dir / "watcher-timer-job-list.json") if (run_dir / "watcher-timer-job-list.json").exists() else "", + "watcher_timer_after_list_json": str(run_dir / "watcher-timer-after-list.json") if (run_dir / "watcher-timer-after-list.json").exists() else "", + "watcher_timer_stop_json": str(run_dir / "watcher-timer-stop.json") if (run_dir / "watcher-timer-stop.json").exists() else "", + "watcher_repair_create_json": str(run_dir / "watcher-repair-create.json") if (run_dir / "watcher-repair-create.json").exists() else "", + "watcher_repair_mark_running_json": str(run_dir / "watcher-repair-mark-running.json") if (run_dir / "watcher-repair-mark-running.json").exists() else "", + "watcher_repair_run_json": str(run_dir / "watcher-repair-run.json") if (run_dir / "watcher-repair-run.json").exists() else "", + "watcher_repair_run_due_json": str(run_dir / "watcher-repair-run-due.json") if (run_dir / "watcher-repair-run-due.json").exists() else "", + "watcher_repair_job_run_due_json": str(run_dir / "watcher-repair-job-run-due.json") if (run_dir / "watcher-repair-job-run-due.json").exists() else "", + "watcher_repair_after_list_json": str(run_dir / "watcher-repair-after-list.json") if (run_dir / "watcher-repair-after-list.json").exists() else "", + "watcher_repair_stop_json": str(run_dir / "watcher-repair-stop.json") if (run_dir / "watcher-repair-stop.json").exists() else "", + "job_repair_create_json": str(run_dir / "job-repair-create.json") if (run_dir / "job-repair-create.json").exists() else "", + "job_repair_mark_running_json": str(run_dir / "job-repair-mark-running.json") if (run_dir / "job-repair-mark-running.json").exists() else "", + "job_repair_run_json": str(run_dir / "job-repair-run.json") if (run_dir / "job-repair-run.json").exists() else "", + "job_repair_run_due_json": str(run_dir / "job-repair-run-due.json") if (run_dir / "job-repair-run-due.json").exists() else "", + "job_repair_list_json": str(run_dir / "job-repair-list.json") if (run_dir / "job-repair-list.json").exists() else "", + "job_repair_stop_json": str(run_dir / "job-repair-stop.json") if (run_dir / "job-repair-stop.json").exists() else "", + "restart_recovery_watcher_create_json": str(run_dir / "restart-recovery-watcher-create.json") if (run_dir / "restart-recovery-watcher-create.json").exists() else "", + "restart_recovery_job_create_json": str(run_dir / "restart-recovery-job-create.json") if (run_dir / "restart-recovery-job-create.json").exists() else "", + "restart_recovery_watcher_mark_running_json": str(run_dir / "restart-recovery-watcher-mark-running.json") if (run_dir / "restart-recovery-watcher-mark-running.json").exists() else "", + "restart_recovery_job_mark_running_json": str(run_dir / "restart-recovery-job-mark-running.json") if (run_dir / "restart-recovery-job-mark-running.json").exists() else "", + "restart_recovery_restart_json": str(run_dir / "restart-recovery-restart.json") if (run_dir / "restart-recovery-restart.json").exists() else "", + "restart_recovery_after_health_json": str(run_dir / "restart-recovery-after-health.json") if (run_dir / "restart-recovery-after-health.json").exists() else "", + "restart_recovery_watcher_list_json": str(run_dir / "restart-recovery-watcher-list.json") if (run_dir / "restart-recovery-watcher-list.json").exists() else "", + "restart_recovery_job_list_json": str(run_dir / "restart-recovery-job-list.json") if (run_dir / "restart-recovery-job-list.json").exists() else "", + "restart_recovery_watcher_stop_json": str(run_dir / "restart-recovery-watcher-stop.json") if (run_dir / "restart-recovery-watcher-stop.json").exists() else "", + "restart_recovery_job_stop_json": str(run_dir / "restart-recovery-job-stop.json") if (run_dir / "restart-recovery-job-stop.json").exists() else "", + "restart_recovery_generated_job_stop_json": str(run_dir / "restart-recovery-generated-job-stop.json") if (run_dir / "restart-recovery-generated-job-stop.json").exists() else "", + "model_status_json": str(run_dir / "model-status.json") if (run_dir / "model-status.json").exists() else "", + "model_loop_run_json": str(run_dir / "model-loop-run.json") if (run_dir / "model-loop-run.json").exists() else "", + "model_loop_trajectory_json": str(run_dir / "model-loop-trajectory.json") if (run_dir / "model-loop-trajectory.json").exists() else "", + "model_loop_repair_run_json": str(run_dir / "model-loop-repair-run.json") if (run_dir / "model-loop-repair-run.json").exists() else "", + "model_loop_repair_trajectory_json": str(run_dir / "model-loop-repair-trajectory.json") if (run_dir / "model-loop-repair-trajectory.json").exists() else "", + "model_loop_cancel_start_json": str(run_dir / "model-loop-cancel-start.json") if (run_dir / "model-loop-cancel-start.json").exists() else "", + "model_loop_cancel_stop_json": str(run_dir / "model-loop-cancel-stop.json") if (run_dir / "model-loop-cancel-stop.json").exists() else "", + "model_loop_cancel_run_json": str(run_dir / "model-loop-cancel-run.json") if (run_dir / "model-loop-cancel-run.json").exists() else "", + "model_loop_cancel_trajectory_json": str(run_dir / "model-loop-cancel-trajectory.json") if (run_dir / "model-loop-cancel-trajectory.json").exists() else "", + "provider_model_configure_json": str(run_dir / "provider-model-configure.json") if (run_dir / "provider-model-configure.json").exists() else "", + "provider_model_status_json": str(run_dir / "provider-model-status.json") if (run_dir / "provider-model-status.json").exists() else "", + "provider_model_run_json": str(run_dir / "provider-model-run.json") if (run_dir / "provider-model-run.json").exists() else "", + "provider_model_trajectory_json": str(run_dir / "provider-model-trajectory.json") if (run_dir / "provider-model-trajectory.json").exists() else "", + "provider_model_reset_json": str(run_dir / "provider-model-reset.json") if (run_dir / "provider-model-reset.json").exists() else "", + "provider_model_broker_log": str(run_dir / "provider-model-broker.log") if (run_dir / "provider-model-broker.log").exists() else "", + "provider_model_ssh_reverse_log": str(run_dir / "provider-model-ssh-reverse.log") if (run_dir / "provider-model-ssh-reverse.log").exists() else "", + "safari_dom_model_marker": str(run_dir / "safari-dom-model-marker.txt") if (run_dir / "safari-dom-model-marker.txt").exists() else "", + "safari_dom_model_backup_json": str(run_dir / "safari-dom-model-backup.json") if (run_dir / "safari-dom-model-backup.json").exists() else "", + "safari_dom_model_before_screen_json": str(run_dir / "safari-dom-model-before-screen.json") if (run_dir / "safari-dom-model-before-screen.json").exists() else "", + "safari_dom_model_open_url_json": str(run_dir / "safari-dom-model-open-url.json") if (run_dir / "safari-dom-model-open-url.json").exists() else "", + "safari_dom_model_pre_screen_json": str(run_dir / "safari-dom-model-pre-screen.json") if (run_dir / "safari-dom-model-pre-screen.json").exists() else "", + "safari_dom_model_configure_json": str(run_dir / "safari-dom-model-configure.json") if (run_dir / "safari-dom-model-configure.json").exists() else "", + "safari_dom_model_status_json": str(run_dir / "safari-dom-model-status.json") if (run_dir / "safari-dom-model-status.json").exists() else "", + "safari_dom_model_run_json": str(run_dir / "safari-dom-model-run.json") if (run_dir / "safari-dom-model-run.json").exists() else "", + "safari_dom_model_trajectory_json": str(run_dir / "safari-dom-model-trajectory.json") if (run_dir / "safari-dom-model-trajectory.json").exists() else "", + "safari_dom_model_after_screen_json": str(run_dir / "safari-dom-model-after-screen.json") if (run_dir / "safari-dom-model-after-screen.json").exists() else "", + "safari_dom_model_safari_state_json": str(run_dir / "safari-dom-model-safari-state.json") if (run_dir / "safari-dom-model-safari-state.json").exists() else "", + "safari_dom_model_reset_json": str(run_dir / "safari-dom-model-reset.json") if (run_dir / "safari-dom-model-reset.json").exists() else "", + "prompt_bridge_marker": str(run_dir / "prompt-bridge-marker.txt") if (run_dir / "prompt-bridge-marker.txt").exists() else "", + "prompt_bridge_request_id": str(run_dir / "prompt-bridge-request-id.txt") if (run_dir / "prompt-bridge-request-id.txt").exists() else "", + "prompt_bridge_before_screen_json": str(run_dir / "prompt-bridge-before-screen.json") if (run_dir / "prompt-bridge-before-screen.json").exists() else "", + "prompt_bridge_open_url_json": str(run_dir / "prompt-bridge-open-url.json") if (run_dir / "prompt-bridge-open-url.json").exists() else "", + "prompt_bridge_pre_screen_json": str(run_dir / "prompt-bridge-pre-screen.json") if (run_dir / "prompt-bridge-pre-screen.json").exists() else "", + "prompt_bridge_model_status_json": str(run_dir / "prompt-bridge-model-status.json") if (run_dir / "prompt-bridge-model-status.json").exists() else "", + "prompt_bridge_response_json": str(run_dir / "prompt-bridge-response.json") if (run_dir / "prompt-bridge-response.json").exists() else "", + "prompt_bridge_agent_status_json": str(run_dir / "prompt-bridge-agent-status.json") if (run_dir / "prompt-bridge-agent-status.json").exists() else "", + "prompt_bridge_trajectory_json": str(run_dir / "prompt-bridge-trajectory.json") if (run_dir / "prompt-bridge-trajectory.json").exists() else "", + "prompt_bridge_after_screen_json": str(run_dir / "prompt-bridge-after-screen.json") if (run_dir / "prompt-bridge-after-screen.json").exists() else "", + "prompt_bridge_safari_state_json": str(run_dir / "prompt-bridge-safari-state.json") if (run_dir / "prompt-bridge-safari-state.json").exists() else "", + "prompt_bridge_tweak_log": str(run_dir / "prompt-bridge-tweak-log.txt") if (run_dir / "prompt-bridge-tweak-log.txt").exists() else "", + "validate_log": str(run_dir / "validate.log"), + }, + "exit_code": exit_code, +} + +(run_dir / "report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") +(run_dir / "exit-code.txt").write_text(str(exit_code) + "\n", encoding="utf-8") +print(json.dumps({ + "status": "ok" if exit_code == 0 else "failed", + "exit_code": exit_code, + "report": str(run_dir / "report.json"), + "health": report["daemon"]["health_status"], + "latest_springboard_crash": latest, + "safe_mode_marker_present": bool(markers), + "gates": report["gates"], + "artifact_hygiene": report["artifact_hygiene"]["status"], +}, indent=2)) +PY +} + +start_iproxy_if_requested +check_remote_target_identity + +if [[ "$mode" != "collect-only" ]]; then + run_local "git-diff-check" 10 git -C "$repo_root" diff --check + run_local "smoke-agentd-local" 10 "$repo_root/tools/mac/agentd/smoke-agentd-local.sh" + run_local "make-package" 20 make -C "$repo_root/agentd" package + if [[ -z "$package" ]]; then + package="$(ls -t "$repo_root"/agentd/packages/com.openphone.agentd_*_iphoneos-arm64.deb 2>/dev/null | head -n 1 || true)" + fi + if [[ -z "$package" || ! -f "$package" ]]; then + fail 20 "package not found after build" + fi + log "Using package $package" + collect_safety "before" + check_preinstall_safety + log "Installing package" + if ! OPENPHONE_AGENTD_DEB="$package" \ + OPENPHONE_IOS_HOST="$host" \ + OPENPHONE_IOS_SSH_PORT="$port" \ + OPENPHONE_IOS_USER="$user" \ + OPENPHONE_IOS_PASSWORD="$password" \ + OPENPHONE_IOS_KNOWN_HOSTS="$known_hosts" \ + OPENPHONE_IOS_UDID="$target_udid" \ + "$repo_root/tools/mac/agentd/install-agentd-package.sh" >"$run_dir/install-agentd-package.log" 2>&1; then + fail 20 "package install failed; see $run_dir/install-agentd-package.log" + fi +fi + +collect_device_state +check_stability_gate +start_provider_broker_if_requested +collect_screenshot_if_requested +collect_unlocked_foreground_sample +collect_app_ui_sample +collect_lockscreen_sample +collect_prefs_backend_sample +collect_prefs_ui_sample +collect_visible_effect_sample +if [[ "$mode" == "full" || "${OPENPHONE_VALIDATE_INCLUDE_STORES:-0}" == "1" ]]; then + collect_safe_store_state +fi +collect_provider_attempt_sample +collect_memory_lifecycle_sample +collect_voice_status_sample +collect_watcher_timer_sample +collect_watcher_repair_sample +collect_job_repair_sample +collect_restart_recovery_sample +collect_model_loop_sample +collect_provider_model_sample +collect_safari_dom_model_sample +collect_prompt_bridge_model_sample +collect_trigger_diagnostics_sample + +generate_report | tee -a "$summary_log" +exit_code="$(cat "$run_dir/exit-code.txt")" +log "Validation report: $run_dir/report.json" +exit "$exit_code" diff --git a/ios/tools/mac/doctor/openphone-agentd-health.sh b/ios/tools/mac/doctor/openphone-agentd-health.sh new file mode 100755 index 0000000..2b11115 --- /dev/null +++ b/ios/tools/mac/doctor/openphone-agentd-health.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +host="${OPENPHONE_IOS_HOST:-127.0.0.1}" +port="${OPENPHONE_IOS_SSH_PORT:-22}" +user="${OPENPHONE_IOS_USER:-mobile}" + +remote_cmd=' +set -eu +if [ ! -e /var/jb ]; then + echo "{\"status\":\"error\",\"reason\":\"rootless_prefix_missing\"}" + exit 1 +fi +if [ ! -x /var/jb/usr/local/bin/openphone-agentctl ]; then + echo "{\"status\":\"error\",\"reason\":\"openphone-agentctl_missing\"}" + exit 1 +fi +/var/jb/usr/local/bin/openphone-agentctl +' + +ssh -p "$port" "$user@$host" "$remote_cmd" From 2ab0ab9b0c0895a36cda3f225848be332abb7512 Mon Sep 17 00:00:00 2001 From: Adam Cohen Hillel Date: Mon, 6 Jul 2026 22:46:17 -0700 Subject: [PATCH 2/3] Add iOS legal boundary and disclaimer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add docs/legal/IOS.md and an ios/README.md legal-boundary section stating, in plain lawful language, the intended scope: OpenPhone-authored code only; personal, non-commercial use on an owned device; proof of concept, no warranty; no Apple affiliation. Explicitly disclaims containing or explaining any exploit, unlocking tool, or circumvention means — the agent is application-layer software that runs on top of an owner-prepared device. Link the notice from the legal index. Co-Authored-By: Claude Opus 4.7 --- docs/legal/IOS.md | 51 +++++++++++++++++++++++++++++++++++++++++++++ docs/legal/index.md | 1 + ios/README.md | 15 +++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 docs/legal/IOS.md diff --git a/docs/legal/IOS.md b/docs/legal/IOS.md new file mode 100644 index 0000000..4a2cedf --- /dev/null +++ b/docs/legal/IOS.md @@ -0,0 +1,51 @@ +# OpenPhone iOS Legal Notice + +This notice governs the OpenPhone iOS materials under [`ios/`](/ios). It is not +legal advice. It states the intended scope and terms under which these +materials are made available. + +## Scope of these materials + +The OpenPhone iOS materials are original, OpenPhone-authored source code and +documentation for a phone-resident agent runtime. They contain **only +OpenPhone-owned code**. + +These materials do **not** contain, bundle, distribute, or link to: + +- any exploit, unlocking tool, bootrom or kernel exploit, or other software + that circumvents a technological protection measure, +- any tool, payload, or package whose purpose is to modify, unlock, or bypass + device security, or +- any instructions, guide, or runbook explaining how to place a device into a + modified-operating-system state. + +The OpenPhone iOS agent is application-layer software that runs **on top of** a +device the owner has already prepared through separate means of their own +choosing. Reaching any such device state is solely the owner's responsibility +and is entirely out of scope for this project. + +## Intended use + +- **Personal, non-commercial use only.** These materials are provided for + personal research and evaluation. Commercial use requires a separate written + commercial license (see [COMMERCIAL.md](/docs/legal/COMMERCIAL)). +- **Owned devices only.** Intended solely for a device that you own. +- **Proof of concept.** Experimental proof-of-concept work, not a supported + product. +- **No warranty.** Provided "as is", without warranty of any kind, to the + fullest extent permitted by law. + +## No affiliation + +OpenPhone is not affiliated with, endorsed by, or sponsored by Apple Inc. Apple, +iPhone, iOS, and related marks are trademarks of Apple Inc. and are used only to +describe interoperability. See also the project [NOTICE](/docs/legal/NOTICE). + +## Licensing + +The OpenPhone iOS materials are OpenPhone-owned materials and are licensed under +the same terms as the rest of this repository: the PolyForm Noncommercial +License 1.0.0 for non-commercial use, with commercial use requiring a separate +written license from Dafdef, inc. See [LICENSE](/LICENSE), +[LICENSE.noncommercial](/docs/legal/LICENSE.noncommercial), and +[COMMERCIAL.md](/docs/legal/COMMERCIAL). diff --git a/docs/legal/index.md b/docs/legal/index.md index c96faa7..39c7dae 100644 --- a/docs/legal/index.md +++ b/docs/legal/index.md @@ -5,3 +5,4 @@ OpenPhone legal and licensing docs live here. - [Licensing Notes](/docs/LICENSING) - [Commercial Licensing](/docs/legal/COMMERCIAL) - [Third-Party Notices](/docs/legal/THIRD_PARTY_NOTICES) +- [OpenPhone iOS Notice](/docs/legal/IOS) diff --git a/ios/README.md b/ios/README.md index 0db5cb1..4e2a528 100644 --- a/ios/README.md +++ b/ios/README.md @@ -10,6 +10,21 @@ This is experimental proof-of-concept work, not a supported product. It targets an owned device that already exposes a rootless runtime prefix at `/var/jb`; reaching that state is the owner's responsibility and out of scope here. +## Legal boundary + +These materials contain **only OpenPhone-authored code**. They do **not** +contain, bundle, distribute, or explain any exploit, unlocking tool, or other +means of circumventing device protections. This is application-layer software +that runs **on top of** a device the owner has already prepared through separate +means of their own choosing. + +Provided for **personal, non-commercial use on a device you own**, as a proof of +concept, "as is" and without warranty. OpenPhone is not affiliated with or +endorsed by Apple Inc.; Apple, iPhone, and iOS are trademarks of Apple Inc. used +only to describe interoperability. Full terms are in +[docs/legal/IOS.md](../docs/legal/IOS.md); licensing is the repository-wide +PolyForm Noncommercial License (see [LICENSE](../LICENSE)). + ## Layout ```text From 277a5148eecb8cc7070d392159fdb1ed3dc66c93 Mon Sep 17 00:00:00 2001 From: Adam Cohen Hillel Date: Fri, 10 Jul 2026 15:31:05 -0700 Subject: [PATCH 3/3] iOS agent: realtime voice loop, volume-trigger gestures, input reliability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Realtime voice pipeline (main.m): the conversation owns the session — the model calling finish_task/fail_task no longer tears down the live mic/socket; only user cancel, a dropped socket, or the duration ceiling ends it. Keep barge-in armed through tool execution and abort an in-flight action batch when the user speaks over the agent. Auto-deliver a fresh screen observation after each UI action (and cache it as the next turn input) and neutralize the wait tool in the realtime path to cut per-step dead air. Tap disabled elements that still expose valid bounds by coordinate. Non-fatal jetsam memory limit raised to 1024MB with kernel readback verification. Volume trigger (volume_trigger.xm): hardened press/hold gesture state machine with generation-counter invalidation of stale hold timers and phase tracking. App introspector: minor input-focus handling refinements. Co-Authored-By: Claude Opus 4.7 --- ios/agentd/OpenPhoneAppIntrospector.plist | 15 +- ios/agentd/src/app_introspector.xm | 31 +- ios/agentd/src/main.m | 1753 +++++++++++++++++++-- ios/agentd/src/volume_trigger.xm | 751 +++++++-- 4 files changed, 2279 insertions(+), 271 deletions(-) diff --git a/ios/agentd/OpenPhoneAppIntrospector.plist b/ios/agentd/OpenPhoneAppIntrospector.plist index 168f5a8..75ee960 100644 --- a/ios/agentd/OpenPhoneAppIntrospector.plist +++ b/ios/agentd/OpenPhoneAppIntrospector.plist @@ -1,20 +1,7 @@ { Filter = { Bundles = ( - "com.apple.mobilesafari", - "com.apple.WebKit.WebContent", - "com.apple.Preferences", - "com.apple.mobilenotes", - "com.apple.MobileSMS", - "com.apple.mobilecal" - ); - Executables = ( - "MobileSafari", - "com.apple.WebKit.WebContent", - "Preferences", - "MobileNotes", - "MobileSMS", - "MobileCal" + "com.apple.UIKit" ); }; } diff --git a/ios/agentd/src/app_introspector.xm b/ios/agentd/src/app_introspector.xm index c18e6f2..81256a3 100644 --- a/ios/agentd/src/app_introspector.xm +++ b/ios/agentd/src/app_introspector.xm @@ -2016,13 +2016,32 @@ static NSDictionary *OPAITypeTextIntoView(UIView *view, } } NSString *afterLength = OPAIEditableTextLength(editable); + long long beforeLen = [beforeLength longLongValue]; + long long afterLen = [afterLength longLongValue]; + // Verify the insert actually landed: some balloon/compose views accept the + // replaceRange/insertText call without applying it (no first responder, an + // input delegate that rejects the edit, etc.) and would otherwise report a + // false success for text that never entered the field. Require the content + // length to have grown by the inserted text before trusting it. + NSUInteger insertedLength = (text ?: @"").length; + if (insertedLength > 0 && afterLen <= beforeLen) { + return OPAIInputResponse(@"unavailable", @"text_input_not_applied", + actionType, target, @{ + @"activation_method": @"text_input_insert", + @"target_class": NSStringFromClass([editable class]) ?: @"", + @"became_first_responder": @(becameFirstResponder), + @"text_length": @(insertedLength), + @"before_text_length": @(beforeLen), + @"after_text_length": @(afterLen) + }); + } return OPAIInputResponse(@"ok", nil, actionType, target, @{ @"activation_method": @"text_input_insert", @"target_class": NSStringFromClass([editable class]) ?: @"", @"became_first_responder": @(becameFirstResponder), - @"text_length": @((text ?: @"").length), - @"before_text_length": @([beforeLength longLongValue]), - @"after_text_length": @([afterLength longLongValue]) + @"text_length": @(insertedLength), + @"before_text_length": @(beforeLen), + @"after_text_length": @(afterLen) }); } @@ -2298,6 +2317,12 @@ static void *OPAIStateThread(void *unused) { __attribute__((constructor)) static void OPAIInit(void) { NSString *bundleId = OPAIAppBundleId(); + // SpringBoard is owned by OpenPhoneVolumeTrigger, which drives the island + // overlay and its own input bridge. Running the introspector's publish/input + // loop there too would fight over the daemon's app-input socket, so stay out. + if ([bundleId isEqualToString:@"com.apple.springboard"]) { + return; + } OPAILog(@"OpenPhoneAppIntrospector loaded bundle=%@ pid=%d", bundleId ?: @"", getpid()); pthread_t thread; diff --git a/ios/agentd/src/main.m b/ios/agentd/src/main.m index a3279cd..e2993aa 100644 --- a/ios/agentd/src/main.m +++ b/ios/agentd/src/main.m @@ -49,8 +49,30 @@ extern int memorystatus_control(uint32_t command, int32_t pid, uint32_t flags, #ifndef MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK #define MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK 5 #endif +// XNU command numbers (bsd/sys/kern_memorystatus.h). Command 6 sets the fatal +// task limit as a bare int; command 7 sets the full memlimit_properties struct +// (active + inactive limits with per-limit fatal/non-fatal flags). Earlier code +// used 7 with a bare int, which is a struct-size mismatch: the kernel rejected +// it with EINVAL and the discarded return hid the failure, so the launchd 6 MB +// default was never actually lifted. We use 7 with the correct struct below. #ifndef MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT -#define MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT 7 +#define MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT 6 +#endif +#ifndef MEMORYSTATUS_CMD_SET_MEMLIMIT_PROPERTIES +#define MEMORYSTATUS_CMD_SET_MEMLIMIT_PROPERTIES 7 +#endif +// Command 8 reads the live memlimit_properties back from the kernel. We use it +// immediately after SET to prove the limit actually took (rc=0 on SET only means +// the call was accepted; the read-back reports the real kernel-enforced ceiling, +// unlike `launchctl print` which shows the stale provisioning-time value). +#ifndef MEMORYSTATUS_CMD_GET_MEMLIMIT_PROPERTIES +#define MEMORYSTATUS_CMD_GET_MEMLIMIT_PROPERTIES 8 +#endif +// Bit in memorystatus_memlimit_properties.memlimit_*_attr: when SET the limit +// is *fatal* (exceed => SIGKILL). We clear it so exceeding the limit only makes +// us a jetsam candidate under pressure rather than an instant kill. +#ifndef MEMORYSTATUS_MEMLIMIT_ATTR_FATAL +#define MEMORYSTATUS_MEMLIMIT_ATTR_FATAL 0x1 #endif // xnu jetsam priority bands are small integers (0..JETSAM_PRIORITY_MAX==21): // 0=idle, 6=phone, 10=foreground app, 12=audio_and_accessory, 19=critical. @@ -63,6 +85,14 @@ extern int memorystatus_control(uint32_t command, int32_t pid, uint32_t flags, #ifndef JETSAM_PRIORITY_AUDIO_AND_ACCESSORY #define JETSAM_PRIORITY_AUDIO_AND_ACCESSORY 12 #endif +// Highest band a userspace daemon can realistically hold. Bands 20/21 are +// effectively kernel-reserved and memorystatus_control rejects them for a +// mobile-uid process, so 19 (critical) is our practical ceiling. We pin here +// so the daemon is never reaped as low-priority collateral when a heavy app +// (e.g. Amazon) foregrounds and spikes system memory mid voice session. +#ifndef JETSAM_PRIORITY_CRITICAL +#define JETSAM_PRIORITY_CRITICAL 19 +#endif #ifndef JETSAM_PRIORITY_MAX #define JETSAM_PRIORITY_MAX 21 #endif @@ -75,6 +105,16 @@ extern int memorystatus_control(uint32_t command, int32_t pid, uint32_t flags, uint64_t user_data; } memorystatus_priority_properties_t; +// XNU bsd/sys/kern_memorystatus.h layout for MEMORYSTATUS_CMD_SET_MEMLIMIT_PROPERTIES. +// memlimit_*_attr carry the per-limit flags (bit 0 = fatal). Must match the +// kernel's size exactly or the call is rejected with EINVAL. +typedef struct { + int32_t memlimit_active; // MB; <=0 means "no active limit" + uint32_t memlimit_active_attr; + int32_t memlimit_inactive; // MB; <=0 means "no inactive limit" + uint32_t memlimit_inactive_attr; +} memorystatus_memlimit_properties_t; + static NSString *const OPAgentVersion = @"0.1.0-dev"; static NSString *const OPDefaultStorePath = @"/var/mobile/Library/OpenPhone"; static NSString *const OPVolumeTriggerPreferencesPath = @"/var/mobile/Library/Preferences/com.openphone.volumetrigger.plist"; @@ -82,6 +122,12 @@ extern int memorystatus_control(uint32_t command, int32_t pid, uint32_t flags, static NSString *const OPLegacyHardwareTriggerGoal = @"Handle the OpenPhone hardware volume trigger using the current phone context."; static NSString *const OPOpenAIRealtimeModel = @"gpt-realtime"; static NSString *const OPOpenAIRealtime2Model = @"gpt-realtime-2"; +// OpenAI Realtime emits pcm16 model audio at 24 kHz; the output AudioQueue must +// play back at the same rate. +#define OP_RT_OUTPUT_SAMPLE_RATE_HZ 24000 +// Step-harness OpenAI model. Matches the Android agent's DEFAULT_MODEL so both +// platforms drive the same backend from the same OpenAI key. +static NSString *const OPOpenAIResponsesModel = @"gpt-5.5"; static volatile sig_atomic_t OPRunning = 1; static int OPServerFd = -1; @@ -91,6 +137,10 @@ extern int memorystatus_control(uint32_t command, int32_t pid, uint32_t flags, static pthread_mutex_t OPVoiceTriggerMutex = PTHREAD_MUTEX_INITIALIZER; static BOOL OPVoiceTriggerRunning = NO; static volatile int OPVoiceCancelRequested = 0; +// Push-to-talk: set by the `voice_stop` command when the user releases the +// long-press. Unlike cancel, this keeps the audio captured so far and finishes +// the STT/transcribe path — it just ends the mic capture immediately. +static volatile int OPVoiceStopRequested = 0; static long long OPVoiceTriggerLastStartedMs = 0; static long long OPVoiceTriggerLastFinishedMs = 0; static NSString *OPVoiceTriggerLastState = nil; @@ -131,6 +181,7 @@ extern int memorystatus_control(uint32_t command, int32_t pid, uint32_t flags, static BOOL OPTaskCancellationRequested(NSString *taskId, NSString **reasonOut); static void OPRecordTrajectory(NSString *taskId, NSString *event, NSDictionary *payload); static BOOL OPModelModeIsOpenAIRealtime(NSString *mode); +static NSString *OPVoiceCredentialValue(NSString **sourceOut); static NSDictionary *OPJetsamPrioritySet(NSDictionary *request); static NSString *OPExplicitURLFromText(NSString *text); static NSDictionary *OPError(NSString *reason); @@ -928,6 +979,20 @@ static BOOL OPWriteProtectedJSONFile(NSString *path, NSDictionary *object) { return [OPStorePath() stringByAppendingPathComponent:@"springboard"]; } +// A "working" mode is a claim that a live daemon is mid-turn (recording, +// transcribing, running a tool, or waiting on the user). The daemon proves it +// still owns that claim by refreshing heartbeat_ms; if it dies, the heartbeat +// stops and the SpringBoard tweak self-heals the stale pill back to idle. +// Resting modes (idle/success/error/hidden) need no owner and never expire. +static BOOL OPIslandModeIsWorking(NSString *mode) { + return [mode isEqualToString:@"listening"] || + [mode isEqualToString:@"realtime"] || + [mode isEqualToString:@"transcribing"] || + [mode isEqualToString:@"thinking"] || + [mode isEqualToString:@"action"] || + [mode isEqualToString:@"needs_review"]; +} + static void OPIslandEnsureState(void) { if (!OPIslandState) { OPIslandState = [NSMutableDictionary dictionary]; @@ -944,13 +1009,22 @@ static void OPIslandEnsureState(void) { OPIslandState[@"accent"] = @"cyan"; OPIslandState[@"sequence"] = @0; OPIslandState[@"updated_at_ms"] = @0; + OPIslandState[@"heartbeat_ms"] = @0; } } -static void OPIslandPublishLocked(void) { - OPIslandSequence += 1; - OPIslandState[@"sequence"] = @(OPIslandSequence); - OPIslandState[@"updated_at_ms"] = @(OPNowMs()); +// Write the current state to disk. Bumps the sequence and posts the Darwin +// notification only on a real state change (bumpSequence=YES); the heartbeat +// path writes with bumpSequence=NO so a liveness refresh doesn't churn the +// tweak's sequence-gated re-render. Caller must hold OPIslandMutex. +static void OPIslandWriteLocked(BOOL bumpSequence) { + long long now = OPNowMs(); + if (bumpSequence) { + OPIslandSequence += 1; + OPIslandState[@"sequence"] = @(OPIslandSequence); + OPIslandState[@"updated_at_ms"] = @(now); + } + OPIslandState[@"heartbeat_ms"] = @(now); NSString *dir = OPIslandDir(); [[NSFileManager defaultManager] createDirectoryAtPath:dir withIntermediateDirectories:YES @@ -959,7 +1033,34 @@ static void OPIslandPublishLocked(void) { if (OPWriteJSONFile(OPIslandStatusPath, snapshot)) { chmod(OPIslandStatusPath.UTF8String, 0644); } - notify_post(OPIslandStatusNotification); + if (bumpSequence) { + notify_post(OPIslandStatusNotification); + } +} + +static void OPIslandPublishLocked(void) { + OPIslandWriteLocked(YES); +} + +// Keeps heartbeat_ms fresh while a working state is showing, so the tweak can +// tell a live long-running turn (e.g. a 45s voice recording) apart from a +// frozen snapshot left by a crashed/jetsammed daemon. +static void *OPIslandHeartbeatThread(void *unused) { + (void)unused; + while (true) { + usleep(5 * 1000 * 1000); + @autoreleasepool { + pthread_mutex_lock(&OPIslandMutex); + OPIslandEnsureState(); + NSString *mode = [OPIslandState[@"mode"] isKindOfClass:[NSString class]] + ? OPIslandState[@"mode"] : @""; + if (OPIslandModeIsWorking(mode)) { + OPIslandWriteLocked(NO); + } + pthread_mutex_unlock(&OPIslandMutex); + } + } + return NULL; } static void OPIslandUpdate(NSDictionary *patch) { @@ -1525,7 +1626,31 @@ static BOOL OPSQLiteTableExists(sqlite3 *db, NSString *name) { return exists; } +static BOOL OPSQLiteMigrationApplied(sqlite3 *db, const char *name) { + sqlite3_stmt *statement = NULL; + BOOL applied = NO; + if (sqlite3_prepare_v2(db, + "SELECT 1 FROM schema_migrations WHERE name = ? LIMIT 1", + -1, &statement, NULL) == SQLITE_OK) { + sqlite3_bind_text(statement, 1, name, -1, SQLITE_TRANSIENT); + applied = sqlite3_step(statement) == SQLITE_ROW; + } + sqlite3_finalize(statement); + return applied; +} + static BOOL OPSQLiteMigrate(sqlite3 *db, NSString **errorOut) { + // Connection-level PRAGMAs don't dirty pages, but the schema bookkeeping below + // (ALTER TABLE + INSERT OR REPLACE into schema_migrations) writes on every call. + // This runs on every OPSQLiteOpen and the 5s scheduler opens the DB constantly; + // re-running the writes each tick dirtied ~4GB/day and got the daemon jetsam-killed + // mid-session. Once the final migration is recorded, skip all writes. + OPSQLiteExec(db, @"PRAGMA journal_mode=WAL", NULL); + OPSQLiteExec(db, @"PRAGMA synchronous=NORMAL", NULL); + if (OPSQLiteTableExists(db, @"schema_migrations") && + OPSQLiteMigrationApplied(db, "openphone_store_v3_scheduler_enabled")) { + return YES; + } NSArray *statements = @[ @"PRAGMA journal_mode=WAL", @"PRAGMA synchronous=NORMAL", @@ -3723,6 +3848,10 @@ static BOOL OPModelModeIsOpenAIRealtime(NSString *mode) { [mode isEqualToString:@"openai_realtime2"]; } +static BOOL OPModelModeIsOpenAIResponses(NSString *mode) { + return [mode isEqualToString:@"openai_responses"]; +} + static NSString *OPModelDefaultModelForMode(NSString *mode) { if ([mode isEqualToString:@"openai_realtime2"]) { return OPOpenAIRealtime2Model; @@ -3730,6 +3859,9 @@ static BOOL OPModelModeIsOpenAIRealtime(NSString *mode) { if ([mode isEqualToString:@"openai_realtime"]) { return OPOpenAIRealtimeModel; } + if (OPModelModeIsOpenAIResponses(mode)) { + return OPOpenAIResponsesModel; + } return @""; } @@ -3746,6 +3878,7 @@ static BOOL OPModelModeIsOpenAIRealtime(NSString *mode) { static BOOL OPModelModeHasDefaultEndpoint(NSString *mode) { return [mode isEqualToString:@"bedrock_converse"] || + OPModelModeIsOpenAIResponses(mode) || OPModelModeIsOpenAIRealtime(mode); } @@ -3753,6 +3886,7 @@ static BOOL OPModelModeIsValid(NSString *mode) { return [mode isEqualToString:@"broker"] || [mode isEqualToString:@"direct_dev"] || [mode isEqualToString:@"bedrock_converse"] || + OPModelModeIsOpenAIResponses(mode) || OPModelModeIsOpenAIRealtime(mode); } @@ -3784,7 +3918,7 @@ static BOOL OPModelModeIsValid(NSString *mode) { config[@"model"] = OPModelEffectiveModel(config) ?: @""; config[@"enabled"] = @(OPBoolFromRequest(config, @"enabled", NO)); BOOL credentialRequired = OPBoolFromRequest(config, @"credential_required", YES); - if (OPModelModeIsOpenAIRealtime(mode)) { + if (OPModelModeIsOpenAIRealtime(mode) || OPModelModeIsOpenAIResponses(mode)) { credentialRequired = YES; } config[@"credential_required"] = @(credentialRequired); @@ -3972,7 +4106,7 @@ static BOOL OPModelModeIsValid(NSString *mode) { long long maxSteps = OPLongLongFromRequest(request, @"max_steps", 5, 1, 120); long long maxDurationMs = OPLongLongFromRequest(request, @"max_duration_ms", 120000, 1000, 3300000); BOOL credentialRequired = OPBoolFromRequest(request, @"credential_required", YES); - if (OPModelModeIsOpenAIRealtime(mode)) { + if (OPModelModeIsOpenAIRealtime(mode) || OPModelModeIsOpenAIResponses(mode)) { credentialRequired = YES; } // Preserve screenshot tuning across model config writes: the screenshot @@ -6579,6 +6713,73 @@ static BOOL OPContactsLooksLikePhone(NSString *value) { return contacts; } +static NSString *OPTelURLForPhoneNumber(NSString *raw) { + if (![raw isKindOfClass:[NSString class]] || raw.length == 0) { + return nil; + } + NSMutableString *out = [NSMutableString string]; + BOOL leadingPlus = NO; + for (NSUInteger i = 0; i < raw.length; i++) { + unichar c = [raw characterAtIndex:i]; + if (c == '+' && out.length == 0) { + leadingPlus = YES; + } else if (c >= '0' && c <= '9') { + [out appendFormat:@"%C", c]; + } + } + if (out.length == 0) { + return nil; + } + return leadingPlus ? [NSString stringWithFormat:@"tel:+%@", out] + : [NSString stringWithFormat:@"tel:%@", out]; +} + +// Pick the single best contact to call for a name-based call goal and return a +// ready tel: URL. gpt-5.5 cannot be trusted to translate contact rows into a +// dial action (it blind-taps the current screen instead), so the daemon owns +// this deterministically: skip the user's own "(me)" entry, require a dialable +// number, and prefer a primary "Main" entry when a first-name query matches +// several people. Returns nil when nothing dialable resolves. +static NSString *OPBestCallDialURL(NSArray *contacts, NSString *query) { + if (![contacts isKindOfClass:[NSArray class]] || contacts.count == 0) { + return nil; + } + NSString *q = [(query ?: @"") lowercaseString]; + NSString *best = nil; + NSInteger bestScore = NSIntegerMin; + for (NSDictionary *c in contacts) { + if (![c isKindOfClass:[NSDictionary class]]) continue; + NSString *name = [c[@"display_name"] isKindOfClass:[NSString class]] + ? c[@"display_name"] : @""; + NSString *nameLower = name.lowercaseString; + // Never dial the user's own record. + if ([nameLower containsString:@"(me)"]) { + continue; + } + NSArray *phones = [c[@"phone_numbers"] isKindOfClass:[NSArray class]] + ? c[@"phone_numbers"] : @[]; + NSString *tel = nil; + for (id p in phones) { + if ([p isKindOfClass:[NSString class]]) { + tel = OPTelURLForPhoneNumber(p); + if (tel) break; + } + } + if (!tel) { + continue; + } + NSInteger score = 0; + if (q.length > 0 && [nameLower isEqualToString:q]) score += 100; + if (q.length > 0 && [nameLower containsString:q]) score += 40; + if ([nameLower containsString:@"main"]) score += 10; + if (score > bestScore) { + bestScore = score; + best = tel; + } + } + return best; +} + static NSArray *OPContactsSummaryContacts(NSArray *contacts, NSUInteger limit) { NSMutableArray *summary = [NSMutableArray array]; for (NSDictionary *contact in contacts) { @@ -6594,7 +6795,18 @@ static BOOL OPContactsLooksLikePhone(NSString *value) { NSArray *emails = [contact[@"emails"] isKindOfClass:[NSArray class]] ? contact[@"emails"] : @[]; if (phones.count > 0) { - item[@"phone_numbers"] = phones.count > 4 ? [phones subarrayWithRange:NSMakeRange(0, 4)] : phones; + NSArray *keptPhones = phones.count > 4 ? [phones subarrayWithRange:NSMakeRange(0, 4)] : phones; + item[@"phone_numbers"] = keptPhones; + NSMutableArray *dialURLs = [NSMutableArray array]; + for (NSString *phone in keptPhones) { + NSString *tel = OPTelURLForPhoneNumber(phone); + if (tel) { + [dialURLs addObject:tel]; + } + } + if (dialURLs.count > 0) { + item[@"dial_urls"] = dialURLs; + } } if (emails.count > 0) { item[@"emails"] = emails.count > 4 ? [emails subarrayWithRange:NSMakeRange(0, 4)] : emails; @@ -11697,6 +11909,56 @@ static BOOL OPCenterFromBoundsObject(id boundsObject, double *outX, double *outY return nil; } +static BOOL OPElementIsEditableField(NSDictionary *element) { + if (![element isKindOfClass:[NSDictionary class]]) { + return NO; + } + NSString *kind = [element[@"kind"] isKindOfClass:[NSString class]] + ? element[@"kind"] : @""; + if ([kind isEqualToString:@"text_field"] || [kind isEqualToString:@"text_area"] || + [kind isEqualToString:@"search"]) { + return YES; + } + // Web/DOM editables report editable:true rather than a native kind. + if ([element[@"editable"] respondsToSelector:@selector(boolValue)] && + [element[@"editable"] boolValue]) { + return YES; + } + return NO; +} + +// Returns YES only when the current screen shows a focused, editable text +// field. Blind daemon HID typing has no target of its own, so if nothing is +// focused the keystrokes are dropped on the floor; callers use this to avoid +// reporting a clean success for typing that went nowhere. +static BOOL OPScreenHasFocusedEditableField(NSDictionary *screen) { + NSDictionary *context = [screen[@"context"] isKindOfClass:[NSDictionary class]] + ? screen[@"context"] : @{}; + NSMutableArray *candidateArrays = [NSMutableArray array]; + if ([context[@"interactive_elements"] isKindOfClass:[NSArray class]]) { + [candidateArrays addObject:context[@"interactive_elements"]]; + } + NSDictionary *uiTree = [context[@"ui_tree"] isKindOfClass:[NSDictionary class]] + ? context[@"ui_tree"] : @{}; + if ([uiTree[@"interactive_elements"] isKindOfClass:[NSArray class]]) { + [candidateArrays addObject:uiTree[@"interactive_elements"]]; + } + for (NSArray *elements in candidateArrays) { + for (id object in elements) { + if (![object isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *element = object; + BOOL focused = [element[@"focused"] respondsToSelector:@selector(boolValue)] && + [element[@"focused"] boolValue]; + if (focused && OPElementIsEditableField(element)) { + return YES; + } + } + } + return NO; +} + static NSDictionary *OPResolvedElementSummary(NSDictionary *element) { if (![element isKindOfClass:[NSDictionary class]]) { return @{}; @@ -12242,18 +12504,19 @@ static BOOL OPPointFromAction(NSDictionary *action, double *outX, double *outY) id enabledValue = resolvedElement[@"enabled"]; BOOL enabled = ![enabledValue respondsToSelector:@selector(boolValue)] || [enabledValue boolValue]; - if (!enabled) { - result = @{ - @"state": @"action.denied.element_disabled", - @"task_id": taskId, - @"capability": capability, - @"detail": [NSString stringWithFormat:@"element_disabled:%@", elementId], - @"target": OPResolvedElementSummary(resolvedElement), - @"source": @"openphone.agentd" - }; - } else if (OPCenterFromBoundsObject(resolvedElement[@"bounds"], &x, &y)) { + // Many real controls (e.g. Amazon's "Search Amazon.com" field, + // which is actually a button that expands the search UI) report + // accessibilityTraits without UIAccessibilityTrait* enabling, so + // our tree marks them enabled=false even though a human taps them + // fine. Hard-denying stalled the agent forever ("Looking at ..." + // with no path forward). So we no longer refuse a disabled element + // outright: if it has valid bounds we tap its center by coordinate + // and flag it, letting the next screen observation confirm whether + // the tap did anything. We only deny when there are no bounds at all. + if (OPCenterFromBoundsObject(resolvedElement[@"bounds"], &x, &y)) { hasPoint = YES; - coordinateSource = @"ui_tree.bounds_center"; + coordinateSource = enabled ? @"ui_tree.bounds_center" + : @"ui_tree.bounds_center.disabled_override"; } else { result = @{ @"state": @"action.denied.missing_coordinates", @@ -12539,17 +12802,38 @@ static BOOL OPPointFromAction(NSDictionary *action, double *outX, double *outY) } } if (!result) { + // Blind daemon HID typing has no target of its own: keystrokes + // land in whatever field currently holds keyboard focus, or are + // dropped entirely if nothing is focused. Reporting success here + // when no editable field is focused is how send-message flows + // "type" a body that never actually enters the field. Require a + // focused editable field before trusting the blind path. + BOOL hasFocusedField = OPScreenHasFocusedEditableField(screen); NSDictionary *typed = OPPerformHIDTypeText(text); + BOOL dispatched = [typed[@"ok"] boolValue]; + NSString *state; + NSString *detail; + if (!dispatched) { + state = @"action.denied.input_failed"; + detail = [NSString stringWithFormat:@"type_text:%lu", (unsigned long)text.length]; + } else if (!hasFocusedField) { + state = @"action.denied.no_focused_field"; + detail = [NSString stringWithFormat:@"type_text:%lu:no_focused_field", (unsigned long)text.length]; + } else { + state = @"action.executed"; + detail = [NSString stringWithFormat:@"type_text:%lu", (unsigned long)text.length]; + } NSMutableDictionary *mutableResult = [@{ - @"state": [typed[@"ok"] boolValue] - ? @"action.executed" : @"action.denied.input_failed", + @"state": state, @"task_id": taskId, @"capability": capability, - @"detail": [NSString stringWithFormat:@"type_text:%lu", - (unsigned long)text.length], + @"detail": detail, @"provider_result": typed, @"source": @"openphone.agentd" } mutableCopy]; + if ([state isEqualToString:@"action.denied.no_focused_field"]) { + mutableResult[@"reason"] = @"No editable text field is focused, so the typed text was not entered. Tap the text field first (to give it keyboard focus), then type again."; + } NSMutableArray *providerAttempts = [NSMutableArray array]; if (appInput) { [providerAttempts addObject:OPInputAttemptSummaryForAction(appInput, type)]; @@ -13200,6 +13484,87 @@ static BOOL OPStringContainsCaseInsensitive(NSString *haystack, NSString *needle return [haystack rangeOfString:needle options:NSCaseInsensitiveSearch].location != NSNotFound; } +// A send-a-message goal must actually enter body text before it can be +// finished. gpt-5.5 sometimes hallucinates a "Sent ..." finish_task after only +// tapping/opening, so we gate finish_task on a real prior type_text for these +// goals. Deliberately narrow: only trip on explicit send-message phrasing. +static BOOL OPGoalIsSendMessage(NSString *goal) { + NSString *lower = [(goal ?: @"") lowercaseString]; + if (lower.length == 0) { + return NO; + } + BOOL hasVerb = [lower containsString:@"send"] || [lower containsString:@"text "] || + [lower hasPrefix:@"text"] || [lower containsString:@"message"] || + [lower containsString:@"imessage"] || [lower containsString:@"reply"]; + if (!hasVerb) { + return NO; + } + return [lower containsString:@"message"] || [lower containsString:@"imessage"] || + [lower containsString:@"text"] || [lower containsString:@"say"] || + [lower containsString:@"tell"] || [lower containsString:@"reply"] || + [lower containsString:@"whatsapp"] || [lower containsString:@"sms"]; +} + +// A "call " goal where the number is NOT already in the goal text. In +// that case gpt-5.5 is unreliable at deciding to look the number up first — +// it either blind-taps the current screen or refuses. So we deterministically +// seed contacts_search as the first step and hand the resolved dial_urls back +// to the model, which then only has to open_url the tel: link (which it does +// reliably when the number is literally in front of it). Returns the contact +// name to search for, or nil when the goal is not a name-based call request. +static NSString *OPCallGoalContactName(NSString *goal) { + NSString *lower = [(goal ?: @"") lowercaseString]; + NSString *trimmed = [(goal ?: @"") stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + if (lower.length == 0) { + return nil; + } + // If the goal already carries a phone number, the model dials fine on its + // own — do not seed a contact search. + NSUInteger digitRun = 0, maxDigitRun = 0; + for (NSUInteger i = 0; i < lower.length; i++) { + unichar c = [lower characterAtIndex:i]; + if (c >= '0' && c <= '9') { digitRun++; if (digitRun > maxDigitRun) maxDigitRun = digitRun; } + else { digitRun = 0; } + } + if (maxDigitRun >= 5) { + return nil; + } + // Match a leading call verb. + NSArray *prefixes = @[@"call ", @"phone ", @"dial ", @"ring ", + @"give a call to ", @"place a call to ", @"call up "]; + NSString *name = nil; + for (NSString *prefix in prefixes) { + if ([lower hasPrefix:prefix]) { + name = [trimmed substringFromIndex:prefix.length]; + break; + } + } + if (name == nil) { + return nil; + } + name = [name stringByTrimmingCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + // Strip a trailing "on his/her cell", "'s phone", etc. — keep the first + // 1-3 words which are almost always the contact name. + NSArray *words = [name componentsSeparatedByCharactersInSet: + [NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSMutableArray *kept = [NSMutableArray array]; + for (NSString *w in words) { + if (w.length == 0) continue; + NSString *wl = w.lowercaseString; + if ([wl isEqualToString:@"on"] || [wl isEqualToString:@"at"] || + [wl isEqualToString:@"please"] || [wl isEqualToString:@"now"] || + [wl hasPrefix:@"'s"]) { + break; + } + [kept addObject:w]; + if (kept.count >= 3) break; + } + NSString *result = [kept componentsJoinedByString:@" "]; + return result.length > 0 ? result : nil; +} + static BOOL OPModelVerifiedTypeTextCompletesGoal(NSString *goal, NSDictionary *decision, NSDictionary *verification) { if (![decision[@"tool"] isEqualToString:@"type_text"] || @@ -13595,47 +13960,79 @@ static BOOL OPModelVerifiedTypeTextCompletesGoal(NSString *goal, NSDictionary *d return [NSString stringWithFormat: @"You are OpenPhone — a voice assistant inside the user's iPhone.\n" @"Speak like a friend. Short. Human. Never recap raw context fields.\n\n" + @"You operate ONE STEP AT A TIME. Each turn you see the CURRENT screen and\n" + @"choose ONE next action. After it runs you get the NEW screen and choose\n" + @"again, until the goal is truly done. You are NOT planning the whole thing\n" + @"up front — decide the single best next step for the screen in front of you.\n\n" @"You route every request into ONE mode:\n\n" - @" answer — user asked a question you can answer from what's already visible or from common knowledge. Fill `reply` with a short human answer. Done in one turn.\n" - @" act — user wants something done on the phone. Fill `proposed_actions[]` with the exact tool calls needed, IN ORDER. If ONE tool call finishes the whole goal (open X, dial Y, open URL Z) put JUST that one call. Only include multiple actions if they are actually required for that goal (e.g. open Messages → tap Alex → type → send).\n" + @" answer — user asked a question you can answer from what's visible or from common knowledge. Fill `reply`. Done in one turn.\n" + @" act — user wants something done on the phone. Put the SINGLE next tool call in `proposed_actions[]` (exactly one). Do the next concrete step toward the goal based on what's on screen right now.\n" @" stop — user asked to cancel or stop the assistant.\n\n" + @"FINISHING (critical — do NOT lie about completion):\n" + @"- Only finish when the goal is ACTUALLY, VISIBLY complete on the current screen.\n" + @"- Opening an app is NOT completing a task. Typing text is NOT sending it.\n" + @"- To finish, put a single {\"tool\":\"finish_task\",\"arguments\":{\"summary\":\"...\"}} in proposed_actions[]. NEVER claim you sent/did something unless the screen proves it (e.g. the sent message now appears in the transcript).\n" + @"- If something clearly failed or is impossible, use {\"tool\":\"fail_task\",\"arguments\":{\"reason\":\"...\"}}.\n\n" + @"SENDING A MESSAGE (multi-step — follow in order, one step per turn):\n" + @" 1. open_app com.apple.MobileSMS.\n" + @" 2. Start a new message (tap the Compose element) OR tap the existing conversation with that contact if one is visible.\n" + @" 3. In a new message, type the recipient into the To field (type_text), then pick the matching contact suggestion (tap_element).\n" + @" 4. Tap the message/text field FIRST and confirm the keyboard appears — this is a SEPARATE step from typing. type_text ONLY works on a field that already has keyboard focus; typing into an unfocused screen silently drops every keystroke and the field stays empty. Then, on the NEXT turn, type the body with type_text.\n" + @" 5. Tap the Send button (the up-arrow, often labeled 'Send'). \n" + @" 6. Only AFTER the sent bubble appears in the transcript → finish_task.\n\n" + @"PLACING A CALL (multi-step — follow in order, one step per turn):\n" + @" 1. If you do not already have the phone number, use contacts_search to look it up. Do this AT MOST ONCE — if it returns a number, move on; never search the same contact twice.\n" + @" 2. open_url with a tel: URL. The contacts_search result gives each contact a ready-to-use `dial_urls` array — pass the FIRST entry straight into open_url, e.g. {\"url\":\"tel:+14155550123\"}. Do NOT reformat it, do NOT second-guess it, do NOT decide the number is unusable: if a `dial_urls` value exists, it IS the number to call. Only if `dial_urls` is absent, build the tel: URL yourself from the phone_numbers digits (strip spaces, dashes and parentheses; keep a leading + and country code). This launches the dialer and starts the call.\n" + @" 3. On the next turn the in-call screen should be visible → finish_task with a summary like \"Calling Adam.\". If the dialer never appears, fail_task — do NOT claim the call was placed.\n\n" @"HARD RULES:\n" - @"- The current phone context (foreground app, visible text, tappable elements) is ALREADY in this prompt. NEVER pick mode=inspect just to look again.\n" - @"- For questions like 'what app am I in', 'can you see my screen', 'what's on my calendar today' → mode=answer with a direct sentence in `reply`.\n" - @"- For 'open X', 'search Y on Z', 'call mom' → mode=act with the minimal proposed_actions[] to finish the goal.\n" - @"- Explicit URLs (goal contains http/https): the FIRST proposed_action MUST be open_url with that URL. Never tap Safari icons first.\n" - @"- Autonomy is full YOLO. Never ask permission. Never say 'I would' or 'shall I'. Just do it.\n\n" + @"- The current phone context (foreground app, visible text, tappable elements) is ALREADY in this prompt. Never ask to look again.\n" + @"- `loop.guidance` and `loop.last_tool` in the context tell you what you just did and what to do next — READ THEM before deciding. Never repeat the identical action twice.\n" + @"- Prefer tap_element with an exact element_id from the screen over raw tap coordinates.\n" + @"- For questions ('what app am I in', 'can you see my screen') → mode=answer with a direct sentence.\n" + @"- Explicit URLs (goal contains http/https): next action is open_url with that URL. Never tap Safari icons first.\n" + @"- Autonomy is full YOLO. Never ask permission. Never say 'I would' or 'shall I'. Just do the next step.\n\n" @"HOW TO WRITE `reply`:\n" - @"- Under 20 words. Talk TO the user. Never mention 'foreground_app', 'SpringBoard', 'element id', 'UI tree', 'metadata'.\n" - @"- For act mode: describe what you're doing right now, present tense. Example: \"Opening Wikipedia for Adam.\"\n" - @"- For answer mode: the direct answer. Example: \"Settings.\"\n\n" - @"EXAMPLES:\n" + @"- Under 20 words. Talk TO the user. Never mention 'foreground_app', 'element id', 'UI tree', 'metadata'.\n" + @"- act mode: describe what you're doing right now, present tense. Example: \"Typing your message to Alex.\"\n" + @"- answer mode: the direct answer. Example: \"Settings.\"\n\n" + @"EXAMPLES (each is ONE turn):\n" @" User: \"Can you see my screen?\" →\n" @" { \"mode\":\"answer\", \"reply\":\"Yes — you're on the Home screen.\" }\n" - @" User: \"What app am I in?\" →\n" - @" { \"mode\":\"answer\", \"reply\":\"Settings.\" }\n" - @" User: \"Open Wikipedia\" →\n" + @" User: \"Open Wikipedia\" (on Home screen) →\n" @" { \"mode\":\"act\", \"reply\":\"Opening Wikipedia.\",\n" @" \"proposed_actions\":[{\"tool\":\"open_url\",\"arguments\":{\"url\":\"https://en.wikipedia.org/\"}}] }\n" - @" User: \"Search Adam on Wikipedia\" →\n" - @" { \"mode\":\"act\", \"reply\":\"Opening the Wikipedia page for Adam.\",\n" - @" \"proposed_actions\":[{\"tool\":\"open_url\",\"arguments\":{\"url\":\"https://en.wikipedia.org/wiki/Adam\"}}] }\n" - @" User: \"Text Alex on my way\" →\n" - @" { \"mode\":\"act\", \"reply\":\"Texting Alex 'on my way'.\", \"task_goal\":\"send SMS 'on my way' to contact Alex\",\n" - @" \"proposed_actions\":[{\"tool\":\"open_app\",\"arguments\":{\"bundle_id\":\"com.apple.MobileSMS\"}}] }\n\n" + @" User: \"Call Adam\" (found +14155550123 via contacts_search) →\n" + @" { \"mode\":\"act\", \"reply\":\"Calling Adam.\",\n" + @" \"proposed_actions\":[{\"tool\":\"open_url\",\"arguments\":{\"url\":\"tel:+14155550123\"}}] }\n" + @" User: \"Text Alex 'on my way'\" (Home screen, step 1) →\n" + @" { \"mode\":\"act\", \"reply\":\"Opening Messages.\", \"task_goal\":\"send 'on my way' to Alex\",\n" + @" \"proposed_actions\":[{\"tool\":\"open_app\",\"arguments\":{\"bundle_id\":\"com.apple.MobileSMS\"}}] }\n" + @" (next turn, Messages open, compose visible) →\n" + @" { \"mode\":\"act\", \"reply\":\"Starting a new message.\",\n" + @" \"proposed_actions\":[{\"tool\":\"tap_element\",\"arguments\":{\"element_id\":\"\"}}] }\n" + @" (next turn, To field focused) →\n" + @" { \"mode\":\"act\", \"reply\":\"Adding Alex.\",\n" + @" \"proposed_actions\":[{\"tool\":\"type_text\",\"arguments\":{\"text\":\"Alex\"}}] }\n" + @" (…continue: pick contact, tap body, type 'on my way', tap Send…)\n" + @" (final turn, sent bubble visible in transcript) →\n" + @" { \"mode\":\"act\", \"reply\":\"Sent 'on my way' to Alex.\",\n" + @" \"proposed_actions\":[{\"tool\":\"finish_task\",\"arguments\":{\"summary\":\"Sent 'on my way' to Alex.\"}}] }\n\n" @"TOOL SCHEMAS (for proposed_actions):\n" @"- open_url: {\"url\":\"https://...\"}\n" @"- open_app: {\"bundle_id\":\"com.example.App\"}\n" @"- tap_element: {\"element_id\":\"exact visible id from the screen context\"}\n" @"- tap: {\"x\":num,\"y\":num}\n" - @"- type_text: {\"text\":\"...\",\"element_id\":\"field id\"}\n" + @"- type_text: {\"text\":\"...\",\"element_id\":\"field id (optional)\"}\n" @"- swipe: {\"start_x\",\"start_y\",\"end_x\",\"end_y\"}\n" - @"- home: {}\n\n" - @"Return ONE JSON object matching this schema — no markdown, no prose outside JSON:\n" + @"- home: {}\n" + @"- finish_task: {\"summary\":\"what you actually accomplished\"}\n" + @"- fail_task: {\"reason\":\"why it can't be done\"}\n\n" + @"Return ONE JSON object matching this schema — no markdown, no prose outside JSON.\n" + @"In act mode, proposed_actions MUST contain EXACTLY ONE action (the next step):\n" @"{\"schema\":\"openphone.model_decision.v3\",\"mode\":\"answer|act|stop\",\"reply\":\"user-facing text\",\"task_goal\":\"only if multi-step\",\"proposed_actions\":[{\"tool\":\"...\",\"arguments\":{...}}],\"reason\":\"private, brief\"}\n\n" @"Allowed tools inside proposed_actions: %@\n" - @"User said: %@\n\n" - @"Current phone context:\n%@", + @"User's original goal: %@\n\n" + @"Current phone context (includes loop.guidance / loop.last_tool — read them):\n%@", toolList, goal, OPModelJSONStringForPrompt(context, 6000)]; @@ -13884,12 +14281,214 @@ static BOOL OPModelVerifiedTypeTextCompletesGoal(NSString *goal, NSDictionary *d return result; } +// Extract the assistant text from an OpenAI /v1/responses envelope. Mirrors the +// Android adapter's extractOutputText: prefer the flattened "output_text", else +// walk output[].content[].text. +static NSString *OPModelOpenAIResponsesOutputText(NSDictionary *object) { + NSString *direct = [object[@"output_text"] isKindOfClass:[NSString class]] + ? object[@"output_text"] : @""; + if (direct.length > 0) { + return direct; + } + NSArray *output = [object[@"output"] isKindOfClass:[NSArray class]] + ? object[@"output"] : @[]; + NSMutableArray *parts = [NSMutableArray array]; + for (id itemValue in output) { + if (![itemValue isKindOfClass:[NSDictionary class]]) { + continue; + } + NSArray *content = [itemValue[@"content"] isKindOfClass:[NSArray class]] + ? itemValue[@"content"] : @[]; + for (id blockValue in content) { + if (![blockValue isKindOfClass:[NSDictionary class]]) { + continue; + } + NSString *text = [blockValue[@"text"] isKindOfClass:[NSString class]] + ? blockValue[@"text"] : @""; + if (text.length > 0) { + [parts addObject:text]; + } + } + } + return [[parts componentsJoinedByString:@"\n"] + stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; +} + +// Read the current screenshot referenced by the prompt context and return it as +// a base64 "data:image/jpeg;base64,..." URL for the Responses input_image block. +// Returns @"" when no usable screenshot is available (model then runs text-only). +static NSString *OPModelScreenshotDataURLFromRequestBody(NSDictionary *requestBody) { + NSDictionary *context = [requestBody[@"context"] isKindOfClass:[NSDictionary class]] + ? requestBody[@"context"] : @{}; + NSDictionary *screen = [context[@"screen"] isKindOfClass:[NSDictionary class]] + ? context[@"screen"] : @{}; + NSDictionary *shot = [screen[@"screenshot"] isKindOfClass:[NSDictionary class]] + ? screen[@"screenshot"] : @{}; + if (![shot[@"status"] isEqualToString:@"ok"]) { + return @""; + } + NSString *path = [shot[@"path"] isKindOfClass:[NSString class]] ? shot[@"path"] : @""; + if (path.length == 0 || ![[NSFileManager defaultManager] fileExistsAtPath:path]) { + return @""; + } + NSData *imageData = [NSData dataWithContentsOfFile:path]; + if (imageData.length == 0) { + return @""; + } + NSString *b64 = [imageData base64EncodedStringWithOptions:0]; + if (b64.length == 0) { + return @""; + } + return [@"data:image/jpeg;base64," stringByAppendingString:b64]; +} + +// Step-harness decision via OpenAI's Responses API. Same backend and key as the +// Android agent (OpenAiResponsesAgentAdapter). Reuses the OpenAI voice +// credential so a single key drives transcription, realtime, and the harness. +static NSDictionary *OPModelOpenAIResponsesDecision(NSDictionary *modelStatus, + NSDictionary *requestBody) { + NSDictionary *config = OPModelConfig(); + NSString *model = [config[@"model"] isKindOfClass:[NSString class]] && + [config[@"model"] length] > 0 ? config[@"model"] : OPOpenAIResponsesModel; + NSString *credential = OPVoiceCredentialValue(NULL); + if (credential.length == 0) { + credential = OPModelCredentialValue(); + } + if (credential.length == 0) { + return OPError(@"model_credential_missing"); + } + NSString *endpoint = [config[@"endpoint_url"] isKindOfClass:[NSString class]] && + [config[@"endpoint_url"] length] > 0 + ? config[@"endpoint_url"] : @"https://api.openai.com/v1/responses"; + NSURL *url = [NSURL URLWithString:endpoint]; + if (!url || !url.scheme || !url.host) { + return OPError(@"model_endpoint_invalid"); + } + long long timeoutMs = [modelStatus[@"timeout_ms"] respondsToSelector:@selector(longLongValue)] + ? [modelStatus[@"timeout_ms"] longLongValue] : 30000; + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url + cachePolicy:NSURLRequestReloadIgnoringLocalCacheData + timeoutInterval:MAX(1.0, (NSTimeInterval)timeoutMs / 1000.0)]; + request.HTTPMethod = @"POST"; + [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"]; + [request setValue:@"application/json" forHTTPHeaderField:@"Accept"]; + [request setValue:[NSString stringWithFormat:@"Bearer %@", credential] + forHTTPHeaderField:@"Authorization"]; + + NSString *instructions = [requestBody[@"instructions"] isKindOfClass:[NSString class]] + ? requestBody[@"instructions"] : @"Return exactly one JSON object."; + NSString *prompt = OPModelPromptText(requestBody); + // Build the user turn's content. Always include the text prompt; when a + // screenshot is available on disk, attach it as an input_image so the vision + // model can actually SEE the current screen (matches the Android adapter). + // Without this the model navigates UI blind and hallucinates completions. + NSMutableArray *content = [NSMutableArray arrayWithObject:@{ + @"type": @"input_text", + @"text": prompt + }]; + NSString *shotDataUrl = OPModelScreenshotDataURLFromRequestBody(requestBody); + if (shotDataUrl.length > 0) { + [content addObject:@{ + @"type": @"input_image", + @"image_url": shotDataUrl + }]; + } + // gpt-5.5 and other reasoning models spend part of max_output_tokens on + // internal reasoning before emitting the visible answer. With a small budget + // the reasoning can consume everything and the response comes back with + // empty output text, so keep a generous budget. Use medium reasoning effort: + // low effort produced sloppy refusals (e.g. claiming "no phone number" when + // the number was plainly in the prompt); medium gives the model enough to + // actually act on the data it was given. + NSDictionary *body = @{ + @"model": model, + @"instructions": instructions, + @"input": @[@{ + @"role": @"user", + @"content": content + }], + @"reasoning": @{@"effort": @"medium"}, + @"max_output_tokens": @4000 + }; + request.HTTPBody = OPCanonicalJSONData(body); + + NSURLResponse *response = nil; + NSError *error = nil; + long long startedMs = OPNowMs(); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + NSData *data = [NSURLConnection sendSynchronousRequest:request + returningResponse:&response + error:&error]; +#pragma clang diagnostic pop + long long latencyMs = OPNowMs() - startedMs; + if (error || !data) { + return OPError([NSString stringWithFormat:@"openai_responses_request_failed:%@", + error.localizedDescription ?: @"unknown"]); + } + NSInteger statusCode = 0; + if ([response isKindOfClass:[NSHTTPURLResponse class]]) { + statusCode = [(NSHTTPURLResponse *)response statusCode]; + } + if (statusCode < 200 || statusCode >= 300) { + return @{ + @"status": @"error", + @"reason": [NSString stringWithFormat:@"openai_responses_http_status:%ld", (long)statusCode], + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"source": @"openphone.agentd" + }; + } + id parsed = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + if (![parsed isKindOfClass:[NSDictionary class]]) { + return @{ + @"status": @"error", + @"reason": @"openai_responses_not_object", + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"source": @"openphone.agentd" + }; + } + NSDictionary *object = parsed; + NSString *decisionText = OPModelOpenAIResponsesOutputText(object); + if (decisionText.length == 0) { + return @{ + @"status": @"error", + @"reason": @"openai_responses_empty_text", + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"source": @"openphone.agentd" + }; + } + NSMutableDictionary *result = [@{ + @"status": @"ok", + @"provider": @"openai_responses", + @"http_status": @(statusCode), + @"response_bytes": @(data.length), + @"decision": decisionText, + @"source": @"openphone.agentd", + @"metadata": @{ + @"provider": @"openai_responses", + @"provider_backed": @YES, + @"model": model, + @"latency_ms": @(latencyMs) + } + } mutableCopy]; + if ([object[@"usage"] isKindOfClass:[NSDictionary class]]) { + result[@"usage"] = object[@"usage"]; + } + return result; +} + static NSDictionary *OPModelProviderDecision(NSDictionary *modelStatus, NSDictionary *requestBody) { NSString *mode = [modelStatus[@"mode"] isKindOfClass:[NSString class]] ? modelStatus[@"mode"] : @"broker"; if ([mode isEqualToString:@"bedrock_converse"]) { return OPModelBedrockConverseDecision(modelStatus, requestBody); } + if (OPModelModeIsOpenAIResponses(mode)) { + return OPModelOpenAIResponsesDecision(modelStatus, requestBody); + } return OPModelBrokerDecision(modelStatus, requestBody); } @@ -14213,6 +14812,11 @@ - (void)close { @"Use clipboard_read when the task depends on copied text, clipboard_write when the user asks to copy text, contacts_search when the task needs a saved person, organization, phone, or email, calendar_search when the task needs saved events or schedule context, calls_search when the task needs recent call-history context, and messages_search when the task needs saved SMS/iMessage context. " @"Do not narrate a plan or answer with plain text when a phone tool can progress the task. " @"For broad choices like random, any, best, or surprise me, choose a reasonable visible/default option yourself. " + @"This is a CONTINUOUS voice conversation, not a one-shot task. The live microphone session stays open the entire time and is ended ONLY by the user. " + @"Calling finish_task marks the CURRENT request complete but does NOT end the session — after it you keep listening for the user's next spoken instruction. " + @"Do not treat merely opening an app or a webpage as completion: carry the user's request all the way through (e.g. if asked to order a coffee, search for it, open a product, and proceed toward checkout) before considering that request done. " + @"SPEED RULES (critical — the user is listening live): (1) Do NOT use the wait tool. The runtime already paces actions and hands you a fresh screen after every action; inserting wait only adds dead air. (2) Do NOT call get_screen yourself between actions — a fresh screen observation is delivered automatically after each action, so just act on it. (3) Emit ONE decisive action per turn and keep moving; never stall re-observing the same screen. " + @"NAVIGATION RULE: stay inside the current native app and drive its real UI (tap the search field, type, tap a result). Do NOT escape to a web page via open_url to route around a sticky control — tap it by coordinate or retry in-app instead. " @"Call finish_task only when the visible screen or tool result verifies the requested outcome. " @"Never say Done unless finish_task has been called.%@", realtime2 ? @" Realtime 2 should use low reasoning effort and concise tool selection." : @""]; @@ -14268,15 +14872,16 @@ - (void)close { // Streaming session config: accepts pcm16 mic audio and lets the server run // voice-activity detection so end-of-turn is decided server-side instead of by -// our local record-then-transcribe VAD. Output stays text so the existing tool -// loop is unchanged; the agent speaks through the island, not TTS. +// our local record-then-transcribe VAD. Output is audio so the agent speaks +// back through the phone speaker (voice in + voice out + screen); the audio +// transcript still drives the island text and the tool loop is unchanged. static NSDictionary *OPRealtimeStreamingSessionUpdateEvent(NSString *mode, NSString *model, long long sampleRateHz) { NSMutableDictionary *session = [@{ @"type": @"realtime", @"model": model ?: @"", @"instructions": OPRealtimeInstructions(mode ?: @"openai_realtime2"), - @"output_modalities": @[@"text"], + @"output_modalities": @[@"audio"], @"tool_choice": @"auto", @"tools": OPRealtimeToolDefinitions(), @"audio": @{ @@ -14285,6 +14890,14 @@ - (void)close { @"type": @"audio/pcm", @"rate": @(sampleRateHz > 0 ? sampleRateHz : 16000) }, + // Ask the server to transcribe the user's own speech. Without + // this the loop never receives + // conversation.item.input_audio_transcription.completed events, + // so the island transcript stays blank and there's no way to + // confirm the mic stream actually carried intelligible audio. + @"transcription": @{ + @"model": @"gpt-4o-mini-transcribe" + }, @"turn_detection": @{ @"type": @"server_vad", @"threshold": @0.5, @@ -14292,6 +14905,13 @@ - (void)close { @"silence_duration_ms": @700, @"create_response": @YES } + }, + @"output": @{ + @"format": @{ + @"type": @"audio/pcm", + @"rate": @(OP_RT_OUTPUT_SAMPLE_RATE_HZ) + }, + @"voice": @"marin" } } } mutableCopy]; @@ -14497,10 +15117,145 @@ static void OPRealtimeCollectResponseDone(NSDictionary *response, NSMutableArray return OPError([NSString stringWithFormat:@"realtime_wait_timeout:%@", expectedType ?: @""]); } -static NSDictionary *OPRealtimeWaitForTurn(OPRealtimeWebSocket *socket, long long timeoutMs) { - NSMutableArray *calls = [NSMutableArray array]; +// Output audio player: the Realtime server streams pcm16 model speech as +// base64 response.audio.delta events. We feed those bytes into a playback +// AudioQueue so the agent literally speaks back through the phone speaker. +// Buffers recycle through a free list; barge-in flushes the queue so the user +// can interrupt mid-sentence. +#define OP_RT_OUTPUT_BUFFER_BYTES 4800 // ~100ms of 24kHz mono pcm16 +#define OP_RT_OUTPUT_MAX_BUFFERS 32 + +typedef struct { + AudioQueueRef queue; + pthread_mutex_t mutex; + AudioQueueBufferRef freeBuffers[OP_RT_OUTPUT_MAX_BUFFERS]; + int freeCount; + int allocatedCount; + BOOL disposed; + UInt32 sampleRate; +} OPRealtimeAudioPlayer; + +static void OPRealtimeAudioOutputCallback(void *userData, AudioQueueRef queue, + AudioQueueBufferRef buffer) { + (void)queue; + OPRealtimeAudioPlayer *player = (OPRealtimeAudioPlayer *)userData; + if (!player || !buffer) { + return; + } + pthread_mutex_lock(&player->mutex); + if (!player->disposed && player->freeCount < OP_RT_OUTPUT_MAX_BUFFERS) { + player->freeBuffers[player->freeCount++] = buffer; + } + pthread_mutex_unlock(&player->mutex); +} + +static NSString *OPRealtimeAudioPlayerStart(OPRealtimeAudioPlayer *player, UInt32 sampleRate) { + memset(player, 0, sizeof(*player)); + pthread_mutex_init(&player->mutex, NULL); + player->sampleRate = sampleRate > 0 ? sampleRate : OP_RT_OUTPUT_SAMPLE_RATE_HZ; + AudioStreamBasicDescription format; + memset(&format, 0, sizeof(format)); + format.mSampleRate = player->sampleRate; + format.mFormatID = kAudioFormatLinearPCM; + format.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + format.mBytesPerPacket = 2; + format.mFramesPerPacket = 1; + format.mBytesPerFrame = 2; + format.mChannelsPerFrame = 1; + format.mBitsPerChannel = 16; + OSStatus status = AudioQueueNewOutput(&format, OPRealtimeAudioOutputCallback, + player, NULL, NULL, 0, &player->queue); + if (status != noErr || !player->queue) { + pthread_mutex_destroy(&player->mutex); + return [NSString stringWithFormat:@"audio_queue_new_output_failed:%d", (int)status]; + } + status = AudioQueueStart(player->queue, NULL); + if (status != noErr) { + AudioQueueDispose(player->queue, true); + player->queue = NULL; + pthread_mutex_destroy(&player->mutex); + return [NSString stringWithFormat:@"audio_queue_output_start_failed:%d", (int)status]; + } + return @""; +} + +static void OPRealtimeAudioPlayerEnqueue(OPRealtimeAudioPlayer *player, NSData *pcm) { + if (!player || !player->queue || pcm.length == 0) { + return; + } + const uint8_t *bytes = (const uint8_t *)pcm.bytes; + NSUInteger remaining = pcm.length; + while (remaining > 0) { + pthread_mutex_lock(&player->mutex); + if (player->disposed) { + pthread_mutex_unlock(&player->mutex); + return; + } + AudioQueueBufferRef buffer = NULL; + if (player->freeCount > 0) { + buffer = player->freeBuffers[--player->freeCount]; + } else if (player->allocatedCount < OP_RT_OUTPUT_MAX_BUFFERS) { + OSStatus st = AudioQueueAllocateBuffer(player->queue, + OP_RT_OUTPUT_BUFFER_BYTES, &buffer); + if (st == noErr && buffer) { + player->allocatedCount += 1; + } else { + buffer = NULL; + } + } + pthread_mutex_unlock(&player->mutex); + if (!buffer) { + // All buffers in flight; wait briefly for the output callback to + // recycle one rather than dropping model speech. + usleep(5000); + continue; + } + UInt32 capacity = buffer->mAudioDataBytesCapacity; + UInt32 chunk = (UInt32)MIN((NSUInteger)capacity, remaining); + memcpy(buffer->mAudioData, bytes, chunk); + buffer->mAudioDataByteSize = chunk; + if (AudioQueueEnqueueBuffer(player->queue, buffer, 0, NULL) != noErr) { + pthread_mutex_lock(&player->mutex); + if (player->freeCount < OP_RT_OUTPUT_MAX_BUFFERS) { + player->freeBuffers[player->freeCount++] = buffer; + } + pthread_mutex_unlock(&player->mutex); + return; + } + bytes += chunk; + remaining -= chunk; + } +} + +// Stop and drop any queued/playing model speech (barge-in or turn hand-off). +static void OPRealtimeAudioPlayerFlush(OPRealtimeAudioPlayer *player) { + if (player && player->queue) { + AudioQueueReset(player->queue); + } +} + +static void OPRealtimeAudioPlayerStop(OPRealtimeAudioPlayer *player) { + if (!player || !player->queue) { + return; + } + pthread_mutex_lock(&player->mutex); + player->disposed = YES; + pthread_mutex_unlock(&player->mutex); + AudioQueueStop(player->queue, true); + AudioQueueDispose(player->queue, true); + player->queue = NULL; + pthread_mutex_destroy(&player->mutex); +} + +static NSDictionary *OPRealtimeWaitForTurn(OPRealtimeWebSocket *socket, long long timeoutMs, + OPRealtimeAudioPlayer *player) { + NSMutableArray *calls = [NSMutableArray array]; NSMutableString *finalText = [NSMutableString string]; NSMutableString *inputTranscript = [NSMutableString string]; + // timeoutMs is an *inactivity* window, not a total-turn budget: it resets + // every time an event arrives. A streaming model response (audio deltas + // every ~100ms) therefore never times out mid-sentence, while a fully + // silent socket returns after timeoutMs so the caller can re-check cancel. long long deadlineMs = OPNowMs() + timeoutMs; while (OPNowMs() < deadlineMs) { NSError *error = nil; @@ -14510,6 +15265,9 @@ static void OPRealtimeCollectResponseDone(NSDictionary *response, NSMutableArray return OPError([NSString stringWithFormat:@"realtime_read_failed:%@", error.localizedDescription ?: @"unknown"]); } + // An event arrived: reset the inactivity window so an in-progress + // response (streaming audio deltas) is never cut off mid-turn. + deadlineMs = OPNowMs() + timeoutMs; NSString *type = [event[@"type"] isKindOfClass:[NSString class]] ? event[@"type"] : @""; if ([type isEqualToString:@"error"]) { return @{ @@ -14531,15 +15289,35 @@ static void OPRealtimeCollectResponseDone(NSDictionary *response, NSMutableArray [inputTranscript appendString:transcript]; } } + // Model speech: base64 pcm16 chunks. Play them straight through the + // output AudioQueue so the agent talks back over the phone speaker. + if (player && ([type isEqualToString:@"response.audio.delta"] || + [type isEqualToString:@"response.output_audio.delta"])) { + NSString *b64 = [event[@"delta"] isKindOfClass:[NSString class]] + ? event[@"delta"] : @""; + if (b64.length > 0) { + NSData *pcm = [[NSData alloc] initWithBase64EncodedString:b64 + options:NSDataBase64DecodingIgnoreUnknownCharacters]; + if (pcm.length > 0) { + OPRealtimeAudioPlayerEnqueue(player, pcm); + } + } + } NSDictionary *call = OPRealtimeFunctionCallFromEvent(event); if (call) { OPRealtimeAddOrUpgradeCall(calls, call); } + // In audio-output mode the assistant's words arrive as the audio + // transcript rather than as output_text; collect both so the island + // still shows what the agent said. if ([type isEqualToString:@"response.output_text.done"] || - [type isEqualToString:@"response.text.done"]) { + [type isEqualToString:@"response.text.done"] || + [type isEqualToString:@"response.audio_transcript.done"] || + [type isEqualToString:@"response.output_audio_transcript.done"]) { NSString *text = [event[@"text"] isKindOfClass:[NSString class]] - ? event[@"text"] : ([event[@"content"] isKindOfClass:[NSString class]] - ? event[@"content"] : @""); + ? event[@"text"] : ([event[@"transcript"] isKindOfClass:[NSString class]] + ? event[@"transcript"] : ([event[@"content"] isKindOfClass:[NSString class]] + ? event[@"content"] : @"")); if (text.length > 0) { if (finalText.length > 0) { [finalText appendString:@"\n"]; @@ -14609,9 +15387,13 @@ static void OPRealtimeCollectResponseDone(NSDictionary *response, NSMutableArray } return nil; } - // v3 router decisions don't need a `tool` field — they carry mode/reply - // instead. Short-circuit here so the tool validation below doesn't reject. - if ([schema isEqualToString:@"openphone.model_decision.v3"]) { + // Router decisions carry a `mode`/`reply` instead of a `tool`. Some models + // (e.g. gpt-5.5) emit the router shape but tag it with the v1/v2 schema, so + // key off the `mode` field rather than the schema string. The loop's router + // stage keys off `mode` too, so any mode-bearing decision belongs there. + NSString *routerMode = [decision[@"mode"] isKindOfClass:[NSString class]] + ? decision[@"mode"] : @""; + if ([schema isEqualToString:@"openphone.model_decision.v3"] || routerMode.length > 0) { return decision; } NSString *tool = [decision[@"tool"] isKindOfClass:[NSString class]] ? decision[@"tool"] : @""; @@ -14751,8 +15533,78 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { return decision; } +// The vision model reads tap/swipe coordinates off the screenshot IMAGE, which +// is the device framebuffer downscaled so its longest edge is +// screenshot_max_dimension_px (default 1024). But the HID input layer expects +// display POINTS (e.g. 430x932). Left unscaled, a y that points at the compose +// field in a 1024-tall image lands ~80pt too low (in the dock), the field never +// focuses, and the following type_text fails with target_not_found. Rescale the +// model's raw image-pixel coordinates into points using the actual screenshot +// dimensions the model was shown. tap_element is unaffected: it resolves via +// ui_tree bounds, which are already in points. +static NSDictionary *OPModelRescaleActionForScreenshot(NSDictionary *arguments, + NSString *tool, NSDictionary *screen) { + if (![arguments isKindOfClass:[NSDictionary class]] || arguments.count == 0) { + return arguments ?: @{}; + } + if (![tool isEqualToString:@"tap"] && ![tool isEqualToString:@"long_press"] && + ![tool isEqualToString:@"swipe"]) { + return arguments; + } + NSDictionary *screenshot = nil; + if ([screen isKindOfClass:[NSDictionary class]]) { + id direct = screen[@"screenshot"]; + if ([direct isKindOfClass:[NSDictionary class]]) { + screenshot = direct; + } else { + NSDictionary *context = [screen[@"context"] isKindOfClass:[NSDictionary class]] + ? screen[@"context"] : nil; + if ([context[@"screenshot"] isKindOfClass:[NSDictionary class]]) { + screenshot = context[@"screenshot"]; + } + } + } + double imgWidth = [screenshot[@"width"] respondsToSelector:@selector(doubleValue)] + ? [screenshot[@"width"] doubleValue] : 0.0; + double imgHeight = [screenshot[@"height"] respondsToSelector:@selector(doubleValue)] + ? [screenshot[@"height"] doubleValue] : 0.0; + if (imgWidth <= 0.0 || imgHeight <= 0.0) { + return arguments; + } + NSDictionary *display = OPScreenDisplayInfo(); + double pointWidth = [display[@"point_width"] doubleValue]; + double pointHeight = [display[@"point_height"] doubleValue]; + if (pointWidth <= 0.0 || pointHeight <= 0.0) { + return arguments; + } + // If the image already matches point space (within 2%), the model's numbers + // are already usable — don't touch them. + if (fabs(imgWidth - pointWidth) <= pointWidth * 0.02 && + fabs(imgHeight - pointHeight) <= pointHeight * 0.02) { + return arguments; + } + double scaleX = pointWidth / imgWidth; + double scaleY = pointHeight / imgHeight; + NSMutableDictionary *scaled = [arguments mutableCopy]; + NSArray *xKeys = @[@"x", @"start_x", @"end_x"]; + NSArray *yKeys = @[@"y", @"start_y", @"end_y"]; + for (NSString *key in xKeys) { + double v = 0.0; + if (OPDoubleForKey(scaled, key, &v)) { + scaled[key] = @(v * scaleX); + } + } + for (NSString *key in yKeys) { + double v = 0.0; + if (OPDoubleForKey(scaled, key, &v)) { + scaled[key] = @(v * scaleY); + } + } + return scaled; +} + static NSDictionary *OPModelExecuteDecision(NSDictionary *decision, NSString *taskId, - NSArray *approvedCapabilities) { + NSArray *approvedCapabilities, NSDictionary *screen) { NSString *tool = decision[@"tool"] ?: @""; NSString *capability = OPModelToolCapability(tool); if (![approvedCapabilities containsObject:capability]) { @@ -14842,7 +15694,7 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { @"source": @"openphone.agentd" }; } - NSMutableDictionary *action = [arguments mutableCopy]; + NSMutableDictionary *action = [OPModelRescaleActionForScreenshot(arguments, tool, screen) mutableCopy]; action[@"type"] = tool; return OPExecuteAction(@{ @"command": @"execute_action", @@ -15128,7 +15980,7 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { @"updated_at_ms": @(OPNowMs()) } }); - NSDictionary *turn = OPRealtimeWaitForTurn(socket, timeoutMs); + NSDictionary *turn = OPRealtimeWaitForTurn(socket, timeoutMs, NULL); if (![turn[@"status"] isEqualToString:@"ok"]) { toolErrors += 1; stopReason = turn[@"reason"] ?: @"realtime_turn_failed"; @@ -15256,7 +16108,7 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { }); OPIslandPublishToolStep(taskId, tool, @"tool_running", step, maxSteps); - NSDictionary *toolResult = OPModelExecuteDecision(decision, taskId, approved); + NSDictionary *toolResult = OPModelExecuteDecision(decision, taskId, approved, screen); lastToolResult = toolResult ?: @{}; NSString *state = [toolResult[@"state"] isKindOfClass:[NSString class]] ? toolResult[@"state"] : @""; @@ -15597,12 +16449,49 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { // we drain them one by one *without* re-prompting the model. Massively // reduces roundtrips + eliminates the source of loop bugs. NSMutableArray *routerActionQueue = [NSMutableArray array]; + long long providerRetries = 0; NSString *routerReply = @""; NSString *routerLastScreenSignature = @""; long long consecutiveNoProgress = 0; BOOL terminal = NO; BOOL succeeded = NO; BOOL cancelled = NO; + // For send-message goals: track whether the model has actually entered body + // text yet, so we can reject a hallucinated finish_task that claims the + // message was sent without ever typing it. + BOOL sendGoalTypedBody = NO; + BOOL isSendMessageGoal = OPGoalIsSendMessage(goal); + long long finishBlockedCount = 0; + // A type_text into an unfocused screen is a recoverable mistake (the model + // just needs to tap the field first), so give it a couple of soft retries + // that steer instead of instantly burning the hard tool-error budget. + long long noFocusedFieldCount = 0; + BOOL lastTypeTextNoFocusedField = NO; + // Hallucinated element_ids are likewise recoverable: re-prompt with the real + // element list rather than instantly failing the task. + long long elementNotFoundCount = 0; + BOOL lastElementNotFound = NO; + + // Deterministically resolve the contact for a "call " goal before we + // ever hit the model. gpt-5.5 is unreliable at deciding to look a number up + // (it blind-taps the current screen or refuses), but it dials reliably once + // the tel: URL is in front of it. So we seed contacts_search as step 1; the + // resolved dial_urls then flow back to the model via last_tool_result. + NSString *callContactName = OPCallGoalContactName(goal); + BOOL callDialEnqueued = NO; + if (callContactName.length > 0) { + [routerActionQueue addObject:@{ + @"tool": @"contacts_search", + @"arguments": @{ + @"query": callContactName, + @"limit": @(10), + @"reason": @"seeded: resolve number for call goal" + } + }]; + OPRecordTrajectory(taskId, @"call_goal_contact_search_seeded", @{ + @"query_length": @(callContactName.length) + }); + } for (long long step = 1; step <= maxSteps; step++) { @autoreleasepool { @@ -15651,15 +16540,30 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { // Loop guidance: tell the model exactly what it did last and, // if that action already succeeded, that it should finish now. NSString *guidance = @""; - if ([lastModelTool isEqualToString:@"get_screen"]) { + if ([lastModelTool isEqualToString:@"same_action_nudged"]) { + guidance = @"HARD RULE: you just issued the EXACT same action twice in a row, which made zero progress. You already have the information/result you need from the previous identical call — it is in last_tool_result. Do NOT run that tool again. Choose a DIFFERENT action that actually advances the goal (e.g. to place a call, use open_url with a tel: URL of the number you already found). If the goal is truly complete and visible on screen, call finish_task. If you cannot proceed, call fail_task — never repeat the same action a third time."; + } else if ([lastModelTool isEqualToString:@"finish_task_rejected"]) { + guidance = @"HARD RULE: you tried to finish but you have NOT typed the message body yet, so nothing has been sent. Do the next REAL step: if the text field isn't focused, tap it; then type_text the message body; then tap the Send button (up-arrow); only call finish_task AFTER the sent bubble is visible in the transcript. Never claim you sent a message you didn't type."; + } else if (lastTypeTextNoFocusedField) { + guidance = @"HARD RULE: your last type_text did NOTHING because no text field was focused — the keystrokes were dropped and the field is still empty. You CANNOT type into a field you have not focused. Next step MUST be a tap on the message text field to give it keyboard focus (use tap with the field's x,y from the CURRENT screen, or tap_element on the text_field element). The keyboard must appear. Only AFTER the field is focused should you type_text the body. Do NOT call type_text again this step."; + } else if (lastElementNotFound) { + guidance = @"HARD RULE: your last tap_element used an element_id that does NOT exist on the current screen, so nothing happened. Do NOT invent element ids. Look at context.interactive_elements in THIS prompt and either use an EXACT id copied from that list, or if the target isn't listed, use a raw `tap` at the x,y of the visible target instead. Choose a different action than the one that just failed."; + } else if (isSendMessageGoal && !sendGoalTypedBody && + OPScreenHasFocusedEditableField(screen)) { + guidance = @"HARD RULE: the message text field is ALREADY focused and the keyboard is up. Do NOT tap it again. Your next action MUST be type_text with the message body. After the body text appears in the field, tap the Send button (up-arrow)."; + } else if ([lastModelTool isEqualToString:@"get_screen"]) { guidance = @"HARD RULE: last_tool was get_screen. You MUST choose an action tool (tap, tap_element, type_text, open_url, open_app, swipe, home, finish_task, fail_task) this step. Do NOT choose get_screen again — the observation is already in this prompt."; + } else if ([lastModelTool hasSuffix:@"_search"]) { + guidance = [NSString stringWithFormat: + @"last_tool was %@ and its results are in last_tool_result. Do NOT search again — you already have the data. Use those results to take the NEXT real action toward the goal. If several contacts matched, PICK THE SINGLE BEST match by name and act on it — do NOT give up just because there is more than one result. Skip any contact that is the user themselves (e.g. a name containing 'me'). Prefer an entry whose name most closely matches the request (e.g. a 'Main'/primary entry over a labeled variant). To place a call, take that contact's `dial_urls[0]` value from last_tool_result and pass it VERBATIM into open_url (e.g. {\"url\":\"tel:+14155550123\"}) — the daemon already validated and formatted it, so a `dial_urls` value is ALWAYS a usable number; never declare it unusable. To send a message, open Messages and follow the send flow. Only call fail_task if the matched contact has NO phone_numbers and NO dial_urls at all.", + lastModelTool]; } else if (lastModelTool.length > 0) { NSString *lastState = [lastToolResult[@"state"] isKindOfClass:[NSString class]] ? lastToolResult[@"state"] : @""; if ([lastState isEqualToString:@"action.executed"] || [lastState isEqualToString:@"task.finished"]) { guidance = [NSString stringWithFormat: - @"HARD RULE: last_tool was %@ and it EXECUTED SUCCESSFULLY. Do NOT repeat the same action. The user's request is now satisfied by that action. Call finish_task with a short human summary of what you did.", + @"last_tool was %@ and it executed successfully. Look at the CURRENT screen in this prompt and decide the next step toward the goal. Do NOT repeat the identical action. If — and only if — the goal is now fully accomplished (e.g. a message was actually sent, not merely composed), call finish_task with a short human summary. Otherwise continue with the next action tool needed to complete the goal.", lastModelTool]; } } @@ -15716,6 +16620,22 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { @"reason": brokerResponse[@"reason"] ?: @"" }); if (![brokerResponse[@"status"] isEqualToString:@"ok"]) { + // Transient empty responses from reasoning models (all reasoning + // budget consumed, no visible text) are retryable — re-prompt + // instead of failing the whole task. Bounded by providerRetries. + NSString *provReason = [brokerResponse[@"reason"] isKindOfClass:[NSString class]] + ? brokerResponse[@"reason"] : @""; + if ([provReason isEqualToString:@"openai_responses_empty_text"] && + providerRetries < 2) { + providerRetries += 1; + OPRecordTrajectory(taskId, @"model_provider_retry", @{ + @"step": @(step), + @"reason": provReason, + @"attempt": @(providerRetries) + }); + step -= 1; + continue; + } toolErrors += 1; lastToolResult = brokerResponse ?: @{}; stopReason = @"model_provider_error"; @@ -15799,10 +16719,15 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { synth[@"confidence"] = @(0.95); decision = synth; } else { - // mode=act: pull first proposed_action as this step's decision, - // queue the rest, and ALWAYS append a synthetic finish_task at - // the end so the loop terminates after the last planned action - // — no re-prompting the model, which is where loops happen. + // mode=act: execute only the FIRST proposed action this step, + // then let the loop re-prompt the model with the resulting + // screen so it can decide the next step. We deliberately do NOT + // pre-plan the remaining proposed_actions: the model planned + // them blind (before observing the screens they act on), so + // tap/type coordinates for not-yet-visible UI would be wrong. + // Multi-step UI flows (e.g. compose + send a message) need the + // model to see each new screen. Loop safety comes from the + // no-progress, same-action, and step/duration guards below. NSDictionary *first = proposed.firstObject; if ([first isKindOfClass:[NSDictionary class]]) { NSMutableDictionary *synth = [NSMutableDictionary dictionary]; @@ -15814,18 +16739,6 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { synth[@"confidence"] = @(0.95); decision = synth; [routerActionQueue removeAllObjects]; - for (NSUInteger i = 1; i < proposed.count; i++) { - id a = proposed[i]; - if ([a isKindOfClass:[NSDictionary class]]) { - [routerActionQueue addObject:a]; - } - } - // Sentinel finish_task at end of the queue. - NSString *finishMsg = reply.length > 0 ? reply : @"Done."; - [routerActionQueue addObject:@{ - @"tool": @"finish_task", - @"arguments": @{@"summary": finishMsg} - }]; } } } @@ -15918,31 +16831,47 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { lastDecisionRepeats = 0; } lastDecisionSig = sig; - if (lastDecisionRepeats >= 1) { - // 2nd identical action in a row → treat previous run as successful - // and finish. Better UX than looping forever. - NSString *msg = @"Done."; - NSString *url = [decArgs[@"url"] isKindOfClass:[NSString class]] - ? decArgs[@"url"] : @""; - if (url.length > 0) { - msg = [NSString stringWithFormat:@"Opened %@.", url]; - } - OPRecordTrajectory(taskId, @"model_loop_guard_same_action", @{ + if (lastDecisionRepeats == 1) { + // First identical repeat: the model is stuck re-issuing the same + // action instead of advancing. Repeating it will not make progress, + // so nudge it with a corrective observation and let it choose a + // different move. Do NOT fabricate success here — a stuck agent has + // not completed the goal. + OPRecordTrajectory(taskId, @"model_loop_guard_same_action_nudge", @{ @"step": @(step), @"tool": tool, @"arguments": decArgs, @"reason": @"same_action_repeated" }); lastToolResult = @{ - @"status": @"ok", - @"state": @"task.finished", + @"status": @"error", + @"state": @"action.no_progress", + @"task_id": taskId ?: @"", + @"reason": @"You just issued this exact action again with no change. Repeating it will not make progress. Choose a DIFFERENT next step to actually advance the goal. If the goal is complete AND verified on screen, call finish_task. If you cannot make progress, call fail_task — do NOT claim the task is done when it is not.", + @"source": @"openphone.agentd" + }; + lastModelTool = @"same_action_nudged"; + continue; + } + if (lastDecisionRepeats >= 2) { + // Still repeating after the nudge: the agent is genuinely stuck. + // Report an honest failure rather than a fabricated "Done." + OPRecordTrajectory(taskId, @"model_loop_guard_same_action", @{ + @"step": @(step), + @"tool": tool, + @"arguments": decArgs, + @"reason": @"stuck_repeated_action" + }); + lastToolResult = @{ + @"status": @"error", + @"state": @"task.failed", @"task_id": taskId ?: @"", - @"summary": msg, + @"reason": @"Stuck repeating the same action without progress; the goal was not completed.", @"source": @"openphone.agentd" }; terminal = YES; - succeeded = YES; - stopReason = @"same_action_repeated"; + succeeded = NO; + stopReason = @"stuck_repeated_action"; break; } OPRecordTrajectory(taskId, @"tool_call", @{ @@ -16019,7 +16948,7 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { @"tool": tool }); } else { - toolResult = OPModelExecuteDecision(decision, taskId, approved); + toolResult = OPModelExecuteDecision(decision, taskId, approved, screen); } lastToolResult = toolResult ?: @{}; NSString *state = [toolResult[@"state"] isKindOfClass:[NSString class]] @@ -16028,7 +16957,19 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { [state isEqualToString:@"action.executed"] || [state isEqualToString:@"task.finished"] || [state isEqualToString:@"task.failed"]; - if (!toolOK || [state hasPrefix:@"action.denied"] || + BOOL noFocusedField = [state isEqualToString:@"action.denied.no_focused_field"]; + BOOL elementNotFound = [state isEqualToString:@"action.denied.element_not_found"]; + lastTypeTextNoFocusedField = noFocusedField; + lastElementNotFound = elementNotFound; + if (noFocusedField && noFocusedFieldCount < 2) { + // Soft error: steer the model to tap the field first without + // spending its hard error budget. + noFocusedFieldCount += 1; + } else if (elementNotFound && elementNotFoundCount < 2) { + // Soft error: the model guessed an element_id that isn't on screen. + // Re-prompt with the real element list instead of failing. + elementNotFoundCount += 1; + } else if (!toolOK || [state hasPrefix:@"action.denied"] || [toolResult[@"status"] isEqualToString:@"error"]) { toolErrors += 1; } @@ -16094,6 +17035,68 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { } lastModelTool = tool; + // Remember a successful body-text entry for send-message goals. + if (isSendMessageGoal && [tool isEqualToString:@"type_text"] && toolOK) { + NSDictionary *typeArgs = [decision[@"arguments"] isKindOfClass:[NSDictionary class]] + ? decision[@"arguments"] : @{}; + NSString *typed = [typeArgs[@"text"] isKindOfClass:[NSString class]] + ? typeArgs[@"text"] : @""; + if (typed.length > 0) { + sendGoalTypedBody = YES; + } + } + + // Guard against a hallucinated finish on send-message goals: if the + // model tries to finish before ever typing the body, reject it and push + // it to keep going. Give it two nudges before giving up. + if (isSendMessageGoal && [tool isEqualToString:@"finish_task"] && + !sendGoalTypedBody && finishBlockedCount < 2) { + finishBlockedCount += 1; + OPRecordTrajectory(taskId, @"model_finish_blocked_no_body", @{ + @"step": @(step), + @"attempt": @(finishBlockedCount), + @"reason": @"finish_task before body text was typed on a send-message goal" + }); + lastToolResult = @{ + @"status": @"error", + @"state": @"task.finish_rejected", + @"task_id": taskId ?: @"", + @"reason": @"You have not typed the message body yet. Tap the text field and type the message, tap Send, and only finish AFTER the sent bubble appears. Do NOT claim it was sent.", + @"source": @"openphone.agentd" + }; + lastModelTool = @"finish_task_rejected"; + continue; + } + + // Daemon-owned dialing for name-based call goals: once our seeded + // contacts_search resolves, pick the best number and enqueue the + // open_url tel: action deterministically. gpt-5.5 will not translate + // contact rows into a dial on its own (it blind-taps the current + // screen), so we do not leave that decision to the model. + if (callContactName.length > 0 && !callDialEnqueued && + [tool isEqualToString:@"contacts_search"] && + [toolResult[@"status"] isEqualToString:@"ok"]) { + NSArray *resolved = [toolResult[@"contacts"] isKindOfClass:[NSArray class]] + ? toolResult[@"contacts"] : @[]; + NSString *dialURL = OPBestCallDialURL(resolved, callContactName); + if (dialURL.length > 0) { + callDialEnqueued = YES; + [routerActionQueue addObject:@{ + @"tool": @"open_url", + @"arguments": @{ + @"url": dialURL, + @"reason": @"seeded: dial resolved contact for call goal" + } + }]; + routerReply = [NSString stringWithFormat:@"Calling %@.", callContactName]; + OPRecordTrajectory(taskId, @"call_goal_dial_enqueued", @{ + @"step": @(step), + @"contact_count": @(resolved.count) + }); + continue; + } + } + if (OPModelVerifiedTypeTextCompletesGoal(goal, decision, verification)) { terminal = YES; succeeded = YES; @@ -16144,9 +17147,25 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { if (consecutiveNoProgress >= 2) { OPRecordTrajectory(taskId, @"model_no_progress_by_signature", @{ @"step": @(step), - @"signature": newSig + @"signature": newSig, + @"send_goal_typed_body": @(sendGoalTypedBody) }); terminal = YES; + // A send-message goal that stalls before the body was ever typed + // did NOT succeed — never report a message as sent when nothing + // was entered. + if (isSendMessageGoal && !sendGoalTypedBody) { + succeeded = NO; + stopReason = @"stalled_before_body_typed"; + lastToolResult = @{ + @"status": @"error", + @"state": @"task.failed", + @"task_id": taskId ?: @"", + @"reason": @"Stalled without ever typing the message body, so nothing was sent.", + @"source": @"openphone.agentd" + }; + break; + } succeeded = YES; stopReason = @"no_progress_signature"; if (routerReply.length > 0) { @@ -16612,7 +17631,7 @@ static id OPModelJSONObjectFromString(NSString *string, NSString **errorOut) { @"goal": goal, @"reason": reason, @"mode": @"model", - @"max_steps": @(OPLongLongFromRequest(request, @"max_steps", 5, 1, 25)), + @"max_steps": @(OPLongLongFromRequest(request, @"max_steps", 12, 1, 25)), @"max_duration_ms": @(OPLongLongFromRequest(request, @"max_duration_ms", 120000, 1000, 600000)), @"source": source ?: @"hardware_trigger", @"trigger_task_id": taskId @@ -16840,6 +17859,63 @@ static void OPVoiceAppendLE32(NSMutableData *data, uint32_t value) { #endif } +// Full-duplex audio session for realtime streaming voice (mic in + speaker out +// at the same time). VoiceChat mode enables acoustic echo cancellation so the +// agent's own speech does not feed back into the mic and falsely trigger +// barge-in. Measurement mode (used by the record-then-transcribe path) has no +// AEC and is wrong for simultaneous playback. +static NSString *OPVoiceActivateDuplexAudioSession(void) { +#if TARGET_OS_IPHONE + NSError *error = nil; + AVAudioSession *session = [AVAudioSession sharedInstance]; + if (!session) { + return @"audio_session_unavailable"; + } + AVAudioSessionCategoryOptions options = + AVAudioSessionCategoryOptionAllowBluetoothHFP | + AVAudioSessionCategoryOptionDefaultToSpeaker; + if (![session setCategory:AVAudioSessionCategoryPlayAndRecord + mode:AVAudioSessionModeVoiceChat + options:options + error:&error]) { + return [NSString stringWithFormat:@"set_category_failed:%@", + error.localizedDescription ?: @"unknown"]; + } + error = nil; + [session setPreferredSampleRate:24000.0 error:&error]; + error = nil; + [session setActive:YES error:&error]; + if (error) { + return [NSString stringWithFormat:@"set_active_failed:%@", + error.localizedDescription ?: @"unknown"]; + } + return @""; +#else + return @"audio_session_unavailable_on_macos"; +#endif +} + +// Tear the shared audio session down after a capture/stream ends. Without this +// the session stays active and holds the mic, so the NEXT trigger records near +// silence (peak_rms ~= ambient floor, heard_speech=false). Notify other apps so +// ducked audio resumes. Best-effort: errors are logged, not fatal. +static void OPVoiceDeactivateAudioSession(void) { +#if TARGET_OS_IPHONE + AVAudioSession *session = [AVAudioSession sharedInstance]; + if (!session) { + return; + } + NSError *error = nil; + [session setActive:NO + withOptions:AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation + error:&error]; + if (error) { + OPLog(@"audio session deactivate failed: %@", + error.localizedDescription ?: @"unknown"); + } +#endif +} + static NSData *OPVoiceRecordWAV(NSDictionary *request, NSDictionary **metadataOut, NSString **errorOut) { NSString *sessionWarning = OPVoiceActivateAudioSession(); @@ -16949,6 +18025,11 @@ static void OPVoiceAppendLE32(NSMutableData *data, uint32_t value) { if (OPVoiceCancelRequested) { OPVoiceSetDoneLocked(&state, "cancelled"); } + // Push-to-talk: the user released the long-press. Stop the mic now and + // keep the audio for transcription (distinct from a cancel). + if (OPVoiceStopRequested && elapsedMs >= minRecordMs) { + OPVoiceSetDoneLocked(&state, "push_to_talk_release"); + } } NSString *stopReason = [NSString stringWithUTF8String:state.stopReason]; BOOL heardSpeech = state.heardSpeech; @@ -16961,6 +18042,7 @@ static void OPVoiceAppendLE32(NSMutableData *data, uint32_t value) { AudioQueueDispose(state.queue, true); pthread_mutex_destroy(&state.mutex); pthread_cond_destroy(&state.cond); + OPVoiceDeactivateAudioSession(); NSData *wav = OPVoiceWAVDataFromPCM16(pcm, sampleRate); if (metadataOut) { @@ -17362,8 +18444,16 @@ static void OPRealtimeStreamInputCallback(void *userData, AudioQueueRef queue, // Start a mic AudioQueue that streams pcm16 to the socket. Returns "" on // success or an error string. Caller owns the returned queue via state. static NSString *OPRealtimeStreamStart(OPRealtimeStreamState *state) { - NSString *sessionWarning = OPVoiceActivateAudioSession(); - (void)sessionWarning; + NSString *sessionWarning = OPVoiceActivateDuplexAudioSession(); + if (sessionWarning.length > 0) { + OPLog(@"realtime duplex audio session warning: %@", sessionWarning); + } else { + AVAudioSession *s = [AVAudioSession sharedInstance]; + OPLog(@"realtime duplex audio session active: inputAvailable=%d inputs=%lu sampleRate=%.0f", + (int)s.isInputAvailable, + (unsigned long)s.availableInputs.count, + s.sampleRate); + } AudioStreamBasicDescription format; memset(&format, 0, sizeof(format)); format.mSampleRate = state->sampleRate; @@ -17412,6 +18502,7 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { AudioQueueStop(state->queue, true); AudioQueueDispose(state->queue, true); state->queue = NULL; + OPVoiceDeactivateAudioSession(); } // Drive the streaming realtime voice loop. Mirrors OPRunOpenAIRealtimeTask's @@ -17426,6 +18517,17 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { NSDictionary *config = OPModelConfig(); NSString *model = [modelStatus[@"model"] isKindOfClass:[NSString class]] ? modelStatus[@"model"] : OPModelEffectiveModel(config); + // The short volume gesture forces streaming even when the configured model + // mode is a text harness (e.g. openai_responses/gpt-5.5). In that case the + // configured model is not a Realtime model, so hardwire the realtime2 model + // and mode here rather than trying to open a Realtime socket to gpt-5.5. + BOOL configuredRealtime = [mode isEqualToString:@"openai_realtime"] || + [mode isEqualToString:@"openai_realtime2"]; + if (!configuredRealtime) { + mode = @"openai_realtime2"; + model = OPOpenAIRealtime2Model; + config = @{}; // ignore text-harness endpoint override; use default wss URL + } // Realtime is always OpenAI, so prefer the OpenAI voice credential. Only // fall back to the model credential when a realtime2 model mode is actually // configured (in which case model-credential.json holds the OpenAI key). @@ -17436,8 +18538,11 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { long long timeoutMs = [modelStatus[@"timeout_ms"] respondsToSelector:@selector(longLongValue)] ? [modelStatus[@"timeout_ms"] longLongValue] : 30000; long long maxSteps = OPLongLongFromRequest(request, @"max_steps", 25, 1, 120); + // Realtime is an open-ended conversation, not a one-shot task: it stays live + // until the user squeezes again (voice_cancel). maxDurationMs is only a + // safety ceiling so a lost cancel can't hold the mic forever. long long maxDurationMs = OPLongLongFromRequest(request, @"max_duration_ms", - 600000, 1000, 3300000); + 3300000, 1000, 3300000); // OpenAI Realtime requires the input PCM rate >= 24000 Hz. UInt32 sampleRate = (UInt32)OPLongLongFromRequest(request, @"sample_rate_hz", 24000, 24000, 48000); @@ -17506,6 +18611,16 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { return OPError(streamStartError); } + // Output playback: the model's pcm16 speech is enqueued here as + // response.audio.delta events arrive during each turn. + OPRealtimeAudioPlayer player; + NSString *playerStartError = OPRealtimeAudioPlayerStart(&player, OP_RT_OUTPUT_SAMPLE_RATE_HZ); + BOOL playerActive = playerStartError.length == 0; + if (!playerActive) { + OPRecordTrajectory(taskId, @"realtime_audio_output_unavailable", + @{@"reason": playerStartError ?: @"unknown"}); + } + // Yellow realtime island: server-VAD is now listening on the live mic. OPIslandReset(@"realtime", @"Listening", @"yellow"); OPIslandUpdate(@{@"task_id": taskId ?: @""}); @@ -17521,7 +18636,20 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { NSDictionary *screen = @{}; NSString *cancelReason = @""; - for (long long step = 1; step <= maxSteps; step++) { + // Open-ended conversation loop. There is no "conversation over" for a + // Realtime session: it keeps listening across silent gaps and only ends + // when the user squeezes again (voice_cancel), the socket genuinely drops, + // or the safety duration ceiling is hit. `step` counts model turns for + // island/trajectory display but never terminates the loop. + for (long long step = 1; ; step++) { + // Drain autoreleased per-turn allocations (screenshots, decoded PCM, JSON + // dicts, transcripts) at the end of every turn. Without this the + // open-ended session accumulates every turn's temporaries until it ends, + // crossing the daemon's per-process jetsam limit and getting SIGKILLed + // mid-conversation (JETSAM_REASON_MEMORY_PERPROCESSLIMIT). ARC keeps the + // outer __strong vars assigned inside (screen, lastTranscript, stopReason) + // alive across the drain, so only genuine per-turn garbage is freed. + @autoreleasepool { if (OPVoiceCancelRequested || OPTaskCancellationRequested(taskId, &cancelReason)) { cancelled = YES; stopReason = @"cancelled"; @@ -17533,30 +18661,47 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { } stepsUsed = step; - // Wait for a server-VAD-delimited model turn. Long timeout: the user - // may take a while to start speaking. + // Wait for a server-VAD-delimited model turn. Short timeout so we + // re-check the cancel flag frequently; a timeout here just means the + // user was silent, so we loop and keep listening. + // + // Keep barge-in ARMED the whole time (not disarmed per turn). The mic + // callback streams pcm16 to the socket continuously regardless, so the + // server always hears the user; arming barge-in only controls whether a + // sustained loud RMS cancels the in-flight response and flushes queued + // agent speech. Leaving it armed through the wait AND through tool + // execution is what lets the user actually interrupt mid-action — the + // whole point of realtime. We only clear the *detection* state here so a + // stale detection from a prior turn doesn't fire. pthread_mutex_lock(&stream.mutex); - stream.bargeInArmed = NO; + stream.bargeInArmed = YES; stream.bargeInStartMs = 0; stream.bargeInDetected = 0; pthread_mutex_unlock(&stream.mutex); - NSDictionary *turn = OPRealtimeWaitForTurn(socket, MAX(timeoutMs, 60000)); + // Short inactivity window: the wait resets on every socket event, so an + // active response never times out, but a silent socket returns quickly + // (~1.5s) so we can re-check the cancel flag and stay responsive to the + // user squeezing again to end the conversation. + NSDictionary *turn = OPRealtimeWaitForTurn(socket, 1500, + playerActive ? &player : NULL); if (![turn[@"status"] isEqualToString:@"ok"]) { - // A read/turn timeout with no speech is a natural end of conversation, - // not an error. The socket's own receive timeout surfaces as - // "realtime_read_failed:realtime_receive_timeout"; treat any timeout - // flavor as a graceful idle end. stopReason = [turn[@"reason"] isKindOfClass:[NSString class]] ? turn[@"reason"] : @"realtime_turn_failed"; - if ([stopReason isEqualToString:@"realtime_turn_timeout"] || - [stopReason rangeOfString:@"timeout"].location != NSNotFound) { - stopReason = @"conversation_idle_timeout"; - terminal = YES; - succeeded = YES; - } else { - toolErrors += 1; + BOOL isTimeout = [stopReason isEqualToString:@"realtime_turn_timeout"] || + [stopReason rangeOfString:@"timeout"].location != NSNotFound; + if (isTimeout) { + // Silent gap, not an end of conversation. Re-arm the mic and + // keep the socket open so the user can speak again after a + // pause without the session closing under them. + step--; // don't burn a "step" on an empty listen window + pthread_mutex_lock(&stream.mutex); + stream.bargeInArmed = YES; + pthread_mutex_unlock(&stream.mutex); + continue; } + // Non-timeout failure means the socket actually dropped. + toolErrors += 1; break; } NSString *transcript = [turn[@"input_transcript"] isKindOfClass:[NSString class]] @@ -17583,6 +18728,7 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { continue; } + BOOL bargedMidBatch = NO; for (id value in calls) { if (![value isKindOfClass:[NSDictionary class]]) continue; NSDictionary *call = value; @@ -17591,6 +18737,24 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { stopReason = @"cancelled"; break; } + // Mid-batch interruption: if the user spoke over the agent while it + // was executing a previous action in this same batch, abort the + // remaining queued actions immediately and return to listening. This + // is what makes the agent feel interruptible — the user shouldn't + // have to wait for a multi-action plan to finish before being heard. + if (stream.bargeInDetected) { + OPRecordTrajectory(taskId, @"realtime_barge_in_midbatch", @{@"step": @(step)}); + [socket sendEvent:@{@"type": @"response.cancel"} timeoutMs:timeoutMs error:&error]; + if (playerActive) { + OPRealtimeAudioPlayerFlush(&player); + } + pthread_mutex_lock(&stream.mutex); + stream.bargeInDetected = 0; + stream.bargeInStartMs = 0; + pthread_mutex_unlock(&stream.mutex); + bargedMidBatch = YES; + break; + } NSDictionary *decision = OPRealtimeDecisionFromCall(call); NSDictionary *guardrail = nil; decision = OPModelDecisionByApplyingGuardrails(decision, lastTranscript, step, &guardrail); @@ -17620,7 +18784,22 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { @"reason": @"realtime stream pre-action observation" })); } - NSDictionary *toolResult = OPModelExecuteDecision(decision, taskId, approved); + // Speed: neutralize `wait` in realtime. The runtime already hands the + // model a fresh screen after every action, so a model-inserted + // wait:800ms is pure dead air while the user is listening live. Treat + // it as an executed no-op instead of blocking the loop. + NSDictionary *toolResult; + if ([tool isEqualToString:@"wait"]) { + toolResult = @{ + @"state": @"action.executed", + @"task_id": taskId ?: @"", + @"capability": @"tasks.observe", + @"detail": @"wait:skipped_realtime", + @"source": @"openphone.agentd" + }; + } else { + toolResult = OPModelExecuteDecision(decision, taskId, approved, screen); + } NSString *state = [toolResult[@"state"] isKindOfClass:[NSString class]] ? toolResult[@"state"] : @""; NSError *outputError = nil; @@ -17628,21 +18807,85 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { timeoutMs, &outputError); OPRealtimeSendScreenFollowupIfUseful(socket, tool, toolResult ?: @{}, timeoutMs, &outputError); + // Speed: auto-deliver a fresh screen after each UI-driving action and + // reuse it as the next turn's pre-action observation. Without this the + // model has to spend an entire extra turn calling get_screen to see + // the result of its own tap/type — a full socket round-trip of dead + // air on every step. Capturing it here (and caching into `screen`) + // lets the SPEED RULE "don't call get_screen yourself" actually hold: + // the model always has the current screen in hand. Non-UI tools and + // failed actions don't get a follow-up screen (nothing changed). + BOOL toolExecuted = [state isEqualToString:@"action.executed"] || + [toolResult[@"status"] isEqualToString:@"ok"]; + if (OPModelToolDrivesUI(tool) && toolExecuted && + ![tool isEqualToString:@"get_screen"]) { + NSDictionary *afterScreen = OPModelCompactScreenForLoop(OPGetScreen(@{ + @"task_id": taskId ?: @"", + @"include_screenshot": @YES, + @"include_activity": @YES, + @"include_ui_tree": @YES, + @"compact_trajectory": @YES, + @"reason": @"realtime stream post-action observation" + })); + if (afterScreen.count > 0) { + screen = afterScreen; + NSString *text = [NSString stringWithFormat: + @"Latest iPhone screen observation (after your %@):\n%@", + tool, OPJSONString(OPRedactedObject(afterScreen, 0))]; + OPRealtimeSendUserMessage(socket, text, timeoutMs, &outputError); + } + } else if (![tool isEqualToString:@"wait"]) { + // Non-UI or non-executed tool: next turn should re-observe fresh. + // (A skipped `wait` changed nothing, so keep the cached screen.) + screen = @{}; + } + // In realtime voice mode the *conversation* owns the session, not the + // task. The model calling finish_task/fail_task means "this sub-task + // is done" — it MUST NOT tear down the live mic/socket. Only a user + // squeeze (voice_cancel), a dropped socket, or the safety duration + // ceiling ends a realtime session. So we record the model's terminal + // intent, mark this turn's outcome, and keep listening for the next + // spoken instruction instead of breaking the loop. if ([tool isEqualToString:@"finish_task"] || [state isEqualToString:@"task.finished"]) { - terminal = YES; succeeded = YES; - stopReason = @"finish_task"; - break; + OPRecordTrajectory(taskId, @"realtime_subtask_finished", @{ + @"step": @(step), + @"summary": [decision[@"arguments"] isKindOfClass:[NSDictionary class]] + ? (decision[@"arguments"][@"summary"] ?: @"") : @"" + }); + continue; } if ([tool isEqualToString:@"fail_task"] || [state isEqualToString:@"task.failed"]) { - terminal = YES; - stopReason = @"fail_task"; - break; + OPRecordTrajectory(taskId, @"realtime_subtask_failed", @{ + @"step": @(step), + @"reason": [decision[@"arguments"] isKindOfClass:[NSDictionary class]] + ? (decision[@"arguments"][@"reason"] ?: @"") : @"" + }); + continue; } } if (terminal || cancelled) { break; } + // We just submitted one or more function_call_output items (the loop + // above only runs when calls.count > 0). The Realtime API does NOT + // auto-continue after a tool result: without an explicit response.create + // the model goes silent and the conversation stalls until the user + // cancels. Prompt the model to produce its spoken follow-up turn. An + // empty response payload inherits the session's audio output modality. + // + // Skip this when the user barged in mid-batch: server-VAD will create a + // response for the user's new utterance on its own, and forcing our own + // response.create here would talk over the interruption we just honored. + if (!bargedMidBatch) { + NSError *continueError = nil; + if (![socket sendEvent:@{@"type": @"response.create"} + timeoutMs:timeoutMs error:&continueError]) { + toolErrors += 1; + stopReason = @"realtime_response_create_failed"; + break; + } + } // Post-action: re-arm barge-in and keep the loop alive for follow-ups. pthread_mutex_lock(&stream.mutex); stream.bargeInArmed = YES; @@ -17650,14 +18893,23 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { if (stream.bargeInDetected) { OPRecordTrajectory(taskId, @"realtime_barge_in", @{@"step": @(step)}); [socket sendEvent:@{@"type": @"response.cancel"} timeoutMs:timeoutMs error:&error]; + // Drop any queued/playing model speech so the user's interruption + // is heard immediately instead of talking over the agent. + if (playerActive) { + OPRealtimeAudioPlayerFlush(&player); + } pthread_mutex_lock(&stream.mutex); stream.bargeInDetected = 0; stream.bargeInStartMs = 0; pthread_mutex_unlock(&stream.mutex); } + } // @autoreleasepool (per-turn drain) } OPRealtimeStreamStop(&stream); + if (playerActive) { + OPRealtimeAudioPlayerStop(&player); + } [socket close]; pthread_mutex_destroy(&stream.mutex); @@ -17704,8 +18956,29 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { OPNowMs(), getpid()]; NSString *source = OPStringFromRequest(request, @"source", @"openphone_agentd_voice"); + + // Decide the route (realtime streaming vs record/transcribe harness) + // BEFORE painting the island, so the first state SpringBoard sees for a + // realtime trigger is already "realtime". Publishing "listening" first + // made SpringBoard hide the edge glow (mode != realtime) and then re-show + // it a beat later when the daemon wrote "realtime", causing the glow to + // flash for a frame instead of coming up cleanly on button release. + NSDictionary *voiceModelStatus = OPModelStatusDictionary(); + NSString *voiceMode = [voiceModelStatus[@"mode"] isKindOfClass:[NSString class]] + ? voiceModelStatus[@"mode"] : @""; + BOOL wantStream = OPBoolFromRequest(request ?: @{}, @"stream", NO) || + OPBoolFromRequest(request ?: @{}, @"require_realtime", NO); + // Long-press harness gesture sends stream:false explicitly; honor it so + // the text harness runs even when realtime2 is the configured mode. + BOOL streamOptOut = [request[@"stream"] respondsToSelector:@selector(boolValue)] && + !OPBoolFromRequest(request ?: @{}, @"stream", YES) && + !OPBoolFromRequest(request ?: @{}, @"require_realtime", NO); + BOOL willStream = !streamOptOut && + ([voiceMode isEqualToString:@"openai_realtime2"] || wantStream); + OPVoiceSetLast(@"voice.recording", @"", @"", @"", YES); - OPIslandReset(@"listening", @"Listening", @"red"); + OPIslandReset(willStream ? @"realtime" : @"listening", + @"Listening", willStream ? @"yellow" : @"red"); OPIslandUpdate(@{@"task_id": voiceTaskId ?: @""}); OPRecordContextEvent(@"voice_trigger_started", source, voiceTaskId, @"volume voice trigger", @"daemon microphone capture started", @{ @@ -17717,12 +18990,7 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { // Realtime-2 streaming path: hand the live mic to the OpenAI Realtime // WebSocket (server-VAD end-of-turn), skipping record-then-transcribe. - // Opt out per-request with stream:false for the legacy capture path. - NSDictionary *voiceModelStatus = OPModelStatusDictionary(); - NSString *voiceMode = [voiceModelStatus[@"mode"] isKindOfClass:[NSString class]] - ? voiceModelStatus[@"mode"] : @""; - if ([voiceMode isEqualToString:@"openai_realtime2"] && - OPBoolFromRequest(request ?: @{}, @"stream", YES)) { + if (willStream) { NSDictionary *streamSummary = OPRunStreamingRealtimeVoice(voiceTaskId, request ?: @{}); OPRecordAudit(@"voice_trigger_finished", voiceTaskId, @"background.run", [streamSummary[@"status"] isEqualToString:@"task.finished"] ? @"completed" : @"stopped", @@ -17863,6 +19131,18 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { OPVoiceSetLast(finalState, transcript, started ? @"" : (agentStart[@"model_loop_status"] ?: agentStart[@"state"] ?: @"agent_not_started"), provider, NO); + // When the agent starts, its async model loop drives the island to a + // terminal state. When it does NOT start, the "Thinking" pill set above + // would otherwise hang forever, so clear it here. + if (!started) { + NSString *loopStatus = [agentStart[@"model_loop_status"] isKindOfClass:[NSString class]] + ? agentStart[@"model_loop_status"] : @""; + if ([loopStatus isEqualToString:@"provider_not_ready"]) { + OPIslandReset(@"error", @"Model not configured", @"orange"); + } else { + OPIslandReset(@"error", @"Agent failed to start", @"red"); + } + } NSDictionary *result = @{ @"status": @"ok", @"state": finalState, @@ -17933,12 +19213,52 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { return result; } + // Realtime gesture (double-tap) demands the realtime model. If the daemon is + // not configured for it, fail synchronously with a distinct state so the + // SpringBoard island can surface "Realtime not configured" instead of + // silently falling back to the record/transcribe harness. + if (OPBoolFromRequest(request, @"require_realtime", NO)) { + NSDictionary *rtStatus = OPModelStatusDictionary(); + NSString *rtMode = [rtStatus[@"mode"] isKindOfClass:[NSString class]] + ? rtStatus[@"mode"] : @""; + // The short gesture forces realtime regardless of the configured model + // mode, so the only hard requirement is a usable OpenAI credential (the + // voice credential, or the model credential when a realtime mode is + // already configured). Do not gate on the text-harness mode/status. + NSString *rtCredential = OPVoiceCredentialValue(NULL); + if (rtCredential.length == 0) { + rtCredential = OPModelCredentialValue(); + } + if (rtCredential.length == 0) { + OPVoiceSetLast(@"voice.realtime_not_configured", @"", + @"openai_credential_missing", @"openai_realtime2", NO); + OPIslandReset(@"error", @"Realtime not configured", @"orange"); + NSDictionary *result = @{ + @"status": @"error", + @"state": @"voice.realtime_not_configured", + @"reason": @"openai_credential_missing", + @"model_mode": rtMode ?: @"", + @"model_status": rtStatus[@"status"] ?: @"", + @"runtime_authority": @"phone_local", + @"microphone_owner": @"openphone-agentd", + @"source": @"openphone.agentd" + }; + OPRecordContextEvent(@"voice_trigger_suppressed", @"openphone_agentd_voice", + [NSString stringWithFormat:@"ios-voice-%lld-%d", OPNowMs(), getpid()], + @"realtime gesture", @"openai_credential_missing", result); + OPLog(@"voice trigger suppressed reason=openai_credential_missing mode=%@ status=%@", + rtMode ?: @"", rtStatus[@"status"] ?: @""); + return result; + } + } + pthread_mutex_lock(&OPVoiceTriggerMutex); OPVoiceTriggerRunning = YES; OPVoiceTriggerLastStartedMs = OPNowMs(); OPVoiceTriggerLastState = @"voice.starting"; OPVoiceTriggerLastError = @""; OPVoiceCancelRequested = 0; + OPVoiceStopRequested = 0; pthread_mutex_unlock(&OPVoiceTriggerMutex); NSMutableDictionary *ownedRequest = [request mutableCopy] ?: [NSMutableDictionary dictionary]; @@ -17949,7 +19269,9 @@ static void OPRealtimeStreamStop(OPRealtimeStreamState *state) { ownedRequest[@"max_steps"] = @25; } if (!ownedRequest[@"max_duration_ms"]) { - ownedRequest[@"max_duration_ms"] = @600000; + // Realtime voice is open-ended (stays live until the user squeezes to + // close); use the safety ceiling rather than a 10-minute task budget. + ownedRequest[@"max_duration_ms"] = @3300000; } pthread_t thread; int rc = pthread_create(&thread, NULL, OPAsyncVoiceTriggerMain, @@ -18489,6 +19811,21 @@ static void OPStartVolumeTriggerListener(void) { @"source": @"openphone.agentd" }; } + if ([command isEqualToString:@"voice_stop"] || + [command isEqualToString:@"openphone.voice.stop"]) { + // Push-to-talk release: end mic capture now but KEEP the audio and run + // the transcribe/STT path (unlike voice_cancel, which discards it). + pthread_mutex_lock(&OPVoiceTriggerMutex); + BOOL voiceRunning = OPVoiceTriggerRunning; + OPVoiceStopRequested = 1; + pthread_mutex_unlock(&OPVoiceTriggerMutex); + return @{ + @"status": @"ok", + @"state": voiceRunning ? @"voice.stopping" : @"voice.stop_noop", + @"voice_was_running": @(voiceRunning), + @"source": @"openphone.agentd" + }; + } if ([command isEqualToString:@"voice_cancel"] || [command isEqualToString:@"openphone.voice.cancel"] || [command isEqualToString:@"cancel_active"]) { @@ -18708,22 +20045,68 @@ static void OPStartAppUIIntakeServer(void) { static int32_t OPApplyJetsamPriority(int32_t pid, int32_t requestedBand, int32_t limitMB, int *rcOut, int *errnoOut) { memorystatus_priority_properties_t props; - props.priority = requestedBand > 0 ? requestedBand : JETSAM_PRIORITY_AUDIO_AND_ACCESSORY; props.user_data = 0; - int rc = memorystatus_control(MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES, - pid, 0, &props, sizeof(props)); - if (rc != 0 && props.priority != JETSAM_PRIORITY_FOREGROUND) { - // Fall back to plain foreground band (100). - props.priority = JETSAM_PRIORITY_FOREGROUND; + // Descending ladder: try the requested (high) band first, then fall back to + // successively lower bands the kernel is more willing to grant a mobile-uid + // daemon. We must never end up *lower* than we asked for by accident, so the + // ladder is ordered and we stop at the first band that sticks. + int32_t ladder[] = { + requestedBand > 0 ? requestedBand : JETSAM_PRIORITY_CRITICAL, + JETSAM_PRIORITY_CRITICAL, + JETSAM_PRIORITY_AUDIO_AND_ACCESSORY, + JETSAM_PRIORITY_FOREGROUND + }; + int rc = -1; + for (size_t i = 0; i < sizeof(ladder) / sizeof(ladder[0]); i++) { + // Skip a rung that is >= a band we already failed to set (the requested + // band may itself equal a later rung). + if (i > 0 && ladder[i] >= ladder[0]) continue; + props.priority = ladder[i]; rc = memorystatus_control(MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES, pid, 0, &props, sizeof(props)); + if (rc == 0) break; } if (rcOut) *rcOut = rc; if (errnoOut) *errnoOut = rc != 0 ? errno : 0; if (limitMB > 0) { - int32_t mb = limitMB; - (void)memorystatus_control(MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT, - pid, 0, &mb, sizeof(mb)); + // Lift the per-process memory limit off launchd's fatal 6 MB default. + // Requires the com.apple.private.memorystatus entitlement (we have it). + // Set BOTH active and inactive limits and clear the FATAL bit so that + // exceeding the limit makes us a jetsam candidate under pressure rather + // than an instant SIGKILL. Try the full struct (cmd 7) first; if the + // kernel rejects it, fall back to the bare-int fatal task limit (cmd 6) + // so we at least raise the ceiling above 6 MB. + memorystatus_memlimit_properties_t limit; + memset(&limit, 0, sizeof(limit)); + limit.memlimit_active = limitMB; + limit.memlimit_active_attr = 0; // non-fatal + limit.memlimit_inactive = limitMB; + limit.memlimit_inactive_attr = 0; // non-fatal + int limitRc = memorystatus_control(MEMORYSTATUS_CMD_SET_MEMLIMIT_PROPERTIES, + pid, 0, &limit, sizeof(limit)); + int limitErrno = limitRc != 0 ? errno : 0; + if (limitRc != 0) { + int32_t mb = limitMB; + int taskRc = memorystatus_control(MEMORYSTATUS_CMD_SET_JETSAM_TASK_LIMIT, + pid, 0, &mb, sizeof(mb)); + OPLog(@"jetsam memlimit_properties rc=%d errno=%d; task_limit fallback rc=%d errno=%d limit_mb=%d", + limitRc, limitErrno, taskRc, taskRc != 0 ? errno : 0, (int)limitMB); + } else { + OPLog(@"jetsam memlimit set rc=0 limit_mb=%d (active+inactive, non-fatal)", (int)limitMB); + } + // Read the live limit straight back from the kernel so the log carries + // authoritative proof of the enforced ceiling (not launchd's stale 6 MB). + memorystatus_memlimit_properties_t got; + memset(&got, 0, sizeof(got)); + int getRc = memorystatus_control(MEMORYSTATUS_CMD_GET_MEMLIMIT_PROPERTIES, + pid, 0, &got, sizeof(got)); + if (getRc == 0) { + OPLog(@"jetsam memlimit readback rc=0 active_mb=%d attr=0x%x inactive_mb=%d attr=0x%x", + (int)got.memlimit_active, (unsigned)got.memlimit_active_attr, + (int)got.memlimit_inactive, (unsigned)got.memlimit_inactive_attr); + } else { + OPLog(@"jetsam memlimit readback rc=%d errno=%d", getRc, errno); + } } return rc == 0 ? props.priority : 0; } @@ -18753,13 +20136,22 @@ static int32_t OPApplyJetsamPriority(int32_t pid, int32_t requestedBand, } static void OPRaiseJetsamPriority(void) { - // Long model tasks temporarily hold screenshots + prompt text + HTTP body — - // 256 MB is conservative headroom, still below iPhone's per-process cap. + // Pin to the highest band we can hold (critical). The audio band (12) is + // not enough: when a heavy foreground app launches, iOS reaps band-12 + // daemons as low-priority collateral. Critical keeps us above the reap line. + // + // The per-process task limit is the OTHER half of the fix: launchd was + // relaunching us with `last exit reason = JETSAM_REASON_MEMORY_PERPROCESSLIMIT`, + // i.e. we were crossing our OWN self-imposed 256 MB cap mid voice session. + // The real leak (missing per-turn autoreleasepool in the realtime loop) is + // fixed separately; 1024 MB here is generous headroom for a transient spike + // (screenshots + decoded PCM + HTTP body), still well below the device's + // multi-GB per-process ceiling. int rc = 0, err = 0; int32_t band = OPApplyJetsamPriority(getpid(), - JETSAM_PRIORITY_AUDIO_AND_ACCESSORY, 256, &rc, &err); + JETSAM_PRIORITY_CRITICAL, 1024, &rc, &err); if (band > 0) { - OPLog(@"jetsam priority raised band=%d rc=%d limit_mb=256 via=self", (int)band, rc); + OPLog(@"jetsam priority raised band=%d rc=%d limit_mb=1024 via=self", (int)band, rc); return; } OPLog(@"jetsam priority set failed rc=%d errno=%d; delegating to root helper", rc, err); @@ -18768,8 +20160,8 @@ static void OPRaiseJetsamPriority(void) { NSDictionary *helper = OPProtectedDataHelperRequest(@{ @"command": @"jetsam_priority_set", @"pid": @(getpid()), - @"band": @(JETSAM_PRIORITY_AUDIO_AND_ACCESSORY), - @"limit_mb": @256 + @"band": @(JETSAM_PRIORITY_CRITICAL), + @"limit_mb": @1024 }); if ([helper[@"status"] isEqualToString:@"ok"]) { OPLog(@"jetsam priority raised band=%@ via=root_helper", helper[@"band"] ?: @0); @@ -18791,6 +20183,30 @@ int main(int argc, char **argv) { OPRaiseJetsamPriority(); OPEnsureDirectories(); + + // Reset the Dynamic Island to idle on startup. A crash/jetsam kill mid-run + // leaves a stale non-idle snapshot (e.g. "thinking") on disk; the tweak + // keeps rendering it and treats it as an active turn, so hardware gestures + // get swallowed as cancels. Seed our in-memory sequence from the existing + // file first so the idle write is strictly newer and the tweak accepts it. + if (!OPProtectedDataHelperRole()) { + NSDictionary *staleIsland = OPReadJSONFile(OPIslandStatusPath); + unsigned long long staleSeq = [staleIsland[@"sequence"] isKindOfClass:[NSNumber class]] + ? [staleIsland[@"sequence"] unsignedLongLongValue] : 0; + pthread_mutex_lock(&OPIslandMutex); + if (staleSeq > OPIslandSequence) { + OPIslandSequence = staleSeq; + } + pthread_mutex_unlock(&OPIslandMutex); + OPIslandReset(@"idle", @"", @"cyan"); + + pthread_t heartbeatThread; + if (pthread_create(&heartbeatThread, NULL, + OPIslandHeartbeatThread, NULL) == 0) { + pthread_detach(heartbeatThread); + } + } + OPServerFd = OPCreateServerSocket(); if (OPServerFd < 0) { return 1; @@ -18830,12 +20246,49 @@ int main(int argc, char **argv) { ? task[@"goal"] : @""; NSString *taskId = [task[@"task_id"] isKindOfClass:[NSString class]] ? task[@"task_id"] : @""; + // Never auto-resume interactive voice/streaming sessions. If the + // daemon dies mid-conversation the session is over; re-enqueueing + // the same "Voice conversation" goal just re-runs the path that + // crashed us, and launchd's KeepAlive turns that into a tight + // crash->restart->resume loop that saturates the device (SSH + // handshakes start failing). These are user-driven and cheap to + // just re-trigger by hand, so fail them instead of resuming. + BOOL isVoiceStreaming = + [task[@"stream"] boolValue] || + [task[@"require_realtime"] boolValue] || + [goal rangeOfString:@"Voice conversation" + options:NSCaseInsensitiveSearch].location != NSNotFound; + // Safety net for every other task type: cap resume attempts so a + // task that reliably crashes the daemon on replay can't loop + // forever. One recovery resume is enough for a genuine jetsam. + long long priorResumes = + [task[@"resume_count"] respondsToSelector:@selector(longLongValue)] + ? [task[@"resume_count"] longLongValue] : 0; + if (isVoiceStreaming || priorResumes >= 1) { + OPUpdateTask(taskId, @"failed", @{ + @"stop_reason": isVoiceStreaming + ? @"voice_session_not_resumed_after_restart" + : @"resume_attempts_exhausted", + @"resume_count": @(priorResumes), + @"result": @{ + @"status": @"error", + @"state": @"task.failed", + @"reason": isVoiceStreaming + ? @"voice_session_not_resumed_after_restart" + : @"resume_attempts_exhausted", + @"task_id": taskId, + @"source": @"openphone.agentd" + } + }); + continue; + } if (ageMs <= 60000 && goal.length > 0 && taskId.length > 0) { // Mark original task as resumed so the fail-repair sweep // below skips it (status != "active"), and enqueue fresh // work with the same goal. OPUpdateTask(taskId, @"resumed_after_restart", @{ @"resumed_at_ms": @(nowMs), + @"resume_count": @(priorResumes + 1), @"stop_reason": @"resumed_after_daemon_restart", @"result": @{ @"status": @"ok", diff --git a/ios/agentd/src/volume_trigger.xm b/ios/agentd/src/volume_trigger.xm index 3732e1c..fb2e999 100644 --- a/ios/agentd/src/volume_trigger.xm +++ b/ios/agentd/src/volume_trigger.xm @@ -51,6 +51,32 @@ static long long OPVTLastComboEventMs = 0; static NSString *OPVTLastButtonEventName = nil; static NSString *OPVTLastButtonEventSource = nil; static NSString *OPVTLastTriggerRoute = nil; + +// Diagnostic-only hold/phase probe (T: long-press feasibility on the safe path). +// Classifies press-down vs press-up from the hooked selector name and records +// hold duration. Publishes into trigger-status.json; does not change routing. +static long long OPVTDownPhaseEventCount = 0; +static long long OPVTUpPhaseEventCount = 0; +static long long OPVTUnknownPhaseEventCount = 0; +static CFAbsoluteTime OPVTLastUpDownTime = 0; // last DOWN phase for volume-up button +static CFAbsoluteTime OPVTLastDownDownTime = 0; // last DOWN phase for volume-down button +static double OPVTLastHoldMs = -1.0; +static NSString *OPVTLastHoldButton = nil; +static NSString *OPVTLastPhaseObserved = nil; + +// Gesture discrimination on the safe SpringBoard path. The probe confirmed the +// PressDown and PressUp selectors fire as a clean pair on this device, so we can +// track each button's held state and split one "squeeze both" gesture by how +// long it is held: +// * squeeze both, release < LongPressMs -> realtime streaming +// * squeeze both, hold > LongPressMs -> local record/transcribe harness +static BOOL OPVTUpPressed = NO; +static BOOL OPVTDownPressed = NO; +static BOOL OPVTBothPressed = NO; +static CFAbsoluteTime OPVTBothPressedStart = 0; +static BOOL OPVTLongPressFired = NO; +static long long OPVTGestureGeneration = 0; // bumped to invalidate stale hold timers +static NSString *OPVTLastGesture = nil; static UIWindow *OPVTOverlayWindow = nil; static UIWindow *OPVTPromptWindow = nil; static NSString *OPVTIslandCurrentMode = @"idle"; @@ -165,7 +191,7 @@ static BOOL OPVTForegroundNoArgBoolReplacement(id self, SEL _cmd); static BOOL OPVTShouldLogHookMiss(NSString *phase); static void OPVTRecordButtonFromSource(BOOL volumeUp, NSString *source); static void OPVTPresentTriggerPrompt(void); -static void OPVTCallDaemonVoiceAgent(void); +static void OPVTCallDaemonVoiceAgent(BOOL realtime); static void OPVTCallAgentWithGoal(NSString *requestedGoal, BOOL userProvidedGoal); static void OPVTPublishTriggerStatus(NSString *eventName, NSDictionary *extra); static void OPVTInstallVolumeNotificationObserver(void); @@ -558,6 +584,14 @@ static void OPVTPublishTriggerStatus(NSString *eventName, NSDictionary *extra) { @"last_button_event_source": OPVTLastButtonEventSource ?: @"", @"last_combo_event_ms": @(OPVTLastComboEventMs), @"last_trigger_route": OPVTLastTriggerRoute ?: @"", + @"phase_probe": @{ + @"down_phase_events": @(OPVTDownPhaseEventCount), + @"up_phase_events": @(OPVTUpPhaseEventCount), + @"unknown_phase_events": @(OPVTUnknownPhaseEventCount), + @"last_phase": OPVTLastPhaseObserved ?: @"", + @"last_hold_ms": @(OPVTLastHoldMs), + @"last_hold_button": OPVTLastHoldButton ?: @"" + }, @"source": @"springboard" } mutableCopy]; if (extra.count > 0) { @@ -1150,14 +1184,16 @@ static NSString *OPVTBundleIdentifierFromObject(id object, NSUInteger depth) { return nil; } -static NSTimeInterval OPVTWindowSeconds(void) { - return OPVTMillisecondsPreference(@"WindowMs", 1200.0, 50.0, 3000.0); -} - static NSTimeInterval OPVTCooldownSeconds(void) { return OPVTMillisecondsPreference(@"CooldownMs", 10000.0, 250.0, 60000.0); } +// Squeeze both buttons and release before this threshold -> realtime; keep both +// held past it -> record/transcribe harness. +static NSTimeInterval OPVTLongPressSeconds(void) { + return OPVTMillisecondsPreference(@"LongPressMs", 550.0, 250.0, 3000.0); +} + static void OPVTPlayHapticSuccess(void) { AudioServicesPlaySystemSound(1519); } @@ -3594,27 +3630,41 @@ static NSString *OPVTVoiceOverlayDetail(NSDictionary *response) { // (Legacy toast-based helpers removed. The island observer now renders live // status directly from the daemon's island-status.json file.) -static void OPVTCallDaemonVoiceAgent(void) { +static void OPVTCallDaemonVoiceAgent(BOOL realtime) { OPVTPublishSpringBoardStateOnMain(); OPVTLastTriggerRoute = @"daemon_voice_agent"; OPVTPublishTriggerStatus(@"voice_request", @{ - @"route": OPVTLastTriggerRoute ?: @"" + @"route": OPVTLastTriggerRoute ?: @"", + @"realtime": @(realtime) }); - NSDictionary *request = @{ + NSMutableDictionary *request = [@{ @"command": @"voice_trigger", - @"trigger": @"volume_up_down_combo", + @"trigger": realtime ? @"volume_double_tap_realtime" : @"volume_long_press_harness", @"source": @"springboard_volume", - @"reason": @"hardware volume combo voice trigger", + @"reason": realtime ? @"hardware volume double-tap realtime trigger" + : @"hardware volume long-press harness trigger", @"mode": @"auto", + // Realtime gesture streams the mic to the realtime model and requires it + // to be configured; long-press gesture forces the record/transcribe path. + @"stream": @(realtime), + @"require_realtime": @(realtime), @"max_steps": @25, @"max_duration_ms": @600000 - }; + } mutableCopy]; + if (!realtime) { + // Long-press harness is push-to-talk: the mic records for as long as the + // user holds both buttons and stops when they release (a voice_stop + // command). Disable silence-based auto-stop so a natural pause mid- + // sentence doesn't cut the recording; keep a large safety ceiling. + request[@"vad"] = @NO; + request[@"record_max_ms"] = @90000; + } OPVTPlayHapticSuccess(); OPVTIslandApplyState(@{ - @"mode": @"listening", + @"mode": realtime ? @"realtime" : @"listening", @"subtitle": @"Listening", - @"accent": @"red" + @"accent": realtime ? @"yellow" : @"red" }); dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ NSDictionary *response = OPVTAgentRequest(request); @@ -3637,9 +3687,13 @@ static void OPVTCallDaemonVoiceAgent(void) { NSString *accent = @"red"; if ([state isEqualToString:@"voice.credential_missing"] || [state isEqualToString:@"voice.already_running"] || + [state isEqualToString:@"voice.realtime_not_configured"] || [state isEqualToString:@"voice.empty_transcript"]) { accent = @"orange"; } + if ([state isEqualToString:@"voice.realtime_not_configured"]) { + detail = @"Realtime not configured"; + } OPVTIslandApplyState(@{ @"mode": @"error", @"subtitle": detail ?: @"", @@ -3647,7 +3701,8 @@ static void OPVTCallDaemonVoiceAgent(void) { }); } if (!ok && OPVTPromptForGoalEnabled() && - ![state isEqualToString:@"voice.credential_missing"]) { + ![state isEqualToString:@"voice.credential_missing"] && + ![state isEqualToString:@"voice.realtime_not_configured"]) { OPVTPresentTriggerPrompt(); } }); @@ -4343,6 +4398,191 @@ static void OPVTHandleSystemVolumeNotification(NSNotification *notification) { } } +// Diagnostic-only: classify the press phase from the hooked selector name. +// Returns @"down", @"up", or @"unknown". Phase markers ("PressDown", "ButtonUp", +// etc.) are chosen so they never collide with the volume direction words +// ("handleVolumeUpButtonPress" is direction=up, phase=unknown, not phase=up). +static NSString *OPVTClassifyPressPhase(NSString *source) { + if (![source isKindOfClass:[NSString class]] || source.length == 0) { + return @"unknown"; + } + // Unambiguous, phase-specific selectors only. Selectors whose name carries a + // phase token regardless of the actual call (e.g. + // "_sendVolumeButtonToSBUIControllerForIncrease:down:", which contains + // ":down:" even on the up-call) are deliberately left "unknown" so they never + // corrupt the pressed-state latch that drives gesture timing. + if ([source containsString:@"PressDown"] || + [source containsString:@"ButtonDownFor"]) { + return @"down"; + } + if ([source containsString:@"PressUp"] || + [source containsString:@"ButtonUpFor"] || + [source containsString:@"IncreaseUp"] || + [source containsString:@"DecreaseUp"]) { + return @"up"; + } + return @"unknown"; +} + +// The gesture latch must see exactly one down and one up per physical press. +// SpringBoard fires several redundant hooked selectors per press (public +// PressDown/PressUp plus internal _handle*/_send* forwarders), which desynced +// OPVTUpPressed/OPVTDownPressed in the first cut (down_phase_events 24 != +// up_phase_events 40). Only the outermost public selectors fire exactly once per +// physical edge, so we drive the latch from those alone; the others still feed +// the diagnostic probe but never touch the pressed state. +static BOOL OPVTIsCanonicalGestureSource(NSString *source) { + if (![source isKindOfClass:[NSString class]] || source.length == 0) { + return NO; + } + return [source containsString:@"PressDownWithModifiers"] || + [source containsString:@"PressUp"]; +} + +// Route a detected gesture. realtime=YES -> realtime streaming; NO -> harness. +// Honors the daemon-voice-agent / prompt / direct-agent preference chain and +// respects the cooldown + active-island-cancel behavior of the old combo path. +static void OPVTFireGesture(NSString *gesture, BOOL realtime) { + CFAbsoluteTime now = CFAbsoluteTimeGetCurrent(); + + // While a voice turn is live, either gesture acts as a cancel. This is + // checked BEFORE the cooldown so a re-squeeze can always stop an active + // turn — otherwise the 10s cooldown swallows the stop gesture and the user + // is stuck in "Listening" with no way out. + BOOL activeIslandMode = [OPVTIslandCurrentMode isEqualToString:@"listening"] || + [OPVTIslandCurrentMode isEqualToString:@"realtime"] || + [OPVTIslandCurrentMode isEqualToString:@"transcribing"] || + [OPVTIslandCurrentMode isEqualToString:@"thinking"] || + [OPVTIslandCurrentMode isEqualToString:@"action"]; + if (activeIslandMode) { + OPVTLastTrigger = now; + OPVTPublishTriggerStatus(@"gesture_cancel", @{ + @"gesture": gesture ?: @"", + @"mode": OPVTIslandCurrentMode ?: @"" + }); + OPVTPlayHapticFailure(); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + OPVTAgentRequest(@{@"command": @"voice_cancel", + @"reason": @"user_gesture_cancel"}); + }); + return; + } + + if ((now - OPVTLastTrigger) < OPVTCooldownSeconds()) { + OPVTPublishTriggerStatus(@"gesture_cooldown", @{ + @"gesture": gesture ?: @"", + @"realtime": @(realtime) + }); + return; + } + + OPVTLastTrigger = now; + OPVTComboEventCount++; + OPVTLastComboEventMs = OPVTNowMs(); + OPVTLastGesture = gesture; + NSString *route = OPVTDaemonVoiceAgentEnabled() + ? @"daemon_voice_agent" + : (OPVTPromptForGoalEnabled() ? @"springboard_prompt" : @"direct_agent"); + OPVTLastTriggerRoute = route; + OPVTPublishTriggerStatus(@"gesture_fired", @{ + @"gesture": gesture ?: @"", + @"realtime": @(realtime), + @"route": route ?: @"", + @"combo_event_count": @(OPVTComboEventCount) + }); + OPVTLog(@"gesture=%@ realtime=%d route=%@", gesture ?: @"", realtime, route); + if (OPVTDaemonVoiceAgentEnabled()) { + OPVTCallDaemonVoiceAgent(realtime); + } else if (OPVTPromptForGoalEnabled()) { + OPVTPresentTriggerPrompt(); + } else { + OPVTCallAgent(); + } +} + +// Fires the record/transcribe harness once both buttons have been held past the +// long-press threshold. Guarded by a generation token so a stale dispatch (from +// an earlier press that was released before the threshold) is ignored. +static void OPVTScheduleLongPressCheck(long long generation) { + NSTimeInterval hold = OPVTLongPressSeconds(); + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(hold * NSEC_PER_SEC)), + dispatch_get_main_queue(), ^{ + if (generation != OPVTGestureGeneration) { + return; // released or superseded before threshold + } + if (!(OPVTUpPressed && OPVTDownPressed) || OPVTLongPressFired) { + return; + } + OPVTLongPressFired = YES; + OPVTPublishTriggerStatus(@"gesture_long_press", @{ + @"hold_ms": @(OPVTLongPressSeconds() * 1000.0) + }); + OPVTLog(@"gesture LONG-PRESS -> harness (hold_ms=%0.0f)", OPVTLongPressSeconds() * 1000.0); + OPVTFireGesture(@"harness_long_press", NO); + }); +} + +// Drives gesture detection from clean down/up phase events. Maintains a per- +// button pressed latch and times how long both buttons stay held. Squeeze both +// and release quickly (< long-press threshold) -> realtime streaming. Keep both +// held past the threshold -> record/transcribe harness. Splitting one natural +// "squeeze both" gesture by hold duration avoids the double-tap timing problem, +// where quick separate taps never actually overlap on both buttons. +static void OPVTHandlePhaseGesture(BOOL volumeUp, BOOL down, CFAbsoluteTime now) { + BOOL wasBoth = OPVTUpPressed && OPVTDownPressed; + if (volumeUp) { + OPVTUpPressed = down; + } else { + OPVTDownPressed = down; + } + BOOL isBoth = OPVTUpPressed && OPVTDownPressed; + OPVTLog(@"gesture phase %@ %@ up_pressed=%d down_pressed=%d was_both=%d is_both=%d", + volumeUp ? @"volume_up" : @"volume_down", down ? @"down" : @"up", + OPVTUpPressed, OPVTDownPressed, wasBoth, isBoth); + + if (isBoth && !wasBoth) { + // Both buttons just became held: start the long-press timer that fires + // the harness if the squeeze is held past the threshold. + OPVTBothPressed = YES; + OPVTBothPressedStart = now; + OPVTLongPressFired = NO; + OPVTGestureGeneration++; + OPVTScheduleLongPressCheck(OPVTGestureGeneration); + OPVTPublishTriggerStatus(@"gesture_both_down", @{ + @"generation": @(OPVTGestureGeneration) + }); + return; + } + + if (!isBoth && wasBoth) { + // A button released while both were held. Invalidate the pending + // long-press timer. If the harness already fired on hold, we're done; + // otherwise this was a quick squeeze -> realtime. + OPVTGestureGeneration++; + double heldMs = (now - OPVTBothPressedStart) * 1000.0; + OPVTBothPressed = NO; + if (OPVTLongPressFired) { + OPVTLongPressFired = NO; + // Push-to-talk: the harness has been recording while both buttons + // were held. Releasing is the signal to stop the mic and run STT. + OPVTLog(@"gesture both-release held_ms=%0.0f -> harness push-to-talk stop", heldMs); + OPVTPublishTriggerStatus(@"gesture_harness_release", @{ + @"held_ms": @(heldMs) + }); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ + OPVTAgentRequest(@{@"command": @"voice_stop", + @"reason": @"harness_push_to_talk_release"}); + }); + return; + } + OPVTLog(@"gesture QUICK-SQUEEZE -> realtime (held_ms=%0.0f)", heldMs); + OPVTPublishTriggerStatus(@"gesture_quick_squeeze", @{ + @"held_ms": @(heldMs) + }); + OPVTFireGesture(@"realtime_quick_squeeze", YES); + } +} + static void OPVTRecordButtonFromSource(BOOL volumeUp, NSString *source) { if (!OPVTEnabled()) { OPVTPublishTriggerStatus(@"button_ignored_disabled", @{ @@ -4358,6 +4598,42 @@ static void OPVTRecordButtonFromSource(BOOL volumeUp, NSString *source) { OPVTLastButtonEventName = volumeUp ? @"volume_up" : @"volume_down"; OPVTLastButtonEventSource = [source isKindOfClass:[NSString class]] && source.length > 0 ? [source copy] : @"unknown"; + + // --- Diagnostic hold/phase probe (does not change routing) --- + NSString *phase = OPVTClassifyPressPhase(OPVTLastButtonEventSource); + OPVTLastPhaseObserved = phase; + double holdMsThisEvent = -1.0; + if ([phase isEqualToString:@"down"]) { + OPVTDownPhaseEventCount++; + if (volumeUp) { + OPVTLastUpDownTime = now; + } else { + OPVTLastDownDownTime = now; + } + } else if ([phase isEqualToString:@"up"]) { + OPVTUpPhaseEventCount++; + CFAbsoluteTime downTime = volumeUp ? OPVTLastUpDownTime : OPVTLastDownDownTime; + if (downTime > 0 && now >= downTime) { + holdMsThisEvent = (now - downTime) * 1000.0; + OPVTLastHoldMs = holdMsThisEvent; + OPVTLastHoldButton = volumeUp ? @"volume_up" : @"volume_down"; + } + } else { + OPVTUnknownPhaseEventCount++; + } + OPVTPublishTriggerStatus(@"button_phase_probe", @{ + @"button": OPVTLastButtonEventName ?: @"", + @"phase": phase ?: @"unknown", + @"source": OPVTLastButtonEventSource ?: @"", + @"hold_ms_this_event": @(holdMsThisEvent), + @"down_phase_events": @(OPVTDownPhaseEventCount), + @"up_phase_events": @(OPVTUpPhaseEventCount), + @"unknown_phase_events": @(OPVTUnknownPhaseEventCount), + @"last_hold_ms": @(OPVTLastHoldMs), + @"last_hold_button": OPVTLastHoldButton ?: @"" + }); + // --- end probe --- + OPVTPublishTriggerStatus(@"button_event", @{ @"button": OPVTLastButtonEventName ?: @"", @"button_event_count": @(OPVTButtonEventCount), @@ -4377,53 +4653,17 @@ static void OPVTRecordButtonFromSource(BOOL volumeUp, NSString *source) { OPVTLastDown = now; } - BOOL combo = fabs(OPVTLastUp - OPVTLastDown) <= OPVTWindowSeconds(); - BOOL cooledDown = (now - OPVTLastTrigger) >= OPVTCooldownSeconds(); - if (combo && cooledDown) { - OPVTLastTrigger = now; - OPVTComboEventCount++; - OPVTLastComboEventMs = OPVTNowMs(); - - // If the island is in an active state, treat the combo as a cancel. - BOOL activeIslandMode = [OPVTIslandCurrentMode isEqualToString:@"listening"] || - [OPVTIslandCurrentMode isEqualToString:@"realtime"] || - [OPVTIslandCurrentMode isEqualToString:@"transcribing"] || - [OPVTIslandCurrentMode isEqualToString:@"thinking"] || - [OPVTIslandCurrentMode isEqualToString:@"action"]; - if (activeIslandMode) { - OPVTLog(@"volume combo -> cancel (island mode=%@)", - OPVTIslandCurrentMode ?: @""); - OPVTPublishTriggerStatus(@"volume_combo_cancel", @{ - @"mode": OPVTIslandCurrentMode ?: @"" - }); - OPVTPlayHapticFailure(); - dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ - OPVTAgentRequest(@{@"command": @"voice_cancel", - @"reason": @"user_double_combo"}); - }); - return; - } - - NSString *route = OPVTDaemonVoiceAgentEnabled() - ? @"daemon_voice_agent" - : (OPVTPromptForGoalEnabled() ? @"springboard_prompt" : @"direct_agent"); - OPVTLastTriggerRoute = route; - OPVTPublishTriggerStatus(@"volume_combo", @{ - @"route": route ?: @"", - @"combo_event_count": @(OPVTComboEventCount), - @"source": OPVTLastButtonEventSource ?: @"", - @"last_up_delta_s": @(now - OPVTLastUp), - @"last_down_delta_s": @(now - OPVTLastDown) - }); - OPVTLog(@"volume combo detected up=%0.3f down=%0.3f source=%@", - OPVTLastUp, OPVTLastDown, OPVTLastButtonEventSource ?: @""); - if (OPVTDaemonVoiceAgentEnabled()) { - OPVTCallDaemonVoiceAgent(); - } else if (OPVTPromptForGoalEnabled()) { - OPVTPresentTriggerPrompt(); - } else { - OPVTCallAgent(); - } + // Gesture detection runs purely on the deterministic phase engine, and only + // for the canonical public selectors (each fires exactly once per physical + // edge). The redundant internal forwarders and the no-arg wrappers still feed + // the diagnostic probe above but must not touch the pressed-state latch. + // + // The old "two volume events within window_ms" combo fallback was removed: it + // treated any two ordinary volume adjustments as a realtime trigger, which is + // a constant false-fire source now that real gestures are deterministic. + if (OPVTIsCanonicalGestureSource(OPVTLastButtonEventSource) && + ([phase isEqualToString:@"down"] || [phase isEqualToString:@"up"])) { + OPVTHandlePhaseGesture(volumeUp, [phase isEqualToString:@"down"], now); } } @@ -5882,6 +6122,12 @@ static UIView *OPVTIslandStatusDot = nil; static CAShapeLayer *OPVTIslandGlowLayer = nil; static CAGradientLayer *OPVTIslandGradientGlowLayer = nil; static CAShapeLayer *OPVTIslandGradientMask = nil; +// Full-screen edge glow shown only during realtime streaming so the user can +// tell at a glance they're in live voice mode (vs. the STT harness). Mirrors +// the Android PointerOverlayController screen-border glow. +static UIView *OPVTRealtimeEdgeView = nil; +static CAGradientLayer *OPVTRealtimeEdgeGradient = nil; +static CAShapeLayer *OPVTRealtimeEdgeMask = nil; static CADisplayLink *OPVTIslandTicker = nil; static CGFloat OPVTIslandGradientShift = 0.0; static NSInteger OPVTIslandThinkingTick = 0; @@ -5913,6 +6159,7 @@ static long long OPVTIslandLastAppliedMs = 0; static NSDictionary *OPVTIslandCurrentState = nil; static int OPVTIslandStatusNotifyToken = 0; static BOOL OPVTIslandStatusNotifyRegistered = NO; +static dispatch_source_t OPVTIslandLivenessTimer = nil; static UIColor *OPVTIslandAccentColor(NSString *accent) { NSString *key = [accent isKindOfClass:[NSString class]] ? accent.lowercaseString : @"cyan"; @@ -5944,6 +6191,24 @@ static BOOL OPVTIslandDeviceHasDynamicIsland(void) { return NO; } +// True when the STT harness is mid-flight AND has text worth showing in the +// compact band. Only then does the compact pill extend below the physical +// notch; otherwise it stays exactly notch-sized so it disappears into the +// hardware Dynamic Island (idle and realtime both read as a bare notch). +static BOOL OPVTIslandCompactShouldExtend(void) { + NSString *mode = OPVTIslandCurrentMode ?: @"idle"; + BOOL sttHarnessActive = [mode isEqualToString:@"listening"] || + [mode isEqualToString:@"transcribing"] || + [mode isEqualToString:@"thinking"] || + [mode isEqualToString:@"action"]; + if (!sttHarnessActive) { + return NO; + } + NSString *subtitle = [OPVTIslandCurrentState[@"subtitle"] isKindOfClass:[NSString class]] + ? OPVTIslandCurrentState[@"subtitle"] : @""; + return subtitle.length > 0; +} + // level: 0 = compact (DI shape), 1 = medium panel, 2 = large full-screen panel. static CGRect OPVTIslandPillFrameForLevel(CGRect bounds, NSInteger level) { CGFloat topBase = OPVTIslandDeviceHasDynamicIsland() ? 11.0 : 12.0; @@ -5957,11 +6222,14 @@ static CGRect OPVTIslandPillFrameForLevel(CGRect bounds, NSInteger level) { } } if (level >= 2) { - // Large: full width minus small margin, ~75% of screen height. - CGFloat pillWidth = bounds.size.width - 16.0; - CGFloat pillHeight = bounds.size.height * 0.75; - CGFloat originX = (bounds.size.width - pillWidth) / 2.0; - return CGRectMake(originX, topBase, pillWidth, pillHeight); + // Large: inset from every screen edge so the rounded panel floats + // clearly inside the device bezel (doesn't touch the screen edges). + CGFloat sideMargin = 12.0; + CGFloat pillWidth = bounds.size.width - sideMargin * 2.0; + CGFloat originY = topBase; + CGFloat pillHeight = bounds.size.height * 0.82; + CGFloat originX = sideMargin; + return CGRectMake(originX, originY, pillWidth, pillHeight); } if (level == 1) { // Medium panel. @@ -5974,8 +6242,20 @@ static CGRect OPVTIslandPillFrameForLevel(CGRect bounds, NSInteger level) { } // Compact: match Dynamic Island exactly on DI devices, plain pill elsewhere. if (OPVTIslandDeviceHasDynamicIsland()) { - CGFloat pillWidth = 122.0; - CGFloat pillHeight = 37.0; + // The hardware Dynamic Island is a physical cutout — its ~37pt tall + // region is not real screen, so anything we draw there is invisible. + // To read as ONE continuous black shape (not a detached blob with a + // gap above), the pill must be a touch WIDER than the physical cutout + // so the hardware notch sits fully inside our black fill, and its top + // edge must align with the TOP of the physical notch (topBase) — not + // the screen top, which would paint black above the notch. It extends + // one text line below the cutout onto visible screen. Top corners + // square so the top edge is flush with the notch top; bottom rounded. + CGFloat pillWidth = 134.0; // slightly wider than the ~126pt cutout. + // Notch-only by default so the pill vanishes into the hardware Dynamic + // Island; extend one text line below only when the STT harness has + // something to say. ~37pt cutout, +20pt visible line when extended. + CGFloat pillHeight = OPVTIslandCompactShouldExtend() ? 57.0 : 37.0; CGFloat originX = (bounds.size.width - pillWidth) / 2.0; return CGRectMake(originX, topBase, pillWidth, pillHeight); } @@ -6024,9 +6304,65 @@ static void OPVTIslandEnsureWindow(void) { UIView *root = controller.view; + // Full-screen realtime edge glow (hidden until realtime mode). A rainbow + // gradient filling the screen, masked to a wide rounded-rect band that is + // gaussian-blurred into a soft feathered glow hugging the display corners. + // Non-interactive so it never eats touches meant for the app underneath. + OPVTRealtimeEdgeView = [[UIView alloc] initWithFrame:bounds]; + OPVTRealtimeEdgeView.userInteractionEnabled = NO; + OPVTRealtimeEdgeView.backgroundColor = [UIColor clearColor]; + OPVTRealtimeEdgeView.hidden = YES; + OPVTRealtimeEdgeView.alpha = 0.0; + + OPVTRealtimeEdgeGradient = [CAGradientLayer layer]; + OPVTRealtimeEdgeGradient.frame = bounds; + OPVTRealtimeEdgeGradient.colors = @[ + (id)[UIColor colorWithRed:0.365 green:0.863 blue:1.0 alpha:1.0].CGColor, + (id)[UIColor colorWithRed:0.580 green:0.424 blue:1.0 alpha:1.0].CGColor, + (id)[UIColor colorWithRed:1.0 green:0.337 blue:0.659 alpha:1.0].CGColor, + (id)[UIColor colorWithRed:1.0 green:0.800 blue:0.424 alpha:1.0].CGColor, + (id)[UIColor colorWithRed:0.365 green:0.863 blue:1.0 alpha:1.0].CGColor + ]; + OPVTRealtimeEdgeGradient.startPoint = CGPointMake(0.0, 0.5); + OPVTRealtimeEdgeGradient.endPoint = CGPointMake(1.0, 0.5); + + OPVTRealtimeEdgeMask = [CAShapeLayer layer]; + OPVTRealtimeEdgeMask.fillColor = [UIColor clearColor].CGColor; + OPVTRealtimeEdgeMask.strokeColor = [UIColor blackColor].CGColor; + // Wide band hugging the device's rounded screen corners. The band is then + // gaussian-blurred (below) so it reads as a soft feathered glow that fades + // toward the screen interior rather than a hard-edged stroke. + OPVTRealtimeEdgeMask.lineWidth = 46.0; + CGFloat screenRadius = 55.0; // iPhone 15 Pro display corner radius + UIBezierPath *edgePath = [UIBezierPath bezierPathWithRoundedRect: + CGRectInset(bounds, 6.0, 6.0) cornerRadius:screenRadius]; + OPVTRealtimeEdgeMask.path = edgePath.CGPath; + OPVTRealtimeEdgeMask.frame = bounds; + // Feather the mask so the reveal has soft edges (a glow, not a stroke). + // CAFilter is private QuartzCore but available inside the SpringBoard + // process we inject into; degrade gracefully to the crisp band if absent. + Class CAFilterClass = NSClassFromString(@"CAFilter"); + SEL filterSel = NSSelectorFromString(@"filterWithType:"); + if (CAFilterClass && [CAFilterClass respondsToSelector:filterSel]) { + id (*makeFilter)(id, SEL, id) = (id (*)(id, SEL, id))objc_msgSend; + id blur = makeFilter(CAFilterClass, filterSel, @"gaussianBlur"); + if (blur) { + [blur setValue:@(22.0) forKey:@"inputRadius"]; + @try { [blur setValue:@NO forKey:@"inputNormalizeEdges"]; } @catch (__unused NSException *e) {} + OPVTRealtimeEdgeMask.filters = @[blur]; + } + } + OPVTRealtimeEdgeGradient.mask = OPVTRealtimeEdgeMask; + OPVTRealtimeEdgeGradient.shadowColor = [UIColor colorWithRed:0.580 green:0.424 blue:1.0 alpha:1.0].CGColor; + OPVTRealtimeEdgeGradient.shadowRadius = 18.0; + OPVTRealtimeEdgeGradient.shadowOpacity = 0.8; + OPVTRealtimeEdgeGradient.shadowOffset = CGSizeZero; + [OPVTRealtimeEdgeView.layer addSublayer:OPVTRealtimeEdgeGradient]; + [root addSubview:OPVTRealtimeEdgeView]; + // Pill container. OPVTIslandPill = [[UIView alloc] initWithFrame:OPVTIslandPillFrame(bounds, NO)]; - OPVTIslandPill.backgroundColor = [UIColor colorWithWhite:0.0 alpha:0.94]; + OPVTIslandPill.backgroundColor = [UIColor blackColor]; // Match the Dynamic Island's exact corner curvature — its height is ~37pt // and it's a full pill (radius = height/2). We keep that ratio on expand // too so the shape stays consistent. @@ -6141,7 +6477,9 @@ static void OPVTIslandEnsureWindow(void) { // Tap toggles expansion. Never hides the pill entirely — it's always // present so the user can tap into chat history at any time. Fresh // volume trigger is the only way to start a new voice turn. - if (OPVTIslandExpanded && !OPVTIslandExpanded.hidden) { + BOOL expanded = OPVTIslandExpanded && !OPVTIslandExpanded.hidden; + OPVTLog(@"island tap expanded=%d level=%ld", expanded, (long)OPVTIslandExpansionLevel); + if (expanded) { OPVTIslandCollapse(); } else { OPVTIslandExpand(); @@ -6229,6 +6567,15 @@ static void OPVTIslandEnsureWindow(void) { OPVTIslandGradientGlowLayer.endPoint = CGPointMake(2.0 + OPVTIslandGradientShift, 0.5); } + if (OPVTRealtimeEdgeGradient && !OPVTRealtimeEdgeView.hidden) { + [CATransaction begin]; + [CATransaction setDisableActions:YES]; + OPVTRealtimeEdgeGradient.startPoint = + CGPointMake(-1.0 + OPVTIslandGradientShift, 0.5); + OPVTRealtimeEdgeGradient.endPoint = + CGPointMake(2.0 + OPVTIslandGradientShift, 0.5); + [CATransaction commit]; + } } - (void)thinkingDotsTick:(NSTimer *)timer { (void)timer; @@ -6310,54 +6657,136 @@ static void OPVTIslandStartTicker(void) { static void OPVTIslandLayoutInterior(void) { if (!OPVTIslandPill) return; CGRect b = OPVTIslandPill.bounds; - BOOL compact = b.size.height < 55.0; + BOOL compact = b.size.height < 120.0; BOOL activeMode = [OPVTIslandCurrentMode isEqualToString:@"listening"] || [OPVTIslandCurrentMode isEqualToString:@"realtime"] || [OPVTIslandCurrentMode isEqualToString:@"transcribing"] || [OPVTIslandCurrentMode isEqualToString:@"thinking"] || [OPVTIslandCurrentMode isEqualToString:@"action"]; - // Show cancel chip only while active. Never during idle/terminal. + // Show the × stop chip while a turn is live so the user can tap to cancel + // (in addition to re-squeezing the volume buttons). Reserve room on the + // right so the subtitle text doesn't run under the chip. if (OPVTIslandCancelChip) OPVTIslandCancelChip.hidden = !activeMode; CGFloat rightReserve = activeMode ? 30.0 : 0.0; + // The compact/minimized small-text row (status dot + subtitle) is only + // meaningful for the STT harness, which narrates its progress as text. + // Realtime uses the edge glow instead, and at idle there is nothing to say, + // so in those cases keep the compact pill a bare black notch with no text. + BOOL sttHarnessActive = [OPVTIslandCurrentMode isEqualToString:@"listening"] || + [OPVTIslandCurrentMode isEqualToString:@"transcribing"] || + [OPVTIslandCurrentMode isEqualToString:@"thinking"] || + [OPVTIslandCurrentMode isEqualToString:@"action"]; if (compact) { - OPVTIslandStatusDot.frame = CGRectMake(11, (b.size.height - 10) / 2.0, 10, 10); + // The top ~37pt sits over (behind) the hardware Dynamic Island cutout + // and is not visible. Render the status dot + text in the LOWER band + // so it lands on real screen. On non-DI devices cutoutTop is 0 and the + // whole pill is visible, so we center within the pill instead. + CGFloat cutoutTop = OPVTIslandDeviceHasDynamicIsland() ? 37.0 : 0.0; + CGFloat visibleH = b.size.height - cutoutTop; + CGFloat dotY = cutoutTop + (visibleH - 10) / 2.0; + OPVTIslandStatusDot.frame = CGRectMake(14, dotY, 10, 10); OPVTIslandStatusDot.layer.cornerRadius = 5.0; + OPVTIslandStatusDot.hidden = !sttHarnessActive; OPVTIslandTitleLabel.hidden = YES; - OPVTIslandSubtitleLabel.frame = CGRectMake(28, 0, b.size.width - 40 - rightReserve, b.size.height); - OPVTIslandSubtitleLabel.font = [UIFont systemFontOfSize:12.0 weight:UIFontWeightSemibold]; + OPVTIslandSubtitleLabel.hidden = !sttHarnessActive; + OPVTIslandSubtitleLabel.frame = CGRectMake(30, cutoutTop, + b.size.width - 42 - rightReserve, visibleH); + OPVTIslandSubtitleLabel.font = [UIFont systemFontOfSize:11.0 weight:UIFontWeightSemibold]; OPVTIslandSubtitleLabel.textAlignment = NSTextAlignmentLeft; if (OPVTIslandCancelChip) { - OPVTIslandCancelChip.frame = CGRectMake(b.size.width - 30, (b.size.height - 22) / 2.0, 22, 22); + OPVTIslandCancelChip.frame = CGRectMake(b.size.width - 28, + cutoutTop + (visibleH - 20) / 2.0, 20, 20); } } else { - OPVTIslandStatusDot.frame = CGRectMake(14, 14, 14, 14); + // Expanded panel. Inset content away from the rounded corners so the + // dot, text, and chip don't crowd the edges. padX/padY are the interior + // margins; the corner radius is ~44pt so these keep content clear of it. + CGFloat padX = 22.0; + CGFloat padY = 16.0; + OPVTIslandStatusDot.hidden = NO; + OPVTIslandSubtitleLabel.hidden = NO; + OPVTIslandStatusDot.frame = CGRectMake(padX, padY + 2, 14, 14); OPVTIslandStatusDot.layer.cornerRadius = 7.0; OPVTIslandTitleLabel.hidden = NO; - OPVTIslandTitleLabel.frame = CGRectMake(34, 6, b.size.width - 48 - rightReserve, 16); - OPVTIslandSubtitleLabel.frame = CGRectMake(34, 20, b.size.width - 48 - rightReserve, 16); + CGFloat textX = padX + 28; + CGFloat textW = b.size.width - textX - padX - rightReserve; + OPVTIslandTitleLabel.frame = CGRectMake(textX, padY, textW, 16); + OPVTIslandSubtitleLabel.frame = CGRectMake(textX, padY + 16, textW, 16); OPVTIslandSubtitleLabel.font = [UIFont systemFontOfSize:13.0 weight:UIFontWeightMedium]; OPVTIslandSubtitleLabel.textAlignment = NSTextAlignmentLeft; if (OPVTIslandCancelChip) { - OPVTIslandCancelChip.frame = CGRectMake(b.size.width - 32, 10, 22, 22); + OPVTIslandCancelChip.frame = CGRectMake(b.size.width - padX - 22, padY, 22, 22); } } } +// Show/hide the full-screen realtime edge glow. Only realtime streaming mode +// gets the border glow; the STT harness ("listening"/"transcribing") does not, +// so the two modes are visually distinct at a glance. +static BOOL OPVTRealtimeEdgeIntendedVisible = NO; +static void OPVTRealtimeEdgeSetVisible(BOOL visible) { + if (!OPVTRealtimeEdgeView) { + return; + } + // Track the INTENDED state, not the current alpha. Keying off a mid-fade + // alpha (> 0.5) meant a quick hide→show could latch the wrong state and + // leave the glow stuck off — the "appears for a millisecond then gone" bug. + if (visible == OPVTRealtimeEdgeIntendedVisible && + (visible ? !OPVTRealtimeEdgeView.hidden : YES)) { + return; + } + OPVTRealtimeEdgeIntendedVisible = visible; + if (visible) { + [OPVTRealtimeEdgeView.layer removeAllAnimations]; + OPVTRealtimeEdgeView.hidden = NO; + [UIView animateWithDuration:0.25 animations:^{ + OPVTRealtimeEdgeView.alpha = 1.0; + }]; + } else { + [UIView animateWithDuration:0.25 animations:^{ + OPVTRealtimeEdgeView.alpha = 0.0; + } completion:^(BOOL finished) { + (void)finished; + // Only actually hide if a newer call hasn't re-requested visibility. + if (!OPVTRealtimeEdgeIntendedVisible) { + OPVTRealtimeEdgeView.hidden = YES; + } + }]; + } +} + static void OPVTIslandUpdateGlowPath(void) { if (!OPVTIslandGlowLayer || !OPVTIslandPill) { return; } CGRect bounds = OPVTIslandPill.bounds; // Keep the rounded-pill feel: radius = 22 when expanded (matches iOS - // system smart-stack corner), 18.5 when compact (matches hardware DI). - CGFloat radius = bounds.size.height > 60 ? 22.0 : 18.5; - OPVTIslandPill.layer.cornerRadius = radius; + // system smart-stack corner), matches the hardware DI curvature when + // compact so the pill reads as a continuation of the physical notch. + BOOL compact = bounds.size.height < 120.0; + // Compact: match the hardware DI curvature (~20pt) so top corners blend + // into the physical notch roundness. Expanded: use the iPhone 15 Pro screen + // corner radius (~55pt) so the panel echoes the device's rounded corners. + CGFloat radius = compact ? 20.0 : 44.0; OPVTIslandLayoutInterior(); - UIBezierPath *path = [UIBezierPath bezierPathWithRoundedRect:bounds - cornerRadius:radius]; + UIBezierPath *path; + OPVTIslandPill.layer.cornerRadius = radius; + // All four corners rounded in every state. + OPVTIslandPill.layer.maskedCorners = + kCALayerMinXMinYCorner | kCALayerMaxXMinYCorner | + kCALayerMinXMaxYCorner | kCALayerMaxXMaxYCorner; + path = [UIBezierPath bezierPathWithRoundedRect:bounds + cornerRadius:radius]; + // When compact the pill IS the notch — a real notch has no colored border + // or glow bloom. Hide every glow layer so it renders as pure solid black. + // The rainbow stroke + shadow return the moment the user expands it. + BOOL hideGlow = compact; + OPVTIslandGlowLayer.hidden = hideGlow; + OPVTIslandGlowLayer.shadowOpacity = hideGlow ? 0.0 : 0.85; OPVTIslandGlowLayer.path = path.CGPath; OPVTIslandGlowLayer.frame = bounds; if (OPVTIslandGradientGlowLayer) { + OPVTIslandGradientGlowLayer.hidden = hideGlow; OPVTIslandGradientGlowLayer.frame = bounds; } if (OPVTIslandGradientMask) { @@ -6668,8 +7097,6 @@ static void OPVTIslandApplyState(NSDictionary *state) { ? state[@"subtitle"] : @""; NSString *transcript = [state[@"transcript"] isKindOfClass:[NSString class]] ? state[@"transcript"] : @""; - NSString *tool = [state[@"tool"] isKindOfClass:[NSString class]] - ? state[@"tool"] : @""; NSString *goal = [state[@"goal"] isKindOfClass:[NSString class]] ? state[@"goal"] : @""; NSString *accent = [state[@"accent"] isKindOfClass:[NSString class]] @@ -6694,7 +7121,13 @@ static void OPVTIslandApplyState(NSDictionary *state) { mode.length == 0 || [mode isEqualToString:@"hidden"] || [mode isEqualToString:@"idle"]; - OPVTIslandPill.alpha = isIdle ? 0.55 : 1.0; + // The pill is ALWAYS solid black — compact it merges with the hardware + // notch, expanded it must not show the app behind it. Never dim the + // pill itself (an expanded translucent panel was the "transparent when + // I expand it" bug). Only dim the status dot/text on idle. + OPVTIslandPill.alpha = 1.0; + OPVTIslandStatusDot.alpha = isIdle ? 0.45 : 1.0; + OPVTIslandSubtitleLabel.alpha = isIdle ? 0.55 : 1.0; // Fire haptic on state transitions to terminal (Android-parity feel). BOOL modeChanged = ![OPVTIslandCurrentMode isEqualToString:mode]; if (modeChanged && [mode isEqualToString:@"success"]) { @@ -6720,6 +7153,26 @@ static void OPVTIslandApplyState(NSDictionary *state) { } } OPVTIslandCurrentMode = mode ?: @"idle"; + // Realtime edge glow. A realtime conversation cycles through + // realtime → thinking/action (tool steps) → realtime, so keying the + // glow strictly on mode=="realtime" made it blink off the instant the + // agent started thinking after a question. Instead latch it: turn ON at + // "realtime", KEEP it on through the working tool-step modes, and only + // turn OFF on a harness/terminal/idle mode. (Harness thinking/action + // never turn it on because the latch starts from "listening", not + // "realtime".) + if ([mode isEqualToString:@"realtime"]) { + OPVTRealtimeEdgeSetVisible(YES); + } else if ([mode isEqualToString:@"listening"] || + [mode isEqualToString:@"transcribing"] || + isIdle || + [mode isEqualToString:@"success"] || + [mode isEqualToString:@"error"] || + [mode isEqualToString:@"needs_review"]) { + OPVTRealtimeEdgeSetVisible(NO); + } + // thinking/action: leave the glow in whatever state the session set — + // keeps it lit during a realtime turn, stays off during a harness turn. UIColor *accentColor = OPVTIslandAccentColor(accent); OPVTIslandStatusDot.backgroundColor = accentColor; OPVTIslandGlowLayer.strokeColor = accentColor.CGColor; @@ -6742,28 +7195,31 @@ static void OPVTIslandApplyState(NSDictionary *state) { OPVTIslandTitleLabel.text = title; OPVTIslandSubtitleLabel.text = subtitle.length > 0 ? subtitle : @""; - // Auto-expand when we have any meaningful content to show. The pill - // stays expanded across states until it collapses on terminal. + // Working states stay COMPACT — the pill should sit as a small top + // island while the agent listens/thinks/acts, so it never covers the + // screen the user (and the agent) needs to see. Only the terminal + // reply auto-expands briefly so the answer is readable; everything + // else expands solely on a user tap. NSString *primaryText = goal.length > 0 ? goal : transcript; - BOOL hasContent = primaryText.length > 0 || - tool.length > 0 || - [mode isEqualToString:@"success"] || - [mode isEqualToString:@"error"] || - [mode isEqualToString:@"needs_review"]; BOOL isTerminal = [mode isEqualToString:@"success"] || [mode isEqualToString:@"error"]; - BOOL shouldExpand = hasContent && !isTerminal; - if (shouldExpand) { - OPVTIslandPill.frame = OPVTIslandPillFrame(OPVTIslandWindow.bounds, YES); - OPVTIslandEnsureExpandedViews(OPVTIslandPill.bounds.size.width - 28); - OPVTIslandExpanded.hidden = NO; - } else if (isTerminal && primaryText.length > 0) { - // Keep expanded on terminal so user can read summary. - OPVTIslandPill.frame = OPVTIslandPillFrame(OPVTIslandWindow.bounds, YES); + BOOL autoExpand = isTerminal && primaryText.length > 0; + // Never shrink below the level the user opened by hand. applyState runs + // on every daemon update AND at the tail of a manual expand, so keying + // the frame purely off content would fight the user's tap and snap the + // pill back to compact (the "island won't open" bug). Honor the manual + // expansion level as a floor. + NSInteger autoLevel = autoExpand ? 1 : 0; + NSInteger effectiveLevel = MAX(OPVTIslandExpansionLevel, autoLevel); + OPVTIslandExpansionLevel = effectiveLevel; + if (effectiveLevel > 0) { + OPVTIslandPill.frame = OPVTIslandPillFrameForLevel( + OPVTIslandWindow.bounds, effectiveLevel); OPVTIslandEnsureExpandedViews(OPVTIslandPill.bounds.size.width - 28); OPVTIslandExpanded.hidden = NO; } else { - OPVTIslandPill.frame = OPVTIslandPillFrame(OPVTIslandWindow.bounds, NO); + OPVTIslandPill.frame = OPVTIslandPillFrameForLevel( + OPVTIslandWindow.bounds, 0); if (OPVTIslandExpanded) { OPVTIslandExpanded.hidden = YES; } @@ -6838,6 +7294,25 @@ static void OPVTIslandApplyState(NSDictionary *state) { }); } +// A "working" mode means the daemon claims it is mid-turn. Such a state is only +// trustworthy while the daemon keeps its heartbeat fresh (see the daemon-side +// OPIslandHeartbeatThread); a stale heartbeat means the owner died and the pill +// must self-heal to idle. Resting modes never expire. +static BOOL OPVTIslandModeIsWorking(NSString *mode) { + return [mode isEqualToString:@"listening"] || + [mode isEqualToString:@"realtime"] || + [mode isEqualToString:@"transcribing"] || + [mode isEqualToString:@"thinking"] || + [mode isEqualToString:@"action"] || + [mode isEqualToString:@"needs_review"]; +} + +// Wall-clock ms matching the daemon's OPNowMs (Unix epoch), used to compare +// against heartbeat_ms. CFAbsoluteTime is a different epoch, so it can't. +static long long OPVTIslandWallClockMs(void) { + return (long long)([[NSDate date] timeIntervalSince1970] * 1000.0); +} + static void OPVTIslandRefreshFromDisk(void) { NSData *data = [NSData dataWithContentsOfFile:OPVTIslandStatusPath]; if (data.length == 0) { @@ -6859,6 +7334,70 @@ static void OPVTIslandRefreshFromDisk(void) { OPVTIslandApplyState(state); } +// Self-heal a stale working pill. If the current mode is "working" but the +// daemon's heartbeat is older than the grace window, the owning daemon has +// died (crash, jetsam, hang) and left a frozen snapshot — collapse to idle. +// The grace window (15s) comfortably clears the daemon's 5s heartbeat cadence +// while still catching a dead owner within a couple of seconds of the miss. +static void OPVTIslandCheckLiveness(void) { + if (!OPVTIslandModeIsWorking(OPVTIslandCurrentMode)) { + return; + } + // The daemon refreshes heartbeat_ms on disk every ~5s *without* bumping the + // sequence (a liveness ping shouldn't churn the tweak's sequence-gated + // re-render). OPVTIslandRefreshFromDisk therefore skips those writes and our + // cached OPVTIslandCurrentState holds a stale heartbeat. Read the live + // heartbeat straight from disk so a healthy long-running turn (e.g. an + // open-ended realtime conversation) isn't misjudged as a dead owner. + NSDictionary *diskState = OPVTIslandCurrentState; + NSData *diskData = [NSData dataWithContentsOfFile:OPVTIslandStatusPath]; + if (diskData.length > 0) { + id parsed = [NSJSONSerialization JSONObjectWithData:diskData options:0 error:nil]; + if ([parsed isKindOfClass:[NSDictionary class]]) { + diskState = parsed; + } + } + long long heartbeat = [diskState[@"heartbeat_ms"] isKindOfClass:[NSNumber class]] + ? [diskState[@"heartbeat_ms"] longLongValue] : 0; + // Fall back to updated_at_ms for states written before heartbeats existed. + if (heartbeat == 0) { + heartbeat = [diskState[@"updated_at_ms"] isKindOfClass:[NSNumber class]] + ? [diskState[@"updated_at_ms"] longLongValue] : 0; + } + if (heartbeat == 0) { + return; + } + if (OPVTIslandWallClockMs() - heartbeat < 15000) { + return; + } + OPVTLog(@"island self-heal: mode=%@ heartbeat stale by %lldms, collapsing to idle", + OPVTIslandCurrentMode, OPVTIslandWallClockMs() - heartbeat); + OPVTIslandExpansionLevel = 0; + NSDictionary *idle = @{ + @"mode": @"idle", + @"subtitle": @"OpenPhone", + @"accent": @"cyan", + @"sequence": @(OPVTIslandLastSequence) + }; + OPVTIslandCurrentState = idle; + OPVTIslandApplyState(idle); +} + +static void OPVTIslandStartLivenessTimer(void) { + if (OPVTIslandLivenessTimer) { + return; + } + OPVTIslandLivenessTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, + dispatch_get_main_queue()); + dispatch_source_set_timer(OPVTIslandLivenessTimer, + dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC), + 5 * NSEC_PER_SEC, 1 * NSEC_PER_SEC); + dispatch_source_set_event_handler(OPVTIslandLivenessTimer, ^{ + OPVTIslandCheckLiveness(); + }); + dispatch_resume(OPVTIslandLivenessTimer); +} + static void OPVTIslandNotificationCallback(int token) { (void)token; OPVTIslandRefreshFromDisk(); @@ -6936,6 +7475,10 @@ static void OPVTStartIslandStatusObserver(void) { @"sequence": @(0) }); } + // Guard against a frozen working pill left by a dead daemon: on boot + // (e.g. a respring that reloaded a stale snapshot) and continuously. + OPVTIslandCheckLiveness(); + OPVTIslandStartLivenessTimer(); }); }