From 42098187b1e0d1abad52e80f44ba60d64715d26a Mon Sep 17 00:00:00 2001 From: somew1nd Date: Mon, 28 Sep 2026 17:29:02 -0700 Subject: [PATCH 1/2] refactor: organize third-party code --- .github/workflows/ci.yml | 1 + Dockerfile | 1 + README.md | 6 +- README_CN.md | 6 +- apps/admin_console/replay_manager.py | 2 +- apps/admin_console/routers/system.py | 2 +- artemis/__init__.py | 4 + artemis/_legacy_imports.py | 94 ++ artemis/agents/checker/checker.py | 2 +- artemis/agents/diagnoser/diagnoser.py | 2 +- artemis/agents/explorer/explorer.py | 2 +- artemis/agents/explorer/native_runner.py | 2 +- artemis/agents/explorer/perception_tools.py | 2 +- artemis/agents/explorer/run_setup.py | 2 +- artemis/agents/explorer/universal_runner.py | 2 +- artemis/agents/flash/context_compressor.py | 2 +- artemis/agents/flash/runner.py | 2 +- artemis/agents/flash/summarizer.py | 2 +- .../history_analyzer/history_analyzer.py | 2 +- .../agents/image_processor/image_processor.py | 2 +- artemis/agents/log_analyzer/log_analyzer.py | 21 +- .../agents/log_analyzer/output_analyzer.py | 2 +- .../agents/object_detector/object_detector.py | 2 +- artemis/agents/operator/operator.py | 10 +- artemis/agents/operator/prompts.py | 2 +- artemis/agents/outputter/outputter.md | 8 + artemis/agents/outputter/outputter.py | 757 +++++++----- artemis/agents/planner/planner.py | 29 +- artemis/agents/summarizer/summarizer.py | 2 +- artemis/agents/validator/action_execution.py | 2 +- artemis/agents/validator/execution_loop.py | 2 +- .../agents/validator/precondition_pixel.py | 2 +- artemis/agents/validator/precondition_xml.py | 2 +- artemis/agents/validator/tool_declarations.py | 2 +- artemis/agents/validator/validator.py | 10 +- artemis/agents/video_analyzer/audio_native.py | 2 +- artemis/agents/video_analyzer/blackboard.py | 4 +- .../agents/video_analyzer/chunk_agentic.py | 2 +- .../video_analyzer/chunk_conversation.py | 2 +- artemis/agents/video_analyzer/chunk_native.py | 2 +- artemis/agents/video_analyzer/gemini_files.py | 2 +- .../video_analyzer/native_coordinator.py | 2 +- .../agents/video_analyzer/universal_engine.py | 2 +- .../agents/video_analyzer/video_analyzer.py | 26 +- artemis/clients/accessibility_client.py | 35 +- artemis/clients/adb_tunnel.py | 330 ----- artemis/clients/screen_client_factory.py | 7 +- artemis/config/__init__.py | 14 +- artemis/config/agent.py | 4 +- artemis/config/constants.py | 2 - artemis/config/core/engine.py | 2 +- artemis/config/llm.py | 187 +-- artemis/config/runtime.py | 2 +- artemis/config/settings.py | 2 +- artemis/context.py | 98 +- artemis/controllers/controller_factory.py | 2 +- artemis/controllers/device_controller.py | 221 +--- artemis/controllers/unified_controller.py | 275 +---- artemis/core/diagnostics/adb_keys.py | 2 +- .../core/diagnostics/adb_server_connection.py | 2 +- artemis/core/diagnostics/device_smoke.py | 2 +- artemis/core/diagnostics/emulator_manager.py | 2 +- artemis/core/diagnostics/engine.py | 2 +- artemis/core/diagnostics/hierarchy_parity.py | 6 +- artemis/core/diagnostics/probes/adb_probe.py | 2 +- artemis/data_engine/engine.py | 2 +- artemis/data_engine/history_reader.py | 2 +- artemis/data_engine/storage.py | 2 +- artemis/drivers/android/adb_driver.py | 12 +- artemis/drivers/android/hierarchy.py | 2 +- artemis/drivers/android/input_ime.py | 2 +- artemis/drivers/android/recorder.py | 2 +- artemis/drivers/base.py | 20 +- artemis/drivers/cloud/cloud_driver.py | 2 +- artemis/drivers/factory.py | 2 +- artemis/drivers/types.py | 78 +- artemis/graph/checkpoints.py | 2 +- artemis/graph/graph.py | 51 +- artemis/graph/perception.py | 2 +- artemis/graph/state.py | 8 +- artemis/interfaces/cli/commands/batch.py | 4 +- artemis/interfaces/cli/commands/helper.py | 2 +- artemis/interfaces/cli/commands/mcp.py | 2 +- artemis/interfaces/cli/commands/run.py | 125 +- artemis/interfaces/cli/commands/server.py | 2 +- artemis/interfaces/cli/commands/trace.py | 2 +- artemis/interfaces/cli/main.py | 2 +- artemis/llm/router.py | 2 +- artemis/mcp/action_executor.py | 2 +- artemis/mcp/action_server.py | 2 +- artemis/mcp/action_session.py | 2 +- artemis/mcp/actuators/adb.py | 4 +- artemis/mcp/adb_server.py | 4 +- artemis/mcp/observation.py | 2 +- artemis/memory/__init__.py | 2 +- artemis/memory/chunking.py | 2 +- artemis/memory/context_policy.py | 2 +- artemis/memory/step_memory.py | 2 +- artemis/memory/transcript.py | 2 +- artemis/runtime/awake_lease.py | 2 +- artemis/runtime/awake_service.py | 2 +- artemis/runtime/daemon_client.py | 2 +- artemis/runtime/device_pool.py | 2 +- artemis/runtime/helper_manager.py | 2 +- artemis/runtime/server_lifecycle.py | 2 +- artemis/runtime/supervisor.py | 2 +- artemis/sdk/agent.py | 1077 +---------------- artemis/sdk/builders/__init__.py | 4 +- artemis/sdk/builders/agent_config_builder.py | 274 +---- artemis/sdk/examples/app_lock_messaging.py | 90 -- .../sdk/examples/simple_photo_organizer.py | 111 -- .../examples/smart_notification_assistant.py | 235 ---- .../examples/video_transcription_example.py | 110 -- artemis/sdk/run_outcome.py | 58 + artemis/sdk/types/__init__.py | 6 +- artemis/sdk/types/agent.py | 93 +- artemis/sdk/types/platform.py | 210 ---- artemis/services/llm.py | 277 +---- artemis/services/token_meter.py | 2 +- artemis/telemetry/tracer.py | 2 +- artemis/tools/base.py | 2 +- artemis/tools/command_tool.py | 8 +- artemis/tools/committee_tool.py | 4 +- artemis/tools/diagnoser_submit_answer_tool.py | 2 +- artemis/tools/diagnostic_tool.py | 4 +- artemis/tools/explorer_tool.py | 4 +- artemis/tools/history/__init__.py | 2 +- artemis/tools/history/screenshot.py | 2 +- artemis/tools/history/search.py | 2 +- artemis/tools/image_processor_tool.py | 4 +- artemis/tools/index.py | 23 +- artemis/tools/log_tool.py | 4 +- artemis/tools/mobile/launch_app.py | 73 +- artemis/tools/mobile/log_utils.py | 4 +- artemis/tools/mobile/ocr.py | 2 +- artemis/tools/mobile/read_hierarchy.py | 4 +- artemis/tools/mobile/read_logs.py | 2 +- artemis/tools/mobile/search_logs.py | 2 +- artemis/tools/object_detection_tool.py | 4 +- artemis/tools/scratchpad.py | 11 +- artemis/tools/tool_wrapper.py | 30 +- artemis/tools/types.py | 72 -- artemis/tools/utils.py | 392 ------ artemis/tools/video_tool.py | 4 +- artemis/tools/wait_tool.py | 4 +- artemis/utils/app_launch_utils.py | 203 +--- artemis/utils/cli_selection.py | 154 --- artemis/utils/conversations.py | 45 - artemis/utils/credentials_validator.py | 2 +- artemis/utils/cython_compat.py | 45 + artemis/utils/image_hash.py | 2 +- artemis/utils/notes.py | 2 +- artemis/utils/ocr_api.py | 2 +- artemis/utils/ocr_xml_fusion.py | 2 +- artemis/utils/recorder.py | 60 - artemis/utils/requests_utils.py | 51 - artemis/utils/ui_filter.py | 2 +- artemis/utils/ui_hierarchy.py | 166 --- artemis/utils/video.py | 453 +------ mcp_server/tools/diagnose.py | 2 +- scripts/quality_ratchet.py | 7 +- scripts/real_device_video_acceptance.py | 2 +- setup.py | 7 +- tests/integration/test_physical_actions.py | 2 +- tests/integration/test_remote_adb.py | 2 +- tests/unit/agents/test_outputter.py | 455 ++++--- .../unit/clients/test_accessibility_client.py | 6 +- .../clients/test_screen_client_factory.py | 2 +- .../test_latency_optimizations_deep.py | 8 +- tests/unit/mcp/test_mcp_stdio_lifecycle.py | 8 +- .../unit/sdk/test_agent_trace_finalization.py | 4 +- tests/unit/test_drivers.py | 3 +- tests/unit/test_hierarchy_fidelity.py | 9 +- tests/unit/test_llm_gateway.py | 43 + tests/unit/test_third_party_boundary.py | 111 ++ tests/unit/test_ui_automator_client_awake.py | 23 +- tests/unit/test_unified_controller_video.py | 8 +- tests/unit/tools/test_launch_app.py | 14 +- tests/unit/tools/test_utils.py | 457 ------- tests/unit/utils/test_app_launch_utils.py | 27 +- tests/unit/utils/test_media.py | 2 +- tests/unit/utils/test_ui_hierarchy.py | 246 ---- third_party/mobile_use/LICENSE | 190 +++ third_party/mobile_use/METADATA | 27 + third_party/mobile_use/NOTICE | 12 + third_party/mobile_use/__init__.py | 19 + .../mobile_use/agents}/__init__.py | 2 - .../mobile_use/agents/hopper/__init__.py | 7 - .../mobile_use}/agents/hopper/hopper.md | 6 +- .../mobile_use}/agents/hopper/hopper.py | 28 +- third_party/mobile_use/clients/__init__.py | 13 + .../clients/ui_automator_client.py | 63 +- third_party/mobile_use/config/__init__.py | 13 + third_party/mobile_use/config/llm.py | 183 +++ .../mobile_use}/config/output.py | 31 +- third_party/mobile_use/context.py | 119 ++ .../mobile_use/controllers/__init__.py | 13 + .../platform_specific_commands_controller.py | 11 +- third_party/mobile_use/controllers/types.py | 92 ++ .../controllers/unified_controller.py | 183 +++ third_party/mobile_use/graph.py | 54 + third_party/mobile_use/llm_service.py | 193 +++ third_party/mobile_use/main.py | 130 ++ third_party/mobile_use/sdk/__init__.py | 13 + third_party/mobile_use/sdk/agent.py | 1060 ++++++++++++++++ .../mobile_use/sdk/builders/__init__.py | 13 + .../sdk/builders/agent_config_builder.py | 225 ++++ .../mobile_use}/sdk/builders/index.py | 8 +- .../sdk/builders/task_request_builder.py | 28 +- third_party/mobile_use/sdk/types/__init__.py | 13 + third_party/mobile_use/sdk/types/agent.py | 98 ++ .../mobile_use}/sdk/types/exceptions.py | 6 +- .../mobile_use}/sdk/types/task.py | 29 +- third_party/mobile_use/testing.py | 59 + third_party/mobile_use/tools/__init__.py | 13 + third_party/mobile_use/tools/index.py | 44 + .../mobile_use/tools/mobile/__init__.py | 13 + .../mobile_use/tools/mobile/launch_app.py | 89 ++ third_party/mobile_use/tools/tool_wrapper.py | 52 + third_party/mobile_use/utils/__init__.py | 13 + .../mobile_use/utils/app_launch_utils.py | 217 ++++ .../mobile_use}/utils/cli_helpers.py | 24 +- .../mobile_use}/utils/decorators.py | 21 +- .../mobile_use}/utils/file.py | 6 +- .../mobile_use}/utils/logger.py | 7 +- .../mobile_use}/utils/media.py | 9 +- .../mobile_use}/utils/shell_utils.py | 6 +- third_party/mobile_use/utils/video.py | 398 ++++++ 228 files changed, 5433 insertions(+), 6877 deletions(-) create mode 100644 artemis/_legacy_imports.py delete mode 100644 artemis/clients/adb_tunnel.py delete mode 100644 artemis/sdk/examples/app_lock_messaging.py delete mode 100644 artemis/sdk/examples/simple_photo_organizer.py delete mode 100644 artemis/sdk/examples/smart_notification_assistant.py delete mode 100644 artemis/sdk/examples/video_transcription_example.py create mode 100644 artemis/sdk/run_outcome.py delete mode 100644 artemis/sdk/types/platform.py delete mode 100644 artemis/tools/types.py delete mode 100644 artemis/tools/utils.py delete mode 100644 artemis/utils/cli_selection.py delete mode 100644 artemis/utils/conversations.py create mode 100644 artemis/utils/cython_compat.py delete mode 100644 artemis/utils/recorder.py delete mode 100644 artemis/utils/requests_utils.py delete mode 100644 artemis/utils/ui_hierarchy.py create mode 100644 tests/unit/test_third_party_boundary.py delete mode 100644 tests/unit/tools/test_utils.py delete mode 100644 tests/unit/utils/test_ui_hierarchy.py create mode 100644 third_party/mobile_use/LICENSE create mode 100644 third_party/mobile_use/METADATA create mode 100644 third_party/mobile_use/NOTICE create mode 100644 third_party/mobile_use/__init__.py rename {artemis/sdk/examples => third_party/mobile_use/agents}/__init__.py (92%) rename artemis/utils/time.py => third_party/mobile_use/agents/hopper/__init__.py (78%) rename {artemis => third_party/mobile_use}/agents/hopper/hopper.md (74%) rename {artemis => third_party/mobile_use}/agents/hopper/hopper.py (80%) create mode 100644 third_party/mobile_use/clients/__init__.py rename {artemis => third_party/mobile_use}/clients/ui_automator_client.py (91%) create mode 100644 third_party/mobile_use/config/__init__.py create mode 100644 third_party/mobile_use/config/llm.py rename {artemis => third_party/mobile_use}/config/output.py (81%) create mode 100644 third_party/mobile_use/context.py create mode 100644 third_party/mobile_use/controllers/__init__.py rename {artemis => third_party/mobile_use}/controllers/platform_specific_commands_controller.py (92%) create mode 100644 third_party/mobile_use/controllers/types.py create mode 100644 third_party/mobile_use/controllers/unified_controller.py create mode 100644 third_party/mobile_use/graph.py create mode 100644 third_party/mobile_use/llm_service.py create mode 100644 third_party/mobile_use/main.py create mode 100644 third_party/mobile_use/sdk/__init__.py create mode 100644 third_party/mobile_use/sdk/agent.py create mode 100644 third_party/mobile_use/sdk/builders/__init__.py create mode 100644 third_party/mobile_use/sdk/builders/agent_config_builder.py rename {artemis => third_party/mobile_use}/sdk/builders/index.py (72%) rename {artemis => third_party/mobile_use}/sdk/builders/task_request_builder.py (91%) create mode 100644 third_party/mobile_use/sdk/types/__init__.py create mode 100644 third_party/mobile_use/sdk/types/agent.py rename {artemis => third_party/mobile_use}/sdk/types/exceptions.py (93%) rename {artemis => third_party/mobile_use}/sdk/types/task.py (91%) create mode 100644 third_party/mobile_use/testing.py create mode 100644 third_party/mobile_use/tools/__init__.py create mode 100644 third_party/mobile_use/tools/index.py create mode 100644 third_party/mobile_use/tools/mobile/__init__.py create mode 100644 third_party/mobile_use/tools/mobile/launch_app.py create mode 100644 third_party/mobile_use/tools/tool_wrapper.py create mode 100644 third_party/mobile_use/utils/__init__.py create mode 100644 third_party/mobile_use/utils/app_launch_utils.py rename {artemis => third_party/mobile_use}/utils/cli_helpers.py (60%) rename {artemis => third_party/mobile_use}/utils/decorators.py (85%) rename {artemis => third_party/mobile_use}/utils/file.py (78%) rename {artemis => third_party/mobile_use}/utils/logger.py (96%) rename {artemis => third_party/mobile_use}/utils/media.py (95%) rename {artemis => third_party/mobile_use}/utils/shell_utils.py (83%) create mode 100644 third_party/mobile_use/utils/video.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2be0ff5d5..8d1ce0c5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,7 @@ jobs: --cov=artemis --cov=mcp_server --cov=apps/admin_console + --cov=third_party --cov-fail-under=60 frontend: diff --git a/Dockerfile b/Dockerfile index 16e63d5d0..61a8c1b43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ WORKDIR /build COPY artemis/ ./artemis/ COPY apps/ ./apps/ COPY mcp_server/ ./mcp_server/ +COPY third_party/ ./third_party/ COPY packages/artemis-client/ ./packages/artemis-client/ COPY config/ ./config/ COPY pyproject.toml setup.py README.md LICENSE ./ diff --git a/README.md b/README.md index d18836053..8683bcce3 100644 --- a/README.md +++ b/README.md @@ -315,4 +315,8 @@ Contributions are warmly welcomed! This project is licensed under the [Apache License 2.0](LICENSE). -This project includes source code developed by [Minitap, Inc.](https://github.com/minitap-ai/mobile-use). \ No newline at end of file +### Third-party code + +Code not owned by Google lives under [`third_party/`](third_party/), one directory per upstream project, each with its own `LICENSE` and `METADATA`. + +- [`third_party/mobile_use`](third_party/mobile_use/) – portions of [mobile-use](https://github.com/minitap-ai/mobile-use) by Minitap, Inc., Apache License 2.0. diff --git a/README_CN.md b/README_CN.md index 7c425c695..67f461c58 100644 --- a/README_CN.md +++ b/README_CN.md @@ -313,4 +313,8 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求： 本项目基于 [Apache License 2.0](LICENSE) 协议开源。 -本项目包含由 [Minitap, Inc.](https://github.com/minitap-ai/mobile-use) 开发的源代码。 +### 第三方代码 + +非 Google 所有的代码位于 [`third_party/`](third_party/) 目录下，每个上游项目一个子目录，并各自附带 `LICENSE` 与 `METADATA`。 + +- [`third_party/mobile_use`](third_party/mobile_use/) – 来自 Minitap, Inc. 的 [mobile-use](https://github.com/minitap-ai/mobile-use) 的部分代码，Apache License 2.0。 diff --git a/apps/admin_console/replay_manager.py b/apps/admin_console/replay_manager.py index 7174d64e9..7a3ea5506 100644 --- a/apps/admin_console/replay_manager.py +++ b/apps/admin_console/replay_manager.py @@ -158,7 +158,7 @@ def _init_device(self, device_id: str = None): from adbutils import AdbClient try: - from artemis.clients.ui_automator_client import UIAutomatorClient + from third_party.mobile_use.clients.ui_automator_client import UIAutomatorClient except ImportError: raise ImportError( "Failed to import UIAutomatorClient. Ensure artemis package is installed in path." diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index 5127eca64..b809d57e6 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -376,7 +376,7 @@ async def get_model_config_and_env(): import os from artemis.config.paths import get_config_path, get_env_file from artemis.config import settings - from artemis.utils.file import load_jsonc + from third_party.mobile_use.utils.file import load_jsonc from artemis.config.settings import is_placeholder_key diff --git a/artemis/__init__.py b/artemis/__init__.py index 658bd193b..d4ba44cfa 100644 --- a/artemis/__init__.py +++ b/artemis/__init__.py @@ -15,6 +15,10 @@ """☕ ARTEMIS: Autonomous Multimodal Android Agent & Testing Framework.""" from artemis._version import __version__ +from artemis import _legacy_imports + +# Old import paths of the modules moved to third_party/ (deprecated). +_legacy_imports.install() from artemis.interfaces.sdk.client import ArtemisClient, ConcurrencyMode from artemis.interfaces.sdk.task import StreamEvent, StreamEventType, Task diff --git a/artemis/_legacy_imports.py b/artemis/_legacy_imports.py new file mode 100644 index 000000000..57b7686bd --- /dev/null +++ b/artemis/_legacy_imports.py @@ -0,0 +1,94 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Old import paths of the modules moved to ``third_party/mobile_use``. + +An old path resolves to the new module object itself, so isinstance checks, +module-level state and ``patch("old.path.x")`` still work. The first import of +an old path emits a ``DeprecationWarning``. + +This is a meta path finder rather than one stub file per path because +``artemis.agents.hopper`` is a package; a stub file would load +``artemis.agents.hopper.hopper`` a second time. +""" + +import importlib +import importlib.abc +import importlib.machinery +import sys +from types import ModuleType +import warnings + +_NEW_ROOT = "third_party.mobile_use" + +# Old module path -> new module path. +LEGACY_MODULES: dict[str, str] = { + "artemis.agents.hopper": f"{_NEW_ROOT}.agents.hopper", + "artemis.agents.hopper.hopper": f"{_NEW_ROOT}.agents.hopper.hopper", + "artemis.clients.ui_automator_client": f"{_NEW_ROOT}.clients.ui_automator_client", + "artemis.config.output": f"{_NEW_ROOT}.config.output", + "artemis.controllers.platform_specific_commands_controller": ( + f"{_NEW_ROOT}.controllers.platform_specific_commands_controller" + ), + "artemis.sdk.builders.index": f"{_NEW_ROOT}.sdk.builders.index", + "artemis.sdk.builders.task_request_builder": f"{_NEW_ROOT}.sdk.builders.task_request_builder", + "artemis.sdk.types.exceptions": f"{_NEW_ROOT}.sdk.types.exceptions", + "artemis.sdk.types.task": f"{_NEW_ROOT}.sdk.types.task", + "artemis.utils.cli_helpers": f"{_NEW_ROOT}.utils.cli_helpers", + "artemis.utils.decorators": f"{_NEW_ROOT}.utils.decorators", + "artemis.utils.file": f"{_NEW_ROOT}.utils.file", + "artemis.utils.logger": f"{_NEW_ROOT}.utils.logger", + "artemis.utils.media": f"{_NEW_ROOT}.utils.media", + "artemis.utils.shell_utils": f"{_NEW_ROOT}.utils.shell_utils", +} + + +class _AliasLoader(importlib.abc.Loader): + """Loader that hands back an already-importable module under another name.""" + + def __init__(self, target: str) -> None: + self._target = target + self._target_spec: importlib.machinery.ModuleSpec | None = None + + def create_module(self, spec: importlib.machinery.ModuleSpec) -> ModuleType: + warnings.warn( + f"'{spec.name}' is deprecated; import '{self._target}' instead.", + DeprecationWarning, + stacklevel=2, + ) + module = importlib.import_module(self._target) + self._target_spec = module.__spec__ + return module + + def exec_module(self, module: ModuleType) -> None: + # Nothing to execute. The import system has overwritten __spec__ with the + # old name; put the real spec back so reload() and pickling still work. + if self._target_spec is not None: + module.__spec__ = self._target_spec + + +class _LegacyModuleFinder(importlib.abc.MetaPathFinder): + """Resolve entries of :data:`LEGACY_MODULES` to their new modules.""" + + def find_spec(self, fullname, path=None, target=None): + new_name = LEGACY_MODULES.get(fullname) + if new_name is None: + return None + return importlib.machinery.ModuleSpec(fullname, _AliasLoader(new_name)) + + +def install() -> None: + """Register the finder once, ahead of the default path-based finders.""" + if not any(isinstance(finder, _LegacyModuleFinder) for finder in sys.meta_path): + sys.meta_path.insert(0, _LegacyModuleFinder()) diff --git a/artemis/agents/checker/checker.py b/artemis/agents/checker/checker.py index 40a90e292..52dff2ca2 100644 --- a/artemis/agents/checker/checker.py +++ b/artemis/agents/checker/checker.py @@ -57,7 +57,6 @@ split_multimodal_result, tool_result_messages, ) -from artemis.utils.logger import get_logger from artemis.utils.task_tree import format_actions_clean from artemis.utils.ocr_api import is_ocr_configured, perform_ocr from artemis.utils.ocr_xml_fusion import ( @@ -67,6 +66,7 @@ fuse_ocr_with_xml, ) from artemis.utils.visualization import format_minimal_list_with_elements +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index bb62ff38a..13d80317c 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -48,12 +48,12 @@ ) from artemis.tools.video_tool import get_video_analyzer_tool from artemis.tools.wait_tool import get_wait_tool -from artemis.utils.logger import get_logger from artemis.memory.context_policy import build_history_for from artemis.utils.task_tree import ( get_active_subgoal_hashes, get_recent_subgoal_hashes, ) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/explorer/explorer.py b/artemis/agents/explorer/explorer.py index 8855c2361..f39dfab28 100644 --- a/artemis/agents/explorer/explorer.py +++ b/artemis/agents/explorer/explorer.py @@ -50,8 +50,8 @@ from artemis.data_engine.trace import trace from artemis.graph.state import State from artemis.llm.google import is_gemini_model, strip_provider_prefix -from artemis.utils.logger import get_logger from artemis.utils.ocr_api import is_ocr_configured +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/explorer/native_runner.py b/artemis/agents/explorer/native_runner.py index 3f62c998b..4b29bd1b0 100644 --- a/artemis/agents/explorer/native_runner.py +++ b/artemis/agents/explorer/native_runner.py @@ -44,7 +44,7 @@ retry_policy_for, ) from artemis.services.llm import _record_llm_event, _record_llm_retry -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/explorer/perception_tools.py b/artemis/agents/explorer/perception_tools.py index a8a635001..32be2a784 100644 --- a/artemis/agents/explorer/perception_tools.py +++ b/artemis/agents/explorer/perception_tools.py @@ -37,8 +37,8 @@ from artemis.agents.object_detector.object_detector import _run_object_detection from artemis.config import settings from artemis.data_engine.trace import trace -from artemis.utils.logger import get_logger from artemis.utils.visualization import draw_dots +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/explorer/run_setup.py b/artemis/agents/explorer/run_setup.py index 60f7f55bc..a25c81feb 100644 --- a/artemis/agents/explorer/run_setup.py +++ b/artemis/agents/explorer/run_setup.py @@ -45,10 +45,10 @@ from artemis.config import settings from artemis.data_engine.storage import StorageManager from artemis.graph.state import State -from artemis.utils.logger import get_logger from artemis.utils.ocr_api import is_ocr_configured, perform_ocr from artemis.utils.ocr_xml_fusion import fuse_ocr_with_xml from artemis.utils.visualization import draw_dots, format_minimal_list_with_elements +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/explorer/universal_runner.py b/artemis/agents/explorer/universal_runner.py index 9a123ca47..c5b5bc89a 100644 --- a/artemis/agents/explorer/universal_runner.py +++ b/artemis/agents/explorer/universal_runner.py @@ -35,7 +35,7 @@ from artemis.agents.explorer.tool_declarations import UNIVERSAL_EXPLORER_TOOLS from artemis.graph.state import State from artemis.services.llm import get_llm -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/flash/context_compressor.py b/artemis/agents/flash/context_compressor.py index 35b494b0a..468f31090 100644 --- a/artemis/agents/flash/context_compressor.py +++ b/artemis/agents/flash/context_compressor.py @@ -30,7 +30,7 @@ from artemis.memory.step_memory import StepMemoryService from artemis.memory.transcript import EPHEMERAL_BLOCKS_KEY -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 936f60655..3ea3f6865 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -94,7 +94,7 @@ COORDINATE_SPACE_NORMALIZED, parse_swipe_parameters, ) -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 56c5e0985..5b8a3cf98 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -38,9 +38,9 @@ from artemis.memory.step_memory import JobKey, StepMemoryService from artemis.services.llm import RobustChatModelWrapper, get_google_llm, get_llm from artemis.services.token_meter import record_llm_usage -from artemis.utils.logger import get_logger from artemis.utils.task_tree import format_actions_clean from artemis.utils.visualization import draw_action_overlay_on_image +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/history_analyzer/history_analyzer.py b/artemis/agents/history_analyzer/history_analyzer.py index ce9739a95..ced146403 100644 --- a/artemis/agents/history_analyzer/history_analyzer.py +++ b/artemis/agents/history_analyzer/history_analyzer.py @@ -40,9 +40,9 @@ split_multimodal_result, tool_result_messages, ) -from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path from artemis.utils.task_tree import get_active_subgoal_hashes +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/image_processor/image_processor.py b/artemis/agents/image_processor/image_processor.py index 05e37abbe..4ce526984 100644 --- a/artemis/agents/image_processor/image_processor.py +++ b/artemis/agents/image_processor/image_processor.py @@ -33,8 +33,8 @@ from artemis.core.tool_declaration import ToolDeclaration from artemis.data_engine.trace import TraceSpan, trace from artemis.services.llm import get_llm -from artemis.utils.logger import get_logger from artemis.utils.python_executor import PythonExecutor +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/log_analyzer/log_analyzer.py b/artemis/agents/log_analyzer/log_analyzer.py index 126b770a9..c95adec7c 100644 --- a/artemis/agents/log_analyzer/log_analyzer.py +++ b/artemis/agents/log_analyzer/log_analyzer.py @@ -19,24 +19,7 @@ from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage from langchain_core.tools import BaseTool, StructuredTool from pydantic import BaseModel, Field - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass +from artemis.utils.cython_compat import CyFunctionDetector class SpawnLogReaderArgs(BaseModel): @@ -71,7 +54,7 @@ class SpawnLogReaderArgs(BaseModel): get_tool_result_content, invoke_tool_with_injection, ) -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/log_analyzer/output_analyzer.py b/artemis/agents/log_analyzer/output_analyzer.py index ca5bd604d..0f9a35b37 100644 --- a/artemis/agents/log_analyzer/output_analyzer.py +++ b/artemis/agents/log_analyzer/output_analyzer.py @@ -20,7 +20,7 @@ from artemis.context import ArtemisContext from artemis.services.llm import get_llm, invoke_llm_with_timeout_message from artemis.tools.mobile.search_logs import search_and_merge_logs -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/object_detector/object_detector.py b/artemis/agents/object_detector/object_detector.py index 22b34f2eb..b2acb4f5f 100644 --- a/artemis/agents/object_detector/object_detector.py +++ b/artemis/agents/object_detector/object_detector.py @@ -25,7 +25,7 @@ from artemis.llm.structured import ParseFailure, parse_structured from artemis.services.llm import get_llm -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index dd1a32d94..d24536270 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -47,13 +47,13 @@ compute_smart_swipe_coordinates, parse_swipe_parameters, ) -from artemis.utils.decorators import wrap_with_callbacks -from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path from artemis.utils.plan_grammar import parse_plan from artemis.memory.context_policy import build_history_for from artemis.utils.task_tree import get_active_subgoal_hashes from artemis.utils.visualization import format_minimal_list_with_elements +from third_party.mobile_use.utils.decorators import agent_lifecycle_logging +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -854,11 +854,7 @@ def normalize_name(name: str) -> str: tool_limit_exceeded, ) - @wrap_with_callbacks( - before=lambda: logger.info("Starting Operator Agent..."), - on_success=lambda _: logger.success("Operator Agent"), - on_failure=lambda _: logger.error("Operator Agent"), - ) + @agent_lifecycle_logging("Operator", logger) @trace(type="agent", name="operator") async def __call__(self, state: State): state = strict_state(state, "operator") diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index 258ffb48d..e800de234 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -31,9 +31,9 @@ _format_long_output_response, _is_output_long, ) -from artemis.utils.logger import get_logger from artemis.utils.plan_grammar import parse_plan, render_plan_grammar_spec from artemis.utils.task_tree import SELF_DESCRIBED_MARKER, action_intent_phrase +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/outputter/outputter.md b/artemis/agents/outputter/outputter.md index 8a21164b2..8c460001e 100644 --- a/artemis/agents/outputter/outputter.md +++ b/artemis/agents/outputter/outputter.md @@ -16,6 +16,14 @@ The user explicitly requested to extract the following information: > "{{ output_description }}" {% endif %} +{% if target_schema %} +## Target Output Fields +Your final answer will afterwards be formatted into the JSON schema below. Gather evidence for every field, and make sure your final answer states each value you found (or that it could not be verified). You do not need to write JSON yourself. +```json +{{ target_schema }} +``` +{% endif %} + --- # Instruction diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py index aed600d82..9b75b427f 100644 --- a/artemis/agents/outputter/outputter.py +++ b/artemis/agents/outputter/outputter.py @@ -11,22 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. +import asyncio import json +from functools import lru_cache from pathlib import Path +from typing import Any from jinja2 import Template -from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage +from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, ToolMessage +from langchain_core.tools import BaseTool from artemis.core.tool_failure import ToolFailure, is_tool_failure from artemis.config import OutputConfig from artemis.context import ArtemisContext from artemis.data_engine.trace import trace from artemis.graph.state import State -from artemis.llm.structured import ParseFailure, parse_structured -from artemis.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback +from artemis.llm.structured import ParseFailure, content_to_text, parse_structured +from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message, with_fallback from artemis.tools.scratchpad import ( get_append_note_tool_pure, get_list_notes_tool_pure, @@ -38,340 +39,514 @@ from artemis.tools.history import get_history_tools from artemis.tools.tool_wrapper import ( invoke_tool_with_injection, - split_multimodal_result, tool_result_messages, ) -from artemis.utils.logger import get_logger from artemis.memory.context_policy import build_history_for +from artemis.utils.notes import read_note_content from artemis.utils.task_tree import get_active_subgoal_hashes +from third_party.mobile_use.utils.logger import get_logger from pydantic import BaseModel logger = get_logger(__name__) +MAX_TURNS = 20 + +# Note key of the report the prompt asks for. The UI Task Report card and the +# MCP docs read it from notes/output.md. +OUTPUT_NOTE_KEY = "output" + +# Note writes run one by one in the order issued; all other tools only read +# and can run in parallel. +_WRITE_TOOLS = frozenset({"save_note", "update_note", "append_note"}) + +_MAX_TURNS_ERROR = "Error: Outputter failed to resolve the query within maximum turns." + +_SYSTEM_MESSAGE = ( + "You are the Output Synthesis Agent for an Android UI automation" + " system. Your sole objective is to verify whether the user's initial" + " goal was achieved and synthesize the final report.\n\n## Core" + " Principles\n1. STRICTLY EVIDENCE-BASED & NO HALLUCINATION: Rely ONLY" + " on the actual observed execution history and successful tool outputs." + " Do not assume, reconstruct, or hallucinate any actions, elements, or" + " steps that did not actually occur during this specific run. If a step" + " was skipped or not observed (e.g., because the device was already in" + " the target state), you must explicitly state that it was 'Not" + " Observed / Already in State' rather than fabricating the interaction." + " If a tool call (like `video_analyzer`) fails or returns an error, you" + " must report the failure and the resulting lack of evidence, rather" + " than assuming success or guessing the details. If the user requests a" + " step-by-step guide, you must only include steps that were actually" + " executed and observed; do not invent 'standard' steps to fill in" + " gaps.\n2. NO JARGON IN FINAL ANSWER: The 'Final Answer' must be" + " written in clear, user-friendly language, completely free of" + " agent-specific jargon (e.g., do not mention 'nodes', 'XPath'," + " 'selectors', 'ReAct', or 'tools').\n\n## Tool Usage Guidelines\n-" + " LAZY VERIFICATION: If the execution history and the final screenshot" + " already in your context provide undeniable proof of the outcome, do" + " not call tools. Directly output your conclusion.\n- ACTIVE RETRIEVAL:" + " You should only call tools (e.g., `search_history` to locate the" + " step, `replay_steps` for its full record, `get_step_screenshot` for" + " its image) if you need to extract specific hidden data (like" + " verification codes, tracking numbers) or if the final state is" + " ambiguous.\n- BATCH INDEPENDENT LOOKUPS: When you need several" + " lookups that do not depend on each other, request them together in" + " the same turn instead of one per turn.\n- VIDEO ANALYSIS COST:" + " The `video_analyzer` tool is expensive. Use it only when necessary" + " (e.g., to verify video playback or motion).\n- PERSISTENT WRITE: If" + " you extract important information requested by the user (such as a" + " verification code, tracking number, or a text summary), you should" + " use `save_note` or `update_note` to write it directly into persistent" + " notes so it can be retrieved by other agents or the system." +) -@trace(type="agent", name="outputter") -async def outputter( - ctx: ArtemisContext, - output_config: OutputConfig, - graph_output: State, - plan_and_history: str | None = None, -) -> dict: - logger.info("Starting Outputter Agent") +_FINAL_TURN_NOTICE = ( + "This is your final turn and tools are no longer available. Write your" + " final answer to the user now, using only the evidence gathered so far." + " State plainly anything you could not verify." +) - # Fetch plan and history if not provided - if plan_and_history is None and ctx.data_engine: - try: - history = ctx.data_engine.get_agent_friendly_steps() - - # Read current plan if it exists and base_dir is valid - current_plan = "" - base_dir = getattr(ctx.data_engine, "base_dir", None) - if base_dir and isinstance(base_dir, (str, Path)): - notes_dir = Path(base_dir) / "notes" - current_path = notes_dir / "task_plan.md" - if current_path.exists(): - try: - current_plan = current_path.read_text(encoding="utf-8") - except Exception as e: - logger.error(f"Failed to read current plan for outputter: {e}") - - if history: - active_subgoal_hash = "default" - if current_plan: - try: - active_subgoal_hash, _ = get_active_subgoal_hashes(current_plan) - except Exception as e: - logger.error(f"Failed to parse active subgoal in outputter: {e}") - - plan_and_history = build_history_for( - "outputter", - current_plan, - history, - active_subgoal_hash, - engine=ctx.data_engine, - ) - except Exception as e: - logger.error(f"Failed to resolve plan and history in outputter: {e}") - - system_message = ( - "You are the Output Synthesis Agent for an Android UI automation" - " system. Your sole objective is to verify whether the user's initial" - " goal was achieved and synthesize the final report.\n\n## Core" - " Principles\n1. STRICTLY EVIDENCE-BASED & NO HALLUCINATION: Rely ONLY" - " on the actual observed execution history and successful tool outputs." - " Do not assume, reconstruct, or hallucinate any actions, elements, or" - " steps that did not actually occur during this specific run. If a step" - " was skipped or not observed (e.g., because the device was already in" - " the target state), you must explicitly state that it was 'Not" - " Observed / Already in State' rather than fabricating the interaction." - " If a tool call (like `video_analyzer`) fails or returns an error, you" - " must report the failure and the resulting lack of evidence, rather" - " than assuming success or guessing the details. If the user requests a" - " step-by-step guide, you must only include steps that were actually" - " executed and observed; do not invent 'standard' steps to fill in" - " gaps.\n2. NO JARGON IN FINAL ANSWER: The 'Final Answer' must be" - " written in clear, user-friendly language, completely free of" - " agent-specific jargon (e.g., do not mention 'nodes', 'XPath'," - " 'selectors', 'ReAct', or 'tools').\n\n## Tool Usage Guidelines\n-" - " LAZY VERIFICATION: If the execution history and the final screenshot" - " already in your context provide undeniable proof of the outcome, do" - " not call tools. Directly output your conclusion.\n- ACTIVE RETRIEVAL:" - " You should only call tools (e.g., `search_history` to locate the" - " step, `replay_steps` for its full record, `get_step_screenshot` for" - " its image) if you need to extract specific hidden data (like" - " verification codes, tracking numbers) or if the final state is" - " ambiguous.\n- VIDEO ANALYSIS COST:" - " The `video_analyzer` tool is expensive. Use it only when necessary" - " (e.g., to verify video playback or motion).\n- PERSISTENT WRITE: If" - " you extract important information requested by the user (such as a" - " verification code, tracking number, or a text summary), you should" - " use `save_note` or `update_note` to write it directly into persistent" - " notes so it can be retrieved by other agents or the system." - ) +_EMPTY_REPLY_NUDGE = ( + "Your previous reply was empty. If you still need evidence, call the" + " relevant tools; otherwise write your final answer to the user now." +) - render_kwargs = { - "initial_goal": graph_output.initial_goal, - "structured_output": output_config.structured_output, - "output_description": output_config.output_description, - "plan_and_history": plan_and_history, - } - - human_message = Template( - Path(__file__).parent.joinpath("outputter.md").read_text(encoding="utf-8") - ).render(**render_kwargs) - - screenshot_b64 = ( - graph_output.operator_raw_data.get("screenshot_b64") - if graph_output.operator_raw_data - else None - ) +_FORMAT_SYSTEM_MESSAGE = ( + "You are a helpful assistant. Your task is to take the" + " raw execution summary and format it into the" + " requested structured output schema. Do not invent any" + " information. If the raw summary does not contain the" + " required info, leave those fields empty or null." +) - human_message_content = [{"type": "text", "text": human_message}] - # Verification material (only present when the run declared check items): - # the full append-only verdict sequence plus the four-state test summary. - if ctx.data_engine: - try: - from artemis.graph.checkpoints import read_ledger, read_run_outcome +@lru_cache(maxsize=1) +def _prompt_template() -> Template: + return Template(Path(__file__).with_name("outputter.md").read_text(encoding="utf-8")) + + +def _notes_base_dir(ctx: ArtemisContext) -> str | Path | None: + base_dir = getattr(ctx.data_engine, "base_dir", None) if ctx.data_engine else None + return base_dir if isinstance(base_dir, (str, Path)) else None + + +def _resolve_plan_and_history(ctx: ArtemisContext) -> str | None: + """Build the plan and history text passed to the outputter prompt.""" + try: + history = ctx.data_engine.get_agent_friendly_steps() + + # Read current plan if it exists and base_dir is valid + current_plan = "" + base_dir = _notes_base_dir(ctx) + if base_dir: + current_path = Path(base_dir) / "notes" / "task_plan.md" + if current_path.exists(): + try: + current_plan = current_path.read_text(encoding="utf-8") + except Exception as e: + logger.error(f"Failed to read current plan for outputter: {e}") + + if not history: + return None + + active_subgoal_hash = "default" + if current_plan: + try: + active_subgoal_hash, _ = get_active_subgoal_hashes(current_plan) + except Exception as e: + logger.error(f"Failed to parse active subgoal in outputter: {e}") + + return build_history_for( + "outputter", + current_plan, + history, + active_subgoal_hash, + engine=ctx.data_engine, + ) + except Exception as e: + logger.error(f"Failed to resolve plan and history in outputter: {e}") + return None + - ledger_records = read_ledger(ctx.data_engine.base_dir) - run_outcome = read_run_outcome(ctx.data_engine.base_dir) or getattr( - graph_output, "run_outcome", None +def _verification_block(ctx: ArtemisContext, graph_output: State) -> str | None: + """Format the verdict ledger and test summary, or None if the run had no check items.""" + try: + from artemis.graph.checkpoints import read_ledger, read_run_outcome + + ledger_records = read_ledger(ctx.data_engine.base_dir) + run_outcome = read_run_outcome(ctx.data_engine.base_dir) or getattr( + graph_output, "run_outcome", None + ) + if not (ledger_records or run_outcome): + return None + lines = ["--- Verification Verdict Ledger (append-only) ---"] + for r in ledger_records: + lines.append( + f"- attempt {r.get('attempt_id')} [{r.get('kind')}]" + f" '{r.get('item_text')}' -> {r.get('status')}:" + f" {r.get('evidence', '')}" ) - if ledger_records or run_outcome: - lines = ["--- Verification Verdict Ledger (append-only) ---"] - for r in ledger_records: - lines.append( - f"- attempt {r.get('attempt_id')} [{r.get('kind')}]" - f" '{r.get('item_text')}' -> {r.get('status')}:" - f" {r.get('evidence', '')}" - ) - if run_outcome: - tests = run_outcome.get("tests") or {} - lines.append( - "--- Test Summary ---\n" - f"task_status={run_outcome.get('task_status')}," - f" passed={tests.get('passed', 0)}," - f" failed={tests.get('failed', 0)}," - f" inconclusive={tests.get('inconclusive', 0)}," - f" unchecked={tests.get('unchecked', 0)}" - ) - lines.append( - "Report every declared check item with its final state" - " (passed / failed / inconclusive / unchecked) and include" - " the complete verdict sequence above. A failed assertion" - " is a test result to report verbatim, not something to" - " explain away. Items reported unchecked were never" - " judged — never present them as passed." - ) - human_message_content.append({"type": "text", "text": "\n".join(lines)}) - except Exception as e: - logger.error(f"Failed to attach verdict ledger to outputter: {e}") + if run_outcome: + tests = run_outcome.get("tests") or {} + lines.append( + "--- Test Summary ---\n" + f"task_status={run_outcome.get('task_status')}," + f" passed={tests.get('passed', 0)}," + f" failed={tests.get('failed', 0)}," + f" inconclusive={tests.get('inconclusive', 0)}," + f" unchecked={tests.get('unchecked', 0)}" + ) + lines.append( + "Report every declared check item with its final state" + " (passed / failed / inconclusive / unchecked) and include" + " the complete verdict sequence above. A failed assertion" + " is a test result to report verbatim, not something to" + " explain away. Items reported unchecked were never" + " judged — never present them as passed." + ) + return "\n".join(lines) + except Exception as e: + logger.error(f"Failed to attach verdict ledger to outputter: {e}") + return None + - # Run environment facts the model must not guess: which UI-hierarchy source - # served the run and whether it switched mid-way (a fallback to UIAutomator2 - # explains slower steps and possible conflicts with other automation tools). +def _environment_block(ctx: ArtemisContext) -> str | None: + """Describe which UI hierarchy backend served the run. + + Mentions a mid-run switch too: falling back to UIAutomator2 explains slower + steps and conflicts with other automation tools. + """ from artemis.clients.screen_client_factory import hierarchy_backend_sentence relative = ctx.data_engine.get_relative_time if ctx.data_engine else None environment_line = hierarchy_backend_sentence( getattr(ctx, "ui_adb_client", None), relative_time=relative ) - if environment_line: - human_message_content.append( - { - "type": "text", - "text": ( - "## Run Environment (facts, include verbatim under a short " - "'Environment' note only if the source changed mid-run or the " - "user asked about it)" + chr(10) + environment_line - ), - } - ) + if not environment_line: + return None + return ( + "## Run Environment (facts, include verbatim under a short " + "'Environment' note only if the source changed mid-run or the " + "user asked about it)" + chr(10) + environment_line + ) + + +def _target_schema_json(output_config: OutputConfig) -> str | None: + """Return the structured output schema as JSON text for the prompt.""" + schema = output_config.structured_output_schema() + if schema is None: + return None + try: + if isinstance(schema, type) and issubclass(schema, BaseModel): + schema = schema.model_json_schema() + return json.dumps(schema, indent=2, ensure_ascii=False) + except Exception as e: + logger.warning(f"Could not render the structured output schema for outputter: {e}") + return None + + +def _initial_messages( + ctx: ArtemisContext, + output_config: OutputConfig, + graph_output: State, + plan_and_history: str | None, +) -> list[BaseMessage]: + human_message = _prompt_template().render( + initial_goal=graph_output.initial_goal, + structured_output=output_config.structured_output, + output_description=output_config.output_description, + target_schema=_target_schema_json(output_config), + plan_and_history=plan_and_history, + ) + content: list[dict[str, Any]] = [{"type": "text", "text": human_message}] + if ctx.data_engine: + verification = _verification_block(ctx, graph_output) + if verification: + content.append({"type": "text", "text": verification}) + + environment = _environment_block(ctx) + if environment: + content.append({"type": "text", "text": environment}) + + raw_data = graph_output.operator_raw_data + screenshot_b64 = raw_data.get("screenshot_b64") if raw_data else None if screenshot_b64: - human_message_content.append( + content.append( { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, } ) - messages: list[BaseMessage] = [ - SystemMessage(content=system_message), - HumanMessage(content=human_message_content), - ] + return [SystemMessage(content=_SYSTEM_MESSAGE), HumanMessage(content=content)] - # Setup tools (the shared history tools plus notes and video) + +def _outputter_tools(ctx: ArtemisContext) -> list[BaseTool]: + """Return the history, notes and video tools available to the outputter.""" search_tool, replay_tool, screenshot_tool = get_history_tools(ctx) - list_notes_tool = get_list_notes_tool_pure(ctx) - read_note_tool = get_read_note_tool_pure(ctx) - save_note_tool = get_save_note_tool_pure(ctx) - update_note_tool = get_update_note_tool_pure(ctx) - append_note_tool = get_append_note_tool_pure(ctx) - video_analyzer_tool = get_video_analyzer_tool_pure(ctx) - - tools = [ + return [ search_tool, replay_tool, screenshot_tool, - list_notes_tool, - read_note_tool, - save_note_tool, - update_note_tool, - append_note_tool, - video_analyzer_tool, + get_list_notes_tool_pure(ctx), + get_read_note_tool_pure(ctx), + get_save_note_tool_pure(ctx), + get_update_note_tool_pure(ctx), + get_append_note_tool_pure(ctx), + get_video_analyzer_tool_pure(ctx), ] - llm = get_llm(ctx=ctx, name="outputter", is_utils=True) - llm_fallback = get_llm(ctx=ctx, name="outputter", is_utils=True, use_fallback=True) - llm_with_tools = llm.bind_tools(tools=tools) - llm_fallback_with_tools = llm_fallback.bind_tools(tools=tools) +def _tool_name(tool_call: dict) -> str: + name = tool_call["name"] + return name.split(":")[-1] if ":" in name else name + - # ReAct Loop - max_turns = 20 - raw_answer = None - for turn in range(max_turns): +class _Investigation: + """Runs the outputter's tool calls and tracks whether the report was saved.""" + + def __init__(self, tool_map: dict[str, BaseTool], graph_output: State, llm: Any): + self.tool_map = tool_map + self.graph_output = graph_output + self.llm = llm + self.report_written = False + + async def _run_tool(self, tc: dict) -> list[BaseMessage]: + tool_name = _tool_name(tc) + args = tc["args"] + logger.info(f"Outputter executing tool {tool_name} with args: {args}") + try: + selected_tool = self.tool_map.get(tool_name) + if selected_tool: + result = await invoke_tool_with_injection( + tool=selected_tool, + args=args, + tool_call_id=tc["id"], + state=self.graph_output, + record_trace=True, + ) + else: + result = ToolFailure(f"Error: Tool {tool_name} is not supported.") + status = "error" if is_tool_failure(result) else "success" + except Exception as e: + logger.error(f"Error running tool {tool_name}: {e}") + result = f"Error running tool {tool_name}: {e}" + status = "error" + + if ( + tool_name in _WRITE_TOOLS + and status == "success" + and isinstance(args, dict) + and args.get("key") == OUTPUT_NOTE_KEY + ): + self.report_written = True + + # Screenshots are wrapped in whatever message type the provider accepts + # for images; text results stay a plain ToolMessage. + return tool_result_messages(tc["id"], result, name=tool_name, status=status, llm=self.llm) + + async def execute_tool_calls(self, tool_calls: list[dict]) -> list[BaseMessage]: + """Run the tool calls of one turn and return the reply messages. + + Consecutive reads run in parallel; each note write runs on its own, in + order. All ToolMessages come first, in call order, followed by any + HumanMessages carrying images: some providers reject a tool call whose + results are not directly after it. + """ + per_call: list[list[BaseMessage]] = [] + batch: list[dict] = [] + + async def flush() -> None: + if batch: + per_call.extend(await asyncio.gather(*(self._run_tool(tc) for tc in batch))) + batch.clear() + + for tc in tool_calls: + if _tool_name(tc) in _WRITE_TOOLS: + await flush() + per_call.append(await self._run_tool(tc)) + else: + batch.append(tc) + await flush() + + produced = [m for msgs in per_call for m in msgs] + tool_messages = [m for m in produced if isinstance(m, ToolMessage)] + carriers = [m for m in produced if not isinstance(m, ToolMessage)] + return tool_messages + carriers + + +def _read_saved_report(ctx: ArtemisContext) -> str | None: + base_dir = _notes_base_dir(ctx) + if not base_dir: + return None + try: + report = read_note_content(base_dir, OUTPUT_NOTE_KEY).strip() + except Exception as e: + logger.warning(f"Could not read the saved outputter report: {e}") + return None + return report or None + + +async def _run_react_loop( + investigation: _Investigation, + messages: list[BaseMessage], + llm_with_tools: Any, + llm_fallback_with_tools: Any, +) -> str | None: + """Call the model and its tools until it answers; return the answer or None.""" + nudged = False + for turn in range(MAX_TURNS): logger.info(f"Outputter ReAct turn {turn + 1}") + final_turn = turn == MAX_TURNS - 1 + if final_turn: + messages.append(HumanMessage(content=_FINAL_TURN_NOTICE)) response = await with_fallback( - main_call=lambda: invoke_llm_with_timeout_message(llm_with_tools.ainvoke(messages)), + main_call=lambda: invoke_llm_with_timeout_message(acomplete(llm_with_tools, messages)), fallback_call=lambda: invoke_llm_with_timeout_message( - llm_fallback_with_tools.ainvoke(messages) + acomplete(llm_fallback_with_tools, messages) ), ) + text = content_to_text(response.content).strip() - messages.append(response) - - if not response.tool_calls: - raw_answer = response.content - break - - for tc in response.tool_calls: - tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"] - args = tc["args"] - - logger.info(f"Outputter executing tool {tool_name} with args: {args}") - tool_map = { - "search_history": search_tool, - "replay_steps": replay_tool, - "get_step_screenshot": screenshot_tool, - "list_notes": list_notes_tool, - "read_note": read_note_tool, - "save_note": save_note_tool, - "update_note": update_note_tool, - "append_note": append_note_tool, - "video_analyzer": video_analyzer_tool, - } + if response.tool_calls and not final_turn: + messages.append(response) + messages.extend(await investigation.execute_tool_calls(response.tool_calls)) + continue - try: - selected_tool = tool_map.get(tool_name) - if selected_tool: - result = await invoke_tool_with_injection( - tool=selected_tool, - args=args, - tool_call_id=tc["id"], - state=graph_output, - record_trace=True, - ) - else: - result = ToolFailure(f"Error: Tool {tool_name} is not supported.") - text, _ = split_multimodal_result(result) - status = "error" if is_tool_failure(result) else "success" - except Exception as e: - logger.error(f"Error running tool {tool_name}: {e}") - result = f"Error running tool {tool_name}: {e}" - status = "error" - - # Step screenshots enter the conversation in the carrier the - # model's provider accepts; text results stay a plain ToolMessage. - messages.extend( - tool_result_messages(tc["id"], result, name=tool_name, status=status, llm=llm) - ) + if text: + messages.append(response) + return text - if raw_answer is None: - raw_answer = "Error: Outputter failed to resolve the query within maximum turns." - logger.error(raw_answer) + if final_turn: + logger.warning("Outputter final turn produced no answer text.") + return None - # Final Formatting if structured output is requested - if output_config.structured_output: - logger.info("Formatting raw output to structured format...") - schema = None - so = output_config.structured_output - if isinstance(so, dict): - schema = so - elif isinstance(so, BaseModel): - schema = type(so) - elif isinstance(so, type) and issubclass(so, BaseModel): - schema = so - - if schema is not None: - structured_llm = llm.with_structured_output(schema) - structured_llm_fallback = llm_fallback.with_structured_output(schema) - - format_messages = [ - SystemMessage( - content=( - "You are a helpful assistant. Your task is to take the" - " raw execution summary and format it into the" - " requested structured output schema. Do not invent any" - " information. If the raw summary does not contain the" - " required info, leave those fields empty or null." - ) - ), - HumanMessage( - content=( - f"Initial Goal: {graph_output.initial_goal}\nRaw Summary:\n{raw_answer}" - ) - ), - ] - - response = await with_fallback( - main_call=lambda: invoke_llm_with_timeout_message( - structured_llm.ainvoke(format_messages) - ), - fallback_call=lambda: invoke_llm_with_timeout_message( - structured_llm_fallback.ainvoke(format_messages) - ), - ) + if nudged: + logger.warning("Outputter reply was empty again after a nudge.") + return None - if isinstance(response, BaseModel): - return response.model_dump() - return response - - # Fallback to old behavior if it didn't match the template - if isinstance(raw_answer, str): - raw_answer_stripped = raw_answer.strip() - if raw_answer_stripped.startswith(("{", "[", "```")): - parsed = parse_structured(raw_answer_stripped) - if not isinstance(parsed, ParseFailure): - return parsed - # The raw text is a legitimate final answer; the miss is only - # noteworthy because the answer *looked* like JSON. - logger.warning( - "Outputter answer looked like JSON but could not be parsed" - f" ({parsed.error}); returning it as plain text." - ) + # An empty model turn is not kept: Gemini rejects messages without parts. + logger.warning("Outputter reply was empty; asking once more for an answer.") + messages.append(HumanMessage(content=_EMPTY_REPLY_NUDGE)) + nudged = True + return None + + +async def _format_structured( + output_config: OutputConfig, + graph_output: State, + raw_answer: str, + saved_report: str | None, + llm: Any, + llm_fallback: Any, +) -> Any: + """Convert the answer to the configured schema with a second model call. + + Returns None if no schema is configured. If the call fails, falls back to + parsing the answer locally, and then to the answer text. Never raises. + """ + bound = output_config.bind_structured_output(llm, llm_fallback) + if bound is None: + return None + structured_llm, structured_llm_fallback = bound + logger.info("Formatting raw output to structured format...") + + format_messages: list[BaseMessage] = [SystemMessage(content=_FORMAT_SYSTEM_MESSAGE)] + if saved_report: + format_messages.append( + HumanMessage(content=f"Supporting Evidence (saved report):\n{saved_report}") + ) + format_messages.append( + HumanMessage( + content=(f"Initial Goal: {graph_output.initial_goal}\nRaw Summary:\n{raw_answer}") + ) + ) + try: + response = await with_fallback( + main_call=lambda: invoke_llm_with_timeout_message( + structured_llm.ainvoke(format_messages) + ), + fallback_call=lambda: invoke_llm_with_timeout_message( + structured_llm_fallback.ainvoke(format_messages) + ), + ) + if response is None: + raise ValueError("the model returned no structured output") + except Exception as e: + logger.warning(f"Structured formatting failed ({e}); parsing the answer locally.") + schema = output_config.structured_output_schema() + parsed = parse_structured( + raw_answer, + schema=schema if isinstance(schema, type) and issubclass(schema, BaseModel) else None, + ) + if isinstance(parsed, ParseFailure): + logger.warning("Local parse failed as well; returning the answer as text.") + return raw_answer + response = parsed + + if isinstance(response, BaseModel): + return response.model_dump() + return response + + +def _coerce_text_answer(raw_answer: str) -> Any: + """Parse the answer if it looks like JSON; otherwise return it unchanged.""" + stripped = raw_answer.strip() + if stripped.startswith(("{", "[", "```")): + parsed = parse_structured(stripped) + if not isinstance(parsed, ParseFailure): + return parsed + # The raw text is a legitimate final answer; the miss is only + # noteworthy because the answer *looked* like JSON. + logger.warning( + "Outputter answer looked like JSON but could not be parsed" + f" ({parsed.error}); returning it as plain text." + ) return raw_answer + + +@trace(type="agent", name="outputter") +async def outputter( + ctx: ArtemisContext, + output_config: OutputConfig, + graph_output: State, + plan_and_history: str | None = None, +) -> dict: + logger.info("Starting Outputter Agent") + + if plan_and_history is None and ctx.data_engine: + plan_and_history = _resolve_plan_and_history(ctx) + + messages = _initial_messages(ctx, output_config, graph_output, plan_and_history) + + tools = _outputter_tools(ctx) + llm = get_llm(ctx=ctx, name="outputter", is_utils=True) + llm_fallback = get_llm(ctx=ctx, name="outputter", is_utils=True, use_fallback=True) + investigation = _Investigation({t.name: t for t in tools}, graph_output, llm) + + raw_answer = await _run_react_loop( + investigation, + messages, + llm.bind_tools(tools=tools), + llm_fallback.bind_tools(tools=tools), + ) + + saved_report = _read_saved_report(ctx) if investigation.report_written else None + if raw_answer is None: + if saved_report: + logger.warning("Outputter produced no answer; using its saved report instead.") + raw_answer = saved_report + else: + raw_answer = _MAX_TURNS_ERROR + logger.error(raw_answer) + + if output_config.structured_output: + formatted = await _format_structured( + output_config, graph_output, raw_answer, saved_report, llm, llm_fallback + ) + if formatted is not None: + return formatted + + return _coerce_text_answer(raw_answer) diff --git a/artemis/agents/planner/planner.py b/artemis/agents/planner/planner.py index 9a46038b8..02be5c7c4 100644 --- a/artemis/agents/planner/planner.py +++ b/artemis/agents/planner/planner.py @@ -43,8 +43,6 @@ get_tool_result_content, invoke_tool_with_injection, ) -from artemis.utils.decorators import wrap_with_callbacks -from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path from artemis.utils.plan_grammar import ( CHECKBOX_LINE_RE, @@ -53,6 +51,9 @@ render_plan_grammar_spec, ) from artemis.memory.context_policy import build_history_for +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.decorators import agent_lifecycle_logging +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -108,24 +109,6 @@ def build_planner_system_blocks(prompts_data: dict, mode: str, include_checks: b return blocks -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - class ValidationResult(BaseModel): model_config = {"ignored_types": (CyFunctionDetector,)} is_approved: bool = Field( @@ -224,11 +207,7 @@ def __init__(self, ctx: ArtemisContext, tools: list[BaseTool] | None = None): self.ctx = ctx self.tools = tools or [] - @wrap_with_callbacks( - before=lambda: logger.info("Starting Planner Agent..."), - on_success=lambda _: logger.success("Planner Agent"), - on_failure=lambda _: logger.error("Planner Agent"), - ) + @agent_lifecycle_logging("Planner", logger) @trace(type="agent", name="planner") async def __call__(self, state: State): state = strict_state(state, "planner") diff --git a/artemis/agents/summarizer/summarizer.py b/artemis/agents/summarizer/summarizer.py index efbbd1674..60f4bfccd 100644 --- a/artemis/agents/summarizer/summarizer.py +++ b/artemis/agents/summarizer/summarizer.py @@ -39,8 +39,8 @@ from artemis.graph.state import State from artemis.graph.visibility import strict_state from artemis.memory import ensure_step_memory -from artemis.utils.logger import get_logger from artemis.utils.task_tree import format_result_clean +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/validator/action_execution.py b/artemis/agents/validator/action_execution.py index f167d476f..215f757d4 100644 --- a/artemis/agents/validator/action_execution.py +++ b/artemis/agents/validator/action_execution.py @@ -27,7 +27,7 @@ from artemis.graph.state import State from artemis.mcp.action_names import to_canonical_call from artemis.mcp.action_session import ActionSession -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/validator/execution_loop.py b/artemis/agents/validator/execution_loop.py index 6467a7c51..cd3b7a4ae 100644 --- a/artemis/agents/validator/execution_loop.py +++ b/artemis/agents/validator/execution_loop.py @@ -52,8 +52,8 @@ from artemis.data_engine.trace import CURRENT_TRACE_ID from artemis.graph.state import State from artemis.utils.coordinates import normalize_action_dict -from artemis.utils.logger import get_logger from artemis.utils.task_tree import format_action_clean +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/validator/precondition_pixel.py b/artemis/agents/validator/precondition_pixel.py index b53c88348..80a542fb8 100644 --- a/artemis/agents/validator/precondition_pixel.py +++ b/artemis/agents/validator/precondition_pixel.py @@ -35,7 +35,7 @@ from artemis.graph.state import State from artemis.services.llm import acomplete_structured from artemis.utils import visualization -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/validator/precondition_xml.py b/artemis/agents/validator/precondition_xml.py index 52f5bdce7..3b369615f 100644 --- a/artemis/agents/validator/precondition_xml.py +++ b/artemis/agents/validator/precondition_xml.py @@ -33,7 +33,7 @@ from artemis.context import ArtemisContext from artemis.graph.state import State from artemis.utils import visualization -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/validator/tool_declarations.py b/artemis/agents/validator/tool_declarations.py index 368e8d989..fba8dd751 100644 --- a/artemis/agents/validator/tool_declarations.py +++ b/artemis/agents/validator/tool_declarations.py @@ -39,12 +39,12 @@ from artemis.mcp.action_specs import tool_declaration from artemis.mcp.observation import observe from artemis.graph.state import State -from artemis.utils.logger import get_logger from artemis.utils.notes import ( LIST_NOTES_DOCSTRING, READ_NOTE_ARG_KEY_DESC, READ_NOTE_DOCSTRING, ) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/validator/validator.py b/artemis/agents/validator/validator.py index d7979b7fd..d8eb8ed9d 100644 --- a/artemis/agents/validator/validator.py +++ b/artemis/agents/validator/validator.py @@ -54,8 +54,8 @@ from artemis.graph.visibility import strict_state from artemis.mcp.action_session import ActionSession, get_action_session from artemis.services.llm import acomplete_structured, get_llm # noqa: F401 -from artemis.utils.decorators import wrap_with_callbacks -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.decorators import agent_lifecycle_logging +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -97,11 +97,7 @@ async def _get_initial_screenshot(self, session, state: State) -> tuple[str, str async def _execute_validation_loop(self, state: State): return await execution_loop.run_validation_loop(self, state) - @wrap_with_callbacks( - before=lambda: logger.info("Starting Validator Agent..."), - on_success=lambda _: logger.success("Validator Agent"), - on_failure=lambda _: logger.error("Validator Agent"), - ) + @agent_lifecycle_logging("Validator", logger) @trace(type="agent", name="validator") async def __call__(self, state: State): state = strict_state(state, "validator") diff --git a/artemis/agents/video_analyzer/audio_native.py b/artemis/agents/video_analyzer/audio_native.py index 84d4334bc..74ab27baf 100644 --- a/artemis/agents/video_analyzer/audio_native.py +++ b/artemis/agents/video_analyzer/audio_native.py @@ -43,7 +43,7 @@ from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE from artemis.data_engine.trace import TraceSpan from artemis.llm.reliability import retry_policy_for -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/video_analyzer/blackboard.py b/artemis/agents/video_analyzer/blackboard.py index 00e62a0a1..5475b3927 100644 --- a/artemis/agents/video_analyzer/blackboard.py +++ b/artemis/agents/video_analyzer/blackboard.py @@ -29,8 +29,8 @@ from uuid import uuid4 from artemis.context import ArtemisContext -from artemis.utils.logger import get_logger -from artemis.utils.video import get_active_session +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import get_active_session logger = get_logger(__name__) diff --git a/artemis/agents/video_analyzer/chunk_agentic.py b/artemis/agents/video_analyzer/chunk_agentic.py index 6c2913ab4..0578c539e 100644 --- a/artemis/agents/video_analyzer/chunk_agentic.py +++ b/artemis/agents/video_analyzer/chunk_agentic.py @@ -48,7 +48,7 @@ from artemis.agents.video_analyzer.universal_tools import UNIVERSAL_SUBMIT_ANSWER_TOOL from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from artemis.llm.google import normalize_usage -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/video_analyzer/chunk_conversation.py b/artemis/agents/video_analyzer/chunk_conversation.py index 1e5fd4a3b..914c09e52 100644 --- a/artemis/agents/video_analyzer/chunk_conversation.py +++ b/artemis/agents/video_analyzer/chunk_conversation.py @@ -33,7 +33,7 @@ from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from artemis.llm.google import normalize_usage -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/video_analyzer/chunk_native.py b/artemis/agents/video_analyzer/chunk_native.py index 1bfd424a8..2a81459b5 100644 --- a/artemis/agents/video_analyzer/chunk_native.py +++ b/artemis/agents/video_analyzer/chunk_native.py @@ -37,7 +37,7 @@ is_agentic_rejection, ) from artemis.llm.reliability import retry_policy_for -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/video_analyzer/gemini_files.py b/artemis/agents/video_analyzer/gemini_files.py index ed4e65559..e3b8318af 100644 --- a/artemis/agents/video_analyzer/gemini_files.py +++ b/artemis/agents/video_analyzer/gemini_files.py @@ -19,7 +19,7 @@ from pathlib import Path import time -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger try: from datetime import UTC diff --git a/artemis/agents/video_analyzer/native_coordinator.py b/artemis/agents/video_analyzer/native_coordinator.py index 10fb3aef8..6cfab85d9 100644 --- a/artemis/agents/video_analyzer/native_coordinator.py +++ b/artemis/agents/video_analyzer/native_coordinator.py @@ -33,7 +33,7 @@ from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from artemis.llm.google import normalize_usage -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/video_analyzer/universal_engine.py b/artemis/agents/video_analyzer/universal_engine.py index 39e6127d5..96ade3fe7 100644 --- a/artemis/agents/video_analyzer/universal_engine.py +++ b/artemis/agents/video_analyzer/universal_engine.py @@ -41,7 +41,7 @@ classify_video_failure, ) from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/agents/video_analyzer/video_analyzer.py b/artemis/agents/video_analyzer/video_analyzer.py index d4d95f1e5..9704a5796 100644 --- a/artemis/agents/video_analyzer/video_analyzer.py +++ b/artemis/agents/video_analyzer/video_analyzer.py @@ -75,13 +75,13 @@ ) from artemis.llm.reliability import retry_policy_for from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm -from artemis.utils.logger import get_logger -from artemis.utils.video import ( +from artemis.utils.video import extract_frames_at_timestamps, extract_keyframes_from_video +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import ( compress_video_for_api, extract_audio_from_video, - extract_frames_at_timestamps, - extract_keyframes_from_video, get_active_session, + probe_duration, ) try: @@ -536,23 +536,7 @@ async def exec_extract_segment_metadata( duration = end_time - start_time else: try: - duration_cmd = [ - "ffprobe", - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - str(path), - ] - proc = await asyncio.create_subprocess_exec( - *duration_cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, _ = await proc.communicate() - duration = float(stdout.decode().strip()) + duration = await probe_duration(path) except Exception as e: logger.warning(f"Failed to resolve duration via ffprobe for {path}: {e}") duration = 0.0 diff --git a/artemis/clients/accessibility_client.py b/artemis/clients/accessibility_client.py index a86384f4a..f14b8ab3c 100644 --- a/artemis/clients/accessibility_client.py +++ b/artemis/clients/accessibility_client.py @@ -23,7 +23,7 @@ The screen-data contract is byte-for-byte the one :class:`UIAutomatorClient` exposes (``UIAutomatorScreenData`` with the element dictionaries produced by -``_parse_hierarchy_xml_to_elements``), so every consumer downstream +``parse_hierarchy_xml_to_elements``), so every consumer downstream (``filter_ui_hierarchy``, the Explorer index, the MCP actuators that compare ``clickable == "true"``) sees identical shapes whichever backend is active. @@ -44,11 +44,6 @@ from PIL import Image -from artemis.clients.ui_automator_client import ( - UIAutomatorScreenData, - _parse_hierarchy_xml_to_elements, - _pil_to_base64, -) from artemis.runtime.adb_endpoint import adb_command from artemis.runtime.awake_service import ensure_device_awake from artemis.runtime.helper_manager import ( @@ -61,7 +56,13 @@ ProvisionEvent, helper_manager, ) -from artemis.utils.logger import get_logger +from third_party.mobile_use.clients.ui_automator_client import ( + UIAUTOMATOR_BOOL_ATTRIBUTES, + UIAutomatorScreenData, + parse_hierarchy_xml_to_elements, + pil_to_base64, +) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -80,19 +81,7 @@ "normalize_helper_elements", ] -_BOOL_ATTRIBUTES = ( - "checkable", - "checked", - "clickable", - "enabled", - "focusable", - "focused", - "scrollable", - "long-clickable", - "password", - "selected", - "visible-to-user", -) +_BOOL_ATTRIBUTES = (*UIAUTOMATOR_BOOL_ATTRIBUTES, "visible-to-user") _GLOBAL_KEYS = ("back", "home", "recents", "notifications", "quick_settings") @@ -345,13 +334,13 @@ def get_screenshot(self) -> Image.Image | None: def get_screenshot_base64(self) -> str | None: screenshot = self.get_screenshot() - return None if screenshot is None else _pil_to_base64(screenshot, format="JPEG") + return None if screenshot is None else pil_to_base64(screenshot, format="JPEG") @staticmethod def _elements_from_dump(dump: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]: xml = dump.get("xml") or "" if isinstance(xml, str) and xml.strip(): - return xml, _parse_hierarchy_xml_to_elements(xml) + return xml, parse_hierarchy_xml_to_elements(xml) return "", normalize_helper_elements(dump.get("elements")) def get_screen_data(self) -> UIAutomatorScreenData: @@ -376,7 +365,7 @@ def get_screen_data(self) -> UIAutomatorScreenData: if screenshot is None: raise RuntimeError("Failed to capture screenshot") return UIAutomatorScreenData( - base64=_pil_to_base64(screenshot, format="JPEG"), + base64=pil_to_base64(screenshot, format="JPEG"), hierarchy_xml=xml, elements=elements, width=screenshot.width, diff --git a/artemis/clients/adb_tunnel.py b/artemis/clients/adb_tunnel.py deleted file mode 100644 index e129a6e08..000000000 --- a/artemis/clients/adb_tunnel.py +++ /dev/null @@ -1,330 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. - -"""ADB WebSocket tunnel implementation. - -Bridges local TCP connections to a remote ADB server via WebSocket. -This replaces the need for the `lim` CLI for ADB tunneling. - -The tunnel runs in a dedicated thread with its own event loop to ensure -it cannot be blocked by synchronous operations in the main thread. - -Based on the Go SDK implementation which: -1. Creates a TCP listener -2. Accepts TCP connections from ADB -3. Dials the WebSocket -4. Forwards traffic bidirectionally -""" - -import asyncio -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager -import socket -import threading - -from artemis.utils.logger import get_logger -import websockets -from websockets.asyncio.client import ClientConnection - -logger = get_logger(__name__) - -BUFFER_SIZE = 32 * 1024 # 32KB, same as Go SDK -PING_INTERVAL = 30 # seconds - - -class AdbTunnel: - """ADB tunnel that bridges local TCP connections to a remote WebSocket endpoint. - - Runs in a dedicated thread with its own event loop to ensure the tunnel - cannot be blocked by synchronous operations in the caller's thread. - - Usage: - tunnel = AdbTunnel(remote_url, token) - addr = await tunnel.start() # Returns "127.0.0.1:PORT" - # Use addr with ADB: adb connect 127.0.0.1:PORT - await tunnel.stop() - """ - - def __init__(self, remote_url: str, token: str): - """Initialize the ADB tunnel. - - Args: - remote_url: WebSocket URL for the remote ADB endpoint - token: Bearer token for authentication - """ - self.remote_url = remote_url - self.token = token - - self._local_port: int | None = None - self._thread: threading.Thread | None = None - self._loop: asyncio.AbstractEventLoop | None = None - self._started = threading.Event() - self._stop_event: asyncio.Event | None = None - - @property - def local_addr(self) -> str: - """Get the local address to connect ADB to.""" - if self._local_port is None: - raise RuntimeError("Tunnel not started") - return f"127.0.0.1:{self._local_port}" - - @property - def local_port(self) -> int: - """Get the local port number.""" - if self._local_port is None: - raise RuntimeError("Tunnel not started") - return self._local_port - - async def start(self) -> str: - """Start the ADB tunnel in a dedicated thread. - - Returns: - The local address (host:port) to connect ADB to. - """ - if self._thread is not None and self._thread.is_alive(): - return self.local_addr - - # Create listener in main thread to get the port immediately - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(("127.0.0.1", 0)) - listener.listen(5) - listener.setblocking(False) - self._local_port = listener.getsockname()[1] - - logger.info(f"ADB tunnel listening on {self.local_addr}") - - # Start tunnel thread - self._thread = threading.Thread( - target=self._run_in_thread, - args=(listener,), - name=f"adb-tunnel-{self._local_port}", - daemon=True, - ) - self._thread.start() - - # Wait for thread to initialize its event loop - self._started.wait(timeout=5.0) - if not self._started.is_set(): - await self.stop() - raise RuntimeError("Tunnel thread failed to start") - - return self.local_addr - - async def stop(self) -> None: - """Stop the ADB tunnel and cleanup resources.""" - if self._loop and self._stop_event: - # Signal the tunnel loop to stop - self._loop.call_soon_threadsafe(self._stop_event.set) - - if self._thread and self._thread.is_alive(): - await asyncio.to_thread(self._thread.join, 5.0) - if self._thread.is_alive(): - logger.warning("ADB tunnel thread did not stop within 5s") - - self._thread = None - self._loop = None - self._stop_event = None - logger.info("ADB tunnel stopped") - - def _run_in_thread(self, listener: socket.socket) -> None: - """Run the tunnel event loop in a dedicated thread.""" - try: - self._loop = asyncio.new_event_loop() - asyncio.set_event_loop(self._loop) - self._stop_event = asyncio.Event() - self._started.set() - - self._loop.run_until_complete(self._run_tunnel(listener)) - except Exception as e: - logger.error(f"Tunnel thread error: {e}") - finally: - listener.close() - if self._loop: - self._loop.close() - - async def _run_tunnel(self, listener: socket.socket) -> None: - """Run the tunnel - accept TCP connections and bridge to WebSocket.""" - loop = asyncio.get_event_loop() - connection_tasks: list[asyncio.Task] = [] - - stop_event = self._stop_event - assert stop_event is not None - - logger.debug("Tunnel: ready to accept connections") - - while not stop_event.is_set(): - try: - # Use wait_for to periodically check stop_event - try: - tcp_conn, addr = await asyncio.wait_for( - loop.sock_accept(listener), - timeout=1.0, - ) - except TimeoutError: - # Check if we should stop - continue - - logger.debug(f"Tunnel: new TCP connection from {addr}") - - task = asyncio.create_task( - self._handle_connection(tcp_conn, loop), - name=f"conn-{addr[1]}", - ) - task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None) - connection_tasks.append(task) - connection_tasks = [t for t in connection_tasks if not t.done()] - - except Exception as e: - if not stop_event.is_set(): - logger.error(f"Tunnel accept error: {e}") - break - - # Cleanup pending connections - for task in connection_tasks: - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - - async def _handle_connection( - self, - tcp_conn: socket.socket, - loop: asyncio.AbstractEventLoop, - ) -> None: - """Handle a single TCP connection.""" - tcp_conn.setblocking(False) - - ws: ClientConnection | None = None - try: - logger.debug(f"Connecting to WebSocket: {self.remote_url[:80]}...") - ws = await websockets.connect( - self.remote_url, - additional_headers={"Authorization": f"Bearer {self.token}"}, - ping_interval=PING_INTERVAL, - ping_timeout=10, - ) - logger.debug("WebSocket connected to remote ADB") - await self._bridge(tcp_conn, ws, loop) - except Exception as e: - logger.error(f"Connection error: {e}") - finally: - if ws is not None: - await ws.close() - tcp_conn.close() - logger.debug("Connection closed") - - async def _bridge( - self, - tcp_conn: socket.socket, - ws: ClientConnection, - loop: asyncio.AbstractEventLoop, - ) -> None: - """Bridge TCP and WebSocket bidirectionally.""" - - stop_event = self._stop_event - assert stop_event is not None - - async def tcp_to_ws() -> None: - """Forward data from TCP to WebSocket.""" - msg_count = 0 - try: - while not stop_event.is_set(): - try: - data = await asyncio.wait_for( - loop.sock_recv(tcp_conn, BUFFER_SIZE), - timeout=1.0, - ) - except TimeoutError: - continue - except (OSError, ConnectionError) as e: - logger.debug(f"tcp->ws: recv error after {msg_count} msgs: {e}") - break - if not data: - logger.debug(f"tcp->ws: closed by client after {msg_count} msgs") - break - msg_count += 1 - logger.debug(f"tcp->ws: [{msg_count}] sending {len(data)} bytes") - await ws.send(data) - logger.debug(f"tcp->ws: loop ended after {msg_count} messages") - except asyncio.CancelledError: - logger.debug(f"tcp->ws: cancelled after {msg_count} messages") - raise - except Exception as e: - logger.error(f"tcp->ws: error after {msg_count} messages: {e}") - - async def ws_to_tcp() -> None: - """Forward data from WebSocket to TCP.""" - msg_count = 0 - try: - logger.debug("ws->tcp: starting message loop") - async for message in ws: - if stop_event.is_set(): - break - if isinstance(message, bytes) and message: - msg_count += 1 - logger.debug(f"ws->tcp: [{msg_count}] received {len(message)} bytes") - await loop.sock_sendall(tcp_conn, message) - elif isinstance(message, str): - logger.debug(f"ws->tcp: received string: {message[:100]}") - logger.debug(f"ws->tcp: loop ended after {msg_count} messages") - except asyncio.CancelledError: - logger.debug(f"ws->tcp: cancelled after {msg_count} messages") - raise - except Exception as e: - logger.error(f"ws->tcp: error after {msg_count} messages: {e}") - - tcp_task = asyncio.create_task(tcp_to_ws()) - ws_task = asyncio.create_task(ws_to_tcp()) - - done, pending = await asyncio.wait( - [tcp_task, ws_task], - return_when=asyncio.FIRST_COMPLETED, - ) - - for task in pending: - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - - -@asynccontextmanager -async def adb_tunnel(remote_url: str, token: str) -> AsyncGenerator[AdbTunnel, None]: - """Context manager for creating and managing an ADB tunnel. - - Args: - remote_url: WebSocket URL for the remote ADB endpoint - token: Bearer token for authentication - - Yields: - AdbTunnel instance with the tunnel running - - Example: - async with adb_tunnel(ws_url, token) as tunnel: - # Connect ADB to tunnel.local_addr - adb_client = AdbClient() - adb_client.connect(tunnel.local_addr) - """ - tunnel = AdbTunnel(remote_url, token) - try: - await tunnel.start() - yield tunnel - finally: - await tunnel.stop() diff --git a/artemis/clients/screen_client_factory.py b/artemis/clients/screen_client_factory.py index 311f2629e..43e380fb1 100644 --- a/artemis/clients/screen_client_factory.py +++ b/artemis/clients/screen_client_factory.py @@ -42,11 +42,14 @@ from PIL import Image from artemis.clients.accessibility_client import AccessibilityClient, HelperUnavailable -from artemis.clients.ui_automator_client import UIAutomatorClient, UIAutomatorScreenData from artemis.config.constants import ENV_ARTEMIS_HIERARCHY_BACKEND from artemis.runtime.adb_endpoint import adb_command from artemis.runtime.helper_manager import ProvisionEvent -from artemis.utils.logger import get_logger +from third_party.mobile_use.clients.ui_automator_client import ( + UIAutomatorClient, + UIAutomatorScreenData, +) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/config/__init__.py b/artemis/config/__init__.py index e539f9333..46862a438 100644 --- a/artemis/config/__init__.py +++ b/artemis/config/__init__.py @@ -93,15 +93,15 @@ LLM_CONFIG_OVERRIDE_FILENAME, LS_ADDRESS_FILENAME, AgentNode, - AgentNodeWithFallback, ExplorerVersion, LLMProvider, LLMUtilsNode, - LLMUtilsNodeWithFallback, ) from artemis.config.llm import ( CyFunctionDetector, LLM, + AgentNodeWithFallback, + LLMUtilsNodeWithFallback, LLMConfig, LLMConfigUtils, LLMWithFallback, @@ -112,11 +112,6 @@ parse_llm_config, validate_vertex_ai_credentials, ) -from artemis.config.output import ( - OutputConfig, - prepare_output_files, - record_events, -) from artemis.config.paths import ( CONFIG_DIR, DB_PATH, @@ -161,6 +156,11 @@ Settings, settings, ) +from third_party.mobile_use.config.output import ( + OutputConfig, + prepare_output_files, + record_events, +) __all__ = [ # Settings & Singleton diff --git a/artemis/config/agent.py b/artemis/config/agent.py index d075d1638..6fb00d398 100644 --- a/artemis/config/agent.py +++ b/artemis/config/agent.py @@ -27,8 +27,8 @@ ) from artemis.config.paths import ROOT_DIR, get_config_path from artemis.llm.google import VideoProcessing -from artemis.utils.file import load_jsonc -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.file import load_jsonc +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/config/constants.py b/artemis/config/constants.py index 972bc6df4..7e4d74258 100644 --- a/artemis/config/constants.py +++ b/artemis/config/constants.py @@ -129,7 +129,6 @@ "video_analyzer", "object_detector", ] -LLMUtilsNodeWithFallback = LLMUtilsNode AgentNode = Literal[ "planner", @@ -150,4 +149,3 @@ "validator", "output_analyzer", ] -AgentNodeWithFallback = AgentNode diff --git a/artemis/config/core/engine.py b/artemis/config/core/engine.py index 9af350254..656c9d5ed 100644 --- a/artemis/config/core/engine.py +++ b/artemis/config/core/engine.py @@ -23,7 +23,7 @@ from artemis.config.core.interpolator import interpolate_config_value from artemis.platform import platform -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/config/llm.py b/artemis/config/llm.py index 36915dd9e..641d2e198 100644 --- a/artemis/config/llm.py +++ b/artemis/config/llm.py @@ -14,106 +14,47 @@ """LLM provider, model hierarchy, fallback chaining, and configuration loaders.""" -import os from pathlib import Path -from typing import Any, Literal +from typing import Any -import google.auth -from google.auth.exceptions import DefaultCredentialsError -from pydantic import BaseModel, ValidationError from artemis.config.constants import ( LLM_CONFIG_FILENAME, AgentNode, - LLMProvider, - LLMUtilsNode, ) from artemis.config.paths import ROOT_DIR, get_config_path -from artemis.config.settings import settings -from artemis.utils.file import load_jsonc -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.config import llm as base_llm_config +from third_party.mobile_use.config.llm import ( + LLM, + AgentNodeWithFallback, + LLMConfigBase, + LLMConfigUtils, + LLMUtilsNodeWithFallback, + LLMWithFallback, + validate_vertex_ai_credentials, +) +from third_party.mobile_use.utils.file import load_jsonc +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) - -def validate_vertex_ai_credentials() -> None: - """Validate Google Application Default Credentials for VertexAI provider.""" - try: - _, project = google.auth.default() - if not project: - raise Exception("VertexAI requires a Google Cloud project to be set.") - except DefaultCredentialsError as e: - raise Exception( - f"VertexAI requires valid Google Application Default Credentials (ADC): {e}" - ) - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - """Utility type detector for Cython and C-extension functions in Pydantic.""" - - pass - - -class LLM(BaseModel): - """Base model representing an LLM model provider and runtime parameters.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - provider: LLMProvider - model: str - temperature: float | None = None - thinking_budget: int | None = None - thinking_level: Literal["minimal", "low", "medium", "high"] | None = None - reasoning_effort: Literal["none", "low", "medium", "high"] | None = None - include_thoughts: bool | None = None - enable_grounding: bool | None = None - - def validate_provider(self, name: str) -> None: - """Ensure the required API key or credentials exist in settings for this provider.""" - if self.provider == "openai": - if not settings.OPENAI_API_KEY: - raise Exception(f"{name} requires OPENAI_API_KEY in .env") - elif self.provider == "google": - if not settings.GOOGLE_API_KEY: - raise Exception(f"{name} requires GOOGLE_API_KEY in .env") - elif self.provider == "vertexai": - validate_vertex_ai_credentials() - elif self.provider == "anthropic": - if not (settings.ANTHROPIC_API_KEY or os.environ.get("ANTHROPIC_API_KEY")): - raise Exception(f"{name} requires ANTHROPIC_API_KEY in .env") - elif self.provider == "openrouter": - if not settings.OPEN_ROUTER_API_KEY: - raise Exception(f"{name} requires OPEN_ROUTER_API_KEY in .env") - elif self.provider == "xai": - if not settings.XAI_API_KEY: - raise Exception(f"{name} requires XAI_API_KEY in .env") - - def __str__(self) -> str: - return f"{self.provider}/{self.model}" - - -class LLMWithFallback(LLM): - """LLM configuration with automatic secondary fallback and timeout specs.""" - - fallback: LLM - fix_model: str | None = None - timeout: float | None = None - - def __str__(self) -> str: - return f"{self.provider}/{self.model} (fallback: {self.fallback})" +__all__ = [ + "LLM", + "AgentNodeWithFallback", + "LLMUtilsNodeWithFallback", + "CyFunctionDetector", + "LLMConfig", + "LLMConfigUtils", + "LLMWithFallback", + "deep_merge_llm_config", + "get_default_llm_config", + "initialize_llm_config", + "lightweight_judge_default", + "load_llm_config_override", + "parse_llm_config", + "validate_vertex_ai_credentials", +] def lightweight_judge_default() -> "LLMWithFallback": @@ -131,22 +72,9 @@ def lightweight_judge_default() -> "LLMWithFallback": ) -class LLMConfigUtils(BaseModel): - """Configuration container for auxiliary utility agents/nodes.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - outputter: LLMWithFallback - hopper: LLMWithFallback - video_analyzer: LLMWithFallback | None = None - object_detector: LLMWithFallback | None = None - - -class LLMConfig(BaseModel): +class LLMConfig(LLMConfigBase): """Comprehensive LLM configuration mapping every node to primary/fallback models.""" - model_config = {"ignored_types": (CyFunctionDetector,)} - planner: LLMWithFallback - utils: LLMConfigUtils summarizer: LLMWithFallback operator: LLMWithFallback operator_summarizer: LLMWithFallback @@ -165,13 +93,7 @@ class LLMConfig(BaseModel): def validate_providers(self) -> None: """Validate credentials across all configured agent nodes.""" - self.planner.validate_provider("Planner") - self.utils.outputter.validate_provider("Outputter") - self.utils.hopper.validate_provider("Hopper") - if self.utils.video_analyzer: - self.utils.video_analyzer.validate_provider("VideoAnalyzer") - if self.utils.object_detector: - self.utils.object_detector.validate_provider("ObjectDetector") + super().validate_providers() self.summarizer.validate_provider("Summarizer") self.operator.validate_provider("Operator") self.operator_summarizer.validate_provider("OperatorSummarizer") @@ -192,16 +114,6 @@ def validate_providers(self) -> None: if self.output_analyzer: self.output_analyzer.validate_provider("OutputAnalyzer") - def __str__(self) -> str: - return f""" -📃 Planner: {self.planner} -🧩 Utils: - 🔽 Hopper: {self.utils.hopper} - 📝 Outputter: {self.utils.outputter} - 🎬 Video Analyzer: {self.utils.video_analyzer or "Not configured"} - 👁️ Object Detector: {self.utils.object_detector or "Not configured"} -""" - def get_agent(self, item: AgentNode) -> LLMWithFallback: """Retrieve model configuration for a specific agent node with sensible defaults.""" val = getattr(self, item) @@ -217,16 +129,6 @@ def get_agent(self, item: AgentNode) -> LLMWithFallback: return self.log_analyzer return val - def get_utils(self, item: LLMUtilsNode) -> LLMWithFallback: - """Retrieve model configuration for a specific utility node.""" - value = getattr(self.utils, item) - if value is None: - raise ValueError( - f"Utils '{item}' is not configured. Please add it to your LLM " - "config or enable it via AgentConfigBuilder." - ) - return value - def _expand_default_into_nodes(config_dict: dict) -> dict: """Expand unified config format with 'default' and 'nodes' into full LLMConfig schema.""" @@ -320,10 +222,7 @@ def parse_llm_config() -> LLMConfig: def initialize_llm_config() -> LLMConfig: """Parse and validate credentials for LLMConfig.""" - llm_config = parse_llm_config() - llm_config.validate_providers() - logger.success("LLM config initialized") - return llm_config + return base_llm_config.initialize_llm_config(parse_llm_config) def get_default_llm_config() -> LLMConfig: @@ -348,26 +247,12 @@ def merge(d1: dict, d2: dict) -> None: def load_llm_config_override(path: Path | str) -> LLMConfig: """Load custom LLM configuration JSON/JSONC overrides on top of default configuration.""" - default_config = get_default_llm_config() - resolved_path = Path(path) if not resolved_path.exists(): try: resolved_path = get_config_path(str(path)) except OSError: pass - - override_config_dict = {} - if resolved_path.exists(): - logger.info(f"Loading custom LLM config from {resolved_path.resolve()}...") - with open(resolved_path, encoding="utf-8") as f: - override_config_dict = load_jsonc(f) - else: - logger.warning(f"Custom LLM config not found at {path} - using default config") - - try: - return deep_merge_llm_config(default_config, override_config_dict) - except ValidationError as e: - logger.error(f"Invalid LLM config: {e}") - logger.info("Falling back to default config") - return default_config + return base_llm_config.load_llm_config_override( + resolved_path, get_default_llm_config, deep_merge_llm_config + ) diff --git a/artemis/config/runtime.py b/artemis/config/runtime.py index 799cc72c8..bf9e4cad4 100644 --- a/artemis/config/runtime.py +++ b/artemis/config/runtime.py @@ -31,7 +31,7 @@ get_temp_dir, is_source_checkout, ) -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/config/settings.py b/artemis/config/settings.py index 7129dd8ab..182d7ec77 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -44,7 +44,7 @@ get_env_file, get_temp_dir, ) -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger # Installed wheels load .env from the user directory, outside site-packages. _canonical_env = get_env_file() diff --git a/artemis/context.py b/artemis/context.py index fa698db03..57360c8c8 100644 --- a/artemis/context.py +++ b/artemis/context.py @@ -11,28 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. - -"""Context variables for global state management. -Uses ContextVar to avoid prop drilling and maintain clean function signatures. -""" +"""Runtime context shared by Artemis agents, tools and drivers for a single task run.""" from __future__ import annotations import asyncio -try: - from enum import StrEnum -except ImportError: - from enum import Enum - - class StrEnum(str, Enum): - pass - - from pathlib import Path from typing import Any, Literal, TYPE_CHECKING @@ -51,62 +36,21 @@ class StrEnum(str, Enum): from artemis.config import ExplorerConfig, LLMConfig, OutputterConfig -from artemis.utils.logger import get_logger +from third_party.mobile_use.context import ( + AppLaunchResult, + DeviceContext, + DeviceClientAccessors, + DevicePlatform, + ExecutionSetupBase, +) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) -class AppLaunchResult(BaseModel): - """Result of initial app launch attempt.""" - - model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) - - locked_app_package: str - locked_app_initial_launch_success: bool | None - locked_app_initial_launch_error: str | None - - -class DevicePlatform(StrEnum): - """Mobile device platform enumeration.""" - - ANDROID = "android" - - -class DeviceContext(BaseModel): - model_config = ConfigDict( - arbitrary_types_allowed=True, - extra="allow", - ) - - host_platform: Literal["WINDOWS", "LINUX", "DARWIN", "MACOS"] | str = "DARWIN" - mobile_platform: DevicePlatform = DevicePlatform.ANDROID - device_id: str = "default-device" - - device_width: int = 1080 - device_height: int = 2400 - - def to_str(self): - return ( - f"Host platform: {self.host_platform}\n" - f"Mobile platform: {self.mobile_platform.value}\n" - f"Device ID: {self.device_id}\n" - f"Device width: {self.device_width}\n" - f"Device height: {self.device_height}\n" - ) - - -class ExecutionSetup(BaseModel): +class ExecutionSetup(ExecutionSetupBase): """Execution setup for a task.""" - model_config = ConfigDict( - arbitrary_types_allowed=True, - extra="allow", - ) - - traces_path: Path | None = None - trace_name: str | None = None - enable_remote_tracing: bool = False - app_lock_status: AppLaunchResult | None = None video_recording_tools_enabled: bool = Field(default_factory=detect_video_tools_enabled) disable_checker: bool = False """Legacy master switch (compat alias): ``True`` disables BOTH the midway @@ -165,18 +109,8 @@ def explorer_pro_mode(self) -> str: def explorer_caching(self) -> bool | None: return self.explorer.caching - def get_locked_app_package(self) -> str | None: - """Get the locked app package name if app locking is enabled. - Returns: - The locked app package name, or None if app locking is not enabled. - """ - if self.app_lock_status: - return self.app_lock_status.locked_app_package - return None - - -class ArtemisContext(BaseModel): +class ArtemisContext(DeviceClientAccessors, BaseModel): model_config = ConfigDict( arbitrary_types_allowed=True, extra="allow", @@ -349,16 +283,6 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): except Exception as exc: logger.debug(f"DataEngine shutdown failed; skipped: {exc}", exc_info=True) - def get_adb_client(self) -> Any: - if self.adb_client is None: - raise ValueError("No ADB client in context.") - return self.adb_client - - def get_ui_adb_client(self) -> Any: - if self.ui_adb_client is None: - raise ValueError("No UIAutomator client in context.") - return self.ui_adb_client - from artemis.data_engine.engine import DataEngine diff --git a/artemis/controllers/controller_factory.py b/artemis/controllers/controller_factory.py index 0d4428f4f..aaf86aa61 100644 --- a/artemis/controllers/controller_factory.py +++ b/artemis/controllers/controller_factory.py @@ -14,7 +14,7 @@ from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/controllers/device_controller.py b/artemis/controllers/device_controller.py index fd5e43a71..20849b4f1 100644 --- a/artemis/controllers/device_controller.py +++ b/artemis/controllers/device_controller.py @@ -11,34 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. -from abc import ABC, abstractmethod from typing import Any -from artemis.controllers.types import Bounds, CoordinatesSelectorRequest, TapOutput -from artemis.utils.video import VideoRecordingResult from pydantic import BaseModel, Field - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass +from artemis.utils.cython_compat import CyFunctionDetector class ScreenDataResponse(BaseModel): @@ -50,199 +27,3 @@ class ScreenDataResponse(BaseModel): width: int = Field(default=1080, description="Device screen width in pixels") height: int = Field(default=2400, description="Device screen height in pixels") platform: str = Field(default="android", description="Platform identifier") - - -class MobileDeviceController(ABC): - @abstractmethod - async def tap( - self, - coords: CoordinatesSelectorRequest, - long_press: bool = False, - long_press_duration: int = 1000, - times: int = 1, - delay_ms: int = 100, - ) -> TapOutput: - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def swipe( - self, - start: CoordinatesSelectorRequest, - end: CoordinatesSelectorRequest, - duration: int = 400, - ) -> str | None: - """Swipe from start to end coordinates. - - Returns error message on failure, None on success. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def screenshot(self) -> str: - """Take a screenshot and return raw image data.""" - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def input_text(self, text: str) -> bool: - """Input text at the currently focused field. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def launch_app(self, package_or_bundle_id: str) -> bool: - """Launch an application by package name (Android). - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def terminate_app(self, package_or_bundle_id: str | None) -> bool: - """Terminate an application. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def open_url(self, url: str) -> bool: - """Open a URL. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def press_back(self) -> bool: - """Press the back button (Android). - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def press_home(self) -> bool: - """Press the home button. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def press_enter(self) -> bool: - """Press the enter/return key. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def press_key(self, keycode: str) -> bool: - """Press a specific key event. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def get_ui_hierarchy(self) -> list[dict]: - """Get the UI element hierarchy. - - Returns a list of UI elements with their properties. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - def find_element( - self, - ui_hierarchy: list[dict], - resource_id: str | None = None, - text: str | None = None, - index: int = 0, - ) -> tuple[dict | None, Bounds | None, str | None]: - """Find a UI element in the hierarchy. - - Returns: - Tuple of (element_dict, bounds, error_message) - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def cleanup(self) -> None: - """Clean up resources (e.g., stop companion processes).""" - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def erase_text(self, nb_chars: int | None = None) -> bool: - """Erase the last nb_chars characters. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def get_screen_data(self) -> "ScreenDataResponse": - """Get screen data including screenshot (base64), UI hierarchy elements, - - screen dimensions, and platform. - - Returns: - ScreenDataResponse with base64 screenshot, elements, width, height, - platform - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - def get_compressed_b64_screenshot(self, image_base64: str, quality: int = 50) -> str: - """Compress a base64 image. - - Returns the compressed base64 image. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def start_video_recording( - self, - max_duration_seconds: int = 900, - ) -> VideoRecordingResult: - """Start screen recording on the device. - - Args: - max_duration_seconds: Maximum recording duration in seconds. - - Returns: - VideoRecordingResult with success status and message. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def stop_video_recording(self) -> VideoRecordingResult: - """Stop screen recording and retrieve the video file. - - Returns: - VideoRecordingResult with success status, message, and video_path if - successful. - """ - raise NotImplementedError("Subclasses must implement this method") - - @abstractmethod - async def extract_segment_metadata( - self, - start_relative_time: float, - end_relative_time: float | None = None, - ) -> VideoRecordingResult: - """Get a video segment for a specific time range. - - Args: - start_relative_time: Start time in seconds relative to session - start. - end_relative_time: End time in seconds relative to session start, or - None for latest. - - Returns: - VideoRecordingResult with success status, message, and video_path if - successful. - """ - raise NotImplementedError("Subclasses must implement this method") diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index 201abfe9a..d0dc1d035 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import asyncio import os @@ -28,53 +25,37 @@ from artemis.config.paths import get_temp_dir from artemis.context import ArtemisContext from artemis.drivers.factory import get_driver -from artemis.drivers.base import BaseDeviceDriver -from artemis.controllers.device_controller import ScreenDataResponse -from artemis.controllers.types import ( - SwipeRequest, - SwipeStartEndCoordinatesRequest, - SwipeStartEndPercentagesRequest, - TapOutput, -) -from artemis.utils.logger import get_logger from artemis.utils.video import ( ANDROID_RECORDING_SEGMENT_SECONDS, - DEFAULT_MAX_DURATION_SECONDS, - RecordingSession, - VideoRecordingResult, await_scrcpy_first_frame, build_scrcpy_record_command, - cleanup_video_segments, - concatenate_videos, get_android_display_state, - get_active_session, - has_active_session, - normalize_recording_to_mp4, remux_recording_to_mp4, render_timeline_clip, + write_recording_manifest, +) +from third_party.mobile_use.controllers.unified_controller import UnifiedMobileControllerBase +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import ( + DEFAULT_MAX_DURATION_SECONDS, + RecordingSession, + VideoRecordingResult, + abort_recording, + get_active_session, + no_active_recording, + recording_already_active, remove_active_session, set_active_session, - trim_video, - write_recording_manifest, ) logger = get_logger(__name__) -class UnifiedMobileController: +class UnifiedMobileController(UnifiedMobileControllerBase): def __init__(self, ctx: ArtemisContext): - self.ctx = ctx - self._driver: BaseDeviceDriver = get_driver(ctx) + super().__init__(ctx, get_driver(ctx)) self._segment_cache: dict[tuple[str, int, float, float], VideoRecordingResult] = {} - @property - def driver(self) -> BaseDeviceDriver: - return self._driver - - @property - def controller(self) -> Any: - return self._driver - @staticmethod async def _spawn_scrcpy(command: list[str]) -> asyncio.subprocess.Process: kwargs: dict[str, Any] = { @@ -98,190 +79,6 @@ async def _stop_scrcpy(process: asyncio.subprocess.Process) -> None: process.terminate() await asyncio.wait_for(process.wait(), timeout=5.0) - async def tap_at( - self, - x: int, - y: int, - long_press: bool = False, - long_press_duration: int = 1000, - times: int = 1, - delay_ms: int = 100, - ) -> TapOutput: - try: - if long_press: - success = await self._driver.long_press(x, y, duration_ms=long_press_duration) - else: - success = await self._driver.tap( - x, y, duration_ms=100, times=times, delay_ms=delay_ms - ) - return TapOutput(error=None if success else f"Tap failed at ({x}, {y})") - except Exception as e: - return TapOutput(error=str(e)) - - async def tap_percentage( - self, - x_percent: int, - y_percent: int, - long_press: bool = False, - long_press_duration: int = 1000, - ) -> TapOutput: - """Tap at percentage-based coordinates (0 to 100).""" - norm_x = int(x_percent * 10) - norm_y = int(y_percent * 10) - success = await self._driver.tap_normalized( - norm_x, norm_y, long_press=long_press, duration_ms=long_press_duration - ) - return TapOutput( - error=None if success else f"Tap percentage failed at ({x_percent}%, {y_percent}%)" - ) - - async def tap_element( - self, - resource_id: str | None = None, - text: str | None = None, - index: int = 0, - long_press: bool = False, - long_press_duration: int = 1000, - ) -> TapOutput: - """Tap on a UI element by finding it in the hierarchy.""" - success = await self._driver.tap_element( - resource_id=resource_id, - text=text, - index=index, - long_press=long_press, - duration_ms=long_press_duration, - ) - return TapOutput( - error=None - if success - else f"Failed to tap element (resource_id={resource_id}, text={text})" - ) - - async def swipe_coords( - self, - start_x: int, - start_y: int, - end_x: int, - end_y: int, - duration: int = 400, - ) -> str | None: - """Swipe between two coordinate points.""" - success = await self._driver.swipe(start_x, start_y, end_x, end_y, duration_ms=duration) - return None if success else f"Swipe failed from ({start_x},{start_y}) to ({end_x},{end_y})" - - async def swipe_percentage( - self, - start_x_percent: int, - start_y_percent: int, - end_x_percent: int, - end_y_percent: int, - duration: int = 400, - ) -> str | None: - """Swipe using percentage-based coordinates (0 to 100).""" - start_norm = [int(start_x_percent * 10), int(start_y_percent * 10)] - end_norm = [int(end_x_percent * 10), int(end_y_percent * 10)] - success = await self._driver.swipe_normalized(start_norm, end_norm, duration_ms=duration) - return None if success else "Swipe percentage failed" - - async def swipe_request(self, request: SwipeRequest) -> str | None: - mode = request.swipe_mode - - if isinstance(mode, SwipeStartEndCoordinatesRequest): - return await self.swipe_coords( - start_x=mode.start.x, - start_y=mode.start.y, - end_x=mode.end.x, - end_y=mode.end.y, - duration=request.duration or 400, - ) - elif isinstance(mode, SwipeStartEndPercentagesRequest): - return await self.swipe_percentage( - start_x_percent=mode.start.x_percent, - start_y_percent=mode.start.y_percent, - end_x_percent=mode.end.x_percent, - end_y_percent=mode.end.y_percent, - duration=request.duration or 400, - ) - else: - return "Unsupported swipe mode" - - async def type_text(self, text: str, clear_existing: bool = True) -> bool: - return await self._driver.input_text(text, clear_existing=clear_existing) - - async def take_screenshot(self) -> str: - screen_data = await self._driver.get_screen_data() - return screen_data.screenshot_base64 - - async def launch_app(self, package_or_bundle_id: str) -> bool: - return await self._driver.launch_app(package_or_bundle_id) - - async def terminate_app(self, package_or_bundle_id: str | None) -> bool: - if not package_or_bundle_id: - return False - return await self._driver.stop_app(package_or_bundle_id) - - async def open_url(self, url: str) -> bool: - await self._driver.execute_shell(f"am start -a android.intent.action.VIEW -d '{url}'") - return True - - async def go_back(self) -> bool: - return await self._driver.press_key("back") - - async def go_home(self) -> bool: - return await self._driver.press_key("home") - - async def press_enter(self) -> bool: - return await self._driver.press_key("enter") - - async def press_key(self, keycode: str) -> bool: - return await self._driver.press_key(keycode) - - async def erase_text(self, nb_chars: int | None = None) -> bool: - if nb_chars is not None and nb_chars > 0: - for _ in range(nb_chars): - await self._driver.press_key("delete") - return True - # Best-effort full clear: End -> Ctrl+A -> Delete - try: - clear_cmd = ( - "input keyevent 123 && " - "input keycombination 113 29 && input keyevent 67 && " - "input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67" - ) - await self._driver.execute_shell(clear_cmd) - except Exception: - for _ in range(30): - await self._driver.press_key("delete") - return True - - async def get_ui_elements(self) -> list[dict]: - screen_data = await self._driver.get_screen_data() - return screen_data.ui_elements or [] - - async def get_screen_data(self) -> "ScreenDataResponse": - """Get screen data including screenshot, UI hierarchy, dimensions, and platform.""" - data = await self._driver.get_screen_data() - return ScreenDataResponse( - base64=data.screenshot_base64, - elements=data.ui_elements or [], - width=data.width, - height=data.height, - platform=data.platform, - ) - - async def find_element( - self, - resource_id: str | None = None, - text: str | None = None, - index: int = 0, - ) -> tuple[dict | None, str | None]: - elem, _, error = await self._driver.find_element( - resource_id=resource_id, - text=text, - index=index, - ) - return elem, error - def _get_device_id(self) -> str: if self.ctx and self.ctx.device and self.ctx.device.device_id: return self.ctx.device.device_id @@ -311,10 +108,7 @@ async def extract_segment_metadata( session = get_active_session(device_id) if not session: - return VideoRecordingResult( - success=False, - message=f"No active recording for device {device_id}", - ) + return no_active_recording(device_id) cache_key = None if end_time is not None: @@ -616,11 +410,8 @@ async def start_video_recording( await self._driver.start_video_recording(output_dir) return VideoRecordingResult(success=True, message="Mock recording started") - if has_active_session(device_id): - return VideoRecordingResult( - success=False, - message=f"Recording already in progress for device {device_id}", - ) + if already_active := recording_already_active(device_id): + return already_active try: if not output_dir: @@ -723,11 +514,7 @@ async def start_video_recording( except Exception as e: logger.error(f"Failed to start scrcpy recording: {e}") - remove_active_session(device_id) - return VideoRecordingResult( - success=False, - message=f"Failed to start recording: {e}", - ) + return abort_recording(device_id, "start", e) async def stop_video_recording(self) -> VideoRecordingResult: """Stop scrcpy recording and return the converted MP4 video file.""" @@ -749,10 +536,7 @@ async def stop_video_recording(self) -> VideoRecordingResult: session = get_active_session(device_id) if not session: - return VideoRecordingResult( - success=False, - message=f"No active recording for device {device_id}", - ) + return no_active_recording(device_id) # Mark inactive and cancel watchdog task session.is_active = False @@ -859,26 +643,7 @@ async def stop_video_recording(self) -> VideoRecordingResult: except Exception as e: logger.error(f"Failed to stop scrcpy recording: {e}") self._record_recording_failure(session, str(e)) - remove_active_session(device_id) - return VideoRecordingResult( - success=False, - message=f"Failed to stop recording: {e}", - ) - - async def _convert_mkv_to_mp4(self, mkv_path: Path, mp4_path: Path) -> bool: - """Normalize MKV into a fixed-size, browser-safe MP4. - - Stream-copying a scrcpy H.264 track is unsafe because historical or - recovered recordings may contain resolution changes. Re-encoding onto - the initial capture canvas guarantees one coded size for the full MP4. - """ - session = get_active_session(self._get_device_id()) - return await normalize_recording_to_mp4( - mkv_path, - mp4_path, - session.capture_width if session else None, - session.capture_height if session else None, - ) + return abort_recording(device_id, "stop", e) async def cleanup(self) -> None: await self._driver.disconnect() diff --git a/artemis/core/diagnostics/adb_keys.py b/artemis/core/diagnostics/adb_keys.py index b3a2d037e..e342b19e1 100644 --- a/artemis/core/diagnostics/adb_keys.py +++ b/artemis/core/diagnostics/adb_keys.py @@ -31,7 +31,7 @@ from typing import Any from artemis.toolchain import toolchain -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/core/diagnostics/adb_server_connection.py b/artemis/core/diagnostics/adb_server_connection.py index 9fdb00a7e..4244306a6 100644 --- a/artemis/core/diagnostics/adb_server_connection.py +++ b/artemis/core/diagnostics/adb_server_connection.py @@ -34,7 +34,7 @@ current_adb_endpoint, ) from artemis.toolchain import toolchain -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py index 2fcc4ac60..3c43c9cf4 100644 --- a/artemis/core/diagnostics/device_smoke.py +++ b/artemis/core/diagnostics/device_smoke.py @@ -57,7 +57,7 @@ from typing import Any, TypeVar from artemis.runtime.device_lock import DeviceExecutionLock -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/core/diagnostics/emulator_manager.py b/artemis/core/diagnostics/emulator_manager.py index 9e2695011..7895b0e5a 100644 --- a/artemis/core/diagnostics/emulator_manager.py +++ b/artemis/core/diagnostics/emulator_manager.py @@ -30,7 +30,7 @@ from artemis.platform import OSType, platform from artemis.toolchain import toolchain -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/core/diagnostics/engine.py b/artemis/core/diagnostics/engine.py index 39ca9633f..97f9d20ca 100644 --- a/artemis/core/diagnostics/engine.py +++ b/artemis/core/diagnostics/engine.py @@ -42,7 +42,7 @@ ) from artemis.toolchain import toolchain from artemis.platform import platform -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/core/diagnostics/hierarchy_parity.py b/artemis/core/diagnostics/hierarchy_parity.py index e1b905a91..19082eec4 100644 --- a/artemis/core/diagnostics/hierarchy_parity.py +++ b/artemis/core/diagnostics/hierarchy_parity.py @@ -34,7 +34,7 @@ import time from typing import Any -from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements +from third_party.mobile_use.clients.ui_automator_client import parse_hierarchy_xml_to_elements MIN_RECALL = 0.9 MIN_PRECISION = 0.8 @@ -62,7 +62,7 @@ def _iou(a: dict[str, int], b: dict[str, int]) -> float: def analyze_xml(xml: str, width: int, height: int) -> dict[str, Any]: """Counts and labelled elements of one dump, plus the bounds problems the helper must not have.""" - elements = _parse_hierarchy_xml_to_elements(xml) + elements = parse_hierarchy_xml_to_elements(xml) labelled: list[tuple[str, dict[str, int]]] = [] negative = offscreen = 0 package = None @@ -167,7 +167,7 @@ def compare_dumps(helper_xml: str, uiautomator_xml: str, width: int, height: int def compare_backends(serial: str, rounds: int = 3) -> dict[str, Any]: """Dump the current screen of ``serial`` with both backends and compare (device required).""" from artemis.clients.accessibility_client import AccessibilityClient - from artemis.clients.ui_automator_client import UIAutomatorClient + from third_party.mobile_use.clients.ui_automator_client import UIAutomatorClient helper = AccessibilityClient(serial) u2 = UIAutomatorClient(serial) diff --git a/artemis/core/diagnostics/probes/adb_probe.py b/artemis/core/diagnostics/probes/adb_probe.py index 4abd9288b..850de1d6b 100644 --- a/artemis/core/diagnostics/probes/adb_probe.py +++ b/artemis/core/diagnostics/probes/adb_probe.py @@ -33,7 +33,7 @@ ) from artemis.platform import OSType, platform from artemis.toolchain import toolchain -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/data_engine/engine.py b/artemis/data_engine/engine.py index e880d3a95..7850082bb 100644 --- a/artemis/data_engine/engine.py +++ b/artemis/data_engine/engine.py @@ -45,8 +45,8 @@ normalize_any_structure, normalize_step_actions, ) -from artemis.utils.logger import get_logger from artemis.utils.text import safe_extract_text +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/data_engine/history_reader.py b/artemis/data_engine/history_reader.py index 942ef0da5..fe7d71345 100644 --- a/artemis/data_engine/history_reader.py +++ b/artemis/data_engine/history_reader.py @@ -27,7 +27,7 @@ from artemis.data_engine.engine import build_image_describer, friendly_step from artemis.data_engine.models import HistoryChunkRecord, StepRecord from artemis.data_engine.storage import StorageManager -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/data_engine/storage.py b/artemis/data_engine/storage.py index 73825fbf8..581737c5f 100644 --- a/artemis/data_engine/storage.py +++ b/artemis/data_engine/storage.py @@ -34,7 +34,7 @@ TraceRecord, VideoRecordingRecord, ) -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger def _safe_uuid(val: Any) -> UUID | str: diff --git a/artemis/drivers/android/adb_driver.py b/artemis/drivers/android/adb_driver.py index 4a543810b..360aa4862 100644 --- a/artemis/drivers/android/adb_driver.py +++ b/artemis/drivers/android/adb_driver.py @@ -21,16 +21,16 @@ from typing import Any, Literal from adbutils import AdbClient, AdbDevice -from artemis.clients.ui_automator_client import ( - UIAutomatorClient, - _parse_hierarchy_xml_to_elements, -) from artemis.config.paths import get_temp_dir from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection from artemis.toolchain import find_ffmpeg, find_scrcpy from artemis.utils.video import build_scrcpy_record_command from artemis.utils.ui_filter import filter_ui_hierarchy -from artemis.utils.logger import get_logger +from third_party.mobile_use.clients.ui_automator_client import ( + UIAutomatorClient, + parse_hierarchy_xml_to_elements, +) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -182,7 +182,7 @@ async def get_screen_data(self, skip_settling: bool = False) -> ScreenData: hierarchy = await res if asyncio.iscoroutine(res) else res if isinstance(hierarchy, str): ui_hierarchy_xml = hierarchy - ui_elements = _parse_hierarchy_xml_to_elements(hierarchy) + ui_elements = parse_hierarchy_xml_to_elements(hierarchy) else: ui_elements = hierarchy elif hasattr(self._ui_adb_client, "get_ui_elements"): diff --git a/artemis/drivers/android/hierarchy.py b/artemis/drivers/android/hierarchy.py index c5762e0b5..25a3d3fa3 100644 --- a/artemis/drivers/android/hierarchy.py +++ b/artemis/drivers/android/hierarchy.py @@ -17,7 +17,7 @@ import re from typing import Any import xml.etree.ElementTree as ET -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/drivers/android/input_ime.py b/artemis/drivers/android/input_ime.py index 723a85bd6..4083790fe 100644 --- a/artemis/drivers/android/input_ime.py +++ b/artemis/drivers/android/input_ime.py @@ -17,7 +17,7 @@ import asyncio import base64 from typing import Any -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/drivers/android/recorder.py b/artemis/drivers/android/recorder.py index a0325f9aa..9c57203e2 100644 --- a/artemis/drivers/android/recorder.py +++ b/artemis/drivers/android/recorder.py @@ -17,7 +17,7 @@ import asyncio from pathlib import Path from typing import Any -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/drivers/base.py b/artemis/drivers/base.py index 54d7ff762..31a3fbc35 100644 --- a/artemis/drivers/base.py +++ b/artemis/drivers/base.py @@ -28,6 +28,8 @@ from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field +from third_party.mobile_use.controllers.types import ElementQuery + class SwipeDirection(str, Enum): UP = "up" @@ -221,13 +223,10 @@ async def swipe_normalized( return await self.swipe(sx, sy, ex, ey, duration_ms=duration_ms) async def find_element( - self, - resource_id: str | None = None, - text: str | None = None, - index: int = 0, - screen_data: ScreenData | None = None, + self, query: ElementQuery, screen_data: ScreenData | None = None ) -> tuple[dict[str, Any] | None, list[int] | None, str | None]: """Finds matching UI element in screen hierarchy by resource_id or text.""" + resource_id, text, index = query.resource_id, query.text, query.index data = screen_data or await self.get_screen_data() elements = data.ui_elements or [] @@ -242,7 +241,7 @@ async def find_element( matched.append(elem) if not matched: - return None, None, f"Element not found (resource_id={resource_id}, text={text})" + return None, None, f"Element not found ({query.describe()})" if index >= len(matched): return None, None, f"Element index {index} out of range (matched {len(matched)})" @@ -280,15 +279,10 @@ async def find_element( return elem, center, None async def tap_element( - self, - resource_id: str | None = None, - text: str | None = None, - index: int = 0, - long_press: bool = False, - duration_ms: int = 1000, + self, query: ElementQuery, long_press: bool = False, duration_ms: int = 1000 ) -> bool: """Finds and taps a UI element by resource_id or text.""" - _, center, error = await self.find_element(resource_id=resource_id, text=text, index=index) + _, center, error = await self.find_element(query) if error or not center: return False if long_press: diff --git a/artemis/drivers/cloud/cloud_driver.py b/artemis/drivers/cloud/cloud_driver.py index 28e39e9f5..4bfcb67df 100644 --- a/artemis/drivers/cloud/cloud_driver.py +++ b/artemis/drivers/cloud/cloud_driver.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Literal from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/drivers/factory.py b/artemis/drivers/factory.py index ba1bcbf87..d6283a1c6 100644 --- a/artemis/drivers/factory.py +++ b/artemis/drivers/factory.py @@ -23,7 +23,7 @@ from artemis.drivers.android.adb_driver import AndroidAdbDriver from artemis.drivers.base import BaseDeviceDriver from artemis.drivers.mock.mock_driver import MockDeviceDriver -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger if TYPE_CHECKING: from artemis.context import ArtemisContext diff --git a/artemis/drivers/types.py b/artemis/drivers/types.py index 7b891734f..fa509b895 100644 --- a/artemis/drivers/types.py +++ b/artemis/drivers/types.py @@ -15,66 +15,24 @@ """Typed data models and geometry representations for device drivers.""" from typing import Literal -from pydantic import BaseModel, ConfigDict, Field - - -class TapOutput(BaseModel): - """Output from tap operations.""" - - error: str | None = Field(default=None, description="Error message if tap failed") - success: bool = Field(default=True, description="Whether the tap was successfully dispatched") - - -class CoordinatesSelectorRequest(BaseModel): - """Absolute pixel coordinate pair.""" - - model_config = ConfigDict(extra="forbid") - x: int - y: int - - def to_str(self) -> str: - return f"{self.x}, {self.y}" - - -class PercentagesSelectorRequest(BaseModel): - """Normalized percentage coordinate pair (0-100%).""" - - model_config = ConfigDict(extra="forbid") - x_percent: int = Field(ge=0, le=100, description="X percentage (0-100)") - y_percent: int = Field(ge=0, le=100, description="Y percentage (0-100)") - - def to_str(self) -> str: - return f"{self.x_percent}%, {self.y_percent}%" - - def to_coords(self, width: int, height: int) -> CoordinatesSelectorRequest: - """Convert percentages to absolute pixel coordinates.""" - x = min(max(int(width * self.x_percent / 100), 0), max(0, width - 1)) - y = min(max(int(height * self.y_percent / 100), 0), max(0, height - 1)) - return CoordinatesSelectorRequest(x=x, y=y) - - -class Bounds(BaseModel): - """Bounding box defined by top-left (x1, y1) and bottom-right (x2, y2).""" - - x1: int - y1: int - x2: int - y2: int - - @property - def width(self) -> int: - return max(0, self.x2 - self.x1) - - @property - def height(self) -> int: - return max(0, self.y2 - self.y1) - - def get_center(self) -> CoordinatesSelectorRequest: - """Get the center point of the bounds.""" - return CoordinatesSelectorRequest( - x=(self.x1 + self.x2) // 2, - y=(self.y1 + self.y2) // 2, - ) +from pydantic import BaseModel, Field + +from third_party.mobile_use.controllers.types import ( + Bounds, + CoordinatesSelectorRequest, + PercentagesSelectorRequest, + TapOutput, +) + +__all__ = [ + "Bounds", + "CoordinatesSelectorRequest", + "DeviceInfo", + "PercentagesSelectorRequest", + "SwipeRequest", + "SwipeStartEndPercentagesRequest", + "TapOutput", +] class SwipeRequest(BaseModel): diff --git a/artemis/graph/checkpoints.py b/artemis/graph/checkpoints.py index ae8ed4ed0..fa03d95ce 100644 --- a/artemis/graph/checkpoints.py +++ b/artemis/graph/checkpoints.py @@ -40,7 +40,6 @@ from pydantic import BaseModel, Field -from artemis.utils.logger import get_logger from artemis.utils.notes import append_note_content, get_note_file_path from artemis.utils.plan_grammar import ( CHECKBOX_LINE_RE, @@ -52,6 +51,7 @@ parse_plan, subgoal_hash, ) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/graph/graph.py b/artemis/graph/graph.py index ab854a1d5..499739110 100644 --- a/artemis/graph/graph.py +++ b/artemis/graph/graph.py @@ -79,7 +79,6 @@ ) from artemis.tools.tool_wrapper import invoke_tool_with_injection from artemis.tools.video_tool import get_video_analyzer_tool -from artemis.utils.logger import get_logger from artemis.utils.notes import ( SAVE_NOTE_ARG_CONTENT_DESC, SAVE_NOTE_ARG_KEY_DESC, @@ -99,15 +98,17 @@ unintended_milestone_edits, ) from artemis.utils.task_tree import get_active_subgoal_hashes +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.graph import ( + CONVERGENCE_NODE, + add_convergence_edges, + add_convergence_node, +) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) -def convergence_node(state: State): - """Convergence point for parallel execution paths.""" - return {} - - def _notify_history_chunker(ctx: ArtemisContext, method: str, *args) -> None: """Best-effort event feed to the transcript's L2/L3 chunk manager (M3). @@ -511,24 +512,6 @@ def _get_active_subgoal_hashes(ctx: ArtemisContext) -> tuple[str, str | None]: return "default", None -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - def check_plan_mutation_rejections( content_before: str, content_after: str, state: State | None = None ) -> str | None: @@ -1036,32 +1019,28 @@ async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph: graph_builder.add_node("summarizer", SummarizerNode(ctx)) graph_builder.add_node("execution_check", functools.partial(execution_check_node, ctx=ctx)) graph_builder.add_node("perception", functools.partial(perception_node, ctx=ctx)) - graph_builder.add_node(node="convergence", action=convergence_node, defer=True) + add_convergence_node(graph_builder) graph_builder.add_node("exit_settlement", functools.partial(exit_settlement_node, ctx=ctx)) ## Linking nodes graph_builder.add_edge(START, "planner") - graph_builder.add_edge("planner", "convergence") + graph_builder.add_edge("planner", CONVERGENCE_NODE) graph_builder.add_edge("operator", "execution_check") graph_builder.add_conditional_edges( "execution_check", execution_check_edge, { - "review_subgoals": "convergence", + "review_subgoals": CONVERGENCE_NODE, "execute_decisions": "validator", }, ) graph_builder.add_edge("validator", "summarizer") - graph_builder.add_edge("summarizer", "convergence") + graph_builder.add_edge("summarizer", CONVERGENCE_NODE) - graph_builder.add_conditional_edges( - source="convergence", - path=functools.partial(convergence_gate, ctx=ctx), - path_map={ - "continue": "perception", - "exit_settlement": "exit_settlement", - "end": END, - }, + add_convergence_edges( + graph_builder, + functools.partial(convergence_gate, ctx=ctx), + {"continue": "perception", "exit_settlement": "exit_settlement", "end": END}, ) graph_builder.add_conditional_edges( source="exit_settlement", diff --git a/artemis/graph/perception.py b/artemis/graph/perception.py index 85a6d214e..d60d82c62 100644 --- a/artemis/graph/perception.py +++ b/artemis/graph/perception.py @@ -23,7 +23,6 @@ from artemis.data_engine.trace import trace from artemis.graph.state import State from artemis.graph.visibility import strict_state -from artemis.utils.logger import get_logger from artemis.utils.ocr_api import is_ocr_configured, perform_ocr from artemis.utils.ocr_xml_fusion import ( _crop_image_remove_status_bar, @@ -31,6 +30,7 @@ _map_coordinates_back, fuse_ocr_with_xml, ) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/graph/state.py b/artemis/graph/state.py index 11fc58cb7..560104ff8 100644 --- a/artemis/graph/state.py +++ b/artemis/graph/state.py @@ -16,16 +16,12 @@ from pydantic import BaseModel, ConfigDict -from artemis.utils.logger import get_logger +from third_party.mobile_use.graph import take_last +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) -def take_last(a, b): - """Reducer function keeping the latest value.""" - return b - - def sticky_or(a, b): """Reducer that latches True: once set, the flag survives later updates.""" return bool(a) or bool(b) diff --git a/artemis/interfaces/cli/commands/batch.py b/artemis/interfaces/cli/commands/batch.py index 13f52a9f6..225ba98fc 100644 --- a/artemis/interfaces/cli/commands/batch.py +++ b/artemis/interfaces/cli/commands/batch.py @@ -23,8 +23,8 @@ from artemis.config import initialize_llm_config from artemis.sdk import Agent from artemis.sdk.builders import Builders -from artemis.sdk.types.task import AgentProfile -from artemis.utils.logger import get_logger +from third_party.mobile_use.sdk.types.task import AgentProfile +from third_party.mobile_use.utils.logger import get_logger from rich.console import Console from rich.table import Table import typer diff --git a/artemis/interfaces/cli/commands/helper.py b/artemis/interfaces/cli/commands/helper.py index 9bc696ad2..a07f7f0a8 100644 --- a/artemis/interfaces/cli/commands/helper.py +++ b/artemis/interfaces/cli/commands/helper.py @@ -31,7 +31,7 @@ from artemis.runtime.device_pool import device_pool from artemis.runtime.helper_manager import helper_manager -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) helper_app = typer.Typer(help="Install, inspect, or remove the Artemis Accessibility Helper APK.") diff --git a/artemis/interfaces/cli/commands/mcp.py b/artemis/interfaces/cli/commands/mcp.py index 6a6ca3596..a819d977f 100644 --- a/artemis/interfaces/cli/commands/mcp.py +++ b/artemis/interfaces/cli/commands/mcp.py @@ -28,7 +28,7 @@ from mcp_server.utils import env_utils from artemis.mcp.adb_server import mcp as adb_mcp from artemis.runtime import shutdown_awake_service, start_awake_service -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger from rich.console import Console from rich.syntax import Syntax import typer diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 91a5d0440..89dca25ab 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -16,19 +16,23 @@ import asyncio import os -from pathlib import Path -from shutil import which from typing import Annotated -from adbutils import AdbClient from langchain_core.callbacks.base import Callbacks -from artemis.config import checker_overrides_for_level, initialize_llm_config, settings +from artemis.config import checker_overrides_for_level, settings from artemis.utils.startup_progress import publish_startup_progress -from artemis import Agent, Builders -from artemis.sdk.types.task import AgentProfile -from artemis.utils.cli_helpers import display_device_status -from artemis.utils.logger import get_logger -from artemis.utils.video import check_ffmpeg_available +from third_party.mobile_use.main import ( + GoalArgument, + OutputDescriptionOption, + TestNameOption, + TracesPathOption, + VideoRecordingToolsOption, + ensure_video_recording_available, + new_default_config_builder, + run_automation, +) +from third_party.mobile_use.utils.cli_helpers import display_local_device_status +from third_party.mobile_use.utils.logger import get_logger import signal from rich.console import Console from rich.panel import Panel @@ -93,9 +97,7 @@ async def execute_task( session_id=str(effective_sid) if effective_sid else None, ) - llm_config = initialize_llm_config() - agent_profile = AgentProfile(name="default", llm_config=llm_config) - config = Builders.AgentConfig.with_default_profile(profile=agent_profile) + config = new_default_config_builder() if video_recording_tools_enabled is not None: config.with_video_recording_tools(enabled=video_recording_tools_enabled) @@ -132,9 +134,6 @@ async def execute_task( pro_mode=explorer_pro_mode, ) - if settings.ADB_HOST: - config.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT) - target_serial = ( device_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") ) @@ -154,39 +153,21 @@ async def execute_task( if graph_config_callbacks: config.with_graph_config_callbacks(graph_config_callbacks) - agent: Agent | None = None - try: - agent = Agent(config=config.build(), session_id=effective_sid) - await agent.init( - retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), - retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)), - ) - - task = agent.new_task(goal) - if locked_app_package: - task.with_locked_app_package(locked_app_package) - if test_name: - trace_path = traces_output_path_str or str(settings.TRACES_PATH) - task.with_name(test_name).with_trace_recording(path=trace_path) - if output_description: - task.with_output_description(output_description) - if profile: - task.using_profile(profile) - if app_path: - task.with_app_path(Path(app_path)) - - llm_result_path = os.getenv("RESULTS_OUTPUT_PATH", None) - if llm_result_path: - task.with_llm_output_saving(path=llm_result_path) - - await agent.run_task(request=task.build()) - finally: - if agent is not None: - await agent.clean() + await run_automation( + config, + goal, + session_id=effective_sid, + locked_app_package=locked_app_package, + test_name=test_name, + traces_output_path_str=traces_output_path_str, + output_description=output_description, + profile=profile, + app_path=app_path, + ) def run_command( - goal: Annotated[str, typer.Argument(help="The main goal for the agent to achieve.")], + goal: GoalArgument, profile: Annotated[ str | None, typer.Option( @@ -203,40 +184,10 @@ def run_command( help="Optional Android package name to restrict the agent to.", ), ] = None, - test_name: Annotated[ - str | None, - typer.Option( - "--test-name", - "-n", - help="Name of the test run for trace recording.", - ), - ] = None, - traces_path: Annotated[ - str | None, - typer.Option( - "--traces-path", - "-t", - help="Directory where execution traces are persisted.", - ), - ] = None, - output_description: Annotated[ - str | None, - typer.Option( - "--output-description", - "-o", - help="Natural language or schema description of the expected output.", - ), - ] = None, - with_video_recording_tools: Annotated[ - bool | None, - typer.Option( - "--with-video-recording-tools/--without-video-recording-tools", - help=( - "Enable or disable dynamic video recording and screen analysis " - "tools (auto-detected if omitted)." - ), - ), - ] = None, + test_name: TestNameOption = None, + traces_path: TracesPathOption = None, + output_description: OutputDescriptionOption = None, + with_video_recording_tools: VideoRecordingToolsOption = None, app_path: Annotated[ str | None, typer.Option( @@ -346,8 +297,7 @@ def run_command( ] = False, ) -> None: """Run an autonomous UI automation task on the connected Android device.""" - if with_video_recording_tools: - check_ffmpeg_available() + ensure_video_recording_available(with_video_recording_tools) console = Console() @@ -440,18 +390,7 @@ def on_status(sess_info): f"[yellow]Daemon routing notice: {exc}. Falling back to local execution...[/yellow]" ) - adb_client = None - try: - if which("adb"): - adb_client = AdbClient( - host=settings.ADB_HOST or "localhost", - port=settings.ADB_PORT or 5037, - ) - except Exception as exc: - # Optional cosmetic device-status display; run continues without it. - logger.debug(f"Could not create ADB client for device status display: {exc}") - - display_device_status(console, adb_client=adb_client) + display_local_device_status(console, host=settings.ADB_HOST, port=settings.ADB_PORT) cancelled = False original_sigterm = None diff --git a/artemis/interfaces/cli/commands/server.py b/artemis/interfaces/cli/commands/server.py index 265fccfa0..96e5faa9d 100644 --- a/artemis/interfaces/cli/commands/server.py +++ b/artemis/interfaces/cli/commands/server.py @@ -28,7 +28,7 @@ status_command, stop_command, ) -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger import typer logger = get_logger(__name__) diff --git a/artemis/interfaces/cli/commands/trace.py b/artemis/interfaces/cli/commands/trace.py index c2c200273..2942ac1cc 100644 --- a/artemis/interfaces/cli/commands/trace.py +++ b/artemis/interfaces/cli/commands/trace.py @@ -20,7 +20,7 @@ from typing import Annotated from artemis.config import settings -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger from rich.console import Console from rich.table import Table import typer diff --git a/artemis/interfaces/cli/main.py b/artemis/interfaces/cli/main.py index d11cbed17..629d6b2cb 100644 --- a/artemis/interfaces/cli/main.py +++ b/artemis/interfaces/cli/main.py @@ -31,7 +31,7 @@ ) from artemis.interfaces.cli.commands.trace import trace_app from artemis.interfaces.cli.commands.ui import ui_command -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger from rich.console import Console from rich.panel import Panel import typer diff --git a/artemis/llm/router.py b/artemis/llm/router.py index 8b44743b5..f2e6b42b6 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -29,7 +29,7 @@ from artemis.config.settings import settings from artemis.llm.google.provider import supports_thinking_level -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/mcp/action_executor.py b/artemis/mcp/action_executor.py index b3e565ebc..79b399cda 100644 --- a/artemis/mcp/action_executor.py +++ b/artemis/mcp/action_executor.py @@ -55,7 +55,6 @@ compute_smart_swipe_coordinates, parse_swipe_parameters, ) -from artemis.utils.logger import get_logger from artemis.utils.notes import ( format_list_notes_failure, format_list_notes_success, @@ -63,6 +62,7 @@ list_notes_info, read_note_content, ) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/mcp/action_server.py b/artemis/mcp/action_server.py index 7369f7493..be32b95c9 100644 --- a/artemis/mcp/action_server.py +++ b/artemis/mcp/action_server.py @@ -45,7 +45,7 @@ from artemis.mcp.action_types import ActionCode, ActionResult, ObserveResult from artemis.mcp.actuators.base import Actuator from artemis.mcp.observation import observe as observe_impl -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/mcp/action_session.py b/artemis/mcp/action_session.py index 4666d43ef..06e512ca7 100644 --- a/artemis/mcp/action_session.py +++ b/artemis/mcp/action_session.py @@ -54,7 +54,7 @@ from mcp.shared.memory import create_connected_server_and_client_session from artemis.mcp.action_types import ActionResult, ObserveResult -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/mcp/actuators/adb.py b/artemis/mcp/actuators/adb.py index d0c1107b8..4580e436b 100644 --- a/artemis/mcp/actuators/adb.py +++ b/artemis/mcp/actuators/adb.py @@ -35,7 +35,7 @@ from artemis.controllers.unified_controller import UnifiedMobileController from artemis.mcp.action_manifest import DEVICE_ACTIONS, ExtensionTool from artemis.mcp.action_types import ActionCode, ActionResult -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -286,7 +286,7 @@ async def press_key(self, key: str) -> ActionResult: async def manage_app(self, action: str, app_name: str) -> ActionResult: # Imported lazily: launch_app pulls in tool wrappers that are costly at import. from artemis.tools.mobile.launch_app import find_package - from artemis.utils.app_launch_utils import launch_app_with_retries + from third_party.mobile_use.utils.app_launch_utils import launch_app_with_retries res = find_package(self.ctx, app_name, use_fallback=False) pkg = await res if inspect.iscoroutine(res) else res diff --git a/artemis/mcp/adb_server.py b/artemis/mcp/adb_server.py index be2b71abb..0c3c9df59 100644 --- a/artemis/mcp/adb_server.py +++ b/artemis/mcp/adb_server.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio import json import logging import os @@ -43,7 +42,7 @@ from artemis.context import ArtemisContext, DeviceContext, DevicePlatform from artemis.controllers.unified_controller import UnifiedMobileController from artemis.platform import platform -from artemis.utils.app_launch_utils import launch_app_with_retries +from third_party.mobile_use.utils.app_launch_utils import launch_app_with_retries def configure_stdio_mode() -> None: @@ -184,7 +183,6 @@ def _get_controller(device_serial: str | None = None): # single implementation in artemis.mcp.actuators.adb. from artemis.mcp.actuators.adb import ( # noqa: E402 pylint: disable=wrong-import-position ensure_focus_at_coords as _ensure_focus_at_coords, - find_element_at_coords as _find_element_at_coords, ) diff --git a/artemis/mcp/observation.py b/artemis/mcp/observation.py index 42fffa63c..7b40eec5b 100644 --- a/artemis/mcp/observation.py +++ b/artemis/mcp/observation.py @@ -28,9 +28,9 @@ from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.mcp.action_types import ActionCode, ObserveResult -from artemis.utils.logger import get_logger from artemis.utils.ocr_xml_fusion import fuse_ocr_with_xml from artemis.utils.visualization import format_minimal_list_with_elements +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/memory/__init__.py b/artemis/memory/__init__.py index 5b005dfdf..d0f2f2e7b 100644 --- a/artemis/memory/__init__.py +++ b/artemis/memory/__init__.py @@ -28,7 +28,7 @@ ) from artemis.memory.step_memory import StepLens, StepMemoryService from artemis.memory.transcript import TranscriptLedger, format_session_offset -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/memory/chunking.py b/artemis/memory/chunking.py index 4e5645409..c0c91ea07 100644 --- a/artemis/memory/chunking.py +++ b/artemis/memory/chunking.py @@ -48,7 +48,7 @@ from artemis.llm.google import is_google_provider from artemis.memory.step_memory import JobKey, StepLens, StepMemoryService from artemis.memory.transcript import format_session_offset -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/memory/context_policy.py b/artemis/memory/context_policy.py index cad063de0..2183ac67d 100644 --- a/artemis/memory/context_policy.py +++ b/artemis/memory/context_policy.py @@ -36,7 +36,7 @@ from dataclasses import dataclass, fields, replace from typing import Any -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/memory/step_memory.py b/artemis/memory/step_memory.py index 98e256b72..4e3312913 100644 --- a/artemis/memory/step_memory.py +++ b/artemis/memory/step_memory.py @@ -27,7 +27,7 @@ from typing import Any from artemis.data_engine.trace import detached_trace -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/memory/transcript.py b/artemis/memory/transcript.py index d08c1d0f7..e87ecf7bf 100644 --- a/artemis/memory/transcript.py +++ b/artemis/memory/transcript.py @@ -55,7 +55,7 @@ from langchain_core.messages import AIMessage, BaseMessage, HumanMessage from artemis.memory.step_memory import StepMemoryService -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/runtime/awake_lease.py b/artemis/runtime/awake_lease.py index 5f03ab720..3f7371d67 100644 --- a/artemis/runtime/awake_lease.py +++ b/artemis/runtime/awake_lease.py @@ -30,7 +30,7 @@ from artemis.config.paths import get_temp_dir from artemis.runtime.adb_endpoint import adb_command from artemis.runtime.process_probe import pid_is_alive -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/runtime/awake_service.py b/artemis/runtime/awake_service.py index 2de284e81..fc29ae3d9 100644 --- a/artemis/runtime/awake_service.py +++ b/artemis/runtime/awake_service.py @@ -27,7 +27,7 @@ from artemis.config import settings from artemis.runtime.adb_endpoint import adb_command from artemis.runtime.awake_lease import ScreenAwakeLease -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/runtime/daemon_client.py b/artemis/runtime/daemon_client.py index 67736e651..77d63bb3c 100644 --- a/artemis/runtime/daemon_client.py +++ b/artemis/runtime/daemon_client.py @@ -28,7 +28,7 @@ import urllib.request from artemis.config.paths import ROOT_DIR -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/runtime/device_pool.py b/artemis/runtime/device_pool.py index e6f33386b..c67aa5381 100644 --- a/artemis/runtime/device_pool.py +++ b/artemis/runtime/device_pool.py @@ -27,7 +27,7 @@ from artemis.runtime.device_lock import DeviceExecutionLock from artemis.toolchain import toolchain -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/runtime/helper_manager.py b/artemis/runtime/helper_manager.py index 3196d5e49..5830ad267 100644 --- a/artemis/runtime/helper_manager.py +++ b/artemis/runtime/helper_manager.py @@ -80,7 +80,7 @@ from artemis.config.constants import ENV_ARTEMIS_HELPER_AUTO_INSTALL from artemis.config.paths import get_temp_dir from artemis.runtime.adb_endpoint import adb_command -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/runtime/server_lifecycle.py b/artemis/runtime/server_lifecycle.py index b0932ace0..820404a7e 100644 --- a/artemis/runtime/server_lifecycle.py +++ b/artemis/runtime/server_lifecycle.py @@ -37,7 +37,7 @@ from artemis.runtime.device_lock import DeviceExecutionLock from artemis.runtime.process_probe import pid_is_alive from artemis.runtime.supervisor import ProcessSupervisor -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/runtime/supervisor.py b/artemis/runtime/supervisor.py index 3aec1afe0..70b814e7e 100644 --- a/artemis/runtime/supervisor.py +++ b/artemis/runtime/supervisor.py @@ -20,7 +20,7 @@ from pathlib import Path from artemis.platform import platform -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index a0028eb19..c0c0da08b 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -11,104 +11,54 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import asyncio import inspect import os import re -try: - from datetime import UTC, datetime -except ImportError: - from datetime import datetime - - UTC = UTC -from pathlib import Path -from platform import system -import shutil -from shutil import which -import sys -import threading -from types import NoneType -from typing import Any, TypeVar, overload import uuid +from pathlib import Path -from adbutils import AdbClient from dotenv import load_dotenv from google import genai from langchain_google_genai import ChatGoogleGenerativeAI -from PIL import Image -from pydantic import BaseModel -from artemis.agents.flash.runner import FlashRunner -from artemis.agents.outputter.outputter import outputter from artemis.clients.screen_client_factory import ( - ScreenClient, - create_screen_client, describe_backend, helper_version, hierarchy_backend_sentence, ) from artemis.config import ( CheckerConfig, - OutputConfig, - record_events, run_tuning_for_profile, settings, ) -from artemis.constants import RECURSION_LIMIT from artemis.context import ( ArtemisContext, - DeviceContext, DevicePlatform, ExecutionSetup, ) -from artemis.controllers.controller_factory import get_controller -from artemis.controllers.platform_specific_commands_controller import ( - get_first_device, -) -from artemis.runtime import DeviceExecutionLock, trace_store -from artemis.runtime.cancel_requests import watch_for_cancel_request from artemis.data_engine.engine import DataEngine -from artemis.data_engine.trace import DataEngineCallbackHandler -from artemis.graph.graph import get_graph from artemis.graph.state import State -from artemis.sdk.builders.agent_config_builder import get_default_agent_config -from artemis.sdk.builders.task_request_builder import TaskRequestBuilder +from artemis.runtime import trace_store +from artemis.runtime.cancel_requests import watch_for_cancel_request +from artemis.sdk.run_outcome import attach_test_summary, resolve_trace_suffix from artemis.sdk.types.agent import AgentConfig -from artemis.sdk.types.exceptions import ( - AgentError, - AgentNotInitializedError, - AgentProfileNotFoundError, - AgentTaskRequestError, - DeviceNotFoundError, - ExecutableNotFoundError, -) -from artemis.sdk.types.task import ( - AgentProfile, - Task, - TaskRequest, -) -from artemis.utils.app_launch_utils import _handle_initial_app_launch -from artemis.utils.logger import get_logger -from artemis.utils.media import ( - create_gif_from_trace_folder, - create_steps_json_from_trace_folder, - remove_images_from_trace_folder, - remove_steps_json_from_trace_folder, -) from artemis.utils.startup_progress import publish_startup_progress +from third_party.mobile_use.sdk.types.exceptions import AgentError +from third_party.mobile_use.sdk.types.task import Task +from third_party.mobile_use.sdk.agent import AgentBase, TOutput +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) -TOutput = TypeVar("TOutput", bound=BaseModel | None) - load_dotenv() +__all__ = ["Agent", "TOutput", "attach_test_summary", "resolve_trace_suffix", "run_tuning_summary"] + + def run_tuning_summary(config: AgentConfig, profile: str | None) -> dict[str, str] | None: """Per-run Pro tuning (verification level, explorer mode) for an SDK config. @@ -129,18 +79,8 @@ def run_tuning_summary(config: AgentConfig, profile: str | None) -> dict[str, st ) -class Agent: - _config: AgentConfig - _tasks: list[Task] = [] - _tmp_traces_dir: Path - _initialized: bool = False - _device_context: DeviceContext - _adb_client: AdbClient | None - _ui_adb_client: ScreenClient | Any | None - - _current_task: asyncio.Task | None = None - _task_lock: asyncio.Lock - _cloud_mobile_id: str | None = None +class Agent(AgentBase): + """Artemis SDK agent.""" def __init__( self, @@ -186,93 +126,6 @@ def __init__( self._initialized = False self._task_lock = asyncio.Lock() - async def init( - self, - api_key: str | None = None, - device_id: str | None = None, - device_serial: str | None = None, - retry_count: int = 5, - retry_wait_seconds: int = 5, - ): - target_dev = device_serial or device_id - if target_dev: - self._config = self._config.model_copy( - update={"device_id": target_dev, "device_platform": DevicePlatform.ANDROID} - ) - - return await self._init_internal( - api_key=api_key, - retry_count=retry_count, - retry_wait_seconds=retry_wait_seconds, - ) - - async def _init_internal( - self, - api_key: str | None = None, - retry_count: int = 5, - retry_wait_seconds: int = 5, - ): - - if os.environ.get("ARTEMIS_CLOUD_MODE") != "1" and not which("adb"): - raise ExecutableNotFoundError("adb") - - if self._initialized: - logger.warning("Agent is already initialized. Skipping...") - return True - - # Get first available device ID - if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": - device_id = os.environ.get("ADB_DEVICE_SERIAL", "cloud_device") - platform = DevicePlatform.ANDROID - elif not self._config.device_id or not self._config.device_platform: - device_id, platform, _ = get_first_device(logger=logger) - else: - device_id, platform = ( - self._config.device_id, - self._config.device_platform, - ) - - if not device_id or not platform: - error_msg = "No device found. Exiting." - logger.error(error_msg) - raise DeviceNotFoundError(error_msg) - - # Initialize clients - publish_startup_progress( - "device_check", "Checking the Android device", session_id=self._session_id - ) - if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": - self._init_clients( - device_id=device_id, - platform=platform, - ) - else: - # Cloud mode is pinned to UIAutomator2 through the gateway; the - # hierarchy backend switch only applies to locally attached devices. - from cloud_service.virtualization import RemoteAdbClient, RemoteUIAutomatorClient - - self._adb_client = RemoteAdbClient() - self._ui_adb_client = RemoteUIAutomatorClient( - device_id=device_id, - adb_client=self._adb_client, - ) - - self._device_context = await self._get_device_context( - device_id=device_id, platform=platform - ) - logger.info(self._device_context.to_str()) - publish_startup_progress( - "device_ready", "Android device connected", session_id=self._session_id - ) - - # Asynchronously pre-warm LLM connection pools in the background - asyncio.create_task(self._prewarm_llm_connections(api_key)) - - logger.info("✅ Artemis agent initialized.") - self._initialized = True - - return True - async def _prewarm_llm_connections(self, api_key: str | None = None): """Pre-warms the HTTP2/gRPC connection pools for both Native GenAI and LangChain clients in the background.""" if os.environ.get("ARTEMIS_FAKE_LLM") == "1": @@ -323,601 +176,6 @@ async def _prewarm_llm_connections(self, api_key: str | None = None): session_id=self._session_id, ) - async def install_apk(self, apk_path: str | Path) -> None: - """Install an APK on the connected device. - - For cloud mobiles, the APK must be x86_64 compatible. - - Args: - apk_path: Path to the local APK file to install - - Raises: - AgentNotInitializedError: If the agent is not initialized - AgentError: If attempting to install on non-Android device or ADB - operations fail - FileNotFoundError: If the APK file doesn't exist - CloudMobileServiceUninitializedError: If cloud service is - unavailable - """ - await self._install_apk_internal(apk_path) - - async def _install_apk_internal(self, apk_path: str | Path) -> None: - if isinstance(apk_path, str): - apk_path = Path(apk_path) - - if not apk_path.exists(): - raise FileNotFoundError(f"APK file not found: {apk_path}") - - if not self._initialized: - raise AgentNotInitializedError() - - device_id = self._device_context.device_id - logger.info(f"Installing APK on Android device '{device_id}'") - if not self._adb_client: - raise AgentError("ADB client not initialized") - - device = self._adb_client.device(serial=device_id) - await asyncio.to_thread(device.install, apk_path) - logger.info(f"APK installed successfully on Android device '{device_id}'") - - async def install_app(self, app_path: str | Path) -> str | None: - """Install an app on the connected device. - - For Android: Installs an APK file using ADB. - - Args: - app_path: Path to the app to install (Android APK file). - - Returns: - None. - - Raises: - AgentNotInitializedError: If the agent is not initialized - AgentError: If installation fails or platform is unsupported - FileNotFoundError: If the app file/folder doesn't exist - """ - return await self._install_app_internal(app_path) - - async def _install_app_internal(self, app_path: str | Path) -> str | None: - if isinstance(app_path, str): - app_path = Path(app_path) - - if not app_path.exists(): - raise FileNotFoundError(f"App not found: {app_path}") - - if not self._initialized: - raise AgentNotInitializedError() - - await self._install_apk_internal(app_path) - return None - - def new_task(self, goal: str): - """Create a new task request builder. - - Args: - goal: Natural language description of what to accomplish - - Returns: - TaskRequestBuilder that can be configured with: - - .with_output_format() for structured output - - .with_output_description() for output description - - .with_locked_app_package() to restrict execution to a specific app - - .using_profile() to specify an LLM profile - - .with_max_steps() to set maximum execution steps - - .with_trace_recording() to enable trace recording - - .with_name() to set a custom task name - """ - return TaskRequestBuilder[None].from_common( - goal=goal, - common=self._config.task_request_defaults, - ) - - @overload - async def run_task( - self, - *, - goal: str, - output: type[TOutput], - profile: str | AgentProfile | None = None, - name: str | None = None, - locked_app_package: str | None = None, - app_path: str | Path | None = None, - ) -> TOutput | None: ... - - @overload - async def run_task( - self, - *, - goal: str, - output: str, - profile: str | AgentProfile | None = None, - name: str | None = None, - locked_app_package: str | None = None, - app_path: str | Path | None = None, - ) -> str | dict | None: ... - - @overload - async def run_task( - self, - *, - goal: str, - output=None, - profile: str | AgentProfile | None = None, - name: str | None = None, - locked_app_package: str | None = None, - app_path: str | Path | None = None, - ) -> str | None: ... - - @overload - async def run_task( - self, - *, - request: TaskRequest[None], - locked_app_package: str | None = None, - app_path: str | Path | None = None, - ) -> str | dict | None: ... - - @overload - async def run_task( - self, - *, - request: TaskRequest[TOutput], - locked_app_package: str | None = None, - app_path: str | Path | None = None, - ) -> TOutput | None: ... - - async def run_task( - self, - *, - goal: str | None = None, - output: type[TOutput] | str | None = None, - profile: str | AgentProfile | None = None, - locked_app_package: str | None = None, - name: str | None = None, - app_path: str | Path | None = None, - request: TaskRequest[TOutput] | None = None, - ) -> str | dict | TOutput | None: - - # Normal local execution path - if request is not None: - if locked_app_package is not None: - if request.locked_app_package: - logger.warning( - "Locked app package specified both in the request and" - " as a parameter. Using the parameter value." - ) - request.locked_app_package = locked_app_package - # Handle app_path parameter override - if app_path is not None: - if request.app_path: - logger.warning( - "App path specified both in the request and as a" - " parameter. Using the parameter value." - ) - request.app_path = Path(app_path) if isinstance(app_path, str) else app_path - return await self._run_task(request=request) - if goal is None: - raise AgentTaskRequestError("Goal is required") - task_request = self.new_task(goal=goal) - if output is not None: - if isinstance(output, str): - task_request.with_output_description(description=output) - elif output is not NoneType: - task_request.with_output_format(output_format=output) - if profile is not None: - task_request.using_profile(profile=profile) - if name is not None: - task_request.with_name(name=name) - if locked_app_package is not None: - task_request.with_locked_app_package(package_name=locked_app_package) - if app_path is not None: - task_request.with_app_path(app_path=app_path) - return await self._run_task(task_request.build()) - - async def _run_task( - self, - request: TaskRequest[TOutput], - ) -> str | dict | TOutput | None: - if not self._initialized: - raise AgentNotInitializedError() - - if request.profile: - agent_profile = self._config.agent_profiles.get(request.profile) - if agent_profile is None: - if request.profile.lower() in ( - "flash", - "pro", - "ultra", - "default", - ): - agent_profile = self._config.default_profile - else: - raise AgentProfileNotFoundError(request.profile) - else: - agent_profile = self._config.default_profile - - logger.info(str(agent_profile)) - - on_status_changed = None - task_id = str( - self._session_id - or os.getenv("ARTEMIS_SESSION_ID") - or os.getenv("ARTEMIS_CLOUD_SESSION_ID") - or uuid.uuid4() - ) - - task = Task( - id=task_id, - device=self._device_context, - status="pending", - request=request, - created_at=datetime.now(), - on_status_changed=on_status_changed, - ) - self._tasks.append(task) - task_name = task.get_name() - - context = ArtemisContext( - device=self._device_context, - adb_client=self._adb_client, - ui_adb_client=self._ui_adb_client, - llm_config=agent_profile.llm_config, - agent_config=self._config, - ) - - output_config = None - if request.output_description or request.output_format: - output_config = OutputConfig( - output_description=request.output_description, - structured_output=request.output_format, # type: ignore - ) - logger.info(str(output_config)) - - logger.info(f"[{task_name}] Starting graph with goal: `{request.goal}`") - state = self._get_graph_state(task=task) - graph_input = state.model_dump() - datetime.now(UTC) - - async def _execute_task_logic(): - last_state: State | None = None - last_state_snapshot: dict | None = None - output = None - effective_mode = getattr(self._config, "concurrency_mode", "per_device") - effective_max = getattr(self._config, "max_concurrency", None) - sess_id = ( - self._session_id - or os.getenv("ARTEMIS_SESSION_ID") - or os.getenv("ARTEMIS_CLOUD_SESSION_ID") - or getattr(task, "id", None) - or getattr(getattr(task, "request", None), "task_name", None) - ) - active_owner = DeviceExecutionLock.get_active_owner(self._device_context.device_id) - already_held = ( - active_owner is not None - and active_owner.pid == os.getpid() - and ( - (sess_id and str(active_owner.session_id) == str(sess_id)) - or active_owner.token == self._device_context.device_id - ) - ) - device_lock = ( - None - if already_held - else DeviceExecutionLock( - self._device_context.device_id, - description=f"{request.goal[:120]}", - concurrency_mode=effective_mode, - max_concurrency=effective_max, - session_id=str(sess_id) if sess_id else None, - ingress=os.getenv("ARTEMIS_TASK_INGRESS") or "agent", - ) - ) - try: - if device_lock is not None and os.environ.get("ARTEMIS_CLOUD_MODE") != "1": - queue_cancel_event = threading.Event() - acquire_task = asyncio.create_task( - asyncio.to_thread( - device_lock.acquire, - cancel_event=queue_cancel_event, - ) - ) - try: - await asyncio.shield(acquire_task) - except asyncio.CancelledError: - queue_cancel_event.set() - try: - await asyncio.shield(acquire_task) - except Exception as exc: # pylint: disable=broad-exception-caught - # Draining the lock acquisition after cancellation is best effort. - logger.debug( - f"Device lock acquisition drain after cancel failed: {exc}", - exc_info=True, - ) - raise - # All device mutation and UI initialization happens only after - # this task reaches the head of the one global FIFO queue. - # Session creation must be inside the same boundary as well: - # DataEngine uses a shared database, so a queued task must not - # publish a new active session while the current task is still - # finishing. - self._prepare_tracing(task=task, context=context) - self._prepare_output_files(task=task) - if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": - if self._ui_adb_client is not None: - await self._connect_screen_client(context, str(sess_id)) - await self._ensure_device_unlocked() - publish_startup_progress( - "environment", "Preparing the device environment", session_id=str(sess_id) - ) - await self._prepare_app_installation(task=task) - await self._prepare_device_environment(context=context) - await self._prepare_app_lock(task=task, context=context) - async with context: - recording_started = False - if self._config.video_recording_tools_enabled: - try: - controller = get_controller(context) - - output_dir = None - if context.execution_setup and context.execution_setup.traces_path: - output_dir = ( - self._tmp_traces_dir / context.execution_setup.trace_name - ).resolve() - - logger.info(f"[{task_name}] Starting automated screen recording...") - start_res = await controller.start_video_recording( - output_dir=output_dir - ) - if start_res and start_res.success: - recording_started = True - else: - logger.warning( - f"[{task_name}] Failed to start screen" - f" recording: {start_res.message if start_res else 'unknown'}" - ) - except Exception as e: - logger.error(f"[{task_name}] Failed to start screen recording: {e}") - - publish_startup_progress( - "environment_ready", "Device environment is ready", session_id=str(sess_id) - ) - try: - if request.profile and request.profile.lower() == "flash": - logger.info(f"[{task_name}] Invoking FlashRunner reactive loop...") - await task.set_status( - status="running", - message="Invoking FlashRunner...", - ) - - # An explicit max_steps caps the reactive loop; the - # default leaves the cap to agent.flash.max_turns - # (0 = unlimited). - max_turns = ( - request.max_steps if request.max_steps != RECURSION_LIMIT else None - ) - runner = FlashRunner(context, goal=request.goal, max_turns=max_turns) - flash_result = await runner.run(state) - - output = flash_result - last_state_snapshot = state.model_dump() - - status = flash_result.get("status") - if status == "completed": - logger.info(f"✅ Automation '{task_name}' is success ✅") - await task.finalize(content=output, state=last_state_snapshot) - if context.data_engine: - context.data_engine.end_session("completed") - else: - err = ( - f"[{task_name}] FlashRunner failed:" - f" {flash_result.get('explanation')}" - ) - logger.warning(err) - await task.finalize( - content=output, - state=last_state_snapshot, - error=err, - ) - if context.data_engine: - context.data_engine.end_session("failed") - return output - else: - logger.info(f"[{task_name}] Invoking graph with input: {graph_input}") - await task.set_status(status="running", message="Invoking graph...") - async for chunk in (await get_graph(context)).astream( - input=graph_input, - config={ - "recursion_limit": task.request.max_steps, - "callbacks": ( - ( - list(self._config.graph_config_callbacks) - if self._config.graph_config_callbacks is not None - else [] - ) - + [DataEngineCallbackHandler(context)] - if context.data_engine - else self._config.graph_config_callbacks - ), - }, - stream_mode=[ - "messages", - "custom", - "updates", - "values", - ], - ): - stream_mode, payload = chunk - if stream_mode == "values": - last_state_snapshot = payload # type: ignore - last_state = State(**last_state_snapshot) # type: ignore - - if not last_state: - err = f"[{task_name}] No result received from graph" - logger.warning(err) - await task.finalize( - content=output, - state=last_state_snapshot, - error=err, - ) - if context.data_engine: - context.data_engine.end_session("failed") - return None - - output = await self._extract_output( - task_name=task_name, - ctx=context, - request=request, - output_config=output_config, - state=last_state, - ) - # Run outcome (goal axis x test axis) drives the - # wrap-up instead of an unconditional success path. - run_outcome = getattr(last_state, "run_outcome", None) - context.run_outcome = run_outcome - output = attach_test_summary(output, run_outcome) - task_status_axis = ( - run_outcome.get("task_status") if run_outcome else None - ) - tests_failed = ( - int((run_outcome.get("tests") or {}).get("failed", 0)) - if run_outcome - else 0 - ) - if task_status_axis == "blocked": - err = ( - f"[{task_name}] Task ended blocked: verify" - " criteria unmet after exhausting the final" - " check budget." - ) - logger.warning(err) - await task.finalize( - content=output, - state=last_state_snapshot, - error=err, - ) - if context.data_engine: - context.data_engine.end_session("failed") - return output - - if tests_failed > 0: - logger.warning( - f"[{task_name}] Task completed, but" - f" {tests_failed} assertion(s) failed —" - " see the test summary." - ) - else: - logger.info(f"✅ Automation '{task_name}' is success ✅") - await task.finalize(content=output, state=last_state_snapshot) - if context.data_engine: - context.data_engine.end_session("completed") - - return output - finally: - if recording_started: - - async def _safe_stop_recording(): - try: - logger.info( - f"[{task_name}] Stopping automated screen recording..." - ) - controller = get_controller(context) - return await controller.stop_video_recording() - except Exception as e: - logger.error( - f"[{task_name}] Error stopping screen recording: {e}" - ) - return None - - stop_task = asyncio.create_task(_safe_stop_recording()) - try: - stop_result = await asyncio.shield(stop_task) - except (asyncio.CancelledError, BaseException): - try: - stop_result = await asyncio.wait_for(stop_task, timeout=15.0) - except Exception: - stop_result = None - - if stop_result and stop_result.success and stop_result.video_path: - logger.info( - f"[{task_name}] Screen recording saved" - f" to: {stop_result.video_path}" - ) - elif stop_result: - logger.warning( - f"[{task_name}] Failed to save screen" - f" recording: {stop_result.message}" - ) - except asyncio.CancelledError: - err = f"[{task_name}] Task cancelled" - logger.warning(err) - await task.finalize( - content=output, - state=last_state_snapshot, - error=err, - cancelled=True, - ) - if context.data_engine: - context.data_engine.end_session("cancelled") - - raise - except Exception as e: - err = f"[{task_name}] Error running automation: {e}" - logger.error(err) - await task.finalize( - content=output, - state=last_state_snapshot, - error=err, - ) - if context.data_engine: - context.data_engine.end_session("failed") - - raise - finally: - try: - # Background ADB processes (logcat, screenrecord, ...) started - # by the Operator must not outlive the automation task. - from artemis.tools.command_tool import shutdown_adb_background_tasks - - await asyncio.wait_for(shutdown_adb_background_tasks(context), timeout=15.0) - except Exception as e: - logger.warning(f"[{task_name}] Failed to stop background ADB tasks: {e}") - try: - await self._finalize_tracing_safely(task=task, context=context) - finally: - if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": - try: - if self._ui_adb_client is not None: - await asyncio.to_thread(self._ui_adb_client.disconnect) - finally: - if device_lock is not None: - await asyncio.to_thread(device_lock.release) - - async with self._task_lock: - if self._current_task and not self._current_task.done(): - logger.warning( - "Another automation task is already running. " - "Stopping it before starting the new one." - ) - self.stop_current_task() - try: - await self._current_task - except asyncio.CancelledError: - pass - - cancel_watcher: asyncio.Task | None = None - try: - self._current_task = asyncio.create_task(_execute_task_logic()) - cancel_watcher = asyncio.create_task(self._watch_external_cancel(task_name)) - return await self._current_task - finally: - self._current_task = None - if cancel_watcher is not None and not cancel_watcher.done(): - cancel_watcher.cancel() - try: - await cancel_watcher - except asyncio.CancelledError: - pass - async def _watch_external_cancel(self, task_name: str) -> None: """Cancel the running task when another process drops a cancel marker. @@ -941,44 +199,6 @@ def _on_cancel() -> None: except Exception as exc: logger.debug(f"[{task_name}] Cancel watcher stopped: {exc}") - def stop_current_task(self): - """Requests cancellation of the currently running automation task.""" - if self._current_task and not self._current_task.done(): - logger.info("Requesting to stop the current automation task...") - was_cancelled = self._current_task.cancel() - if was_cancelled: - logger.success("Cancellation request for the current task was sent.") - else: - logger.warning( - "Could not send cancellation request for the current task " - "(it may already be completing)." - ) - else: - logger.info("No active automation task to stop.") - - async def get_screenshot(self) -> Image.Image: - """Capture a screenshot from the mobile device. - - Returns: - Screenshot as PIL Image - - Raises: - AgentNotInitializedError: If the agent is not initialized - Exception: If screenshot capture fails - """ - if not self._initialized: - raise AgentNotInitializedError() - - # Use ADB to capture screenshot - logger.info("Capturing screenshot from local Android device") - if not self._adb_client: - raise Exception("ADB client not initialized") - - device = self._adb_client.device(serial=self._device_context.device_id) - screenshot = await asyncio.to_thread(device.screenshot) - logger.info("Screenshot captured from local Android device") - return screenshot - async def clean(self, force: bool = False): if not self._initialized and not force: return @@ -1012,22 +232,6 @@ async def _ensure_device_unlocked(self) -> None: "running Artemis; automation will not guess a PIN, password, or pattern." ) - async def _prepare_app_installation(self, task: Task) -> str | None: - """Install app if app_path is specified in the task request. - - Returns: - None. - """ - if not task.request.app_path: - return None - - task_name = task.get_name() - logger.info(f"[{task_name}] Installing app from: {task.request.app_path}") - - await self.install_app(task.request.app_path) - - return None - async def _prepare_device_environment(self, context: ArtemisContext): """Prepare device environment flags (like forcing Web Accessibility) before the task runs.""" if not self._config.force_web_accessibility: @@ -1076,38 +280,9 @@ async def _prepare_device_environment(self, context: ArtemisContext): f" root/userdebug device): {e}" ) - async def _prepare_app_lock(self, task: Task, context: ArtemisContext): - """Prepare app lock by launching the locked app if specified.""" - if not task.request.locked_app_package: - return - - task_name = task.get_name() - logger.info(f"[{task_name}] Preparing app lock for: {task.request.locked_app_package}") - - app_lock_status = await _handle_initial_app_launch( - ctx=context, locked_app_package=task.request.locked_app_package - ) - - if context.execution_setup is None: - context.execution_setup = ExecutionSetup(app_lock_status=app_lock_status) - else: - context.execution_setup.app_lock_status = app_lock_status - - if app_lock_status.locked_app_initial_launch_success is False: - error = app_lock_status.locked_app_initial_launch_error - logger.warning(f"[{task_name}] Failed to launch locked app: {error}") - def _prepare_tracing(self, task: Task, context: ArtemisContext): """Prepare tracing and data engine setup.""" - task_name = task.get_name() - temp_trace_path = Path(self._tmp_traces_dir / task_name).resolve() - temp_trace_path.mkdir(parents=True, exist_ok=True) - - if task.request.record_trace: - traces_output_path = Path(task.request.trace_path).resolve() - logger.info(f"[{task_name}] 📂 Traces output path: {traces_output_path}") - logger.info(f"[{task_name}] 📄📂 Traces temp path: {temp_trace_path}") - traces_output_path.mkdir(parents=True, exist_ok=True) + task_name = self._prepare_trace_paths(task) context.execution_setup = ExecutionSetup( traces_path=self._tmp_traces_dir, @@ -1175,118 +350,9 @@ async def _finalize_tracing_safely(self, task: Task, context: ArtemisContext): exc_info=True, ) - async def _finalize_tracing(self, task: Task, context: ArtemisContext): - if context.data_engine: - session_status = "completed" - if task.status == "failed": - session_status = "failed" - elif task.status == "cancelled": - session_status = "cancelled" - context.data_engine.end_session(status=session_status) - - exec_setup_ctx = context.execution_setup - if not exec_setup_ctx: - return - - if exec_setup_ctx.traces_path is None or exec_setup_ctx.trace_name is None: - return - - task_name = task.get_name() - temp_trace_path = (self._tmp_traces_dir / exec_setup_ctx.trace_name).resolve() - - if not task.request.record_trace: - try: - if temp_trace_path.exists(): - shutil.rmtree(temp_trace_path) - logger.info(f"[{task_name}] Cleaned up temporary trace folder.") - except Exception as e: - logger.error(f"[{task_name}] Failed to clean up temp trace folder: {e}") - return - - status = resolve_trace_suffix(task.status, getattr(context, "run_outcome", None)) - ts = task.created_at.strftime("%Y-%m-%dT%H-%M-%S") - new_name = f"{exec_setup_ctx.trace_name}{status}_{ts}" - - traces_output_path = Path(task.request.trace_path).resolve() - - logger.info(f"[{task_name}] Compiling trace FROM FOLDER: " + str(temp_trace_path)) - create_gif_from_trace_folder(temp_trace_path) - create_steps_json_from_trace_folder(temp_trace_path) - - logger.info(f"[{task_name}] Video created, removing dust...") - remove_images_from_trace_folder(temp_trace_path) - remove_steps_json_from_trace_folder(temp_trace_path) - logger.info(f"[{task_name}] 📽️ Trace compiled, moving to output path 📽️") - - output_folder_path = temp_trace_path.rename(traces_output_path / new_name).resolve() - logger.info(f"[{task_name}] 📂✅ Traces located in: {output_folder_path}") - - new_video_path = output_folder_path / "recording.mp4" - if new_video_path.exists() and context.data_engine: - context.data_engine.update_video_path(new_video_path) - - def _prepare_output_files(self, task: Task): - if task.request.llm_output_path: - _validate_and_prepare_file(file_path=task.request.llm_output_path) - - async def _extract_output( - self, - task_name: str, - ctx: ArtemisContext, - request: TaskRequest[TOutput], - output_config: OutputConfig | None, - state: State, - ) -> str | dict | TOutput | None: - exec_setup = getattr(ctx, "execution_setup", None) - outputter_cfg = getattr(exec_setup, "outputter", None) or ( - getattr(ctx.agent_config, "outputter", None) if ctx and ctx.agent_config else None - ) - outputter_enabled = getattr(outputter_cfg, "enabled", True) if outputter_cfg else True - force_synthesis = ( - getattr(outputter_cfg, "force_synthesis", False) if outputter_cfg else False - ) - if exec_setup and getattr(exec_setup, "disable_outputter", False): - outputter_enabled = False - - should_run = outputter_enabled and ( - ( - output_config - and output_config.needs_structured_format(default_enabled=outputter_enabled) - ) - or force_synthesis - or (output_config and output_config.enable_outputter is True) - ) - if output_config and output_config.enable_outputter is False: - should_run = False - - if should_run: - logger.info(f"[{task_name}] Generating structured output via Outputter...") - try: - structured_output = await outputter( - ctx=ctx, - output_config=output_config or OutputConfig(), - graph_output=state, - ) - logger.info(f"[{task_name}] Structured output: {structured_output}") - record_events( - output_path=request.llm_output_path, - events=structured_output, - ) - if request.output_format is not None and request.output_format is not NoneType: - return request.output_format.model_validate(structured_output) - return structured_output - except Exception as e: - logger.error(f"[{task_name}] Failed to generate structured output: {e}") - return None - return None - def _get_graph_state(self, task: Task): return State.initial(task.request.goal) - # ------------------------------------------------------------------ # - # UI hierarchy backend: connect, and keep the user told which one serves - # ------------------------------------------------------------------ # - async def _connect_screen_client(self, context: ArtemisContext, session_id: str) -> None: """Connect the screen client with visible progress for slow first-time steps.""" client = self._ui_adb_client @@ -1417,118 +483,3 @@ def _announce_hierarchy_backend( ) except (OSError, ValueError, RuntimeError) as exc: logger.debug(f"Could not record hierarchy backend in status.json: {exc}") - - def _init_clients( - self, - device_id: str, - platform: DevicePlatform, - ): - self._adb_client = AdbClient( - host=self._config.servers.adb_host, - port=self._config.servers.adb_port, - ) - self._ui_adb_client = create_screen_client(device_id) - - async def _get_device_context( - self, - device_id: str, - platform: DevicePlatform, - ) -> DeviceContext: - - host_platform = system() - if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": - width, height = (1080, 2400) - if self._adb_client: - try: - width, height = self._adb_client.device(device_id).window_size() - logger.info( - f"Retrieved remote device screen dimensions dynamically: {width}x{height}" - ) - except Exception as e: - logger.debug( - f"Failed to query remote window size: {e}. Defaulting to 1080x2400." - ) - return DeviceContext( - host_platform="LINUX", - mobile_platform=platform, - device_id=device_id, - device_width=width, - device_height=height, - ) - - # Query dimensions without starting UIAutomator or acquiring an awake - # strategy. Those belong inside the global execution queue lease. - if self._adb_client: - try: - device_width, device_height = self._adb_client.device(device_id).window_size() - logger.info(f"Retrieved Android screen dimensions: {device_width}x{device_height}") - except Exception as e: - logger.warning(f"Failed to get Android window size: {e}, using defaults") - device_width, device_height = 1080, 2340 - else: - logger.warning("ADB client not available, using default dimensions") - device_width, device_height = 1080, 2340 - - from artemis.platform import platform as pal_platform - - return DeviceContext( - host_platform=pal_platform.os_type.name, - mobile_platform=platform, - device_id=device_id, - device_width=device_width, - device_height=device_height, - ) - - -def _validate_and_prepare_file(file_path: Path): - path_obj = Path(file_path) - if path_obj.exists() and path_obj.is_dir(): - raise AgentTaskRequestError(f"Error: Path '{file_path}' is a directory, not a file.") - try: - path_obj.parent.mkdir(parents=True, exist_ok=True) - path_obj.touch(exist_ok=True) - except OSError as e: - raise AgentTaskRequestError(f"Error creating file '{file_path}': {e}") - - -def attach_test_summary(output, run_outcome: dict | None): - """Surfaces the machine-readable test summary in the task's return value. - - Only applies when the run actually had check items (the summary carries at - least one counted item) — otherwise the historical return shape is kept - untouched. String outputs are wrapped into a dict so that callers such as - ``mobile_run_task`` receive the summary without parsing report prose. - """ - if not run_outcome: - return output - tests = run_outcome.get("tests") or {} - total = ( - int(tests.get("passed", 0)) - + int(tests.get("failed", 0)) - + int(tests.get("inconclusive", 0)) - + int(tests.get("unchecked", 0)) - ) - if total <= 0: - return output - summary = {"task_status": run_outcome.get("task_status"), **tests} - if isinstance(output, dict): - merged = dict(output) - merged.setdefault("test_summary", summary) - return merged - if output is None: - return {"test_summary": summary} - if isinstance(output, str): - return {"result": output, "test_summary": summary} - # Typed/structured outputs keep their shape; the summary stays available in - # run_outcome.json and the session state. - return output - - -def resolve_trace_suffix(task_status: str, run_outcome: dict | None) -> str: - """Trace naming: assertion failures must be distinguishable from _PASS.""" - if task_status != "completed": - return "_FAIL" - tests = (run_outcome or {}).get("tests") or {} - if int(tests.get("failed", 0)) > 0: - return "_TESTFAIL" - return "_PASS" diff --git a/artemis/sdk/builders/__init__.py b/artemis/sdk/builders/__init__.py index dbdba77e7..ade62e36d 100644 --- a/artemis/sdk/builders/__init__.py +++ b/artemis/sdk/builders/__init__.py @@ -15,8 +15,8 @@ """Builder classes for configuring ARTEMIS components.""" from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder -from artemis.sdk.builders.index import Builders -from artemis.sdk.builders.task_request_builder import ( +from third_party.mobile_use.sdk.builders.index import Builders +from third_party.mobile_use.sdk.builders.task_request_builder import ( TaskRequestBuilder, TaskRequestCommonBuilder, ) diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index 4d4b87003..16893864d 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -11,68 +11,35 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """Builder for AgentConfig objects using a fluent interface.""" -import copy import os -from typing import Any +from typing import Any, cast -from langchain_core.callbacks.base import Callbacks from artemis.config import ( ExplorerVersion, checker_overrides_for_level, - get_default_llm_config, load_agent_config, settings, ) from artemis.context import DevicePlatform -from artemis.sdk.constants import DEFAULT_PROFILE_NAME -from artemis.sdk.types.agent import ( - AgentConfig, - AgentProfile, - ServerConfig, -) -from artemis.sdk.types.task import TaskRequestCommon +from artemis.sdk.types.agent import AgentConfig, ServerConfig from artemis.utils.video import detect_video_tools_enabled +from third_party.mobile_use.sdk.builders.agent_config_builder import AgentConfigBuilderBase -class AgentConfigBuilder: - """Builder class providing a fluent interface for creating AgentConfig objects. - - This builder allows for step-by-step construction of an AgentConfig with - clear methods that make the configuration process intuitive and type-safe. +class AgentConfigBuilder(AgentConfigBuilderBase): + """Artemis AgentConfig builder; see :class:`AgentConfigBuilderBase` for usage.""" - Examples: - >>> builder = AgentConfigBuilder() - >>> config = (builder - ... .add_profile(AgentProfile(name="HighReasoning", - llm_config=LLMConfig(...))) - ... .add_profile(AgentProfile(name="LowReasoning", - llm_config=LLMConfig(...))) - ... .for_device(DevicePlatform.ANDROID, "device123") - ... .with_default_task_config(TaskRequestCommon(max_steps=30)) - ... .with_default_profile("HighReasoning") - ... .build() - ... ) - """ + config_class = AgentConfig def __init__(self): - """Initialize an empty AgentConfigBuilder.""" - self._agent_profiles: dict[str, AgentProfile] = {} - self._task_request_defaults: TaskRequestCommon | None = None - self._default_profile: str | AgentProfile | None = None - self._device_id: str | None = None - self._device_platform: DevicePlatform | None = None - self._servers: ServerConfig = get_default_servers() - self._graph_config_callbacks: Callbacks = None + """Initialize an empty AgentConfigBuilder with Artemis defaults.""" + super().__init__(servers=get_default_servers()) self._video_recording_tools_enabled: bool = detect_video_tools_enabled() self._force_web_accessibility: bool = False self._disable_checker: bool = False - self._cloud_mobile_id_or_ref: str | None = None self._concurrency_mode: str = "per_device" self._max_concurrency: int | None = None @@ -105,70 +72,6 @@ def __init__(self): self._flash = agent_cfg.flash self._pro = agent_cfg.pro - def add_profile(self, profile: AgentProfile, validate: bool = True) -> "AgentConfigBuilder": - """Add an agent profile to the ARTEMIS agent. - - Args: - profile: The agent profile to add - """ - self._agent_profiles[profile.name] = profile - if validate: - profile.llm_config.validate_providers() - return self - - def add_profiles( - self, - profiles: list[AgentProfile], - validate: bool = True, - ) -> "AgentConfigBuilder": - """Add multiple agent profiles to the ARTEMIS agent. - - Args: - profiles: List of agent profiles to add - """ - for profile in profiles: - self.add_profile(profile=profile, validate=validate) - return self - - def with_default_profile(self, profile: str | AgentProfile) -> "AgentConfigBuilder": - """Set the default agent profile used for tasks. - - Args: - profile: The name or instance of the default agent profile - """ - self._default_profile = profile - return self - - def for_device( - self, - platform_or_device_id: DevicePlatform | str, - device_id: str | None = None, - ) -> "AgentConfigBuilder": - """Configure the ARTEMIS agent for a specific device. - - Supports both: - builder.for_device(DevicePlatform.ANDROID, "emulator-5554") - and: - builder.for_device("emulator-5554") (defaults to DevicePlatform.ANDROID) - - Args: - platform_or_device_id: DevicePlatform or unique identifier for the device - device_id: The unique identifier for the device (if platform was passed first) - """ - if self._cloud_mobile_id_or_ref is not None: - raise ValueError( - "Device ID cannot be set when a cloud mobile is already" - " configured.\n> for_device() and for_cloud_mobile() are" - " mutually exclusive" - ) - if isinstance(platform_or_device_id, DevicePlatform): - self._device_platform = platform_or_device_id - self._device_id = device_id - else: - self._device_platform = DevicePlatform.ANDROID - self._device_id = str(platform_or_device_id) - return self - def for_device_serial(self, device_serial: str) -> "AgentConfigBuilder": """Target a specific Android device by its ADB serial number.""" return self.for_device(DevicePlatform.ANDROID, device_serial) @@ -183,45 +86,6 @@ def with_max_concurrency(self, max_concurrency: int) -> "AgentConfigBuilder": self._max_concurrency = max_concurrency return self - def with_default_task_config(self, config: TaskRequestCommon) -> "AgentConfigBuilder": - """Set the default task configuration. - - Args: - config: The task configuration to use as default - """ - self._task_request_defaults = copy.deepcopy(config) - return self - - def with_adb_server(self, host: str, port: int | None = None) -> "AgentConfigBuilder": - """Set the ADB server host and port. - - Args: - host: The ADB server host - port: The ADB server port - """ - self._servers.adb_host = host - if port is not None: - self._servers.adb_port = port - return self - - def with_servers(self, servers: ServerConfig) -> "AgentConfigBuilder": - """Set the server settings. - - Args: - servers: The server settings to use - """ - self._servers = copy.deepcopy(servers) - return self - - def with_graph_config_callbacks(self, callbacks: Callbacks) -> "AgentConfigBuilder": - """Set the graph config callbacks. - - Args: - callbacks: The graph config callbacks to use - """ - self._graph_config_callbacks = callbacks - return self - def with_video_recording_tools(self, enabled: bool = True) -> "AgentConfigBuilder": """Enable or disable video recording tools. @@ -554,94 +418,46 @@ def with_denylisted_tools(self, tools: dict[str, list[str]]) -> "AgentConfigBuil self._denylisted_tools = tools return self - def build(self, validate_profiles: bool = True) -> AgentConfig: - """Build the ARTEMIS AgentConfig object. - - Args: - default_profile: Name of the default agent profile to use - - Returns: - A configured AgentConfig object - - Raises: - ValueError: If default_profile is specified but not found in - configured profiles - """ - nb_profiles = len(self._agent_profiles) - - if isinstance(self._default_profile, str): - profile_name = self._default_profile - default_profile = self._agent_profiles.get(profile_name, None) - if default_profile is None: - raise ValueError(f"Profile '{profile_name}' not found in configured agents") - elif isinstance(self._default_profile, AgentProfile): - default_profile = self._default_profile - if default_profile.name not in self._agent_profiles: - self.add_profile(default_profile, validate=validate_profiles) - elif nb_profiles <= 0: - llm_config = get_default_llm_config() - default_profile = AgentProfile( - name=DEFAULT_PROFILE_NAME, - llm_config=llm_config, - ) - self.add_profile(default_profile, validate=validate_profiles) - elif nb_profiles == 1: - # Select the only one available - default_profile = next(iter(self._agent_profiles.values())) - else: - available_profiles = ", ".join(self._agent_profiles.keys()) - raise ValueError( - f"You must call with_default_profile() to select one among: {available_profiles}" - ) - - device_id = ( - self._device_id - or os.environ.get("ARTEMIS_DEVICE_ID") - or os.environ.get("ADB_DEVICE_SERIAL") - ) - - return AgentConfig( - agent_profiles=self._agent_profiles, - task_request_defaults=self._task_request_defaults or TaskRequestCommon(), - default_profile=default_profile, - device_id=device_id, - device_platform=self._device_platform, - servers=self._servers, - graph_config_callbacks=self._graph_config_callbacks, - video_recording_tools_enabled=self._video_recording_tools_enabled, - force_web_accessibility=self._force_web_accessibility, - disable_checker=self._disable_checker, - disable_midway_checks=self._disable_midway_checks, - disable_final_check=self._disable_final_check, - checker_max_iterations=self._checker_max_iterations, - final_check_max_attempts=self._final_check_max_attempts, - checkpoint_max_repairs=self._checkpoint_max_repairs, - max_concurrent_checkpoints=self._max_concurrent_checkpoints, - checkpoint_timeout=self._checkpoint_timeout, - settlement_timeout=self._settlement_timeout, - assert_failure_policy=self._assert_failure_policy, - disable_device_probes=self._disable_device_probes, - disable_planner_validation=self._disable_planner_validation, - enable_committee=self._enable_committee, - committee_debate_rounds=self._committee_debate_rounds, - disable_outputter=self._disable_outputter, - outputter=self._outputter, - flash=self._flash, - pro=self._pro, - explorer=self._explorer, - explorer_versions=self._explorer_versions, - denylisted_tools=self._denylisted_tools, - enable_video_ledger=self._enable_video_ledger, - video_analyzer=self._video_analyzer.model_copy( + def _extra_config_fields(self) -> dict[str, Any]: + return { + "device_id": ( + self._device_id + or os.environ.get("ARTEMIS_DEVICE_ID") + or os.environ.get("ADB_DEVICE_SERIAL") + ), + "video_recording_tools_enabled": self._video_recording_tools_enabled, + "force_web_accessibility": self._force_web_accessibility, + "disable_checker": self._disable_checker, + "disable_midway_checks": self._disable_midway_checks, + "disable_final_check": self._disable_final_check, + "checker_max_iterations": self._checker_max_iterations, + "final_check_max_attempts": self._final_check_max_attempts, + "checkpoint_max_repairs": self._checkpoint_max_repairs, + "max_concurrent_checkpoints": self._max_concurrent_checkpoints, + "checkpoint_timeout": self._checkpoint_timeout, + "settlement_timeout": self._settlement_timeout, + "assert_failure_policy": self._assert_failure_policy, + "disable_device_probes": self._disable_device_probes, + "disable_planner_validation": self._disable_planner_validation, + "enable_committee": self._enable_committee, + "committee_debate_rounds": self._committee_debate_rounds, + "disable_outputter": self._disable_outputter, + "outputter": self._outputter, + "flash": self._flash, + "pro": self._pro, + "explorer": self._explorer, + "explorer_versions": self._explorer_versions, + "denylisted_tools": self._denylisted_tools, + "enable_video_ledger": self._enable_video_ledger, + "video_analyzer": self._video_analyzer.model_copy( update={"enable_ledger": self._enable_video_ledger} ), - concurrency_mode=self._concurrency_mode, - max_concurrency=self._max_concurrency, - ) + "concurrency_mode": self._concurrency_mode, + "max_concurrency": self._max_concurrency, + } - -def get_default_agent_config(): - return AgentConfigBuilder().build() + def build(self, validate_profiles: bool = True) -> AgentConfig: + return cast(AgentConfig, super().build(validate_profiles=validate_profiles)) def get_default_servers(): diff --git a/artemis/sdk/examples/app_lock_messaging.py b/artemis/sdk/examples/app_lock_messaging.py deleted file mode 100644 index e598aeb4c..000000000 --- a/artemis/sdk/examples/app_lock_messaging.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. - -import asyncio - -from artemis.sdk import Agent -from pydantic import BaseModel, Field - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class MessageResult(BaseModel): - """Structured result from messaging task.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - messages_sent: int = Field(..., description="Number of messages successfully sent") - contacts: list[str] = Field(..., description="List of contacts messaged") - success: bool = Field(..., description="Whether all messages were sent successfully") - - -async def main() -> None: - # Create agent with default configuration - agent = Agent() - - try: - await agent.init() - - # Use app lock to keep execution in WhatsApp - # This ensures the agent stays in the app and relaunches if needed - task = ( - agent.new_task("Send 'Happy New Year!' message to Alice, Bob, and Charlie on WhatsApp") - .with_name("send_new_year_messages") - .with_locked_app_package("com.whatsapp") # Lock to WhatsApp - .with_output_format(MessageResult) - .with_max_steps(600) # Messaging tasks may need more steps - .build() - ) - - print("Sending messages with app lock enabled...") - print("The agent will stay in WhatsApp and relaunch if needed.\n") - - result = await agent.run_task(request=task) - - if result: - print("\n=== Messaging Complete ===") - print(f"Messages sent: {result.messages_sent}") - print(f"Contacts: {', '.join(result.contacts)}") - print(f"Success: {result.success}") - else: - print("Failed to send messages") - - except Exception as e: - print(f"Error: {e}") - finally: - await agent.clean() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/examples/simple_photo_organizer.py b/artemis/sdk/examples/simple_photo_organizer.py deleted file mode 100644 index 6650d6693..000000000 --- a/artemis/sdk/examples/simple_photo_organizer.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Simple Photo Organizer - Basic SDK Usage Example - -This example demonstrates a straightforward way to use the Artemis SDK -without builders or advanced configuration. It performs a real-world automation -task: -1. Opens the photo gallery -2. Finds photos from a specific date -3. Creates an album and moves those photos into it - -Run: -- python artemis/sdk/examples/simple_photo_organizer.py -""" - -import asyncio -from datetime import date, timedelta -from artemis.sdk import Agent -from pydantic import BaseModel, Field - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class PhotosResult(BaseModel): - """Structured result from photo search.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - found_photos: int = Field(..., description="Number of photos found") - date_range: str = Field(..., description="Date range of photos found") - album_created: bool = Field(..., description="Whether an album was created") - album_name: str = Field(..., description="Name of the created album") - photos_moved: int = Field(0, description="Number of photos moved to the album") - - -async def main() -> None: - # Create a simple agent with default configuration - agent = Agent() - - try: - # Initialize agent (finds a device, starts required servers) - await agent.init() - - # Calculate yesterday's date for the example - yesterday = date.today() - timedelta(days=1) - formatted_date = yesterday.strftime("%B %d") # e.g. "August 22" - - print(f"Looking for photos from {formatted_date}...") - - # First task: search for photos and organize them, with typed output - result = await agent.run_task( - goal=( - "Open the Photos/Gallery app. Find photos taken on" - f" {formatted_date}. Create a new album named '{formatted_date}" - " Memories' and move those photos into it. Count how many" - " photos were moved." - ), - output=PhotosResult, - name="organize_photos", - ) - - # Handle and display the result - if result: - print("\n=== Photo Organization Complete ===") - print(f"Found: {result.found_photos} photos from {result.date_range}") - - if result.album_created: - print(f"Created album: '{result.album_name}'") - print(f"Moved {result.photos_moved} photos to the album") - else: - print("No album was created") - else: - print("Failed to organize photos") - - except Exception as e: - print(f"Error: {e}") - finally: - # Always clean up resources - await agent.clean() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/examples/smart_notification_assistant.py b/artemis/sdk/examples/smart_notification_assistant.py deleted file mode 100644 index 297ea08a1..000000000 --- a/artemis/sdk/examples/smart_notification_assistant.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Smart Notification Assistant - Intermediate SDK Usage Example - -This example demonstrates more advanced SDK features including: -- TaskRequestBuilder pattern -- Multiple agent profiles for different reasoning tasks -- Tracing for debugging/visualization -- Structured output with Pydantic -- Exception handling - -It performs a practical automation task: -1. Checks notification panel for unread notifications -2. Categorizes them by priority/app -3. Performs actions based on notification content - -Run: -- python artemis/sdk/examples/smart_notification_assistant.py -""" - -import asyncio -from datetime import datetime - -try: - from enum import StrEnum -except ImportError: - from enum import Enum - - class StrEnum(str, Enum): - pass - - -from artemis.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback -from artemis.sdk import Agent -from artemis.sdk.builders import Builders -from artemis.sdk.types import AgentProfile -from artemis.sdk.types.exceptions import AgentError -from pydantic import BaseModel, Field - - -class NotificationPriority(StrEnum): - HIGH = "high" - MEDIUM = "medium" - LOW = "low" - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class Notification(BaseModel): - """Individual notification details.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - app_name: str = Field(..., description="Name of the app that sent the notification") - title: str = Field(..., description="Title/header of the notification") - message: str = Field(..., description="Message content of the notification") - priority: NotificationPriority = Field( - default=NotificationPriority.MEDIUM, - description="Priority level of notification", - ) - - -class NotificationSummary(BaseModel): - """Summary of all notifications.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - total_count: int = Field(..., description="Total number of notifications found") - high_priority_count: int = Field(0, description="Count of high priority notifications") - notifications: list[Notification] = Field( - default_factory=list, description="List of individual notifications" - ) - - -def get_agent() -> Agent: - # Create two specialized profiles: - # 1. An analyzer profile for detailed inspection tasks - analyzer_profile = AgentProfile( - name="analyzer", - llm_config=LLMConfig( - planner=LLMWithFallback( - provider="openrouter", - model="meta-llama/llama-4-scout", - fallback=LLM(provider="openrouter", model="meta-llama/llama-4-maverick"), - ), - utils=LLMConfigUtils( - outputter=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - hopper=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - ), - ), - # from_file="/tmp/analyzer.jsonc" # can be loaded from file - ) - - # 2. An action profile for handling easy & fast actions based on notifications - action_profile = AgentProfile( - name="note_taker", - llm_config=LLMConfig( - planner=LLMWithFallback( - provider="openai", - model="o3", - fallback=LLM(provider="openai", model="gpt-5"), - ), - utils=LLMConfigUtils( - outputter=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - hopper=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - ), - ), - ) - - # Configure default task settings with tracing - task_defaults = Builders.TaskDefaults.with_max_steps(200).build() - - # Configure the agent - config = ( - Builders.AgentConfig.add_profiles(profiles=[analyzer_profile, action_profile]) - .with_default_profile(profile=action_profile) - .with_default_task_config(config=task_defaults) - .build() - ) - return Agent(config=config) - - -async def main(): - # Set up traces directory with timestamp for uniqueness - timestamp = datetime.now().strftime("%Y%m%d_%H%M") - traces_dir = f"/tmp/notification_traces/{timestamp}" - agent = get_agent() - - try: - # Initialize agent (finds a device, starts required servers) - await agent.init() - - print("Checking for notifications...") - - # Task 1: Get and analyze notifications with analyzer profile - notification_task = ( - agent.new_task( - goal=( - "Open the notification panel (swipe down from top). Scroll" - " through the first 3 unread notifications. For each" - " notification, identify the app name, title, and content." - " Tag messages from messaging apps or email as high" - " priority." - ) - ) - .with_output_format(NotificationSummary) - .using_profile("analyzer") - .with_name("notification_scan") - .with_max_steps(400) - .with_trace_recording(enabled=True, path=traces_dir) - .build() - ) - - # Execute the task with proper exception handling - try: - notifications = await agent.run_task(request=notification_task) - - # Display the structured results - if notifications: - print("\n=== Notification Summary ===") - print(f"Total notifications: {notifications.total_count}") - print(f"High priority: {notifications.high_priority_count}") - - # Task 2: Create a note to store the notification summary - response = await agent.run_task( - goal=( - "Open my Notes app and create a new note summarizing" - f" the following information:\n{notifications}" - ), - name="email_action", - profile="note_taker", - ) - print(f"Action result: {response}") - - else: - print("Failed to retrieve notifications") - - except AgentError as e: - print(f"Agent error occurred: {e}") - except Exception as e: - print(f"Unexpected error: {type(e).__name__}: {e}") - raise - - finally: - # Clean up - await agent.clean() - print(f"\nTraces saved to: {traces_dir}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/examples/video_transcription_example.py b/artemis/sdk/examples/video_transcription_example.py deleted file mode 100644 index 5b1267374..000000000 --- a/artemis/sdk/examples/video_transcription_example.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Video Transcription Example - -This example demonstrates how to use the video recording tools to capture -and analyze video content from a mobile device screen. - -The agent can: -1. Start a screen recording -2. Perform actions while recording -3. Stop the recording and analyze its content using Gemini models - -Use case: Recording a video playing on the screen and transcribing its content. -""" - -import asyncio - -from artemis.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback -from artemis.sdk.agent import Agent -from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder -from artemis.sdk.types.agent import AgentConfig -from artemis.sdk.types.task import AgentProfile, TaskRequest - - -def get_video_capable_llm_config() -> LLMConfig: - """Returns an LLM config with video_analyzer configured. - - The video_analyzer must use a video-capable Gemini model: - - gemini-3.8-flash (recommended - fast and capable) - - gemini-3.7-flash - - gemini-3-pro-preview - - gemini-2.5-flash - - gemini-2.5-pro - - gemini-2.0-flash - """ - return LLMConfig( - planner=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - utils=LLMConfigUtils( - outputter=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - hopper=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - video_analyzer=LLMWithFallback( - provider="google", - model="gemini-3.8-flash", - fallback=LLM(provider="google", model="gemini-3.7-flash"), - ), - ), - ) - - -async def main(): - config: AgentConfig = ( - AgentConfigBuilder() - .add_profile( - AgentProfile( - name="VideoCapable", - llm_config=get_video_capable_llm_config(), - ) - ) - .with_video_recording_tools() - .build() - ) - - agent = Agent(config=config) - try: - await agent.init() - - result = await agent.run_task( - request=TaskRequest( - goal=""" - 1. Open YouTube app - 2. Search for "Python tutorial" - 3. Start recording the screen - 4. Play the first video - 5. Wait for the first 30 seconds of the video to play - 6. Stop recording and tell me what was said in the video - """, - profile="VideoCapable", - ) - ) - print(f"Task result: {result}") - finally: - await agent.clean() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/run_outcome.py b/artemis/sdk/run_outcome.py new file mode 100644 index 000000000..91ef78ee1 --- /dev/null +++ b/artemis/sdk/run_outcome.py @@ -0,0 +1,58 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run outcome helpers: test summary attachment and trace naming.""" + + +def attach_test_summary(output, run_outcome: dict | None): + """Surfaces the machine-readable test summary in the task's return value. + + Only applies when the run actually had check items (the summary carries at + least one counted item) — otherwise the historical return shape is kept + untouched. String outputs are wrapped into a dict so that callers such as + ``mobile_run_task`` receive the summary without parsing report prose. + """ + if not run_outcome: + return output + tests = run_outcome.get("tests") or {} + total = ( + int(tests.get("passed", 0)) + + int(tests.get("failed", 0)) + + int(tests.get("inconclusive", 0)) + + int(tests.get("unchecked", 0)) + ) + if total <= 0: + return output + summary = {"task_status": run_outcome.get("task_status"), **tests} + if isinstance(output, dict): + merged = dict(output) + merged.setdefault("test_summary", summary) + return merged + if output is None: + return {"test_summary": summary} + if isinstance(output, str): + return {"result": output, "test_summary": summary} + # Typed/structured outputs keep their shape; the summary stays available in + # run_outcome.json and the session state. + return output + + +def resolve_trace_suffix(task_status: str, run_outcome: dict | None) -> str: + """Trace naming: assertion failures must be distinguishable from _PASS.""" + if task_status != "completed": + return "_FAIL" + tests = (run_outcome or {}).get("tests") or {} + if int(tests.get("failed", 0)) > 0: + return "_TESTFAIL" + return "_PASS" diff --git a/artemis/sdk/types/__init__.py b/artemis/sdk/types/__init__.py index 9c59ed1f1..e1cc0b380 100644 --- a/artemis/sdk/types/__init__.py +++ b/artemis/sdk/types/__init__.py @@ -14,13 +14,13 @@ """Type definitions for the ARTEMIS SDK.""" +from artemis.context import DevicePlatform from artemis.sdk.types.agent import ( AgentConfig, ApiBaseUrl, - DevicePlatform, ServerConfig, ) -from artemis.sdk.types.exceptions import ( +from third_party.mobile_use.sdk.types.exceptions import ( AgentError, AgentNotInitializedError, AgentProfileNotFoundError, @@ -31,7 +31,7 @@ ServerError, ServerStartupError, ) -from artemis.sdk.types.task import ( +from third_party.mobile_use.sdk.types.task import ( AgentProfile, Task, TaskRequest, diff --git a/artemis/sdk/types/agent.py b/artemis/sdk/types/agent.py index 2b4359abe..10f4e1d9a 100644 --- a/artemis/sdk/types/agent.py +++ b/artemis/sdk/types/agent.py @@ -11,14 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. from typing import Literal -from urllib.parse import urlparse -from langchain_core.callbacks.base import Callbacks from artemis.config import ( ExplorerConfig, FlashProfileConfig, @@ -27,91 +22,25 @@ VideoAnalyzerConfig, ) from artemis.context import DevicePlatform -from artemis.sdk.types.task import AgentProfile, TaskRequestCommon from artemis.utils.video import detect_video_tools_enabled -from pydantic import BaseModel, Field - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class ApiBaseUrl(BaseModel): - """Defines an API base URL.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - scheme: Literal["http", "https"] - host: str - port: int | None = None - - def __eq__(self, other): - if not isinstance(other, ApiBaseUrl): - return False - return self.to_url() == other.to_url() - - def to_url(self): - return ( - f"{self.scheme}://{self.host}:{self.port}" - if self.port is not None - else f"{self.scheme}://{self.host}" - ) - - @classmethod - def from_url(cls, url: str) -> "ApiBaseUrl": - parsed_url = urlparse(url) - if parsed_url.scheme not in ["http", "https"]: - raise ValueError(f"Invalid scheme: {parsed_url.scheme}") - if parsed_url.hostname is None: - raise ValueError("Invalid hostname") - return cls( - scheme=parsed_url.scheme, # type: ignore - host=parsed_url.hostname, - port=parsed_url.port, - ) - - -class ServerConfig(BaseModel): - """Configuration for the required servers.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} +from third_party.mobile_use.sdk.types.agent import ( + AgentConfigBase, + ApiBaseUrl, + ServerConfig, +) +from third_party.mobile_use.sdk.types.task import AgentProfile +from pydantic import Field - adb_host: str - adb_port: int +__all__ = ["AgentConfig", "AgentProfile", "ApiBaseUrl", "DevicePlatform", "ServerConfig"] -class AgentConfig(BaseModel): +class AgentConfig(AgentConfigBase): """ARTEMIS agent configuration. - Attributes: - agent_profiles: Map an agent profile name to its configuration. - task_config_defaults: Default task request configuration. - default_profile: default profile to use for tasks - device_id: Specific device to target (if None, first available is used). - device_platform: Platform of the device to target. - servers: Custom server configurations. + Adds the Checker, Outputter, Explorer, video-analysis and concurrency + settings to :class:`AgentConfigBase`. """ - agent_profiles: dict[str, AgentProfile] - task_request_defaults: TaskRequestCommon - default_profile: AgentProfile - device_id: str | None = None - device_platform: DevicePlatform | None = None - servers: ServerConfig - graph_config_callbacks: Callbacks = None video_recording_tools_enabled: bool = Field(default_factory=detect_video_tools_enabled) force_web_accessibility: bool = False disable_checker: bool = False diff --git a/artemis/sdk/types/platform.py b/artemis/sdk/types/platform.py deleted file mode 100644 index 5bceb4fd9..000000000 --- a/artemis/sdk/types/platform.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from datetime import datetime -from typing import Annotated, Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, PlainSerializer -from pydantic.v1.utils import to_lower_camel - -TaskRunStatus = Literal["pending", "running", "completed", "failed", "cancelled"] - -IsoDatetime = Annotated[ - datetime, - PlainSerializer( - func=lambda v: v.isoformat() if v else None, - return_type=str, - when_used="unless-none", - ), -] - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class BaseApiModel(BaseModel): - model_config = ConfigDict( - alias_generator=to_lower_camel, - populate_by_name=True, - str_strip_whitespace=True, - ) - - -class LLMProfileResponse(BaseApiModel): - """Response model for LLM profile.""" - - id: str = Field(..., description="Profile ID") - name: str = Field(..., description="Profile name") - description: str | None = Field(None, description="Profile description") - llms: dict[str, Any] = Field(..., description="LLM configuration") - created_at: str = Field(..., description="Creation timestamp (ISO format)") - updated_at: str = Field(..., description="Last update timestamp (ISO format)") - - -class TaskOptionsResponse(BaseApiModel): - """Response model for task options.""" - - id: str = Field(..., description="Options ID") - enable_tracing: bool = Field(..., description="Whether tracing is enabled") - max_steps: int = Field(..., description="Maximum number of steps") - enable_video_tools: bool = Field(..., description="Whether video tools are enabled") - created_at: str = Field(..., description="Creation timestamp (ISO format)") - updated_at: str = Field(..., description="Last update timestamp (ISO format)") - - -class TaskResponse(BaseApiModel): - """Response model for task.""" - - id: str = Field(..., description="Task ID") - name: str = Field(..., description="Task name") - description: str | None = Field(None, description="Task description") - input_prompt: str = Field(..., description="Input prompt") - output_description: str | None = Field(None, description="Output description") - options: TaskOptionsResponse = Field(..., description="Task options") - created_at: str = Field(..., description="Creation timestamp (ISO format)") - updated_at: str = Field(..., description="Last update timestamp (ISO format)") - - -class CreateTaskRunRequest(BaseApiModel): - """Request model for creating a task run.""" - - task_id: str = Field(..., description="ID of the task to run") - llm_profile_id: str = Field(..., description="LLM profile ID to use") - virtual_mobile_id: str | None = Field(None, description="Virtual mobile ID to use") - locked_app_package: str | None = Field(None, description="App package to lock for the task run") - execution_origin: str | None = Field(None, description="Origin of the task execution") - enable_video_tools: bool = Field( - default=False, - description="Whether video tools are enabled for this task run", - ) - - -class CreateOrphanTaskRunRequest(BaseApiModel): - """Request model for creating an orphan task run (without a parent task).""" - - input_prompt: str = Field( - ..., - description="Input prompt for the task run", - ) - output_description: str | None = Field( - None, - description="Optional output description", - ) - llm_profile_id: str = Field( - ..., - description="LLM profile ID to use", - ) - virtual_mobile_id: str | None = Field( - None, - description="Optional virtual mobile ID on which it runs", - ) - locked_app_package: str | None = Field( - None, - description="Optional app package to lock execution to", - ) - execution_origin: str | None = Field( - None, - description="Origin of the task run execution", - ) - max_steps: int | None = Field( - None, - description="Maximum number of steps for the task run", - ) - enable_video_tools: bool = Field( - default=False, - description="Whether video tools are enabled for this task run", - ) - - -class UpdateTaskRunStatusRequest(BaseApiModel): - """Request model for updating task run status.""" - - status: TaskRunStatus = Field(..., description="New status of the task run") - message: str | None = Field(None, description="Message associated with the status") - output: str | None = Field(None, description="Output of the task run") - - -class TaskRunResponse(BaseApiModel): - """Response model for a single task run.""" - - id: str = Field(..., description="Unique identifier for the task run") - task: TaskResponse | None = Field( - ..., - description=("ID of the task this run is for or None if manually created"), - ) - llm_profile: LLMProfileResponse = Field(..., description="LLM profile ID used for this run") - status: TaskRunStatus = Field(..., description="Current status of the task run") - input_prompt: str = Field(..., description="Input prompt for this task run") - output_description: str | None = Field(None, description="Description of expected output") - created_at: datetime = Field(..., description="When the task run was created") - started_at: datetime | None = Field(None, description="When the task run started") - finished_at: datetime | None = Field(None, description="When the task run finished") - locked_app_package: str | None = Field( - None, description="The app package to lock for the task run" - ) - - -SubgoalState = Literal["pending", "started", "completed", "failed"] - - -class ArtemisSubgoal(BaseModel): - """Upsert ArtemisSubgoal API model.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - name: str = Field(..., description="Name of the subgoal") - state: SubgoalState = Field(default="pending", description="Current state of the subgoal") - started_at: IsoDatetime | None = Field(default=None, description="When the subgoal started") - ended_at: IsoDatetime | None = Field(default=None, description="When the subgoal ended") - - -class UpsertTaskRunPlanRequest(BaseApiModel): - """Upsert task run plan request model.""" - - started_at: IsoDatetime = Field(..., description="When the plan started") - subgoals: list[ArtemisSubgoal] = Field(..., description="Subgoals of the plan") - ended_at: IsoDatetime | None = Field( - default=None, - description="When the plan ended (replanned or completed)", - ) - - -class TaskRunPlanResponse(UpsertTaskRunPlanRequest): - """Response model for a task run plan.""" - - id: str = Field(..., description="Unique identifier for the task run plan") - task_run_id: str = Field(..., description="ID of the task run this plan is for") - - -class TrajectoryGifUploadResponse(BaseApiModel): - """Response containing trajectory GIF signed upload URL.""" - - signed_url: str = Field(..., description="Signed upload URL for the trajectory GIF") - task_run_id: str = Field(..., description="Task run ID for which the GIF URL was generated") - file_path: str = Field(..., description="Path where the GIF will be stored") diff --git a/artemis/services/llm.py b/artemis/services/llm.py index 784e971f3..ec91e6fa0 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -19,7 +19,8 @@ """ import asyncio -from collections.abc import Awaitable, Callable, Coroutine +from collections.abc import Awaitable, Callable, Iterator +from contextlib import contextmanager from contextvars import ContextVar, Token import functools import logging @@ -27,7 +28,7 @@ import re import sys import time -from typing import Any, Literal, TypeVar, overload +from typing import Any, TypeVar from uuid import uuid4 from langchain_core.language_models.chat_models import BaseChatModel @@ -35,10 +36,6 @@ from artemis.config import ( PAUSE_FILE, - AgentNode, - AgentNodeWithFallback, - LLMUtilsNode, - LLMUtilsNodeWithFallback, LLMWithFallback, get_default_llm_config, settings, @@ -63,7 +60,8 @@ content_to_text, parse_structured, ) -from artemis.utils.logger import get_logger +from third_party.mobile_use.llm_service import FallbackRunner, GetLLM, LLMWaitNotice +from third_party.mobile_use.utils.logger import get_logger # Logger for internal messages llm_logger = logging.getLogger(__name__) @@ -862,48 +860,14 @@ async def acomplete_structured( return parsed -async def invoke_llm_with_timeout_message[T]( - llm_call: Coroutine[Any, Any, T], - timeout_seconds: int = 10, - hard_timeout: int = 180, -) -> T: - """Send an LLM call and display a countdown / timeout message if delayed.""" - llm_task = asyncio.create_task(llm_call) - waiter_task = asyncio.create_task(asyncio.sleep(timeout_seconds)) - try: - done, _ = await asyncio.wait({llm_task, waiter_task}, return_when=asyncio.FIRST_COMPLETED) - - if llm_task in done: - return llm_task.result() - - user_messages_logger.info("Waiting for LLM call response...") - start_time = asyncio.get_event_loop().time() - - while True: - try: - return await asyncio.wait_for(asyncio.shield(llm_task), timeout=1.0) - except TimeoutError: - if llm_task.done(): - return llm_task.result() - - pause_file = PAUSE_FILE - if pause_file.exists(): - start_time = asyncio.get_event_loop().time() - continue - - elapsed = asyncio.get_event_loop().time() - start_time - if elapsed > max(0, hard_timeout - timeout_seconds): - user_messages_logger.error(f"LLM call timed out after {hard_timeout} seconds.") - raise TimeoutError(f"LLM call timed out after {hard_timeout} seconds.") - except BaseException: - if not llm_task.done(): - llm_task.cancel() - await asyncio.gather(llm_task, return_exceptions=True) - raise - finally: - if not waiter_task.done(): - waiter_task.cancel() - await asyncio.gather(waiter_task, return_exceptions=True) +# Wraps every LLM call: tells the user when a call is slow and enforces a hard +# timeout. The timeout does not count while the task is paused (PAUSE_FILE). +invoke_llm_with_timeout_message = LLMWaitNotice( + notify=user_messages_logger.info, + on_timeout=user_messages_logger.error, + hold=lambda: PAUSE_FILE.exists(), + default_hard_timeout=180, +).__call__ # Backward compatible factory functions delegating to ModelFactory @@ -929,108 +893,6 @@ def get_google_llm( return ModelFactory.create_model(ep) -def get_vertex_llm( - model_name: str = "gemini-3.8-flash", - temperature: float | None = None, - timeout: float | None = None, - thinking_budget: int | None = None, -) -> BaseChatModel: - ep = ModelEndpoint( - provider=ModelProvider.VERTEX_AI, - model_name=model_name, - temperature=temperature or 0.0, - timeout_seconds=timeout or 60.0, - thinking_budget=thinking_budget, - ) - return ModelFactory.create_model(ep) - - -def get_openai_llm( - model_name: str = "o3", - temperature: float | None = None, - timeout: float | None = None, -) -> BaseChatModel: - ep = ModelEndpoint( - provider=ModelProvider.OPENAI, - model_name=model_name, - temperature=temperature or 0.0, - timeout_seconds=timeout or 60.0, - ) - return ModelFactory.create_model(ep) - - -def get_openrouter_llm( - model_name: str, - temperature: float | None = None, - timeout: float | None = None, -) -> BaseChatModel: - ep = ModelEndpoint( - provider=ModelProvider.OPENROUTER, - model_name=model_name, - temperature=temperature or 0.0, - timeout_seconds=timeout or 60.0, - ) - return ModelFactory.create_model(ep) - - -def get_grok_llm( - model_name: str, - temperature: float | None = None, - timeout: float | None = None, -) -> BaseChatModel: - ep = ModelEndpoint( - provider=ModelProvider.XAI, - model_name=model_name, - temperature=temperature or 0.0, - timeout_seconds=timeout or 60.0, - ) - return ModelFactory.create_model(ep) - - -def get_anthropic_llm( - model_name: str, - temperature: float | None = None, - timeout: float | None = None, - thinking_budget: int | None = None, - reasoning_effort: str | None = None, -) -> BaseChatModel: - ep = ModelEndpoint( - provider=ModelProvider.ANTHROPIC, - model_name=model_name, - temperature=temperature or 0.0, - timeout_seconds=timeout or 60.0, - thinking_budget=thinking_budget, - reasoning_effort=reasoning_effort, - ) - return ModelFactory.create_model(ep) - - -def get_cached_raw_model( - provider: str, - model_name: str, - temperature: float | None = None, - timeout: float | None = None, - thinking_budget: int | None = None, - thinking_level: str | None = None, - include_thoughts: bool | None = None, - reasoning_effort: str | None = None, - enable_grounding: bool = False, -) -> BaseChatModel: - """Retrieves or instantiates a cached raw LangChain chat model.""" - ep = ModelEndpoint( - provider=ModelProvider.from_string(provider), - model_name=model_name, - temperature=temperature or 0.0, - timeout_seconds=timeout or 60.0, - thinking_budget=thinking_budget, - thinking_level=thinking_level, - include_thoughts=include_thoughts, - reasoning_effort=reasoning_effort, - enable_grounding=enable_grounding, - ) - return ModelFactory.get_model(ep) - - def _resolve_endpoint( ctx: ArtemisContext, name: str, @@ -1077,46 +939,15 @@ def _get_val(obj, attr, expected_type): ) -@overload -def get_llm( - ctx: ArtemisContext, - name: AgentNodeWithFallback, - *, - use_fallback: bool = False, - temperature: float | None = None, -) -> BaseChatModel: ... - - -@overload -def get_llm( - ctx: ArtemisContext, - name: LLMUtilsNode, - *, - is_utils: Literal[True], - temperature: float | None = None, -) -> BaseChatModel: ... - - -@overload -def get_llm( +def _get_llm( ctx: ArtemisContext, - name: LLMUtilsNodeWithFallback, - *, - is_utils: Literal[True], - use_fallback: bool = False, - temperature: float | None = None, -) -> BaseChatModel: ... - - -def get_llm( - ctx: ArtemisContext, - name: AgentNode | LLMUtilsNode | AgentNodeWithFallback, + name: str, is_utils: bool = False, use_fallback: bool = False, temperature: float | None = None, ) -> BaseChatModel: """Resolves and instantiates the appropriate LLM wrapper for the given agent role.""" - endpoint = _resolve_endpoint(ctx, str(name), is_utils=is_utils, use_fallback=use_fallback) + endpoint = _resolve_endpoint(ctx, name, is_utils=is_utils, use_fallback=use_fallback) if temperature is not None: endpoint = endpoint.model_copy(update={"temperature": temperature}) raw_model = ModelFactory.get_model(endpoint) @@ -1127,23 +958,39 @@ def get_llm( return RobustChatModelWrapper(bound_model, ctx, endpoint=endpoint) # type: ignore -async def with_fallback[T]( - main_call: Callable[[], Awaitable[T]], - fallback_call: Callable[[], Awaitable[T]], - none_should_fallback: bool = True, -) -> T: - """Run main_call, switching to fallback_call only when it can actually help. - - Falling back is an explicit, observed decision: the failure is classified - and only categories where a different endpoint might succeed trigger the - fallback (a bad request would just hide the bug inside a weaker model's - output). Every switch is logged at WARNING and recorded as an - llm_fallback telemetry event. While main_call runs, the recovery layer - knows a fallback exists and hands over immediately on retry exhaustion - instead of pausing the task. +get_llm: GetLLM = _get_llm + + +class _ObservedFallback(FallbackRunner): + """Fall back only on failures another endpoint could fix, and record it. + + Errors are classified first; a bad request, for example, is re-raised + because a weaker model would only hide the bug. Each switch is logged at + WARNING and recorded as an ``llm_fallback`` event. While the main call + runs, the retry layer knows a fallback exists and hands over as soon as + retries run out instead of pausing the task. """ - def _switch(reason: str, category: str | None, error: str | None) -> None: + @contextmanager + def main_scope(self) -> Iterator[None]: + # Reset before the fallback call starts: nothing is behind the + # fallback, so if it runs out of retries the task should pause. + token = _FALLBACK_AVAILABLE.set(True) + try: + yield + finally: + _FALLBACK_AVAILABLE.reset(token) + + def fallback_reason(self, error: Exception) -> tuple[str, str | None] | None: + if isinstance(error, LLMCallError): + reason, failure = "terminal_error", error.failure + else: + reason, failure = "error", classify_failure(error) + if failure.category is FailureCategory.CANCELLED: + return None + return (reason, failure.category.value) if failure.should_fallback else None + + def on_fallback(self, reason: str, category: str | None, error: str | None) -> None: llm_logger.warning( f"❗ Main LLM inference failed ({reason}" f"{f': {error}' if error else ''}). Falling back..." @@ -1157,30 +1004,6 @@ def _switch(reason: str, category: str | None, error: str | None) -> None: }, ) - # The contextvar must be reset BEFORE fallback_call runs: the fallback has - # no further fallback behind it, so its own exhaustion should pause. - fallback_token = _FALLBACK_AVAILABLE.set(True) - try: - result = await main_call() - except (KeyboardInterrupt, asyncio.CancelledError): - raise - except LLMCallError as e: - _FALLBACK_AVAILABLE.reset(fallback_token) - if not e.failure.should_fallback: - raise - _switch("terminal_error", e.failure.category.value, str(e)) - return await fallback_call() - except Exception as e: - _FALLBACK_AVAILABLE.reset(fallback_token) - failure = classify_failure(e) - if failure.category is FailureCategory.CANCELLED or not failure.should_fallback: - raise - _switch("error", failure.category.value, str(e)) - return await fallback_call() - else: - _FALLBACK_AVAILABLE.reset(fallback_token) - - if result is None and none_should_fallback: - _switch("empty_result", None, None) - return await fallback_call() - return result + +# Runs main_call; switches to fallback_call when another endpoint could succeed. +with_fallback = _ObservedFallback().run diff --git a/artemis/services/token_meter.py b/artemis/services/token_meter.py index 8a48e95cb..f7bd5a6bc 100644 --- a/artemis/services/token_meter.py +++ b/artemis/services/token_meter.py @@ -41,7 +41,7 @@ from artemis.data_engine.context_vars import CURRENT_NODE_NAME from artemis.llm.google import usage_from_message -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/telemetry/tracer.py b/artemis/telemetry/tracer.py index 241dec705..5364a15df 100644 --- a/artemis/telemetry/tracer.py +++ b/artemis/telemetry/tracer.py @@ -23,7 +23,7 @@ from uuid import uuid4 from artemis.telemetry.models import SpanType, TelemetrySpan -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/base.py b/artemis/tools/base.py index 9e70bdbdb..05d8cbedb 100644 --- a/artemis/tools/base.py +++ b/artemis/tools/base.py @@ -33,7 +33,7 @@ from artemis.core.tool_declaration import ToolDeclaration from artemis.drivers.base import BaseDeviceDriver from artemis.drivers.factory import get_driver -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/command_tool.py b/artemis/tools/command_tool.py index 4c97a8d48..96619f6bf 100644 --- a/artemis/tools/command_tool.py +++ b/artemis/tools/command_tool.py @@ -50,14 +50,14 @@ from artemis.core.tool_failure import ToolFailure from artemis.agents.log_analyzer.output_analyzer import TaskOutputAnalyzerNode from artemis.context import ArtemisContext -from artemis.controllers.platform_specific_commands_controller import ( - get_adb_device, -) from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper -from artemis.utils.logger import get_logger +from third_party.mobile_use.controllers.platform_specific_commands_controller import ( + get_adb_device, +) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/committee_tool.py b/artemis/tools/committee_tool.py index 5db3b342c..28735c903 100644 --- a/artemis/tools/committee_tool.py +++ b/artemis/tools/committee_tool.py @@ -46,15 +46,15 @@ invoke_tool_with_injection, tool_result_messages, ) -from artemis.tools.types import CyFunctionDetector +from artemis.utils.cython_compat import CyFunctionDetector from artemis.tools.video_tool import get_video_analyzer_tool -from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path, get_notes_dir from artemis.memory.context_policy import build_history_for from artemis.utils.task_tree import ( get_active_subgoal_hashes, get_recent_subgoal_hashes, ) +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/diagnoser_submit_answer_tool.py b/artemis/tools/diagnoser_submit_answer_tool.py index abdb7f322..e970e548a 100644 --- a/artemis/tools/diagnoser_submit_answer_tool.py +++ b/artemis/tools/diagnoser_submit_answer_tool.py @@ -23,7 +23,7 @@ from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver from artemis.tools.base import ArtemisTool -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/diagnostic_tool.py b/artemis/tools/diagnostic_tool.py index 426261402..0c68ece06 100644 --- a/artemis/tools/diagnostic_tool.py +++ b/artemis/tools/diagnostic_tool.py @@ -27,8 +27,8 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/explorer_tool.py b/artemis/tools/explorer_tool.py index 951108b06..14a95b94f 100644 --- a/artemis/tools/explorer_tool.py +++ b/artemis/tools/explorer_tool.py @@ -58,10 +58,10 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector +from artemis.utils.cython_compat import CyFunctionDetector from artemis.utils.element_hit_test import find_element_at_point -from artemis.utils.logger import get_logger from artemis.utils.visualization import draw_dots +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/history/__init__.py b/artemis/tools/history/__init__.py index 17aba4aad..09c0cd24f 100644 --- a/artemis/tools/history/__init__.py +++ b/artemis/tools/history/__init__.py @@ -40,7 +40,7 @@ from artemis.tools.history.screenshot import ScreenshotResult, load_step_screenshot from artemis.tools.history.search import search_history_text from artemis.tools.tool_wrapper import ToolWrapper -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/history/screenshot.py b/artemis/tools/history/screenshot.py index 51906b966..94f55d343 100644 --- a/artemis/tools/history/screenshot.py +++ b/artemis/tools/history/screenshot.py @@ -30,8 +30,8 @@ from typing import Any from artemis.core.tool_failure import ToolFailure -from artemis.utils.logger import get_logger from artemis.utils.visualization import overlay_action_on_screenshot +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/history/search.py b/artemis/tools/history/search.py index eb9ec2f4a..c2a847970 100644 --- a/artemis/tools/history/search.py +++ b/artemis/tools/history/search.py @@ -47,8 +47,8 @@ from typing import Any from artemis.core.tool_failure import ToolFailure -from artemis.utils.logger import get_logger from artemis.utils.task_tree import format_actions_clean +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/image_processor_tool.py b/artemis/tools/image_processor_tool.py index 33b22c3b6..b70194ce5 100644 --- a/artemis/tools/image_processor_tool.py +++ b/artemis/tools/image_processor_tool.py @@ -25,8 +25,8 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/index.py b/artemis/tools/index.py index 71c90e564..cf1762ce6 100644 --- a/artemis/tools/index.py +++ b/artemis/tools/index.py @@ -16,27 +16,20 @@ from artemis.context import ArtemisContext from artemis.data_engine.trace import trace_langchain_tool -from artemis.tools.tool_wrapper import CompositeToolWrapper, ToolWrapper +from artemis.tools.tool_wrapper import ToolWrapper +from third_party.mobile_use.tools.index import ( + get_tools_from_wrappers as build_tools_from_wrappers, +) def get_tools_from_wrappers( ctx: "ArtemisContext", wrappers: list[ToolWrapper], ) -> list[BaseTool]: - """Instantiate and wrap LangChain tools from a list of ToolWrappers.""" - tools: list[BaseTool] = [] - for wrapper in wrappers: - if wrapper.is_available_fn is not None and not wrapper.is_available_fn(ctx): - continue - if isinstance(wrapper, CompositeToolWrapper): - comp_tools = wrapper.composite_tools_fn_getter(ctx) - for t in comp_tools: - tools.append(trace_langchain_tool(t, ctx)) - continue - - t = wrapper.tool_fn_getter(ctx) - tools.append(trace_langchain_tool(t, ctx)) - return tools + """Instantiate the available tools of ``wrappers``, traced into the data engine.""" + return build_tools_from_wrappers( + ctx, wrappers, wrap_tool=lambda t: trace_langchain_tool(t, ctx) + ) def get_tool_by_name(name: str, tools: list[BaseTool]) -> BaseTool | None: diff --git a/artemis/tools/log_tool.py b/artemis/tools/log_tool.py index dc6190151..8885c57e9 100644 --- a/artemis/tools/log_tool.py +++ b/artemis/tools/log_tool.py @@ -27,8 +27,8 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/mobile/launch_app.py b/artemis/tools/mobile/launch_app.py index 5c0de077f..23cde2c7f 100644 --- a/artemis/tools/mobile/launch_app.py +++ b/artemis/tools/mobile/launch_app.py @@ -18,21 +18,15 @@ from langchain_core.tools import BaseTool from pydantic import BaseModel, Field -from artemis.agents.hopper.hopper import HopperOutput, hopper from artemis.context import ArtemisContext -from artemis.controllers.platform_specific_commands_controller import ( - list_packages_async, -) from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector -from artemis.utils.app_launch_utils import launch_app_with_retries -from artemis.utils.logger import get_logger - -logger = get_logger(__name__) +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.tools.mobile.launch_app import LAUNCH_APP_DOCSTRING, find_package +from third_party.mobile_use.utils.app_launch_utils import launch_app_with_retries class LaunchAppArgs(BaseModel): @@ -45,67 +39,6 @@ class LaunchAppArgs(BaseModel): ) -LAUNCH_APP_DOCSTRING = ( - "[ACTION] Finds and launches an application on the device using its natural language name." -) - - -async def find_package(ctx: ArtemisContext, app_name: str, use_fallback: bool = True) -> str | None: - """Finds the package name for a given application name. - - Returns None if package not found or on error. - """ - package_cache = getattr(ctx, "package_cache", None) - if package_cache is None or not isinstance(package_cache, dict): - try: - ctx.package_cache = {} - package_cache = ctx.package_cache - except Exception: # pylint: disable=broad-exception-caught - package_cache = {} - - if isinstance(package_cache, dict) and app_name in package_cache: - logger.info(f"Cache hit for app '{app_name}': {package_cache[app_name]}") - return package_cache[app_name] - - try: - all_packages = await list_packages_async(ctx=ctx) - package_set = {p.strip() for p in all_packages.split("\n") if p.strip()} - - # Fast path: If app_name is already directly an installed package name - if app_name in package_set: - if isinstance(package_cache, dict): - package_cache[app_name] = app_name - return app_name - - hopper_output: HopperOutput = await hopper( - ctx=ctx, - request=(f"I'm looking for the package name of the following app: '{app_name}'"), - data=all_packages, - use_fallback=use_fallback, - ) - if not hopper_output.found or not hopper_output.output: - if isinstance(package_cache, dict): - package_cache[app_name] = None - return None - - package_name = hopper_output.output.strip() - if package_name not in package_set: - logger.warning( - f"Hopper returned package '{package_name}' for '{app_name}', " - "but it is NOT physically installed on the device!" - ) - if isinstance(package_cache, dict): - package_cache[app_name] = None - return None - - if isinstance(package_cache, dict): - package_cache[app_name] = package_name - return package_name - except Exception as e: # pylint: disable=broad-exception-caught - logger.error(f"Failed to find package for '{app_name}': {e}") - return None - - class LaunchAppTool(ArtemisTool): """Universal tool for finding and launching an application on the device.""" diff --git a/artemis/tools/mobile/log_utils.py b/artemis/tools/mobile/log_utils.py index 39d5e4f20..bcaf2b036 100644 --- a/artemis/tools/mobile/log_utils.py +++ b/artemis/tools/mobile/log_utils.py @@ -15,8 +15,8 @@ from datetime import datetime, timedelta import time from artemis.context import ArtemisContext -from artemis.controllers.platform_specific_commands_controller import get_adb_device -from artemis.utils.logger import get_logger +from third_party.mobile_use.controllers.platform_specific_commands_controller import get_adb_device +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/mobile/ocr.py b/artemis/tools/mobile/ocr.py index a3f3ffc3f..6ce93fbf6 100644 --- a/artemis/tools/mobile/ocr.py +++ b/artemis/tools/mobile/ocr.py @@ -27,8 +27,8 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.utils.logger import get_logger from artemis.utils.ocr_api import is_ocr_configured, perform_ocr +from third_party.mobile_use.utils.logger import get_logger class OcrArgs(BaseModel): diff --git a/artemis/tools/mobile/read_hierarchy.py b/artemis/tools/mobile/read_hierarchy.py index bd8e47c2d..02930e82c 100644 --- a/artemis/tools/mobile/read_hierarchy.py +++ b/artemis/tools/mobile/read_hierarchy.py @@ -25,8 +25,8 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/mobile/read_logs.py b/artemis/tools/mobile/read_logs.py index 2487f539d..b84af4c78 100644 --- a/artemis/tools/mobile/read_logs.py +++ b/artemis/tools/mobile/read_logs.py @@ -24,7 +24,7 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.mobile.log_utils import fetch_and_filter_logs -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/mobile/search_logs.py b/artemis/tools/mobile/search_logs.py index eed853740..2c43dfbc0 100644 --- a/artemis/tools/mobile/search_logs.py +++ b/artemis/tools/mobile/search_logs.py @@ -25,7 +25,7 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.mobile.log_utils import fetch_and_filter_logs -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/object_detection_tool.py b/artemis/tools/object_detection_tool.py index c922f9025..95e5c42d8 100644 --- a/artemis/tools/object_detection_tool.py +++ b/artemis/tools/object_detection_tool.py @@ -35,8 +35,8 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/scratchpad.py b/artemis/tools/scratchpad.py index 6032e431d..8ed3928ca 100644 --- a/artemis/tools/scratchpad.py +++ b/artemis/tools/scratchpad.py @@ -25,8 +25,7 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector from artemis.utils.notes import ( APPEND_NOTE_ARG_CONTENT_DESC, APPEND_NOTE_ARG_KEY_DESC, @@ -51,6 +50,8 @@ save_note_content, update_note_content, ) +from third_party.mobile_use.tools.tool_wrapper import make_save_note_wrapper +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -789,11 +790,7 @@ def get_append_note_tool_pure(ctx: ArtemisContext) -> BaseTool: return append_note_pure.to_langchain_tool(ctx, name="append_note") -save_note_wrapper = ToolWrapper( - tool_fn_getter=get_save_note_tool, - on_success_fn=lambda key: f"Saved note '{key}'.", - on_failure_fn=lambda key: f"Failed to save note '{key}'.", -) +save_note_wrapper = make_save_note_wrapper(get_save_note_tool) read_note_wrapper = ToolWrapper( tool_fn_getter=get_read_note_tool, diff --git a/artemis/tools/tool_wrapper.py b/artemis/tools/tool_wrapper.py index 0d9784198..89f0908a6 100644 --- a/artemis/tools/tool_wrapper.py +++ b/artemis/tools/tool_wrapper.py @@ -24,30 +24,22 @@ from langchain_core.tools import BaseTool from langchain_core.tools.base import InjectedToolCallId from langgraph.prebuilt import InjectedState -from pydantic import BaseModel from artemis.core.tool_failure import is_tool_failure -from artemis.context import ArtemisContext from artemis.data_engine import engine as engine_mod from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize from artemis.graph.state import State -from artemis.tools.types import CyFunctionDetector - - -class ToolWrapper(BaseModel): - """Wrapper holding a tool factory and lifecycle callbacks.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - tool_fn_getter: Callable[[ArtemisContext], BaseTool] - on_success_fn: Callable[..., str] - on_failure_fn: Callable[..., str] - is_available_fn: Callable[[ArtemisContext], bool] | None = None - - -class CompositeToolWrapper(ToolWrapper): - """Wrapper holding a composite tool factory and lifecycle callbacks.""" - - composite_tools_fn_getter: Callable[[ArtemisContext], list[BaseTool]] +from third_party.mobile_use.tools.tool_wrapper import CompositeToolWrapper, ToolWrapper + +__all__ = [ + "CompositeToolWrapper", + "ToolWrapper", + "get_tool_result_content", + "invoke_tool_with_injection", + "resolve_image_carrier", + "split_multimodal_result", + "tool_result_messages", +] # pylint: disable=too-many-branches,too-many-locals,too-many-statements diff --git a/artemis/tools/types.py b/artemis/tools/types.py deleted file mode 100644 index 0fa8a9415..000000000 --- a/artemis/tools/types.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pydantic import BaseModel, Field, model_validator - -from artemis.utils.ui_hierarchy import ElementBounds - - -class _CyFunctionDetectorMeta(type): - """Metaclass for detecting cython compiled functions.""" - - def __instancecheck__(cls, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -# pylint: disable=too-few-public-methods -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - """Detector for cython compiled functions.""" - - -class Target(BaseModel): - """A comprehensive locator for a UI element, supporting a fallback mechanism.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - resource_id: str | None = Field(None, description="The resource-id of the element.") - resource_id_index: int | None = Field( - None, - description=("The zero-based index if multiple elements share the same resource-id."), - ) - text: str | None = Field( - None, - description=("The text content of the element (e.g., a label or placeholder)."), - ) - text_index: int | None = Field( - None, - description=("The zero-based index if multiple elements share the same text."), - ) - bounds: ElementBounds | None = Field( - None, description="The x, y, width, and height of the element." - ) - - @model_validator(mode="after") - def _default_indices(self): - # Treat empty strings like “not provided” - if ( - self.resource_id is not None and self.resource_id != "" - ) and self.resource_id_index is None: - self.resource_id_index = 0 - if (self.text is not None and self.text != "") and self.text_index is None: - self.text_index = 0 - return self diff --git a/artemis/tools/utils.py b/artemis/tools/utils.py deleted file mode 100644 index ecc8e48a7..000000000 --- a/artemis/tools/utils.py +++ /dev/null @@ -1,392 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. - -from __future__ import annotations - -from typing import Literal - -from pydantic import BaseModel, ConfigDict - -from artemis.context import ArtemisContext -from artemis.controllers.types import ( - CoordinatesSelectorRequest, - PercentagesSelectorRequest, -) -from artemis.controllers.unified_controller import UnifiedMobileController -from artemis.graph.state import State -from artemis.tools.types import Target -from artemis.utils.logger import get_logger -from artemis.utils.ui_hierarchy import ( - ElementBounds, - Point, - find_element_by_resource_id, - get_bounds_for_element, - get_element_text, - is_element_focused, -) - -logger = get_logger(__name__) - - -def find_element_by_text( - ui_hierarchy: list[dict], text: str, index: int | None = None -) -> dict | None: - """Find a UI element by its text content (adapted to both flat and rich hierarchy) - - This function performs a recursive, case-insensitive search. - - Args: - ui_hierarchy: List of UI element dictionaries. - text: The text content to search for. - index: Optional index to select nth matching element. - - Returns: - The complete UI element dictionary if found, None otherwise. - """ - matches: list[dict] = [] - - def search_recursive(elements: list[dict]) -> None: - for element in elements: - if isinstance(element, dict): - src = element.get("attributes", element) - element_text = src.get("text", "") - - # Guard against non-string values - if not isinstance(element_text, str): - element_text = "" - - text_lower = text.lower() - if element_text and text_lower == element_text.lower(): - matches.append(element) - - if children := element.get("children", []): - search_recursive(children) - - search_recursive(ui_hierarchy) - - if not matches: - return None - - if index is None: - idx = 0 - elif index < 0: - return None - else: - idx = index - if idx < len(matches): - return matches[idx] - return None - - -async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext): - """Tap the bottom-right corner of the specified element bounds.""" - bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99) - await tap( - ctx=ctx, - selector_request=SelectorRequestWithCoordinates( - coordinates=CoordinatesSelectorRequest( - x=bottom_right.x, - y=bottom_right.y, - ), - ), - ) - - -# pylint: disable=too-many-return-statements -async def move_cursor_to_end_if_bounds( - ctx: ArtemisContext, - state: State, - target: Target, - elt: dict | None = None, -) -> dict | None: - """Best-effort move of the text cursor near the end of the input by tapping the - - bottom-right area of the focused element (if bounds are available). - """ - if target.resource_id: - if not elt: - elt = find_element_by_resource_id( - ui_hierarchy=state.latest_ui_hierarchy or [], - resource_id=target.resource_id, - index=target.resource_id_index, - ) - if not elt: - return None - - bounds = get_bounds_for_element(elt) - if not bounds: - return elt - - logger.debug("Tapping near the end of the input to move the cursor") - await tap_bottom_right_of_element(bounds=bounds, ctx=ctx) - logger.debug(f"Tapped end of input {target.resource_id}") - return elt - - if target.bounds: - await tap_bottom_right_of_element(target.bounds, ctx=ctx) - logger.debug("Tapped end of input by coordinates") - return elt - - if target.text: - text_elt = find_element_by_text( - state.latest_ui_hierarchy or [], - target.text, - index=target.text_index, - ) - if text_elt: - bounds = get_bounds_for_element(text_elt) - if bounds: - await tap_bottom_right_of_element(bounds=bounds, ctx=ctx) - logger.debug(f"Tapped end of input that had text'{target.text}'") - return text_elt - return None - - return None - - -async def focus_element_if_needed( - ctx: ArtemisContext, target: Target -) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None: - """Ensures the element is focused, with a sanity check to prevent trusting misleading IDs. - - If no target locator is provided (empty target), assumes the field is - already focused - and returns "already_focused" to allow typing without re-focusing. - """ - # If no locator is provided, assume already focused (e.g., keyboard is visible) - if not target.resource_id and not target.bounds and not target.text: - logger.debug("No target locator provided, assuming element is already focused") - return "already_focused" - - controller = UnifiedMobileController(ctx) - rich_hierarchy = await controller.get_ui_elements() - elt_from_id = None - if target.resource_id: - elt_from_id = find_element_by_resource_id( - ui_hierarchy=rich_hierarchy, - resource_id=target.resource_id, - index=target.resource_id_index, - is_rich_hierarchy=False, - ) - - if elt_from_id and target.text: - text_from_id_elt = get_element_text(elt_from_id) - if not text_from_id_elt or target.text.lower() != text_from_id_elt.lower(): - logger.warning( - f"ID '{target.resource_id}' and text '{target.text}' seem to be" - " on different elements. Ignoring the resource_id and falling" - " back to other locators." - ) - elt_from_id = None - - if elt_from_id: - if not is_element_focused(elt_from_id): - await tap( - ctx=ctx, - selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore - index=target.resource_id_index, - ) - logger.debug(f"Focused (tap) on resource_id={target.resource_id}") - rich_hierarchy = await controller.get_ui_elements() - elt_from_id = find_element_by_resource_id( - ui_hierarchy=rich_hierarchy, - resource_id=target.resource_id, # type: ignore - index=target.resource_id_index, - is_rich_hierarchy=False, - ) - if elt_from_id and is_element_focused(elt_from_id): - logger.debug(f"Text input is focused: {target.resource_id}") - return "resource_id" - logger.warning(f"Failed to focus using resource_id='{target.resource_id}'. Fallback...") - - if target.bounds: - relative_point = target.bounds.get_center() - await tap( - ctx=ctx, - selector_request=SelectorRequestWithCoordinates( - coordinates=CoordinatesSelectorRequest(x=relative_point.x, y=relative_point.y) - ), - ) - logger.debug(f"Tapped on coordinates ({relative_point.x}, {relative_point.y}) to focus.") - return "coordinates" - - if target.text: - text_elt = find_element_by_text(rich_hierarchy, target.text, index=target.text_index) - if text_elt: - bounds = get_bounds_for_element(text_elt) - if bounds: - relative_point = bounds.get_center() - await tap( - ctx=ctx, - selector_request=SelectorRequestWithCoordinates( - coordinates=CoordinatesSelectorRequest( - x=relative_point.x, y=relative_point.y - ) - ), - ) - logger.debug(f"Tapped on text element '{target.text}' to focus.") - return "text" - - logger.error( - "Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded." - ) - return None - - -def validate_coordinates_bounds( - target: Target, screen_width: int, screen_height: int -) -> str | None: - """Validate that coordinates are within screen bounds. - - Returns error message if invalid, None if valid. - """ - if not target.bounds: - return None - - center = target.bounds.get_center() - errors = [] - - if center.x < 0 or center.x >= screen_width: - errors.append(f"x={center.x} is outside screen width (0-{screen_width})") - if center.y < 0 or center.y >= screen_height: - errors.append(f"y={center.y} is outside screen height (0-{screen_height})") - - return "; ".join(errors) if errors else None - - -def has_valid_selectors(target: Target) -> bool: - """Check if target has at least one valid selector.""" - has_coordinates = target.bounds is not None - has_resource_id = target.resource_id is not None and target.resource_id != "" - has_text = target.text is not None and target.text != "" - return has_coordinates or has_resource_id or has_text - - -class IdSelectorRequest(BaseModel): - """Selector request using resource id.""" - - model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) - id: str - - def to_dict(self) -> dict[str, str | int]: - """Convert selector to dictionary.""" - return {"id": self.id} - - -class TextSelectorRequest(BaseModel): - """Selector request using element text.""" - - model_config = ConfigDict(extra="forbid") - text: str - - def to_dict(self) -> dict[str, str | int]: - """Convert selector to dictionary.""" - return {"text": self.text} - - -class SelectorRequestWithCoordinates(BaseModel): - """Selector request using absolute coordinates.""" - - model_config = ConfigDict(extra="forbid") - coordinates: CoordinatesSelectorRequest - - def to_dict(self) -> dict[str, str | int]: - """Convert selector to dictionary.""" - return {"point": self.coordinates.to_str()} - - -class SelectorRequestWithPercentages(BaseModel): - """Selector request using percentage coordinates.""" - - model_config = ConfigDict(extra="forbid") - percentages: PercentagesSelectorRequest - - def to_dict(self) -> dict[str, str | int]: - """Convert selector to dictionary.""" - return {"point": self.percentages.to_str()} - - -class IdWithTextSelectorRequest(BaseModel): - """Selector request using both resource id and text.""" - - model_config = ConfigDict(extra="forbid") - id: str - text: str - - def to_dict(self) -> dict[str, str | int]: - """Convert selector to dictionary.""" - return {"id": self.id, "text": self.text} - - -SelectorRequest = ( - IdSelectorRequest - | SelectorRequestWithCoordinates - | SelectorRequestWithPercentages - | TextSelectorRequest - | IdWithTextSelectorRequest -) - - -def _extract_resource_id_and_text_from_selector( - selector: SelectorRequest, -) -> tuple[str | None, str | None]: - """Extract resource_id and text from a selector.""" - resource_id = None - text = None - - if isinstance(selector, IdSelectorRequest): - resource_id = selector.id - elif isinstance(selector, TextSelectorRequest): - text = selector.text - elif isinstance(selector, IdWithTextSelectorRequest): - resource_id = selector.id - text = selector.text - - return resource_id, text - - -async def tap( - ctx: ArtemisContext, - selector_request: SelectorRequest, - index: int | None = None, -): - """Tap on a selector. - - Index is optional and is used when you have multiple views matching the same - selector. ui_hierarchy is optional and used for ADB taps to find elements. - """ - controller = UnifiedMobileController(ctx) - if isinstance(selector_request, SelectorRequestWithCoordinates): - result = await controller.tap_at( - x=selector_request.coordinates.x, y=selector_request.coordinates.y - ) - return result.error if result.error else None - if isinstance(selector_request, SelectorRequestWithPercentages): - coords = selector_request.percentages.to_coords( - width=ctx.device.device_width, - height=ctx.device.device_height, - ) - return await controller.tap_at(coords.x, coords.y) - - # For other selectors, we need the UI hierarchy - resource_id, text = _extract_resource_id_and_text_from_selector(selector_request) - - return await controller.tap_element( - resource_id=resource_id, - text=text, - index=index if index is not None else 0, - ) diff --git a/artemis/tools/video_tool.py b/artemis/tools/video_tool.py index da4209fbf..7685a9450 100644 --- a/artemis/tools/video_tool.py +++ b/artemis/tools/video_tool.py @@ -34,8 +34,8 @@ from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/tools/wait_tool.py b/artemis/tools/wait_tool.py index 96371d667..f7cf89fb8 100644 --- a/artemis/tools/wait_tool.py +++ b/artemis/tools/wait_tool.py @@ -23,8 +23,8 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/utils/app_launch_utils.py b/artemis/utils/app_launch_utils.py index 6a9ecfa65..56f90240e 100644 --- a/artemis/utils/app_launch_utils.py +++ b/artemis/utils/app_launch_utils.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """Utilities for handling app locking and initial app launch logic.""" @@ -21,14 +18,12 @@ from dataclasses import dataclass, field import re -from artemis.context import AppLaunchResult, ArtemisContext -from artemis.controllers.platform_specific_commands_controller import ( +from artemis.context import ArtemisContext +from third_party.mobile_use.controllers.platform_specific_commands_controller import ( get_adb_device, get_current_foreground_package_async, ) -from artemis.controllers.unified_controller import UnifiedMobileController -from artemis.data_engine.trace import TraceSpan -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -185,7 +180,7 @@ async def get_foreground_task_async(ctx: ArtemisContext) -> ForegroundTask | Non return await asyncio.to_thread(get_foreground_task, ctx) -async def _observe_foreground( +async def observe_foreground( ctx: ArtemisContext, app_package: str ) -> tuple[bool, str | None, ForegroundTask | None]: """Report whether ``app_package`` is in the foreground. @@ -207,190 +202,6 @@ async def _observe_foreground( return task is not None and task.owns(app_package), current_package, task -async def _poll_for_app_ready( - ctx: ArtemisContext, - app_package: str, - max_poll_seconds: int = 15, - poll_interval: float = 1.0, -) -> tuple[bool, str | None]: - """Poll for app to be ready after launch. - - Treats mCurrentFocus=null as a loading state and keeps polling. - Only fails if we get a different (non-null) package or timeout. - - Args: - ctx: Mobile use context - app_package: Expected package name - max_poll_seconds: Maximum time to poll (default: 15s) - poll_interval: Time between polls (default: 1s) - - Returns: - Tuple of (success: bool, error_message: str | None) - """ - polls = int(max_poll_seconds / poll_interval) - - for i in range(polls): - ready, current_package, task = await _observe_foreground(ctx, app_package) - - if ready: - if current_package == app_package: - logger.success(f"App {app_package} is ready (took ~{i * poll_interval:.1f}s)") - else: - logger.success( - f"App {app_package} is ready (focused window belongs to" - f" '{current_package}', but the foreground {task.describe()} is owned" - f" by {app_package}, took ~{i * poll_interval:.1f}s)" - ) - return True, None - - if current_package is None: - logger.debug(f"Poll {i + 1}/{polls}: App loading (mCurrentFocus=null)...") - else: - logger.debug( - f"Poll {i + 1}/{polls}: Wrong app in foreground (expected" - f" '{app_package}', got '{current_package}', foreground" - f" {task.describe() if task else 'task unknown'}). Still waiting..." - ) - - if i < polls - 1: - await asyncio.sleep(poll_interval) - - current_package = await get_current_foreground_package_async(ctx) - task = await get_foreground_task_async(ctx) - error_msg = ( - f"Timeout waiting for {app_package} to load after {max_poll_seconds}s. " - f"Current foreground: {current_package}; foreground" - f" {task.describe() if task else 'task unknown'}" - ) - logger.error(error_msg) - return False, error_msg - - -async def launch_app_with_retries( - ctx: ArtemisContext, - app_package: str, - max_retries: int = 3, - max_poll_seconds: int = 15, -) -> tuple[bool, str | None]: - """Launch an app with retry logic and smart polling. - - Args: - ctx: Mobile use context - app_package: Package name (Android) to launch - max_retries: Maximum number of launch attempts (default: 3) - max_poll_seconds: Maximum time to wait for app to load per attempt - (default: 15s) - - Returns: - Tuple of (success: bool, error_message: str | None) - """ - - for attempt in range(1, max_retries + 1): - logger.info(f"Launch attempt {attempt}/{max_retries} for app {app_package}") - - with TraceSpan( - name=f"Launch Attempt {attempt}", - trace_type="span", - ctx=ctx, - ) as span: - span.payload = {"attempt": attempt, "app_package": app_package} - - controller = UnifiedMobileController(ctx) - if attempt > 1: - logger.warning( - f"Attempt {attempt - 1} failed. Force stopping" - f" '{app_package}' to clear frozen state before retrying..." - ) - await controller.terminate_app(app_package) - await asyncio.sleep(1.0) - - launch_success = await controller.launch_app(app_package) - if not launch_success: - error_msg = f"Failed to execute launch command for {app_package}" - logger.error(error_msg) - span.status = "failed" - span.error = error_msg - if attempt == max_retries: - return False, error_msg - await asyncio.sleep(2) - continue - - await asyncio.sleep(1) - - success, error_msg = await _poll_for_app_ready(ctx, app_package, max_poll_seconds) - - if success: - span.status = "success" - span.result = "App is ready" - return True, None - - span.status = "failed" - span.error = error_msg - - if attempt < max_retries: - logger.warning(f"Attempt {attempt} failed: {error_msg}. Retrying...") - await asyncio.sleep(1) - - error_msg = f"Failed to launch {app_package} after {max_retries} attempts" - logger.error(error_msg) - return False, error_msg - - -async def _handle_initial_app_launch( - ctx: ArtemisContext, - locked_app_package: str, -) -> AppLaunchResult: - """Handle initial app launch verification and launching if needed. - - If locked_app_package is set: - 1. Check if the app is already in the foreground - 2. If not, attempt to launch it (with retries) - 3. Return status with success/error information - - Args: - ctx: Mobile use context - locked_app_package: Package name (Android) to lock to - - Returns: - AppLaunchResult with launch status and error information - """ - if not locked_app_package: - error_msg = f"Invalid locked_app_package: '{locked_app_package}'" - logger.error(error_msg) - return AppLaunchResult( - locked_app_package=locked_app_package, - locked_app_initial_launch_success=False, - locked_app_initial_launch_error=error_msg, - ) - - logger.info(f"Starting initial app launch for package: {locked_app_package}") - - try: - already_foreground, current_package, _ = await _observe_foreground(ctx, locked_app_package) - logger.info(f"Current foreground app: {current_package}") - - if already_foreground: - logger.info(f"App {locked_app_package} is already in foreground") - return AppLaunchResult( - locked_app_package=locked_app_package, - locked_app_initial_launch_success=True, - locked_app_initial_launch_error=None, - ) - - logger.info(f"App {locked_app_package} not in foreground, attempting to launch") - success, error_msg = await launch_app_with_retries(ctx, locked_app_package) - - return AppLaunchResult( - locked_app_package=locked_app_package, - locked_app_initial_launch_success=success, - locked_app_initial_launch_error=error_msg, - ) - - except Exception as e: - error_msg = f"Exception during initial app launch: {str(e)}" - logger.error(error_msg) - return AppLaunchResult( - locked_app_package=locked_app_package, - locked_app_initial_launch_success=False, - locked_app_initial_launch_error=error_msg, - ) +async def current_foreground(ctx: ArtemisContext) -> tuple[str | None, ForegroundTask | None]: + """Return the focused window's package and the foreground task, for diagnostics.""" + return await get_current_foreground_package_async(ctx), await get_foreground_task_async(ctx) diff --git a/artemis/utils/cli_selection.py b/artemis/utils/cli_selection.py deleted file mode 100644 index 8b288c5a5..000000000 --- a/artemis/utils/cli_selection.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -import inquirer -from rich.console import Console -from rich.prompt import Prompt -from rich.text import Text - - -def select_provider_and_model( - console: Console, - available_providers: list[str], - available_models: dict, - default_provider: str, - default_model: str, - provider: str | None = None, - model: str | None = None, -) -> tuple[str, str]: - """Interactive selection of LLM provider and model with arrow-key dropdowns when available. - - Args: - console: Rich console for output - available_providers: List of available provider names - available_models: Dict mapping providers to their available models - default_provider: Default provider to use - default_model: Default model to use - provider: Pre-selected provider (optional) - model: Pre-selected model (optional) - - Returns: - Tuple of (selected_provider, selected_model) - """ - final_provider = provider - final_model = model - - # Interactive provider selection - if not final_provider: - console.print("\n🤖 [bold cyan]LLM Provider Selection[/bold cyan]") - final_provider = _select_from_list( - console=console, - item_type="provider", - choices=available_providers, - default=default_provider, - message="Select LLM provider", - ) - - # Interactive model selection - if not final_model: - console.print(f"\n🎯 [bold green]Model Selection for {final_provider}[/bold green]") - available_model_list = ( - available_models[final_provider] - if final_provider - else available_models[default_provider] - ) - - default_model_for_provider = ( - default_model if default_model in available_model_list else available_model_list[0] - ) - - final_model = _select_from_list( - console=console, - item_type="model", - choices=available_model_list, - default=default_model_for_provider, - message=f"Select model for {final_provider}", - ) - - return final_provider, final_model - - -def _select_from_list( - console: Console, - item_type: str, - choices: list[str], - default: str, - message: str, -) -> str: - """Select an item from a list using arrow keys when available, fallback to numbered selection. - - Args: - console: Rich console for output - item_type: Type of item being selected (for error messages) - choices: List of choices to select from - default: Default choice - message: Message to display in dropdown - - Returns: - Selected choice - """ - # Try arrow-key dropdown if TTY is available, fallback to numbered selection - if sys.stdin.isatty(): - try: - questions = [ - inquirer.List( - "selection", - message=f"{message} (use arrow keys)", - choices=choices, - default=default, - ), - ] - answers = inquirer.prompt(questions) - return answers["selection"] if answers else default - except Exception: - # Fallback to numbered selection - return _numbered_selection(console, item_type, choices, default) - else: - return _numbered_selection(console, item_type, choices, default) - - -def _numbered_selection(console: Console, item_type: str, choices: list[str], default: str) -> str: - """Fallback numbered selection when arrow keys aren't available.""" - choices_text = "\n".join([f" {i + 1}. {choice}" for i, choice in enumerate(choices)]) - console.print(f"Available {item_type}s:\n{choices_text}") - - default_idx = choices.index(default) + 1 if default in choices else 1 - - while True: - choice = Prompt.ask( - f"Select {item_type} (1-{len(choices)}) or press Enter for default", - default=str(default_idx), - ) - try: - choice_idx = int(choice) - 1 - if 0 <= choice_idx < len(choices): - return choices[choice_idx] - else: - console.print("[red]Invalid choice. Please try again.[/red]") - except ValueError: - console.print("[red]Please enter a number.[/red]") - - -def display_llm_config(console: Console, provider: str, model: str) -> None: - """Display the selected LLM configuration with colors.""" - config_text = Text() - config_text.append("🤖 LLM Configuration: ", style="bold white") - config_text.append("Provider: ", style="white") - config_text.append(f"{provider}", style="bold cyan") - config_text.append(" | Model: ", style="white") - config_text.append(f"{model}", style="bold green") - - console.print(config_text) diff --git a/artemis/utils/conversations.py b/artemis/utils/conversations.py deleted file mode 100644 index c7a2058cb..000000000 --- a/artemis/utils/conversations.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import TypeGuard - -from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage - - -def is_ai_message(message: BaseMessage) -> TypeGuard[AIMessage]: - return isinstance(message, AIMessage) - - -def is_human_message(message: BaseMessage) -> TypeGuard[HumanMessage]: - return isinstance(message, HumanMessage) - - -def is_tool_message(message: BaseMessage) -> TypeGuard[ToolMessage]: - return isinstance(message, ToolMessage) - - -def is_tool_for_name(tool_message: ToolMessage, name: str) -> bool: - return tool_message.name == name - - -def get_screenshot_message_for_llm(screenshot_base64: str): - prefix = "" if screenshot_base64.startswith("data:image") else "data:image/jpeg;base64," - return HumanMessage( - content=[ - { - "type": "image_url", - "image_url": {"url": f"{prefix}{screenshot_base64}"}, - } - ] - ) diff --git a/artemis/utils/credentials_validator.py b/artemis/utils/credentials_validator.py index e71e8ba49..29b224e60 100644 --- a/artemis/utils/credentials_validator.py +++ b/artemis/utils/credentials_validator.py @@ -18,7 +18,7 @@ from artemis.llm.google.provider import is_google_family_provider -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/utils/cython_compat.py b/artemis/utils/cython_compat.py new file mode 100644 index 000000000..4008a47cd --- /dev/null +++ b/artemis/utils/cython_compat.py @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pydantic support for Cython builds. + +With ``USE_CYTHON=1``, methods defined in a class body compile to +``cyfunction`` objects, which pydantic would take for fields. Models add +:class:`CyFunctionDetector` to ``model_config["ignored_types"]`` to prevent it. + +The models in ``third_party/`` are not compiled but still need the setting: +compiled Artemis subclasses inherit their ``model_config``, for example +``artemis.config.llm.LLMConfig`` from ``LLMConfigBase``. +""" + + +class _CyFunctionDetectorMeta(type): + """Metaclass for detecting cython compiled functions.""" + + def __instancecheck__(cls, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +# pylint: disable=too-few-public-methods +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + """Detector for cython compiled functions.""" diff --git a/artemis/utils/image_hash.py b/artemis/utils/image_hash.py index a42400141..f85f050ae 100644 --- a/artemis/utils/image_hash.py +++ b/artemis/utils/image_hash.py @@ -26,7 +26,7 @@ import io -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/utils/notes.py b/artemis/utils/notes.py index 855b39a0c..c668827ff 100644 --- a/artemis/utils/notes.py +++ b/artemis/utils/notes.py @@ -20,8 +20,8 @@ from artemis.core.tool_failure import ToolFailure from artemis.data_engine.engine import _CURRENT_DATA_ENGINE -from artemis.utils.logger import get_logger from artemis.utils.plan_grammar import parse_plan +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/utils/ocr_api.py b/artemis/utils/ocr_api.py index b56adf55d..56b7c2356 100644 --- a/artemis/utils/ocr_api.py +++ b/artemis/utils/ocr_api.py @@ -17,7 +17,7 @@ import httpx from artemis.config import settings -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) _HTTP_CLIENT: httpx.AsyncClient | None = None diff --git a/artemis/utils/ocr_xml_fusion.py b/artemis/utils/ocr_xml_fusion.py index 57d69db85..ab8ee5d5c 100644 --- a/artemis/utils/ocr_xml_fusion.py +++ b/artemis/utils/ocr_xml_fusion.py @@ -17,8 +17,8 @@ from io import BytesIO from typing import Any -from artemis.utils.logger import get_logger from artemis.utils.ui_filter import _parse_bounds +from third_party.mobile_use.utils.logger import get_logger from PIL import Image logger = get_logger(__name__) diff --git a/artemis/utils/recorder.py b/artemis/utils/recorder.py deleted file mode 100644 index 70dbee134..000000000 --- a/artemis/utils/recorder.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base64 -import time - -from langchain_core.messages import BaseMessage -from artemis.context import ArtemisContext -from artemis.controllers.controller_factory import create_device_controller -from artemis.utils.logger import get_logger - -logger = get_logger(__name__) - - -async def record_interaction(ctx: ArtemisContext, response: BaseMessage): - if not ctx.execution_setup: - raise ValueError("No execution setup found") - if not ctx.execution_setup.traces_path or not ctx.execution_setup.trace_name: - raise ValueError("No traces path or trace name found") - - logger.info("Recording interaction") - controller = create_device_controller(ctx) - screenshot_base64 = await controller.screenshot() - logger.info("Screenshot taken") - try: - controller = create_device_controller(ctx) - compressed_screenshot_base64 = controller.get_compressed_b64_screenshot(screenshot_base64) - except Exception as e: - logger.error(f"Error compressing screenshot: {e}") - return "Could not record this interaction" - timestamp = time.time() - folder = ctx.execution_setup.traces_path.joinpath(ctx.execution_setup.trace_name).resolve() - folder.mkdir(parents=True, exist_ok=True) - try: - with open( - folder.joinpath(f"{int(timestamp)}.jpeg").resolve(), - "wb", - ) as f: - f.write(base64.b64decode(compressed_screenshot_base64)) - - with open( - folder.joinpath(f"{int(timestamp)}.json").resolve(), - "w", - encoding="utf-8", - ) as f: - f.write(response.model_dump_json()) - except Exception as e: - logger.error(f"Error recording interaction: {e}") - return "Screenshot recorded successfully" diff --git a/artemis/utils/requests_utils.py b/artemis/utils/requests_utils.py deleted file mode 100644 index aa56194da..000000000 --- a/artemis/utils/requests_utils.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from artemis.utils.logger import get_logger -import requests - -logger = get_logger(__name__) - - -def curl_from_request(req: requests.PreparedRequest) -> str: - """Converts a requests.PreparedRequest object to a valid cURL command string.""" - command = ["curl", f"-X {req.method}"] - - for key, value in req.headers.items(): - command.append(f'-H "{key}: {value}"') - - if req.body: - body = req.body - if isinstance(body, bytes): - body = body.decode("utf-8") - # Escape single quotes in the body for shell safety - body = body.replace("'", "'\\''") - command.append(f"-d '{body}'") - - command.append(f"'{req.url}'") - - return " ".join(command) - - -def logging_hook(response, *args, **kwargs): - """Hook to log the request as a cURL command.""" - curl_command = curl_from_request(response.request) - logger.debug(f"\n--- cURL Command ---\n{curl_command}\n--------------------") - - -def get_session_with_curl_logging() -> requests.Session: - """Returns a requests.Session with cURL logging enabled.""" - session = requests.Session() - session.hooks["response"] = [logging_hook] - return session diff --git a/artemis/utils/ui_filter.py b/artemis/utils/ui_filter.py index c68d768ed..64aebfe72 100644 --- a/artemis/utils/ui_filter.py +++ b/artemis/utils/ui_filter.py @@ -15,7 +15,7 @@ import re from typing import Any -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/artemis/utils/ui_hierarchy.py b/artemis/utils/ui_hierarchy.py deleted file mode 100644 index 611449716..000000000 --- a/artemis/utils/ui_hierarchy.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from artemis.utils.logger import get_logger -from pydantic import BaseModel, Field - -logger = get_logger(__name__) - - -def __find_element_by_resource_id_in_rich_hierarchy( - hierarchy: list[dict], resource_id: str -) -> dict | None: - """Retrieves all the sibling elements for a given resource ID from a nested dictionary. - - Args: - hierarchy (dict): The nested dictionary representing the UI hierarchy. - resource_id (str): The resource-id to find. - - Returns: - list: A list of the sibling elements, or None if the resource_id is not - found. - """ - if not hierarchy: - return None - - for child in hierarchy: - if child.get("attributes", {}).get("resource-id") == resource_id: - return child.get("attributes", {}) - - for child in hierarchy: - result = __find_element_by_resource_id_in_rich_hierarchy( - child.get("children", []), resource_id - ) - if result is not None: - return result - - return None - - -def text_input_is_empty(text: str | None, hint_text: str | None) -> bool: - return not text or text == hint_text - - -def find_element_by_resource_id( - ui_hierarchy: list[dict], - resource_id: str, - index: int | None = None, - is_rich_hierarchy: bool = False, -) -> dict | None: - """Find a UI element by its resource-id in the UI hierarchy. - - Args: - ui_hierarchy: List of UI element dictionaries - resource_id: The resource-id to search for (e.g., - "com.google.android.settings.intelligence:id/open_search_view_edit_text") - - Returns: - The complete UI element dictionary if found, None otherwise - """ - if is_rich_hierarchy: - return __find_element_by_resource_id_in_rich_hierarchy(ui_hierarchy, resource_id) - - target_idx = index or 0 - - def search_recursive(elements: list[dict]) -> dict | None: - nonlocal target_idx - for element in elements: - if isinstance(element, dict): - if element.get("resourceId") == resource_id: - if target_idx == 0: - return element - target_idx -= 1 - continue - - children = element.get("children", []) - if children: - result = search_recursive(children) - if result: - return result - return None - - return search_recursive(ui_hierarchy) - - -def is_element_focused(element: dict) -> bool: - return element.get("focused", None) == "true" - - -def get_element_text(element: dict, hint_text: bool = False) -> str | None: - if hint_text: - return element.get("hintText", None) - return element.get("text", None) - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class Point(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} - x: int - y: int - - -class ElementBounds(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} - x: int = Field(description="The x coordinate of the top-left corner of the element.") - y: int = Field(description="The y coordinate of the top-left corner of the element.") - width: int = Field(description="The width of the element.") - height: int = Field(description="The height of the element.") - - def get_center(self) -> Point: - return Point(x=self.x + self.width // 2, y=self.y + self.height // 2) - - def get_relative_point(self, x_percent: float, y_percent: float) -> Point: - """Returns the coordinates of the point at x_percent of the width and y_percent - - of the height of the element. - - Ex if x_percent = 0.95 and y_percent = 0.95, - the point is at the bottom right of the element: - <------> - | | - | x| - <------> - """ - return Point( - x=int(self.x + self.width * x_percent), - y=int(self.y + self.height * y_percent), - ) - - -def get_bounds_for_element(element: dict) -> ElementBounds | None: - bounds = element.get("bounds", None) - if bounds: - try: - return ElementBounds(**bounds) - except Exception as e: - logger.error(f"Failed to validate bounds: {e}") - return None - return None diff --git a/artemis/utils/video.py b/artemis/utils/video.py index a069d1244..9c6ab31a8 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -11,52 +11,39 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. -"""Video recording utilities for mobile devices. +"""scrcpy recording, remuxing, timeline rendering and frame extraction. -Provides shared types and utilities for video recording across platforms. +Recording session types and the session registry live in +``third_party.mobile_use.utils.video``. """ import asyncio import importlib.util import json from pathlib import Path -import platform import re import shutil import time import subprocess from typing import Any -from uuid import UUID import cv2 -from pydantic import BaseModel, ConfigDict -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) -DEFAULT_MAX_DURATION_SECONDS = 900 # 15 minutes ANDROID_RECORDING_SEGMENT_SECONDS = 1800 # The recording marker closely tracks the first frame; startup measurements # put the fallback near three quarters of a second after process creation. SCRCPY_RECORDING_STARTED_MARKER = "Recording started" SCRCPY_STARTUP_FALLBACK_SECONDS = 0.75 SCRCPY_STARTUP_TIMEOUT_SECONDS = 6.0 -VIDEO_READY_DELAY_SECONDS = 1 -ANDROID_DEVICE_VIDEO_PATH = "/sdcard/screen_recording.mp4" -ANDROID_MAX_RECORDING_DURATION_SECONDS = 180 # Android screenrecord limit # Ignore small timing differences at segment boundaries. TIMELINE_GAP_EPSILON_SECONDS = 0.05 -# Expanded for Gemini File API (Supports up to 2GB). -# Target 100MB to allow 3-5min crisp video and prevent blurring for long durations. -MAX_VIDEO_SIZE_MB = 500 -MAX_VIDEO_SIZE_BYTES = MAX_VIDEO_SIZE_MB * 1024 * 1024 - def build_scrcpy_record_command( scrcpy_executable: str, @@ -126,94 +113,6 @@ async def await_scrcpy_first_frame( return max(spawned_at, time.time()) -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class RecordingSession(BaseModel): - """Tracks an active video recording session.""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - - video_id: UUID - device_id: str - start_time: float - process: Any = None - data_engine_start_time: float | None = None - local_video_path: Path | None = None - capture_width: int | None = None - capture_height: int | None = None - android_device_path: str = ANDROID_DEVICE_VIDEO_PATH - android_video_segments: list[Path] = [] - android_segment_index: int = 0 - android_restart_task: asyncio.Task | None = None - watchdog_task: asyncio.Task | None = None - android_rotation: int | None = None - android_segment_started_at: float | None = None - android_segment_records: list[dict[str, Any]] = [] - android_conversion_tasks: list[asyncio.Task] = [] - generation: int = 0 - sealed_until: float = 0.0 - is_active: bool = True - errors: list[str] = [] - - -class VideoRecordingResult(BaseModel): - """Result of a video recording operation.""" - - success: bool - message: str - video_path: Path | None = None - file_size_mb: float | None = None - duration_seconds: float | None = None - actual_start_relative_time: float | None = None - warning: str | None = None - video_id: UUID | None = None - generation: int | None = None - sealed_until: float | None = None - source_revision: str | None = None - - model_config = ConfigDict(arbitrary_types_allowed=True) - - -# Global session storage - keyed by device_id -_active_recordings: dict[str, RecordingSession] = {} - - -def get_active_session(device_id: str) -> RecordingSession | None: - """Get the active recording session for a device.""" - return _active_recordings.get(device_id) - - -def set_active_session(device_id: str, session: RecordingSession) -> None: - """Set the active recording session for a device.""" - _active_recordings[device_id] = session - - -def remove_active_session(device_id: str) -> RecordingSession | None: - """Remove and return the active recording session for a device.""" - return _active_recordings.pop(device_id, None) - - -def has_active_session(device_id: str) -> bool: - """Check if there's an active recording session for a device.""" - return device_id in _active_recordings - - def is_ffmpeg_installed() -> bool: """Check if ffmpeg is available via imageio_ffmpeg or system PATH.""" @@ -667,80 +566,6 @@ def detect_video_tools_enabled() -> bool: return is_ffmpeg_installed() and is_scrcpy_installed() -class FFmpegNotInstalledError(Exception): - """Raised when ffmpeg is required but not installed.""" - - def __init__(self): - os_name = platform.system().lower() - if os_name == "darwin": # macOS - install_instructions = "brew install ffmpeg" - elif os_name == "windows": - install_instructions = "Download from https://www.ffmpeg.org/download.html" - else: # Linux and others - install_instructions = ( - "Install via your package manager (e.g., apt install ffmpeg," - " dnf install ffmpeg) or download from" - " https://www.ffmpeg.org/download.html" - ) - - message = ( - "\n\n❌ ffmpeg is required for video recording but is not" - " installed.\n\nPlease install ffmpeg first:\n →" - f" {install_instructions}\n\nAfter installation, restart Artemis.\n" - ) - super().__init__(message) - - -def check_ffmpeg_available() -> None: - """Check if ffmpeg is installed and raise an error if not. - - Raises: - FFmpegNotInstalledError: If ffmpeg is not found in PATH. - """ - if not is_ffmpeg_installed(): - raise FFmpegNotInstalledError() - - -async def concatenate_videos(segments: list[Path], output_path: Path) -> bool: - """Concatenate multiple video segments using ffmpeg.""" - if not segments: - return False - - if len(segments) == 1: - shutil.move(segments[0], output_path) - return True - - list_file = output_path.parent / "segments.txt" - with open(list_file, "w") as f: - for segment in segments: - f.write(f"file '{segment}'\n") - - try: - process = await asyncio.create_subprocess_exec( - get_ffmpeg_path(), - "-y", - "-f", - "concat", - "-safe", - "0", - "-i", - str(list_file), - "-c", - "copy", - str(output_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - await process.wait() - return output_path.exists() - except Exception as e: - logger.error(f"Failed to concatenate videos: {e}") - return False - finally: - if list_file.exists(): - list_file.unlink() - - _drawtext_supported: bool | None = None @@ -762,274 +587,6 @@ def is_ffmpeg_drawtext_supported() -> bool: return _drawtext_supported -async def trim_video( - input_path: Path, - start_time: float, - end_time: float | None, - output_path: Path, - fast_copy: bool = True, -) -> bool: - """Trim a video file using ffmpeg. - - Uses fast stream copy (-c copy) by default with fallback to re-encoding. - """ - try: - cmd = [ - get_ffmpeg_path(), - "-y", - "-ss", - str(start_time), - "-i", - str(input_path), - ] - if end_time is not None: - duration = end_time - start_time - cmd.extend(["-t", str(duration)]) - - if fast_copy: - cmd.extend(["-c", "copy", str(output_path)]) - else: - cmd.extend( - [ - "-vf", - "setpts=PTS-STARTPTS", - "-c:v", - "libx264", - "-pix_fmt", - "yuv420p", - str(output_path), - ] - ) - - process = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await process.communicate() - if process.returncode == 0 and output_path.exists(): - return True - - if fast_copy: - logger.warning( - f"ffmpeg fast copy trim failed (code {process.returncode})," - f" falling back to re-encoding: {stderr.decode()[:200]}" - ) - return await trim_video(input_path, start_time, end_time, output_path, fast_copy=False) - - logger.error(f"ffmpeg trim failed (code {process.returncode}): {stderr.decode()}") - return False - except Exception as e: - logger.error(f"Failed to trim video: {e}") - return False - - -def cleanup_video_segments(segments: list[Path], keep_path: Path | None = None) -> None: - """Clean up temporary video segments, optionally keeping one path.""" - for segment in segments: - try: - if segment.exists() and segment != keep_path: - segment.unlink() - if segment.parent.exists() and not any(segment.parent.iterdir()): - segment.parent.rmdir() - except OSError: - # Best-effort cleanup of temporary segments; leftovers are harmless. - pass - - -async def compress_video_for_api( - input_path: Path, - target_size_bytes: int = MAX_VIDEO_SIZE_BYTES, - force_compress: bool = False, - start_offset_seconds: float = 0.0, - slowdown_factor: float = 1.0, -) -> Path: - """Compress a video to fit within API size limits using ffmpeg. - - Uses a two-pass approach: - 1. First check if video is already small enough - 2. If not, compress with reduced resolution and bitrate - - Args: - input_path: Path to the input video file - target_size_bytes: Target maximum file size in bytes - force_compress: If True, always perform compression (e.g., to extract - frames at 15fps) - start_offset_seconds: Offset to add to the burned-in timestamp - slowdown_factor: Factor to slow down the video by (e.g. 5.0 for 5x - slower) to increase API frame sampling rate - - Returns: - Path to the compressed video (may be same as input if no compression - needed) - """ - if not input_path.exists(): - raise FileNotFoundError(f"Video file not found: {input_path}") - - current_size = input_path.stat().st_size - logger.info(f"Video size: {current_size / 1024 / 1024:.2f} MB") - - if current_size <= target_size_bytes and not force_compress and slowdown_factor == 1.0: - logger.info( - "Video already within size limit and force_compress=False, no compression needed" - ) - return input_path - - logger.info( - f"Compressing video to fit within {target_size_bytes / 1024 / 1024:.1f}" - f" MB (slowdown={slowdown_factor})" - ) - - output_path = input_path.parent / f"compressed_{input_path.name}" - - # Use Constant Rate Factor (CRF) for high-fidelity UI text readability. - # CRF dynamically allocates bitrates depending on scene motion. - logger.info("Compressing video using CRF=26 for high-fidelity UI text readability.") - - # Compress with ffmpeg: reduce resolution to 720p max, use CRF - # and check if drawtext is supported - vf_parts = ["setpts=PTS-STARTPTS", "scale='min(720,iw)':'-2'"] - if is_ffmpeg_drawtext_supported(): - vf_parts.append( - "drawtext=text='TS\\:" - f" %{{expr_int_format\\:trunc(t+{start_offset_seconds})\\:d}}" - " s':x=w-tw-30:y=120+th+20:fontcolor=white@0.6:fontsize=44:borderw=4:" - "bordercolor=red@0.6:box=1:boxcolor=yellow@0.4:boxborderw=10:font='Sans" - " Bold'" - ) - else: - logger.warning( - "ffmpeg 'drawtext' filter not supported on this system. Skipping" - " burned-in timestamp overlay." - ) - - if slowdown_factor != 1.0: - vf_parts.append(f"setpts={slowdown_factor}*PTS") - - vf_filter = ",".join(vf_parts) - - compress_cmd = [ - get_ffmpeg_path(), - "-y", - "-i", - str(input_path), - "-vf", - vf_filter, - "-r", - "15", # Set framerate to 15 fps - "-c:v", - "libx264", - "-preset", - "fast", - "-crf", - "26", # High quality for UI elements - "-pix_fmt", - "yuv420p", - "-c:a", - "aac", - "-b:a", - "64k", - str(output_path), - ] - - try: - proc = await asyncio.create_subprocess_exec( - *compress_cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - - if proc.returncode != 0: - err_msg = stderr.decode().strip() - logger.error(f"ffmpeg compression failed (code {proc.returncode}): {err_msg}") - if force_compress or slowdown_factor != 1.0: - raise RuntimeError( - f"Video compression/slowdown failed (code {proc.returncode}): {err_msg}" - ) - return input_path # Return original if compression fails and was optional - - new_size = output_path.stat().st_size - logger.info( - f"Compressed: {current_size / 1024 / 1024:.2f} MB -> {new_size / 1024 / 1024:.2f} MB" - ) - - return output_path - - except Exception as e: - logger.error(f"Video compression failed: {e}") - if force_compress or slowdown_factor != 1.0: - raise RuntimeError(f"Video compression/slowdown failed: {e}") - return input_path # Return original if compression fails and was optional - - -async def extract_audio_from_video(input_path: Path) -> Path: - """Extract audio from a video file using ffmpeg. - - Saves as .mp3. - """ - if not input_path.exists(): - raise FileNotFoundError(f"Video file not found: {input_path}") - - probe_cmd = [ - get_ffprobe_path(), - "-v", - "error", - "-select_streams", - "a:0", - "-show_entries", - "stream=index", - "-of", - "csv=p=0", - str(input_path), - ] - probe = await asyncio.create_subprocess_exec( - *probe_cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - probe_stdout, _ = await probe.communicate() - if probe.returncode != 0 or not probe_stdout.strip(): - raise ValueError("Video has no audio stream") - - output_path = input_path.parent / f"audio_{input_path.stem}.mp3" - logger.info(f"Extracting audio from {input_path} to {output_path}") - - cmd = [ - get_ffmpeg_path(), - "-y", - "-i", - str(input_path), - "-vn", - "-c:a", - "libmp3lame", - "-b:a", - "128k", - str(output_path), - ] - - try: - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - - if proc.returncode != 0: - err_msg = stderr.decode(errors="replace").strip() - concise_error = "\n".join(err_msg.splitlines()[-8:]) - logger.error(f"ffmpeg audio extraction failed: {concise_error}") - raise RuntimeError( - f"ffmpeg audio extraction failed (code {proc.returncode}): {concise_error}" - ) - - return output_path - - except Exception: - raise - - def extract_keyframes_from_video( video_path: Path | str, fps: float = 1.0, diff --git a/mcp_server/tools/diagnose.py b/mcp_server/tools/diagnose.py index c62b52263..4f336463d 100644 --- a/mcp_server/tools/diagnose.py +++ b/mcp_server/tools/diagnose.py @@ -46,7 +46,7 @@ from artemis.runtime import DeviceExecutionLock, trace_store from artemis.runtime.helper_manager import helper_manager from artemis.utils.credentials_validator import validate_api_key -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) diff --git a/scripts/quality_ratchet.py b/scripts/quality_ratchet.py index ea6b135c1..2eafbc256 100644 --- a/scripts/quality_ratchet.py +++ b/scripts/quality_ratchet.py @@ -14,7 +14,12 @@ ROOT = Path(__file__).resolve().parents[1] -SOURCE_ROOTS = (ROOT / "artemis", ROOT / "mcp_server", ROOT / "apps" / "admin_console") +SOURCE_ROOTS = ( + ROOT / "artemis", + ROOT / "mcp_server", + ROOT / "apps" / "admin_console", + ROOT / "third_party", +) BASELINE_PATH = ROOT / ".quality-baseline.json" diff --git a/scripts/real_device_video_acceptance.py b/scripts/real_device_video_acceptance.py index 3bb905d83..4df14cfef 100644 --- a/scripts/real_device_video_acceptance.py +++ b/scripts/real_device_video_acceptance.py @@ -37,7 +37,7 @@ from artemis.runtime import DeviceExecutionLock from artemis.sdk.builders import Builders from artemis.sdk.types import AgentProfile -from artemis.clients.ui_automator_client import UIAutomatorClient +from third_party.mobile_use.clients.ui_automator_client import UIAutomatorClient CALCULATOR_PACKAGE = "com.google.android.calculator" diff --git a/setup.py b/setup.py index e6465cad6..054260cbc 100644 --- a/setup.py +++ b/setup.py @@ -119,6 +119,7 @@ def make_release_tree(self, base_dir: str, files: list[str]) -> None: if module_name in ( "artemis.__init__", + "artemis._legacy_imports", "artemis.main", "artemis.mcp.adb_server", "artemis.mcp.xml_search_server", @@ -144,12 +145,16 @@ def make_release_tree(self, base_dir: str, files: list[str]) -> None: "apps", "apps.admin_console", "apps.admin_console.*", - ] + "third_party.*", + ], ), package_data={ "artemis": ["**/*.json", "**/*.md"], "artemis.resources": ["config/*.jsonc", "showcase_ui/*", "showcase_ui/**/*"], "apps.admin_console": ["index.html"], + # third_party has no __init__.py (namespace package), so it does not clash + # with other distributions that also ship a third_party directory. + "third_party.mobile_use": ["LICENSE", "NOTICE", "METADATA", "**/*.md"], }, include_package_data=False, ext_modules=ext_modules, diff --git a/tests/integration/test_physical_actions.py b/tests/integration/test_physical_actions.py index fd5487c9e..3ae74d0c0 100644 --- a/tests/integration/test_physical_actions.py +++ b/tests/integration/test_physical_actions.py @@ -22,9 +22,9 @@ sys.path.insert(0, str(root_dir)) from adbutils import AdbClient -from artemis.clients.ui_automator_client import UIAutomatorClient from artemis.context import DeviceContext, DevicePlatform, ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController +from third_party.mobile_use.clients.ui_automator_client import UIAutomatorClient import pytest diff --git a/tests/integration/test_remote_adb.py b/tests/integration/test_remote_adb.py index 359fba772..513bc5903 100644 --- a/tests/integration/test_remote_adb.py +++ b/tests/integration/test_remote_adb.py @@ -32,7 +32,7 @@ ) from artemis.context import ArtemisContext, DeviceContext, DevicePlatform -from artemis.controllers.platform_specific_commands_controller import ( +from third_party.mobile_use.controllers.platform_specific_commands_controller import ( get_adb_device, list_packages, get_current_foreground_package, diff --git a/tests/unit/agents/test_outputter.py b/tests/unit/agents/test_outputter.py index 668e5b3f8..8def19fa9 100644 --- a/tests/unit/agents/test_outputter.py +++ b/tests/unit/agents/test_outputter.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio from unittest.mock import AsyncMock, Mock, patch from pydantic import BaseModel @@ -21,73 +22,13 @@ from artemis.config import LLM, OutputConfig # noqa: E402 from artemis.context import ArtemisContext # noqa: E402 from artemis.core.tool_failure import ToolFailure # noqa: E402 -from artemis.utils.logger import get_logger # noqa: E402 +from third_party.mobile_use import testing as agent_fixtures +from third_party.mobile_use.utils.logger import get_logger # noqa: E402 logger = get_logger(__name__) - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class MockPydanticSchema(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} - color: str - price: float - currency_symbol: str - website_url: str - - -mock_dict = { - "color": "green", - "price": 20, - "currency_symbol": "$", - "website_url": "http://superwebsite.fr", -} - - -class DummyState: - def __init__(self, messages, initial_goal, operator_raw_data=None): - self.messages = messages - self.initial_goal = initial_goal - self.operator_raw_data = operator_raw_data - - -@pytest.fixture -def mock_context(): - """Create a properly mocked context with all required fields.""" - ctx = Mock(spec=ArtemisContext) - ctx.llm_config = { - "planner": LLM(provider="openai", model="gpt-5-nano"), - "operator": LLM(provider="openai", model="gpt-5-nano"), - "validator": LLM(provider="openai", model="gpt-5-nano"), - } - ctx.device = Mock() - ctx.data_engine = None - return ctx - - -@pytest.fixture -def mock_state(): - """Create a mock state with test data.""" - return DummyState( - messages=[], - initial_goal="Find a green product on my website", - ) +mock_context = agent_fixtures.mock_context +mock_state = agent_fixtures.mock_state def setup_mock_llm(mock_get_llm, react_response_content="Paris", structured_response=None): @@ -123,87 +64,45 @@ def get_llm_side_effect(ctx, name, is_utils=False, use_fallback=False): return mock_llm, mock_llm_with_tools, mock_structured_llm -@patch("artemis.agents.outputter.outputter.get_llm") -@pytest.mark.asyncio -async def test_outputter_with_pydantic_model(mock_get_llm, mock_context, mock_state): - """Test outputter with Pydantic model output.""" - expected_structured = MockPydanticSchema( - color="green", - price=20, - currency_symbol="$", - website_url="http://superwebsite.fr", - ) - setup_mock_llm( - mock_get_llm, - react_response_content="Raw details about green product", - structured_response=expected_structured, - ) - - config = OutputConfig( - structured_output=MockPydanticSchema, - output_description=None, - ) - - result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state) - - assert isinstance(result, dict) - assert result.get("color") == "green" - - -@patch("artemis.agents.outputter.outputter.get_llm") -@pytest.mark.asyncio -async def test_outputter_with_dict(mock_get_llm, mock_context, mock_state): - """Test outputter with dictionary output.""" - expected_dict = { - "color": "green", - "price": 20, - "currency_symbol": "$", - "website_url": "http://superwebsite.fr", - } - setup_mock_llm( - mock_get_llm, - react_response_content="Raw details", - structured_response=expected_dict, - ) - - config = OutputConfig( - structured_output=mock_dict, - output_description=None, - ) +class _Verdict(BaseModel): + achieved: bool + evidence: str - result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state) - assert isinstance(result, dict) - assert result.get("color") == "green" - assert result.get("price") == 20 - assert result.get("currency_symbol") == "$" - assert result.get("website_url") == "http://superwebsite.fr" +# A Pydantic model or a JSON schema dict; the result is a plain dict in both cases. +_FORMAT_CASES = [ + pytest.param(_Verdict, _Verdict(achieved=True, evidence="Order placed"), id="pydantic_model"), + pytest.param( + _Verdict.model_json_schema(), + {"achieved": True, "evidence": "Order placed"}, + id="json_schema", + ), +] @patch("artemis.agents.outputter.outputter.get_llm") @pytest.mark.asyncio -async def test_outputter_with_natural_language_output(mock_get_llm, mock_context, mock_state): - """Test outputter with natural language description output (returns JSON string).""" - expected_json = ( - '{"color": "green", "price": 20, "currency_symbol": "$", "website_url":' - ' "http://superwebsite.fr"}' +@pytest.mark.parametrize("schema, formatted_reply", _FORMAT_CASES) +async def test_outputter_applies_schema_in_separate_formatting_pass( + mock_get_llm, schema, formatted_reply, mock_context, mock_state +): + """The ReAct answer is re-shaped by a second, schema-bound call that sees the goal.""" + react_answer = "The confirmation screen reads 'Order placed'." + mock_llm, _, mock_structured_llm = setup_mock_llm( + mock_get_llm, react_response_content=react_answer, structured_response=formatted_reply ) - setup_mock_llm(mock_get_llm, react_response_content=expected_json) - config = OutputConfig( - structured_output=None, - output_description=( - "A JSON object with a color, a price, a currency_symbol and a website_url key" - ), + result = await outputter( + ctx=mock_context, + output_config=OutputConfig(structured_output=schema), + graph_output=mock_state, ) - result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state) - - assert isinstance(result, dict) - assert result.get("color") == "green" - assert result.get("price") == 20 - assert result.get("currency_symbol") == "$" - assert result.get("website_url") == "http://superwebsite.fr" + assert result == {"achieved": True, "evidence": "Order placed"} + mock_llm.with_structured_output.assert_called_once_with(schema) + format_request = mock_structured_llm.ainvoke.call_args.args[0][-1].content + assert mock_state.initial_goal in format_request + assert react_answer in format_request @patch("artemis.agents.outputter.outputter.get_llm") @@ -559,3 +458,289 @@ async def test_outputter_tool_message_status_is_structural( assert tool_msgs[0].tool_call_id == "c1" assert tool_msgs[0].status == expected_status assert tool_msgs[0].content == str(result) + + +# Runs that end without a normal answer. + + +def _tool_call(name: str, call_id: str, **args): # noqa: D103 + return {"name": name, "args": args, "id": call_id} + + +def _recording_tool(name: str, result, events: list, barrier=None): # noqa: D103 + from langchain_core.tools import StructuredTool + + async def _run(**_kwargs): + events.append(f"start:{name}") + if barrier is not None: + await asyncio.wait_for(barrier.wait(), timeout=2) + await asyncio.sleep(0) + events.append(f"end:{name}") + return result + + return StructuredTool.from_function(coroutine=_run, name=name, description=name) + + +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +async def test_outputter_returns_text_for_content_block_replies( + mock_get_llm, mock_context, mock_state +): + """Gemini 3+ replies arrive as content blocks; the caller gets their text.""" + from langchain_core.messages import AIMessage + + _, bound, _ = setup_mock_llm(mock_get_llm) + bound.ainvoke.return_value = AIMessage( + content=[ + {"type": "thinking", "thinking": "internal"}, + {"type": "text", "text": "The order was placed."}, + ] + ) + + result = await outputter( + ctx=mock_context, output_config=OutputConfig(), graph_output=mock_state + ) + + assert result == "The order was placed." + + +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +async def test_outputter_nudges_once_after_empty_reply(mock_get_llm, mock_context, mock_state): + from langchain_core.messages import AIMessage, HumanMessage + + _, bound, _ = setup_mock_llm(mock_get_llm) + bound.ainvoke.side_effect = [AIMessage(content=""), AIMessage(content="Done.")] + + result = await outputter( + ctx=mock_context, output_config=OutputConfig(), graph_output=mock_state + ) + + assert result == "Done." + assert bound.ainvoke.call_count == 2 + sent = bound.ainvoke.call_args_list[1].args[0] + assert isinstance(sent[2], HumanMessage) # the nudge, not the empty AI turn + assert not any(isinstance(m, AIMessage) and not m.content for m in sent) + + +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +async def test_outputter_nudges_only_once(mock_get_llm, mock_context, mock_state): + from langchain_core.messages import AIMessage + + _, bound, _ = setup_mock_llm(mock_get_llm) + bound.ainvoke.side_effect = [AIMessage(content=""), AIMessage(content=""), AssertionError()] + + result = await outputter( + ctx=mock_context, output_config=OutputConfig(), graph_output=mock_state + ) + + assert bound.ainvoke.call_count == 2 + assert result.startswith("Error: Outputter failed") + + +@patch("artemis.agents.outputter.outputter.MAX_TURNS", 2) +@patch("artemis.agents.outputter.outputter.get_read_note_tool_pure") +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +async def test_outputter_final_turn_runs_no_tools_and_returns_text( + mock_get_llm, mock_get_read_note, mock_context, mock_state +): + from langchain_core.messages import AIMessage, HumanMessage + + events: list[str] = [] + mock_get_read_note.return_value = _recording_tool("read_note", "note body", events) + _, bound, _ = setup_mock_llm(mock_get_llm) + bound.ainvoke.side_effect = [ + AIMessage(content="", tool_calls=[_tool_call("read_note", "c1", key="a")]), + AIMessage(content="Best answer so far.", tool_calls=[_tool_call("read_note", "c2")]), + ] + + result = await outputter( + ctx=mock_context, output_config=OutputConfig(), graph_output=mock_state + ) + + assert result == "Best answer so far." + assert events == ["start:read_note", "end:read_note"] # the final turn's call never ran + final_request = bound.ainvoke.call_args_list[1].args[0] + assert any( + isinstance(m, HumanMessage) and "final turn" in str(m.content) for m in final_request + ) + + +@patch("artemis.agents.outputter.outputter.MAX_TURNS", 2) +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +async def test_outputter_falls_back_to_saved_report( + mock_get_llm, mock_context, mock_state, tmp_path +): + """Without a final answer, the report written this run is returned, not an error.""" + from langchain_core.messages import AIMessage + + mock_context.data_engine = Mock(base_dir=str(tmp_path)) + _, bound, _ = setup_mock_llm(mock_get_llm) + report = "# Report\nStatus: SUCCESS" + bound.ainvoke.side_effect = [ + AIMessage( + content="", tool_calls=[_tool_call("save_note", "c1", key="output", content=report)] + ), + AIMessage(content=""), + ] + + result = await outputter( + ctx=mock_context, + output_config=OutputConfig(), + graph_output=mock_state, + plan_and_history="Step 1: placed order", + ) + + assert result == report + + +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +async def test_outputter_formatting_pass_sees_saved_report( + mock_get_llm, mock_context, mock_state, tmp_path +): + from langchain_core.messages import AIMessage + + mock_context.data_engine = Mock(base_dir=str(tmp_path)) + _, bound, structured = setup_mock_llm( + mock_get_llm, structured_response={"achieved": True, "evidence": "e"} + ) + bound.ainvoke.side_effect = [ + AIMessage( + content="", tool_calls=[_tool_call("save_note", "c1", key="output", content="REPORT")] + ), + AIMessage(content="Order placed."), + ] + + await outputter( + ctx=mock_context, + output_config=OutputConfig(structured_output=_Verdict), + graph_output=mock_state, + plan_and_history="Step 1", + ) + + format_messages = structured.ainvoke.call_args.args[0] + assert "REPORT" in format_messages[1].content + assert "Order placed." in format_messages[-1].content + assert mock_state.initial_goal in format_messages[-1].content + + +@patch("artemis.agents.outputter.outputter.get_save_note_tool_pure") +@patch("artemis.agents.outputter.outputter.get_history_tools") +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +async def test_outputter_runs_reads_concurrently_and_writes_in_order( + mock_get_llm, mock_history_tools, mock_get_save_note, mock_context, mock_state +): + from langchain_core.messages import AIMessage, ToolMessage + + events: list[str] = [] + barrier = asyncio.Barrier(2) # only passable if both reads are in flight together + mock_history_tools.return_value = ( + _recording_tool("search_history", "hit", events, barrier), + _recording_tool("replay_steps", "replay", events, barrier), + _recording_tool("get_step_screenshot", "no image", events), + ) + mock_get_save_note.return_value = _recording_tool("save_note", "saved", events) + _, bound, _ = setup_mock_llm(mock_get_llm) + calls = [ + _tool_call("search_history", "a"), + _tool_call("replay_steps", "b"), + _tool_call("save_note", "c", key="k", content="v"), + _tool_call("get_step_screenshot", "d"), + ] + bound.ainvoke.side_effect = [AIMessage(content="", tool_calls=calls), AIMessage("ok")] + + await outputter(ctx=mock_context, output_config=OutputConfig(), graph_output=mock_state) + + save_start = events.index("start:save_note") + assert {"end:search_history", "end:replay_steps"} <= set(events[:save_start]) + assert events.index("end:save_note") < events.index("start:get_step_screenshot") + sent = bound.ainvoke.call_args_list[1].args[0] + assert [m.tool_call_id for m in sent if isinstance(m, ToolMessage)] == ["a", "b", "c", "d"] + + +@patch("artemis.agents.outputter.outputter.get_history_tools") +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +async def test_outputter_puts_image_messages_after_all_tool_results( + mock_get_llm, mock_history_tools, mock_context, mock_state +): + """Providers that carry images in a HumanMessage need tool results contiguous.""" + from langchain_core.messages import AIMessage, HumanMessage, ToolMessage + + shot = [ + {"type": "text", "text": "Step 3 screenshot"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAAA"}}, + ] + events: list[str] = [] + mock_history_tools.return_value = ( + _recording_tool("search_history", "hit at step 3", events), + _recording_tool("replay_steps", "replay", events), + _recording_tool("get_step_screenshot", shot, events), + ) + mock_llm, bound, _ = setup_mock_llm(mock_get_llm) + mock_llm.endpoint.provider = "openai" + turn = AIMessage( + content="", + tool_calls=[_tool_call("get_step_screenshot", "a"), _tool_call("search_history", "b")], + ) + bound.ainvoke.side_effect = [turn, AIMessage("done")] + + await outputter(ctx=mock_context, output_config=OutputConfig(), graph_output=mock_state) + + sent = bound.ainvoke.call_args_list[1].args[0] + after = sent[sent.index(turn) + 1 : -1] + assert [type(m) for m in after] == [ToolMessage, ToolMessage, HumanMessage] + assert [m.tool_call_id for m in after[:2]] == ["a", "b"] + + +_DEGRADE_CASES = [ + pytest.param( + '{"achieved": true, "evidence": "Order placed"}', + {"achieved": True, "evidence": "Order placed"}, + id="parsed_locally", + ), + pytest.param("The order was placed.", "The order was placed.", id="text_answer"), +] + + +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +@pytest.mark.parametrize("answer, expected", _DEGRADE_CASES) +async def test_outputter_formatting_failure_degrades( + mock_get_llm, answer, expected, mock_context, mock_state +): + _, _, structured = setup_mock_llm(mock_get_llm, react_response_content=answer) + structured.ainvoke.side_effect = RuntimeError("schema rejected") + + result = await outputter( + ctx=mock_context, + output_config=OutputConfig(structured_output=_Verdict), + graph_output=mock_state, + ) + + assert result == expected + + +@patch("artemis.agents.outputter.outputter.get_llm") +@pytest.mark.asyncio +async def test_outputter_shows_target_schema_while_investigating( + mock_get_llm, mock_context, mock_state +): + _, bound, _ = setup_mock_llm( + mock_get_llm, structured_response={"achieved": True, "evidence": "e"} + ) + + await outputter( + ctx=mock_context, + output_config=OutputConfig(structured_output=_Verdict), + graph_output=mock_state, + ) + + prompt = bound.ainvoke.call_args.args[0][1].content[0]["text"] + assert "## Target Output Fields" in prompt + assert '"achieved"' in prompt and '"evidence"' in prompt diff --git a/tests/unit/clients/test_accessibility_client.py b/tests/unit/clients/test_accessibility_client.py index 2d722dc6b..bcee3e7ac 100644 --- a/tests/unit/clients/test_accessibility_client.py +++ b/tests/unit/clients/test_accessibility_client.py @@ -19,8 +19,8 @@ HelperUnavailable, normalize_helper_elements, ) -from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements from artemis.runtime.helper_manager import HelperSession +from third_party.mobile_use.clients.ui_automator_client import parse_hierarchy_xml_to_elements XML = ( '' @@ -217,7 +217,7 @@ def test_screen_data_is_one_snapshot_request_with_xml_only(client): assert data.base64 == "QUJD" and data.width == 1080 and data.height == 2424 assert data.hierarchy_xml == XML - assert data.elements == _parse_hierarchy_xml_to_elements(XML) + assert data.elements == parse_hierarchy_xml_to_elements(XML) element = data.elements[1] assert element["clickable"] == "true" assert element["visible-to-user"] == "true" @@ -246,7 +246,7 @@ def test_screen_data_without_screenshot_reuses_the_hierarchy_and_adds_screencap( assert urlopen.call_count == 1 # the hierarchy is not dumped a second time assert data.width == 4 and data.height == 6 assert Image.open(BytesIO(base64.b64decode(data.base64))).size == (4, 6) - assert data.elements == _parse_hierarchy_xml_to_elements(XML) + assert data.elements == parse_hierarchy_xml_to_elements(XML) def test_screen_data_falls_back_to_normalized_json_without_xml(client): diff --git a/tests/unit/clients/test_screen_client_factory.py b/tests/unit/clients/test_screen_client_factory.py index 836af3b94..6a3e1d4e9 100644 --- a/tests/unit/clients/test_screen_client_factory.py +++ b/tests/unit/clients/test_screen_client_factory.py @@ -17,8 +17,8 @@ hierarchy_backend_summary, resolve_backend, ) -from artemis.clients.ui_automator_client import UIAutomatorClient from artemis.config.constants import ENV_ARTEMIS_HIERARCHY_BACKEND +from third_party.mobile_use.clients.ui_automator_client import UIAutomatorClient # --------------------------------------------------------------------------- # diff --git a/tests/unit/data_engine/test_latency_optimizations_deep.py b/tests/unit/data_engine/test_latency_optimizations_deep.py index ee8caccfe..661b60548 100644 --- a/tests/unit/data_engine/test_latency_optimizations_deep.py +++ b/tests/unit/data_engine/test_latency_optimizations_deep.py @@ -24,7 +24,7 @@ from artemis.context import ArtemisContext from artemis.data_engine.engine import DataEngine from artemis.graph.state import State -from artemis.utils.logger import DataEngineHandler +from third_party.mobile_use.utils.logger import DataEngineHandler import pytest @@ -302,12 +302,12 @@ async def test_ocr_api_persistent_http_client_singleton_and_tls_reuse(): async def test_ui_filter_and_ui_automator_client_pre_parsed_bounds_o1_lookup(): """Verify XML parsing pre-populates parsed_bounds and _parse_bounds hits this cache in O(1) without regex matches.""" import re - from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements + from third_party.mobile_use.clients.ui_automator_client import parse_hierarchy_xml_to_elements from artemis.utils.ui_filter import _parse_bounds - # 1. Test ingestion pre-parsing in _parse_hierarchy_xml_to_elements + # 1. Test ingestion pre-parsing in parse_hierarchy_xml_to_elements xml_data = '' - elements = _parse_hierarchy_xml_to_elements(xml_data) + elements = parse_hierarchy_xml_to_elements(xml_data) assert len(elements) == 1 assert elements[0]["parsed_bounds"] == { "left": 15, diff --git a/tests/unit/mcp/test_mcp_stdio_lifecycle.py b/tests/unit/mcp/test_mcp_stdio_lifecycle.py index 7861f7a6e..05dea8801 100644 --- a/tests/unit/mcp/test_mcp_stdio_lifecycle.py +++ b/tests/unit/mcp/test_mcp_stdio_lifecycle.py @@ -29,10 +29,10 @@ import pytest -from artemis.clients import ui_automator_client from artemis.runtime.awake_lease import ScreenAwakeLease from artemis.runtime.awake_service import _run_awake_adb_command -from artemis.utils.logger import get_logger +from third_party.mobile_use.clients import ui_automator_client +from third_party.mobile_use.utils.logger import get_logger from mcp_server.utils import device_utils, env_utils @@ -205,13 +205,13 @@ def test_device_utils_isolates_stdin(): def test_ui_automator_client_isolates_stdin(): """Verify ui_automator_client helper commands isolate stdin.""" - with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run: + with patch("third_party.mobile_use.clients.ui_automator_client.subprocess.run") as mock_run: mock_run.return_value = MagicMock(returncode=0, stdout="package:com.test\n", stderr="") ui_automator_client._is_package_installed("dev-1", "com.test") assert mock_run.called assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL - with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run: + with patch("third_party.mobile_use.clients.ui_automator_client.subprocess.run") as mock_run: mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") ui_automator_client._uninstall_package("dev-1", "com.test") assert mock_run.called diff --git a/tests/unit/sdk/test_agent_trace_finalization.py b/tests/unit/sdk/test_agent_trace_finalization.py index 8da140189..82000b56c 100644 --- a/tests/unit/sdk/test_agent_trace_finalization.py +++ b/tests/unit/sdk/test_agent_trace_finalization.py @@ -19,7 +19,7 @@ from artemis.sdk.agent import Agent from artemis.context import DeviceContext, DevicePlatform from artemis.runtime.device_lock import DeviceBusyError -from artemis.sdk.types.exceptions import AgentError +from third_party.mobile_use.sdk.types.exceptions import AgentError @pytest.mark.asyncio @@ -79,7 +79,7 @@ async def test_task_that_never_acquires_queue_does_not_create_trace_session(): agent._prepare_tracing = MagicMock() with patch( - "artemis.sdk.agent.DeviceExecutionLock.acquire", + "artemis.runtime.DeviceExecutionLock.acquire", side_effect=DeviceBusyError("queue cancelled"), ): with pytest.raises(DeviceBusyError, match="queue cancelled"): diff --git a/tests/unit/test_drivers.py b/tests/unit/test_drivers.py index 4efba167e..6d5b2493a 100644 --- a/tests/unit/test_drivers.py +++ b/tests/unit/test_drivers.py @@ -23,6 +23,7 @@ from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection from artemis.drivers.mock.mock_driver import MockDeviceDriver from artemis.drivers.android.adb_driver import AndroidAdbDriver +from third_party.mobile_use.controllers.types import ElementQuery _ONE_PIXEL_PNG = base64.b64encode( @@ -194,7 +195,7 @@ async def test_find_element_prefers_fresh_bounds_over_stale_center(): ) ) - _, center, error = await driver.find_element(resource_id="app:id/digit_4") + _, center, error = await driver.find_element(ElementQuery(resource_id="app:id/digit_4")) assert error is None assert center == [140, 1820] diff --git a/tests/unit/test_hierarchy_fidelity.py b/tests/unit/test_hierarchy_fidelity.py index bda15ea03..5f233e486 100644 --- a/tests/unit/test_hierarchy_fidelity.py +++ b/tests/unit/test_hierarchy_fidelity.py @@ -15,10 +15,13 @@ from artemis.clients.accessibility_client import AccessibilityClient, HelperEmptyHierarchy from artemis.clients.screen_client_factory import FallbackScreenClient -from artemis.clients.ui_automator_client import UIAutomatorClient, _parse_hierarchy_xml_to_elements from artemis.core.diagnostics.hierarchy_parity import compare_dumps from artemis.utils import ui_filter from artemis.utils.visualization import format_minimal_list_with_elements +from third_party.mobile_use.clients.ui_automator_client import ( + UIAutomatorClient, + parse_hierarchy_xml_to_elements, +) def _node(**attrs: str) -> str: @@ -47,7 +50,7 @@ def _xml(*nodes: str) -> str: def test_negative_bounds_keep_their_coordinates_and_get_clipped(): - elements = _parse_hierarchy_xml_to_elements(_xml(_node(text="Row", bounds="[-20,-40][300,60]"))) + elements = parse_hierarchy_xml_to_elements(_xml(_node(text="Row", bounds="[-20,-40][300,60]"))) row = elements[1] assert row["parsed_bounds"] == {"left": -20, "top": -40, "right": 300, "bottom": 60} @@ -68,7 +71,7 @@ def test_negative_bounds_keep_their_coordinates_and_get_clipped(): def test_minimal_list_shows_hint_for_empty_inputs_and_errors(): - elements = _parse_hierarchy_xml_to_elements( + elements = parse_hierarchy_xml_to_elements( _xml( _node(text="", hint="Search settings", **{"class": "android.widget.EditText"}), _node(text="abc", error="Password too short", bounds="[0,200][100,300]"), diff --git a/tests/unit/test_llm_gateway.py b/tests/unit/test_llm_gateway.py index b2d6be915..73c8eb730 100644 --- a/tests/unit/test_llm_gateway.py +++ b/tests/unit/test_llm_gateway.py @@ -1,5 +1,6 @@ """Tests for the LLM gateway: complete(), classified recovery, and fallback.""" +import asyncio from types import SimpleNamespace from langchain_core.messages import AIMessage, AIMessageChunk @@ -13,6 +14,7 @@ acomplete_structured, with_fallback, ) +from third_party.mobile_use.llm_service import WAITING_MESSAGE, LLMWaitNotice @pytest.fixture(autouse=True) @@ -168,6 +170,47 @@ async def fallback_call(): assert calls["fallback"] == 0 +@pytest.mark.asyncio +async def test_with_fallback_clears_fallback_flag_when_cancelled(): + seen = {} + + async def main_call(): + seen["during"] = llm_service._FALLBACK_AVAILABLE.get() + raise asyncio.CancelledError + + async def fallback_call(): # pragma: no cover - must not run + raise AssertionError("fallback must not run on cancellation") + + with pytest.raises(asyncio.CancelledError): + await with_fallback(main_call=main_call, fallback_call=fallback_call) + assert seen["during"] is True + assert llm_service._FALLBACK_AVAILABLE.get() is False + + +@pytest.mark.asyncio +async def test_timeout_wrapper_enforces_hard_timeout(): + notices = [] + wait = LLMWaitNotice(notify=notices.append, on_timeout=notices.append, poll_interval=0.01) + + with pytest.raises(TimeoutError): + await wait(asyncio.sleep(10), timeout_seconds=0.01, hard_timeout=0.05) + assert notices == [WAITING_MESSAGE, "LLM call timed out after 0.05 seconds."] + + +@pytest.mark.asyncio +async def test_timeout_wrapper_holds_clock_while_paused(): + wait = LLMWaitNotice( + notify=lambda _: None, on_timeout=lambda _: None, hold=lambda: True, poll_interval=0.01 + ) + + async def slow(): + await asyncio.sleep(0.1) + return "done" + + # The 0.02s hard timeout never runs out while the task is held (paused). + assert await wait(slow(), timeout_seconds=0.01, hard_timeout=0.02) == "done" + + @pytest.mark.asyncio async def test_acomplete_structured_corrective_reask(): class CorrectsItself: diff --git a/tests/unit/test_third_party_boundary.py b/tests/unit/test_third_party_boundary.py new file mode 100644 index 000000000..29d42a5d9 --- /dev/null +++ b/tests/unit/test_third_party_boundary.py @@ -0,0 +1,111 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the imports between ``artemis`` and ``third_party/mobile_use``. + +The two packages import each other. Every name imported across them, in either +direction, must exist and be public, so renaming or removing it on one side +fails here instead of at runtime. +""" + +import ast +import importlib +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +THIRD_PARTY_DIRS = [ROOT / "third_party"] +ARTEMIS_DIRS = [ROOT / "artemis", ROOT / "apps" / "admin_console", ROOT / "mcp_server"] + + +def _py_files(dirs: list[Path]): + for base in dirs: + for path in sorted(base.rglob("*.py")): + if "node_modules" not in path.parts and "__pycache__" not in path.parts: + yield path + + +def _cross_imports(dirs: list[Path], target_prefix: str) -> list[tuple[str, str, str | None]]: + """Return (location, module, name) for imports of ``target_prefix`` modules.""" + found: list[tuple[str, str, str | None]] = [] + for path in _py_files(dirs): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + loc = f"{path.relative_to(ROOT)}:{getattr(node, 'lineno', 0)}" + if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + if node.module == target_prefix or node.module.startswith(target_prefix + "."): + found.extend((loc, node.module, alias.name) for alias in node.names) + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.name == target_prefix or alias.name.startswith(target_prefix + "."): + found.append((loc, alias.name, None)) + return found + + +# Imports of artemis from third_party. +THIRD_PARTY_TO_ARTEMIS = _cross_imports(THIRD_PARTY_DIRS, "artemis") +# Imports of third_party from artemis. +ARTEMIS_TO_THIRD_PARTY = _cross_imports(ARTEMIS_DIRS, "third_party") + + +def _is_private(dotted: str) -> bool: + return any(part.startswith("_") and not part.startswith("__") for part in dotted.split(".")) + + +def test_boundary_scan_found_imports(): + # Fails if the scan finds nothing, e.g. after a directory move. + assert THIRD_PARTY_TO_ARTEMIS + assert ARTEMIS_TO_THIRD_PARTY + + +@pytest.mark.parametrize( + ("loc", "module", "name"), + THIRD_PARTY_TO_ARTEMIS + ARTEMIS_TO_THIRD_PARTY, + ids=lambda v: v if isinstance(v, str) else "", +) +def test_cross_boundary_import_resolves_and_is_public(loc, module, name): + full = module if name is None else f"{module}.{name}" + assert not _is_private(full), ( + f"{loc}: imports private '{full}' across the artemis/third_party boundary; " + "make it public on the defining side." + ) + mod = importlib.import_module(module) + if name is None or name == "*": + return + if not hasattr(mod, name): + # `from pkg import submodule` + importlib.import_module(f"{module}.{name}") + + +_THIRD_PARTY_HEADER_MARKERS = ("derived from", "originally from", "minitap") + + +def _leading_comment_block(path: Path) -> str: + lines = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.startswith("#"): + break + lines.append(line) + return "\n".join(lines).lower() + + +def test_first_party_files_carry_no_third_party_header(): + # Code that needs an upstream copyright header belongs in third_party/. + offenders = [ + str(path.relative_to(ROOT)) + for path in _py_files(ARTEMIS_DIRS + [ROOT / "tests", ROOT / "scripts"]) + if any(marker in _leading_comment_block(path) for marker in _THIRD_PARTY_HEADER_MARKERS) + ] + assert not offenders, f"third-party attribution header outside third_party/: {offenders}" diff --git a/tests/unit/test_ui_automator_client_awake.py b/tests/unit/test_ui_automator_client_awake.py index 12e3a821f..3dfb7cc0e 100644 --- a/tests/unit/test_ui_automator_client_awake.py +++ b/tests/unit/test_ui_automator_client_awake.py @@ -2,16 +2,16 @@ import pytest -from artemis.clients.ui_automator_client import UIAutomatorClient from artemis.drivers.android.adb_driver import AndroidAdbDriver +from third_party.mobile_use.clients.ui_automator_client import UIAutomatorClient -@patch("artemis.clients.ui_automator_client.u2.connect") +@patch("third_party.mobile_use.clients.ui_automator_client.u2.connect") @patch( - "artemis.clients.ui_automator_client.ensure_device_awake", + "third_party.mobile_use.clients.ui_automator_client.ensure_device_awake", return_value="host_heartbeat", ) -@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed") +@patch("third_party.mobile_use.clients.ui_automator_client._ensure_maestro_not_installed") def test_new_ui_connection_enrolls_device_in_shared_awake_strategy( mock_remove_maestro, mock_ensure_awake, mock_connect ): @@ -25,12 +25,15 @@ def test_new_ui_connection_enrolls_device_in_shared_awake_strategy( mock_connect.assert_called_once_with("device-123") -@patch("artemis.clients.ui_automator_client.u2.connect", side_effect=RuntimeError("offline")) @patch( - "artemis.clients.ui_automator_client.ensure_device_awake", + "third_party.mobile_use.clients.ui_automator_client.u2.connect", + side_effect=RuntimeError("offline"), +) +@patch( + "third_party.mobile_use.clients.ui_automator_client.ensure_device_awake", return_value="host_heartbeat", ) -@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed") +@patch("third_party.mobile_use.clients.ui_automator_client._ensure_maestro_not_installed") def test_failed_ui_connection_does_not_stop_process_awake_service( _mock_remove_maestro, mock_ensure_awake, _mock_connect ): @@ -43,12 +46,12 @@ def test_failed_ui_connection_does_not_stop_process_awake_service( assert client._awake_strategy is None -@patch("artemis.clients.ui_automator_client.u2.connect") +@patch("third_party.mobile_use.clients.ui_automator_client.u2.connect") @patch( - "artemis.clients.ui_automator_client.ensure_device_awake", + "third_party.mobile_use.clients.ui_automator_client.ensure_device_awake", return_value="host_heartbeat", ) -@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed") +@patch("third_party.mobile_use.clients.ui_automator_client._ensure_maestro_not_installed") def test_client_disconnect_does_not_send_power_cleanup_commands( _mock_remove_maestro, mock_ensure_awake, _mock_connect ): diff --git a/tests/unit/test_unified_controller_video.py b/tests/unit/test_unified_controller_video.py index eea6bfe9f..e87330dd4 100644 --- a/tests/unit/test_unified_controller_video.py +++ b/tests/unit/test_unified_controller_video.py @@ -28,15 +28,17 @@ from artemis.controllers.unified_controller import UnifiedMobileController from artemis.drivers.mock.mock_driver import MockDeviceDriver from artemis.utils.video import ( - RecordingSession, build_scrcpy_record_command, - extract_audio_from_video, extract_frames_at_timestamps, get_ffmpeg_path, - get_active_session, normalize_recording_to_mp4, plan_timeline_pieces, render_timeline_clip, +) +from third_party.mobile_use.utils.video import ( + RecordingSession, + extract_audio_from_video, + get_active_session, remove_active_session, set_active_session, ) diff --git a/tests/unit/tools/test_launch_app.py b/tests/unit/tools/test_launch_app.py index bc4561179..cdc6f2846 100644 --- a/tests/unit/tools/test_launch_app.py +++ b/tests/unit/tools/test_launch_app.py @@ -13,9 +13,9 @@ # limitations under the License. from unittest.mock import MagicMock, patch -from artemis.agents.hopper.hopper import HopperOutput from artemis.context import ArtemisContext from artemis.tools.mobile.launch_app import find_package +from third_party.mobile_use.agents.hopper.hopper import HopperOutput import pytest @@ -27,8 +27,8 @@ def mock_context(): @pytest.mark.asyncio -@patch("artemis.tools.mobile.launch_app.list_packages_async") -@patch("artemis.tools.mobile.launch_app.hopper") +@patch("third_party.mobile_use.tools.mobile.launch_app.list_packages_async") +@patch("third_party.mobile_use.tools.mobile.launch_app.hopper") async def test_find_package_cache_hit(mock_hopper, mock_list_packages, mock_context): mock_list_packages.return_value = "com.example.app1\ncom.example.app2" @@ -53,8 +53,8 @@ async def test_find_package_cache_hit(mock_hopper, mock_list_packages, mock_cont @pytest.mark.asyncio -@patch("artemis.tools.mobile.launch_app.list_packages_async") -@patch("artemis.tools.mobile.launch_app.hopper") +@patch("third_party.mobile_use.tools.mobile.launch_app.list_packages_async") +@patch("third_party.mobile_use.tools.mobile.launch_app.hopper") async def test_find_package_not_found_caches_none(mock_hopper, mock_list_packages, mock_context): mock_list_packages.return_value = "com.example.app1\ncom.example.app2" @@ -74,8 +74,8 @@ async def test_find_package_not_found_caches_none(mock_hopper, mock_list_package @pytest.mark.asyncio -@patch("artemis.tools.mobile.launch_app.list_packages_async") -@patch("artemis.tools.mobile.launch_app.hopper") +@patch("third_party.mobile_use.tools.mobile.launch_app.list_packages_async") +@patch("third_party.mobile_use.tools.mobile.launch_app.hopper") async def test_find_package_passes_use_fallback(mock_hopper, mock_list_packages, mock_context): mock_list_packages.return_value = "com.example.app1\ncom.example.app2" mock_hopper.return_value = HopperOutput( diff --git a/tests/unit/tools/test_utils.py b/tests/unit/tools/test_utils.py deleted file mode 100644 index 63b9fd961..000000000 --- a/tests/unit/tools/test_utils.py +++ /dev/null @@ -1,457 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import sys -from unittest.mock import Mock, patch - -import pytest - -# Mock the problematic langgraph import at module level -_orig_chat_agent_executor = sys.modules.get("langgraph.prebuilt.chat_agent_executor") -_orig_state = sys.modules.get("artemis.graph.state") - -sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock() -sys.modules["artemis.graph.state"] = Mock() - -from artemis.context import DeviceContext, DevicePlatform, ArtemisContext # noqa: E402 -from artemis.tools.types import Target # noqa: E402 -from artemis.tools.utils import ( # noqa: E402 - IdSelectorRequest, - SelectorRequestWithCoordinates, - focus_element_if_needed, - move_cursor_to_end_if_bounds, -) -from artemis.utils.ui_hierarchy import ElementBounds # noqa: E402 - -# Restore original modules to prevent contaminating other tests -if _orig_chat_agent_executor is not None: - sys.modules["langgraph.prebuilt.chat_agent_executor"] = _orig_chat_agent_executor -else: - del sys.modules["langgraph.prebuilt.chat_agent_executor"] - -if _orig_state is not None: - sys.modules["artemis.graph.state"] = _orig_state -else: - del sys.modules["artemis.graph.state"] - - -@pytest.fixture -def mock_context(): - """Create a mock ArtemisContext for testing.""" - ctx = Mock(spec=ArtemisContext) - - # Create device context with necessary attributes - ctx.device = Mock(spec=DeviceContext) - ctx.device.mobile_platform = DevicePlatform.ANDROID - ctx.device.device_id = "test_device_123" - ctx.device.device_width = 1080 - ctx.device.device_height = 2340 - ctx.device.host_platform = "LINUX" - - ctx.ui_adb_client = Mock() - - # Mock the ADB client for Android - ctx.adb_client = Mock() - mock_device = Mock() - mock_device.shell = Mock(return_value="") - ctx.adb_client.device = Mock(return_value=mock_device) - - # Mock the ADB client for Android - mock_response = Mock() - mock_response.json.return_value = {"elements": []} - ctx.ui_adb_client.get_screen_data = Mock(return_value=mock_response) - ctx.ui_adb_client.get_hierarchy = Mock( - return_value=( - b'' - ) - ) - - return ctx - - -@pytest.fixture -def mock_state(): - """Create a mock State for testing.""" - state = Mock() - state.latest_ui_hierarchy = [] - return state - - -@pytest.fixture -def sample_element(): - """Create a sample UI element for testing.""" - return { - "resourceId": "com.example:id/text_input", - "text": "Sample text", - "bounds": {"x": 100, "y": 200, "width": 300, "height": 50}, - "focused": "false", - } - - -@pytest.fixture -def sample_rich_element(): - """Create a sample rich UI element for testing.""" - return { - "attributes": { - "resource-id": "com.example:id/text_input", - "focused": "false", - "text": "Sample text", - "bounds": {"x": 100, "y": 200, "width": 300, "height": 50}, - }, - "children": [], - } - - -class TestMoveCursorToEndIfBounds: - """Test cases for move_cursor_to_end_if_bounds function.""" - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_resource_id") - def test_move_cursor_with_resource_id( - self, - mock_find_element, - mock_tap, - mock_context, - mock_state, - sample_element, - ): - """Test moving cursor using resource_id (highest priority).""" - mock_state.latest_ui_hierarchy = [sample_element] - mock_find_element.return_value = sample_element - - target = Target( - resource_id="com.example:id/text_input", - resource_id_index=None, - text=None, - text_index=None, - bounds=None, - ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) - ) - - mock_find_element.assert_called_once_with( - ui_hierarchy=[sample_element], - resource_id="com.example:id/text_input", - index=0, - ) - mock_tap.assert_called_once() - call_args = mock_tap.call_args[1] - selector_request = call_args["selector_request"] - assert isinstance(selector_request, SelectorRequestWithCoordinates) - coords = selector_request.coordinates - assert coords.x == 397 # 100 + 300 * 0.99 - assert coords.y == 249 # 200 + 50 * 0.99 - assert result == sample_element - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_resource_id") - def test_move_cursor_with_coordinates_only( - self, mock_find_element, mock_tap, mock_context, mock_state - ): - """Test moving cursor when only coordinates are provided.""" - bounds = ElementBounds(x=50, y=150, width=200, height=40) - target = Target( - resource_id=None, - resource_id_index=None, - text=None, - text_index=None, - bounds=bounds, - ) - - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) - ) - - mock_find_element.assert_not_called() - mock_tap.assert_called_once() - call_args = mock_tap.call_args[1] - selector_request = call_args["selector_request"] - coords = selector_request.coordinates - assert coords.x == 248 # 50 + 200 * 0.99 - assert coords.y == 189 # 150 + 40 * 0.99 - assert result is None # No element is returned when using coords directly - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_text") - def test_move_cursor_with_text_only_success( - self, mock_find_text, mock_tap, mock_context, mock_state, sample_element - ): - """Test moving cursor when only text is provided and succeeds.""" - mock_state.latest_ui_hierarchy = [sample_element] - mock_find_text.return_value = sample_element - - target = Target( - resource_id=None, - resource_id_index=None, - text="Sample text", - text_index=0, - bounds=None, - ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) - ) - - mock_find_text.assert_called_once_with([sample_element], "Sample text", index=0) - mock_tap.assert_called_once() - assert result == sample_element - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_text") - def test_move_cursor_with_text_only_element_not_found( - self, mock_find_text, mock_tap, mock_context, mock_state - ): - """Test when searching by text finds no element.""" - mock_state.latest_ui_hierarchy = [] - mock_find_text.return_value = None - - target = Target( - resource_id=None, - resource_id_index=None, - text="Nonexistent text", - text_index=None, - bounds=None, - ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) - ) - - mock_tap.assert_not_called() - assert result is None - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_text") - def test_move_cursor_with_text_only_no_bounds( - self, mock_find_text, mock_tap, mock_context, mock_state - ): - """Test when element is found by text but has no bounds.""" - element_no_bounds = {"text": "Text without bounds"} - mock_state.latest_ui_hierarchy = [element_no_bounds] - mock_find_text.return_value = element_no_bounds - - target = Target( - resource_id=None, - resource_id_index=None, - text="Text without bounds", - text_index=None, - bounds=None, - ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) - ) - - mock_tap.assert_not_called() - assert result is None # Should return None as no action was taken - - @patch("artemis.tools.utils.find_element_by_resource_id") - def test_move_cursor_element_not_found_by_id(self, mock_find_element, mock_context, mock_state): - """Test when element is not found by resource_id.""" - mock_find_element.return_value = None - - target = Target( - resource_id="com.example:id/nonexistent", - resource_id_index=None, - text=None, - text_index=None, - bounds=None, - ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) - ) - - assert result is None - - -class TestFocusElementIfNeeded: - """Test cases for focus_element_if_needed function.""" - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_resource_id") - def test_focus_element_already_focused( - self, mock_find_element, mock_tap, mock_context, sample_rich_element - ): - """Test when element is already focused.""" - focused_element = sample_rich_element.copy() - focused_element["attributes"]["focused"] = "true" - - mock_response = Mock() - mock_response.json.return_value = {"elements": [focused_element]} - mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) - mock_find_element.return_value = focused_element["attributes"] - - target = Target( - resource_id="com.example:id/text_input", - resource_id_index=None, - text=None, - text_index=None, - bounds=None, - ) - result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) - - mock_tap.assert_not_called() - assert result == "resource_id" - mock_context.ui_adb_client.get_hierarchy.assert_called_once() - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_resource_id") - def test_focus_element_needs_focus_success( - self, mock_find_element, mock_tap, mock_context, sample_rich_element - ): - """Test when element needs focus and focusing succeeds.""" - unfocused_element = sample_rich_element - focused_element = { - "attributes": { - "resource-id": "com.example:id/text_input", - "focused": "true", - }, - "children": [], - } - - mock_find_element.side_effect = [ - unfocused_element["attributes"], - focused_element["attributes"], - ] - - target = Target( - resource_id="com.example:id/text_input", - resource_id_index=None, - text=None, - text_index=None, - bounds=None, - ) - result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) - - mock_tap.assert_called_once_with( - ctx=mock_context, - selector_request=IdSelectorRequest(id="com.example:id/text_input"), - index=0, - ) - assert mock_context.ui_adb_client.get_hierarchy.call_count == 2 - assert result == "resource_id" - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.logger") - @patch("artemis.tools.utils.find_element_by_resource_id") - def test_focus_id_and_text_mismatch_fallback_to_text( - self, - mock_find_id, - mock_logger, - mock_tap, - mock_context, - sample_rich_element, - ): - """Test fallback when resource_id and text point to different elements.""" - element_from_id = sample_rich_element["attributes"].copy() - element_from_id["text"] = "Different text" - - element_from_text = sample_rich_element.copy() - element_from_text["attributes"]["bounds"] = { - "x": 10, - "y": 20, - "width": 100, - "height": 30, - } - - mock_response = Mock() - mock_response.json.return_value = {"elements": [element_from_text]} - mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) - mock_find_id.return_value = element_from_id - - with patch("artemis.tools.utils.find_element_by_text") as mock_find_text: - mock_find_text.return_value = element_from_text["attributes"] - - target = Target( - resource_id="com.example:id/text_input", - resource_id_index=None, - text="Sample text", - text_index=None, - bounds=None, - ) - result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) - - mock_logger.warning.assert_called_once() - mock_tap.assert_called_once() - assert result == "text" - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_text") - def test_focus_fallback_to_text( - self, mock_find_text, mock_tap, mock_context, sample_rich_element - ): - """Test fallback to focusing using text.""" - element_with_bounds = sample_rich_element.copy() - element_with_bounds["attributes"]["bounds"] = { - "x": 10, - "y": 20, - "width": 100, - "height": 30, - } - - mock_response = Mock() - mock_response.json.return_value = {"elements": [element_with_bounds]} - mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) - mock_find_text.return_value = element_with_bounds["attributes"] - - target = Target( - resource_id=None, - resource_id_index=None, - text="Sample text", - text_index=None, - bounds=None, - ) - result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) - - mock_find_text.assert_called_once() - mock_tap.assert_called_once() - call_args = mock_tap.call_args[1] - selector = call_args["selector_request"] - assert isinstance(selector, SelectorRequestWithCoordinates) - assert selector.coordinates.x == 60 # 10 + 100/2 - assert selector.coordinates.y == 35 # 20 + 30/2 - assert result == "text" - - @patch("artemis.tools.utils.logger") - def test_focus_all_locators_fail(self, mock_logger, mock_context): - """Test failure when no locator can find an element.""" - - mock_response = Mock() - mock_response.json.return_value = {"elements": []} - mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) - with ( - patch("artemis.tools.utils.find_element_by_resource_id") as mock_find_id, - patch("artemis.tools.utils.find_element_by_text") as mock_find_text, - ): - mock_find_id.return_value = None - mock_find_text.return_value = None - - target = Target( - resource_id="nonexistent", - resource_id_index=None, - text="nonexistent", - text_index=None, - bounds=None, - ) - result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) - - mock_logger.error.assert_called_once_with( - "Failed to focus element." - + " No valid locator (resource_id, coordinates, or text) succeeded." - ) - assert result is None - - -if __name__ == "__main__": - pytest.main([__file__]) diff --git a/tests/unit/utils/test_app_launch_utils.py b/tests/unit/utils/test_app_launch_utils.py index 7c988ab1a..ab7767270 100644 --- a/tests/unit/utils/test_app_launch_utils.py +++ b/tests/unit/utils/test_app_launch_utils.py @@ -17,12 +17,15 @@ from artemis.context import ArtemisContext from artemis.utils.app_launch_utils import ( ForegroundTask, - _handle_initial_app_launch, get_foreground_task, - launch_app_with_retries, parse_foreground_task, ) import pytest +from third_party.mobile_use.utils.app_launch_utils import ( + _handle_initial_app_launch, + launch_app_with_retries, +) + SETTINGS = "com.android.settings" SETTINGS_SEARCH = "com.google.android.settings.intelligence" @@ -273,7 +276,7 @@ def test_get_foreground_task_tolerates_missing_device_and_errors(mock_get_device @pytest.mark.asyncio @patch("artemis.utils.app_launch_utils.get_foreground_task") -@patch("artemis.utils.app_launch_utils.UnifiedMobileController") +@patch("third_party.mobile_use.utils.app_launch_utils.UnifiedMobileController") @patch("artemis.utils.app_launch_utils.get_current_foreground_package_async") async def test_launch_app_success_immediate( mock_get_foreground, mock_controller_cls, mock_get_task, mock_context @@ -294,7 +297,7 @@ async def test_launch_app_success_immediate( @pytest.mark.asyncio @patch("artemis.utils.app_launch_utils.get_foreground_task") -@patch("artemis.utils.app_launch_utils.UnifiedMobileController") +@patch("third_party.mobile_use.utils.app_launch_utils.UnifiedMobileController") @patch("artemis.utils.app_launch_utils.get_current_foreground_package_async") async def test_launch_app_success_when_top_task_belongs_to_app( mock_get_foreground, mock_controller_cls, mock_get_task, mock_context @@ -317,7 +320,7 @@ async def test_launch_app_success_when_top_task_belongs_to_app( @pytest.mark.asyncio @patch("artemis.utils.app_launch_utils.get_foreground_task") -@patch("artemis.utils.app_launch_utils.UnifiedMobileController") +@patch("third_party.mobile_use.utils.app_launch_utils.UnifiedMobileController") @patch("artemis.utils.app_launch_utils.get_current_foreground_package_async") async def test_launch_app_success_permission_overlay_in_app_task( mock_get_foreground, mock_controller_cls, mock_get_task, mock_context @@ -343,7 +346,7 @@ async def test_launch_app_success_permission_overlay_in_app_task( @pytest.mark.asyncio @patch("artemis.utils.app_launch_utils.get_foreground_task") -@patch("artemis.utils.app_launch_utils.UnifiedMobileController") +@patch("third_party.mobile_use.utils.app_launch_utils.UnifiedMobileController") @patch("artemis.utils.app_launch_utils.get_current_foreground_package_async") async def test_launch_app_failure_when_top_task_is_another_app( mock_get_foreground, mock_controller_cls, mock_get_task, mock_context @@ -365,7 +368,7 @@ async def test_launch_app_failure_when_top_task_is_another_app( @pytest.mark.asyncio @patch("artemis.utils.app_launch_utils.get_foreground_task") -@patch("artemis.utils.app_launch_utils.UnifiedMobileController") +@patch("third_party.mobile_use.utils.app_launch_utils.UnifiedMobileController") @patch("artemis.utils.app_launch_utils.get_current_foreground_package_async") async def test_launch_timeout_message_names_foreground_package_and_task( mock_get_foreground, mock_controller_cls, mock_get_task, mock_context @@ -374,7 +377,7 @@ async def test_launch_timeout_message_names_foreground_package_and_task( mock_get_foreground.return_value = SETTINGS_SEARCH mock_get_task.return_value = parse_foreground_task(DUMP_SEARCH_IN_OWN_TASK) - with patch("artemis.utils.app_launch_utils.logger") as mock_logger: + with patch("third_party.mobile_use.utils.app_launch_utils.logger") as mock_logger: success, _ = await launch_app_with_retries( mock_context, SETTINGS, max_retries=1, max_poll_seconds=1 ) @@ -395,7 +398,7 @@ async def test_launch_timeout_message_names_foreground_package_and_task( @pytest.mark.asyncio @patch("artemis.utils.app_launch_utils.get_foreground_task") -@patch("artemis.utils.app_launch_utils.UnifiedMobileController") +@patch("third_party.mobile_use.utils.app_launch_utils.UnifiedMobileController") @patch("artemis.utils.app_launch_utils.get_current_foreground_package_async") async def test_launch_app_null_focus_keeps_polling_without_reading_task( mock_get_foreground, mock_controller_cls, mock_get_task, mock_context @@ -419,7 +422,7 @@ async def test_launch_app_null_focus_keeps_polling_without_reading_task( @pytest.mark.asyncio -@patch("artemis.utils.app_launch_utils.launch_app_with_retries") +@patch("third_party.mobile_use.utils.app_launch_utils.launch_app_with_retries") @patch("artemis.utils.app_launch_utils.get_foreground_task") @patch("artemis.utils.app_launch_utils.get_current_foreground_package_async") async def test_initial_launch_skips_launch_when_app_task_is_already_foreground( @@ -436,7 +439,9 @@ async def test_initial_launch_skips_launch_when_app_task_is_already_foreground( @pytest.mark.asyncio -@patch("artemis.utils.app_launch_utils.launch_app_with_retries", new_callable=AsyncMock) +@patch( + "third_party.mobile_use.utils.app_launch_utils.launch_app_with_retries", new_callable=AsyncMock +) @patch("artemis.utils.app_launch_utils.get_foreground_task") @patch("artemis.utils.app_launch_utils.get_current_foreground_package_async") async def test_initial_launch_launches_when_another_app_is_foreground( diff --git a/tests/unit/utils/test_media.py b/tests/unit/utils/test_media.py index bad64abeb..423b72c7a 100644 --- a/tests/unit/utils/test_media.py +++ b/tests/unit/utils/test_media.py @@ -14,7 +14,7 @@ import json -from artemis.utils.media import ( +from third_party.mobile_use.utils.media import ( create_steps_json_from_trace_folder, remove_steps_json_from_trace_folder, ) diff --git a/tests/unit/utils/test_ui_hierarchy.py b/tests/unit/utils/test_ui_hierarchy.py deleted file mode 100644 index 3ceade43b..000000000 --- a/tests/unit/utils/test_ui_hierarchy.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. - -from unittest.mock import patch - -from artemis.utils.ui_hierarchy import ( - ElementBounds, - Point, - find_element_by_resource_id, - get_bounds_for_element, - get_element_text, - is_element_focused, - text_input_is_empty, -) - - -def test_text_input_is_empty(): - assert text_input_is_empty(text=None, hint_text=None) - assert text_input_is_empty(text="", hint_text=None) - assert text_input_is_empty(text="", hint_text="") - assert text_input_is_empty(text="text", hint_text="text") - - assert not text_input_is_empty(text="text", hint_text=None) - assert not text_input_is_empty(text="text", hint_text="") - - -def test_find_element_by_resource_id(): - ui_hierarchy = [ - { - "resourceId": "com.example:id/button1", - "text": "Button 1", - "children": [], - }, - { - "resourceId": "com.example:id/container", - "children": [ - { - "resourceId": "com.example:id/nested_button", - "text": "Nested Button", - "children": [], - } - ], - }, - ] - - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button1") - assert result is not None - assert result["resourceId"] == "com.example:id/button1" - assert result["text"] == "Button 1" - - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nested_button") - assert result is not None - assert result["resourceId"] == "com.example:id/nested_button" - assert result["text"] == "Nested Button" - - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nonexistent") - assert result is None - - result = find_element_by_resource_id([], "com.example:id/button1") - assert result is None - - -def test_find_element_by_resource_id_with_index(): - ui_hierarchy = [ - { - "resourceId": "com.example:id/button", - "text": "Button 1", - "children": [], - }, - { - "resourceId": "com.example:id/button", - "text": "Button 2", - "children": [], - }, - { - "resourceId": "com.example:id/container", - "children": [ - { - "resourceId": "com.example:id/button", - "text": "Button 3", - "children": [], - } - ], - }, - ] - - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=0) - assert result is not None - assert result["text"] == "Button 1" - - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=1) - assert result is not None - assert result["text"] == "Button 2" - - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=2) - assert result is not None - assert result["text"] == "Button 3" - - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=3) - assert result is None - - -def test_find_element_by_resource_id_rich_hierarchy(): - rich_hierarchy = [ - { - "attributes": {"resource-id": "com.example:id/button1"}, - "children": [], - }, - { - "attributes": {"resource-id": "com.example:id/container"}, - "children": [ - { - "attributes": {"resource-id": "com.example:id/nested_button"}, - "children": [], - } - ], - }, - ] - - result = find_element_by_resource_id( - rich_hierarchy, "com.example:id/button1", is_rich_hierarchy=True - ) - assert result is not None - assert result["resource-id"] == "com.example:id/button1" - - result = find_element_by_resource_id( - rich_hierarchy, "com.example:id/nested_button", is_rich_hierarchy=True - ) - assert result is not None - assert result["resource-id"] == "com.example:id/nested_button" - - result = find_element_by_resource_id( - rich_hierarchy, "com.example:id/nonexistent", is_rich_hierarchy=True - ) - assert result is None - - -def test_is_element_focused(): - focused_element = {"focused": "true"} - assert is_element_focused(focused_element) - - non_focused_element = {"focused": "false"} - assert not is_element_focused(non_focused_element) - - no_focused_element = {"text": "some text"} - assert not is_element_focused(no_focused_element) - - none_focused_element = {"focused": None} - assert not is_element_focused(none_focused_element) - - -def test_get_element_text(): - element = {"text": "Button Text", "hintText": "Hint Text"} - assert get_element_text(element) == "Button Text" - assert get_element_text(element, hint_text=False) == "Button Text" - assert get_element_text(element, hint_text=True) == "Hint Text" - - element_no_text = {"hintText": "Hint Text"} - assert get_element_text(element_no_text) is None - assert get_element_text(element_no_text, hint_text=True) == "Hint Text" - element_no_hint = {"text": "Button Text"} - assert get_element_text(element_no_hint) == "Button Text" - assert get_element_text(element_no_hint, hint_text=True) is None - - empty_element = {} - assert get_element_text(empty_element) is None - assert get_element_text(empty_element, hint_text=True) is None - - -def test_get_bounds_for_element(): - element_with_bounds = {"bounds": {"x": 10, "y": 20, "width": 100, "height": 50}} - bounds = get_bounds_for_element(element_with_bounds) - assert bounds is not None - assert isinstance(bounds, ElementBounds) - assert bounds.x == 10 - assert bounds.y == 20 - assert bounds.width == 100 - assert bounds.height == 50 - - element_no_bounds = {"text": "Button"} - bounds = get_bounds_for_element(element_no_bounds) - assert bounds is None - - # Suppress logger output for the invalid bounds test case - with patch("artemis.utils.ui_hierarchy.logger.error"): - element_invalid_bounds = { - "bounds": { - "x": "invalid", # Should be int - "y": 20, - "width": 100, - "height": 50, - } - } - bounds = get_bounds_for_element(element_invalid_bounds) - assert bounds is None - - -def test_element_bounds(): - bounds = ElementBounds(x=10, y=20, width=100, height=50) - - center = bounds.get_center() - assert isinstance(center, Point) - assert center.x == 60 - assert center.y == 45 - - center_point = bounds.get_relative_point(0.5, 0.5) - assert isinstance(center_point, Point) - assert center_point.x == 60 - assert center_point.y == 45 - - top_left = bounds.get_relative_point(0.0, 0.0) - assert top_left.x == 10 - assert top_left.y == 20 - - bottom_right = bounds.get_relative_point(1.0, 1.0) - assert bottom_right.x == 110 - assert bottom_right.y == 70 - custom_point = bounds.get_relative_point(0.95, 0.95) - assert custom_point.x == 105 - assert custom_point.y == 67 - - -if __name__ == "__main__": - test_text_input_is_empty() - test_find_element_by_resource_id() - test_find_element_by_resource_id_with_index() - test_find_element_by_resource_id_rich_hierarchy() - test_is_element_focused() - test_get_element_text() - test_get_bounds_for_element() - test_element_bounds() - print("All tests passed") diff --git a/third_party/mobile_use/LICENSE b/third_party/mobile_use/LICENSE new file mode 100644 index 000000000..055b75aaf --- /dev/null +++ b/third_party/mobile_use/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2025 Minitap, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/mobile_use/METADATA b/third_party/mobile_use/METADATA new file mode 100644 index 000000000..c1f539427 --- /dev/null +++ b/third_party/mobile_use/METADATA @@ -0,0 +1,27 @@ +name: "mobile-use" +description: + "AI-powered multi-agent framework for mobile device automation. " + "Only the portions used by Artemis are included." + +third_party { + identifier { + type: "Git" + value: "https://github.com/minitap-ai/mobile-use" + version: "3.6.3" + primary_source: true + } + last_upgrade_date { + year: 2026 + month: 9 + day: 24 + } + license_type: NOTICE + + local_modifications: + "Only portions of upstream are included. Modifications by Google:\n" + " - Renamed imports and identifiers to Artemis equivalents.\n" + " - Removed code not used by Artemis (platform/cloud APIs, iOS, telemetry, etc.).\n" + " - Restructured and extended some code for use by Artemis.\n" + " - Added Apache-2.0 file headers carrying the upstream copyright.\n" + "All code derived from mobile-use lives in this directory.\n" +} diff --git a/third_party/mobile_use/NOTICE b/third_party/mobile_use/NOTICE new file mode 100644 index 000000000..b6a536e8c --- /dev/null +++ b/third_party/mobile_use/NOTICE @@ -0,0 +1,12 @@ +Mobile-Use +Copyright 2025-2026 Minitap, Inc. + +This product includes software and architectural methodologies developed by +Minitap, Inc. (https://github.com/minitap-ai/mobile-use) + +ATTRIBUTION REQUEST: +While the Apache 2.0 license only requires preserving copyright notices, +we kindly request that derivative works, redistributions, or products +inspired by the logic/architecture of this repository include a prominent +attribution to "Minitap, Inc." in the documentation and "About" sections +of the software. This is a request, not a legal requirement. diff --git a/third_party/mobile_use/__init__.py b/third_party/mobile_use/__init__.py new file mode 100644 index 000000000..54774e078 --- /dev/null +++ b/third_party/mobile_use/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Parts of mobile-use (Minitap, Inc.) used by Artemis. See LICENSE and METADATA.""" + +# artemis and these modules import each other. Import artemis first so that +# importing one of these modules directly works the same as going through artemis. +import artemis # noqa: F401, E402 diff --git a/artemis/sdk/examples/__init__.py b/third_party/mobile_use/agents/__init__.py similarity index 92% rename from artemis/sdk/examples/__init__.py rename to third_party/mobile_use/agents/__init__.py index d8e762845..58d482ea3 100644 --- a/artemis/sdk/examples/__init__.py +++ b/third_party/mobile_use/agents/__init__.py @@ -11,5 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -"""Example scripts for the ARTEMIS SDK.""" diff --git a/artemis/utils/time.py b/third_party/mobile_use/agents/hopper/__init__.py similarity index 78% rename from artemis/utils/time.py rename to third_party/mobile_use/agents/hopper/__init__.py index 5ec443a56..58d482ea3 100644 --- a/artemis/utils/time.py +++ b/third_party/mobile_use/agents/hopper/__init__.py @@ -11,10 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from datetime import datetime - - -def convert_timestamp_to_str(ts: float) -> str: - dt = datetime.fromtimestamp(ts) - return dt.strftime("%Y-%m-%dT%H-%M-%S") diff --git a/artemis/agents/hopper/hopper.md b/third_party/mobile_use/agents/hopper/hopper.md similarity index 74% rename from artemis/agents/hopper/hopper.md rename to third_party/mobile_use/agents/hopper/hopper.md index 6637ff54a..8d11dfd9f 100644 --- a/artemis/agents/hopper/hopper.md +++ b/third_party/mobile_use/agents/hopper/hopper.md @@ -1,6 +1,8 @@ ## Hopper diff --git a/artemis/agents/hopper/hopper.py b/third_party/mobile_use/agents/hopper/hopper.py similarity index 80% rename from artemis/agents/hopper/hopper.py rename to third_party/mobile_use/agents/hopper/hopper.py index ad9e23b80..8a0080eb6 100644 --- a/artemis/agents/hopper/hopper.py +++ b/third_party/mobile_use/agents/hopper/hopper.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. from pathlib import Path @@ -22,30 +23,13 @@ from artemis.context import ArtemisContext from artemis.data_engine.trace import trace from artemis.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback -from artemis.utils.logger import get_logger from pydantic import BaseModel, Field +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - class HopperOutput(BaseModel): model_config = {"ignored_types": (CyFunctionDetector,)} found: bool = Field(description="True if the requested data was found, False otherwise.") diff --git a/third_party/mobile_use/clients/__init__.py b/third_party/mobile_use/clients/__init__.py new file mode 100644 index 000000000..58d482ea3 --- /dev/null +++ b/third_party/mobile_use/clients/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/artemis/clients/ui_automator_client.py b/third_party/mobile_use/clients/ui_automator_client.py similarity index 91% rename from artemis/clients/ui_automator_client.py rename to third_party/mobile_use/clients/ui_automator_client.py index 12a73fb04..aac7bca06 100644 --- a/artemis/clients/ui_automator_client.py +++ b/third_party/mobile_use/clients/ui_automator_client.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. """UIAutomator2 client for Android device screen data retrieval. @@ -36,7 +37,8 @@ from artemis.runtime.adb_endpoint import adb_command from artemis.runtime.awake_service import ensure_device_awake -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger if TYPE_CHECKING: from uiautomator2 import Device @@ -46,23 +48,19 @@ MAESTRO_PACKAGE = "dev.mobile.maestro" - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass +# Boolean attributes of a UIAutomator hierarchy node. +UIAUTOMATOR_BOOL_ATTRIBUTES = ( + "checkable", + "checked", + "clickable", + "enabled", + "focusable", + "focused", + "scrollable", + "long-clickable", + "password", + "selected", +) class UIAutomatorScreenData(BaseModel): @@ -77,7 +75,7 @@ class UIAutomatorScreenData(BaseModel): height: int -def _parse_hierarchy_xml_to_elements(hierarchy_xml: str) -> list[dict]: +def parse_hierarchy_xml_to_elements(hierarchy_xml: str) -> list[dict]: """Parse uiautomator2 XML hierarchy into a flat list of element dictionaries. The output format matches the existing screen API format with attributes @@ -133,18 +131,7 @@ def _extract_element(node: ET.Element) -> None: element["class"] = attr_value elif attr_name == "package": element["package"] = attr_value - elif attr_name in ( - "checkable", - "checked", - "clickable", - "enabled", - "focusable", - "focused", - "scrollable", - "long-clickable", - "password", - "selected", - ): + elif attr_name in UIAUTOMATOR_BOOL_ATTRIBUTES: element[attr_name] = attr_value else: element[attr_name] = attr_value @@ -160,7 +147,7 @@ def _extract_element(node: ET.Element) -> None: return elements -def _pil_to_base64(img: Image.Image, format: str = "JPEG", quality: int = 80) -> str: +def pil_to_base64(img: Image.Image, format: str = "JPEG", quality: int = 80) -> str: """Convert PIL Image to base64 string with high-quality compression.""" buffer = BytesIO() if format.upper() in ("JPEG", "JPG"): @@ -419,7 +406,7 @@ def get_screenshot_base64(self) -> str | None: screenshot = self.get_screenshot() if screenshot is None: return None - return _pil_to_base64(screenshot, format="JPEG") + return pil_to_base64(screenshot, format="JPEG") def get_screen_data(self) -> UIAutomatorScreenData: """Get complete screen data including screenshot and hierarchy. @@ -439,10 +426,10 @@ def get_screen_data(self) -> UIAutomatorScreenData: hierarchy_xml = device.dump_hierarchy(compressed=True) # Parse XML to flat elements list - elements = _parse_hierarchy_xml_to_elements(hierarchy_xml) + elements = parse_hierarchy_xml_to_elements(hierarchy_xml) return UIAutomatorScreenData( - base64=_pil_to_base64(screenshot, format="JPEG"), + base64=pil_to_base64(screenshot, format="JPEG"), hierarchy_xml=hierarchy_xml, elements=elements, width=screenshot.width, diff --git a/third_party/mobile_use/config/__init__.py b/third_party/mobile_use/config/__init__.py new file mode 100644 index 000000000..58d482ea3 --- /dev/null +++ b/third_party/mobile_use/config/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/third_party/mobile_use/config/llm.py b/third_party/mobile_use/config/llm.py new file mode 100644 index 000000000..be9f509a9 --- /dev/null +++ b/third_party/mobile_use/config/llm.py @@ -0,0 +1,183 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""LLM provider models, the base LLM configuration model and its loaders.""" + +from collections.abc import Callable +import os +from pathlib import Path +from typing import Any, Literal + +import google.auth +from google.auth.exceptions import DefaultCredentialsError +from pydantic import BaseModel, ValidationError + +from artemis.config.constants import AgentNode, LLMProvider, LLMUtilsNode +from artemis.config.settings import settings +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.file import load_jsonc +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +LLMUtilsNodeWithFallback = LLMUtilsNode +AgentNodeWithFallback = AgentNode + + +def validate_vertex_ai_credentials() -> None: + """Validate Google Application Default Credentials for VertexAI provider.""" + try: + _, project = google.auth.default() + if not project: + raise Exception("VertexAI requires a Google Cloud project to be set.") + except DefaultCredentialsError as e: + raise Exception( + f"VertexAI requires valid Google Application Default Credentials (ADC): {e}" + ) + + +class LLM(BaseModel): + """Base model representing an LLM model provider and runtime parameters.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + provider: LLMProvider + model: str + temperature: float | None = None + thinking_budget: int | None = None + thinking_level: Literal["minimal", "low", "medium", "high"] | None = None + reasoning_effort: Literal["none", "low", "medium", "high"] | None = None + include_thoughts: bool | None = None + enable_grounding: bool | None = None + + def validate_provider(self, name: str) -> None: + """Ensure the required API key or credentials exist in settings for this provider.""" + if self.provider == "openai": + if not settings.OPENAI_API_KEY: + raise Exception(f"{name} requires OPENAI_API_KEY in .env") + elif self.provider == "google": + if not settings.GOOGLE_API_KEY: + raise Exception(f"{name} requires GOOGLE_API_KEY in .env") + elif self.provider == "vertexai": + validate_vertex_ai_credentials() + elif self.provider == "anthropic": + if not (settings.ANTHROPIC_API_KEY or os.environ.get("ANTHROPIC_API_KEY")): + raise Exception(f"{name} requires ANTHROPIC_API_KEY in .env") + elif self.provider == "openrouter": + if not settings.OPEN_ROUTER_API_KEY: + raise Exception(f"{name} requires OPEN_ROUTER_API_KEY in .env") + elif self.provider == "xai": + if not settings.XAI_API_KEY: + raise Exception(f"{name} requires XAI_API_KEY in .env") + + def __str__(self) -> str: + return f"{self.provider}/{self.model}" + + +class LLMWithFallback(LLM): + """LLM configuration with automatic secondary fallback and timeout specs.""" + + fallback: LLM + fix_model: str | None = None + timeout: float | None = None + + def __str__(self) -> str: + return f"{self.provider}/{self.model} (fallback: {self.fallback})" + + +class LLMConfigUtils(BaseModel): + """Configuration container for auxiliary utility agents/nodes.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + outputter: LLMWithFallback + hopper: LLMWithFallback + video_analyzer: LLMWithFallback | None = None + object_detector: LLMWithFallback | None = None + + +class LLMConfigBase(BaseModel): + """Base LLM configuration: planner, utility nodes and their accessors.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + planner: LLMWithFallback + utils: LLMConfigUtils + + def __str__(self) -> str: + return f""" +📃 Planner: {self.planner} +🧩 Utils: + 🔽 Hopper: {self.utils.hopper} + 📝 Outputter: {self.utils.outputter} + 🎬 Video Analyzer: {self.utils.video_analyzer or "Not configured"} + 👁️ Object Detector: {self.utils.object_detector or "Not configured"} +""" + + def validate_providers(self) -> None: + """Validate credentials of the planner and the utility nodes.""" + self.planner.validate_provider("Planner") + self.utils.outputter.validate_provider("Outputter") + self.utils.hopper.validate_provider("Hopper") + if self.utils.video_analyzer: + self.utils.video_analyzer.validate_provider("VideoAnalyzer") + if self.utils.object_detector: + self.utils.object_detector.validate_provider("ObjectDetector") + + def get_utils(self, item: LLMUtilsNode) -> LLMWithFallback: + """Retrieve model configuration for a specific utility node.""" + value = getattr(self.utils, item) + if value is None: + raise ValueError( + f"Utils '{item}' is not configured. Please add it to your LLM " + "config or enable it via AgentConfigBuilder." + ) + return value + + +def initialize_llm_config[C: LLMConfigBase](parse: Callable[[], C]) -> C: + """Parse the LLM config with ``parse`` and validate provider credentials.""" + llm_config = parse() + llm_config.validate_providers() + logger.success("LLM config initialized") + return llm_config + + +def load_llm_config_override[C: LLMConfigBase]( + path: Path, + get_default: Callable[[], C], + merge: Callable[[C, dict[str, Any]], C], +) -> C: + """Merge the JSON/JSONC overrides at ``path`` onto the default config. + + Falls back to the default config when the file is missing or the merged + config is invalid. + """ + default_config = get_default() + + override_config_dict: dict[str, Any] = {} + if path.exists(): + logger.info(f"Loading custom LLM config from {path.resolve()}...") + with open(path, encoding="utf-8") as f: + override_config_dict = load_jsonc(f) + else: + logger.warning(f"Custom LLM config not found at {path} - using default config") + + try: + return merge(default_config, override_config_dict) + except ValidationError as e: + logger.error(f"Invalid LLM config: {e}") + logger.info("Falling back to default config") + return default_config diff --git a/artemis/config/output.py b/third_party/mobile_use/config/output.py similarity index 81% rename from artemis/config/output.py rename to third_party/mobile_use/config/output.py index 1955b9a93..19ea9195f 100644 --- a/artemis/config/output.py +++ b/third_party/mobile_use/config/output.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. """Structured output schemas, event recording, and output file management.""" @@ -26,8 +30,8 @@ ENV_EVENTS_OUTPUT_PATH, ENV_RESULTS_OUTPUT_PATH, ) -from artemis.config.llm import CyFunctionDetector -from artemis.utils.logger import get_logger +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -105,6 +109,27 @@ def needs_structured_format(self, default_enabled: bool = True) -> bool: return False return bool(self.structured_output or self.output_description) + def structured_output_schema(self) -> dict | type[BaseModel] | None: + """The structured output as a schema accepted by ``with_structured_output``.""" + so = self.structured_output + if isinstance(so, dict): + return so + elif isinstance(so, BaseModel): + return type(so) + elif isinstance(so, type) and issubclass(so, BaseModel): + return so + return None + + def bind_structured_output(self, llm: Any, llm_fallback: Any) -> tuple[Any, Any] | None: + """Bind the structured output schema to the main and fallback models. + + Returns None when no structured output is configured. + """ + schema = self.structured_output_schema() + if schema is None: + return None + return llm.with_structured_output(schema), llm_fallback.with_structured_output(schema) + def prepare_output_files() -> tuple[str | None, str | None]: """Validate and prepare events and results output files from environment variables.""" diff --git a/third_party/mobile_use/context.py b/third_party/mobile_use/context.py new file mode 100644 index 000000000..89533dc7c --- /dev/null +++ b/third_party/mobile_use/context.py @@ -0,0 +1,119 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Context variables for global state management. + +Uses ContextVar to avoid prop drilling and maintain clean function signatures. +""" + +from pathlib import Path +from typing import Any, Literal + +try: + from enum import StrEnum +except ImportError: + from enum import Enum + + class StrEnum(str, Enum): + pass + + +from pydantic import BaseModel, ConfigDict + + +class AppLaunchResult(BaseModel): + """Result of initial app launch attempt.""" + + model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) + + locked_app_package: str + locked_app_initial_launch_success: bool | None + locked_app_initial_launch_error: str | None + + +class DevicePlatform(StrEnum): + """Mobile device platform enumeration.""" + + ANDROID = "android" + + +class DeviceContext(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + extra="allow", + ) + + host_platform: Literal["WINDOWS", "LINUX", "DARWIN", "MACOS"] | str = "DARWIN" + mobile_platform: DevicePlatform = DevicePlatform.ANDROID + device_id: str = "default-device" + + device_width: int = 1080 + device_height: int = 2400 + + def to_str(self): + return ( + f"Host platform: {self.host_platform}\n" + f"Mobile platform: {self.mobile_platform.value}\n" + f"Device ID: {self.device_id}\n" + f"Device width: {self.device_width}\n" + f"Device height: {self.device_height}\n" + ) + + +class ExecutionSetupBase(BaseModel): + """Execution setup for a task.""" + + model_config = ConfigDict( + arbitrary_types_allowed=True, + extra="allow", + ) + + traces_path: Path | None = None + trace_name: str | None = None + enable_remote_tracing: bool = False + app_lock_status: AppLaunchResult | None = None + + def get_locked_app_package(self) -> str | None: + """Get the locked app package name if app locking is enabled. + + Returns: + The locked app package name, or None if app locking is not enabled. + """ + if self.app_lock_status: + return self.app_lock_status.locked_app_package + return None + + +class DeviceClientAccessors: + """Checked accessors for the device clients held by a context model. + + The inheriting model declares the ``adb_client`` and ``ui_adb_client`` fields. + """ + + adb_client: Any + ui_adb_client: Any + + def get_adb_client(self) -> Any: + if self.adb_client is None: + raise ValueError("No ADB client in context.") + return self.adb_client + + def get_ui_adb_client(self) -> Any: + if self.ui_adb_client is None: + raise ValueError("No UIAutomator client in context.") + return self.ui_adb_client diff --git a/third_party/mobile_use/controllers/__init__.py b/third_party/mobile_use/controllers/__init__.py new file mode 100644 index 000000000..58d482ea3 --- /dev/null +++ b/third_party/mobile_use/controllers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/artemis/controllers/platform_specific_commands_controller.py b/third_party/mobile_use/controllers/platform_specific_commands_controller.py similarity index 92% rename from artemis/controllers/platform_specific_commands_controller.py rename to third_party/mobile_use/controllers/platform_specific_commands_controller.py index 97041fb1e..36c006f25 100644 --- a/artemis/controllers/platform_specific_commands_controller.py +++ b/third_party/mobile_use/controllers/platform_specific_commands_controller.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. import os from shutil import which @@ -21,8 +22,8 @@ from typing import Any from artemis.context import ArtemisContext, DevicePlatform -from artemis.utils.logger import ArtemisLogger, get_logger -from artemis.utils.shell_utils import run_shell_command_on_host +from third_party.mobile_use.utils.logger import ArtemisLogger, get_logger +from third_party.mobile_use.utils.shell_utils import run_shell_command_on_host logger = get_logger(__name__) diff --git a/third_party/mobile_use/controllers/types.py b/third_party/mobile_use/controllers/types.py new file mode 100644 index 000000000..e9c80566b --- /dev/null +++ b/third_party/mobile_use/controllers/types.py @@ -0,0 +1,92 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Geometry, element selector and tap result models shared by device drivers.""" + +from pydantic import BaseModel, ConfigDict, Field + + +class TapOutput(BaseModel): + """Output from tap operations.""" + + error: str | None = Field(default=None, description="Error message if tap failed") + success: bool = Field(default=True, description="Whether the tap was successfully dispatched") + + +class ElementQuery(BaseModel): + """Selects a UI element by resource id or text; ``index`` picks among the matches.""" + + model_config = ConfigDict(frozen=True) + resource_id: str | None = None + text: str | None = None + index: int = 0 + + def describe(self) -> str: + return f"resource_id={self.resource_id}, text={self.text}" + + +class CoordinatesSelectorRequest(BaseModel): + """Absolute pixel coordinate pair.""" + + model_config = ConfigDict(extra="forbid") + x: int + y: int + + def to_str(self) -> str: + return f"{self.x}, {self.y}" + + +class PercentagesSelectorRequest(BaseModel): + """Normalized percentage coordinate pair (0-100%).""" + + model_config = ConfigDict(extra="forbid") + x_percent: int = Field(ge=0, le=100, description="X percentage (0-100)") + y_percent: int = Field(ge=0, le=100, description="Y percentage (0-100)") + + def to_str(self) -> str: + return f"{self.x_percent}%, {self.y_percent}%" + + def to_coords(self, width: int, height: int) -> CoordinatesSelectorRequest: + """Convert percentages to absolute pixel coordinates.""" + x = min(max(int(width * self.x_percent / 100), 0), max(0, width - 1)) + y = min(max(int(height * self.y_percent / 100), 0), max(0, height - 1)) + return CoordinatesSelectorRequest(x=x, y=y) + + +class Bounds(BaseModel): + """Bounding box defined by top-left (x1, y1) and bottom-right (x2, y2).""" + + x1: int + y1: int + x2: int + y2: int + + @property + def width(self) -> int: + return max(0, self.x2 - self.x1) + + @property + def height(self) -> int: + return max(0, self.y2 - self.y1) + + def get_center(self) -> CoordinatesSelectorRequest: + """Get the center point of the bounds.""" + return CoordinatesSelectorRequest( + x=(self.x1 + self.x2) // 2, + y=(self.y1 + self.y2) // 2, + ) diff --git a/third_party/mobile_use/controllers/unified_controller.py b/third_party/mobile_use/controllers/unified_controller.py new file mode 100644 index 000000000..bc4035be8 --- /dev/null +++ b/third_party/mobile_use/controllers/unified_controller.py @@ -0,0 +1,183 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Device-control facade base: the upstream ``UnifiedMobileController`` API. + +Artemis extends it as ``artemis.controllers.unified_controller.UnifiedMobileController`` +with scrcpy video recording and timeline segment extraction. +""" + +from typing import Any + +from artemis.context import ArtemisContext +from artemis.controllers.device_controller import ScreenDataResponse +from artemis.drivers.base import BaseDeviceDriver +from third_party.mobile_use.controllers.types import ElementQuery, TapOutput + + +class UnifiedMobileControllerBase: + def __init__(self, ctx: ArtemisContext, driver: BaseDeviceDriver): + self.ctx = ctx + self._driver: BaseDeviceDriver = driver + + @property + def driver(self) -> BaseDeviceDriver: + return self._driver + + @property + def controller(self) -> Any: + return self._driver + + async def tap_at( + self, + x: int, + y: int, + long_press: bool = False, + long_press_duration: int = 1000, + times: int = 1, + delay_ms: int = 100, + ) -> TapOutput: + try: + if long_press: + success = await self._driver.long_press(x, y, duration_ms=long_press_duration) + else: + success = await self._driver.tap( + x, y, duration_ms=100, times=times, delay_ms=delay_ms + ) + return TapOutput(error=None if success else f"Tap failed at ({x}, {y})") + except Exception as e: + return TapOutput(error=str(e)) + + async def tap_percentage( + self, + x_percent: int, + y_percent: int, + long_press: bool = False, + long_press_duration: int = 1000, + ) -> TapOutput: + """Tap at percentage-based coordinates (0 to 100).""" + norm_x = int(x_percent * 10) + norm_y = int(y_percent * 10) + success = await self._driver.tap_normalized( + norm_x, norm_y, long_press=long_press, duration_ms=long_press_duration + ) + return TapOutput( + error=None if success else f"Tap percentage failed at ({x_percent}%, {y_percent}%)" + ) + + async def tap_element( + self, + resource_id: str | None = None, + text: str | None = None, + index: int = 0, + long_press: bool = False, + long_press_duration: int = 1000, + ) -> TapOutput: + """Tap on a UI element by finding it in the hierarchy.""" + query = ElementQuery(resource_id=resource_id, text=text, index=index) + success = await self._driver.tap_element( + query, long_press=long_press, duration_ms=long_press_duration + ) + return TapOutput(error=None if success else f"Failed to tap element ({query.describe()})") + + async def swipe_coords( + self, + start_x: int, + start_y: int, + end_x: int, + end_y: int, + duration: int = 400, + ) -> str | None: + """Swipe between two coordinate points.""" + success = await self._driver.swipe(start_x, start_y, end_x, end_y, duration_ms=duration) + return None if success else f"Swipe failed from ({start_x},{start_y}) to ({end_x},{end_y})" + + async def type_text(self, text: str, clear_existing: bool = True) -> bool: + return await self._driver.input_text(text, clear_existing=clear_existing) + + async def take_screenshot(self) -> str: + screen_data = await self._driver.get_screen_data() + return screen_data.screenshot_base64 + + async def launch_app(self, package_or_bundle_id: str) -> bool: + return await self._driver.launch_app(package_or_bundle_id) + + async def terminate_app(self, package_or_bundle_id: str | None) -> bool: + if not package_or_bundle_id: + return False + return await self._driver.stop_app(package_or_bundle_id) + + async def open_url(self, url: str) -> bool: + await self._driver.execute_shell(f"am start -a android.intent.action.VIEW -d '{url}'") + return True + + async def go_back(self) -> bool: + return await self._driver.press_key("back") + + async def go_home(self) -> bool: + return await self._driver.press_key("home") + + async def press_enter(self) -> bool: + return await self._driver.press_key("enter") + + async def press_key(self, keycode: str) -> bool: + return await self._driver.press_key(keycode) + + async def erase_text(self, nb_chars: int | None = None) -> bool: + if nb_chars is not None and nb_chars > 0: + for _ in range(nb_chars): + await self._driver.press_key("delete") + return True + # Best-effort full clear: End -> Ctrl+A -> Delete + try: + clear_cmd = ( + "input keyevent 123 && " + "input keycombination 113 29 && input keyevent 67 && " + "input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67" + ) + await self._driver.execute_shell(clear_cmd) + except Exception: + for _ in range(30): + await self._driver.press_key("delete") + return True + + async def get_ui_elements(self) -> list[dict]: + screen_data = await self._driver.get_screen_data() + return screen_data.ui_elements or [] + + async def get_screen_data(self) -> "ScreenDataResponse": + """Get screen data including screenshot, UI hierarchy, dimensions, and platform.""" + data = await self._driver.get_screen_data() + return ScreenDataResponse( + base64=data.screenshot_base64, + elements=data.ui_elements or [], + width=data.width, + height=data.height, + platform=data.platform, + ) + + async def find_element( + self, + resource_id: str | None = None, + text: str | None = None, + index: int = 0, + ) -> tuple[dict | None, str | None]: + elem, _, error = await self._driver.find_element( + ElementQuery(resource_id=resource_id, text=text, index=index) + ) + return elem, error diff --git a/third_party/mobile_use/graph.py b/third_party/mobile_use/graph.py new file mode 100644 index 000000000..341aae523 --- /dev/null +++ b/third_party/mobile_use/graph.py @@ -0,0 +1,54 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""State reducer and convergence node used when building the agent graph.""" + +from collections.abc import Callable, Hashable +from typing import Any + +from langgraph.graph import StateGraph + +CONVERGENCE_NODE = "convergence" + + +def take_last(a, b): + """Reducer function keeping the latest value.""" + return b + + +def convergence_node(state: Any): + """Convergence point for parallel execution paths.""" + return {} + + +def add_convergence_node(graph_builder: StateGraph) -> None: + """Add the deferred node where parallel branches join.""" + graph_builder.add_node(node=CONVERGENCE_NODE, action=convergence_node, defer=True) + + +def add_convergence_edges( + graph_builder: StateGraph, + gate: Callable[..., Hashable], + path_map: dict[Hashable, str], +) -> None: + """Add the conditional edges leaving the convergence node, chosen by ``gate``.""" + graph_builder.add_conditional_edges( + source=CONVERGENCE_NODE, + path=gate, + path_map=path_map, + ) diff --git a/third_party/mobile_use/llm_service.py b/third_party/mobile_use/llm_service.py new file mode 100644 index 000000000..e06f61005 --- /dev/null +++ b/third_party/mobile_use/llm_service.py @@ -0,0 +1,193 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Helpers for LLM calls, from upstream ``services/llm.py``. + +``LLMWaitNotice`` waits for a call, tells the user when it is slow and enforces +a hard timeout. ``FallbackRunner`` retries a failed call on a fallback model. +``GetLLM`` is the type of ``get_llm``. + +Artemis passes in its own behaviour (pause detection, error classification, +telemetry); this module does not import artemis at runtime. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Coroutine +from contextlib import AbstractContextManager, nullcontext +import logging +from typing import TYPE_CHECKING, Any, Literal, Protocol, overload + +if TYPE_CHECKING: + from langchain_core.language_models.chat_models import BaseChatModel + + from artemis.config import AgentNodeWithFallback, LLMUtilsNode, LLMUtilsNodeWithFallback + from artemis.context import ArtemisContext + +_logger = logging.getLogger(__name__) + +WAITING_MESSAGE = "Waiting for LLM call response..." + + +class LLMWaitNotice: + """Wait for LLM calls, report slow ones and enforce a hard timeout. + + Args: + notify: Called with :data:`WAITING_MESSAGE` once a call outlives its notice delay. + on_timeout: Called with the timeout message right before ``TimeoutError`` is raised. + hold: While it returns ``True`` the hard-timeout clock is held at zero + (e.g. the task is paused by the user). + default_hard_timeout: Hard timeout in seconds when the call does not pass + one; ``None`` waits indefinitely. + poll_interval: Seconds between ``hold``/timeout checks after the notice. + """ + + def __init__( + self, + *, + notify: Callable[[str], object], + on_timeout: Callable[[str], object], + hold: Callable[[], bool] = lambda: False, + default_hard_timeout: float | None = None, + poll_interval: float = 1.0, + ) -> None: + self._notify = notify + self._on_timeout = on_timeout + self._hold = hold + self._default_hard_timeout = default_hard_timeout + self._poll_interval = poll_interval + + async def __call__[T]( + self, + llm_call: Coroutine[Any, Any, T], + timeout_seconds: float = 10, + hard_timeout: float | None = None, + ) -> T: + """Run ``llm_call``; show a notice after ``timeout_seconds``. + + The call is cancelled if the caller is cancelled or the hard timeout + (counted from the start of the call) expires. + """ + if hard_timeout is None: + hard_timeout = self._default_hard_timeout + llm_task = asyncio.create_task(llm_call) + try: + # asyncio.wait never raises on timeout and needs no timer task. + done, _ = await asyncio.wait({llm_task}, timeout=timeout_seconds) + if llm_task in done: + return llm_task.result() + + self._notify(WAITING_MESSAGE) + loop = asyncio.get_running_loop() + budget = None if hard_timeout is None else max(0.0, hard_timeout - timeout_seconds) + start_time = loop.time() + while True: + done, _ = await asyncio.wait({llm_task}, timeout=self._poll_interval) + if llm_task in done: + return llm_task.result() + if self._hold(): + start_time = loop.time() + continue + if budget is not None and loop.time() - start_time > budget: + message = f"LLM call timed out after {hard_timeout} seconds." + self._on_timeout(message) + raise TimeoutError(message) + except BaseException: + if not llm_task.done(): + llm_task.cancel() + await asyncio.gather(llm_task, return_exceptions=True) + raise + + +class FallbackRunner: + """Runs a main LLM call and switches to a fallback call when it fails. + + The base class falls back on every ``Exception`` and on a ``None`` result. + Subclasses narrow that down and report switches by overriding the hooks + :meth:`main_scope`, :meth:`fallback_reason` and :meth:`on_fallback`. + """ + + def main_scope(self) -> AbstractContextManager[object]: + """Context active while the main call runs; exited before the fallback starts.""" + return nullcontext() + + def fallback_reason(self, error: Exception) -> tuple[str, str | None] | None: + """``(reason, category)`` if ``error`` should trigger the fallback, ``None`` to re-raise.""" + return "error", None + + def on_fallback(self, reason: str, category: str | None, error: str | None) -> None: + """Report a switch to the fallback call.""" + detail = f": {error}" if error else "" + _logger.warning(f"❗ Main LLM inference failed ({reason}{detail}). Falling back...") + + async def run[T]( + self, + main_call: Callable[[], Awaitable[T]], + fallback_call: Callable[[], Awaitable[T]], + none_should_fallback: bool = True, + ) -> T: + try: + with self.main_scope(): + result = await main_call() + except Exception as e: + decision = self.fallback_reason(e) + if decision is None: + raise + self.on_fallback(decision[0], decision[1], str(e)) + return await fallback_call() + + if result is None and none_should_fallback: + self.on_fallback("empty_result", None, None) + return await fallback_call() + return result + + +class GetLLM(Protocol): + """Call signature of ``get_llm``: resolve the chat model configured for a node.""" + + @overload + def __call__( + self, + ctx: ArtemisContext, + name: AgentNodeWithFallback, + *, + use_fallback: bool = False, + temperature: float | None = None, + ) -> BaseChatModel: ... + + @overload + def __call__( + self, + ctx: ArtemisContext, + name: LLMUtilsNode, + *, + is_utils: Literal[True], + temperature: float | None = None, + ) -> BaseChatModel: ... + + @overload + def __call__( + self, + ctx: ArtemisContext, + name: LLMUtilsNodeWithFallback, + *, + is_utils: Literal[True], + use_fallback: bool = False, + temperature: float | None = None, + ) -> BaseChatModel: ... diff --git a/third_party/mobile_use/main.py b/third_party/mobile_use/main.py new file mode 100644 index 000000000..3e194c8dc --- /dev/null +++ b/third_party/mobile_use/main.py @@ -0,0 +1,130 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Single-task automation driver (upstream ``run_automation``). + +Used by the ``artemis run`` CLI command in ``artemis.interfaces.cli.commands.run``. +""" + +import os +from pathlib import Path +from typing import Annotated + +import typer + +from artemis import Agent, Builders +from artemis.config import initialize_llm_config, settings +from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder +from third_party.mobile_use.sdk.types.task import AgentProfile +from third_party.mobile_use.utils.video import check_ffmpeg_available + +# CLI parameters of the ``run`` command. +GoalArgument = Annotated[str, typer.Argument(help="The main goal for the agent to achieve.")] +TestNameOption = Annotated[ + str | None, + typer.Option( + "--test-name", + "-n", + help="Name of the test run for trace recording.", + ), +] +TracesPathOption = Annotated[ + str | None, + typer.Option( + "--traces-path", + "-t", + help="Directory where execution traces are persisted.", + ), +] +OutputDescriptionOption = Annotated[ + str | None, + typer.Option( + "--output-description", + "-o", + help="Natural language or schema description of the expected output.", + ), +] +VideoRecordingToolsOption = Annotated[ + bool | None, + typer.Option( + "--with-video-recording-tools/--without-video-recording-tools", + help=( + "Enable or disable dynamic video recording and screen analysis " + "tools (auto-detected if omitted)." + ), + ), +] + + +def ensure_video_recording_available(with_video_recording_tools: bool | None) -> None: + """Fail early when video recording tools are requested but ffmpeg is missing.""" + if with_video_recording_tools: + check_ffmpeg_available() + + +def new_default_config_builder() -> AgentConfigBuilder: + """Return a config builder set up with the default LLM profile and ADB server.""" + llm_config = initialize_llm_config() + agent_profile = AgentProfile(name="default", llm_config=llm_config) + config = Builders.AgentConfig.with_default_profile(profile=agent_profile) + if settings.ADB_HOST: + config.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT) + return config + + +async def run_automation( + config: AgentConfigBuilder, + goal: str, + session_id: str | None = None, + locked_app_package: str | None = None, + test_name: str | None = None, + traces_output_path_str: str | None = None, + output_description: str | None = None, + profile: str | None = None, + app_path: str | None = None, +) -> None: + """Initialize an agent from ``config``, run ``goal`` as a single task, then clean up.""" + agent: Agent | None = None + try: + agent = Agent(config=config.build(), session_id=session_id) + await agent.init( + retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)), + retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)), + ) + + task = agent.new_task(goal) + if locked_app_package: + task.with_locked_app_package(locked_app_package) + if test_name: + trace_path = traces_output_path_str or str(settings.TRACES_PATH) + task.with_name(test_name).with_trace_recording(path=trace_path) + if output_description: + task.with_output_description(output_description) + if profile: + task.using_profile(profile) + if app_path: + task.with_app_path(Path(app_path)) + + llm_result_path = os.getenv("RESULTS_OUTPUT_PATH", None) + if llm_result_path: + task.with_llm_output_saving(path=llm_result_path) + + await agent.run_task(request=task.build()) + finally: + if agent is not None: + await agent.clean() diff --git a/third_party/mobile_use/sdk/__init__.py b/third_party/mobile_use/sdk/__init__.py new file mode 100644 index 000000000..58d482ea3 --- /dev/null +++ b/third_party/mobile_use/sdk/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/third_party/mobile_use/sdk/agent.py b/third_party/mobile_use/sdk/agent.py new file mode 100644 index 000000000..c6a812031 --- /dev/null +++ b/third_party/mobile_use/sdk/agent.py @@ -0,0 +1,1060 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Base SDK agent: device/client setup, task creation and the task run loop. + +Artemis extends this class in ``artemis.sdk.agent.Agent`` with tracing setup, device +environment preparation, cancellation watching and UI hierarchy backend reporting. +""" + +import asyncio +import os + +try: + from datetime import UTC, datetime +except ImportError: + from datetime import datetime + + UTC = UTC +import shutil +import threading +import uuid +from pathlib import Path +from platform import system +from shutil import which +from types import NoneType +from typing import Any, TypeVar, overload + +from adbutils import AdbClient +from PIL import Image +from pydantic import BaseModel + +from artemis.agents.flash.runner import FlashRunner +from artemis.agents.outputter.outputter import outputter +from artemis.clients.screen_client_factory import ( + ScreenClient, + create_screen_client, +) +from artemis.config import ( + OutputConfig, + record_events, +) +from artemis.constants import RECURSION_LIMIT +from artemis.context import ( + ArtemisContext, + DeviceContext, + DevicePlatform, + ExecutionSetup, +) +from artemis.controllers.controller_factory import get_controller +from artemis.data_engine.trace import DataEngineCallbackHandler +from artemis.graph.graph import get_graph +from artemis.graph.state import State +from artemis.runtime import DeviceExecutionLock +from artemis.sdk.run_outcome import attach_test_summary, resolve_trace_suffix +from artemis.sdk.types.agent import AgentConfig +from artemis.utils.startup_progress import publish_startup_progress +from third_party.mobile_use.controllers.platform_specific_commands_controller import ( + get_first_device, +) +from third_party.mobile_use.sdk.builders.task_request_builder import TaskRequestBuilder +from third_party.mobile_use.sdk.types.exceptions import ( + AgentError, + AgentNotInitializedError, + AgentProfileNotFoundError, + AgentTaskRequestError, + DeviceNotFoundError, + ExecutableNotFoundError, +) +from third_party.mobile_use.sdk.types.task import ( + AgentProfile, + Task, + TaskRequest, +) +from third_party.mobile_use.utils.app_launch_utils import _handle_initial_app_launch +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.media import ( + create_gif_from_trace_folder, + create_steps_json_from_trace_folder, + remove_images_from_trace_folder, + remove_steps_json_from_trace_folder, +) + +logger = get_logger(__name__) + +TOutput = TypeVar("TOutput", bound=BaseModel | None) + + +class AgentBase: + _config: AgentConfig + _tasks: list[Task] = [] + _tmp_traces_dir: Path + _initialized: bool = False + _device_context: DeviceContext + _adb_client: AdbClient | None + _ui_adb_client: ScreenClient | Any | None + _current_task: asyncio.Task | None = None + _task_lock: asyncio.Lock + _cloud_mobile_id: str | None = None + + async def init( + self, + api_key: str | None = None, + device_id: str | None = None, + device_serial: str | None = None, + retry_count: int = 5, + retry_wait_seconds: int = 5, + ): + target_dev = device_serial or device_id + if target_dev: + self._config = self._config.model_copy( + update={"device_id": target_dev, "device_platform": DevicePlatform.ANDROID} + ) + + return await self._init_internal( + api_key=api_key, + retry_count=retry_count, + retry_wait_seconds=retry_wait_seconds, + ) + + async def _init_internal( + self, + api_key: str | None = None, + retry_count: int = 5, + retry_wait_seconds: int = 5, + ): + + if os.environ.get("ARTEMIS_CLOUD_MODE") != "1" and not which("adb"): + raise ExecutableNotFoundError("adb") + + if self._initialized: + logger.warning("Agent is already initialized. Skipping...") + return True + + # Get first available device ID + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + device_id = os.environ.get("ADB_DEVICE_SERIAL", "cloud_device") + platform = DevicePlatform.ANDROID + elif not self._config.device_id or not self._config.device_platform: + device_id, platform, _ = get_first_device(logger=logger) + else: + device_id, platform = ( + self._config.device_id, + self._config.device_platform, + ) + + if not device_id or not platform: + error_msg = "No device found. Exiting." + logger.error(error_msg) + raise DeviceNotFoundError(error_msg) + + # Initialize clients + publish_startup_progress( + "device_check", "Checking the Android device", session_id=self._session_id + ) + if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": + self._init_clients( + device_id=device_id, + platform=platform, + ) + else: + # Cloud mode is pinned to UIAutomator2 through the gateway; the + # hierarchy backend switch only applies to locally attached devices. + from cloud_service.virtualization import RemoteAdbClient, RemoteUIAutomatorClient + + self._adb_client = RemoteAdbClient() + self._ui_adb_client = RemoteUIAutomatorClient( + device_id=device_id, + adb_client=self._adb_client, + ) + + self._device_context = await self._get_device_context( + device_id=device_id, platform=platform + ) + logger.info(self._device_context.to_str()) + publish_startup_progress( + "device_ready", "Android device connected", session_id=self._session_id + ) + + # Asynchronously pre-warm LLM connection pools in the background + asyncio.create_task(self._prewarm_llm_connections(api_key)) + + logger.info("✅ Artemis agent initialized.") + self._initialized = True + + return True + + async def install_apk(self, apk_path: str | Path) -> None: + """Install an APK on the connected device. + + For cloud mobiles, the APK must be x86_64 compatible. + + Args: + apk_path: Path to the local APK file to install + + Raises: + AgentNotInitializedError: If the agent is not initialized + AgentError: If attempting to install on non-Android device or ADB + operations fail + FileNotFoundError: If the APK file doesn't exist + CloudMobileServiceUninitializedError: If cloud service is + unavailable + """ + await self._install_apk_internal(apk_path) + + async def _install_apk_internal(self, apk_path: str | Path) -> None: + if isinstance(apk_path, str): + apk_path = Path(apk_path) + + if not apk_path.exists(): + raise FileNotFoundError(f"APK file not found: {apk_path}") + + if not self._initialized: + raise AgentNotInitializedError() + + device_id = self._device_context.device_id + logger.info(f"Installing APK on Android device '{device_id}'") + if not self._adb_client: + raise AgentError("ADB client not initialized") + + device = self._adb_client.device(serial=device_id) + await asyncio.to_thread(device.install, apk_path) + logger.info(f"APK installed successfully on Android device '{device_id}'") + + async def install_app(self, app_path: str | Path) -> str | None: + """Install an app on the connected device. + + For Android: Installs an APK file using ADB. + + Args: + app_path: Path to the app to install (Android APK file). + + Returns: + None. + + Raises: + AgentNotInitializedError: If the agent is not initialized + AgentError: If installation fails or platform is unsupported + FileNotFoundError: If the app file/folder doesn't exist + """ + return await self._install_app_internal(app_path) + + async def _install_app_internal(self, app_path: str | Path) -> str | None: + if isinstance(app_path, str): + app_path = Path(app_path) + + if not app_path.exists(): + raise FileNotFoundError(f"App not found: {app_path}") + + if not self._initialized: + raise AgentNotInitializedError() + + await self._install_apk_internal(app_path) + return None + + def new_task(self, goal: str): + """Create a new task request builder. + + Args: + goal: Natural language description of what to accomplish + + Returns: + TaskRequestBuilder that can be configured with: + - .with_output_format() for structured output + - .with_output_description() for output description + - .with_locked_app_package() to restrict execution to a specific app + - .using_profile() to specify an LLM profile + - .with_max_steps() to set maximum execution steps + - .with_trace_recording() to enable trace recording + - .with_name() to set a custom task name + """ + return TaskRequestBuilder[None].from_common( + goal=goal, + common=self._config.task_request_defaults, + ) + + @overload + async def run_task( + self, + *, + goal: str, + output: type[TOutput], + profile: str | AgentProfile | None = None, + name: str | None = None, + locked_app_package: str | None = None, + app_path: str | Path | None = None, + ) -> TOutput | None: ... + + @overload + async def run_task( + self, + *, + goal: str, + output: str, + profile: str | AgentProfile | None = None, + name: str | None = None, + locked_app_package: str | None = None, + app_path: str | Path | None = None, + ) -> str | dict | None: ... + + @overload + async def run_task( + self, + *, + goal: str, + output=None, + profile: str | AgentProfile | None = None, + name: str | None = None, + locked_app_package: str | None = None, + app_path: str | Path | None = None, + ) -> str | None: ... + + @overload + async def run_task( + self, + *, + request: TaskRequest[None], + locked_app_package: str | None = None, + app_path: str | Path | None = None, + ) -> str | dict | None: ... + + @overload + async def run_task( + self, + *, + request: TaskRequest[TOutput], + locked_app_package: str | None = None, + app_path: str | Path | None = None, + ) -> TOutput | None: ... + + async def run_task( + self, + *, + goal: str | None = None, + output: type[TOutput] | str | None = None, + profile: str | AgentProfile | None = None, + locked_app_package: str | None = None, + name: str | None = None, + app_path: str | Path | None = None, + request: TaskRequest[TOutput] | None = None, + ) -> str | dict | TOutput | None: + + # Normal local execution path + if request is not None: + if locked_app_package is not None: + if request.locked_app_package: + logger.warning( + "Locked app package specified both in the request and" + " as a parameter. Using the parameter value." + ) + request.locked_app_package = locked_app_package + # Handle app_path parameter override + if app_path is not None: + if request.app_path: + logger.warning( + "App path specified both in the request and as a" + " parameter. Using the parameter value." + ) + request.app_path = Path(app_path) if isinstance(app_path, str) else app_path + return await self._run_task(request=request) + if goal is None: + raise AgentTaskRequestError("Goal is required") + task_request = self.new_task(goal=goal) + if output is not None: + if isinstance(output, str): + task_request.with_output_description(description=output) + elif output is not NoneType: + task_request.with_output_format(output_format=output) + if profile is not None: + task_request.using_profile(profile=profile) + if name is not None: + task_request.with_name(name=name) + if locked_app_package is not None: + task_request.with_locked_app_package(package_name=locked_app_package) + if app_path is not None: + task_request.with_app_path(app_path=app_path) + return await self._run_task(task_request.build()) + + async def _run_task( + self, + request: TaskRequest[TOutput], + ) -> str | dict | TOutput | None: + if not self._initialized: + raise AgentNotInitializedError() + + if request.profile: + agent_profile = self._config.agent_profiles.get(request.profile) + if agent_profile is None: + if request.profile.lower() in ( + "flash", + "pro", + "ultra", + "default", + ): + agent_profile = self._config.default_profile + else: + raise AgentProfileNotFoundError(request.profile) + else: + agent_profile = self._config.default_profile + + logger.info(str(agent_profile)) + + on_status_changed = None + task_id = str( + self._session_id + or os.getenv("ARTEMIS_SESSION_ID") + or os.getenv("ARTEMIS_CLOUD_SESSION_ID") + or uuid.uuid4() + ) + + task = Task( + id=task_id, + device=self._device_context, + status="pending", + request=request, + created_at=datetime.now(), + on_status_changed=on_status_changed, + ) + self._tasks.append(task) + task_name = task.get_name() + + context = ArtemisContext( + device=self._device_context, + adb_client=self._adb_client, + ui_adb_client=self._ui_adb_client, + llm_config=agent_profile.llm_config, + agent_config=self._config, + ) + + output_config = None + if request.output_description or request.output_format: + output_config = OutputConfig( + output_description=request.output_description, + structured_output=request.output_format, # type: ignore + ) + logger.info(str(output_config)) + + logger.info(f"[{task_name}] Starting graph with goal: `{request.goal}`") + state = self._get_graph_state(task=task) + graph_input = state.model_dump() + datetime.now(UTC) + + async def _execute_task_logic(): + last_state: State | None = None + last_state_snapshot: dict | None = None + output = None + effective_mode = getattr(self._config, "concurrency_mode", "per_device") + effective_max = getattr(self._config, "max_concurrency", None) + sess_id = ( + self._session_id + or os.getenv("ARTEMIS_SESSION_ID") + or os.getenv("ARTEMIS_CLOUD_SESSION_ID") + or getattr(task, "id", None) + or getattr(getattr(task, "request", None), "task_name", None) + ) + active_owner = DeviceExecutionLock.get_active_owner(self._device_context.device_id) + already_held = ( + active_owner is not None + and active_owner.pid == os.getpid() + and ( + (sess_id and str(active_owner.session_id) == str(sess_id)) + or active_owner.token == self._device_context.device_id + ) + ) + device_lock = ( + None + if already_held + else DeviceExecutionLock( + self._device_context.device_id, + description=f"{request.goal[:120]}", + concurrency_mode=effective_mode, + max_concurrency=effective_max, + session_id=str(sess_id) if sess_id else None, + ingress=os.getenv("ARTEMIS_TASK_INGRESS") or "agent", + ) + ) + try: + if device_lock is not None and os.environ.get("ARTEMIS_CLOUD_MODE") != "1": + queue_cancel_event = threading.Event() + acquire_task = asyncio.create_task( + asyncio.to_thread( + device_lock.acquire, + cancel_event=queue_cancel_event, + ) + ) + try: + await asyncio.shield(acquire_task) + except asyncio.CancelledError: + queue_cancel_event.set() + try: + await asyncio.shield(acquire_task) + except Exception as exc: # pylint: disable=broad-exception-caught + # Draining the lock acquisition after cancellation is best effort. + logger.debug( + f"Device lock acquisition drain after cancel failed: {exc}", + exc_info=True, + ) + raise + # All device mutation and UI initialization happens only after + # this task reaches the head of the one global FIFO queue. + # Session creation must be inside the same boundary as well: + # DataEngine uses a shared database, so a queued task must not + # publish a new active session while the current task is still + # finishing. + self._prepare_tracing(task=task, context=context) + self._prepare_output_files(task=task) + if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": + if self._ui_adb_client is not None: + await self._connect_screen_client(context, str(sess_id)) + await self._ensure_device_unlocked() + publish_startup_progress( + "environment", "Preparing the device environment", session_id=str(sess_id) + ) + await self._prepare_app_installation(task=task) + await self._prepare_device_environment(context=context) + await self._prepare_app_lock(task=task, context=context) + async with context: + recording_started = False + if self._config.video_recording_tools_enabled: + try: + controller = get_controller(context) + + output_dir = None + if context.execution_setup and context.execution_setup.traces_path: + output_dir = ( + self._tmp_traces_dir / context.execution_setup.trace_name + ).resolve() + + logger.info(f"[{task_name}] Starting automated screen recording...") + start_res = await controller.start_video_recording( + output_dir=output_dir + ) + if start_res and start_res.success: + recording_started = True + else: + logger.warning( + f"[{task_name}] Failed to start screen" + f" recording: {start_res.message if start_res else 'unknown'}" + ) + except Exception as e: + logger.error(f"[{task_name}] Failed to start screen recording: {e}") + + publish_startup_progress( + "environment_ready", "Device environment is ready", session_id=str(sess_id) + ) + try: + if request.profile and request.profile.lower() == "flash": + logger.info(f"[{task_name}] Invoking FlashRunner reactive loop...") + await task.set_status( + status="running", + message="Invoking FlashRunner...", + ) + + # An explicit max_steps caps the reactive loop; the + # default leaves the cap to agent.flash.max_turns + # (0 = unlimited). + max_turns = ( + request.max_steps if request.max_steps != RECURSION_LIMIT else None + ) + runner = FlashRunner(context, goal=request.goal, max_turns=max_turns) + flash_result = await runner.run(state) + + output = flash_result + last_state_snapshot = state.model_dump() + + status = flash_result.get("status") + if status == "completed": + logger.info(f"✅ Automation '{task_name}' is success ✅") + await task.finalize(content=output, state=last_state_snapshot) + if context.data_engine: + context.data_engine.end_session("completed") + else: + err = ( + f"[{task_name}] FlashRunner failed:" + f" {flash_result.get('explanation')}" + ) + logger.warning(err) + await task.finalize( + content=output, + state=last_state_snapshot, + error=err, + ) + if context.data_engine: + context.data_engine.end_session("failed") + return output + else: + logger.info(f"[{task_name}] Invoking graph with input: {graph_input}") + await task.set_status(status="running", message="Invoking graph...") + async for chunk in (await get_graph(context)).astream( + input=graph_input, + config={ + "recursion_limit": task.request.max_steps, + "callbacks": ( + ( + list(self._config.graph_config_callbacks) + if self._config.graph_config_callbacks is not None + else [] + ) + + [DataEngineCallbackHandler(context)] + if context.data_engine + else self._config.graph_config_callbacks + ), + }, + stream_mode=[ + "messages", + "custom", + "updates", + "values", + ], + ): + stream_mode, payload = chunk + if stream_mode == "values": + last_state_snapshot = payload # type: ignore + last_state = State(**last_state_snapshot) # type: ignore + + if not last_state: + err = f"[{task_name}] No result received from graph" + logger.warning(err) + await task.finalize( + content=output, + state=last_state_snapshot, + error=err, + ) + if context.data_engine: + context.data_engine.end_session("failed") + return None + + output = await self._extract_output( + task_name=task_name, + ctx=context, + request=request, + output_config=output_config, + state=last_state, + ) + # Run outcome (goal axis x test axis) drives the + # wrap-up instead of an unconditional success path. + run_outcome = getattr(last_state, "run_outcome", None) + context.run_outcome = run_outcome + output = attach_test_summary(output, run_outcome) + task_status_axis = ( + run_outcome.get("task_status") if run_outcome else None + ) + tests_failed = ( + int((run_outcome.get("tests") or {}).get("failed", 0)) + if run_outcome + else 0 + ) + if task_status_axis == "blocked": + err = ( + f"[{task_name}] Task ended blocked: verify" + " criteria unmet after exhausting the final" + " check budget." + ) + logger.warning(err) + await task.finalize( + content=output, + state=last_state_snapshot, + error=err, + ) + if context.data_engine: + context.data_engine.end_session("failed") + return output + + if tests_failed > 0: + logger.warning( + f"[{task_name}] Task completed, but" + f" {tests_failed} assertion(s) failed —" + " see the test summary." + ) + else: + logger.info(f"✅ Automation '{task_name}' is success ✅") + await task.finalize(content=output, state=last_state_snapshot) + if context.data_engine: + context.data_engine.end_session("completed") + + return output + finally: + if recording_started: + + async def _safe_stop_recording(): + try: + logger.info( + f"[{task_name}] Stopping automated screen recording..." + ) + controller = get_controller(context) + return await controller.stop_video_recording() + except Exception as e: + logger.error( + f"[{task_name}] Error stopping screen recording: {e}" + ) + return None + + stop_task = asyncio.create_task(_safe_stop_recording()) + try: + stop_result = await asyncio.shield(stop_task) + except (asyncio.CancelledError, BaseException): + try: + stop_result = await asyncio.wait_for(stop_task, timeout=15.0) + except Exception: + stop_result = None + + if stop_result and stop_result.success and stop_result.video_path: + logger.info( + f"[{task_name}] Screen recording saved" + f" to: {stop_result.video_path}" + ) + elif stop_result: + logger.warning( + f"[{task_name}] Failed to save screen" + f" recording: {stop_result.message}" + ) + except asyncio.CancelledError: + err = f"[{task_name}] Task cancelled" + logger.warning(err) + await task.finalize( + content=output, + state=last_state_snapshot, + error=err, + cancelled=True, + ) + if context.data_engine: + context.data_engine.end_session("cancelled") + + raise + except Exception as e: + err = f"[{task_name}] Error running automation: {e}" + logger.error(err) + await task.finalize( + content=output, + state=last_state_snapshot, + error=err, + ) + if context.data_engine: + context.data_engine.end_session("failed") + + raise + finally: + try: + # Background ADB processes (logcat, screenrecord, ...) started + # by the Operator must not outlive the automation task. + from artemis.tools.command_tool import shutdown_adb_background_tasks + + await asyncio.wait_for(shutdown_adb_background_tasks(context), timeout=15.0) + except Exception as e: + logger.warning(f"[{task_name}] Failed to stop background ADB tasks: {e}") + try: + await self._finalize_tracing_safely(task=task, context=context) + finally: + if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": + try: + if self._ui_adb_client is not None: + await asyncio.to_thread(self._ui_adb_client.disconnect) + finally: + if device_lock is not None: + await asyncio.to_thread(device_lock.release) + + async with self._task_lock: + if self._current_task and not self._current_task.done(): + logger.warning( + "Another automation task is already running. " + "Stopping it before starting the new one." + ) + self.stop_current_task() + try: + await self._current_task + except asyncio.CancelledError: + pass + + cancel_watcher: asyncio.Task | None = None + try: + self._current_task = asyncio.create_task(_execute_task_logic()) + cancel_watcher = asyncio.create_task(self._watch_external_cancel(task_name)) + return await self._current_task + finally: + self._current_task = None + if cancel_watcher is not None and not cancel_watcher.done(): + cancel_watcher.cancel() + try: + await cancel_watcher + except asyncio.CancelledError: + pass + + def stop_current_task(self): + """Requests cancellation of the currently running automation task.""" + if self._current_task and not self._current_task.done(): + logger.info("Requesting to stop the current automation task...") + was_cancelled = self._current_task.cancel() + if was_cancelled: + logger.success("Cancellation request for the current task was sent.") + else: + logger.warning( + "Could not send cancellation request for the current task " + "(it may already be completing)." + ) + else: + logger.info("No active automation task to stop.") + + async def get_screenshot(self) -> Image.Image: + """Capture a screenshot from the mobile device. + + Returns: + Screenshot as PIL Image + + Raises: + AgentNotInitializedError: If the agent is not initialized + Exception: If screenshot capture fails + """ + if not self._initialized: + raise AgentNotInitializedError() + + # Use ADB to capture screenshot + logger.info("Capturing screenshot from local Android device") + if not self._adb_client: + raise Exception("ADB client not initialized") + + device = self._adb_client.device(serial=self._device_context.device_id) + screenshot = await asyncio.to_thread(device.screenshot) + logger.info("Screenshot captured from local Android device") + return screenshot + + async def _prepare_app_installation(self, task: Task) -> str | None: + """Install app if app_path is specified in the task request. + + Returns: + None. + """ + if not task.request.app_path: + return None + + task_name = task.get_name() + logger.info(f"[{task_name}] Installing app from: {task.request.app_path}") + + await self.install_app(task.request.app_path) + + return None + + async def _prepare_app_lock(self, task: Task, context: ArtemisContext): + """Prepare app lock by launching the locked app if specified.""" + if not task.request.locked_app_package: + return + + task_name = task.get_name() + logger.info(f"[{task_name}] Preparing app lock for: {task.request.locked_app_package}") + + app_lock_status = await _handle_initial_app_launch( + ctx=context, locked_app_package=task.request.locked_app_package + ) + + if context.execution_setup is None: + context.execution_setup = ExecutionSetup(app_lock_status=app_lock_status) + else: + context.execution_setup.app_lock_status = app_lock_status + + if app_lock_status.locked_app_initial_launch_success is False: + error = app_lock_status.locked_app_initial_launch_error + logger.warning(f"[{task_name}] Failed to launch locked app: {error}") + + def _prepare_trace_paths(self, task: Task) -> str: + """Create the temp (and, when recording, output) trace directories; return the task name.""" + task_name = task.get_name() + temp_trace_path = Path(self._tmp_traces_dir / task_name).resolve() + temp_trace_path.mkdir(parents=True, exist_ok=True) + + if task.request.record_trace: + traces_output_path = Path(task.request.trace_path).resolve() + logger.info(f"[{task_name}] 📂 Traces output path: {traces_output_path}") + logger.info(f"[{task_name}] 📄📂 Traces temp path: {temp_trace_path}") + traces_output_path.mkdir(parents=True, exist_ok=True) + return task_name + + async def _finalize_tracing(self, task: Task, context: ArtemisContext): + if context.data_engine: + session_status = "completed" + if task.status == "failed": + session_status = "failed" + elif task.status == "cancelled": + session_status = "cancelled" + context.data_engine.end_session(status=session_status) + + exec_setup_ctx = context.execution_setup + if not exec_setup_ctx: + return + + if exec_setup_ctx.traces_path is None or exec_setup_ctx.trace_name is None: + return + + task_name = task.get_name() + temp_trace_path = (self._tmp_traces_dir / exec_setup_ctx.trace_name).resolve() + + if not task.request.record_trace: + try: + if temp_trace_path.exists(): + shutil.rmtree(temp_trace_path) + logger.info(f"[{task_name}] Cleaned up temporary trace folder.") + except Exception as e: + logger.error(f"[{task_name}] Failed to clean up temp trace folder: {e}") + return + + status = resolve_trace_suffix(task.status, getattr(context, "run_outcome", None)) + ts = task.created_at.strftime("%Y-%m-%dT%H-%M-%S") + new_name = f"{exec_setup_ctx.trace_name}{status}_{ts}" + + traces_output_path = Path(task.request.trace_path).resolve() + + logger.info(f"[{task_name}] Compiling trace FROM FOLDER: " + str(temp_trace_path)) + create_gif_from_trace_folder(temp_trace_path) + create_steps_json_from_trace_folder(temp_trace_path) + + logger.info(f"[{task_name}] Video created, removing dust...") + remove_images_from_trace_folder(temp_trace_path) + remove_steps_json_from_trace_folder(temp_trace_path) + logger.info(f"[{task_name}] 📽️ Trace compiled, moving to output path 📽️") + + output_folder_path = temp_trace_path.rename(traces_output_path / new_name).resolve() + logger.info(f"[{task_name}] 📂✅ Traces located in: {output_folder_path}") + + new_video_path = output_folder_path / "recording.mp4" + if new_video_path.exists() and context.data_engine: + context.data_engine.update_video_path(new_video_path) + + def _prepare_output_files(self, task: Task): + if task.request.llm_output_path: + _validate_and_prepare_file(file_path=task.request.llm_output_path) + + async def _extract_output( + self, + task_name: str, + ctx: ArtemisContext, + request: TaskRequest[TOutput], + output_config: OutputConfig | None, + state: State, + ) -> str | dict | TOutput | None: + exec_setup = getattr(ctx, "execution_setup", None) + outputter_cfg = getattr(exec_setup, "outputter", None) or ( + getattr(ctx.agent_config, "outputter", None) if ctx and ctx.agent_config else None + ) + outputter_enabled = getattr(outputter_cfg, "enabled", True) if outputter_cfg else True + force_synthesis = ( + getattr(outputter_cfg, "force_synthesis", False) if outputter_cfg else False + ) + if exec_setup and getattr(exec_setup, "disable_outputter", False): + outputter_enabled = False + + should_run = outputter_enabled and ( + ( + output_config + and output_config.needs_structured_format(default_enabled=outputter_enabled) + ) + or force_synthesis + or (output_config and output_config.enable_outputter is True) + ) + if output_config and output_config.enable_outputter is False: + should_run = False + + if should_run: + logger.info(f"[{task_name}] Generating structured output via Outputter...") + try: + structured_output = await outputter( + ctx=ctx, + output_config=output_config or OutputConfig(), + graph_output=state, + ) + logger.info(f"[{task_name}] Structured output: {structured_output}") + record_events( + output_path=request.llm_output_path, + events=structured_output, + ) + if request.output_format is not None and request.output_format is not NoneType: + return request.output_format.model_validate(structured_output) + return structured_output + except Exception as e: + logger.error(f"[{task_name}] Failed to generate structured output: {e}") + return None + return None + + def _init_clients( + self, + device_id: str, + platform: DevicePlatform, + ): + self._adb_client = AdbClient( + host=self._config.servers.adb_host, + port=self._config.servers.adb_port, + ) + self._ui_adb_client = create_screen_client(device_id) + + async def _get_device_context( + self, + device_id: str, + platform: DevicePlatform, + ) -> DeviceContext: + + host_platform = system() + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + width, height = (1080, 2400) + if self._adb_client: + try: + width, height = self._adb_client.device(device_id).window_size() + logger.info( + f"Retrieved remote device screen dimensions dynamically: {width}x{height}" + ) + except Exception as e: + logger.debug( + f"Failed to query remote window size: {e}. Defaulting to 1080x2400." + ) + return DeviceContext( + host_platform="LINUX", + mobile_platform=platform, + device_id=device_id, + device_width=width, + device_height=height, + ) + + # Query dimensions without starting UIAutomator or acquiring an awake + # strategy. Those belong inside the global execution queue lease. + if self._adb_client: + try: + device_width, device_height = self._adb_client.device(device_id).window_size() + logger.info(f"Retrieved Android screen dimensions: {device_width}x{device_height}") + except Exception as e: + logger.warning(f"Failed to get Android window size: {e}, using defaults") + device_width, device_height = 1080, 2340 + else: + logger.warning("ADB client not available, using default dimensions") + device_width, device_height = 1080, 2340 + + from artemis.platform import platform as pal_platform + + return DeviceContext( + host_platform=pal_platform.os_type.name, + mobile_platform=platform, + device_id=device_id, + device_width=device_width, + device_height=device_height, + ) + + +def _validate_and_prepare_file(file_path: Path): + path_obj = Path(file_path) + if path_obj.exists() and path_obj.is_dir(): + raise AgentTaskRequestError(f"Error: Path '{file_path}' is a directory, not a file.") + try: + path_obj.parent.mkdir(parents=True, exist_ok=True) + path_obj.touch(exist_ok=True) + except OSError as e: + raise AgentTaskRequestError(f"Error creating file '{file_path}': {e}") diff --git a/third_party/mobile_use/sdk/builders/__init__.py b/third_party/mobile_use/sdk/builders/__init__.py new file mode 100644 index 000000000..58d482ea3 --- /dev/null +++ b/third_party/mobile_use/sdk/builders/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/third_party/mobile_use/sdk/builders/agent_config_builder.py b/third_party/mobile_use/sdk/builders/agent_config_builder.py new file mode 100644 index 000000000..450ebde5d --- /dev/null +++ b/third_party/mobile_use/sdk/builders/agent_config_builder.py @@ -0,0 +1,225 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Builder for AgentConfig objects using a fluent interface.""" + +import copy +from typing import Any, ClassVar, Self + +from langchain_core.callbacks.base import Callbacks +from artemis.config import get_default_llm_config +from artemis.sdk.constants import DEFAULT_PROFILE_NAME +from third_party.mobile_use.context import DevicePlatform +from third_party.mobile_use.sdk.types.agent import AgentConfigBase, ServerConfig +from third_party.mobile_use.sdk.types.task import AgentProfile, TaskRequestCommon + + +class AgentConfigBuilderBase: + """Builder class providing a fluent interface for creating AgentConfig objects. + + This builder allows for step-by-step construction of an AgentConfig with + clear methods that make the configuration process intuitive and type-safe. + + Examples: + >>> builder = AgentConfigBuilder() + >>> config = (builder + ... .add_profile(AgentProfile(name="HighReasoning", + llm_config=LLMConfig(...))) + ... .add_profile(AgentProfile(name="LowReasoning", + llm_config=LLMConfig(...))) + ... .for_device(DevicePlatform.ANDROID, "device123") + ... .with_default_task_config(TaskRequestCommon(max_steps=30)) + ... .with_default_profile("HighReasoning") + ... .build() + ... ) + + Subclasses extend the built configuration by setting ``config_class`` and + overriding :meth:`_extra_config_fields`. + """ + + config_class: ClassVar[type[AgentConfigBase]] = AgentConfigBase + + def __init__(self, servers: ServerConfig): + """Initialize an empty AgentConfigBuilder.""" + self._agent_profiles: dict[str, AgentProfile] = {} + self._task_request_defaults: TaskRequestCommon | None = None + self._default_profile: str | AgentProfile | None = None + self._device_id: str | None = None + self._device_platform: DevicePlatform | None = None + self._servers: ServerConfig = servers + self._graph_config_callbacks: Callbacks = None + self._cloud_mobile_id_or_ref: str | None = None + + def add_profile(self, profile: AgentProfile, validate: bool = True) -> Self: + """Add an agent profile to the ARTEMIS agent. + + Args: + profile: The agent profile to add + """ + self._agent_profiles[profile.name] = profile + if validate: + profile.llm_config.validate_providers() + return self + + def add_profiles( + self, + profiles: list[AgentProfile], + validate: bool = True, + ) -> Self: + """Add multiple agent profiles to the ARTEMIS agent. + + Args: + profiles: List of agent profiles to add + """ + for profile in profiles: + self.add_profile(profile=profile, validate=validate) + return self + + def with_default_profile(self, profile: str | AgentProfile) -> Self: + """Set the default agent profile used for tasks. + + Args: + profile: The name or instance of the default agent profile + """ + self._default_profile = profile + return self + + def for_device( + self, + platform_or_device_id: DevicePlatform | str, + device_id: str | None = None, + ) -> Self: + """Configure the ARTEMIS agent for a specific device. + + Supports both: + builder.for_device(DevicePlatform.ANDROID, "emulator-5554") + and: + builder.for_device("emulator-5554") (defaults to DevicePlatform.ANDROID) + + Args: + platform_or_device_id: DevicePlatform or unique identifier for the device + device_id: The unique identifier for the device (if platform was passed first) + """ + if self._cloud_mobile_id_or_ref is not None: + raise ValueError( + "Device ID cannot be set when a cloud mobile is already" + " configured.\n> for_device() and for_cloud_mobile() are" + " mutually exclusive" + ) + if isinstance(platform_or_device_id, DevicePlatform): + self._device_platform = platform_or_device_id + self._device_id = device_id + else: + self._device_platform = DevicePlatform.ANDROID + self._device_id = str(platform_or_device_id) + return self + + def with_default_task_config(self, config: TaskRequestCommon) -> Self: + """Set the default task configuration. + + Args: + config: The task configuration to use as default + """ + self._task_request_defaults = copy.deepcopy(config) + return self + + def with_adb_server(self, host: str, port: int | None = None) -> Self: + """Set the ADB server host and port. + + Args: + host: The ADB server host + port: The ADB server port + """ + self._servers.adb_host = host + if port is not None: + self._servers.adb_port = port + return self + + def with_servers(self, servers: ServerConfig) -> Self: + """Set the server settings. + + Args: + servers: The server settings to use + """ + self._servers = copy.deepcopy(servers) + return self + + def with_graph_config_callbacks(self, callbacks: Callbacks) -> Self: + """Set the graph config callbacks. + + Args: + callbacks: The graph config callbacks to use + """ + self._graph_config_callbacks = callbacks + return self + + def _extra_config_fields(self) -> dict[str, Any]: + """Fields a subclass adds to (or overrides in) the built configuration.""" + return {} + + def build(self, validate_profiles: bool = True) -> AgentConfigBase: + """Build the AgentConfig object. + + Args: + default_profile: Name of the default agent profile to use + + Returns: + A configured AgentConfig object + + Raises: + ValueError: If default_profile is specified but not found in + configured profiles + """ + nb_profiles = len(self._agent_profiles) + + if isinstance(self._default_profile, str): + profile_name = self._default_profile + default_profile = self._agent_profiles.get(profile_name, None) + if default_profile is None: + raise ValueError(f"Profile '{profile_name}' not found in configured agents") + elif isinstance(self._default_profile, AgentProfile): + default_profile = self._default_profile + if default_profile.name not in self._agent_profiles: + self.add_profile(default_profile, validate=validate_profiles) + elif nb_profiles <= 0: + llm_config = get_default_llm_config() + default_profile = AgentProfile( + name=DEFAULT_PROFILE_NAME, + llm_config=llm_config, + ) + self.add_profile(default_profile, validate=validate_profiles) + elif nb_profiles == 1: + # Select the only one available + default_profile = next(iter(self._agent_profiles.values())) + else: + available_profiles = ", ".join(self._agent_profiles.keys()) + raise ValueError( + f"You must call with_default_profile() to select one among: {available_profiles}" + ) + + fields: dict[str, Any] = { + "agent_profiles": self._agent_profiles, + "task_request_defaults": self._task_request_defaults or TaskRequestCommon(), + "default_profile": default_profile, + "device_id": self._device_id, + "device_platform": self._device_platform, + "servers": self._servers, + "graph_config_callbacks": self._graph_config_callbacks, + } + fields.update(self._extra_config_fields()) + return self.config_class(**fields) diff --git a/artemis/sdk/builders/index.py b/third_party/mobile_use/sdk/builders/index.py similarity index 72% rename from artemis/sdk/builders/index.py rename to third_party/mobile_use/sdk/builders/index.py index 7ecc3d15c..0439f7e9b 100644 --- a/artemis/sdk/builders/index.py +++ b/third_party/mobile_use/sdk/builders/index.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,9 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder -from artemis.sdk.builders.task_request_builder import TaskRequestCommonBuilder +from third_party.mobile_use.sdk.builders.task_request_builder import TaskRequestCommonBuilder class BuildersWrapper: diff --git a/artemis/sdk/builders/task_request_builder.py b/third_party/mobile_use/sdk/builders/task_request_builder.py similarity index 91% rename from artemis/sdk/builders/task_request_builder.py rename to third_party/mobile_use/sdk/builders/task_request_builder.py index 706802c56..3cba3e958 100644 --- a/artemis/sdk/builders/task_request_builder.py +++ b/third_party/mobile_use/sdk/builders/task_request_builder.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. """Builder for TaskRequest objects using a fluent interface.""" @@ -27,31 +28,14 @@ from artemis.constants import RECURSION_LIMIT from artemis.sdk.types.agent import AgentProfile -from artemis.sdk.types.task import TaskRequest, TaskRequestCommon from pydantic import BaseModel +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.sdk.types.task import TaskRequest, TaskRequestCommon TIn = TypeVar("TIn", bound=BaseModel | None) TOut = TypeVar("TOut", bound=BaseModel) -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - class TaskRequestCommonBuilder(BaseModel): """Builder class providing a fluent interface for creating TaskRequestCommon objects.""" diff --git a/third_party/mobile_use/sdk/types/__init__.py b/third_party/mobile_use/sdk/types/__init__.py new file mode 100644 index 000000000..58d482ea3 --- /dev/null +++ b/third_party/mobile_use/sdk/types/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/third_party/mobile_use/sdk/types/agent.py b/third_party/mobile_use/sdk/types/agent.py new file mode 100644 index 000000000..9a0f194ab --- /dev/null +++ b/third_party/mobile_use/sdk/types/agent.py @@ -0,0 +1,98 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Agent configuration base models. + +Artemis extends :class:`AgentConfigBase` as ``artemis.sdk.types.agent.AgentConfig``. +""" + +from typing import Literal +from urllib.parse import urlparse + +from langchain_core.callbacks.base import Callbacks +from pydantic import BaseModel + +from artemis.utils.cython_compat import CyFunctionDetector +from third_party.mobile_use.context import DevicePlatform +from third_party.mobile_use.sdk.types.task import AgentProfile, TaskRequestCommon + + +class ApiBaseUrl(BaseModel): + """Defines an API base URL.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + scheme: Literal["http", "https"] + host: str + port: int | None = None + + def __eq__(self, other): + if not isinstance(other, ApiBaseUrl): + return False + return self.to_url() == other.to_url() + + def to_url(self): + return ( + f"{self.scheme}://{self.host}:{self.port}" + if self.port is not None + else f"{self.scheme}://{self.host}" + ) + + @classmethod + def from_url(cls, url: str) -> "ApiBaseUrl": + parsed_url = urlparse(url) + if parsed_url.scheme not in ["http", "https"]: + raise ValueError(f"Invalid scheme: {parsed_url.scheme}") + if parsed_url.hostname is None: + raise ValueError("Invalid hostname") + return cls( + scheme=parsed_url.scheme, # type: ignore + host=parsed_url.hostname, + port=parsed_url.port, + ) + + +class ServerConfig(BaseModel): + """Configuration for the required servers.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + + adb_host: str + adb_port: int + + +class AgentConfigBase(BaseModel): + """ARTEMIS agent configuration. + + Attributes: + agent_profiles: Map an agent profile name to its configuration. + task_config_defaults: Default task request configuration. + default_profile: default profile to use for tasks + device_id: Specific device to target (if None, first available is used). + device_platform: Platform of the device to target. + servers: Custom server configurations. + """ + + agent_profiles: dict[str, AgentProfile] + task_request_defaults: TaskRequestCommon + default_profile: AgentProfile + device_id: str | None = None + device_platform: DevicePlatform | None = None + servers: ServerConfig + graph_config_callbacks: Callbacks = None + + model_config = {"arbitrary_types_allowed": True} diff --git a/artemis/sdk/types/exceptions.py b/third_party/mobile_use/sdk/types/exceptions.py similarity index 93% rename from artemis/sdk/types/exceptions.py rename to third_party/mobile_use/sdk/types/exceptions.py index 63288ccd7..13639966f 100644 --- a/artemis/sdk/types/exceptions.py +++ b/third_party/mobile_use/sdk/types/exceptions.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. """Exceptions for the ARTEMIS SDK. diff --git a/artemis/sdk/types/task.py b/third_party/mobile_use/sdk/types/task.py similarity index 91% rename from artemis/sdk/types/task.py rename to third_party/mobile_use/sdk/types/task.py index 7c4a0be66..8529e6343 100644 --- a/artemis/sdk/types/task.py +++ b/third_party/mobile_use/sdk/types/task.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,40 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. """Task-related type definitions for the ARTEMIS SDK.""" from collections.abc import Callable, Coroutine from datetime import datetime from pathlib import Path -from typing import Any, Generic, TypeVar, overload +from typing import Any, Generic, Literal, TypeVar, overload from artemis.config import LLMConfig, get_default_llm_config from artemis.constants import RECURSION_LIMIT from artemis.context import DeviceContext -from artemis.sdk.types.platform import TaskRunStatus from artemis.sdk.utils import load_llm_config_override from pydantic import BaseModel, Field +from artemis.utils.cython_compat import CyFunctionDetector - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass +TaskRunStatus = Literal["pending", "running", "completed", "failed", "cancelled"] class AgentProfile(BaseModel): diff --git a/third_party/mobile_use/testing.py b/third_party/mobile_use/testing.py new file mode 100644 index 000000000..f4c487bcc --- /dev/null +++ b/third_party/mobile_use/testing.py @@ -0,0 +1,59 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Pytest fixtures for agent node tests (upstream ``agents/outputter/test_outputter.py``). + +Test-only: imports pytest, so it must not be imported by runtime code. +""" + +from unittest.mock import Mock + +import pytest + + +class DummyState: + def __init__(self, messages, initial_goal, operator_raw_data=None): + self.messages = messages + self.initial_goal = initial_goal + self.operator_raw_data = operator_raw_data + + +@pytest.fixture +def mock_context(): + """Create a properly mocked context with all required fields.""" + from artemis.config import LLM + from artemis.context import ArtemisContext + + ctx = Mock(spec=ArtemisContext) + ctx.llm_config = { + "planner": LLM(provider="openai", model="gpt-5-nano"), + "operator": LLM(provider="openai", model="gpt-5-nano"), + "validator": LLM(provider="openai", model="gpt-5-nano"), + } + ctx.device = Mock() + ctx.data_engine = None + return ctx + + +@pytest.fixture +def mock_state(): + """Create a mock state with test data.""" + return DummyState( + messages=[], + initial_goal="Find a green product on my website", + ) diff --git a/third_party/mobile_use/tools/__init__.py b/third_party/mobile_use/tools/__init__.py new file mode 100644 index 000000000..58d482ea3 --- /dev/null +++ b/third_party/mobile_use/tools/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/third_party/mobile_use/tools/index.py b/third_party/mobile_use/tools/index.py new file mode 100644 index 000000000..90ee10198 --- /dev/null +++ b/third_party/mobile_use/tools/index.py @@ -0,0 +1,44 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Instantiate LangChain tools from tool wrappers.""" + +from collections.abc import Callable + +from langchain_core.tools import BaseTool + +from artemis.context import ArtemisContext +from third_party.mobile_use.tools.tool_wrapper import CompositeToolWrapper, ToolWrapper + + +def get_tools_from_wrappers( + ctx: "ArtemisContext", + wrappers: list[ToolWrapper], + wrap_tool: Callable[[BaseTool], BaseTool] | None = None, +) -> list[BaseTool]: + """Instantiate and wrap LangChain tools from a list of ToolWrappers.""" + tools: list[BaseTool] = [] + for wrapper in wrappers: + if wrapper.is_available_fn is not None and not wrapper.is_available_fn(ctx): + continue + if isinstance(wrapper, CompositeToolWrapper): + new_tools = wrapper.composite_tools_fn_getter(ctx) + else: + new_tools = [wrapper.tool_fn_getter(ctx)] + tools.extend(wrap_tool(t) if wrap_tool else t for t in new_tools) + return tools diff --git a/third_party/mobile_use/tools/mobile/__init__.py b/third_party/mobile_use/tools/mobile/__init__.py new file mode 100644 index 000000000..58d482ea3 --- /dev/null +++ b/third_party/mobile_use/tools/mobile/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/third_party/mobile_use/tools/mobile/launch_app.py b/third_party/mobile_use/tools/mobile/launch_app.py new file mode 100644 index 000000000..f149b79c9 --- /dev/null +++ b/third_party/mobile_use/tools/mobile/launch_app.py @@ -0,0 +1,89 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Resolve a natural-language app name to an installed package via the Hopper agent.""" + +from artemis.context import ArtemisContext +from third_party.mobile_use.agents.hopper.hopper import HopperOutput, hopper +from third_party.mobile_use.controllers.platform_specific_commands_controller import ( + list_packages_async, +) +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + + +LAUNCH_APP_DOCSTRING = ( + "[ACTION] Finds and launches an application on the device using its natural language name." +) + + +async def find_package(ctx: ArtemisContext, app_name: str, use_fallback: bool = True) -> str | None: + """Finds the package name for a given application name. + + Returns None if package not found or on error. + """ + package_cache = getattr(ctx, "package_cache", None) + if package_cache is None or not isinstance(package_cache, dict): + try: + ctx.package_cache = {} + package_cache = ctx.package_cache + except Exception: # pylint: disable=broad-exception-caught + package_cache = {} + + if isinstance(package_cache, dict) and app_name in package_cache: + logger.info(f"Cache hit for app '{app_name}': {package_cache[app_name]}") + return package_cache[app_name] + + try: + all_packages = await list_packages_async(ctx=ctx) + package_set = {p.strip() for p in all_packages.split("\n") if p.strip()} + + # Fast path: If app_name is already directly an installed package name + if app_name in package_set: + if isinstance(package_cache, dict): + package_cache[app_name] = app_name + return app_name + + hopper_output: HopperOutput = await hopper( + ctx=ctx, + request=(f"I'm looking for the package name of the following app: '{app_name}'"), + data=all_packages, + use_fallback=use_fallback, + ) + if not hopper_output.found or not hopper_output.output: + if isinstance(package_cache, dict): + package_cache[app_name] = None + return None + + package_name = hopper_output.output.strip() + if package_name not in package_set: + logger.warning( + f"Hopper returned package '{package_name}' for '{app_name}', " + "but it is NOT physically installed on the device!" + ) + if isinstance(package_cache, dict): + package_cache[app_name] = None + return None + + if isinstance(package_cache, dict): + package_cache[app_name] = package_name + return package_name + except Exception as e: # pylint: disable=broad-exception-caught + logger.error(f"Failed to find package for '{app_name}': {e}") + return None diff --git a/third_party/mobile_use/tools/tool_wrapper.py b/third_party/mobile_use/tools/tool_wrapper.py new file mode 100644 index 000000000..9ace7eeac --- /dev/null +++ b/third_party/mobile_use/tools/tool_wrapper.py @@ -0,0 +1,52 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Tool wrappers: a LangChain tool factory and the messages shown for its result.""" + +from collections.abc import Callable + +from langchain_core.tools import BaseTool +from pydantic import BaseModel + +from artemis.context import ArtemisContext +from artemis.utils.cython_compat import CyFunctionDetector + + +class ToolWrapper(BaseModel): + """Wrapper holding a tool factory and lifecycle callbacks.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + tool_fn_getter: Callable[[ArtemisContext], BaseTool] + on_success_fn: Callable[..., str] + on_failure_fn: Callable[..., str] + is_available_fn: Callable[[ArtemisContext], bool] | None = None + + +class CompositeToolWrapper(ToolWrapper): + """Wrapper holding a composite tool factory and lifecycle callbacks.""" + + composite_tools_fn_getter: Callable[[ArtemisContext], list[BaseTool]] + + +def make_save_note_wrapper(tool_fn_getter: Callable[[ArtemisContext], BaseTool]) -> ToolWrapper: + """Wrapper for the scratchpad ``save_note`` tool with its outcome messages.""" + return ToolWrapper( + tool_fn_getter=tool_fn_getter, + on_success_fn=lambda key: f"Saved note '{key}'.", + on_failure_fn=lambda key: f"Failed to save note '{key}'.", + ) diff --git a/third_party/mobile_use/utils/__init__.py b/third_party/mobile_use/utils/__init__.py new file mode 100644 index 000000000..58d482ea3 --- /dev/null +++ b/third_party/mobile_use/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/third_party/mobile_use/utils/app_launch_utils.py b/third_party/mobile_use/utils/app_launch_utils.py new file mode 100644 index 000000000..3e7f76458 --- /dev/null +++ b/third_party/mobile_use/utils/app_launch_utils.py @@ -0,0 +1,217 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Utilities for handling app locking and initial app launch logic.""" + +import asyncio + +from artemis.context import AppLaunchResult, ArtemisContext +from artemis.controllers.unified_controller import UnifiedMobileController +from artemis.data_engine.trace import TraceSpan +from artemis.utils.app_launch_utils import current_foreground, observe_foreground +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + + +async def _poll_for_app_ready( + ctx: ArtemisContext, + app_package: str, + max_poll_seconds: int = 15, + poll_interval: float = 1.0, +) -> tuple[bool, str | None]: + """Poll for app to be ready after launch. + + Treats mCurrentFocus=null as a loading state and keeps polling. + Only fails if we get a different (non-null) package or timeout. + + Args: + ctx: Mobile use context + app_package: Expected package name + max_poll_seconds: Maximum time to poll (default: 15s) + poll_interval: Time between polls (default: 1s) + + Returns: + Tuple of (success: bool, error_message: str | None) + """ + polls = int(max_poll_seconds / poll_interval) + + for i in range(polls): + ready, current_package, task = await observe_foreground(ctx, app_package) + + if ready: + if current_package == app_package: + logger.success(f"App {app_package} is ready (took ~{i * poll_interval:.1f}s)") + else: + logger.success( + f"App {app_package} is ready (focused window belongs to" + f" '{current_package}', but the foreground {task.describe()} is owned" + f" by {app_package}, took ~{i * poll_interval:.1f}s)" + ) + return True, None + + if current_package is None: + logger.debug(f"Poll {i + 1}/{polls}: App loading (mCurrentFocus=null)...") + else: + logger.debug( + f"Poll {i + 1}/{polls}: Wrong app in foreground (expected" + f" '{app_package}', got '{current_package}', foreground" + f" {task.describe() if task else 'task unknown'}). Still waiting..." + ) + + if i < polls - 1: + await asyncio.sleep(poll_interval) + + current_package, task = await current_foreground(ctx) + error_msg = ( + f"Timeout waiting for {app_package} to load after {max_poll_seconds}s. " + f"Current foreground: {current_package}; foreground" + f" {task.describe() if task else 'task unknown'}" + ) + logger.error(error_msg) + return False, error_msg + + +async def launch_app_with_retries( + ctx: ArtemisContext, + app_package: str, + max_retries: int = 3, + max_poll_seconds: int = 15, +) -> tuple[bool, str | None]: + """Launch an app with retry logic and smart polling. + + Args: + ctx: Mobile use context + app_package: Package name (Android) to launch + max_retries: Maximum number of launch attempts (default: 3) + max_poll_seconds: Maximum time to wait for app to load per attempt + (default: 15s) + + Returns: + Tuple of (success: bool, error_message: str | None) + """ + + for attempt in range(1, max_retries + 1): + logger.info(f"Launch attempt {attempt}/{max_retries} for app {app_package}") + + with TraceSpan( + name=f"Launch Attempt {attempt}", + trace_type="span", + ctx=ctx, + ) as span: + span.payload = {"attempt": attempt, "app_package": app_package} + + controller = UnifiedMobileController(ctx) + if attempt > 1: + logger.warning( + f"Attempt {attempt - 1} failed. Force stopping" + f" '{app_package}' to clear frozen state before retrying..." + ) + await controller.terminate_app(app_package) + await asyncio.sleep(1.0) + + launch_success = await controller.launch_app(app_package) + if not launch_success: + error_msg = f"Failed to execute launch command for {app_package}" + logger.error(error_msg) + span.status = "failed" + span.error = error_msg + if attempt == max_retries: + return False, error_msg + await asyncio.sleep(2) + continue + + await asyncio.sleep(1) + + success, error_msg = await _poll_for_app_ready(ctx, app_package, max_poll_seconds) + + if success: + span.status = "success" + span.result = "App is ready" + return True, None + + span.status = "failed" + span.error = error_msg + + if attempt < max_retries: + logger.warning(f"Attempt {attempt} failed: {error_msg}. Retrying...") + await asyncio.sleep(1) + + error_msg = f"Failed to launch {app_package} after {max_retries} attempts" + logger.error(error_msg) + return False, error_msg + + +async def _handle_initial_app_launch( + ctx: ArtemisContext, + locked_app_package: str, +) -> AppLaunchResult: + """Handle initial app launch verification and launching if needed. + + If locked_app_package is set: + 1. Check if the app is already in the foreground + 2. If not, attempt to launch it (with retries) + 3. Return status with success/error information + + Args: + ctx: Mobile use context + locked_app_package: Package name (Android) to lock to + + Returns: + AppLaunchResult with launch status and error information + """ + if not locked_app_package: + error_msg = f"Invalid locked_app_package: '{locked_app_package}'" + logger.error(error_msg) + return AppLaunchResult( + locked_app_package=locked_app_package, + locked_app_initial_launch_success=False, + locked_app_initial_launch_error=error_msg, + ) + + logger.info(f"Starting initial app launch for package: {locked_app_package}") + + try: + already_foreground, current_package, _ = await observe_foreground(ctx, locked_app_package) + logger.info(f"Current foreground app: {current_package}") + + if already_foreground: + logger.info(f"App {locked_app_package} is already in foreground") + return AppLaunchResult( + locked_app_package=locked_app_package, + locked_app_initial_launch_success=True, + locked_app_initial_launch_error=None, + ) + + logger.info(f"App {locked_app_package} not in foreground, attempting to launch") + success, error_msg = await launch_app_with_retries(ctx, locked_app_package) + + return AppLaunchResult( + locked_app_package=locked_app_package, + locked_app_initial_launch_success=success, + locked_app_initial_launch_error=error_msg, + ) + + except Exception as e: + error_msg = f"Exception during initial app launch: {str(e)}" + logger.error(error_msg) + return AppLaunchResult( + locked_app_package=locked_app_package, + locked_app_initial_launch_success=False, + locked_app_initial_launch_error=error_msg, + ) diff --git a/artemis/utils/cli_helpers.py b/third_party/mobile_use/utils/cli_helpers.py similarity index 60% rename from artemis/utils/cli_helpers.py rename to third_party/mobile_use/utils/cli_helpers.py index f7056b246..4b9c687bc 100644 --- a/artemis/utils/cli_helpers.py +++ b/third_party/mobile_use/utils/cli_helpers.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,12 +12,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. +from shutil import which import sys from adbutils import AdbClient from rich.console import Console +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + def display_device_status(console: Console, adb_client: AdbClient | None = None): """Checks for connected devices and displays the status.""" @@ -36,3 +45,16 @@ def display_device_status(console: Console, adb_client: AdbClient | None = None) console.print( f"You can start an emulator using a command like: [bold]'{command}'[/bold]" ) + + +def display_local_device_status(console: Console, host: str | None, port: int | None) -> None: + """Display connected Android devices, using a local ADB client when ``adb`` is available.""" + adb_client = None + try: + if which("adb"): + adb_client = AdbClient(host=host or "localhost", port=port or 5037) + except Exception as exc: + # Optional cosmetic device-status display; run continues without it. + logger.debug(f"Could not create ADB client for device status display: {exc}") + + display_device_status(console, adb_client=adb_client) diff --git a/artemis/utils/decorators.py b/third_party/mobile_use/utils/decorators.py similarity index 85% rename from artemis/utils/decorators.py rename to third_party/mobile_use/utils/decorators.py index 4ef07b9a7..a1ffe861a 100644 --- a/artemis/utils/decorators.py +++ b/third_party/mobile_use/utils/decorators.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. import asyncio from collections.abc import Awaitable, Callable @@ -137,3 +141,18 @@ def decorator(func: Callable[..., Any]) -> Any: return decorator else: return decorator(fn) + + +def agent_lifecycle_logging( + agent_name: str, logger: Any +) -> Callable[[Callable[..., R]], Callable[..., R]]: + """Log the start, success and failure of an agent node call. + + ``agent_name`` is the display name, e.g. ``"Planner"``. + """ + label = f"{agent_name} Agent" + return wrap_with_callbacks( + before=lambda: logger.info(f"Starting {label}..."), + on_success=lambda _: logger.success(label), + on_failure=lambda _: logger.error(label), + ) diff --git a/artemis/utils/file.py b/third_party/mobile_use/utils/file.py similarity index 78% rename from artemis/utils/file.py rename to third_party/mobile_use/utils/file.py index 7b8d97d6f..e7930014e 100644 --- a/artemis/utils/file.py +++ b/third_party/mobile_use/utils/file.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. import json import re diff --git a/artemis/utils/logger.py b/third_party/mobile_use/utils/logger.py similarity index 96% rename from artemis/utils/logger.py rename to third_party/mobile_use/utils/logger.py index 321d2789f..5f39a6be1 100644 --- a/artemis/utils/logger.py +++ b/third_party/mobile_use/utils/logger.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. from enum import Enum import logging diff --git a/artemis/utils/media.py b/third_party/mobile_use/utils/media.py similarity index 95% rename from artemis/utils/media.py rename to third_party/mobile_use/utils/media.py index b6069afc1..164a88989 100644 --- a/artemis/utils/media.py +++ b/third_party/mobile_use/utils/media.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. import json import os @@ -21,7 +22,7 @@ import subprocess import tempfile -from artemis.utils.logger import get_logger +from third_party.mobile_use.utils.logger import get_logger from PIL import Image logger = get_logger(__name__) diff --git a/artemis/utils/shell_utils.py b/third_party/mobile_use/utils/shell_utils.py similarity index 83% rename from artemis/utils/shell_utils.py rename to third_party/mobile_use/utils/shell_utils.py index 1972a7345..6dcd9a17a 100644 --- a/artemis/utils/shell_utils.py +++ b/third_party/mobile_use/utils/shell_utils.py @@ -1,4 +1,5 @@ -# Copyright 2026 Google LLC +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. import subprocess diff --git a/third_party/mobile_use/utils/video.py b/third_party/mobile_use/utils/video.py new file mode 100644 index 000000000..a02e0f2cd --- /dev/null +++ b/third_party/mobile_use/utils/video.py @@ -0,0 +1,398 @@ +# Copyright 2025-2026 Minitap, Inc. +# Modifications Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Originally from mobile-use (https://github.com/minitap-ai/mobile-use). +# See third_party/mobile_use/METADATA for the upstream source and local modifications. + +"""Video recording utilities for mobile devices. + +Provides shared types and utilities for video recording across platforms. +""" + +import asyncio +from pathlib import Path +import platform +from typing import Any +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + +from artemis.utils.video import ( + get_ffmpeg_path, + get_ffprobe_path, + is_ffmpeg_drawtext_supported, + is_ffmpeg_installed, +) +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +DEFAULT_MAX_DURATION_SECONDS = 900 # 15 minutes +ANDROID_DEVICE_VIDEO_PATH = "/sdcard/screen_recording.mp4" + +# Expanded for Gemini File API (Supports up to 2GB). +# Target 100MB to allow 3-5min crisp video and prevent blurring for long durations. +MAX_VIDEO_SIZE_MB = 500 +MAX_VIDEO_SIZE_BYTES = MAX_VIDEO_SIZE_MB * 1024 * 1024 + + +class RecordingSession(BaseModel): + """Tracks an active video recording session.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + video_id: UUID + device_id: str + start_time: float + process: Any = None + data_engine_start_time: float | None = None + local_video_path: Path | None = None + capture_width: int | None = None + capture_height: int | None = None + android_device_path: str = ANDROID_DEVICE_VIDEO_PATH + android_video_segments: list[Path] = [] + android_segment_index: int = 0 + android_restart_task: asyncio.Task | None = None + watchdog_task: asyncio.Task | None = None + android_rotation: int | None = None + android_segment_started_at: float | None = None + android_segment_records: list[dict[str, Any]] = [] + android_conversion_tasks: list[asyncio.Task] = [] + generation: int = 0 + sealed_until: float = 0.0 + is_active: bool = True + errors: list[str] = [] + + +class VideoRecordingResult(BaseModel): + """Result of a video recording operation.""" + + success: bool + message: str + video_path: Path | None = None + file_size_mb: float | None = None + duration_seconds: float | None = None + actual_start_relative_time: float | None = None + warning: str | None = None + video_id: UUID | None = None + generation: int | None = None + sealed_until: float | None = None + source_revision: str | None = None + + model_config = ConfigDict(arbitrary_types_allowed=True) + + +# Global session storage - keyed by device_id +_active_recordings: dict[str, RecordingSession] = {} + + +def get_active_session(device_id: str) -> RecordingSession | None: + """Get the active recording session for a device.""" + return _active_recordings.get(device_id) + + +def set_active_session(device_id: str, session: RecordingSession) -> None: + """Set the active recording session for a device.""" + _active_recordings[device_id] = session + + +def remove_active_session(device_id: str) -> RecordingSession | None: + """Remove and return the active recording session for a device.""" + return _active_recordings.pop(device_id, None) + + +def has_active_session(device_id: str) -> bool: + """Check if there's an active recording session for a device.""" + return device_id in _active_recordings + + +def recording_already_active(device_id: str) -> VideoRecordingResult | None: + """Failure result if the device is already recording, otherwise ``None``.""" + if has_active_session(device_id): + return VideoRecordingResult( + success=False, + message=f"Recording already in progress for device {device_id}", + ) + return None + + +def no_active_recording(device_id: str) -> VideoRecordingResult: + """Failure result for an operation that needs an active recording.""" + return VideoRecordingResult( + success=False, + message=f"No active recording for device {device_id}", + ) + + +def abort_recording(device_id: str, action: str, error: Exception) -> VideoRecordingResult: + """Drop the device's session after a failed ``action`` ("start"/"stop") and report it.""" + remove_active_session(device_id) + return VideoRecordingResult( + success=False, + message=f"Failed to {action} recording: {error}", + ) + + +class FFmpegNotInstalledError(Exception): + """Raised when ffmpeg is required but not installed.""" + + def __init__(self): + os_name = platform.system().lower() + if os_name == "darwin": # macOS + install_instructions = "brew install ffmpeg" + elif os_name == "windows": + install_instructions = "Download from https://www.ffmpeg.org/download.html" + else: # Linux and others + install_instructions = ( + "Install via your package manager (e.g., apt install ffmpeg," + " dnf install ffmpeg) or download from" + " https://www.ffmpeg.org/download.html" + ) + + message = ( + "\n\n❌ ffmpeg is required for video recording but is not" + " installed.\n\nPlease install ffmpeg first:\n →" + f" {install_instructions}\n\nAfter installation, restart Artemis.\n" + ) + super().__init__(message) + + +def check_ffmpeg_available() -> None: + """Check if ffmpeg is installed and raise an error if not. + + Raises: + FFmpegNotInstalledError: If ffmpeg is not found in PATH. + """ + if not is_ffmpeg_installed(): + raise FFmpegNotInstalledError() + + +async def probe_duration(input_path: Path) -> float: + """Return the container duration of a video in seconds, using ffprobe. + + Raises: + ValueError: If ffprobe reports no parseable duration. + OSError: If ffprobe cannot be started. + """ + duration_cmd = [ + get_ffprobe_path(), + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + str(input_path), + ] + proc = await asyncio.create_subprocess_exec( + *duration_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return float(stdout.decode().strip()) + + +async def compress_video_for_api( + input_path: Path, + target_size_bytes: int = MAX_VIDEO_SIZE_BYTES, + force_compress: bool = False, + start_offset_seconds: float = 0.0, + slowdown_factor: float = 1.0, +) -> Path: + """Compress a video to fit within API size limits using ffmpeg. + + Uses a two-pass approach: + 1. First check if video is already small enough + 2. If not, compress with reduced resolution and bitrate + + Args: + input_path: Path to the input video file + target_size_bytes: Target maximum file size in bytes + force_compress: If True, always perform compression (e.g., to extract + frames at 15fps) + start_offset_seconds: Offset to add to the burned-in timestamp + slowdown_factor: Factor to slow down the video by (e.g. 5.0 for 5x + slower) to increase API frame sampling rate + + Returns: + Path to the compressed video (may be same as input if no compression + needed) + """ + if not input_path.exists(): + raise FileNotFoundError(f"Video file not found: {input_path}") + + current_size = input_path.stat().st_size + logger.info(f"Video size: {current_size / 1024 / 1024:.2f} MB") + + if current_size <= target_size_bytes and not force_compress and slowdown_factor == 1.0: + logger.info( + "Video already within size limit and force_compress=False, no compression needed" + ) + return input_path + + logger.info( + f"Compressing video to fit within {target_size_bytes / 1024 / 1024:.1f}" + f" MB (slowdown={slowdown_factor})" + ) + + output_path = input_path.parent / f"compressed_{input_path.name}" + + # Use Constant Rate Factor (CRF) for high-fidelity UI text readability. + # CRF dynamically allocates bitrates depending on scene motion. + logger.info("Compressing video using CRF=26 for high-fidelity UI text readability.") + + # Compress with ffmpeg: reduce resolution to 720p max, use CRF + # and check if drawtext is supported + vf_parts = ["setpts=PTS-STARTPTS", "scale='min(720,iw)':'-2'"] + if is_ffmpeg_drawtext_supported(): + vf_parts.append( + "drawtext=text='TS\\:" + f" %{{expr_int_format\\:trunc(t+{start_offset_seconds})\\:d}}" + " s':x=w-tw-30:y=120+th+20:fontcolor=white@0.6:fontsize=44:borderw=4:" + "bordercolor=red@0.6:box=1:boxcolor=yellow@0.4:boxborderw=10:font='Sans" + " Bold'" + ) + else: + logger.warning( + "ffmpeg 'drawtext' filter not supported on this system. Skipping" + " burned-in timestamp overlay." + ) + + if slowdown_factor != 1.0: + vf_parts.append(f"setpts={slowdown_factor}*PTS") + + vf_filter = ",".join(vf_parts) + + compress_cmd = [ + get_ffmpeg_path(), + "-y", + "-i", + str(input_path), + "-vf", + vf_filter, + "-r", + "15", # Set framerate to 15 fps + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "26", # High quality for UI elements + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-b:a", + "64k", + str(output_path), + ] + + try: + proc = await asyncio.create_subprocess_exec( + *compress_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _, stderr = await proc.communicate() + + if proc.returncode != 0: + err_msg = stderr.decode().strip() + logger.error(f"ffmpeg compression failed (code {proc.returncode}): {err_msg}") + if force_compress or slowdown_factor != 1.0: + raise RuntimeError( + f"Video compression/slowdown failed (code {proc.returncode}): {err_msg}" + ) + return input_path # Return original if compression fails and was optional + + new_size = output_path.stat().st_size + logger.info( + f"Compressed: {current_size / 1024 / 1024:.2f} MB -> {new_size / 1024 / 1024:.2f} MB" + ) + + return output_path + + except Exception as e: + logger.error(f"Video compression failed: {e}") + if force_compress or slowdown_factor != 1.0: + raise RuntimeError(f"Video compression/slowdown failed: {e}") + return input_path # Return original if compression fails and was optional + + +async def extract_audio_from_video(input_path: Path) -> Path: + """Extract audio from a video file using ffmpeg. + + Saves as .mp3. + """ + if not input_path.exists(): + raise FileNotFoundError(f"Video file not found: {input_path}") + + probe_cmd = [ + get_ffprobe_path(), + "-v", + "error", + "-select_streams", + "a:0", + "-show_entries", + "stream=index", + "-of", + "csv=p=0", + str(input_path), + ] + probe = await asyncio.create_subprocess_exec( + *probe_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + probe_stdout, _ = await probe.communicate() + if probe.returncode != 0 or not probe_stdout.strip(): + raise ValueError("Video has no audio stream") + + output_path = input_path.parent / f"audio_{input_path.stem}.mp3" + logger.info(f"Extracting audio from {input_path} to {output_path}") + + cmd = [ + get_ffmpeg_path(), + "-y", + "-i", + str(input_path), + "-vn", + "-c:a", + "libmp3lame", + "-b:a", + "128k", + str(output_path), + ] + + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _, stderr = await proc.communicate() + + if proc.returncode != 0: + err_msg = stderr.decode(errors="replace").strip() + concise_error = "\n".join(err_msg.splitlines()[-8:]) + logger.error(f"ffmpeg audio extraction failed: {concise_error}") + raise RuntimeError( + f"ffmpeg audio extraction failed (code {proc.returncode}): {concise_error}" + ) + + return output_path + + except Exception: + raise From 351ca8422f7b5b54e80a9c1ce03a222e02415b6b Mon Sep 17 00:00:00 2001 From: somew1nd Date: Mon, 28 Sep 2026 17:29:02 -0700 Subject: [PATCH 2/2] fix: server status, CLI entrypoint, replay and test issues --- apps/admin_console/replay_manager.py | 10 +- apps/admin_console/routers/system.py | 50 +++++-- artemis/main.py | 43 ++++-- artemis/mcp/adb_server.py | 51 +++++-- artemis/runtime/daemon_client.py | 2 +- artemis/utils/video.py | 7 +- .../backend_manager/app/auth/jwt_handler.py | 8 +- .../backend_manager/app/auth/otp_service.py | 6 +- .../app/services/bigquery_service.py | 3 +- .../app/services/docker_service.py | 3 +- .../app/services/session_manager.py | 8 +- tests/e2e/test_multi_port_end_to_end.py | 129 +++++++++++++----- tests/integration/test_thin_sdk_live.py | 57 ++++++-- tests/unit/admin_console/test_replay_state.py | 113 +++++++++++++++ .../test_server_status_endpoint.py | 73 ++++++++++ tests/unit/mcp/test_device_utils.py | 10 ++ tests/unit/test_main_entrypoint.py | 90 ++++++++++++ 17 files changed, 552 insertions(+), 111 deletions(-) create mode 100644 tests/unit/admin_console/test_replay_state.py create mode 100644 tests/unit/admin_console/test_server_status_endpoint.py create mode 100644 tests/unit/test_main_entrypoint.py diff --git a/apps/admin_console/replay_manager.py b/apps/admin_console/replay_manager.py index 7a3ea5506..c708629dd 100644 --- a/apps/admin_console/replay_manager.py +++ b/apps/admin_console/replay_manager.py @@ -1766,7 +1766,8 @@ def instantiate_state( initial_goal = self.load_session_goal(session_id, step_dir, str(self.db_path)) - ui_hier, decisions, app_info, dev_date = self.extract_state_prepopulation_data( + # State has no fields for the app info or device date (extra="forbid"). + ui_hier, decisions, _app_info, _dev_date = self.extract_state_prepopulation_data( step_dir, step_data, pre_image_meta ) @@ -1776,19 +1777,14 @@ def instantiate_state( raise ImportError(f"Failed to import State module: {import_err}") state = State( - messages=[], initial_goal=initial_goal, latest_screenshot=str(step_dir / "pre.jpg") if (step_dir / "pre.jpg").exists() else str(step_dir / "post.jpg"), operator_raw_data={"width": self.w, "height": self.h}, current_step_id=step_data["step_id"], - complete_subgoals_by_ids=[], - validator_messages=[], - subagent_calls=step_data.get("subagent_calls", []), + subagent_calls=step_data.get("subagent_calls") or [], latest_ui_hierarchy=ui_hier, - focused_app_info=app_info, - device_date=dev_date, structured_decisions=decisions, ) return state diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index b809d57e6..c8ce18343 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -495,9 +495,17 @@ def mask_key(k: str | None) -> str | None: @router.get("/server-status") async def get_server_runtime_status(): - """Retrieve runtime status, PID, port, and uptime of the Artemis server.""" + """Retrieve runtime status, PID, port, and uptime of the Artemis server. + + Answers from in-process state. ``server_lifecycle.get_server_status`` is + not used here: it runs ``lsof``/``fuser``, which can take over a second and + is too slow for the ``is_artemis_daemon`` probe. + """ import os - from artemis.runtime.server_lifecycle import get_server_status + import time + + from artemis.runtime.process_probe import pid_is_alive + from artemis.runtime.server_lifecycle import read_server_info try: from apps.admin_console.core.state import state @@ -505,18 +513,38 @@ async def get_server_runtime_status(): from admin_console.core.state import state port = getattr(state, "port", 8000) - status = get_server_status(port=port) + current_pid = os.getpid() + pids = {current_pid} + started_at = None + + info = read_server_info() + if info and info.get("port") == port: + saved_pid = info.get("pid") + if isinstance(saved_pid, int) and saved_pid != current_pid and pid_is_alive(saved_pid): + pids.add(saved_pid) + if isinstance(info.get("started_at"), (int, float)): + started_at = float(info["started_at"]) + if started_at is None: + try: + import psutil + + started_at = psutil.Process(current_pid).create_time() + except Exception: # pylint: disable=broad-exception-caught + # psutil is optional; uptime is best-effort. + started_at = None + + uptime_seconds = max(0.0, time.time() - started_at) if started_at is not None else None # Explicit DTO: the raw metadata file additionally holds the lifecycle # token, cmdline, and filesystem paths, none of which belong on the wire. return { - "running": status["running"], - "port": status["port"], - "pids": status["pids"], - "active_pid": status["active_pid"], - "uptime_seconds": status["uptime_seconds"], - "url": status["url"], - "admin_url": status["admin_url"], - "current_pid": os.getpid(), + "running": True, + "port": port, + "pids": sorted(pids), + "active_pid": current_pid, + "uptime_seconds": uptime_seconds, + "url": f"http://localhost:{port}", + "admin_url": f"http://localhost:{port}/admin", + "current_pid": current_pid, } diff --git a/artemis/main.py b/artemis/main.py index 2de9eaa50..f3434597f 100644 --- a/artemis/main.py +++ b/artemis/main.py @@ -15,25 +15,42 @@ """Backward compatibility entrypoint for artemis CLI.""" import sys + +import typer + from artemis.interfaces.cli.main import app +# Tests replace `app`; command discovery still needs the real Typer app. +from artemis.interfaces.cli.main import app as _typer_app + + +def _root_tokens() -> frozenset[str]: + """Command names and root options registered on the Typer app.""" + group = typer.main.get_command(_typer_app) + tokens: set[str] = {"-h"} + # Typer may vendor its own click, so use getattr instead of isinstance. + tokens.update(getattr(group, "commands", {}) or {}) + tokens.update(group.get_help_option_names(group.context_class(group))) + for param in group.params: + tokens.update(getattr(param, "opts", ())) + tokens.update(getattr(param, "secondary_opts", ())) + return frozenset(tokens) + + +def normalize_argv(argv: list[str]) -> list[str]: + """Insert ``run`` when the first argument is not a known root command/flag. + + Keeps ``python -m artemis.main "some goal"`` working as ``run "some goal"``. + """ + if len(argv) > 1 and argv[1] not in _root_tokens(): + return [argv[0], "run", *argv[1:]] + return list(argv) + def cli(): # If invoked directly as python -m artemis.main without subcommand 'run', # check if first argument is a goal rather than a subcommand - if len(sys.argv) > 1 and sys.argv[1] not in ( - "run", - "init", - "doctor", - "batch", - "bench", - "mcp", - "server", - "trace", - "--help", - "-h", - ): - sys.argv.insert(1, "run") + sys.argv[:] = normalize_argv(sys.argv) app() diff --git a/artemis/mcp/adb_server.py b/artemis/mcp/adb_server.py index 0c3c9df59..14da4d5c5 100644 --- a/artemis/mcp/adb_server.py +++ b/artemis/mcp/adb_server.py @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections.abc import Callable import json import logging import os from pathlib import Path import sys +import textwrap from typing import Any # Ensure repository root is in sys.path when executed directly or via MCP runner @@ -86,6 +88,29 @@ def configure_stdio_mode() -> None: # Create minimal MCP server mcp = FastMCP("Android_ADB_Controller") + +def _tool_description(fn: Callable[..., Any]) -> str: + """Returns ``fn``'s docstring normalized identically on every Python version. + + FastMCP publishes ``__doc__`` verbatim as the tool description. Python 3.13+ + strips the common indentation of docstring continuation lines at compile time, + while 3.12 keeps it, so the schema external MCP clients see would otherwise + depend on the interpreter. This reproduces the 3.13 form (first line as-is, + remaining lines dedented, whitespace-only lines emptied); it is a no-op there. + """ + first, sep, rest = (fn.__doc__ or "").partition("\n") + return first + sep + textwrap.dedent(rest) + + +def _tool() -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """``mcp.tool()`` with a Python-version-independent description.""" + + def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: + return mcp.tool(description=_tool_description(fn))(fn) + + return decorator + + _GLOBAL_CONTROLLER = None _CONTROLLERS: dict[str, Any] = {} @@ -186,7 +211,7 @@ def _get_controller(device_serial: str | None = None): ) -@mcp.tool() +@_tool() async def tap( ctx: Context, coordinates: list[int], @@ -217,7 +242,7 @@ async def tap( return "Success" -@mcp.tool() +@_tool() async def long_press_on( ctx: Context, coordinates: list[int], @@ -248,7 +273,7 @@ async def long_press_on( return "Success" -@mcp.tool() +@_tool() async def swipe( ctx: Context, coordinates: list[int], @@ -282,7 +307,7 @@ async def swipe( return "Success" -@mcp.tool() +@_tool() async def back(ctx: Context) -> str: """Simulates pressing the system back button.""" try: @@ -294,7 +319,7 @@ async def back(ctx: Context) -> str: return "Success" if success else "Failed" -@mcp.tool() +@_tool() async def launch_app(ctx: Context, package_name: str) -> str: """Launches an application by its Android package name with retries and smart polling.""" try: @@ -306,7 +331,7 @@ async def launch_app(ctx: Context, package_name: str) -> str: return "Success" if success else f"Failed: {error_msg}" -@mcp.tool() +@_tool() async def stop_app(ctx: Context, package_name: str) -> str: """Force stops an application by its Android package name.""" try: @@ -318,7 +343,7 @@ async def stop_app(ctx: Context, package_name: str) -> str: return "Success" if success else "Failed" -@mcp.tool() +@_tool() async def open_link(ctx: Context, url: str) -> str: """Opens a URL or deep link on the device.""" try: @@ -330,7 +355,7 @@ async def open_link(ctx: Context, url: str) -> str: return "Success" if success else "Failed" -@mcp.tool() +@_tool() async def focus_and_input_text( ctx: Context, coordinates: list[int], @@ -367,7 +392,7 @@ async def focus_and_input_text( return "Success" if success else "Failed" -@mcp.tool() +@_tool() async def focus_and_clear_text( ctx: Context, coordinates: list[int], @@ -392,7 +417,7 @@ async def focus_and_clear_text( return "Success" if success else "Failed" -@mcp.tool() +@_tool() async def erase_one_char(ctx: Context) -> str: """Erases a single character (simulates Backspace).""" try: @@ -404,7 +429,7 @@ async def erase_one_char(ctx: Context) -> str: return "Success" if success else "Failed" -@mcp.tool() +@_tool() async def press_key(ctx: Context, keycode: str) -> str: """Presses a specific Android key event (e.g., KEYCODE_ENTER, KEYCODE_HOME).""" try: @@ -419,7 +444,7 @@ async def press_key(ctx: Context, keycode: str) -> str: return f"Error: {e}" -@mcp.tool() +@_tool() async def take_screenshot(ctx: Context) -> str: """Takes a screenshot of the device screen. @@ -437,7 +462,7 @@ async def take_screenshot(ctx: Context) -> str: return f"Error: {e}" -@mcp.tool() +@_tool() async def get_ui_hierarchy(ctx: Context) -> str: """Retrieves the current UI elements hierarchy from the device.""" try: diff --git a/artemis/runtime/daemon_client.py b/artemis/runtime/daemon_client.py index 77d63bb3c..85471489c 100644 --- a/artemis/runtime/daemon_client.py +++ b/artemis/runtime/daemon_client.py @@ -66,7 +66,7 @@ def is_daemon_running( def is_artemis_daemon( host: str = DEFAULT_DAEMON_HOST, port: int = DEFAULT_DAEMON_PORT, - timeout: float = 0.5, + timeout: float = 2.0, ) -> bool: """Identity probe: is the process on ``host:port`` the Artemis daemon? diff --git a/artemis/utils/video.py b/artemis/utils/video.py index 9c6ab31a8..76afb5df0 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -513,10 +513,13 @@ async def render_timeline_clip( f"setsar=1,fps={fps},format=yuv420p[{label}]" ) labels.append(f"[{label}]") + # ``concat`` does not advertise a frame rate on its output link, so without an + # explicit rate ffmpeg falls back to 25 fps CFR and duplicates frames, breaking + # the ``start_time + frame_index / fps`` mapping the analyzer relies on. if len(labels) == 1: - filter_parts.append(f"{labels[0]}null[outv]") + filter_parts.append(f"{labels[0]}fps={fps}[outv]") else: - filter_parts.append(f"{''.join(labels)}concat=n={len(labels)}:v=1:a=0[outv]") + filter_parts.append(f"{''.join(labels)}concat=n={len(labels)}:v=1:a=0,fps={fps}[outv]") command.extend( [ diff --git a/playground/backend_manager/app/auth/jwt_handler.py b/playground/backend_manager/app/auth/jwt_handler.py index f87991fa7..c3c801d18 100644 --- a/playground/backend_manager/app/auth/jwt_handler.py +++ b/playground/backend_manager/app/auth/jwt_handler.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from fastapi import Depends, HTTPException, Security, status from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from jose import JWTError, jwt @@ -23,13 +23,11 @@ def create_access_token(user_id: str, extra_claims: dict | None = None) -> str: """Generate a signed JWT access token.""" - expire = datetime.now(timezone.utc) + timedelta( - minutes=settings.JWT_ACCESS_TOKEN_EXPIRE_MINUTES - ) + expire = datetime.now(UTC) + timedelta(minutes=settings.JWT_ACCESS_TOKEN_EXPIRE_MINUTES) to_encode = { "sub": user_id, "exp": expire, - "iat": datetime.now(timezone.utc), + "iat": datetime.now(UTC), "iss": "artemis-backend-manager", } if extra_claims: diff --git a/playground/backend_manager/app/auth/otp_service.py b/playground/backend_manager/app/auth/otp_service.py index bb1f0d625..bbff4f8cb 100644 --- a/playground/backend_manager/app/auth/otp_service.py +++ b/playground/backend_manager/app/auth/otp_service.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta import logging import secrets from app.config import settings @@ -38,7 +38,7 @@ def generate_otp(self, identifier: str) -> str: else: code = f"{secrets.randbelow(900000) + 100000}" - expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.OTP_EXPIRE_SECONDS) + expires_at = datetime.now(UTC) + timedelta(seconds=settings.OTP_EXPIRE_SECONDS) self._store[ident] = { "code": code, "expires_at": expires_at, @@ -65,7 +65,7 @@ def verify_otp(self, identifier: str, code: str) -> bool: logger.warning(f"[OTP Service] No active OTP found for {ident}") return False - if datetime.now(timezone.utc) > record["expires_at"]: + if datetime.now(UTC) > record["expires_at"]: logger.warning(f"[OTP Service] OTP for {ident} has expired") self._store.pop(ident, None) return False diff --git a/playground/backend_manager/app/services/bigquery_service.py b/playground/backend_manager/app/services/bigquery_service.py index 8588ec169..b6e063d03 100644 --- a/playground/backend_manager/app/services/bigquery_service.py +++ b/playground/backend_manager/app/services/bigquery_service.py @@ -14,7 +14,6 @@ from datetime import datetime, timezone import logging -from typing import Optional from app.config import settings from app.schemas.session_schema import SessionRecord, SessionStatus @@ -34,7 +33,7 @@ class BigQueryMappingService: def __init__(self): self._local_cache: dict[str, SessionRecord] = {} - self._client: Optional["bigquery.Client"] = None + self._client: bigquery.Client | None = None self._table_ref: str = ( f"{settings.GCP_PROJECT_ID}.{settings.BQ_DATASET}.{settings.BQ_TABLE}" ) diff --git a/playground/backend_manager/app/services/docker_service.py b/playground/backend_manager/app/services/docker_service.py index 85d040e56..e07be54f3 100644 --- a/playground/backend_manager/app/services/docker_service.py +++ b/playground/backend_manager/app/services/docker_service.py @@ -15,7 +15,6 @@ import asyncio import logging import os -from typing import Optional from app.config import settings logger = logging.getLogger("artemis.docker") @@ -33,7 +32,7 @@ class DockerManagerService: """Manages creation, monitoring, and deletion of ephemeral Artemis session containers via Docker Socket.""" def __init__(self): - self._client: Optional["docker.DockerClient"] = None + self._client: docker.DockerClient | None = None if DOCKER_AVAILABLE: try: self._client = docker.DockerClient(base_url=settings.DOCKER_SOCKET_PATH) diff --git a/playground/backend_manager/app/services/session_manager.py b/playground/backend_manager/app/services/session_manager.py index 17bf6d64a..522cdbf7b 100644 --- a/playground/backend_manager/app/services/session_manager.py +++ b/playground/backend_manager/app/services/session_manager.py @@ -13,7 +13,7 @@ # limitations under the License. import asyncio -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta import logging import uuid from app.config import settings @@ -49,7 +49,7 @@ async def _reaper_loop(self): while True: try: await asyncio.sleep(settings.REAPER_CHECK_INTERVAL_SECONDS) - now = datetime.now(timezone.utc) + now = datetime.now(UTC) active_sessions = await bigquery_service.list_all_active_sessions() for s in active_sessions: @@ -78,7 +78,7 @@ async def _reaper_loop(self): async def create_session(self, user_id: str, request: CreateSessionRequest) -> SessionResponse: """Provision a new Cuttlefish emulator + Artemis container and link them via ADB.""" session_id = str(uuid.uuid4()) - now = datetime.now(timezone.utc) + now = datetime.now(UTC) expires_at = now + timedelta(minutes=request.ttl_minutes) logger.info( @@ -164,7 +164,7 @@ async def heartbeat(self, session_id: str, user_id: str) -> HeartbeatResponse | if not record or record.user_id != user_id or record.status == SessionStatus.TERMINATED: return None - now = datetime.now(timezone.utc) + now = datetime.now(UTC) record.last_heartbeat_at = now record.expires_at = now + timedelta(minutes=settings.SESSION_TTL_MINUTES) await bigquery_service.save_session(record) diff --git a/tests/e2e/test_multi_port_end_to_end.py b/tests/e2e/test_multi_port_end_to_end.py index 20b31da51..7c624e9af 100644 --- a/tests/e2e/test_multi_port_end_to_end.py +++ b/tests/e2e/test_multi_port_end_to_end.py @@ -61,7 +61,8 @@ @pytest.fixture(scope="module", autouse=True) def ensure_daemon(): """Ensure Artemis Daemon is running for E2E tests.""" - assert ensure_daemon_running(wait_ready=True), "Artemis Daemon failed to start" + ok, _base_url = ensure_daemon_running(wait_ready=True) + assert ok, "Artemis Daemon failed to start" yield @@ -174,64 +175,120 @@ def test_05_sdk_client_targeting_and_concurrency(): client_phone = ArtemisClient(device_id=phone_serial, concurrency_mode="per_device") assert client_phone.device_serial == phone_serial - # Verify multi-device lock independence + # Verify multi-device lock independence. acquire() returns None and raises + # DeviceBusyError on failure, so success means "no exception + we own it". DeviceExecutionLock.cleanup_stale_locks() - lock_emu = DeviceExecutionLock(device_id=emulator_serial) - lock_phone = DeviceExecutionLock(device_id=phone_serial) + lock_emu = DeviceExecutionLock( + device_id=emulator_serial, + description="e2e lock emulator", + concurrency_mode=ConcurrencyMode.PER_DEVICE, + ) + lock_phone = DeviceExecutionLock( + device_id=phone_serial, + description="e2e lock phone", + concurrency_mode=ConcurrencyMode.PER_DEVICE, + ) - # Both locks can be acquired simultaneously because device serials differ - acq1 = lock_emu.acquire(timeout=5.0) - assert acq1 is True, "Failed to acquire emulator lock" + def _owned_by(lock: DeviceExecutionLock) -> bool: + owner = DeviceExecutionLock.get_active_owner(lock.device_id, lock.lock_scope) + return owner is not None and owner.pid == os.getpid() and owner.token == lock.token + + # Both locks can be held simultaneously because device serials differ. try: - acq2 = lock_phone.acquire(timeout=5.0) - assert acq2 is True, ( - "Failed to acquire phone lock concurrently (multi-device parallel mode)" + lock_emu.acquire(timeout=5.0) + assert _owned_by(lock_emu), "Emulator lock not held after acquire()" + lock_phone.acquire(timeout=5.0) + assert _owned_by(lock_phone), ( + "Failed to hold phone lock concurrently (multi-device parallel mode)" ) - lock_phone.release() + assert _owned_by(lock_emu), "Emulator lock lost while acquiring phone lock" finally: + lock_phone.release() lock_emu.release() + assert not _owned_by(lock_emu) + assert not _owned_by(lock_phone) + + +def _run_cli(*args: str) -> subprocess.CompletedProcess: + # A wide, colorless console keeps Rich from wrapping or styling option names. + env = {**os.environ, "COLUMNS": "200", "NO_COLOR": "1", "TERM": "dumb"} + return subprocess.run( + [sys.executable, "-m", "artemis.main", *args], + capture_output=True, + text=True, + env=env, + timeout=120, + ) def test_06_cli_daemon_and_standalone_options(): """Verify CLI commands properly expose and route through Daemon or standalone.""" - # artemis status - res = subprocess.run( - [sys.executable, "-m", "artemis.main", "status"], capture_output=True, text=True - ) - assert res.returncode == 0 - assert "online" in res.stdout.lower() or "running" in res.stdout.lower() + # artemis status renders the Daemon status panel ("● ONLINE / RUNNING"). + res = _run_cli("status") + assert res.returncode == 0, res.stdout + res.stderr + assert "online / running" in res.stdout.lower(), res.stdout # artemis run --help exposes --standalone and --device - res_run = subprocess.run( - [sys.executable, "-m", "artemis.main", "run", "--help"], capture_output=True, text=True - ) - assert res_run.returncode == 0 + res_run = _run_cli("run", "--help") + assert res_run.returncode == 0, res_run.stdout + res_run.stderr assert "--standalone" in res_run.stdout assert "--device" in res_run.stdout # artemis batch --help exposes --standalone - res_batch = subprocess.run( - [sys.executable, "-m", "artemis.main", "batch", "--help"], capture_output=True, text=True - ) - assert res_batch.returncode == 0 + res_batch = _run_cli("batch", "--help") + assert res_batch.returncode == 0, res_batch.stdout + res_batch.stderr assert "--standalone" in res_batch.stdout +def _scheduler_session_ids() -> set[str]: + status = get_daemon_status() or {} + return { + str(item.get("session_id")) + for key in ("active_tasks", "queue") + for item in status.get(key) or [] + if isinstance(item, dict) + } + + def test_07_daemon_session_details_api(): """Verify Daemon GET /api/sessions/{session_id} API endpoint.""" + goal = "E2E session query test: open Android Settings and do not change anything." + target_serial = os.environ.get("ARTEMIS_TEST_DEVICE_SERIAL") or next( + (d.serial for d in device_pool.list_devices() if d.is_emulator), None + ) + assert target_serial, "No target device for the session details test" + res = submit_task_to_daemon( - goal="E2E session query test", + goal=goal, profile="flash", + device_serial=target_serial, ingress="e2e_test", ) - assert "session_id" in res - session_id = res["session_id"] - - # Poll session details - session_data = get_daemon_session(session_id) - assert session_data is not None - assert session_data["session_id"] == session_id - assert session_data["initial_goal"] == "E2E session query test" + assert res is not None, "Daemon rejected or did not answer /api/run" + # /api/run returns {"status", "tasks": [{"session_id", ...}], "enqueued_count", ...}. + assert res.get("status") in ("started", "queued"), res + assert res.get("enqueued_count") == 1, res + tasks = res.get("tasks") or [] + assert len(tasks) == 1 and tasks[0].get("session_id"), res + session_id = str(tasks[0]["session_id"]) - # Clean up test task - stop_task_on_daemon(session_id=session_id) + try: + # The session row is persisted once the worker picks the task up. + session_data = None + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + session_data = get_daemon_session(session_id) + if session_data is not None: + break + time.sleep(1.0) + assert session_data is not None, f"Session {session_id} never became queryable" + assert session_data["session_id"] == session_id + assert session_data["initial_goal"] == goal + finally: + # Clean up: stop the task and wait until it no longer owns a scheduler slot. + stop_task_on_daemon(session_id=session_id) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline and session_id in _scheduler_session_ids(): + time.sleep(0.5) + DeviceExecutionLock.cleanup_stale_locks() + assert session_id not in _scheduler_session_ids(), "Stopped task still owns a slot" diff --git a/tests/integration/test_thin_sdk_live.py b/tests/integration/test_thin_sdk_live.py index 9a72c606a..52a89f096 100644 --- a/tests/integration/test_thin_sdk_live.py +++ b/tests/integration/test_thin_sdk_live.py @@ -16,6 +16,8 @@ import os import asyncio +import shutil +import subprocess import time import uuid @@ -39,6 +41,22 @@ def _live_client() -> tuple[ArtemisClient, str]: ) +def _device_prop(serial: str, prop: str) -> str: + adb = shutil.which("adb") + if not adb: + pytest.skip("adb is required to read the target device's properties") + result = subprocess.run( + [adb, "-s", serial, "shell", "getprop", prop], + capture_output=True, + text=True, + timeout=30, + ) + value = result.stdout.strip() + if result.returncode != 0 or not value: + pytest.skip(f"Could not read {prop} from {serial}: {result.stderr.strip()}") + return value + + @pytest.mark.android @pytest.mark.asyncio async def test_thin_sdk_remote_contract_on_real_device(): @@ -53,25 +71,40 @@ async def test_thin_sdk_remote_contract_on_real_device(): devices = await client.list_devices() assert any(device.serial == serial and device.state == "device" for device in devices) + # Verify against the target device's own values so the goal is achievable on + # any phone or emulator rather than one hardcoded model. + model = _device_prop(serial, "ro.product.model") + android_version = _device_prop(serial, "ro.build.version.release") + goal = ( + "Open Android Settings, navigate to About phone (About emulated device on " + f"emulators), and verify that the visible model is {model} and the Android " + f"version is {android_version}. Do not change any setting. Stop after both " + "values are observed." + ) + task_id = str(uuid.uuid4()) handle = await client.submit( - "Open Android Settings, navigate to About phone, and verify that the visible model is Pixel 10 " - "and the Android version is 16. Do not change any setting. Stop after both values are observed.", + goal, task_id=task_id, locked_app_package="com.android.settings", ) assert handle.task_id == task_id - duplicate = await client.submit( - "Open Android Settings, navigate to About phone, and verify that the visible model is Pixel 10 " - "and the Android version is 16. Do not change any setting. Stop after both values are observed.", - task_id=task_id, - locked_app_package="com.android.settings", - ) - assert duplicate.task_id == task_id - - result = await client.wait_for_task(task_id, timeout=600) - assert result.succeeded, result.error or result.raw + finished = False + try: + duplicate = await client.submit( + goal, + task_id=task_id, + locked_app_package="com.android.settings", + ) + assert duplicate.task_id == task_id + + result = await client.wait_for_task(task_id, timeout=600) + finished = True + assert result.succeeded, result.error or result.raw + finally: + if not finished: + await client.stop(task_id) cleanup_deadline = time.monotonic() + 30 while time.monotonic() < cleanup_deadline: diff --git a/tests/unit/admin_console/test_replay_state.py b/tests/unit/admin_console/test_replay_state.py new file mode 100644 index 000000000..0c5594600 --- /dev/null +++ b/tests/unit/admin_console/test_replay_state.py @@ -0,0 +1,113 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests that ``ReplayManager.instantiate_state`` builds a valid ``State``. + +``State`` uses ``extra="forbid"``, so any unknown field makes replay fail. +""" + +import json +from pathlib import Path + +import pytest + +from apps.admin_console import replay_manager as rm_module +from apps.admin_console.replay_manager import ReplayManager +from artemis.graph.state import State + +SESSION_ID = "11111111-2222-3333-4444-555555555555" +REAL_DATA_DIR = Path(rm_module.TRACES_PATH) / "replay" / "data" + + +def _make_manager(tmp_path: Path, monkeypatch) -> ReplayManager: + manager = ReplayManager(workspace_root=tmp_path, original_db_path=tmp_path / "missing.db") + manager.test_data_dir = tmp_path / "data" + # Chunking reads the master DB; the fixture dir is already "chunked". + monkeypatch.setattr(manager, "_ensure_session_chunked", lambda session_id: None) + return manager + + +def _write_step_dir(tmp_path: Path) -> Path: + session_dir = tmp_path / "data" / f"{SESSION_ID}_chunked" + step_dir = session_dir / "step_01" + step_dir.mkdir(parents=True) + (session_dir / ".chunked").touch() + (session_dir / "session.json").write_text( + json.dumps({"initial_goal": "Open Settings and enable dark mode"}), encoding="utf-8" + ) + (step_dir / "step.json").write_text( + json.dumps( + { + "step_id": "step-uuid-1", + "session_id": SESSION_ID, + "step_number": 1, + "timestamp": 1790638731.79, + "action_taken": {"action": "manage_app", "app_name": "Settings"}, + } + ), + encoding="utf-8", + ) + (step_dir / "pre_image_meta.json").write_text( + json.dumps( + { + "timestamp": 1790638724.78, + "ui_tree": [ + {"index": "0", "text": "16:38", "package": "com.android.systemui"}, + {"index": "1", "text": "Display", "package": "com.android.settings"}, + ], + } + ), + encoding="utf-8", + ) + (step_dir / "pre.jpg").write_bytes(b"\xff\xd8\xff") + return step_dir + + +def test_instantiate_state_builds_valid_state_from_step_dir(tmp_path, monkeypatch): + step_dir = _write_step_dir(tmp_path) + manager = _make_manager(tmp_path, monkeypatch) + + state = manager.instantiate_state(SESSION_ID, 1) + + assert isinstance(state, State) + assert state.initial_goal == "Open Settings and enable dark mode" + assert state.current_step_id == "step-uuid-1" + assert state.latest_screenshot == str(step_dir / "pre.jpg") + assert state.latest_ui_hierarchy[1]["package"] == "com.android.settings" + assert json.loads(state.structured_decisions)["action"] == "manage_app" + assert state.subagent_calls == [] + # Must round-trip through the strict schema. + State.model_validate(state.model_dump()) + + +def _real_step_dirs() -> list[Path]: + if not REAL_DATA_DIR.is_dir(): + return [] + return sorted(REAL_DATA_DIR.glob("*_chunked/step_01"))[:5] + + +@pytest.mark.parametrize("step_dir", _real_step_dirs(), ids=lambda p: p.parent.name[:8]) +def test_instantiate_state_on_recorded_step_dirs(step_dir, monkeypatch): + """Exercise real chunked traces when present locally (traces/ is gitignored).""" + manager = ReplayManager() + manager.test_data_dir = step_dir.parent.parent + monkeypatch.setattr(manager, "_ensure_session_chunked", lambda session_id: None) + monkeypatch.setattr(manager, "load_session_goal", lambda *a, **k: "recorded goal") + session_id = step_dir.parent.name.removesuffix("_chunked") + + state = manager.instantiate_state(session_id, 1) + + assert isinstance(state, State) + assert state.initial_goal == "recorded goal" + assert state.current_step_id diff --git a/tests/unit/admin_console/test_server_status_endpoint.py b/tests/unit/admin_console/test_server_status_endpoint.py new file mode 100644 index 000000000..9e46c3d68 --- /dev/null +++ b/tests/unit/admin_console/test_server_status_endpoint.py @@ -0,0 +1,73 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for GET /api/system/server-status. + +``is_artemis_daemon`` calls this route with a short timeout, so it must not +use ``server_lifecycle.get_server_status`` (which runs ``lsof``). +""" + +import os + +import pytest +from httpx import ASGITransport, AsyncClient + +from apps.admin_console.server import app + + +def _boom(*args, **kwargs): + raise AssertionError("server-status must not scan ports/processes") + + +@pytest.mark.asyncio +async def test_server_status_answers_in_process_without_port_scans(monkeypatch): + from artemis.runtime import server_lifecycle + + monkeypatch.setattr(server_lifecycle, "find_server_pids", _boom) + monkeypatch.setattr(server_lifecycle, "get_server_status", _boom) + monkeypatch.setattr(server_lifecycle.subprocess, "run", _boom) + monkeypatch.setattr(server_lifecycle, "read_server_info", lambda: None) + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://localhost") as ac: + res = await ac.get("/api/system/server-status") + + assert res.status_code == 200 + data = res.json() + assert data["running"] is True + assert data["current_pid"] == os.getpid() + assert data["active_pid"] == os.getpid() + assert os.getpid() in data["pids"] + assert data["url"] == f"http://localhost:{data['port']}" + assert data["uptime_seconds"] is None or data["uptime_seconds"] >= 0 + assert "metadata" not in data + + +@pytest.mark.asyncio +async def test_server_status_uses_metadata_start_time(monkeypatch): + from apps.admin_console.core.state import state + from artemis.runtime import server_lifecycle + + monkeypatch.setattr( + server_lifecycle, + "read_server_info", + lambda: {"pid": os.getpid(), "port": state.port, "started_at": 1.0, "lifecycle_token": "x"}, + ) + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://localhost") as ac: + res = await ac.get("/api/system/server-status") + + data = res.json() + assert data["uptime_seconds"] > 1_000_000 + assert data["pids"] == [os.getpid()] + assert "lifecycle_token" not in res.text diff --git a/tests/unit/mcp/test_device_utils.py b/tests/unit/mcp/test_device_utils.py index 02fa97f3d..a10bafb6b 100644 --- a/tests/unit/mcp/test_device_utils.py +++ b/tests/unit/mcp/test_device_utils.py @@ -9,6 +9,16 @@ def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None: popen = MagicMock() monkeypatch.setattr(device_utils.sys, "platform", "win32") + # CREATE_NEW_PROCESS_GROUP / DETACHED_PROCESS only exist in the Windows build of + # ``subprocess``; supply them (keeping the real values on Windows) so faking + # win32 works on any host instead of raising AttributeError inside the code. + for name, value in (("CREATE_NEW_PROCESS_GROUP", 0x200), ("DETACHED_PROCESS", 0x8)): + monkeypatch.setattr( + device_utils.subprocess, + name, + getattr(device_utils.subprocess, name, value), + raising=False, + ) monkeypatch.setattr(device_utils, "is_emulator_running", lambda _adb: False) monkeypatch.setattr(device_utils.os.path, "exists", lambda _path: True) monkeypatch.setattr(device_utils.subprocess, "Popen", popen) diff --git a/tests/unit/test_main_entrypoint.py b/tests/unit/test_main_entrypoint.py new file mode 100644 index 000000000..190a41adf --- /dev/null +++ b/tests/unit/test_main_entrypoint.py @@ -0,0 +1,90 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for argv handling in ``python -m artemis.main``. + +Only a free-text goal is rewritten into ``run ``; registered commands +and root options are passed through unchanged. +""" + +import pytest +import typer + +from artemis import main as legacy_main +from artemis.interfaces.cli.main import app + + +@pytest.mark.parametrize( + "token", + [ + "status", + "ui", + "stop", + "restart", + "helper", + "run", + "init", + "doctor", + "batch", + "mcp", + "server", + "trace", + "--version", + "-v", + "--help", + "-h", + ], +) +def test_known_commands_and_flags_are_not_rewritten(token): + argv = ["artemis", token, "extra"] + assert legacy_main.normalize_argv(argv) == argv + + +@pytest.mark.parametrize( + "argv", + [ + ["artemis", "Open Settings and enable dark mode"], + ["artemis", "status of my battery"], + ["artemis", "--device", "emulator-5554", "open settings"], + ], +) +def test_free_text_goal_is_rewritten_to_run(argv): + assert legacy_main.normalize_argv(argv) == [argv[0], "run", *argv[1:]] + + +def test_no_args_is_left_alone(): + assert legacy_main.normalize_argv(["artemis"]) == ["artemis"] + + +def test_every_registered_command_is_known(): + """Every command registered on the Typer app is a known root token.""" + registered = set(typer.main.get_command(app).commands) + assert registered + assert registered <= legacy_main._root_tokens() + + +def test_cli_status_dispatches_status_not_run(monkeypatch): + calls = [] + + def fake_app(): + calls.append(list(legacy_main.sys.argv)) + + monkeypatch.setattr(legacy_main, "app", fake_app) + monkeypatch.setattr(legacy_main.sys, "argv", ["artemis", "status"]) + legacy_main.cli() + assert calls == [["artemis", "status"]] + + monkeypatch.setattr(legacy_main.sys, "argv", ["artemis", "open settings"]) + legacy_main.cli() + assert calls[-1] == ["artemis", "run", "open settings"]