diff --git a/.cursor/Dockerfile b/.cursor/Dockerfile new file mode 100644 index 0000000000..a42c723c6b --- /dev/null +++ b/.cursor/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.13-slim-trixie + +ENV DEBIAN_FRONTEND=noninteractive +ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + git \ + sudo \ + curl \ + ca-certificates \ + build-essential \ + gcc \ + binutils \ + libffi-dev \ + libssl-dev \ + libxml2-dev \ + libxslt1-dev \ + libjpeg62-turbo-dev \ + libopenblas-dev \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace diff --git a/.cursor/README.md b/.cursor/README.md new file mode 100644 index 0000000000..94e0292a94 --- /dev/null +++ b/.cursor/README.md @@ -0,0 +1,51 @@ +# OctoBot Cursor Cloud environment + +CI-parity install and validation for [Cursor Cloud Agents](https://cursor.com/docs/cloud-agent/setup). + +## Profiles (`OCTOBOT_CLOUD_PROFILE`) + +| Profile | Layers | +|---------|--------| +| `package-only` | L1–L3 (no tentacles install) | +| `ci-tentacles` | L1–L5 (GitHub `cloud-env-validate`) | +| `ui-node-web` | L1–L6 (**default** in `environment.json`; includes `node_web_interface` build) | + +Install: `bash .cursor/cloud-install.sh` (profile via env var). + +## Agent / shell usage + +```bash +source .cursor/env.sh +# Python, OctoBot CLI, pytest +``` + +Node web UI demo (seed, debug view): skill **agent-seed** (`.cursor/skills/agent-seed/SKILL.md`). + +After editing `packages/tentacles/`: `bash .cursor/reinstall-tentacles.sh` (do not edit repo-root `tentacles/`). + +## Validate environment (on demand) + +```bash +source .cursor/env.sh +bash .cursor/cloud-validate.sh --profile ci-tentacles --json +``` + +Not run from install; use after Build changes or when debugging a broken env. + +## Debugging failures + +| Surface | What to search | +|---------|----------------| +| Cursor Cloud Build log | `OCTOBOT_CLOUD_FAILED` or `==> [L` step markers | +| GitHub `cloud-env-validate` | Same markers; `cloud-validate.json` artifact on failure | +| Success install | `OCTOBOT_CLOUD_INSTALL_OK` | + +If a dashboard environment snapshot shadows this repo config, remove the saved snapshot so committed `environment.json` wins. + +## PRs and policy + +- Target branch: **`dev`** (unless release flow to `master`). +- Before handoff: `python -m tools.extended_linter --base origin/dev` (match PR base ref). +- CI: **OctoBot-CI** job **`extended_linter`** (wheel + tentacles + `pytest tools/tests`; PR policy step uses `--skip-tentacles-reinstall`, no `cloud-install`). Cloud agents use **`ui-node-web`** (`environment.json`). GitHub **`cloud-env-validate`** still installs **`ci-tentacles`**. +- Agent docs: colocated `AGENTS.md` (see root index), skills **octobot-cloud** and **agent-seed** (Node UI QA). +- Plans whose title or description include **roadmap**: skill **cloud-roadmap** (`.cursor/skills/cloud-roadmap/SKILL.md`) before `CreatePlan`; execution uses **octobot-cloud**. diff --git a/.cursor/agent-seed.env b/.cursor/agent-seed.env new file mode 100644 index 0000000000..7c34db42bb --- /dev/null +++ b/.cursor/agent-seed.env @@ -0,0 +1,2 @@ +OCTOBOT_AGENT_SEED_USER_FOLDER=user/agent-seed +NODE_SQLITE_FILE=user/agent-seed/tasks.db diff --git a/.cursor/cloud-install.sh b/.cursor/cloud-install.sh new file mode 100644 index 0000000000..35bab8e91c --- /dev/null +++ b/.cursor/cloud-install.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/cloud-shell-lib.sh" +cloud_lib_init "cloud-install.sh" + +echo "OCTOBOT_CLOUD_PROFILE=${CLOUD_PROFILE}" +log_step "INIT" "repo=${REPO_ROOT}" + +if cloud_profile_includes_layer "L1"; then + log_step "L1" "Toolchain (Pants, optional Rust)" + if ! command -v pants >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash + fi + export PATH="${HOME}/.local/bin:${PATH}" + if compgen -G "packages/*/crates/*/Cargo.toml" >/dev/null 2>&1; then + if ! command -v cargo >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + # shellcheck disable=SC1091 + source "${HOME}/.cargo/env" + fi + fi +fi + +if cloud_profile_includes_layer "L2"; then + log_step "L2" "Pants package OctoBot wheel" + export PATH="${HOME}/.local/bin:${PATH}" + pants package :OctoBot $(pants list --filter-target-type=package_shell_command ::) +fi + +if cloud_profile_includes_layer "L3"; then + log_step "L3" "Python venv and OctoBot wheel" + python3 -m venv "${REPO_ROOT}/.cursor/venv" + cloud_write_env_files + # shellcheck disable=SC1091 + source "${REPO_ROOT}/.cursor/env.sh" + pip install --upgrade pip setuptools wheel + pip install -r dev_requirements.txt + pip install dist/*.whl + if compgen -G "packages/*/crates/*/Cargo.toml" >/dev/null 2>&1; then + pip install maturin + fi +fi + +if cloud_profile_includes_layer "L4"; then + cloud_npm_client_build +fi + +if cloud_profile_includes_layer "L5"; then + cloud_tentacles_pack_and_install +fi + +if [[ "${CLOUD_PROFILE}" == "ui-node-web" ]]; then + cloud_node_web_interface_build +fi + +echo "OCTOBOT_CLOUD_INSTALL_OK profile=${CLOUD_PROFILE}" diff --git a/.cursor/cloud-shell-lib.sh b/.cursor/cloud-shell-lib.sh new file mode 100644 index 0000000000..eba8d03ec8 --- /dev/null +++ b/.cursor/cloud-shell-lib.sh @@ -0,0 +1,109 @@ +# Shared helpers for OctoBot Cursor Cloud shell scripts. +# shellcheck shell=bash + +cloud_lib_init() { + local script_name="$1" + CLOUD_SCRIPT_NAME="$script_name" + set -euo pipefail + CLOUD_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REPO_ROOT="$(cd "${CLOUD_LIB_DIR}/.." && pwd)" + cd "${REPO_ROOT}" + CLOUD_PROFILE="${OCTOBOT_CLOUD_PROFILE:-ci-tentacles}" + CLOUD_LAST_LOG_TAIL="" + trap 'cloud_err_trap' ERR +} + +log_step() { + local step_id="$1" + local message="$2" + echo "==> [${step_id}] ${message}" +} + +die() { + local step_id="$1" + local message="$2" + echo "OCTOBOT_CLOUD_FAILED script=${CLOUD_SCRIPT_NAME} step=${step_id} profile=${CLOUD_PROFILE} message=${message}" >&2 + if [[ -n "${CLOUD_LAST_LOG_TAIL}" && -f "${CLOUD_LAST_LOG_TAIL}" ]]; then + echo "--- last command output (tail) ---" >&2 + tail -n 30 "${CLOUD_LAST_LOG_TAIL}" >&2 || true + fi + exit 1 +} + +cloud_err_trap() { + local step_id="${CLOUD_TRAP_STEP:-ERR}" + die "${step_id}" "command failed: ${BASH_COMMAND}" +} + +cloud_profile_includes_layer() { + local layer="$1" + case "${CLOUD_PROFILE}" in + package-only) + [[ "${layer}" == "L1" || "${layer}" == "L2" || "${layer}" == "L3" ]] && return 0 + return 1 + ;; + ci-tentacles) + [[ "${layer}" == "L6" ]] && return 1 + return 0 + ;; + ui-node-web) + return 0 + ;; + *) + die "INIT" "unknown OCTOBOT_CLOUD_PROFILE=${CLOUD_PROFILE}" + ;; + esac +} + +cloud_write_env_files() { + log_step "ENV_FILES" "Writing .cursor/env.sh and .cursor/pythonpath.sh" + cat >"${REPO_ROOT}/.cursor/env.sh" </dev/null | cut -d. -f1)" +for _octobot_node in /usr/bin/node /usr/local/bin/node; do + if [ -x "\$_octobot_node" ]; then + _octobot_node_major="\$("\$_octobot_node" --version 2>/dev/null | tr -dc '0-9.' | cut -d. -f1)" + if [ -n "\$_octobot_nvmrc_major" ] && [ "\$_octobot_node_major" = "\$_octobot_nvmrc_major" ]; then + export PATH="\$(dirname "\$_octobot_node"):\${PATH}" + break + fi + fi +done +unset _octobot_nvmrc_major _octobot_node _octobot_node_major +# venv bin must remain first so python/OctoBot resolve to the OctoBot venv. +export PATH="\${OCTOBOT_CLOUD_VENV}/bin:\${PATH}" +EOF + cat >"${REPO_ROOT}/.cursor/pythonpath.sh" <<'EOF' +# For packages/node, packages/flow, packages/copy tests from repo root (matches CI) +export PYTHONPATH=".:${PYTHONPATH:-}" +EOF +} + +cloud_tentacles_pack_and_install() { + log_step "TENTACLES_PACK" "OctoBot tentacles pack from packages/tentacles" + mkdir -p "${REPO_ROOT}/output" + export ALLOW_UNSIGNED_TENTACLES=true + # shellcheck disable=SC1091 + source "${REPO_ROOT}/.cursor/env.sh" + # -p is package filename only; default output_dir is output/ (see main.yml tentacles install) + OctoBot tentacles -d packages/tentacles -p any_platform.zip + log_step "TENTACLES_INSTALL" "OctoBot tentacles install to repo root" + OctoBot tentacles --install --location output/any_platform.zip --all +} + +cloud_npm_client_build() { + log_step "L4" "npm ci and build octobot_client_ts" + npm ci + npm run build -w packages/client/octobot_client_ts +} + +cloud_node_web_interface_build() { + local ui_dir="${REPO_ROOT}/packages/tentacles/Services/Interfaces/node_web_interface" + log_step "L6" "npm ci and build node_web_interface" + (cd "${ui_dir}" && npm ci && npm run build) +} diff --git a/.cursor/cloud-validate.sh b/.cursor/cloud-validate.sh new file mode 100644 index 0000000000..10f6ae9c67 --- /dev/null +++ b/.cursor/cloud-validate.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/cloud-shell-lib.sh" +cloud_lib_init "cloud-validate.sh" + +EMIT_JSON=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --profile) CLOUD_PROFILE="$2"; shift 2 ;; + --json) EMIT_JSON=1; shift ;; + *) die "ARGS" "unknown argument $1" ;; + esac +done + +VALIDATE_STEPS=() +VALIDATE_OK=1 +VALIDATE_FAILED_STEP="" +VALIDATE_MESSAGE="" + +validate_record() { + local name="$1" + local ok="$2" + local detail="$3" + VALIDATE_STEPS+=("${name}|${ok}|${detail}") + if [[ "${ok}" != "1" ]]; then + VALIDATE_OK=0 + VALIDATE_FAILED_STEP="${name}" + VALIDATE_MESSAGE="${detail}" + return 1 + fi + return 0 +} + +validate_emit_json() { + export VALIDATE_OK="${VALIDATE_OK}" + export VALIDATE_FAILED_STEP="${VALIDATE_FAILED_STEP}" + export VALIDATE_MESSAGE="${VALIDATE_MESSAGE}" + export CLOUD_PROFILE="${CLOUD_PROFILE}" + export VALIDATE_STEP_LINES + VALIDATE_STEP_LINES="$(printf '%s\n' "${VALIDATE_STEPS[@]}")" + python3 - <<'PY' +import json +import os + +steps = [] +for entry in os.environ.get("VALIDATE_STEP_LINES", "").split("\n"): + if not entry.strip(): + continue + name, ok, detail = entry.split("|", 2) + steps.append({"name": name, "ok": ok == "1", "detail": detail}) + +failed = os.environ.get("VALIDATE_FAILED_STEP") or None +message = os.environ.get("VALIDATE_MESSAGE") or None +if failed == "": + failed = None +if message == "": + message = None + +payload = { + "ok": os.environ.get("VALIDATE_OK") == "1", + "profile": os.environ.get("CLOUD_PROFILE"), + "failed_step": failed, + "message": message, + "steps": steps, +} +print(json.dumps(payload, indent=2)) +PY +} + +validate_finish() { + if [[ "${EMIT_JSON}" == "1" ]]; then + validate_emit_json | tee "${REPO_ROOT}/cloud-validate.json" + fi + if [[ "${VALIDATE_OK}" != "1" ]]; then + die "${VALIDATE_FAILED_STEP}" "${VALIDATE_MESSAGE}" + fi + exit 0 +} + +[[ -f "${REPO_ROOT}/.cursor/env.sh" ]] || die "VALIDATE_ENV" "missing .cursor/env.sh — run cloud-install.sh" +# shellcheck disable=SC1091 +source "${REPO_ROOT}/.cursor/env.sh" + +log_step "VALIDATE_PYTHON_VERSION" "python 3.13.x" +pyver="$(python --version 2>&1)" || die "VALIDATE_PYTHON_VERSION" "python not runnable" +if [[ "${pyver}" != *"3.13"* ]]; then + validate_record "VALIDATE_PYTHON_VERSION" 0 "${pyver}" || validate_finish +fi +validate_record "VALIDATE_PYTHON_VERSION" 1 "ok" || validate_finish + +log_step "VALIDATE_OCTOBOT_CLI" "OctoBot on PATH" +if ! OctoBot --version >/dev/null; then + validate_record "VALIDATE_OCTOBOT_CLI" 0 "OctoBot CLI failed" || validate_finish +fi +validate_record "VALIDATE_OCTOBOT_CLI" 1 "ok" || validate_finish + +log_step "VALIDATE_IMPORT_OCTOBOT" "import octobot" +if ! python -c "import octobot"; then + validate_record "VALIDATE_IMPORT_OCTOBOT" 0 "import failed" || validate_finish +fi +validate_record "VALIDATE_IMPORT_OCTOBOT" 1 "ok" || validate_finish + +needs_tentacles=false +[[ "${CLOUD_PROFILE}" != "package-only" ]] && needs_tentacles=true + +if [[ "${needs_tentacles}" == "true" ]]; then + log_step "VALIDATE_TENTACLES_DIR" "repo-root tentacles/" + if [[ ! -d tentacles ]] || [[ -z "$(ls -A tentacles 2>/dev/null)" ]]; then + validate_record "VALIDATE_TENTACLES_DIR" 0 "missing or empty" || validate_finish + fi + validate_record "VALIDATE_TENTACLES_DIR" 1 "ok" || validate_finish + + log_step "VALIDATE_IMPORT_TENTACLES" "import tentacles" + if ! python -c "import tentacles"; then + validate_record "VALIDATE_IMPORT_TENTACLES" 0 "import failed" || validate_finish + fi + validate_record "VALIDATE_IMPORT_TENTACLES" 1 "ok" || validate_finish + + client_dist="packages/client/octobot_client_ts/dist/index.js" + log_step "VALIDATE_CLIENT_BUILD" "${client_dist}" + if [[ ! -f "${client_dist}" ]]; then + validate_record "VALIDATE_CLIENT_BUILD" 0 "missing" || validate_finish + fi + validate_record "VALIDATE_CLIENT_BUILD" 1 "ok" || validate_finish +fi + +log_step "VALIDATE_SENTINEL_OCTOBOT" "pytest collect-only" +if ! pytest --collect-only tests/unit_tests/test_octobot_version.py >/dev/null; then + validate_record "VALIDATE_SENTINEL_OCTOBOT" 0 "collect failed" || validate_finish +fi +validate_record "VALIDATE_SENTINEL_OCTOBOT" 1 "ok" || validate_finish + +log_step "VALIDATE_SENTINEL_COMMONS" "pytest collect-only" +if ! (cd packages/commons && pytest --collect-only tests/test_copy_util.py >/dev/null); then + validate_record "VALIDATE_SENTINEL_COMMONS" 0 "collect failed" || validate_finish +fi +validate_record "VALIDATE_SENTINEL_COMMONS" 1 "ok" || validate_finish + +if [[ "${needs_tentacles}" == "true" ]]; then + log_step "VALIDATE_SENTINEL_FLOW_COLLECT" "pytest collect-only with PYTHONPATH" + # shellcheck disable=SC1091 + source "${REPO_ROOT}/.cursor/pythonpath.sh" + if ! pytest --collect-only packages/flow/tests/protocol/test_wire_compat.py >/dev/null; then + validate_record "VALIDATE_SENTINEL_FLOW_COLLECT" 0 "collect failed" || validate_finish + fi + validate_record "VALIDATE_SENTINEL_FLOW_COLLECT" 1 "ok" || validate_finish +fi + +if [[ "${CLOUD_PROFILE}" == "ui-node-web" ]]; then + ui_dist="packages/tentacles/Services/Interfaces/node_web_interface/dist" + log_step "VALIDATE_UI_BUILD" "${ui_dist}" + if [[ ! -d "${ui_dist}" ]]; then + validate_record "VALIDATE_UI_BUILD" 0 "missing" || validate_finish + fi + validate_record "VALIDATE_UI_BUILD" 1 "ok" || validate_finish +fi + +validate_finish diff --git a/.cursor/environment.json b/.cursor/environment.json new file mode 100644 index 0000000000..c5ab45e722 --- /dev/null +++ b/.cursor/environment.json @@ -0,0 +1,7 @@ +{ + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "install": "PANTS_PANTSD=false OCTOBOT_CLOUD_PROFILE=ui-node-web bash .cursor/cloud-install.sh" +} diff --git a/.cursor/reinstall-tentacles.sh b/.cursor/reinstall-tentacles.sh new file mode 100644 index 0000000000..17a91c729d --- /dev/null +++ b/.cursor/reinstall-tentacles.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/cloud-shell-lib.sh" +cloud_lib_init "reinstall-tentacles.sh" +# shellcheck disable=SC1091 +[[ -f "${REPO_ROOT}/.cursor/env.sh" ]] || die "INIT" "run cloud-install.sh first" +source "${REPO_ROOT}/.cursor/env.sh" +cloud_tentacles_pack_and_install +if [[ "${CLOUD_PROFILE}" == "ui-node-web" ]]; then + cloud_node_web_interface_build +fi +echo "OCTOBOT_CLOUD_INSTALL_OK profile=${CLOUD_PROFILE} script=reinstall-tentacles" diff --git a/.cursor/rules/octobot-cloud.mdc b/.cursor/rules/octobot-cloud.mdc new file mode 100644 index 0000000000..89e5df06fd --- /dev/null +++ b/.cursor/rules/octobot-cloud.mdc @@ -0,0 +1,22 @@ +--- +description: OctoBot Cursor Cloud — env, tentacles, extended_linter, PR workflow +alwaysApply: true +--- + +# OctoBot cloud (always on) + +- Work at the **OctoBot git repo root** (this repository). Do not use sibling monorepo paths or workspace-only skills. +- Before Python, `OctoBot`, or pytest: `source .cursor/env.sh` +- **Pytest failure:** follow [When tests fail](.cursor/skills/octobot-cloud/reference-pytest.md#when-tests-fail) in `reference-pytest.md`, starting with [step 1 — visible logs](.cursor/skills/octobot-cloud/reference-pytest.md#1-re-run-with-visible-logs); do not patch symptoms (mocks, seeds, timeouts) until the failure boundary is identified. +- **Tentacles:** edit only `packages/tentacles/`; after changes run `bash .cursor/reinstall-tentacles.sh`. Never commit repo-root `tentacles/`. +- **Git:** checkout `dev` (or user base) → feature branch → commit → open PR to **`dev`** unless the user specifies another target. Do not commit agent plan files (`PLAN-*.md`, `*.plan.md`, `.cursor/plans/`); use Plan mode or chat only — delete scratch plans before staging (`path.deny_agent_plans`). Do **not** edit agent docs (`**/AGENTS.md`, `.cursor/skills/**`, `.cursor/rules/**`, `.cursor/README.md`, `CONTRIBUTING-agent.md`, `tools/**/README.md`, `tools/**/ARCHITECTURE.md`) unless the task owns them; if diffs are out of scope, `git checkout origin/ -- ` before commit (`agent_docs.no_regression_vs_merge_base`). +- **Node UI tests:** Vitest (`npm test`) + Python/API tests; do not add `node_web_interface/e2e/` or Playwright specs (`path.deny_node_web_playwright_e2e`). Optional live UI QA via skill **agent-seed**, not committed e2e. +- **UI copy/layout:** For `packages/tentacles/Services/Interfaces/node_web_interface/` or `.../web_interface/`, read skill **end-user-ui** (entry-level trading OK, no pro jargon, minimal, no slop, no em dash in UI strings). +- **Node journal:** From outside `octobot/community/node_journal/`, **record only** (`record_*`); do not `read_events` or use journal as source of truth. Reads belong to journal export inside that package. Skill **node-journal**; [`octobot/community/node_journal/AGENTS.md`](octobot/community/node_journal/AGENTS.md). +- **Python conventions:** Shared literals in package-top `constants.py` / `enums.py` (e.g. `octobot/constants.py`); top-level `import module as alias`; no lazy imports unless exceptional; public API via `__init__.py` re-exports (`from` imports only in `__init__.py`). Full detail: [CONTRIBUTING-agent.md](CONTRIBUTING-agent.md) — **Python conventions (agents)**. +- Before handoff: `python -m tools.extended_linter --base origin/dev` (use `origin/` matching the PR target). +- No `pip install` / `npm install` to fix imports. No edits under `user/` or secret/env files. +- Cross-package or tentacles work: read root `AGENTS.md` and colocated `AGENTS.md` for every area you touch (`octobot/`, `packages//`, `packages/tentacles/`). +- Install success: Build log must contain `OCTOBOT_CLOUD_INSTALL_OK`; optional `bash .cursor/cloud-validate.sh --profile ci-tentacles --json`. +- Before **CreatePlan**, if the plan title or description contains **roadmap** (any case): skill **cloud-roadmap** (`.cursor/skills/cloud-roadmap/SKILL.md`); implement with **octobot-cloud**. +- Details: skill **octobot-cloud**; policy `tools/extended_linter/config/policy.yaml`; architecture `tools/extended_linter/ARCHITECTURE.md`. diff --git a/.cursor/seed-agent.sh b/.cursor/seed-agent.sh new file mode 100644 index 0000000000..f0e3c5e8b3 --- /dev/null +++ b/.cursor/seed-agent.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Demo agent-seed orchestration: sync seed, optional node start, HTTP bootstrap. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +source "${SCRIPT_DIR}/env.sh" +set -a +source "${SCRIPT_DIR}/agent-seed.env" +set +a + +export OCTOBOT_AGENT_SEED_USER_FOLDER="${REPO_ROOT}/${OCTOBOT_AGENT_SEED_USER_FOLDER}" +export NODE_SQLITE_FILE="${REPO_ROOT}/${NODE_SQLITE_FILE}" + +run_seed() { + local clear_flag=() + if [[ "${1:-}" == "--clear" ]]; then + clear_flag=(--clear) + fi + python -m tools.agent_seed seed --user-folder "${OCTOBOT_AGENT_SEED_USER_FOLDER}" "${clear_flag[@]}" +} + +run_bootstrap() { + python -m tools.agent_seed bootstrap --base-url "${AGENT_SEED_BASE_URL:-http://127.0.0.1:8000}" "$@" +} + +run_start() { + export EXIT_BEFORE_TENTACLES_AUTO_REINSTALL=true + python start.py --master --user-folder "${OCTOBOT_AGENT_SEED_USER_FOLDER}" +} + +case "${1:-}" in + seed) + run_seed "${2:-}" + ;; + bootstrap) + shift + run_bootstrap "$@" + ;; + start) + run_start + ;; + --full) + run_seed + run_start & + run_bootstrap + ;; + --clear) + run_seed --clear + ;; + *) + echo "Usage: $0 {seed [--clear]|bootstrap [--scenario grid|index|completed|lifecycle|all]|start|--full|--clear}" >&2 + exit 1 + ;; +esac diff --git a/.cursor/skills/agent-seed/SKILL.md b/.cursor/skills/agent-seed/SKILL.md new file mode 100644 index 0000000000..31b6b4695f --- /dev/null +++ b/.cursor/skills/agent-seed/SKILL.md @@ -0,0 +1,63 @@ +--- +name: agent-seed +description: >- + Demo Node UI QA: seed/clear/bootstrap, first login passphrase, full /app UI for + bugs and experiments, /app/debug to send user actions. Skip for pytest-only work. +--- + +# Agent seed (Node UI demo) + +Full reference: [`tools/agent_seed/README.md`](../../../tools/agent_seed/README.md). Code layout: [`tools/agent_seed/ARCHITECTURE.md`](../../../tools/agent_seed/ARCHITECTURE.md). + +## When to use / when to skip + +| Task | Agent-seed? | +|------|-------------| +| `pytest`, linters, package tests | **Skip** | +| Live Node web UI (any `/app` page or debug) | **Use** this skill | + +| Goal | Command (node stopped unless noted) | +|------|-------------------------------------| +| First UI fixture data | `--full` or `seed` then `start` (+ optional `bootstrap`) | +| Wipe + fresh fixtures | `--clear` → `start` or `--full` | +| Node already up, need grid RUNNING | `bootstrap` only | +| Node up, UI only | Login → use `/app` as needed; `/app/debug` when sending user actions | + +- **`seed`**: fixtures + wallet (required for UI). +- **`bootstrap`**: start grid automation (optional unless task needs RUNNING grid). + +## Prep + +```bash +source .cursor/env.sh +source .cursor/agent-seed.env +``` + +Master tentacles on `user/reference_tentacles_config/` must exist (see `tools/agent_seed/README.md`). + +## Commands (OctoBot repo root) + +```bash +bash .cursor/seed-agent.sh --full # seed + background start + bootstrap — stop node first; do NOT run start again +bash .cursor/seed-agent.sh --clear # wipe + re-seed — stop node first +bash .cursor/seed-agent.sh bootstrap # node must already listen on AGENT_SEED_BASE_URL +bash .cursor/seed-agent.sh bootstrap --scenario index --scenario completed # more automations (grid|index|completed|lifecycle|all) +bash .cursor/seed-agent.sh start # foreground start only +``` + +**`--full` starts OctoBot.** Never double-start. Stop before `--full` or `--clear`. + +## After seed: interact with the UI + +1. Open **`http://127.0.0.1:8000/app/login`** (or `/app` → login). +2. Enter passphrase **`demodemo`** (every new browser session). +3. Use the **full Node UI under `/app`** to find bugs, reproduce issues, or experiment with seeded accounts and strategies. +4. Use **`/app/debug`** when the task is to **send commands to the node** — user actions, automation controls, and debug tables (automations, accounts, user-action history). This is usually the most useful page for driving the scheduler, not the only page you may visit. + +Credentials: wallet `0x70997970c51812dc3a010c7d01b50e0d17dc79c8`, display name `demo` — see fixture README. + +HTTP Basic (`wallet:passphrase`) works for **`/api/v1/debug/`** without a browser session (headless equivalent of debug-view user actions). + +## Demo-only + +Committed insecure wallet in `tools/agent_seed/secrets.py` — never mainnet / real trading. Sandbox blocks live exchange/automation creation (`octobot_node/agent_seed/demo_wallet.py`). diff --git a/.cursor/skills/cloud-roadmap/SKILL.md b/.cursor/skills/cloud-roadmap/SKILL.md new file mode 100644 index 0000000000..f2059f5b15 --- /dev/null +++ b/.cursor/skills/cloud-roadmap/SKILL.md @@ -0,0 +1,135 @@ +--- +name: cloud-roadmap +description: >- + Authors Cursor Plans for the OctoBot git repo that run on Cursor Cloud Agents. + Automatically applies whenever CreatePlan is used and the plan title or + overview/description contains "roadmap" (case-insensitive). Also applies when + the user asks for a cloud roadmap, cloud-safe multi-step plan, or names + cloud-roadmap. Requires repo-relative paths only, package/module responsibility + maps, clarify-before-assume, validation milestones with unit tests (pytest or + npm test/vitest) and functional tests when relevant, package ownership, and + clarification before guessing. +--- + +# Cloud roadmap (Plan authoring) + +Execution after the plan: skill **octobot-cloud** (`.cursor/skills/octobot-cloud/SKILL.md`). + +## Activation + +**Mandatory** before `CreatePlan` when the plan **name, title, or overview/description** contains `roadmap` (any case). + +Also when the user invokes **cloud-roadmap** or asks for a **cloud-safe multi-step plan**. + +1. Read **octobot-cloud** first. +2. Skim root `AGENTS.md` and every colocated `AGENTS.md` for packages in scope. +3. If not in Plan mode but the user wants a roadmap-style plan, use the same rules in the reply or switch to Plan + `CreatePlan`. + +**Output:** submit via **CreatePlan** only. Do **not** write `.cursor/roadmaps/*.md` unless the user asks to persist a file. Never commit plan artifacts in the repo (`PLAN-*.md`, `*.plan.md`, `.cursor/plans/`) — `path.deny_agent_plans`. + +## Clarify first; never guess + +Cloud agents cannot load workspace-only skills. Mirror clarify-dont-assume here. + +- If scope, PR target, behavior, boundaries, or **which package owns a change** is unclear, **stop and ask** (**AskQuestion** when available; otherwise 1–2 focused questions). Do **not** call **CreatePlan** until resolved or labeled **user must choose**. +- Plans may only use: (1) user input, (2) facts **verified** in this repo (`AGENTS.md`, search), (3) answers after you asked. +- Ask when: multiple owners, cross-package impact, tentacles vs `octobot` vs `tools/`, missing acceptance criteria, fix vs refactor, unknown PR base. + +## Hard constraints (every roadmap plan) + +| Rule | Detail | +|------|--------| +| Scope | OctoBot repo only: `octobot/`, `packages/`, `tools/`, `.cursor/`, `tests/`. No monorepo siblings, `cursor_cloud_env/`, or workspace-only skills. | +| Paths | Repo-relative from OctoBot root. Forbidden: `C:\...`, `/Users/...`, `../OctoBot/`, `OctoBot/` prefix at repo root. | +| Env | `source .cursor/env.sh` before Python/shell test steps. | +| Install | No `pip install` / `npm install` to fix imports. | +| Tentacles | Edit `packages/tentacles/` only; after edits `bash .cursor/reinstall-tentacles.sh`; never edit repo-root `tentacles/`. | + +## Responsibility map (required section in the plan) + +Place **before** phased steps. Verify paths; cite `AGENTS.md`. + +```markdown +## Responsibility map + +| Concern / behavior | Package | Module or path (repo-relative) | Role in this roadmap | +|--------------------|---------|--------------------------------|----------------------| +| … | `packages/` or `octobot` | `…/module.py` or subpath | Owns … / calls … / exposes … | +``` + +- One **primary owner** per concern; others as **depends on** or **integration point**. +- **Module** = real file or directory (search repo; read **Owns** in `AGENTS.md`). If still ambiguous, **ask** before locking the table. +- Each implementation **step** references at least one row. + +## Plan shape + +Use **phases** and **steps**. Each step ends with a **Validation milestone** including **tests**. + +```markdown +### Step N — [short title] +**Work:** … +**Owns:** `packages//…` (from responsibility map) +**Validation milestone:** +- **Unit tests (required):** + - Test file(s): `…` + - Cases: each behavior (happy path, regression, edges for edited APIs) + - **Command:** `pytest …` or `npm test` (cwd per below) + - **Pass criteria:** all listed tests green; no untested new public behavior +- **Functional tests:** propose explicitly, or **N/A — reason** +- **Other checks:** reinstall tentacles, cloud-validate, etc. if needed +- **If fail:** … +``` + +Rules: + +- Steps that **create or edit** code or runtime policy YAML need a **complete** unit-test list for that step’s delta (map each new/changed symbol or config key to a scenario). +- Unit tests in the **same step** as the code (test-first or test-same-step). +- **Functional tests:** required to propose or decline with reason when the step touches HTTP/API, tentacles loading, multi-package flows, CLI, Node UI, or trading/backtesting pipelines. +- Final phase: handoff `python -m tools.extended_linter --base origin/dev` (or `origin/`). +- Prefer steps sized for **one Cloud session**; note dependencies. + +## Testing in milestones + +Pytest matrix: [reference-pytest.md](../octobot-cloud/reference-pytest.md) (skill **octobot-cloud**); on failure see [When tests fail](../octobot-cloud/reference-pytest.md#when-tests-fail). + +| Layer | Requirement | Where | +|-------|-------------|--------| +| Unit (Python) | Test path + enumerated behaviors | Package `tests/` or `tools/tests/…` per `AGENTS.md` **Tests** | +| Unit (Node UI) | TS under `packages/tentacles/Services/Interfaces/node_web_interface/` | `src/**/__tests__`; cwd that directory; `npm test` (vitest). Profile **`ui-node-web`** if build env needed. | +| UI copy / UX | User-visible strings or layout in `node_web_interface` or `web_interface` | Review against skill **end-user-ui** (incl. no em dash in copy); Vitest or `web_interface` pytest per stack. | +| Functional / integration | Cross-package or I/O flows | `tools/tests`, integration dirs, tentacles-dependent pytest | +| UI functional | Browser / seeded grid | Default **N/A — Vitest + API tests**; **agent-seed** only when the user explicitly wants manual `/app` QA — **not** new Playwright `e2e/` files in git (`path.deny_node_web_playwright_e2e`) | + +**Milestone catalog** + +| Type | When | +|------|------| +| Env | `source .cursor/env.sh` at start of implementation | +| Unit tests | Every code/policy step — `pytest` or `npm test` | +| Install health | After `.cursor/` changes — `bash .cursor/cloud-validate.sh --profile ci-tentacles --json` | +| Tentacles sync | After `packages/tentacles/**` — `bash .cursor/reinstall-tentacles.sh` + related pytest if needed | +| PR gate | Handoff — `python -m tools.extended_linter --base origin/dev` | +| Functional / UI | Per step when relevant; **agent-seed** for `/app` | + +## Skill and doc routing + +| Situation | Use | +|-----------|-----| +| Implement / verify in cloud | **octobot-cloud** | +| Pytest cwd / PYTHONPATH | **octobot-cloud** → `reference-pytest.md` | +| Node UI unit tests | `packages/tentacles/Services/Interfaces/node_web_interface/` → `npm test` | +| End-user UI copy | skill **end-user-ui** + colocated UI `AGENTS.md` | +| Node UI browser QA | **agent-seed** | +| Package boundaries | colocated `AGENTS.md` | +| Contributor workflow | `CONTRIBUTING-agent.md` | +| Cloud install | `.cursor/README.md` | + +## Checklist before CreatePlan + +- [ ] Ambiguities resolved or marked **user must choose** +- [ ] **Responsibility map** complete +- [ ] Every code step: unit tests (files, scenarios, command) +- [ ] Every code step: functional tests proposed or **N/A — reason** +- [ ] No absolute or out-of-repo paths +- [ ] Steps name owning package/module +- [ ] Git/PR to **`dev`** (or stated base) if the roadmap includes git work diff --git a/.cursor/skills/end-user-ui/SKILL.md b/.cursor/skills/end-user-ui/SKILL.md new file mode 100644 index 0000000000..a65cf8c6cd --- /dev/null +++ b/.cursor/skills/end-user-ui/SKILL.md @@ -0,0 +1,65 @@ +--- +name: end-user-ui +description: >- + End-user UX for node_web_interface (React) and web_interface (Flask/templates): + entry-level trading terms OK, no pro-trader jargon, minimal UI, situational focus, + no AI slop, no em dash in UI strings. Apply on any UI copy, + layout, error, or help change. +paths: + - packages/tentacles/Services/Interfaces/node_web_interface/** + - packages/tentacles/Services/Interfaces/web_interface/** +--- + +# End-user UI (node + classic web) + +Apply whenever you change user-visible copy, layout, errors, or help in the Node React UI or the classic Flask web UI tentacles. + +## Sources only (tentacles) + +- Edit UI only under `packages/tentacles/Services/Interfaces/node_web_interface/` or `packages/tentacles/Services/Interfaces/web_interface/`. +- Do not open or patch repo-root `tentacles/` (install output). After tentacles source changes: `bash .cursor/reinstall-tentacles.sh`. + +Colocated guides: `packages/tentacles/Services/Interfaces/node_web_interface/AGENTS.md`, `.../web_interface/AGENTS.md`. Technical Node UI workflow (when present in the monorepo): workspace skill **node-web-interface-workflow**; cloud agents: skill **octobot-cloud** and `node_web_interface/README.md` for npm cwd. + +## Audience + +- Assume an **average non-technical** user who may be new to OctoBot but is using a **trading bot**. Not a developer, not a pro desk. +- One primary goal per screen or step: show status, fix one problem, or confirm one action. + +## Language: entry-level trading OK; pro/advanced not + +- **OK:** everyday trading vocabulary users expect: buy/sell, order, pair, exchange, balance, profit/loss, stop loss, take profit, market/limit, position (when the feature uses it). Keep terms **short and consistent** with labels elsewhere in the app. +- **Avoid:** advanced or pro-trader jargon: maker/taker fee tiers, funding rate mechanics, delta-neutral, basis, slippage models, OCO/iceberg unless the UI is explicitly an advanced screen; desk slang; dense acronyms without expansion on first use in that flow. +- **Dev/infra:** still translate. Avoid "credentials", "auth payload", "session invalidated" without plain-language equivalent ("password" / "passphrase" per product). + +## Understandable + +- Short sentences; active voice; say what happened and what to do next. +- If a term is required, prefer **entry-level trading** over **pro**; add one short gloss only when the screen cannot work without it (not a glossary block). +- Errors: **title + one explanation**; optional **at most 1–2** tips only when they change behavior. Not a wall of suggestions. + +## Simple (less is better) + +- Remove or defer secondary info, debug detail, and duplicate controls. +- Progressive disclosure: advanced fields behind "Advanced" or debug routes only (`/app/debug` stays operator-facing). +- Empty states: one line what this is plus one action. Not essays. + +## No AI slop + +- Ban list (examples): filler openers ("Certainly", "Great question"), **marketing** buzzwords ("seamless", "robust", "empower", "leverage" as hype), emoji decoration, generic tip lists, three-panel "guidance" when one line suffices, marketing tone in errors. +- **No em dash (`—`) in user-visible UI copy** (labels, titles, errors, toasts, template text). Use a period, comma, colon, or parentheses instead; match punctuation already used on the same screen. +- Match **existing** OctoBot/Node UI tone in nearby components; read siblings before inventing new patterns. +- Do not add UI "just to be helpful" if Vitest/API already cover behavior; user-visible text must earn its space. + +## By stack + +| Stack | Where copy lives | +|-------|------------------| +| `node_web_interface` | React components, route copy, toasts; keep logic in code/tests per skill **octobot-cloud** (Vitest, no e2e). | +| `web_interface` | Jinja templates, flash messages, modal titles; Python strings visible in templates. | + +## Before handoff (UI PR) + +- Re-read changed strings aloud as a non-expert. +- `npm test` in `packages/tentacles/Services/Interfaces/node_web_interface/` when touched; pytest for `web_interface/tests` when touched. +- No new Playwright e2e; agent-seed only if the user asked for live QA. diff --git a/.cursor/skills/node-journal/SKILL.md b/.cursor/skills/node-journal/SKILL.md new file mode 100644 index 0000000000..67473cd48f --- /dev/null +++ b/.cursor/skills/node-journal/SKILL.md @@ -0,0 +1,34 @@ +--- +name: node-journal +description: >- + Node journal (octobot.community.node_journal): record-only from outside the + package; reads only for export inside node_journal; not a source of truth. + Apply when editing node_journal or adding/changing record_* calls elsewhere. +paths: + - octobot/community/node_journal/** +--- + +# Node journal + +Full rules: [`octobot/community/node_journal/AGENTS.md`](../../../octobot/community/node_journal/AGENTS.md). + +## Purpose + +Append-only **event log** for diagnostics, journey analytics, and **export/sharing** (upload envelope). Not authoritative application state. + +## Integrators (outside `node_journal/`) + +- Import `octobot.community.node_journal` and call Tier-1 **`record_*`** / `record` only. +- Do **not** call `read_events`, read journal files on disk, or branch product logic on journal contents. +- Do **not** store data in journal payloads to reuse later for non-journal features. Use config, DB, sync collections, or domain stores as source of truth. + +## Inside `octobot/community/node_journal/` + +- **`read_events`**, `build_journey_summary`, and `build_upload_envelope` belong to the **export pipeline** (plus unit tests). +- Event and wire literals: extend [`events.py`](../../../octobot/community/node_journal/events.py), [`constants.py`](../../../octobot/community/node_journal/constants.py), [`enums.py`](../../../octobot/community/node_journal/enums.py); do not inline strings in recording code. +- Respect import tiers in package [`__init__.py`](../../../octobot/community/node_journal/__init__.py); feature code must not reach into `store` / `state` except via the public API. + +## Related + +- Core app boundaries: [`octobot/AGENTS.md`](../../../octobot/AGENTS.md) +- Python conventions (shared literals, imports): [CONTRIBUTING-agent.md](../../../CONTRIBUTING-agent.md) — **Python conventions (agents)** diff --git a/.cursor/skills/octobot-cloud/SKILL.md b/.cursor/skills/octobot-cloud/SKILL.md new file mode 100644 index 0000000000..3fd926fcc2 --- /dev/null +++ b/.cursor/skills/octobot-cloud/SKILL.md @@ -0,0 +1,84 @@ +--- +name: octobot-cloud +description: >- + OctoBot git repo on Cursor Cloud Agent: bash, cloud-install, tentacles reinstall, + pytest/pylint, debugging failing pytest (integration/functional, cross-package), + extended_linter, changes under octobot/, packages/, packages/tentacles/. +--- + +# OctoBot Cloud Agent + +Multi-step **roadmap** plans (title or description contains `roadmap`): author with skill **cloud-roadmap** (`.cursor/skills/cloud-roadmap/SKILL.md`); execute with this skill. + +## Workflow + +1. `source .cursor/env.sh` before Python tooling. +2. Checkout **`dev`** (or user base) → create feature branch → implement → commit → PR to **`dev`** (match `--base` on extended_linter to PR target). +3. After `packages/tentacles/` edits: `bash .cursor/reinstall-tentacles.sh`. +4. Before commit: remove any scratch plan files (`PLAN-*.md`, `*.plan.md`, `.cursor/plans/`) from the tree; do not stage them (`path.deny_agent_plans`). Do not change agent docs (see `octobot-cloud.mdc` path list) unless the task owns them — otherwise restore from `origin/` (`agent_docs.no_regression_vs_merge_base`). +5. After `git rebase` or fast-forward onto `origin/dev`, re-checkout unrelated agent-doc paths from `origin/dev` if the working tree still has stale copies. Squash commits must reflect the **current** branch tip, not an old cloud workspace snapshot. +6. Before handoff: `python -m tools.extended_linter --base origin/dev` (or `origin/`). +7. Run targeted pytest per [reference-pytest.md](reference-pytest.md) (CI matrix). **If any test fails,** start with [step 1 — visible logs](reference-pytest.md#1-re-run-with-visible-logs), then follow [When tests fail](reference-pytest.md#when-tests-fail) **before** changing production code or the harness. + +## Install and env + +- Profile default: **`ui-node-web`** (`environment.json` → `PANTS_PANTSD=false OCTOBOT_CLOUD_PROFILE=ui-node-web bash .cursor/cloud-install.sh`). GitHub `cloud-env-validate` still uses `ci-tentacles`. +- **Success:** Build log contains `OCTOBOT_CLOUD_INSTALL_OK`. Do not guess from UI alone. +- **Validate:** `bash .cursor/cloud-validate.sh --profile ci-tentacles --json` +- Profiles: `package-only` (no tentacles), `ci-tentacles` (CI parity), `ui-node-web` (+ web UI build). + +## Layout + +- Repo root: OctoBot application, `packages/*`, `tools/`, root `AGENTS.md` (package boundaries colocated as `AGENTS.md`). +- Installed tentacles output: repo-root `tentacles/` (generated — never edit). +- Tentacles sources: `packages/tentacles/`. + +## Coding conventions + +- **Python (imports, literals, public API):** [CONTRIBUTING-agent.md](../../CONTRIBUTING-agent.md) — **Python conventions (agents)**. +- **Node journal:** skill **node-journal** (`.cursor/skills/node-journal/SKILL.md`); colocated [`octobot/community/node_journal/AGENTS.md`](../../octobot/community/node_journal/AGENTS.md). Integrators record only; not a source of truth. + +## Policy and verify + +- Machine rules: `tools/extended_linter/config/policy.yaml` (extend via `tools/extended_linter/ARCHITECTURE.md`) +- CLI: `python -m tools.extended_linter --base origin/dev --continue --report json` +- Read reports: `rule_id`, `file`, `line`, `hint` blocks. + +## Architecture docs + +- Index: root `AGENTS.md` (when to read, CI matrix table) +- Per-package: `octobot/AGENTS.md`, `packages/tentacles/AGENTS.md`, `packages//AGENTS.md` +- Read colocated `AGENTS.md` before cross-package or tentacles changes. + +## Formatting (touched paths only in 1b) + +- Python: `ruff format` / `ruff check` when configured (CI enforce later). +- `node_web_interface`: Biome in `packages/tentacles/Services/Interfaces/node_web_interface/`. + +## Pytest + +See **[reference-pytest.md](reference-pytest.md)** for cwd and PYTHONPATH per CI matrix package. + +### When a test fails + +1. **Re-run** the failing node with log-cli per [step 1](reference-pytest.md#1-re-run-with-visible-logs) (single process; omit `-n auto`). +2. State the **invariant** (one sentence: what must be true when the test passes). +3. Find the **failure boundary** (last good layer vs first bad layer); do not widen mocks, seeds, or timeouts until that boundary is clear. +4. Apply the full protocol in [When tests fail](reference-pytest.md#when-tests-fail) before editing production code or the harness. + +## Node UI tests (agents) + +- **User-visible UI:** For copy, layout, errors, or help in `node_web_interface` or classic `web_interface`, read skill **end-user-ui** (`.cursor/skills/end-user-ui/SKILL.md`) and colocated `AGENTS.md` under those tentacle paths. +- **In PRs:** Vitest in `packages/tentacles/Services/Interfaces/node_web_interface/` (`npm test`) plus Python/API tests (`node_api_interface`, `packages/node`, tentacles pytest as needed). **Do not** add Playwright e2e under `e2e/` or `test:e2e` scripts (`path.deny_node_web_playwright_e2e`). +- CI runs Vitest for tentacles `package.json` projects; e2e is not in the matrix. + +## Node UI manual QA (agent seed) + +**Not required** for normal pytest / `ci-tentacles` package work — only when the task uses the live Node web UI. + +For **`ui-node-web`**: follow skill **agent-seed** (`.cursor/skills/agent-seed/SKILL.md`). Typical one-shot: `source .cursor/env.sh`, `source .cursor/agent-seed.env`, stop any running node, then `bash .cursor/seed-agent.sh --full`. After startup, log in with passphrase **`demodemo`**; use the full **`/app`** UI as needed, and **`/app/debug`** to send user actions. + +## Out of scope unless asked + +- Live bot ports, docker image publish, signed tentacles release pipeline. +- `pip install` / `npm install` to fix imports. diff --git a/.cursor/skills/octobot-cloud/reference-pytest.md b/.cursor/skills/octobot-cloud/reference-pytest.md new file mode 100644 index 0000000000..b029a657a5 --- /dev/null +++ b/.cursor/skills/octobot-cloud/reference-pytest.md @@ -0,0 +1,106 @@ +# Pytest (OctoBot agents) + +> **Test failing?** Jump to [When tests fail](#when-tests-fail). **Step 1:** re-run the failing test with [visible logs](#1-re-run-with-visible-logs)—then follow the rest of the protocol; do not patch symptoms first. + +## CI run matrix + +Run from **OctoBot repo root** after `source .cursor/env.sh`. `DISABLE_SENTRY=True` matches CI. + +| Package | cwd | PYTHONPATH / notes | Example | +|---------|-----|-------------------|---------| +| `octobot` | repo root | default venv | `pytest tests -n auto --dist loadfile` | +| `octobot` (tentacles tests) | repo root | installed tentacles | `pytest --ignore=tentacles/Trading/Exchange tentacles -n auto --dist loadfile` | +| `packages/node` | repo root | `PYTHONPATH=.:$PYTHONPATH` | `pytest packages/node/tests -n auto --dist loadfile` | +| `packages/flow` | repo root | `PYTHONPATH=.:$PYTHONPATH` | `pytest packages/flow/tests -n auto --dist loadfile` | +| `packages/copy` | repo root | `PYTHONPATH=.:$PYTHONPATH` | `pytest packages/copy/tests -n auto --dist loadfile` | +| `packages/tentacles_manager` | `packages/tentacles_manager` | — | `pytest tests` (no xdist in CI) | +| `packages/protocol` | `packages/protocol` | — | `pytest test` | +| `packages/agents` | `packages/agents` | — | `pytest tests -n auto --dist loadfile` | +| `packages/async_channel` | `packages/async_channel` | — | `pytest tests -n auto --dist loadfile` | +| `packages/backtesting` | `packages/backtesting` | — | `pytest tests -n auto --dist loadfile` | +| `packages/commons` | `packages/commons` | — | `pytest tests -n auto --dist loadfile` | +| `packages/evaluators` | `packages/evaluators` | — | `pytest tests -n auto --dist loadfile` | +| `packages/services` | `packages/services` | — | `pytest tests -n auto --dist loadfile` | +| `packages/sync` | `packages/sync` | — | `pytest tests -n auto --dist loadfile` | +| `packages/trading` | `packages/trading` | — | `pytest tests -n auto --dist loadfile` | + +**Tentacles install** required for: `octobot`, `packages/node`, `packages/flow`, `packages/copy` (CI `USES_TENTACLES`). + +**Pylint:** `pylint --rcfile=/standard.rc /` or root `standard.rc` (see CI `main.yml`). + +**tools tests (CI `extended_linter` job):** `PYTHONPATH=.:$PYTHONPATH pytest tools/tests -q` + +**extended_linter only (no tentacles):** `PYTHONPATH=. pytest tools/tests/extended_linter -q` + +## When tests fail + +Use this protocol **before** widening mocks, seeds, timeouts, or expectations—especially for integration, functional, scheduler, or cross-package tests (`packages/node`, `packages/flow`, `packages/sync`, `tentacles/`, and similar). + +### 1. Re-run with visible logs + +Re-run the **single** failing file, class, or test node with logging enabled **before** changing code or the harness. The first pytest traceback alone is often not enough for integration failures. + +- **Cloud:** `source .cursor/env.sh`; `source .cursor/pythonpath.sh` when the matrix requires `PYTHONPATH=.`; use `python -m pytest` if `pytest` is not on `PATH`. +- **Monorepo local:** full `PYTHONPATH` from `.vscode/settings.json`; `OctoBot/venv13/Scripts/python.exe -m pytest` on Windows. +- Append: `-s --log-cli-level INFO --log-cli-format "%(asctime)s.%(msecs)03d - %(levelname)-8s %(name)-24s %(message)s"` (PowerShell: quote the format string). +- **Omit** `-n auto` on debug reruns (single process; worker logs are hard to read). +- Quick green checks may still use `-q` from the matrix; this step is for **investigation**, not full CI sweeps. + +```bash +python -m pytest path/to/test_module.py::TestClass::test_name -s --log-cli-level INFO --log-cli-format '%(asctime)s.%(msecs)03d - %(levelname)-8s %(name)-24s %(message)s' +``` + +Read the log output while you work through steps 2–9. If logs are still empty, check whether the code path uses `logging` vs `print` before adding more flags. + +### 2. State the invariant + +Write one sentence: what must be true when the test passes (workflow state, sync write, HTTP status, portfolio field, order created, and so on). Everything else is diagnosis. + +### 3. Find the failure boundary + +Locate the **last layer where the invariant still holds** and the **first layer where it breaks** (zero, empty, wrong type, exception). Fix at the **smallest layer that explains the break**, not necessarily in the test file. If setup looks correct but computed output is zero or empty, suspect **transformation logic** in another package (`packages/trading`, `packages/protocol`, `packages/commons`, tentacles) before blaming seed data in the harness. + +### 4. Contradictory evidence + +| Observation | Default hypothesis | +|-------------|-------------------| +| Upstream state looks correct; downstream output is zero or empty | Bug in **transformation** between them | +| Two logs disagree about the same entity | Different **code paths** or **representations** | +| Error implies missing resource; resource appears elsewhere | Message may be **generic**; trace **computed inputs** to the failing check | + +Do not increase fixtures, balances, or retries until the failure boundary is identified. + +### 5. Parallel paths must agree + +When one code path correctly handles a domain concept (simulated vs live, user-action type, sync vs debug API, and similar): + +1. Identify the **canonical helper or branch** used there. +2. Search for other operations on the same concept that do not use it. +3. Prefer aligning call sites to the canonical path over duplicating logic in tests. + +Read colocated **`AGENTS.md`** before edits across package boundaries; the fix may live outside the test’s package. + +### 6. Test metadata is not root cause + +`xfail`, `skip`, TODO comments, and “known broken” notes describe **process**, not technical proof. When fixing behavior, do not stop at metadata that excuses failure. If you keep `xfail` or `skip`, name **observable failure** (layer + symptom), not a vague narrative. + +### 7. Before proposing edits + +For integration or functional failures, note briefly: + +- **Invariant** +- **Last good layer** (package + what was still true) +- **First bad layer** (package + what broke) +- **Planned fix location** (may differ from the test package) + +Only then change production code or the test harness. + +### 8. What not to do by default + +- Do not widen mocks or inflate seeds to mask zero or empty computed values. +- Do not adjust only timeouts or polling unless the failure boundary is timing, not wrong logic. +- Do not assume the bug lives in the same package as the test file. + +### 9. After the fix + +Run targeted pytest per the [CI run matrix](#ci-run-matrix) above. Prefer the smallest test that reproduces the failure boundary, then the original failing test. diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..c1dc6f80c2 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 4 + +[*.{json,yml,yaml,md}] +indent_size = 2 + +[*.ts] +indent_size = 2 diff --git a/.github/workflows/cloud-env-validate.yml b/.github/workflows/cloud-env-validate.yml new file mode 100644 index 0000000000..775c531e06 --- /dev/null +++ b/.github/workflows/cloud-env-validate.yml @@ -0,0 +1,54 @@ +name: cloud-env-validate + +on: + pull_request: + paths: + - ".cursor/**" + - ".github/workflows/cloud-env-validate.yml" + +permissions: read-all + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.13.x" + + - uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc + + - name: Cloud install (ci-tentacles) + run: OCTOBOT_CLOUD_PROFILE=ci-tentacles bash .cursor/cloud-install.sh + + - name: Cloud validate + id: validate + run: | + source .cursor/env.sh + bash .cursor/cloud-validate.sh --profile ci-tentacles --json | tee cloud-validate.json + + - name: Upload validate report on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cloud-validate.json + path: cloud-validate.json + + - name: Step summary on failure + if: failure() + run: | + echo "## cloud-env-validate failed" >> "$GITHUB_STEP_SUMMARY" + grep OCTOBOT_CLOUD_FAILED cloud-validate.json 2>/dev/null || cat cloud-validate.json >> "$GITHUB_STEP_SUMMARY" + + notify: + if: ${{ failure() }} + needs: + - validate + uses: Drakkar-Software/.github/.github/workflows/failure_notify_workflow.yml@master + secrets: + DISCORD_GITHUB_WEBHOOK: ${{ secrets.DISCORD_GITHUB_WEBHOOK }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 58f2c64487..e21b660ba6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -169,21 +169,21 @@ jobs: - name: Run tests run: | if [ "${{ matrix.package }}" = "octobot" ]; then - pytest tests -n auto --dist loadfile - pytest --ignore=tentacles/Trading/Exchange tentacles -n auto --dist loadfile + pytest tests -n auto --dist loadfile -r ps + pytest --ignore=tentacles/Trading/Exchange tentacles -n auto --dist loadfile -r ps else if [ "${{ matrix.package }}" = "packages/node" ] || [ "${{ matrix.package }}" = "packages/flow" ] || [ "${{ matrix.package }}" = "packages/copy" ]; then echo "Running tests from root dir to allow tentacles import" - PYTHONPATH=.:$PYTHONPATH pytest ${{ matrix.package }}/tests -n auto --dist loadfile + PYTHONPATH=.:$PYTHONPATH pytest ${{ matrix.package }}/tests -n auto --dist loadfile -r ps else cd ${{ matrix.package }} if [ "${{ matrix.package }}" = "packages/tentacles_manager" ]; then - pytest tests + pytest tests -r ps else if [ "${{ matrix.package }}" = "packages/protocol" ]; then - pytest test + pytest test -r ps else - pytest tests -n auto --dist loadfile + pytest tests -n auto --dist loadfile -r ps fi fi fi @@ -206,6 +206,79 @@ jobs: cd - > /dev/null done + extended_linter: + name: extended_linter + needs: [ build ] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13.x" + + - name: Download build wheel artifact + uses: actions/download-artifact@v8 + with: + name: octobot-wheel + path: dist/ + + - name: Install OctoBot + run: | + pip install -r dev_requirements.txt + pip install dist/*.whl + + - uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc + + - name: Build web client SDK + run: | + npm ci + npm run build -w packages/client/octobot_client_ts + + - name: Install tentacles + env: + ALLOW_UNSIGNED_TENTACLES: "true" + run: | + mkdir -p output + OctoBot tentacles -d packages/tentacles -p any_platform.zip + OctoBot tentacles --install --location output/any_platform.zip --all + + - name: tools tests + run: PYTHONPATH=.:$PYTHONPATH pytest tools/tests -q + + - name: Fetch base branch + if: github.event_name == 'pull_request' + run: git fetch origin ${{ github.base_ref }} + + - name: PR policy check + if: github.event_name == 'pull_request' + id: linter + run: | + PYTHONPATH=. python -m tools.extended_linter \ + --base origin/${{ github.base_ref }} \ + --continue \ + --skip-tentacles-reinstall \ + --write-report-json extended-linter.json | tee extended-linter.txt + + - name: Upload extended_linter report on failure + if: failure() && github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: extended-linter-report + path: | + extended-linter.txt + extended-linter.json + + - name: Step summary on failure + if: failure() && github.event_name == 'pull_request' + run: | + echo "## extended_linter failed" >> "$GITHUB_STEP_SUMMARY" + cat extended-linter.txt >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + docker: name: Build & Push Docker images needs: [ build, tests ] @@ -808,6 +881,7 @@ jobs: needs: - build - tests + - extended_linter - docker - binary - tentacles diff --git a/.gitignore b/.gitignore index 3889479639..d1920dbe92 100644 --- a/.gitignore +++ b/.gitignore @@ -113,6 +113,9 @@ celerybeat-schedule # virtualenv .venv venv/ +.cursor/venv/ +.cursor/env.sh +.cursor/pythonpath.sh ENV/ # Spyder project settings @@ -194,3 +197,8 @@ installer/ # ai analysis/ + +# Agent session plans (never commit; see path.deny_agent_plans in extended_linter) +**/PLAN-*.md +**/*.plan.md +.cursor/plans/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..c95b1c7528 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,81 @@ +# Agent quick pointer (OctoBot repo) + +This repository is the **OctoBot** application and its `packages/*` monorepo. Default integration branch: **`dev`** (open PRs to `dev` unless told otherwise). + +## Cursor Cloud + +- Environment: [`.cursor/README.md`](.cursor/README.md), `environment.json`, `source .cursor/env.sh` +- Skill (procedures): `.cursor/skills/octobot-cloud/SKILL.md` +- Roadmap plans: before `CreatePlan`, if the plan title or description contains `roadmap`, follow `.cursor/skills/cloud-roadmap/SKILL.md` +- Always-on rules: `.cursor/rules/octobot-cloud.mdc` + +## Before you finish + +```bash +source .cursor/env.sh +python -m tools.extended_linter --base origin/dev +``` + +Use `origin/` matching your PR target. Policy: `tools/extended_linter/config/policy.yaml` — see `tools/extended_linter/ARCHITECTURE.md`. No agent plan files or Node UI Playwright e2e in the diff (`path.deny_agent_plans`, `path.deny_node_web_playwright_e2e`). No accidental agent-doc regressions (`agent_docs.no_regression_vs_merge_base`; see `octobot-cloud.mdc`). + +## Architecture for agents + +Colocated **`AGENTS.md`** files describe package boundaries (owns, deps, tests). They are for automation; human docs live under `docs/content/` (Docusaurus). + +### When to read + +| Situation | Read | +|-----------|------| +| Single package task | `packages//AGENTS.md` | +| Core app / CLI / config | [`octobot/AGENTS.md`](octobot/AGENTS.md) | +| Tentacles sources vs install | [`packages/tentacles/AGENTS.md`](packages/tentacles/AGENTS.md) | +| Node UI or classic web UI (copy, layout, errors) | [`packages/tentacles/Services/Interfaces/node_web_interface/AGENTS.md`](packages/tentacles/Services/Interfaces/node_web_interface/AGENTS.md), [`packages/tentacles/Services/Interfaces/web_interface/AGENTS.md`](packages/tentacles/Services/Interfaces/web_interface/AGENTS.md), skill **end-user-ui** (`.cursor/skills/end-user-ui/SKILL.md`) | +| Node journal (record / export) | [`octobot/community/node_journal/AGENTS.md`](octobot/community/node_journal/AGENTS.md), skill **node-journal** (`.cursor/skills/node-journal/SKILL.md`) | +| Python style (imports, literals, `__init__.py` API) | [CONTRIBUTING-agent.md](CONTRIBUTING-agent.md) — **Python conventions (agents)** | +| Cross-package or tentacles | This file + every involved colocated `AGENTS.md` | +| Node UI demo / agent-seed QA | [`tools/agent_seed/README.md`](tools/agent_seed/README.md) + skill **agent-seed** (`.cursor/skills/agent-seed/SKILL.md`) | + +### Dependency sketch + +```mermaid +flowchart TB + octobot[octobot app] + commons[commons] + trading[trading] + evaluators[evaluators] + services[services] + tentacles_pkg[packages/tentacles sources] + octobot --> commons + octobot --> trading + octobot --> evaluators + octobot --> services + trading --> commons + evaluators --> commons + tentacles_pkg --> services +``` + +### CI matrix packages (colocated `AGENTS.md`) + +| Package | Path | +|---------|------| +| Core application | [`octobot/AGENTS.md`](octobot/AGENTS.md) | +| Tentacles sources | [`packages/tentacles/AGENTS.md`](packages/tentacles/AGENTS.md) | +| agents | [`packages/agents/AGENTS.md`](packages/agents/AGENTS.md) | +| async_channel | [`packages/async_channel/AGENTS.md`](packages/async_channel/AGENTS.md) | +| backtesting | [`packages/backtesting/AGENTS.md`](packages/backtesting/AGENTS.md) | +| commons | [`packages/commons/AGENTS.md`](packages/commons/AGENTS.md) | +| copy | [`packages/copy/AGENTS.md`](packages/copy/AGENTS.md) | +| evaluators | [`packages/evaluators/AGENTS.md`](packages/evaluators/AGENTS.md) | +| flow | [`packages/flow/AGENTS.md`](packages/flow/AGENTS.md) | +| node | [`packages/node/AGENTS.md`](packages/node/AGENTS.md) | +| protocol | [`packages/protocol/AGENTS.md`](packages/protocol/AGENTS.md) | +| services | [`packages/services/AGENTS.md`](packages/services/AGENTS.md) | +| sync | [`packages/sync/AGENTS.md`](packages/sync/AGENTS.md) | +| tentacles_manager | [`packages/tentacles_manager/AGENTS.md`](packages/tentacles_manager/AGENTS.md) | +| trading | [`packages/trading/AGENTS.md`](packages/trading/AGENTS.md) | + +New packages: follow **AGENTS.md sections** in [CONTRIBUTING-agent.md](CONTRIBUTING-agent.md). + +## Humans + +- [CONTRIBUTING-agent.md](CONTRIBUTING-agent.md) — agents, policy, boundaries diff --git a/BUILD b/BUILD index 52aa2b3242..9b79ce5ece 100644 --- a/BUILD +++ b/BUILD @@ -161,7 +161,7 @@ python_distribution( ] + PACKAGE_SOURCES + PACKAGE_REQS + PACKAGE_FULL_REQS, provides=python_artifact( name="octobot", - version="3.0.0-beta2", + version="3.0.0-beta3", url='https://github.com/Drakkar-Software/OctoBot', license='GPL-3.0', author="Drakkar-Software", diff --git a/CHANGELOG.md b/CHANGELOG.md index 66217c7ea2..820a221af7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 *It is strongly advised to perform an update of your tentacles after updating OctoBot. (start.py tentacles --install --all)* +## [3.0.0-beta3] - 2026-09-28 +### Added +- [Node Interface] Much more help and guidance to connect to the web & app interfaces +- [Exchanges] KucoinEu, Bybit EU, and Coinrabbit support +- [Portfolio] Portfolio history from trades and transactions +- [Node Interface] Mandatory seed phrase backup step and quiz when creating a wallet; rate-limited seed recovery at login +- [Node Interface] Send feedback via downloadable zip +- [Node] Disable automation schedules; signal-based automations; cancel open orders when stopping an automation +- [Symbols] Support `@network` trading pairs +- [Docs] Tailscale guide for exposing the webhook port +- [Client] Pair the OctoBot client with your node (device code) and sync a read-only mirror per wallet +- [UI] Scan QR codes and copy/paste to confirm action proposals +### Updated +- [Node] View outputs for failed or cancelled automations +- [Docker] Expose child OctoBots ports in the default Docker image +### Fixed +- [DBOS] Reduce tasks.db file size and CPU load +- [Inteface] Fix Exchange & trading pairs selector +- [Trading] Errors when the portfolio cannot be loaded from the exchange +- [Copy trading] Stale copy-trading signals +- [Coinbase] Invalid API key (PEM) handling +- [Node Interface] Wallet import (template, CSV, manual edit); process bot links; seed backup quiz and recovery flow; clearer unlock and login errors +- [Node] Automations stopping unexpectedly; missing or wrong tasks in the task list; wrong passphrase errors with multiple wallets +- [Tentacles] Child bot process failing to start (decoding error) +- [ProcessBot] Incorrect state when using several exchanges +- [Market making] Wrong prices from invalid references; clearer error messages + ## [3.0.0-beta2] - 2026-08-06 ### Added - [Weex] Support Weex exchange diff --git a/CONTRIBUTING-agent.md b/CONTRIBUTING-agent.md new file mode 100644 index 0000000000..3c9476077b --- /dev/null +++ b/CONTRIBUTING-agent.md @@ -0,0 +1,107 @@ +# Contributing (agents and automation) + +## Cloud environment + +See [`.cursor/README.md`](.cursor/README.md). Install profile **`ci-tentacles`** for CI parity. Confirm `OCTOBOT_CLOUD_INSTALL_OK` in Build logs. + +## Workflow + +1. Branch from `dev` (or user-specified base). +2. For cross-package work, read root [`AGENTS.md`](AGENTS.md) and each colocated `AGENTS.md` for packages you touch. +3. Run `python -m tools.extended_linter --base origin/` before handoff. +4. Do not commit agent session plans (`path.deny_agent_plans`: `PLAN-*.md`, `*.plan.md`, `.cursor/plans/`). Node UI: Vitest + Python/API tests only — no Playwright e2e under `node_web_interface/e2e/` (`path.deny_node_web_playwright_e2e`). +5. **Agent docs:** edit `AGENTS.md`, `.cursor/skills/**`, `.cursor/rules/**`, `.cursor/README.md`, `CONTRIBUTING-agent.md`, and `tools/**/README.md` / `tools/**/ARCHITECTURE.md` only when the PR owns them. `extended_linter` blocks shrink/backdated **Last reviewed** (where present) vs merge-base (`agent_docs.no_regression_vs_merge_base`). Large trims need explicit reviewer intent. +6. CI: **OctoBot-CI** job **`extended_linter`** (`pytest tools/tests` with wheel + tentacles; PR policy with `--skip-tentacles-reinstall`, no `cloud-install`). Package **`tests`** matrix unchanged. + +## Adding a policy rule + +See [tools/extended_linter/README.md#adding-a-rule](tools/extended_linter/README.md) and [ARCHITECTURE.md](tools/extended_linter/ARCHITECTURE.md). + +1. Edit `tools/extended_linter/config/policy.yaml` with a new stable `rule_id`. +2. Extend `layers/` if a new `kind` is needed. +3. Add tests under `tools/tests/extended_linter/layers/`. +4. Summarize in `octobot-cloud` skill only for human context; YAML is the contract. + +## AGENTS.md hygiene + +When you change package boundaries (owns, public API, test cwd), update the matching colocated `AGENTS.md` (`octobot/`, `packages//`, or `packages/tentacles/`) and bump **Last reviewed**. `extended_linter` enforces no shrink/backdate on agent-doc paths vs merge-base (`agent_docs.no_regression_vs_merge_base`); it does not require updates when boundaries change. + +### AGENTS.md sections (new packages) + +Use this outline in `packages//AGENTS.md` (or `octobot/AGENTS.md` for core app): + +```markdown +# Agents: `` + +## Role + +One paragraph: what this package does in OctoBot. + +## Owns + +- `packages//...` paths agents may edit for tasks here. + +## Public surface + +- Import paths / modules other packages should use. + +## May depend on + +- Upstream packages (import direction). + +## Do not + +- Forbidden imports, layers to bypass, paths owned elsewhere. + +## Tests + +- **cwd:** `packages/` or repo root +- **PYTHONPATH:** if any +- **Example:** `pytest ...` + +## Related human doc + +- Link to `docs/content/developers/packages/...` when it exists. + +## Last reviewed + +- YYYY-MM-DD +``` + +## Node journal + +The node journal (`octobot.community.node_journal`) is an append-only event log for diagnostics, journey analytics, and **export/sharing**. It is **not** a source of truth. + +- **Outside `octobot/community/node_journal/`:** call Tier-1 **`record_*`** / `record` only. Do not call `read_events`, read journal files on disk, or branch product logic on journal contents. +- **Inside the package:** `read_events`, `build_journey_summary`, and `build_upload_envelope` are for the **export pipeline** (plus unit tests). +- Do not store data in journal payloads to reuse later for non-journal features; use config, DB, sync collections, or domain stores instead. +- Details: [`octobot/community/node_journal/AGENTS.md`](octobot/community/node_journal/AGENTS.md) and skill **node-journal** (`.cursor/skills/node-journal/SKILL.md`). + +## Python conventions (agents) + +Authoritative rules for Python in `octobot/` and `packages/`. Cloud agents: also summarized in `.cursor/rules/octobot-cloud.mdc`. + +### Shared literals (magic strings) + +- Put cross-module identifiers in the **owning package** at **package top level** when only a few literals are needed: e.g. [`octobot/constants.py`](octobot/constants.py), [`octobot/enums.py`](octobot/enums.py), or `packages//octobot_/constants.py` and `enums.py`. +- TypeScript: `constants.ts` / `wireConstants.ts` where applicable; see [`docs/content/client-sdk/wire-contract.md`](docs/content/client-sdk/wire-contract.md) for cross-language wire literals. +- Callers use `import module as alias` and `alias.NAME`; do not duplicate the same string in multiple files. +- **OK inline:** log/debug text; truly local one-off values never compared or reused elsewhere. +- **Submodule `constants.py` / `enums.py`:** only when a sub-area owns a **large** dedicated literal surface (exemplar: [`octobot/community/node_journal/constants.py`](octobot/community/node_journal/constants.py), [`enums.py`](octobot/community/node_journal/enums.py)). Do not add deep per-folder constant files for a handful of strings. + +### Imports + +- Prefer imports at **module top level**. +- Use `import xxx` or `import xxx as yy`; access via `xxx.name` or `yy.name`. +- Avoid `from xxx import yy` in normal module code; avoid lazy (function-scoped) imports unless breaking a documented import cycle or loading an optional heavy dependency on a rare path. +- **`from xxx import yyy` is allowed only in `__init__.py`** when re-exporting the public surface (see below). + +### Package `__init__.py` as public bridge + +- Re-export the **public surface** from each package/subpackage `__init__.py` so callers use one stable import (e.g. `import octobot_trading.personal_data as personal_data`) instead of deep internal paths. +- Example: [`packages/trading/octobot_trading/personal_data/__init__.py`](packages/trading/octobot_trading/personal_data/__init__.py). +- When adding a new public symbol, export it from the appropriate `__init__.py`. + +## Formatting + +Ruff/Biome configs are present; mass format and CI enforce land in later phases. Do not run repo-wide reformat in routine agent PRs unless requested. diff --git a/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py b/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py index f70507f84a..0f09ebcc5f 100644 --- a/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py +++ b/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py @@ -142,6 +142,7 @@ class AbstractAuthenticatedExchangeTester: DEFAULT_MAX_DEFAULT_ORDERS_COUNT = trading_constants.DEFAULT_MAX_DEFAULT_ORDERS_COUNT DEFAULT_MAX_STOP_ORDERS_COUNT = trading_constants.DEFAULT_MAX_STOP_ORDERS_COUNT SLEEP_SECONDS_BEFORE_CHECKING_PORTFOLIO = 0 # used to wait before fetching portfolio after creating/cancelling an order + ENABLE_MARKET_ORDER_CONVERSION_CHECKS = False # set True to run after_market_order_created during market order tests # Implement all "test_[name]" methods, call super() to run the test, pass to ignore it. # Override the "inner_test_[name]" method to override a test content. @@ -361,6 +362,116 @@ def proxy_callback(url: str, method: str, headers: dict, body) -> typing.Optiona async with self.local_exchange_manager(http_proxy_callback_factory=_http_proxy_callback_factory): await self.inner_is_authenticated_request(rest_exchange_data) + def _is_authenticated_request_assert_empty_inputs_false(self): + assert self.exchange_manager.exchange.is_authenticated_request("", "", {}, None) is False + assert self.exchange_manager.exchange.is_authenticated_request("", "", {}, "") is False + assert self.exchange_manager.exchange.is_authenticated_request("", "", {}, b"") is False + assert self.exchange_manager.exchange.is_authenticated_request(None, None, None, b"") is False + + def _is_authenticated_request_assert_calls_public(self, latest_calls): + for latest_call in latest_calls: + if self.exchange_manager.exchange.get_option_value( + trading_enums.ExchangeClientOptions.ALWAYS_REQUIRES_AUTHENTICATION + ): + assert latest_call[1] is True, f"{latest_call} should be authenticated" # authenticated request + else: + assert latest_call[1] is False, f"{latest_call} should be NOT authenticated" # authenticated request + + def _is_authenticated_request_assert_url_changed(self, rest_exchange_data, latest_calls): + assert rest_exchange_data["calls"][-1][0][0] != latest_calls[-1][0][0] # assert latest call's url changed + + async def _assert_is_authenticated_request_full_exchange( + self, rest_exchange_data, get_latest_calls, assert_has_at_least_one_authenticated_call + ): + assert await self.exchange_manager.exchange.get_symbol_prices( + self.SYMBOL, commons_enums.TimeFrames.ONE_HOUR + ) + latest_calls = get_latest_calls() + self._is_authenticated_request_assert_calls_public(latest_calls) + ticker = await self.exchange_manager.exchange.get_price_ticker(self.SYMBOL) + assert ticker + last_price = ticker[trading_enums.ExchangeConstantsTickersColumns.CLOSE.value] + self._is_authenticated_request_assert_url_changed(rest_exchange_data, latest_calls) + latest_calls = get_latest_calls() + self._is_authenticated_request_assert_calls_public(latest_calls) + + portfolio = await self.get_portfolio() + if self.CHECK_EMPTY_ACCOUNT: + assert portfolio == {} + else: + assert portfolio + self._is_authenticated_request_assert_url_changed(rest_exchange_data, latest_calls) + latest_calls = get_latest_calls() + assert_has_at_least_one_authenticated_call(latest_calls) + price = decimal.Decimal(str(last_price)) * decimal.Decimal("0.7") + amount = self.get_order_size( + portfolio, price, symbol=self.SYMBOL, settlement_currency=self.SETTLEMENT_CURRENCY + ) * 100000 + if amount == 0: + amount = 100000 + if self.CHECK_EMPTY_ACCOUNT: + amount = 10 + with pytest.raises(ccxt.ExchangeError): + await self.exchange_manager.exchange.connector.create_limit_buy_order( + self.SYMBOL, amount, price=price, params={} + ) + self._is_authenticated_request_assert_url_changed(rest_exchange_data, latest_calls) + latest_calls = get_latest_calls() + assert_has_at_least_one_authenticated_call(latest_calls) + with pytest.raises(ccxt.BaseError): + await self.exchange_manager.exchange.connector.client.cancel_order(self.VALID_ORDER_ID, self.SYMBOL) + self._is_authenticated_request_assert_url_changed(rest_exchange_data, latest_calls) + latest_calls = get_latest_calls() + assert_has_at_least_one_authenticated_call(latest_calls) + + async def _assert_is_authenticated_request_market_only_swap( + self, rest_exchange_data, get_latest_calls, assert_has_at_least_one_authenticated_call + ): + markets = await self.exchange_manager.exchange.connector.client.fetch_markets() + assert markets + latest_calls = get_latest_calls() + self._is_authenticated_request_assert_calls_public(latest_calls) + ticker = await self.exchange_manager.exchange.get_price_ticker(self.SYMBOL) + assert ticker + last_price = ticker[trading_enums.ExchangeConstantsTickersColumns.CLOSE.value] + self._is_authenticated_request_assert_url_changed(rest_exchange_data, latest_calls) + latest_calls = get_latest_calls() + self._is_authenticated_request_assert_calls_public(latest_calls) + + portfolio = await self.get_portfolio() + if self.CHECK_EMPTY_ACCOUNT: + assert portfolio == {} + else: + assert portfolio + self._is_authenticated_request_assert_url_changed(rest_exchange_data, latest_calls) + latest_calls = get_latest_calls() + assert_has_at_least_one_authenticated_call(latest_calls) + price = decimal.Decimal(str(last_price)) * decimal.Decimal("0.7") + amount = self.get_order_size( + portfolio, price, symbol=self.SYMBOL, settlement_currency=self.SETTLEMENT_CURRENCY + ) * 100000 + if amount == 0: + amount = 100000 + if self.CHECK_EMPTY_ACCOUNT: + amount = 10 + with pytest.raises((ccxt.ExchangeError, ccxt.NotSupported, trading_errors.NotSupported)): + await self.exchange_manager.exchange.connector.create_limit_buy_order( + self.SYMBOL, amount, price=price, params={} + ) + self._is_authenticated_request_assert_url_changed(rest_exchange_data, latest_calls) + latest_calls = get_latest_calls() + assert_has_at_least_one_authenticated_call(latest_calls) + client = self.exchange_manager.exchange.connector.client + if client.has.get("cancelOrder"): + with pytest.raises(ccxt.BaseError): + await client.cancel_order(self.VALID_ORDER_ID, self.SYMBOL) + else: + open_orders = await client.fetch_open_orders(self.SYMBOL) + assert isinstance(open_orders, list) + self._is_authenticated_request_assert_url_changed(rest_exchange_data, latest_calls) + latest_calls = get_latest_calls() + assert_has_at_least_one_authenticated_call(latest_calls) + async def inner_is_authenticated_request(self, rest_exchange_data): if self.IS_AUTHENTICATED_REQUEST_CHECK_AVAILABLE: latest_call_indexes = [len(rest_exchange_data["calls"])] @@ -372,76 +483,21 @@ def get_latest_calls(): def assert_has_at_least_one_authenticated_call(calls): has_authenticated_call = False for latest_call in calls: - # should be at least 1 authenticated call if latest_call[1] is True: has_authenticated_call = True assert has_authenticated_call, f"{calls} should contain at last 1 authenticated call" # authenticated request - # 1. test using different values - assert self.exchange_manager.exchange.is_authenticated_request("", "", {}, None) is False - assert self.exchange_manager.exchange.is_authenticated_request("", "", {}, "") is False - assert self.exchange_manager.exchange.is_authenticated_request("", "", {}, b"") is False - assert self.exchange_manager.exchange.is_authenticated_request(None, None, None, b"") is False - - # 2. make public requests - assert await self.exchange_manager.exchange.get_symbol_prices( - self.SYMBOL, commons_enums.TimeFrames.ONE_HOUR - ) - latest_calls = get_latest_calls() - for latest_call in latest_calls: - if self.exchange_manager.exchange.get_option_value( - trading_enums.ExchangeClientOptions.ALWAYS_REQUIRES_AUTHENTICATION - ): - assert latest_call[1] is True, f"{latest_call} should be authenticated" # authenticated request - else: - assert latest_call[1] is False, f"{latest_call} should be NOT authenticated" # authenticated request - ticker = await self.exchange_manager.exchange.get_price_ticker(self.SYMBOL) - assert ticker - last_price = ticker[trading_enums.ExchangeConstantsTickersColumns.CLOSE.value] - assert rest_exchange_data["calls"][-1][0][0] != latest_calls[-1][0][0] # assert latest call's url changed - latest_calls = get_latest_calls() - for latest_call in latest_calls: - if self.exchange_manager.exchange.get_option_value( - trading_enums.ExchangeClientOptions.ALWAYS_REQUIRES_AUTHENTICATION - ): - assert latest_call[1] is True, f"{latest_call} should be authenticated" # authenticated request - else: - assert latest_call[1] is False, f"{latest_call} should be NOT authenticated" # authenticated request - - # 3. make private requests - # balance (usually a GET) - portfolio = await self.get_portfolio() - if self.CHECK_EMPTY_ACCOUNT: - assert portfolio == {} + self._is_authenticated_request_assert_empty_inputs_false() + client = self.exchange_manager.exchange.connector.client + trading_type = trading_enums.ExchangeTypes(self.EXCHANGE_TYPE) + if ccxt_client_util.supports_order_type(client, trading_type, trading_enums.TradeOrderType.LIMIT): + await self._assert_is_authenticated_request_full_exchange( + rest_exchange_data, get_latest_calls, assert_has_at_least_one_authenticated_call + ) else: - assert portfolio - assert rest_exchange_data["calls"][-1][0][0] != latest_calls[-1][0][0] # assert latest call's url changed - latest_calls = get_latest_calls() - assert_has_at_least_one_authenticated_call(latest_calls) - # create order (usually a POST) - price = decimal.Decimal(str(last_price)) * decimal.Decimal("0.7") - amount = self.get_order_size( - portfolio, price, symbol=self.SYMBOL, settlement_currency=self.SETTLEMENT_CURRENCY - ) * 100000 - if amount == 0: - amount = 100000 - if self.CHECK_EMPTY_ACCOUNT: - amount = 10 - # (amount is too large, creating buy order will fail) - with pytest.raises(ccxt.ExchangeError): - await self.exchange_manager.exchange.connector.create_limit_buy_order( - self.SYMBOL, amount, price=price, params={} + await self._assert_is_authenticated_request_market_only_swap( + rest_exchange_data, get_latest_calls, assert_has_at_least_one_authenticated_call ) - assert rest_exchange_data["calls"][-1][0][0] != latest_calls[-1][0][0] # assert latest call's url changed - latest_calls = get_latest_calls() - assert_has_at_least_one_authenticated_call(latest_calls) - # cancel order (usually a DELETE) - with pytest.raises(ccxt.BaseError): - # use client call directly to avoid any octobot error conversion - await self.exchange_manager.exchange.connector.client.cancel_order(self.VALID_ORDER_ID, self.SYMBOL) - assert rest_exchange_data["calls"][-1][0][0] != latest_calls[-1][0][0] # assert latest call's url changed - latest_calls = get_latest_calls() - assert_has_at_least_one_authenticated_call(latest_calls) else: with pytest.raises(trading_errors.NotSupported): self.exchange_manager.exchange.is_authenticated_request("", "", {}, None) @@ -714,6 +770,8 @@ async def inner_test_create_and_fill_market_orders(self): post_buy_portfolio = {} try: self.check_created_market_order(first_market_order, size, side) + if self.ENABLE_MARKET_ORDER_CONVERSION_CHECKS: + await self.after_market_order_created(first_market_order) filled_order = await self.wait_for_fill(first_market_order) parsed_filled_order = personal_data.create_order_instance_from_raw( self.exchange_manager.trader, @@ -735,6 +793,8 @@ async def inner_test_create_and_fill_market_orders(self): other_side = trading_enums.TradeOrderSide.SELL if side == trading_enums.TradeOrderSide.BUY else trading_enums.TradeOrderSide.BUY second_market_order = await self.create_market_order(current_price, mirror_size, other_side) self.check_created_market_order(second_market_order, mirror_size, other_side) + if self.ENABLE_MARKET_ORDER_CONVERSION_CHECKS: + await self.after_market_order_created(second_market_order) await self.wait_for_fill(second_market_order) await self.sleep_before_checking_portfolio() post_sell_portfolio = await self.get_portfolio() @@ -1702,6 +1762,9 @@ async def _get_recent_trades_until(self, validation_func, timeout): f"message: {message}" ) + async def after_market_order_created(self, market_order) -> None: + pass + async def wait_for_fill(self, order): def parse_is_filled(raw_order): return personal_data.parse_order_status(raw_order) in {trading_enums.OrderStatus.FILLED, diff --git a/additional_tests/exchanges_tests/test_coinrabbit.py b/additional_tests/exchanges_tests/test_coinrabbit.py index eb1ac22192..b8dae691ff 100644 --- a/additional_tests/exchanges_tests/test_coinrabbit.py +++ b/additional_tests/exchanges_tests/test_coinrabbit.py @@ -56,7 +56,40 @@ class TestCoinRabbitAuthenticatedExchange( TOP_UP_CODE = "usdt" TOP_UP_NETWORK = "eth" TOP_UP_AMOUNT = "0" - + ENABLE_MARKET_ORDER_CONVERSION_CHECKS = True + MARKET_FILL_TIMEOUT = 600 # 10 min provisional for first live run; tune after observing fill duration + IS_AUTHENTICATED_REQUEST_CHECK_AVAILABLE = True + + def check_raw_closed_orders(self, closed_orders): + print(f"{len(closed_orders)} closed orders: {closed_orders}") + for raw_order in closed_orders: + order_info = raw_order.get("info", {}) + assert raw_order.get("status") == "closed" + assert order_info.get("status", "").lower() == "closed" + if order_info.get("updated_at"): + last_update_timestamp = raw_order.get("lastUpdateTimestamp") + created_at_timestamp = raw_order.get("timestamp") + assert last_update_timestamp, "closed order with updated_at must expose lastUpdateTimestamp" + assert created_at_timestamp and last_update_timestamp >= created_at_timestamp + super().check_raw_closed_orders(closed_orders) + + async def after_market_order_created(self, market_order) -> None: + client = self.exchange_manager.exchange.connector.client + open_orders = await client.fetch_open_orders(self.SYMBOL) + assert isinstance(open_orders, list) + for open_order in open_orders: + assert open_order["status"] == "open" + active_orders = await client.fetch_orders(self.SYMBOL, params={"status": "active"}) + assert isinstance(active_orders, list) + for active_order in active_orders: + assert active_order["status"] == "open" + assert active_order["info"]["status"].lower() == "active" + created_order_id = str(market_order.exchange_order_id) + open_order_ids = {str(open_order.get("id")) for open_order in open_orders} + assert created_order_id in open_order_ids, ( + f"market order {created_order_id} must appear in fetch_open_orders while converting" + ) + print(f"{created_order_id} in {len(open_order_ids)} open_order_ids: {open_order_ids}") async def test_get_portfolio(self): async with self.local_exchange_manager(): @@ -115,8 +148,7 @@ async def test_cancel_uncancellable_order(self): async def test_create_and_cancel_limit_orders(self): await super().test_create_and_cancel_limit_orders() - # TODO: wait for portfolio settlement (not only order fill) before check_portfolio_changed; fetchOrder closed - # status from the ccxt fee heuristic does not mean balances have settled (used→free can lag). + # Portfolio settlement (used→free) can lag behind order closed status; sleep_before_checking_portfolio handles this. async def test_create_and_fill_market_orders(self): await super().test_create_and_fill_market_orders() diff --git a/additional_tests/historical_backend_tests/__init__.py b/additional_tests/historical_backend_tests/__init__.py index e0a94a67c6..59eb9141ce 100644 --- a/additional_tests/historical_backend_tests/__init__.py +++ b/additional_tests/historical_backend_tests/__init__.py @@ -37,9 +37,3 @@ def _load_historical_backend_creds_env_variables_if_necessary(): async def clickhouse_client(): async with octobot.community.history_backend_client(octobot.enums.CommunityHistoricalBackendType.Clickhouse) as client: yield client - - -@pytest_asyncio.fixture -async def iceberg_client(): - async with octobot.community.history_backend_client(octobot.enums.CommunityHistoricalBackendType.Iceberg) as client: - yield client diff --git a/additional_tests/historical_backend_tests/test_iceberg_candles.py b/additional_tests/historical_backend_tests/test_iceberg_candles.py deleted file mode 100644 index bc1d4e2a5e..0000000000 --- a/additional_tests/historical_backend_tests/test_iceberg_candles.py +++ /dev/null @@ -1,252 +0,0 @@ -# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) -# Copyright (c) 2025 Drakkar-Software, All rights reserved. -# -# OctoBot is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# OctoBot is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public -# License along with OctoBot. If not, see . -import math -import time -import mock -import pytest -import asyncio -import datetime -import enum - -from additional_tests.historical_backend_tests import iceberg_client - -import octobot.constants as constants -import octobot.community.history_backend.iceberg_historical_backend_client as iceberg_historical_backend_client -import octobot.community.history_backend.util as history_backend_util - -import octobot_commons.enums as commons_enums -import octobot_commons.constants as commons_constants - - -# All test coroutines will be treated as marked. -pytestmark = pytest.mark.asyncio - - -EXCHANGE = "binance" -SYMBOL = "BTC/USDC" -SHORT_TIME_FRAME = commons_enums.TimeFrames.FIFTEEN_MINUTES - - -async def test_fetch_candles_history_range(iceberg_client): - # unknown candles - min_time, max_time = await iceberg_client.fetch_candles_history_range( - EXCHANGE+"plop", SYMBOL, SHORT_TIME_FRAME - ) - assert min_time == max_time == 0 - - # known candles - min_time, max_time = await iceberg_client.fetch_candles_history_range( - EXCHANGE, SYMBOL, SHORT_TIME_FRAME - ) - assert 0 < min_time < max_time < time.time() - - -async def test_fetch_candles_history(iceberg_client): - start_time = 1718785679 - end_time = 1721377495 - candles_count = math.floor((end_time - start_time) / ( - commons_enums.TimeFramesMinutes[SHORT_TIME_FRAME] * 60 - )) - # requires multiple fetches - assert candles_count == 2879 - candles = await iceberg_client.fetch_candles_history( - EXCHANGE, SYMBOL, SHORT_TIME_FRAME, start_time, end_time - ) - assert sorted(candles, key=lambda c: c[0]) == candles - fetched_count = candles_count + 1 - assert len(candles) == fetched_count - # will fail if parsed time is not UTC - assert candles[0][commons_enums.PriceIndexes.IND_PRICE_TIME.value] == 1718785800 - assert ( - candles[0][commons_enums.PriceIndexes.IND_PRICE_TIME.value] - != candles[0][commons_enums.PriceIndexes.IND_PRICE_OPEN.value] - != candles[0][commons_enums.PriceIndexes.IND_PRICE_HIGH.value] - != candles[0][commons_enums.PriceIndexes.IND_PRICE_LOW.value] - != candles[0][commons_enums.PriceIndexes.IND_PRICE_CLOSE.value] - != candles[0][commons_enums.PriceIndexes.IND_PRICE_VOL.value] - ) - # candles are unique - assert len(set(c[0] for c in candles)) == fetched_count - - -async def test_fetch_candles_history_asynchronousness(iceberg_client): - start_time = 1718785679 - end_time_1 = 1721377495 - end_time_2 = 1721377495 + 2 * commons_constants.DAYS_TO_SECONDS - end_time_3 = 1721377495 + 23 * commons_constants.DAYS_TO_SECONDS - - scan_call_times = [] - _to_arrow_call_times = [] - _to_arrow_return_times = [] - - def _get_or_create_table(*args, **kwargs): - table = original_get_or_create_table(*args, **kwargs) - original_scan = table.scan - - - def _scan(*args, **kwargs): - scan_call_times.append(time.time()) - scan_result = original_scan(*args, **kwargs) - original_to_arrow = scan_result.to_arrow - - def _to_arrow(*args, **kwargs): - _to_arrow_call_times.append(time.time()) - try: - return original_to_arrow(*args, **kwargs) - finally: - _to_arrow_return_times.append(time.time()) - - scan_result.to_arrow = _to_arrow - return scan_result - - table.scan = mock.Mock(side_effect=_scan) - return table - - original_get_or_create_table = iceberg_client._get_or_create_table - with ( - mock.patch.object(iceberg_client, "_get_or_create_table", mock.Mock(side_effect=_get_or_create_table)) as get_or_create_table_mock - ): - candles_1, candles_2, candles_3 = await asyncio.gather( - iceberg_client.fetch_candles_history( - EXCHANGE, SYMBOL, SHORT_TIME_FRAME, start_time, end_time_1 - ), - iceberg_client.fetch_candles_history( - EXCHANGE, SYMBOL, SHORT_TIME_FRAME, start_time, end_time_2 - ), - iceberg_client.fetch_candles_history( - EXCHANGE, SYMBOL, SHORT_TIME_FRAME, start_time, end_time_3 - ), - ) - assert get_or_create_table_mock.call_count == 3 - assert len(scan_call_times) == 3 - assert len(_to_arrow_call_times) == 3 - assert len(_to_arrow_return_times) == 3 - - assert scan_call_times[0] <= scan_call_times[1] <= scan_call_times[2] - assert _to_arrow_call_times[0] <= _to_arrow_call_times[1] <= _to_arrow_call_times[2] - assert _to_arrow_return_times[0] < _to_arrow_return_times[1] < _to_arrow_return_times[2] - - # all to_arrow calls have been performed before the first to_arrow return, - # which means they are running concurrently in this async context - assert max(_to_arrow_call_times) < min(_to_arrow_return_times) - - assert len(candles_1) > 2000 - assert len(candles_2) > len(candles_1) - assert len(candles_3) > len(candles_2) - - -async def test_concurrent_insert_candles_history(iceberg_client): - assert iceberg_client.enable_async_batch_inserts is True # default value - temp_test_table = "temp_test_ohlcv_history_table" # use temp test table - - class _TableNames(enum.Enum): - OHLCV_HISTORY = "temp_test_ohlcv_history_table" - - with ( - mock.patch.object(iceberg_historical_backend_client, "TableNames", _TableNames), - mock.patch.object(constants, "CREATE_ICEBERG_DB_IF_MISSING", True), - ): - exchange = "binance" - symbol = "BTC/USDC" - time_frame = commons_enums.TimeFrames.FIFTEEN_MINUTES - # iceberg_client.drop_table(temp_test_table) # in case table is not empty - assert len(await iceberg_client.fetch_extended_candles_history( - exchange, [symbol], [time_frame], 0, 1818785688 - )) == 0, ( - f"Test table is not empty" - ) - try: - rows_1 = [ - [iceberg_client.get_formatted_time(1718785679), exchange, symbol, time_frame.value, 80, 100, 79, 85, 12312], - [iceberg_client.get_formatted_time(1718785680), exchange, symbol, time_frame.value, 85, 86, 84, 86, 12312], - ] - rows_2 = [ - [iceberg_client.get_formatted_time(1718785681), exchange, symbol, time_frame.value, 86, 87, 85, 87, 12312], - [iceberg_client.get_formatted_time(1718785682), exchange, symbol, time_frame.value, 87, 88, 86, 88, 12312], - ] - rows_3 = [ - [iceberg_client.get_formatted_time(1718785683), exchange, symbol, time_frame.value, 88, 89, 87, 89, 12312], - [iceberg_client.get_formatted_time(1718785684), exchange, symbol, time_frame.value, 89, 90, 88, 90, 12312], - ] - column_names = [ - "timestamp", "exchange_internal_name", "symbol", "time_frame", "open", "high", "low", "close", "volume" - ] - await asyncio.gather( - iceberg_client.insert_candles_history(rows_1, column_names), - iceberg_client.insert_candles_history(rows_2, column_names), - iceberg_client.insert_candles_history(rows_3, column_names), - ) - assert len(iceberg_client._pending_insert_data_by_table) == 1 - assert sum( - len(pending_data.data) - for pending_data in iceberg_client._pending_insert_data_by_table[ - iceberg_historical_backend_client.TableNames.OHLCV_HISTORY - ] - ) == len(rows_1) + len(rows_2) + len(rows_3) - # didn't insert yet: max pending rows is not reached, it will now be reached - assert await iceberg_client.fetch_extended_candles_history( - exchange, [symbol], [time_frame], 0, 1818785688 - ) == [] - with mock.patch.object(iceberg_historical_backend_client, "_MAX_PENDING_BATCH_INSERT_SIZE", 3): - rows_4 = [ - # will trigger insert of previous pending rows - [iceberg_client.get_formatted_time(1718785685), exchange, symbol, time_frame.value, 90, 91, 89, 91, 12312], - [iceberg_client.get_formatted_time(1718785686), exchange, symbol, time_frame.value, 91, 92, 90, 92, 12312], - [iceberg_client.get_formatted_time(1718785687), exchange, symbol, time_frame.value, 91.5, 92, 90, 92, 12312], - ] - rows_5 = [ - # will trigger another insert - [iceberg_client.get_formatted_time(1718785688), exchange, symbol, time_frame.value, 92, 93, 91, 93, 12312], - [iceberg_client.get_formatted_time(1718785689), exchange, symbol, time_frame.value, 93, 94, 92, 94, 12312], - [iceberg_client.get_formatted_time(1718785690), exchange, symbol, time_frame.value, 93, 94, 92, 94, 12312], - ] - # inserting: max pending rows is reached, it will now be inserted - await asyncio.gather( - iceberg_client.insert_candles_history(rows_4, column_names), - iceberg_client.insert_candles_history(rows_5, column_names), - ) - # rows have been inserted - fetched_rows = await iceberg_client.fetch_extended_candles_history( - exchange, [symbol], [time_frame], 0, 1818785688 - ) - expected_rows = [ - [row[3], row[2], history_backend_util.get_utc_timestamp_from_datetime( - datetime.datetime.strptime(row[0], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=datetime.timezone.utc) - )] + row[4:] - for row in rows_1 + rows_2 + rows_3 + rows_4 + rows_5 - ] - assert fetched_rows == expected_rows - # no remaining pending rows - assert iceberg_client._pending_insert_data_by_table == {} - finally: - # test complete: delete test table - iceberg_client.drop_table(temp_test_table) - - -async def test_deduplicate(iceberg_client): - start_time = 1718785679 - end_time = 1721377495 - candles = await iceberg_client.fetch_extended_candles_history( - EXCHANGE, [SYMBOL], [SHORT_TIME_FRAME], start_time, end_time - ) - assert all(c[0] == SHORT_TIME_FRAME.value for c in candles) - assert all(c[1] == SYMBOL for c in candles) - duplicated = candles + candles - assert len(duplicated) == len(candles) * 2 - assert sorted(candles, key=lambda c: c[2]) == candles - deduplicated = history_backend_util.deduplicate(duplicated, [0, 1, 2]) - # deduplicated and still sorted - assert deduplicated == candles diff --git a/bin/start.spec b/bin/start.spec index 18c0e03875..13d981545b 100644 --- a/bin/start.spec +++ b/bin/start.spec @@ -1,6 +1,6 @@ # -*- mode: python -*- -from PyInstaller.utils.hooks import collect_data_files +from PyInstaller.utils.hooks import collect_data_files, copy_metadata block_cipher = None @@ -8,6 +8,21 @@ block_cipher = None # hiddenimports only bundles Python modules; collect_data_files includes those data files. eth_account_datas = collect_data_files("eth_account") +# importlib.metadata.version() at import time; PyInstaller does not bundle .dist-info by default. +_eth_web3_dist_names = ( + "py-ecc", + "eth-account", + "eth-keyfile", + "eth-utils", + "eth-typing", + "eth-keys", + "hexbytes", + "web3", +) +eth_web3_metadata = [] +for _dist_name in _eth_web3_dist_names: + eth_web3_metadata += copy_metadata(_dist_name) + OCTOBOT_PACKAGES_FILES = REQUIRED = [s.strip() for s in open('bin/octobot_packages_files.txt').readlines()] # hiddenimports=['numpy.core._dtype_ctypes'] from https://github.com/pyinstaller/pyinstaller/issues/3982 a = Analysis( @@ -16,21 +31,24 @@ a = Analysis( datas=[ ('../octobot/config', 'octobot/config'), ('../octobot/strategy_optimizer/optimizer_data_files', 'octobot/strategy_optimizer/optimizer_data_files'), - ] + eth_account_datas, # required for node wallet mnemonic generation (web3.Account.create_with_mnemonic) + ] + eth_account_datas + eth_web3_metadata, # required for node wallet mnemonic generation (web3.Account.create_with_mnemonic) hiddenimports=[ "colorlog", "numpy.core._dtype_ctypes", "dotenv", "pgpy", "imghdr", "web3", "eth_account", "aiosqlite", "aiohttp", - "pyarrow", "pyiceberg", "psutil", - "telegram", "telegram.ext", "telethon", "jsonschema", + "telegram", "telegram.ext", "jsonschema", "tulipy", "asyncpraw", "simplifiedpytrends", "simplifiedpytrends.exceptions", "simplifiedpytrends.request", "pyngrok", "pyngrok.ngrok", "openai", - "flask", "flask_login", "flask_wtf", "flask_caching", "flask_compress", "flask_socketio", "flask_cors", + "flask", "flask_login", "flask_wtf", "flask_socketio", "flask_cors", + # Tentacles (excluded from Analysis) — ASGI stack per packages/services/full_requirements.txt + "asgiref.sync", "asgiref.wsgi", + "uvicorn", "uvicorn.config", "uvicorn.server", + "starlette", "starlette.applications", "starlette.routing", "starlette.websockets", "werkzeug.middleware", "werkzeug.middleware.proxy_fix", - "wtforms", "wtforms.fields", "gevent", "geventwebsocket", + "wtforms", "wtforms.fields", "vaderSentiment", "vaderSentiment.vaderSentiment", "coingecko_openapi_client", "certifi", diff --git a/dev_requirements.txt b/dev_requirements.txt index 4b65f4bb08..ebb485e46f 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -14,3 +14,6 @@ wheel pur pylint + +ruff==0.16.7 +PyYAML==6.0.3 diff --git a/full_requirements.txt b/full_requirements.txt index 74430e325d..65d62fdb36 100644 --- a/full_requirements.txt +++ b/full_requirements.txt @@ -5,10 +5,7 @@ colorlog==6.10.1 gmqtt==0.7.0 pgpy==0.6.0 clickhouse-connect==0.10.0 -pyiceberg==0.10.0 -# pydantic==2.11.10 # required for pyiceberg 0.10.0 https://github.com/apache/iceberg-python/issues/2590. -pydantic==2.12.5 # 2.12.5 is compataible with pyiceberg 0.10.0, but logs tons of deprecation warnings, wait for pyiceberg new version -pyarrow==23.0.0 +pydantic==2.13.5 # used by ccxt for protobuf "websockets" such as mexc # lock protobuf to avoid using .rc versions diff --git a/octobot/AGENTS.md b/octobot/AGENTS.md new file mode 100644 index 0000000000..d6c81db92f --- /dev/null +++ b/octobot/AGENTS.md @@ -0,0 +1,43 @@ +# Agents: octobot (core application) + +## Role + +Top-level OctoBot app: CLI entry, configuration, community features, orchestration of packages. Wires trading, evaluators, services, and tentacles at runtime. + +## Owns + +- `octobot/` Python package at repo root +- Root `tests/` (application integration) +- User-facing config templates under app paths (not `user/` runtime data) + +## Public surface + +- `OctoBot` CLI (installed wheel) +- `octobot` modules consumed by tentacles and startup + +## May depend on + +- `octobot_commons`, `octobot_trading`, `octobot_evaluators`, `octobot_services`, tentacles manager, installed `tentacles/` + +## Do not + +- Edit repo-root `tentacles/` (install output) +- Put secrets or `user/` data in tree +- Duplicate logic that belongs in `packages/*` +- Use **node journal** for reads or control flow outside export: record-only via `octobot.community.node_journal` from the rest of the app (see [`community/node_journal/AGENTS.md`](community/node_journal/AGENTS.md), skill **node-journal**) +- Duplicate shared wire/config/event strings; use package-top [`constants.py`](constants.py) / [`enums.py`](enums.py) (or the owning package’s equivalents) +- Use `from xxx import yyy` or lazy imports in normal modules (only `__init__.py` re-exports; see [CONTRIBUTING-agent.md](../CONTRIBUTING-agent.md) — **Python conventions (agents)**) + +## Tests + +- **cwd:** repo root +- **Tentacles:** required (install via `ci-tentacles` / `reinstall-tentacles.sh`) +- **Example:** `pytest tests -n auto --dist loadfile` and tentacles test pass per [reference-pytest.md](../.cursor/skills/octobot-cloud/reference-pytest.md) ([When tests fail](../.cursor/skills/octobot-cloud/reference-pytest.md#when-tests-fail) for red tests) + +## Related human doc + +- `docs/content/developers/environment/setup-your-environment.md` + +## Last reviewed + +- 2026-09-21 diff --git a/octobot/__init__.py b/octobot/__init__.py index dfb31867b3..6c1b20131c 100644 --- a/octobot/__init__.py +++ b/octobot/__init__.py @@ -16,5 +16,5 @@ PROJECT_NAME = "OctoBot" AUTHOR = "Drakkar-Software" -VERSION = "3.0.0-beta2" # major.minor.revision +VERSION = "3.0.0-beta3" # major.minor.revision LONG_VERSION = f"{VERSION}" diff --git a/octobot/cli.py b/octobot/cli.py index e4d7b59f4d..241d4602e1 100644 --- a/octobot/cli.py +++ b/octobot/cli.py @@ -57,6 +57,7 @@ import octobot.logger as octobot_logger import octobot.community as octobot_community import octobot.community.errors + import octobot.community.node_journal.lifecycle as journal_startup import octobot.limits as limits except ImportError as err: traceback.print_exc() @@ -478,8 +479,36 @@ def _log_startup_distribution_mode(logger, args) -> None: logger.info(startup_message) +def _record_cli_startup_failure( + error: Exception, + startup_phase: str, + config: configuration.Configuration | None = None, +) -> None: + try: + if config is not None: + distribution = configuration_manager.get_distribution(config.config) + elif constants.FORCED_DISTRIBUTION: + distribution = enums.OctoBotDistribution(constants.FORCED_DISTRIBUTION) + else: + return + if distribution is not enums.OctoBotDistribution.NODE: + return + journal_startup.record_node_startup_failed( + error, + startup_phase=startup_phase, + force_exit=True, + config=config, + ) + except Exception: + octobot_commons.logging.get_logger("OctoBot").debug( + "Failed to record CLI startup failure in node journal", + exc_info=True, + ) + + def start_octobot(args, default_config_file=None): logger = None + config = None try: if args.version: print(constants.LONG_VERSION) @@ -518,8 +547,6 @@ def start_octobot(args, default_config_file=None): if not config.is_loaded(): raise errors.ConfigError - octobot_community.ActivityMetrics.initialize_tracker(config) - # Handle utility methods before bot initializing if possible if args.encrypter: commands.exchange_keys_encrypter() @@ -531,6 +558,8 @@ def start_octobot(args, default_config_file=None): community_auth = None if args.backtesting else asyncio.run( _get_authenticated_community_if_possible(config, logger) ) + octobot_community.init_sentry_tracker() + journal_startup.initialize_journal(config) # Startup order matters: sync user and tentacles/community config must run before # profile activation. First boot (empty user/) has no profiles until tentacles @@ -604,20 +633,23 @@ def start_octobot(args, default_config_file=None): force_error_exit = False except errors.RemoteConfigError as err: logger.exception(err) + _record_cli_startup_failure(err, "remote_config", config) force_error_exit = True except errors.ConfigError as err: logger.error("OctoBot can't start without a valid " + common_constants.CONFIG_FILE + " configuration file.\nError: " + str(err) + "\nYou can use " + constants.DEFAULT_CONFIG_FILE + " as an example to fix it.") + _record_cli_startup_failure(err, "config", config) force_error_exit = True - except errors.NoProfileError: + except errors.NoProfileError as err: logger.error("Missing default profiles. OctoBot can't start without a valid default profile configuration. " "Please make sure that the {config.profiles_path} " f"folder is accessible. To reinstall default profiles, delete the " f"'{tentacles_manager_constants.TENTACLES_PATH}' " f"folder or start OctoBot with the following arguments: tentacles --install --all") + _record_cli_startup_failure(err, "no_profile", config) force_error_exit = True except ModuleNotFoundError as err: @@ -626,18 +658,21 @@ def start_octobot(args, default_config_file=None): "please use the following command:\nstart.py tentacles --install --all") else: logger.exception(err) + _record_cli_startup_failure(err, "module_not_found", config) force_error_exit = True - except errors.ConfigEvaluatorError: + except errors.ConfigEvaluatorError as err: logger.error("OctoBot can't start without a valid configuration file.\n" "This file is generated on tentacle " "installation using the following command:\nstart.py tentacles --install --all") + _record_cli_startup_failure(err, "config_evaluator", config) force_error_exit = True - except errors.ConfigTradingError: + except errors.ConfigTradingError as err: logger.error("OctoBot can't start without a valid configuration file.\n" "This file is generated on tentacle " "installation using the following command:\nstart.py tentacles --install --all") + _record_cli_startup_failure(err, "config_trading", config) force_error_exit = True if force_error_exit: octobot_community.flush_tracker() diff --git a/octobot/commands.py b/octobot/commands.py index 7bda986211..0e7f19cb60 100644 --- a/octobot/commands.py +++ b/octobot/commands.py @@ -38,6 +38,8 @@ import octobot.constants as constants import octobot.community.tentacles_packages as community_tentacles_packages import octobot.configuration_manager as configuration_manager +import octobot.enums as enums +import octobot.community.node_journal.lifecycle as journal_startup COMMANDS_LOGGER_NAME = "Commands" IGNORED_COMMAND_WHEN_RESTART = ["-u", "--update"] @@ -340,6 +342,18 @@ def run_bot(bot, logger): bot.task_manager.run_forever(start_bot(bot, logger)) +def _record_bot_initialize_failure(bot, error: Exception, *, catch: bool) -> None: + if configuration_manager.get_distribution(bot.config) is not enums.OctoBotDistribution.NODE: + return + edited_config = bot.get_edited_config(constants.CONFIG_KEY, dict_only=False) + journal_startup.record_node_startup_failed( + error, + startup_phase="bot_initialize", + force_exit=not catch, + config=edited_config, + ) + + async def start_bot(bot, logger, catch=False): try: # load tentacles details @@ -354,6 +368,7 @@ async def start_bot(bot, logger, catch=False): except Exception as e: logger.exception(e) + _record_bot_initialize_failure(bot, e, catch=catch) if not catch: raise stop_bot(bot) diff --git a/octobot/community/__init__.py b/octobot/community/__init__.py index 63be4d8210..ec069b2c34 100644 --- a/octobot/community/__init__.py +++ b/octobot/community/__init__.py @@ -64,9 +64,6 @@ get_current_octobots_stats, can_read_metrics, ) -from octobot.community.activity_analysis.activity_metrics import ( - ActivityMetrics, -) from octobot.community.authentication import ( CommunityAuthentication, ) @@ -100,7 +97,6 @@ history_backend_client, HistoricalBackendClient, ClickhouseHistoricalBackendClient, - IcebergHistoricalBackendClient, ) from octobot.community.community_bot import ( CommunityBot, @@ -123,7 +119,6 @@ "get_community_metrics", "get_current_octobots_stats", "can_read_metrics", - "ActivityMetrics", "CommunityAuthentication", "CommunityTentaclesPackage", "CommunitySupports", @@ -165,7 +160,6 @@ "history_backend_client", "HistoricalBackendClient", "ClickhouseHistoricalBackendClient", - "IcebergHistoricalBackendClient", "CommunityBot", "MissingDeploymentError", "MissingProductsSubscriptionError", diff --git a/octobot/community/activity_analysis/bot_id_resolver.py b/octobot/community/activity_analysis/bot_id_resolver.py deleted file mode 100644 index ec3387e489..0000000000 --- a/octobot/community/activity_analysis/bot_id_resolver.py +++ /dev/null @@ -1,50 +0,0 @@ -# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) -# Copyright (c) 2025 Drakkar-Software, All rights reserved. -# -# OctoBot is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# OctoBot is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public -# License along with OctoBot. If not, see . -import dataclasses -import uuid - -import octobot_commons.configuration as configuration -import octobot_commons.constants as commons_constants - - -@dataclasses.dataclass(frozen=True) -class BotIdResolution: - bot_id: str - was_created: bool - - -def _get_metrics_section(config: configuration.Configuration) -> dict: - metrics_section = config.config.setdefault(commons_constants.CONFIG_METRICS, {}) - if not isinstance(metrics_section, dict): - raise ValueError(f"{commons_constants.CONFIG_METRICS} must be a mapping in config") - return metrics_section - - -def ensure_activity_bot_id(config: configuration.Configuration) -> BotIdResolution: - metrics_section = _get_metrics_section(config) - stored_bot_id = metrics_section.get(commons_constants.CONFIG_METRICS_ACTIVITY_BOT_ID) - if stored_bot_id: - return BotIdResolution( - bot_id=str(stored_bot_id), - was_created=False, - ) - new_bot_id = str(uuid.uuid4()) - metrics_section[commons_constants.CONFIG_METRICS_ACTIVITY_BOT_ID] = new_bot_id - config.save() - return BotIdResolution( - bot_id=new_bot_id, - was_created=True, - ) diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index 4964ab2ef3..7684eb8322 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -20,8 +20,7 @@ import time import threading import typing -import octobot.community.activity_analysis.config_path_binding as config_path_binding -import octobot.community.activity_analysis.activity_metrics as activity_metrics +import octobot.community.config_path_binding as config_path_binding import decimal import octobot.constants as constants @@ -40,6 +39,7 @@ import octobot.community.feeds as community_feeds import octobot.community.tentacles_packages as community_tentacles_packages import octobot.community.community_bot as community_bot +import octobot_commons.asyncio_tools as asyncio_tools import octobot_commons.constants as commons_constants import octobot_commons.enums as commons_enums import octobot_commons.authentication as authentication @@ -150,6 +150,7 @@ def create(configuration: commons_configuration.Configuration, **kwargs): ) def update(self, configuration: commons_configuration.Configuration): + self.config = configuration self.configuration_storage.set_configuration(configuration) self._wallet_backend = wallet_backend.WalletBackend( self._get_wallet_sync_storage(), self.logger @@ -316,8 +317,9 @@ async def ensure_async_loop(self): await self._re_create_client() def is_using_the_current_loop(self): - return self.supabase_client.event_loop is None \ - or self.supabase_client.event_loop is asyncio.get_event_loop() + if self.supabase_client.event_loop is None: + return True + return asyncio_tools.is_on_async_loop(self.supabase_client.event_loop) def is_initialized(self): return self.initialized_event is not None and self.initialized_event.is_set() @@ -583,7 +585,6 @@ def _clear_bot_scoped_config(self): "and webhook url will be different on this bot." ) self._save_bot_id("") - activity_metrics.ActivityMetrics.clear_activity_bot_id(self.config) self.save_tradingview_email("") # also reset mqtt id to force a new mqtt id creation self._save_mqtt_device_uuid("") @@ -691,6 +692,20 @@ def authenticate_wallet(self, address: str, passphrase: str) -> dict: def verify_wallet_passphrase(self, address: str, passphrase: str) -> bool: return self._wallet_backend.verify_wallet_passphrase(address, passphrase) + def recover_passphrase_from_ownership_proof( + self, + address: str, + new_passphrase: str, + seed: typing.Optional[str] = None, + private_key: typing.Optional[str] = None, + ) -> None: + return self._wallet_backend.recover_passphrase_from_ownership_proof( + address, + new_passphrase, + seed=seed, + private_key=private_key, + ) + def decrypt_wallet_by_address(self, address: str, passphrase: str): return self._wallet_backend.decrypt_wallet_by_address(address, passphrase) @@ -723,6 +738,9 @@ def get_wallet(self, address: str) -> sync_chain.Wallet: def get_wallet_by_user_id(self, user_id: str) -> sync_chain.Wallet: return self._wallet_backend.get_wallet_by_user_id(user_id) + def has_wallet_for_user_id(self, user_id: str) -> bool: + return self._wallet_backend.has_wallet_for_user_id(user_id) + async def get_session_for_address(self, address: str) -> starfish_spaces.Session: """Build (and cache) a dk-namespace starfish_spaces Session for the given wallet. diff --git a/octobot/community/community_analysis.py b/octobot/community/community_analysis.py index 4ae2dfd330..093e545c59 100644 --- a/octobot/community/community_analysis.py +++ b/octobot/community/community_analysis.py @@ -86,7 +86,7 @@ async def get_community_metrics(): def can_read_metrics(config): - return constants.IS_CLOUD_ENV or config.get_metrics_enabled() + return False def _format_top_elements(top_elements): diff --git a/octobot/community/activity_analysis/activity_metrics.py b/octobot/community/community_bot_stats.py similarity index 62% rename from octobot/community/activity_analysis/activity_metrics.py rename to octobot/community/community_bot_stats.py index 901bb93cfb..6a0563eda8 100644 --- a/octobot/community/activity_analysis/activity_metrics.py +++ b/octobot/community/community_bot_stats.py @@ -13,62 +13,30 @@ # # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . + +"""Periodic community bot config/stats updates. + +``CommunityBotStats`` is temporary until full migration to OctoBot v3 replaces this path. +""" + import asyncio import octobot_commons.logging as logging -import octobot_commons.configuration as configuration import octobot_commons.authentication as authentication import octobot_commons.constants as common_constants import octobot_trading.api as trading_api -import octobot.community.activity_analysis.bot_id_resolver as bot_id_resolver -import octobot.community.errors_upload.sentry_tracker as tracker -import octobot.constants as constants -import octobot.enums as enums - -class ActivityMetrics: +class CommunityBotStats: def __init__(self, octobot_api): self.octobot_api = octobot_api - self.edited_config: configuration.Configuration = octobot_api.get_edited_config(dict_only=False) - self.enabled = constants.IS_CLOUD_ENV or self.edited_config.get_metrics_enabled() self.logger = logging.get_logger(self.__class__.__name__) self.keep_running = True - - @staticmethod - def initialize_tracker(config: configuration.Configuration) -> None: - tracker.init_sentry_tracker(metrics_enabled=config.get_metrics_enabled()) - - @staticmethod - def clear_activity_bot_id(config: configuration.Configuration) -> None: - metrics_section = config.config.setdefault(common_constants.CONFIG_METRICS, {}) - if isinstance(metrics_section, dict): - metrics_section[common_constants.CONFIG_METRICS_ACTIVITY_BOT_ID] = "" - - def setup_activity_tracking(self, distribution: enums.OctoBotDistribution) -> None: - if not self.enabled: - return - resolution = bot_id_resolver.ensure_activity_bot_id(self.edited_config) - if tracker.activity_tracking_is_active(): - tracker.update_tracker_bot_id(resolution.bot_id) - if distribution is enums.OctoBotDistribution.NODE and resolution.was_created: - tracker.track_usage_event( - "node_first_start", - distribution="node", - version=constants.LONG_VERSION, - ) - - @staticmethod - def report_child_octobot_first_start() -> None: - if not tracker.has_tracker_bot_id(): - return - tracker.track_usage_event("child_octobot_first_start") + self.task_enabled = False async def start_community_task(self): - if not self.enabled: - return try: while self.keep_running: await asyncio.sleep(common_constants.TIMER_BETWEEN_METRICS_UPTIME_UPDATE) diff --git a/octobot/community/activity_analysis/config_path_binding.py b/octobot/community/config_path_binding.py similarity index 99% rename from octobot/community/activity_analysis/config_path_binding.py rename to octobot/community/config_path_binding.py index e97ebf3198..da8789baf5 100644 --- a/octobot/community/activity_analysis/config_path_binding.py +++ b/octobot/community/config_path_binding.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . + import dataclasses import hashlib import os diff --git a/octobot/community/errors_upload/__init__.py b/octobot/community/errors_upload/__init__.py index 6af2d0b4a7..725fb45502 100644 --- a/octobot/community/errors_upload/__init__.py +++ b/octobot/community/errors_upload/__init__.py @@ -1,25 +1,10 @@ # This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) # Copyright (c) 2025 Drakkar-Software, All rights reserved. -# -# OctoBot is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# OctoBot is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public -# License along with OctoBot. If not, see . from octobot.community.errors_upload import sentry_tracker from octobot.community.errors_upload.sentry_tracker import ( - init_sentry_tracker, flush_tracker, - update_tracker_bot_id, - track_usage_event, + init_sentry_tracker, ) __all__ = [ diff --git a/octobot/community/errors_upload/sentry_aiohttp_transport.py b/octobot/community/errors_upload/sentry_aiohttp_transport.py deleted file mode 100644 index 8cce3b0767..0000000000 --- a/octobot/community/errors_upload/sentry_aiohttp_transport.py +++ /dev/null @@ -1,239 +0,0 @@ -# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) -# Copyright (c) 2025 Drakkar-Software, All rights reserved. -# -# OctoBot is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# OctoBot is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public -# License along with OctoBot. If not, see . -import logging -import typing -import asyncio -import aiohttp - -import sentry_sdk -import sentry_sdk.consts -import sentry_sdk.utils -import sentry_sdk.envelope -import sentry_sdk.types - - -class SentryAiohttpTransport(sentry_sdk.HttpTransport): - def __init__( - self, options: typing.Dict[str, typing.Any] - ): - super().__init__(options) - # WARNING: override default "br" value: not supported by Glitchtip yet - self._compression_algo = "gzip" - # use custom async worker instead of default sentry thread worker - # does not support proxies, at least for now - self._worker = AiohttpWorker(queue_size=options["transport_queue_size"]) - - async def _async_send_request( - self, - body, # type: bytes - headers, # type: typing.Dict[str, str] - endpoint_type="store", # type: sentry_sdk.consts.EndpointType - envelope=None, # type: typing.Optional[sentry_sdk.envelope.Envelope] - ) -> None: - - def record_loss(reason: str) -> None: - if envelope is None: - self.record_lost_event(reason, data_category="error") - else: - for item in envelope.items: - self.record_lost_event(reason, item=item) - - headers.update( - { - "User-Agent": str(self._auth.client), - "X-Sentry-Auth": str(self._auth.to_header()), - } - ) - try: - async with self._worker.session.post( - str(self._auth.get_api_url(endpoint_type)), - data=body, - headers=headers, - ) as response: - self._update_rate_limits(response) - - if response.status == 429: - # if we hit a 429. Something was rate limited but we already - # acted on this in `self._update_rate_limits`. Note that we - # do not want to record event loss here as we will have recorded - # an outcome in relay already. - self.on_dropped_event("status_429") - pass - - elif response.status >= 300 or response.status < 200: - logging.getLogger(self.__class__.__name__).warning( - "Unexpected status code: %s (body: %s)", - response.status, - await response.text(), - ) - self.on_dropped_event("status_{}".format(response.status)) - record_loss("network_error") - except BaseException as err: - logging.getLogger(self.__class__.__name__).warning(f"Sentry post error: {err} {err.__class__.__name__}") - self.on_dropped_event("network") - record_loss("network_error") - raise - - async def _async_send_envelope( - self, envelope: sentry_sdk.envelope.Envelope - ) -> None: - """ - Async version of super()._send_envelope - """ - - # remove all items from the envelope which are over quota - new_items = [] - for item in envelope.items: - if self._check_disabled(item.data_category): - if item.data_category in ("transaction", "error", "default", "statsd"): - self.on_dropped_event("self_rate_limits") - self.record_lost_event("ratelimit_backoff", item=item) - else: - new_items.append(item) - - # Since we're modifying the envelope here make a copy so that others - # that hold references do not see their envelope modified. - envelope = sentry_sdk.envelope.Envelope(headers=envelope.headers, items=new_items) - - if not envelope.items: - return None - - # since we're already in the business of sending out an envelope here - # check if we have one pending for the stats session envelopes so we - # can attach it to this enveloped scheduled for sending. This will - # currently typically attach the client report to the most recent - # session update. - client_report_item = self._fetch_pending_client_report(interval=30) - if client_report_item is not None: - envelope.items.append(client_report_item) - - content_encoding, body = self._serialize_envelope(envelope) - - assert self.parsed_dsn is not None - sentry_sdk.utils.logger.debug( - "Sending envelope [%s] project:%s host:%s", - envelope.description, - self.parsed_dsn.project_id, - self.parsed_dsn.host, - ) - - headers = { - "Content-Type": "application/x-sentry-envelope", - } - if content_encoding: - headers["Content-Encoding"] = content_encoding - - await self._async_send_request( - body.getvalue(), - headers=headers, - endpoint_type=sentry_sdk.consts.EndpointType.ENVELOPE, - envelope=envelope, - ) - return None - - def capture_event( - self, event: sentry_sdk.types.Event - ) -> None: - """ - DEPRECATED: Please use capture_envelope instead. - """ - return super().capture_event(event) - - def capture_envelope( - self, envelope: sentry_sdk.envelope.Envelope - ) -> None: - - async def send_envelope_wrapper() -> None: - with sentry_sdk.utils.capture_internal_exceptions(): - await self._async_send_envelope(envelope) - self._flush_client_reports() - - if not self._worker.submit(send_envelope_wrapper): - self.on_dropped_event("full_queue") - for item in envelope.items: - self.record_lost_event("queue_overflow", item=item) - - async def async_kill(self): - await self._worker.async_kill() - -class AiohttpWorker: - # do not inherit from sentry_sdk.worker.BackgroundWorker to avoid implementing all its methods - # only implement required methods - def __init__(self, queue_size=sentry_sdk.consts.DEFAULT_QUEUE_SIZE): - self.session = None - self.call_tasks = [] - self._queue_size = queue_size - self._kill_task = None - self._stopped = False - - @property - def is_alive(self) -> bool: - if self.session is None or self.session.closed: - return False - # session is not closed - # no pending kill task - if self._kill_task is None or self._kill_task.done(): - return not self.session.closed - # pending kill task, will stop - return False - - def _ensure_session(self) -> None: - if not self.is_alive: - self.start() - - def start(self) -> None: - if self._kill_task and not self._kill_task.done(): - self._kill_task.cancel() - self.session = aiohttp.ClientSession() - - def kill(self) -> None: - for task in self.call_tasks: - if not task.done(): - task.cancel() - self.call_tasks = [] - if self.is_alive: - self._kill_task = asyncio.create_task(self.session.close()) - - async def async_kill(self): - self._stopped = True - self.kill() - if self._kill_task and not self._kill_task.done(): - await self._kill_task - - def full(self) -> bool: - return len(self.call_tasks) > self._queue_size - - def flush(self, timeout: float, callback=None) -> None: - sentry_sdk.utils.logger.debug("Custom background worker got flush request, ignored") - - async def _async_call(self, callback): - try: - await callback() - finally: - if asyncio.current_task() in self.call_tasks: - self.call_tasks.remove(asyncio.current_task()) - - def _schedule_async_send(self, callback): - self.call_tasks.append(asyncio.create_task(self._async_call(callback))) - - def submit(self, callback) -> bool: - if self._stopped: - return False - self._ensure_session() - if self.full(): - return False - self._schedule_async_send(callback) - return True diff --git a/octobot/community/errors_upload/sentry_tracker.py b/octobot/community/errors_upload/sentry_tracker.py index 1e015ad107..7febf133cf 100644 --- a/octobot/community/errors_upload/sentry_tracker.py +++ b/octobot/community/errors_upload/sentry_tracker.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . import time -import typing import sentry_sdk @@ -25,72 +24,50 @@ _sentry_initialized = False -_activity_tracking_active = False -_tracker_bot_id_set: bool = False -def init_sentry_tracker(metrics_enabled: bool) -> None: - """ - Initialize Sentry when ACTIVITY_TRACKER_DSN (metrics enabled) or ERROR_TRACKER_DSN is set. - Tracker DSN takes priority when metrics are enabled. - """ - global _activity_tracking_active, _sentry_initialized +def init_sentry_tracker() -> None: + """Initialize Sentry error reporting when ERROR_TRACKER_DSN is set.""" + global _sentry_initialized logger = octobot_commons.logging.get_logger("sentry_tracker") - activity_dsn = octobot.constants.ACTIVITY_TRACKER_DSN error_dsn = octobot.constants.ERROR_TRACKER_DSN - use_activity_dsn = bool(activity_dsn and metrics_enabled) - active_dsn = activity_dsn if use_activity_dsn else error_dsn - - if not active_dsn: - logger.debug("Sentry tracker disabled: no applicable DSN") - _activity_tracking_active = False + if not error_dsn: _sentry_initialized = False return + logger.debug("Sentry tracker enabled: ERROR_TRACKER_DSN is set") if _sentry_initialized: - _activity_tracking_active = use_activity_dsn return environment = "cloud" if octobot.constants.IS_CLOUD_ENV else "self hosted" app_name = f"{octobot.constants.PROJECT_NAME} open source" - init_kwargs: dict[str, typing.Any] = { - "dsn": active_dsn, + # Error DSN: auto-capture log errors; strip bot log prefix before upload. + sentry_sdk.init( + dsn=error_dsn, # Percent of error events to send to the server. # 0.5 would be 50%. Defaults to 1.0. - "sample_rate": 1, + sample_rate=1, # Set traces_sample_rate to 1.0 to capture 100% # of transactions for performance monitoring. # We recommend adjusting this value in production. - "traces_sample_rate": 0, + traces_sample_rate=0, # By default the SDK will try to use the SENTRY_RELEASE # environment variable, or infer a git commit # SHA as release, however you may want to set # something more human-readable. - "release": octobot.constants.LONG_VERSION, - "include_local_variables": False, # careful not to upload sensitive data + release=octobot.constants.LONG_VERSION, + include_local_variables=False, # careful not to upload sensitive data # breadcrumbs from multiple tasks and activities are mixed, only take the last ones - "max_breadcrumbs": 5, - "environment": environment, - } - if use_activity_dsn: - init_kwargs["sample_rate"] = 0 - # Activity-only: no default integrations (especially LoggingIntegration). - # Only explicit track_usage_event calls send data. - init_kwargs["default_integrations"] = False - init_kwargs["integrations"] = [] - else: - # Error DSN: auto-capture log errors; strip bot log prefix before upload. - init_kwargs["before_send"] = _before_send - - sentry_sdk.init(**init_kwargs) - _activity_tracking_active = use_activity_dsn + max_breadcrumbs=5, + environment=environment, + before_send=_before_send, + ) _sentry_initialized = True - mode = "activity" if use_activity_dsn else "error" logger.info( - f"Initialized {mode} tracking with environment: {environment}, " - f"release: {octobot.constants.LONG_VERSION}, dsn: {active_dsn}" + f"Initialized error tracking with environment: {environment}, " + f"release: {octobot.constants.LONG_VERSION}, dsn: {error_dsn}" ) sentry_sdk.set_context("app", { "app_start_time": str(round(time.time())), @@ -110,34 +87,6 @@ def flush_tracker() -> None: time.sleep(delay) -def activity_tracking_is_active() -> bool: - return _activity_tracking_active - - -def has_tracker_bot_id() -> bool: - return _tracker_bot_id_set - - -def update_tracker_bot_id(bot_id: str) -> None: - global _tracker_bot_id_set - _tracker_bot_id_set = True - sentry_sdk.set_user({"id": bot_id}) - sentry_sdk.set_tag("bot_id", bot_id) - - -def track_usage_event(event_name: str, **attributes: typing.Any) -> None: - metric_attributes = {"event": event_name} - for attribute_key, attribute_value in attributes.items(): - if attribute_value is not None: - metric_attributes[attribute_key] = str(attribute_value) - sentry_sdk.metrics.count("octobot.usage", 1, attributes=metric_attributes) - octobot_commons.logging.get_logger("sentry_tracker").debug( - "Tracked usage event %s with attributes %s", - event_name, - metric_attributes, - ) - - def _get_log_prefix() -> str: return f"[{octobot.constants.COMMUNITY_BOT_ID}] " if octobot.constants.COMMUNITY_BOT_ID else "[self-hosted]" diff --git a/octobot/community/history_backend/__init__.py b/octobot/community/history_backend/__init__.py index b5b3c51b44..fe51fed092 100644 --- a/octobot/community/history_backend/__init__.py +++ b/octobot/community/history_backend/__init__.py @@ -29,14 +29,8 @@ ClickhouseHistoricalBackendClient, ) -from octobot.community.history_backend import iceberg_historical_backend_client -from octobot.community.history_backend.iceberg_historical_backend_client import ( - IcebergHistoricalBackendClient, -) - __all__ = [ "history_backend_client", "HistoricalBackendClient", "ClickhouseHistoricalBackendClient", - "IcebergHistoricalBackendClient", ] diff --git a/octobot/community/history_backend/history_backend_factory.py b/octobot/community/history_backend/history_backend_factory.py index 236fb44fe9..f851550ae2 100644 --- a/octobot/community/history_backend/history_backend_factory.py +++ b/octobot/community/history_backend/history_backend_factory.py @@ -16,7 +16,6 @@ import contextlib import octobot.community.history_backend.clickhouse_historical_backend_client as clickhouse_historical_backend_client -import octobot.community.history_backend.iceberg_historical_backend_client as iceberg_historical_backend_client import octobot.enums @@ -41,8 +40,6 @@ def _create_client( async with history_backend_client(backend_type) as client: await client.xxxx() """ - if backend_type is octobot.enums.CommunityHistoricalBackendType.Iceberg: - return iceberg_historical_backend_client.IcebergHistoricalBackendClient(**kwargs) if backend_type is octobot.enums.CommunityHistoricalBackendType.Clickhouse: return clickhouse_historical_backend_client.ClickhouseHistoricalBackendClient(**kwargs) raise NotImplementedError(f"Unsupported historical backend type: {backend_type}") diff --git a/octobot/community/history_backend/iceberg_historical_backend_client.py b/octobot/community/history_backend/iceberg_historical_backend_client.py deleted file mode 100644 index 6ba1f8406c..0000000000 --- a/octobot/community/history_backend/iceberg_historical_backend_client.py +++ /dev/null @@ -1,982 +0,0 @@ -# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) -# Copyright (c) 2025 Drakkar-Software, All rights reserved. -# -# OctoBot is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# OctoBot is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public -# License along with OctoBot. If not, see . -import typing -import enum -import datetime -import os -import asyncio -import concurrent.futures -import logging -import json -import time -import dataclasses - -import octobot_commons.logging as commons_logging -import octobot_commons.os_util as os_util - -try: - if os_util.is_raspberry_pi_machine(): - raise ImportError("pyarrow is not available on Raspberry Pi") - else: - import pyarrow -except ImportError as err: - commons_logging.get_logger().info(f"Skipped pyarrow import: {err}") - class PyArrowMock(): - # mock to allow typing hints - Table = None - Schema = None - pyarrow = PyArrowMock() - -try: - import pyiceberg.catalog - import pyiceberg.schema - import pyiceberg.types - import pyiceberg.exceptions - import pyiceberg.expressions - import pyiceberg.table - import pyiceberg.table.sorting - import pyiceberg.table.statistics - import pyiceberg.table.update - import pyiceberg.table.refs -except ImportError as err: - commons_logging.get_logger().info(f"Skipped pyiceberg import: {err}") - class PyIcebergImportMock(): - # type hints mocks only - class catalog: - class Catalog: - def __init__(self, *args): - raise ImportError("pyiceberg not installed") - def load_catalog(self, *args, **kwargs): - raise ImportError("pyiceberg not installed") - class table: - class statistics: - class StatisticsFile: - def __init__(self, *args): - raise ImportError("pyiceberg not installed") - class BlobMetadata: - def __init__(self, *args): - raise ImportError("pyiceberg not installed") - class Table: - def __init__(self, *args): - raise ImportError("pyiceberg not installed") - class DataScan: - def __init__(self, *args): - raise ImportError("pyiceberg not installed") - class exceptions: - class NoSuchTableError(Exception): - def __init__(self, *args): - raise ImportError("pyiceberg not installed") - class schema: - class Schema: - def __init__(self, *args): - raise ImportError("pyiceberg not installed") - class expressions: - class BooleanExpression: - def __init__(self, *args): - raise ImportError("pyiceberg not installed") - class Or: - def __init__(self, *args): - raise ImportError("pyiceberg not installed") - class EqualTo: - def __init__(self, *args): - raise ImportError("pyiceberg not installed") - pyiceberg = PyIcebergImportMock() - -import octobot_commons.enums as commons_enums -import octobot.constants as constants -import octobot.community.history_backend.historical_backend_client as historical_backend_client -import octobot.community.history_backend.util as history_backend_util -import octobot_commons.os_util as os_util - - -# avoid "Loaded FileIO: pyiceberg.io.pyarrow.PyArrowFileIO" info logs -logging.getLogger("pyiceberg.io").setLevel(logging.WARNING) - - -class TableNames(enum.Enum): - OHLCV_HISTORY = constants.ICEBERG_OHLCV_HISTORY_TABLE - - -# pyiceberg itself uses a thread pool so each thread will create children threads, limit the number of "master" threads -_MAX_EXECUTOR_WORKERS = min(4, (os.cpu_count() or 1)) # use a max of 4 workers -_METADATA_SEPARATOR = "|" -_METADATA_VERSION = "1.0.0" -_MAX_STATISTICS_FILES = 10 # keep only the last X statistics files -_MAX_PENDING_BATCH_INSERT_SIZE = 500000 # max number of pending rows to store before inserting when enable_async_batch_inserts is True - -class _MetadataIdentifiers(enum.Enum): - UPDATED_AT = "updated_at" - VERSION = "version" - - -@dataclasses.dataclass -class _PendingInsertData: - data: typing.Iterable[list] - column_names: list[str] - - def get_columns_fingerprint(self) -> str: - return ",".join(self.column_names) - - -class IcebergHistoricalBackendClient(historical_backend_client.HistoricalBackendClient): - - def __init__(self, enable_async_batch_inserts: bool = True, **kwargs): - # enable_async_batch_inserts is used to avoid concurrent inserts, which are not properly supported by iceberg - if pyarrow.Table is None: - raise ImportError(f"The pyarrow dependency is required to use {self.__class__.__name__}") - self.enable_async_batch_inserts: bool = enable_async_batch_inserts - self.namespace: typing.Optional[str] = None - self.catalog: pyiceberg.catalog.Catalog = None # type: ignore - self._executor: typing.Optional[concurrent.futures.ThreadPoolExecutor] = None # only availble when client is open - self._fetching_candles_history_range: asyncio.Lock() = None # type: ignore - self._inserting_pending_data: asyncio.Lock() = None # type: ignore - self._fetched_min_max_per_symbol_per_time_frame_per_exchange: typing.Optional[ - dict[str, dict[str, dict[str, tuple[float, float]]]] - ] = None - self._updated_min_max_per_symbol_per_time_frame_per_exchange: typing.Optional[ - dict[str, dict[str, dict[str, tuple[float, float]]]] - ] = None - self._pending_insert_data_by_table: dict[TableNames, list[_PendingInsertData]] = {} - - async def open(self): - try: - self._fetched_min_max_per_symbol_per_time_frame_per_exchange = None - self._updated_min_max_per_symbol_per_time_frame_per_exchange = None - self._fetching_candles_history_range = asyncio.Lock() - self._inserting_pending_data = asyncio.Lock() - self._executor = concurrent.futures.ThreadPoolExecutor( - max_workers=_MAX_EXECUTOR_WORKERS, - thread_name_prefix="IcebergHistoricalBackendClient" - ) - self.catalog = self._load_catalog() - except Exception as err: - message = f"Error when connecting to Iceberg server, {err.__class__.__name__}: {err}" - self._get_logger().exception(err, True, message) - raise err.__class__(message) from err - - async def close(self): - if self._executor is not None: - if self.enable_async_batch_inserts: - async with self._inserting_pending_data: - await self._insert_and_reset_pending_data() - if self._has_metadata_to_update(): - # should always be called when metadata have to be updated - # otherwise their update will be lost and candle ranges won't be updated - await self._update_metadata() - self._updated_min_max_per_symbol_per_time_frame_per_exchange = None - self._fetched_min_max_per_symbol_per_time_frame_per_exchange = None - self._executor.shutdown() - self._fetching_candles_history_range = None - self._inserting_pending_data = None - self._executor = None - - async def _run_in_executor(self, func, *args): - if self._executor is None: - raise ValueError(f"{self.__class__.__name__} is not open") - # run blocking IO code in a dedicated thread poool executor that is shutdown on close - # to avoid leaving threads alive when the client is closed - return await asyncio.get_running_loop().run_in_executor(self._executor, func, *args) - - async def fetch_candles_history( - self, - exchange: str, - symbol: str, - time_frame: commons_enums.TimeFrames, - first_open_time: float, - last_open_time: float - ) -> list[list[float]]: - return await self._run_in_executor( - self._sync_fetch_candles_history, - exchange, symbol, time_frame, None, None, first_open_time, last_open_time, - False - ) - - async def fetch_extended_candles_history( - self, - exchange: str, - symbols: list[str], - time_frames: list[commons_enums.TimeFrames], - first_open_time: typing.Optional[float] = None, - last_open_time: typing.Optional[float] = None, - ) -> list[list[typing.Union[float, str]]]: - return await self._run_in_executor( - self._sync_fetch_candles_history, - exchange, None, None, symbols, time_frames, first_open_time, last_open_time, - True - ) - - def _get_filter( - self, element: str, value: typing.Union[None, str, list[str]] - ) -> typing.Optional[pyiceberg.expressions.BooleanExpression]: - if value is None: - return None - if isinstance(value, list): - if len(value) > 1: - return pyiceberg.expressions.Or(*(pyiceberg.expressions.EqualTo(element, _value) for _value in value)) - return pyiceberg.expressions.EqualTo(element, value[0]) - return pyiceberg.expressions.EqualTo(element, value) - - - def _sync_fetch_candles_history( - self, - exchange: str, - symbol: typing.Optional[str], - time_frame: typing.Optional[commons_enums.TimeFrames], - symbols: typing.Optional[list[str]], - time_frames: typing.Optional[list[commons_enums.TimeFrames]], - first_open_time: typing.Optional[float], - last_open_time: typing.Optional[float], - extended: bool, - ) -> list[list[typing.Union[float, str]]]: - table = self._get_or_create_table(TableNames.OHLCV_HISTORY) - and_filters = [ - pyiceberg.expressions.EqualTo("exchange_internal_name", exchange), - ] - if tf_filter := self._get_filter( - "time_frame", - [tf.value for tf in time_frames] if time_frames else time_frame.value if time_frame else None - ): - and_filters.append(tf_filter) - if symbol_filter := self._get_filter("symbol", symbol or symbols): - and_filters.append(symbol_filter) - if first_open_time: - and_filters.append(pyiceberg.expressions.GreaterThanOrEqual("timestamp", self.get_formatted_time(first_open_time))) - if last_open_time: - and_filters.append(pyiceberg.expressions.LessThanOrEqual("timestamp", self.get_formatted_time(last_open_time))) - filter = pyiceberg.expressions.And(*and_filters) # pylint: disable=no-value-for-parameter - selected_fields = ["timestamp", "open", "high", "low", "close", "volume"] - if extended: - selected_fields = ["time_frame", "symbol"] + selected_fields - result = table.scan( - row_filter=filter, - selected_fields=selected_fields, - case_sensitive=True, - ) - - formatted = self._format_ohlcvs(result.to_arrow(), extended) - # ensure no duplicates as they can happen due to no unicity constraint - return history_backend_util.deduplicate(formatted, [0, 1, 2] if extended else [0]) - - async def fetch_candles_history_range( - self, - exchange: str, - symbol: str, - time_frame: commons_enums.TimeFrames, - ) -> tuple[float, float]: - if self._fetched_min_max_per_symbol_per_time_frame_per_exchange is None: - async with self._fetching_candles_history_range: - # now that lock is acquired, check again if metadata still needs to be fetched - if self._fetched_min_max_per_symbol_per_time_frame_per_exchange is None: - # fetches table metadata: do it in executor to avoid blocking the main thread - table = await self._run_in_executor( - self._get_or_create_table, TableNames.OHLCV_HISTORY - ) - self._fetched_min_max_per_symbol_per_time_frame_per_exchange = self._parse_min_max_per_symbol_per_time_frame_per_exchange( - table - ) - if not self._fetched_min_max_per_symbol_per_time_frame_per_exchange: - return (0, 0) - - return self._get_latest_min_max_per_symbol_per_time_frame_per_exchange(exchange, symbol, time_frame) - - def _get_latest_min_max_per_symbol_per_time_frame_per_exchange( - self, exchange: str, symbol: str, time_frame: commons_enums.TimeFrames - ) -> tuple[float, float]: - if self._updated_min_max_per_symbol_per_time_frame_per_exchange and ( - min_max := self._updated_min_max_per_symbol_per_time_frame_per_exchange.get(exchange, {}).get(symbol, {}).get(time_frame.value) - ): - return min_max - if not self._fetched_min_max_per_symbol_per_time_frame_per_exchange: - return (0, 0) - return self._fetched_min_max_per_symbol_per_time_frame_per_exchange.get(exchange, {}).get(symbol, {}).get(time_frame.value) or (0, 0) - - async def fetch_all_candles_for_exchange(self, exchange: str) -> list[list[typing.Union[float, str]]]: - return await self._run_in_executor( - self._sync_fetch_all_candles_for_exchange, - exchange - ) - - def _sync_fetch_all_candles_for_exchange(self, exchange: str) -> list[list[typing.Union[float, str]]]: - table = self._get_or_create_table(TableNames.OHLCV_HISTORY) - selected_fields = ("timestamp", "symbol", "time_frame", "open", "high", "low", "close", "volume") - scan = self._scan_all_candles_for_exchange(table, exchange, selected_fields) - return self._format_ohlcvs(scan.to_arrow(), True) - - async def insert_candles_history(self, rows: list, column_names: list) -> None: - if self.enable_async_batch_inserts: - await self._async_insert(TableNames.OHLCV_HISTORY, rows, column_names) - else: - await self._run_in_executor( - self._sync_insert_candles_history, - rows, column_names - ) - - def _sync_insert_candles_history(self, rows: typing.Iterable[list], column_names: list[str]) -> None: - # warning: try not to insert duplicate candles, - # however duplicates will be deduplicated later on anyway - if not rows: - return - schema = self._pyarrow_get_ohlcv_schema() - table = self._get_or_create_table(TableNames.OHLCV_HISTORY) - pa_arrays = [ - pyarrow.array([ - row[i] - for row in rows - ]) - for i, _ in enumerate(column_names) - ] - pa_table = pyarrow.Table.from_arrays(pa_arrays, schema=schema) - try: - table.append(pa_table) - # note: alternative upsert syntax could prevent duplicates but is really slow and silentlycrashes the process - # when used with a few thousand rows - # table.upsert(pa_table, join_cols=["timestamp", "exchange_internal_name", "symbol", "time_frame"]) - except pyiceberg.exceptions.CommitFailedException as err: - # if this happens, it means there are conflicts. Log error and let maintenance - # perform the rollback to previous snapshot - self._get_logger().exception( - err, - True, - f"Commit failed: conflicts. Rolling back to previous snapshot might fix this {err}" - ) - raise - # now that candles have been inserted, update metadata - self._register_updated_min_max(table, pa_table) - self._get_logger().info( - f"Successfully inserted {len(rows)} rows into " - f"{TableNames.OHLCV_HISTORY.value}: {self._get_candles_summary(pa_table)}" - ) - - async def _async_insert( - self, table_name: TableNames, rows: typing.Iterable[list], column_names: list[str] - ) -> None: - if table_name not in self._pending_insert_data_by_table: - self._pending_insert_data_by_table[table_name] = [] - self._pending_insert_data_by_table[table_name].append(_PendingInsertData(rows, column_names)) - self._get_logger().info(f"Registered {len(rows)} rows to insert into {table_name.value}") - await self._insert_pending_data_if_necessary() - - async def _insert_pending_data_if_necessary(self) -> None: - if self.enable_async_batch_inserts: - async with self._inserting_pending_data: - pending_rows_count = sum( - len(data.data) - for pending_data_list in self._pending_insert_data_by_table.values() - for data in pending_data_list - ) - if pending_rows_count >= _MAX_PENDING_BATCH_INSERT_SIZE: - self._get_logger().info(f"Inserting {pending_rows_count} pending rows") - await self._insert_and_reset_pending_data() - - async def _insert_and_reset_pending_data(self): - to_insert_pending_data = self._pending_insert_data_by_table - # reset pending data - self._pending_insert_data_by_table = {} - for table_name, pending_data in to_insert_pending_data.items(): - pending_rows_count = sum( - len(data.data) - for data in pending_data - ) - self._get_logger().info(f"Inserting {table_name.value} {pending_rows_count} pending rows") - await self._run_in_executor(self._sync_insert_table_pending_data, table_name, pending_data) - - def _sync_insert_table_pending_data(self, table_name: TableNames, pending_data: list[_PendingInsertData]): - # group pending data by columns to insert compatible ones all at once - pending_data_by_columns_fingerprint: dict[str, list[_PendingInsertData]] = {} - for data in pending_data: - columns_fingerprint = data.get_columns_fingerprint() - if columns_fingerprint not in pending_data_by_columns_fingerprint: - pending_data_by_columns_fingerprint[columns_fingerprint] = [] - pending_data_by_columns_fingerprint[columns_fingerprint].append(data) - if table_name.name == TableNames.OHLCV_HISTORY.name: - for pending_data_list in pending_data_by_columns_fingerprint.values(): - rows = list(pending_data_list[0].data) - for other_rows in pending_data_list[1:]: - rows.extend(other_rows.data) - column_names = pending_data_list[0].column_names - self._get_logger().info(f"Inserting {len(rows)} rows into {table_name.value} (columns: {column_names})") - self._sync_insert_candles_history(rows, column_names) - else: - raise NotImplementedError(f"Inserting pending data for table {table_name.value} is not implemented") - - @staticmethod - def get_formatted_time(timestamp: float) -> str: - return datetime.datetime.fromtimestamp( - timestamp, tz=datetime.timezone.utc - ).isoformat(sep='T').replace("+00:00", "") - - @staticmethod - def _format_ohlcvs(ohlcvs_table: pyarrow.Table, extended: bool) -> list[list[typing.Union[float, str]]]: - # ohlcv are fetched in "random" order, sort them before switching to python objects - batches = ohlcvs_table.sort_by([("timestamp", "ascending")]).to_batches() - ohlcvs = [ - # time frame - # symbol - # then uses PriceIndexes order - # IND_PRICE_TIME = 0 - # IND_PRICE_OPEN = 1 - # IND_PRICE_HIGH = 2 - # IND_PRICE_LOW = 3 - # IND_PRICE_CLOSE = 4 - # IND_PRICE_VOL = 5 - [tf, s, history_backend_util.get_utc_timestamp_from_datetime(t), o, h, l, c, v] - for batch in batches - if (batch_dict := batch.to_pydict()) - for tf, s, t, o, h, l, c, v in zip( - batch_dict['time_frame'], batch_dict['symbol'], batch_dict['timestamp'], batch_dict['open'], batch_dict['high'], batch_dict['low'], batch_dict['close'], batch_dict['volume'] - ) - ] if extended else [ - # uses PriceIndexes order - # IND_PRICE_TIME = 0 - # IND_PRICE_OPEN = 1 - # IND_PRICE_HIGH = 2 - # IND_PRICE_LOW = 3 - # IND_PRICE_CLOSE = 4 - # IND_PRICE_VOL = 5 - [history_backend_util.get_utc_timestamp_from_datetime(t), o, h, l, c, v] - for batch in batches - if (batch_dict := batch.to_pydict()) - for t, o, h, l, c, v in zip( - batch_dict['timestamp'], batch_dict['open'], batch_dict['high'], batch_dict['low'], batch_dict['close'], batch_dict['volume'] - ) - ] - return ohlcvs - - def _load_catalog(self) -> pyiceberg.catalog.Catalog: - self.namespace = constants.ICEBERG_CATALOG_NAMESPACE - catalog = pyiceberg.catalog.load_catalog(constants.ICEBERG_CATALOG_NAME, **self._get_catalog_properties()) - if constants.CREATE_ICEBERG_DB_IF_MISSING: - self._ensure_namespace(catalog) - self._get_logger().info(f"PyIceberg catalog '{constants.ICEBERG_CATALOG_NAME}' successfully loaded") - return catalog - - def drop_table(self, table_name: str) -> None: - self.catalog.purge_table(f"{self.namespace}.{table_name}") - self._get_logger().info(f"Table {table_name} deleted successfully") - - async def resynchronize_min_max_per_symbol_per_time_frame_per_exchange_metadata(self, exchanges: list[str]): - # Maintenance operation - # To use if db metadata got deleted or need to be recreated from scratch for any reason. - # Warning: will sequentially scan the olhcv history tables for all exchanges, requires a lot of ram - # (usually around 1 GB) - self._updated_min_max_per_symbol_per_time_frame_per_exchange = await self._run_in_executor( - self._sync_get_min_max_per_symbol_per_time_frame_per_exchange_metadata, - exchanges - ) - - def _sync_get_min_max_per_symbol_per_time_frame_per_exchange_metadata( - self, exchanges: list[str] - ) -> dict[str, dict[str, dict[str, tuple[float, float]]]]: - updated_metadata = {} - global_t0 = time.time() - table = self._get_or_create_table(TableNames.OHLCV_HISTORY) - selected_fields = ("timestamp", "exchange_internal_name", "symbol", "time_frame") - for exchange in exchanges: - t0 = time.time() - self._update_min_max_per_symbol_per_time_frame_per_exchange_for_table( - self._scan_all_candles_for_exchange(table, exchange, selected_fields).to_arrow(), updated_metadata - ) - used_ram = os_util.get_cpu_and_ram_usage(0)[3] - self._get_logger().info(f"Updated metadata for {exchange} in {time.time() - t0} seconds, using {used_ram} GB of RAM") - self._get_logger().info( - f"Total time to update metadata for {len(exchanges)} exchanges: {exchange} = {len(exchanges)}: {time.time() - global_t0} seconds" - ) - return updated_metadata - - async def _update_metadata(self) -> None: - await self._run_in_executor(self._sync_update_metadata) - - def _get_snapshot_id(self, table: pyiceberg.table.Table) -> int: - return table.current_snapshot().snapshot_id if table.current_snapshot() else 0 - - def _sync_update_metadata(self) -> None: - ohlcv_table = self._get_or_create_table(TableNames.OHLCV_HISTORY) - # Get current snapshot ID from the table - current_snapshot_id = self._get_snapshot_id(ohlcv_table) - - blob_metadata_list = self._create_blob_metadata_list(current_snapshot_id, ohlcv_table) - - # Calculate total file size (approximate) - total_data_size = sum(len(json.dumps(blob.properties or {})) for blob in blob_metadata_list) - - # Create a unique statistics file path - statistics_path = f"{ohlcv_table.location()}/statistics/{current_snapshot_id}.json" - - # Create the StatisticsFile with the blob_metadata - # Using field names (Pydantic will map to aliases automatically) - statistics_file = pyiceberg.table.statistics.StatisticsFile( - snapshot_id=current_snapshot_id, # type: ignore - statistics_path=statistics_path, # type: ignore - file_size_in_bytes=total_data_size, # type: ignore # warning: required to store all metadata blobs - file_footer_size_in_bytes=total_data_size, # type: ignore # warning: required to store all metadata blobs - key_metadata=None, # type: ignore - blob_metadata=blob_metadata_list, # type: ignore - ) - t0 = time.time() - self._get_logger().info(f"Updating statistics for {ohlcv_table.name()} with {len(blob_metadata_list)} blob metadata") - self._update_statistics(ohlcv_table, statistics_file) - self._get_logger().info( - f"Completed updating statistics for {ohlcv_table.name()} with {len(blob_metadata_list)} " - f"blob metadata in {round(time.time() - t0, 2)} seconds" - ) - - def _scan_all_candles_for_exchange(self, table: pyiceberg.table.Table, exchange: str, selected_fields: tuple[str]) -> pyiceberg.table.DataScan: - filter = pyiceberg.expressions.EqualTo("exchange_internal_name", exchange) - return table.scan( - row_filter=filter, - selected_fields=selected_fields, - case_sensitive=True, - ) - - def _update_statistics( - self, table: pyiceberg.table.Table, statistics_file: pyiceberg.table.statistics.StatisticsFile - ) -> None: - # only keep the last _MAX_STATISTICS_FILES most recent statistics files to avoid growing metadata size forever - # -1 as we are also adding a new statistics file - to_remove_statistics_files = self._get_recent_to_older_statistics_files(table)[(_MAX_STATISTICS_FILES - 1):] - # each update can only process 1 operation - for to_delete_statistics_file in to_remove_statistics_files: - with table.update_statistics() as update: - self._get_logger().info( - f"Clearing statistics for {table.name()} with snapshot id: {to_delete_statistics_file.snapshot_id} to free up space." - ) - update.remove_statistics(snapshot_id=to_delete_statistics_file.snapshot_id) - with table.update_statistics() as update: - self._get_logger().info( - f"Updating statistics for {table.name()} with snapshot id: {statistics_file.snapshot_id}" - ) - update.set_statistics(statistics_file=statistics_file) - - def _get_latest_statistics_file_range_metadata( - self, table: pyiceberg.table.Table - ) -> pyiceberg.table.statistics.BlobMetadata: - range_metadata = None - for metadata in self._get_latest_statistics_file(table).blob_metadata: - if version := metadata.properties.get(_MetadataIdentifiers.VERSION.value): - # ensure version compatibility - if version == _METADATA_VERSION: - # version is compatible - pass - else: - raise ValueError( - f"Latest statistics file version {version} is not compatible with current version {_METADATA_VERSION}" - ) - else: - range_metadata = metadata - if range_metadata is None: - raise ValueError( - f"No range metadata found for table {table.name()}" - ) - return range_metadata - - def _get_latest_statistics_file( - self, table: pyiceberg.table.Table - ) -> pyiceberg.table.statistics.StatisticsFile: - return self._get_recent_to_older_statistics_files(table)[0] - - def _get_recent_to_older_statistics_files( - self, table: pyiceberg.table.Table - ) -> list[pyiceberg.table.statistics.StatisticsFile]: - return sorted( - table.metadata.statistics, - key=lambda x: -self._get_statistic_file_created_at(x) - ) - - def _get_statistic_file_created_at( - self, statistics_file: pyiceberg.table.statistics.StatisticsFile - ) -> float: - for blob_metadata in statistics_file.blob_metadata: - if updated_at := blob_metadata.properties.get(_MetadataIdentifiers.UPDATED_AT.value): - return float(updated_at) - return 0 - - def _create_blob_metadata_list( - self, current_snapshot_id: int, ohlcv_table: pyiceberg.table.Table - ) -> list[pyiceberg.table.statistics.BlobMetadata]: - current_min_max_per_symbol_per_time_frame_per_exchange = self._parse_min_max_per_symbol_per_time_frame_per_exchange( - ohlcv_table - ) - updated_min_max = { - # keep previous min max for exchanges that were not updated, replace others - **current_min_max_per_symbol_per_time_frame_per_exchange, - **(self._updated_min_max_per_symbol_per_time_frame_per_exchange or {}) - } - # Create BlobMetadata objects for all exchange/symbol/timeframe combinations - updated_properties = { - f"{exchange}{_METADATA_SEPARATOR}{symbol}{_METADATA_SEPARATOR}{time_frame}": - f"{min_timestamp}{_METADATA_SEPARATOR}{max_timestamp}" - for exchange, symbol_data in updated_min_max.items() - for symbol, time_frame_data in symbol_data.items() - for time_frame, (min_timestamp, max_timestamp) in time_frame_data.items() - } - - # Get field IDs from the table schema for timestamp field (field ID 1) - timestamp_field_id = 1 # Based on the schema definition in _get_ohlcv_schema() - blob_metadata_list = [ - pyiceberg.table.statistics.BlobMetadata( - # actual data - type="apache-datasketches-theta-v1", - snapshot_id=current_snapshot_id, # type: ignore - sequence_number=0, # type: ignore - fields=[timestamp_field_id], - properties=updated_properties - ), - pyiceberg.table.statistics.BlobMetadata( - # to keep track of the latest update in case snapshot id is not enough - type="apache-datasketches-theta-v1", - snapshot_id=current_snapshot_id, # type: ignore - sequence_number=0, # type: ignore - fields=[timestamp_field_id], - properties={ - _MetadataIdentifiers.UPDATED_AT.value: str(int(time.time())), - _MetadataIdentifiers.VERSION.value: _METADATA_VERSION, - } - ), - ] - return blob_metadata_list - - def _parse_min_max_per_symbol_per_time_frame_per_exchange( - self, ohlcv_table: pyiceberg.table.Table - ) -> dict[str, dict[str, dict[str, tuple[float, float]]]]: - try: - properties = self._get_latest_statistics_file_range_metadata(ohlcv_table).properties - except ValueError: - # should never happen, raise if it does - raise - except IndexError: - # no metadata found - return {} - min_max_per_symbol_per_time_frame_per_exchange = {} - for key, values in properties.items(): - exchange, symbol, time_frame = key.split(_METADATA_SEPARATOR) - if exchange not in min_max_per_symbol_per_time_frame_per_exchange: - min_max_per_symbol_per_time_frame_per_exchange[exchange] = {} - if symbol not in min_max_per_symbol_per_time_frame_per_exchange[exchange]: - min_max_per_symbol_per_time_frame_per_exchange[exchange][symbol] = {} - min_max_per_symbol_per_time_frame_per_exchange[exchange][symbol][time_frame] = tuple(map(float, values.split(_METADATA_SEPARATOR))) - return min_max_per_symbol_per_time_frame_per_exchange - - def _register_updated_min_max(self, table: pyiceberg.table.Table, update_table: pyarrow.Table) -> None: - if self._updated_min_max_per_symbol_per_time_frame_per_exchange is None: - self._updated_min_max_per_symbol_per_time_frame_per_exchange = self._parse_min_max_per_symbol_per_time_frame_per_exchange( - table - ) - self._update_min_max_per_symbol_per_time_frame_per_exchange_for_table( - update_table, self._updated_min_max_per_symbol_per_time_frame_per_exchange - ) - - def _get_candles_summary(self, table: pyarrow.Table) -> dict[str, dict[str, int]]: - grouped_result = table.group_by( - ["exchange_internal_name", "symbol", "time_frame"] - ).aggregate([ - ("timestamp", "count"), - ]) - summary = {} - for exchange, symbol, time_frame, count in zip( - grouped_result['exchange_internal_name'], grouped_result['symbol'], grouped_result['time_frame'], grouped_result['timestamp_count'] - ): - py_exchange = exchange.as_py() # type: ignore - py_symbol = symbol.as_py() # type: ignore - py_time_frame = time_frame.as_py() # type: ignore - if py_exchange not in summary: - summary[py_exchange] = {} - if py_symbol not in summary[py_exchange]: - summary[py_exchange][py_symbol] = {} - summary[py_exchange][py_symbol][py_time_frame] = count.as_py() # type: ignore - return summary - - @staticmethod - def _update_min_max_per_symbol_per_time_frame_per_exchange_for_table( - update_table: pyarrow.Table, - current_min_max_per_symbol_per_time_frame_per_exchange: dict[str, dict[str, dict[str, tuple[float, float]]]] - ): - grouped_result = update_table.group_by( - ["exchange_internal_name", "symbol", "time_frame"] - ).aggregate([ - ("timestamp", "hash_min"), - ("timestamp", "hash_max") - ]) - - update_min_max_per_symbol_per_time_frame_per_exchange = {} - for exchange, symbol, time_frame, ts_min, ts_max in zip( - grouped_result['exchange_internal_name'], grouped_result['symbol'], grouped_result['time_frame'], grouped_result['timestamp_min'], grouped_result['timestamp_max'] - ): - py_exchange = exchange.as_py() # type: ignore - py_symbol = symbol.as_py() # type: ignore - py_time_frame = time_frame.as_py() # type: ignore - if py_exchange not in update_min_max_per_symbol_per_time_frame_per_exchange: - update_min_max_per_symbol_per_time_frame_per_exchange[py_exchange] = {} - if py_symbol not in update_min_max_per_symbol_per_time_frame_per_exchange[py_exchange]: - update_min_max_per_symbol_per_time_frame_per_exchange[py_exchange][py_symbol] = {} - update_min_max_per_symbol_per_time_frame_per_exchange[py_exchange][py_symbol][py_time_frame] = ( - history_backend_util.get_utc_timestamp_from_datetime(ts_min.as_py()), - history_backend_util.get_utc_timestamp_from_datetime(ts_max.as_py()) - ) - # save updated min max per symbol per time frame per exchange in current_min_max_per_symbol_per_time_frame_per_exchange - for exchange, symbol_data in update_min_max_per_symbol_per_time_frame_per_exchange.items(): - if exchange not in current_min_max_per_symbol_per_time_frame_per_exchange: - current_min_max_per_symbol_per_time_frame_per_exchange[exchange] = symbol_data - else: - current_exchange_data = current_min_max_per_symbol_per_time_frame_per_exchange[exchange] - for symbol, time_frame_data in symbol_data.items(): - if symbol not in current_exchange_data: - current_exchange_data[symbol] = time_frame_data - else: - current_symbol_data = current_exchange_data[symbol] - for time_frame, (min_timestamp, max_timestamp) in time_frame_data.items(): - if time_frame not in current_symbol_data: - current_symbol_data[time_frame] = (min_timestamp, max_timestamp) - else: - min_ts = min(min_timestamp, current_symbol_data[time_frame][0]) - max_ts = max(max_timestamp, current_symbol_data[time_frame][1]) - current_symbol_data[time_frame] = (min_ts, max_ts) - - def _get_or_create_table(self, table_name: TableNames) -> pyiceberg.table.Table: - try: - return self.catalog.load_table(f"{self.namespace}.{table_name.value}") - except pyiceberg.exceptions.NoSuchTableError: - table_by_ns = {ns: self.catalog.list_tables(ns) for ns in self.catalog.list_namespaces()} - self._get_logger().info( - f"Table {table_name.value} does not exist in {self.namespace} namespace. Tables by namespace: {table_by_ns}" - ) - if constants.CREATE_ICEBERG_DB_IF_MISSING: - self.catalog.create_table( - identifier=f"{self.namespace}.{table_name.value}", - schema=self._get_schema_for_table(table_name), - sort_order=pyiceberg.table.sorting.SortOrder( - pyiceberg.table.sorting.SortField( - source_id=1, - direction=pyiceberg.table.sorting.SortDirection.DESC, # warning: not applied in selects - ) - ) - ) - self._get_logger().info(f"Table {table_name.value} successfully created") - return self.catalog.load_table(f"{self.namespace}.{table_name.value}") - raise - - def _ensure_namespace(self, catalog: pyiceberg.catalog.Catalog): - # Check if namespace exists, create if not - namespaces = [ns[0] for ns in catalog.list_namespaces()] - if self.namespace not in namespaces: - catalog.create_namespace(self.namespace) - self._get_logger().info(f"Namespace {self.namespace} created") - - @staticmethod - def _get_catalog_properties() -> dict: - catalog_properties = { - "type": "rest", - "uri": constants.ICEBERG_CATALOG_URI, - "warehouse": constants.ICEBERG_CATALOG_WAREHOUSE, - "py-io-impl": "pyiceberg.io.pyarrow.PyArrowFileIO", - "s3.force-virtual-addressing": False, - } - if constants.ICEBERG_CATALOG_TOKEN: - catalog_properties["token"] = constants.ICEBERG_CATALOG_TOKEN - if constants.ICEBERG_S3_ACCESS_KEY and constants.ICEBERG_S3_SECRET_KEY: - catalog_properties.update({ - "s3.access-key-id": constants.ICEBERG_S3_ACCESS_KEY, - "s3.secret-access-key": constants.ICEBERG_S3_SECRET_KEY, - "s3.region": constants.ICEBERG_S3_REGION, - }) - - if constants.ICEBERG_S3_ENDPOINT: - catalog_properties["s3.endpoint"] = constants.ICEBERG_S3_ENDPOINT - return catalog_properties - - @staticmethod - def _get_schema_for_table(table_name: TableNames) -> typing.Union[pyarrow.Schema, pyiceberg.schema.Schema]: - if table_name is TableNames.OHLCV_HISTORY: - return IcebergHistoricalBackendClient._get_ohlcv_schema() - raise ValueError(f"No schema found for table '{table_name}'. Available schemas: {list(TableNames)}") - - @staticmethod - def _get_ohlcv_schema() -> pyiceberg.schema.Schema: - """Schema for OHLCV data""" - return pyiceberg.schema.Schema( - pyiceberg.types.NestedField(1, "timestamp", pyiceberg.types.TimestampType(), required=True), - pyiceberg.types.NestedField(2, "exchange_internal_name", pyiceberg.types.StringType(), required=True), - pyiceberg.types.NestedField(3, "symbol", pyiceberg.types.StringType(), required=True), - pyiceberg.types.NestedField(4, "time_frame", pyiceberg.types.StringType(), required=True), - pyiceberg.types.NestedField(5, "open", pyiceberg.types.DoubleType(), required=True), - pyiceberg.types.NestedField(6, "high", pyiceberg.types.DoubleType(), required=True), - pyiceberg.types.NestedField(7, "low", pyiceberg.types.DoubleType(), required=True), - pyiceberg.types.NestedField(8, "close", pyiceberg.types.DoubleType(), required=True), - pyiceberg.types.NestedField(9, "volume", pyiceberg.types.DoubleType(), required=True), - ) - @staticmethod - def _pyarrow_get_ohlcv_schema() -> pyarrow.Schema: - """Schema for OHLCV data""" - # constructor is pyarrow.schema([...]), not pyarrow.Schema([...]) - return pyarrow.schema([ - pyarrow.field("timestamp", pyarrow.timestamp("us"), False), # Adjust precision as needed (e.g., "ms" for milliseconds) - pyarrow.field("exchange_internal_name", pyarrow.string(), False), - pyarrow.field("symbol", pyarrow.string(), False), - pyarrow.field("time_frame", pyarrow.string(), False), - pyarrow.field("open", pyarrow.float64(), False), - pyarrow.field("high", pyarrow.float64(), False), - pyarrow.field("low", pyarrow.float64(), False), - pyarrow.field("close", pyarrow.float64(), False), - pyarrow.field("volume", pyarrow.float64(), False), - ]) - - @classmethod - def _get_logger(cls): - return commons_logging.get_logger(cls.__name__) - - def _has_metadata_to_update(self) -> bool: - return bool(self._updated_min_max_per_symbol_per_time_frame_per_exchange) - - async def rollback_to_previous_snapshot( - self, table_name: TableNames, snapshot_id: typing.Optional[int] = None - ) -> int: - return await self._run_in_executor( - self._sync_rollback_to_previous_snapshot, table_name, snapshot_id - ) - - def _sync_rollback_to_previous_snapshot( - self, table_name: TableNames, snapshot_id: typing.Optional[int] = None - ) -> int: - """Synchronous implementation of rollback_to_previous_snapshot""" - table = self._get_or_create_table(table_name) - - # Get the snapshot history - history = table.history() - - if not history: - raise ValueError(f"No snapshot history found for table {table_name.value}") - - # Sort snapshots by timestamp (most recent first) - sorted_snapshots = sorted( - history, - key=lambda s: s.timestamp_ms, - reverse=True - ) - - if snapshot_id is not None: - # Rollback to specific snapshot - target_snapshot_id = snapshot_id - if not any(s.snapshot_id == snapshot_id for s in sorted_snapshots): - raise ValueError( - f"Snapshot ID {snapshot_id} not found in table {table_name.value} history" - ) - else: - # Rollback to previous snapshot (second most recent) - if len(sorted_snapshots) < 2: - raise ValueError( - f"No previous snapshot to rollback to for table {table_name.value}. " - f"Only {len(sorted_snapshots)} snapshot(s) available." - ) - # sorted_snapshots[0] is the current snapshot, [1] is the previous one - target_snapshot_id = sorted_snapshots[1].snapshot_id - - # Perform the rollback by updating the main branch to point to the target snapshot - # This is the PyIceberg way to "rollback" - you update the branch reference to a previous snapshot - self._get_logger().info( - f"Rolling back table {table_name.value} to snapshot {target_snapshot_id}" - ) - - # Use transaction to update the main branch reference to the target snapshot - # We create the update directly to avoid the AssertRefSnapshotId requirement that causes 409 conflicts - with table.transaction() as txn: - # Create the SetSnapshotRefUpdate directly without assertion requirements - update = pyiceberg.table.update.SetSnapshotRefUpdate( - ref_name=pyiceberg.table.refs.MAIN_BRANCH, - type=pyiceberg.table.refs.SnapshotRefType.BRANCH, - snapshot_id=target_snapshot_id, - ) - txn._updates += (update,) - # Warning: don't add requirements - this allows the update to succeed even if there are concurrent changes - # (otherwise, the update will fail with a 409 conflict in case the target branch is corrupted) - - self._get_logger().info( - f"Successfully rolled back table {table_name.value} to snapshot {target_snapshot_id}" - ) - - return target_snapshot_id - - - async def cleanup_old_snapshots_and_branches( - self, - table_name: TableNames, - older_than_s: int, - branches_to_delete: typing.Optional[list[str]] = None - ) -> tuple[int, int]: - return await self._run_in_executor( - self._sync_cleanup_old_snapshots_and_branches, - table_name, - older_than_s, - branches_to_delete - ) - - def _sync_cleanup_old_snapshots_and_branches( - self, - table_name: TableNames, - older_than_s: float, - branches_to_delete: typing.Optional[list[str]] = None - ) -> tuple[int, int]: - """Synchronous implementation of cleanup_old_snapshots_and_branches""" - table = self._get_or_create_table(table_name) - snapshots_expired = 0 - branches_deleted = 0 - - table.maintenance.expire_snapshots().older_than( - datetime.datetime.fromtimestamp( - older_than_s, tz=datetime.timezone.utc - ) - ).commit() - - # Delete branches if branches_to_delete is provided - if branches_to_delete: - # Refresh table metadata to get current branches - table.refresh() - current_refs = table.metadata.refs - - # Filter to only delete branches that exist and are not the main branch - valid_branches_to_delete = [ - branch for branch in branches_to_delete - if branch in current_refs and branch != pyiceberg.table.refs.MAIN_BRANCH - ] - - if valid_branches_to_delete: - self._get_logger().info( - f"Deleting {len(valid_branches_to_delete)} branches for table {table_name.value}: " - f"{', '.join(valid_branches_to_delete)}" - ) - - # Delete branches one by one using manage_snapshots - with table.manage_snapshots() as ms: - for branch in valid_branches_to_delete: - ms.remove_branch(branch_name=branch) - - branches_deleted = len(valid_branches_to_delete) - self._get_logger().info( - f"Successfully deleted {branches_deleted} branches for table {table_name.value}" - ) - elif branches_to_delete: - # Log warning if some branches were requested but don't exist or are protected - invalid_branches = [b for b in branches_to_delete if b not in current_refs or b == pyiceberg.table.refs.MAIN_BRANCH] - if invalid_branches: - self._get_logger().warning( - f"Skipped deletion of {len(invalid_branches)} branches (non-existent or protected): " - f"{', '.join(invalid_branches)}" - ) - - self._get_logger().info( - f"Cleanup complete for table {table_name.value}: " - f"{snapshots_expired} snapshots expired, {branches_deleted} branches deleted" - ) - - return snapshots_expired, branches_deleted \ No newline at end of file diff --git a/octobot/community/node_journal/AGENTS.md b/octobot/community/node_journal/AGENTS.md new file mode 100644 index 0000000000..e374080809 --- /dev/null +++ b/octobot/community/node_journal/AGENTS.md @@ -0,0 +1,47 @@ +# Agents: node_journal + +## Role + +Append-only **node journal**: record lifecycle and user-journey events for diagnostics, analytics, and **export/sharing** (e.g. upload envelope). Tier-1 API: `octobot.community.node_journal` (see [`__init__.py`](__init__.py) import tiers). + +## Owns + +- `octobot/community/node_journal/` (recording, store, state, journey export) + +## Public surface (integrators) + +- **Write:** `record`, `record_*` helpers exported from `octobot.community.node_journal` +- **Read (export only):** `read_events`, `build_journey_summary`, `build_upload_envelope` for journal export pipeline inside this package (and tests) + +## External integrators (`packages/sync`, wallet, node, tentacles, etc.) + +- **Record only.** Do not call `read_events`, read journal files on disk, or use journal contents for control flow. +- Do not treat journal or [`state.py`](state.py) persistence as source of truth for app behavior. + +## Not a source of truth + +- Do not store data in journal payloads to reuse later for non-journal features. +- Authoritative state lives in config, databases, sync collections, and domain modules, not the journal. + +## Literals + +- Journal-specific events and wire keys: [`events.py`](events.py), [`constants.py`](constants.py), [`enums.py`](enums.py) (large dedicated set; submodule files are the exception to package-top-level constants). +- Extend those modules; do not inline strings in recording code. + +## Layers + +- Follow layer rules in package docstring: `journal` / `store` / `state` must not import from `recording/` or `lifecycle` in forbidden directions; feature code uses the public API only. + +## Tests + +- **cwd:** repo root +- **Example:** `pytest tests/unit_tests/community/node_journal` + +## Related + +- Skill **node-journal** (`.cursor/skills/node-journal/SKILL.md`) +- [`octobot/AGENTS.md`](../../AGENTS.md) + +## Last reviewed + +- 2026-09-21 diff --git a/octobot/community/node_journal/__init__.py b/octobot/community/node_journal/__init__.py new file mode 100644 index 0000000000..57072fa51e --- /dev/null +++ b/octobot/community/node_journal/__init__.py @@ -0,0 +1,127 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +"""Public node journal API. + +Import tiers: +- Tier 1 (feature code): ``octobot.community.node_journal`` — record/read helpers and events. +- Tier 2 (infrastructure): ``octobot.community.node_journal.lifecycle``, + ``octobot.community.node_journal.recording_context``. +- Tier 3 (internal/tests): ``octobot.community.node_journal.store``, + ``octobot.community.node_journal.state``. + +Layer rules (documentation only): +- ``journal``, ``store``, ``state`` must not import from ``recording/`` or ``lifecycle``. +- ``recording/*`` may import ``journal``, ``state``, and ``classify``; not ``lifecycle``. +- ``lifecycle`` may import ``journal``, ``recording``, and external infrastructure. +- ``journey_summary`` is read-only; no ``recording`` imports. +""" + +from octobot.community.node_journal.events import ( + NodeJournalEvent, + UI_JOURNAL_EVENTS, +) +from octobot.community.node_journal.journal import ( + initialize_for_config, + is_journal_enabled, + read_events, + record, + record_failure, +) +from octobot.community.node_journal.journey_summary import ( + build_journey_summary, + build_upload_envelope, +) +import octobot.community.node_journal.recording as journal_recording + +on_user_data_pull_succeeded = journal_recording.on_user_data_pull_succeeded +record_account_auth_create_succeeded = journal_recording.record_account_auth_create_succeeded +record_account_auth_deleted = journal_recording.record_account_auth_deleted +record_account_deleted = journal_recording.record_account_deleted +record_account_edit_succeeded = journal_recording.record_account_edit_succeeded +record_account_validated = journal_recording.record_account_validated +record_account_validated_from_account = journal_recording.record_account_validated_from_account +record_account_validation_failed = journal_recording.record_account_validation_failed +record_accounts_refreshed = journal_recording.record_accounts_refreshed +record_automation_restarted = journal_recording.record_automation_restarted +record_automation_run_errored = journal_recording.record_automation_run_errored +record_automation_started = journal_recording.record_automation_started +record_automation_stopped = journal_recording.record_automation_stopped +record_executor_failure = journal_recording.record_executor_failure +record_existing_config_detected = journal_recording.record_existing_config_detected +record_external_action_failed = journal_recording.record_external_action_failed +record_external_action_received = journal_recording.record_external_action_received +record_first_automation_started = journal_recording.record_first_automation_started +record_new_automation_created = journal_recording.record_new_automation_created +record_new_automation_created_from_strategy = journal_recording.record_new_automation_created_from_strategy +record_process_startup_failed = journal_recording.record_process_startup_failed +record_process_startup_succeeded = journal_recording.record_process_startup_succeeded +record_reconcile_completed = journal_recording.record_reconcile_completed +record_scheduler_init_failed = journal_recording.record_scheduler_init_failed +record_strategy_create_succeeded = journal_recording.record_strategy_create_succeeded +record_strategy_edit_succeeded = journal_recording.record_strategy_edit_succeeded +record_sync_read_failed = journal_recording.record_sync_read_failed +record_sync_storage_event = journal_recording.record_sync_storage_event +record_wallet_operation_failed = journal_recording.record_wallet_operation_failed +record_wallet_setup_attempt = journal_recording.record_wallet_setup_attempt +record_wallet_setup_failed = journal_recording.record_wallet_setup_failed +record_wallet_setup_succeeded = journal_recording.record_wallet_setup_succeeded +reset_sync_tracker_after_node_startup = journal_recording.reset_sync_tracker_after_node_startup + +__all__ = [ + "NodeJournalEvent", + "UI_JOURNAL_EVENTS", + "build_journey_summary", + "build_upload_envelope", + "initialize_for_config", + "is_journal_enabled", + "on_user_data_pull_succeeded", + "read_events", + "record", + "record_account_auth_create_succeeded", + "record_account_auth_deleted", + "record_account_deleted", + "record_account_edit_succeeded", + "record_account_validated", + "record_account_validated_from_account", + "record_account_validation_failed", + "record_accounts_refreshed", + "record_automation_restarted", + "record_automation_run_errored", + "record_automation_started", + "record_automation_stopped", + "record_executor_failure", + "record_existing_config_detected", + "record_external_action_failed", + "record_external_action_received", + "record_failure", + "record_first_automation_started", + "record_new_automation_created", + "record_new_automation_created_from_strategy", + "record_strategy_create_succeeded", + "record_strategy_edit_succeeded", + "record_process_startup_failed", + "record_process_startup_succeeded", + "record_reconcile_completed", + "record_scheduler_init_failed", + "record_sync_read_failed", + "record_sync_storage_event", + "record_wallet_operation_failed", + "record_wallet_setup_attempt", + "record_wallet_setup_failed", + "record_wallet_setup_succeeded", + "reset_sync_tracker_after_node_startup", +] diff --git a/octobot/community/node_journal/constants.py b/octobot/community/node_journal/constants.py new file mode 100644 index 0000000000..3b510fff11 --- /dev/null +++ b/octobot/community/node_journal/constants.py @@ -0,0 +1,36 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot.community.node_journal.enums as journal_enums + +JOURNAL_MAX_EVENTS = 50_000 +JOURNAL_SCHEMA_VERSION = 1 +SYNC_SESSION_GAP_SECONDS = 14_400 +JOURNAL_DIR_NAME = "node_journal" +MANIFEST_FILE_NAME = "manifest.json" +STORAGE_CTX_KEY = "_ctx" +EVENTS_FILE_NAME = "events.jsonl" +ONBOARDING_SEGMENT_FILE_NAME = "onboarding_segment.jsonl" +CONFIG_JOURNAL_SECTION = "journal" +CONFIG_INSTALL_ID = journal_enums.JournalManifestField.INSTALL_ID.value +CONFIG_ONBOARDING_STARTED_AT = "onboarding_started_at" +CONFIG_FIRST_AUTOMATION_STARTED_AT = "first_automation_started_at" +CONFIG_CONNECTION_SEQUENCE = "connection_sequence" +CONFIG_LAST_USER_DATA_PULL_AT = "last_user_data_pull_at" +CONFIG_TRACKED_AUTOMATION_IDS = "tracked_automation_ids" +DISTRIBUTION_NODE = "node" +JOURNAL_ENABLED_ENV_VAR = "OCTOBOT_NODE_JOURNAL_ENABLED" +JOURNAL_ENABLED_DEFAULT = "true" diff --git a/octobot/community/node_journal/enums.py b/octobot/community/node_journal/enums.py new file mode 100644 index 0000000000..bf56cca6fa --- /dev/null +++ b/octobot/community/node_journal/enums.py @@ -0,0 +1,121 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import enum + + +class JournalEventLineField(enum.StrEnum): + EVENT = "event" + TIMESTAMP = "timestamp" + SESSION_ID = "session_id" + INSTALL_ID = "install_id" + APP_VERSION = "app_version" + DISTRIBUTION = "distribution" + ONBOARDING_COMPLETE = "onboarding_complete" + ATTRIBUTES = "attributes" + RECORDED = "recorded" + + +class JournalStorageContextField(enum.StrEnum): + SESSION_ID = "session_id" + APP_VERSION = "app_version" + + +class JournalManifestField(enum.StrEnum): + SCHEMA = "schema" + INSTALL_ID = "install_id" + + +class JournalInitPhase(enum.StrEnum): + DBOS_CREATE = "dbos_create" + REGISTER_WORKFLOWS = "register_workflows" + VERSION_MIGRATION = "version_migration" + DBOS_LAUNCH = "dbos_launch" + REGISTER_SCHEDULES = "register_schedules" + + +class JournalStartupPhase(enum.StrEnum): + NODE_API_START = "node_api_start" + BOT_INITIALIZE = "bot_initialize" + CLI_START = "cli_start" + + +class JournalSchedulerBackend(enum.StrEnum): + POSTGRES = "postgres" + SQLITE = "sqlite" + + +class WalletSetupFailureReason(enum.StrEnum): + SERVICE_UNAVAILABLE = "service_unavailable" + ALREADY_CONFIGURED = "already_configured" + CONCURRENT_RACE = "concurrent_race" + WALLET_ERROR = "wallet_error" + + +class WalletSetupMethod(enum.StrEnum): + CREATE = "create" + IMPORT = "import" + + +class WalletOperation(enum.StrEnum): + IMPORT = "import" + CREATE = "create" + DECRYPT = "decrypt" + LOOKUP = "lookup" + DELETE = "delete" + RENAME = "rename" + + +class SyncReadFailureReason(enum.StrEnum): + CAP_AUTH = "cap_auth" + WALLET_NOT_FOUND = "wallet_not_found" + IDENTITY_MISSING = "identity_missing" + STORAGE_ERROR = "storage_error" + OTHER = "other" + + +class SyncStorageRecoveryAction(enum.StrEnum): + DROP_INVALID_ITEMS = "drop_invalid_items" + SANITIZE_STATE = "sanitize_state" + SKIP_ITEM = "skip_item" + REBUILD_ITEM = "rebuild_item" + FALLBACK_ITEM = "fallback_item" + + +class SyncStorageProvider(enum.StrEnum): + LOCAL = "local" + + +class ConfigurationType(enum.StrEnum): + GENERIC_PROCESS = "generic_process" + MARKET_MAKING = "market_making" + COPY = "copy" + SIGNAL_BOT = "signal_bot" + GENERIC_WORKFLOW = "generic_workflow" + TRADING_TENTACLES = "trading_tentacles" + + +class OctobotKind(enum.StrEnum): + MANUAL = "manual" + MARKET_MAKING = "market_making" + FLOW = "flow" + + +class FlowSubtype(enum.StrEnum): + COPY = "copy" + SIGNAL_BOT = "signal_bot" + AI_AGENTS = "ai_agents" + OTHER = "other" diff --git a/octobot/community/node_journal/events.py b/octobot/community/node_journal/events.py new file mode 100644 index 0000000000..f54694bbbb --- /dev/null +++ b/octobot/community/node_journal/events.py @@ -0,0 +1,105 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import enum + + +class NodeJournalEvent(enum.StrEnum): + UNKNOWN = "unknown" + NODE_PROCESS_STARTUP_SUCCEEDED = "node_process_startup_succeeded" + NODE_PROCESS_STARTUP_FAILED = "node_process_startup_failed" + WALLET_SETUP_ATTEMPT = "wallet_setup_attempt" + WALLET_SETUP_SUCCEEDED = "wallet_setup_succeeded" + WALLET_SETUP_FAILED = "wallet_setup_failed" + WALLET_OPERATION_FAILED = "wallet_operation_failed" + EXTERNAL_INTERFACE_CONNECTED = "external_interface_connected" + SYNC_READ_FAILED = "sync_read_failed" + EXTERNAL_ACTION_RECEIVED = "external_action_received" + EXTERNAL_ACTION_FAILED = "external_action_failed" + ACCOUNT_AUTH_CREATE_ATTEMPT = "account_auth_create_attempt" + ACCOUNT_AUTH_CREATE_SUCCEEDED = "account_auth_create_succeeded" + ACCOUNT_AUTH_CREATE_FAILED = "account_auth_create_failed" + ACCOUNT_CREATE_ATTEMPT = "account_create_attempt" + ACCOUNT_VALIDATED = "account_validated" + ACCOUNT_VALIDATION_FAILED = "account_validation_failed" + ACCOUNT_EDIT_ATTEMPT = "account_edit_attempt" + ACCOUNT_EDIT_SUCCEEDED = "account_edit_succeeded" + ACCOUNT_EDIT_FAILED = "account_edit_failed" + STRATEGY_CREATE_ATTEMPT = "strategy_create_attempt" + STRATEGY_CREATE_SUCCEEDED = "strategy_create_succeeded" + STRATEGY_CREATE_FAILED = "strategy_create_failed" + STRATEGY_EDIT_ATTEMPT = "strategy_edit_attempt" + STRATEGY_EDIT_SUCCEEDED = "strategy_edit_succeeded" + STRATEGY_EDIT_FAILED = "strategy_edit_failed" + AUTOMATION_CREATE_ATTEMPT = "automation_create_attempt" + AUTOMATION_CREATE_FAILED = "automation_create_failed" + FIRST_AUTOMATION_STARTED = "first_automation_started" + AUTOMATION_EDIT_ATTEMPT = "automation_edit_attempt" + AUTOMATION_EDIT_SUCCEEDED = "automation_edit_succeeded" + AUTOMATION_EDIT_FAILED = "automation_edit_failed" + AUTOMATION_STARTED = "automation_started" + AUTOMATION_STOPPED = "automation_stopped" + AUTOMATION_RESTARTED = "automation_restarted" + AUTOMATION_RUN_ERRORED = "automation_run_errored" + ACCOUNT_DELETED = "account_deleted" + ACCOUNT_AUTH_DELETED = "account_auth_deleted" + ACCOUNTS_REFRESHED = "accounts_refreshed" + EXISTING_CONFIG_DETECTED = "existing_config_detected" + RECONCILE_COMPLETED = "reconcile_completed" + UI_BOOT_FAILED = "ui_boot_failed" + UI_AUTH_STATE_BROKEN = "ui_auth_state_broken" + UI_FATAL_RENDER_ERROR = "ui_fatal_render_error" + UI_CLIENT_STORAGE_RESET = "ui_client_storage_reset" + UI_INSECURE_CONTEXT = "ui_insecure_context" + SYNC_STORAGE_DECRYPT_FAILED = "sync_storage_decrypt_failed" + SYNC_STORAGE_FORMAT_ERROR = "sync_storage_format_error" + SYNC_STORAGE_SCHEMA_RECOVERY = "sync_storage_schema_recovery" + SYNC_STORAGE_LOAD_FAILED = "sync_storage_load_failed" + SCHEDULER_INIT_FAILED = "scheduler_init_failed" + + +# Must match node_web_interface UiJournalEvent union (5 client-emitted events). +UI_JOURNAL_EVENTS = frozenset({ + NodeJournalEvent.UI_BOOT_FAILED, + NodeJournalEvent.UI_AUTH_STATE_BROKEN, + NodeJournalEvent.UI_FATAL_RENDER_ERROR, + NodeJournalEvent.UI_CLIENT_STORAGE_RESET, + NodeJournalEvent.UI_INSECURE_CONTEXT, +}) + +UI_BLOCKING_JOURNAL_EVENTS = frozenset( + event for event in UI_JOURNAL_EVENTS if event != NodeJournalEvent.UI_CLIENT_STORAGE_RESET +) + +FAILURE_EVENTS = frozenset( + event + for event in NodeJournalEvent + if event.value.endswith("_failed") or event.value.endswith("_errored") +) + + +def coerce_node_journal_event(raw_event: NodeJournalEvent | str) -> tuple[NodeJournalEvent, str | None]: + if isinstance(raw_event, NodeJournalEvent): + if raw_event == NodeJournalEvent.UNKNOWN: + return NodeJournalEvent.UNKNOWN, None + return raw_event, None + event_name = str(raw_event) + if event_name in NodeJournalEvent._value2member_map_: + parsed_event = NodeJournalEvent(event_name) + if parsed_event == NodeJournalEvent.UNKNOWN: + return NodeJournalEvent.UNKNOWN, None + return parsed_event, None + return NodeJournalEvent.UNKNOWN, event_name diff --git a/octobot/community/node_journal/events_metadata.py b/octobot/community/node_journal/events_metadata.py new file mode 100644 index 0000000000..36b1572f70 --- /dev/null +++ b/octobot/community/node_journal/events_metadata.py @@ -0,0 +1,64 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot.community.node_journal.events as journal_events + +FUNNEL_STEP_ORDER: tuple[journal_events.NodeJournalEvent, ...] = ( + journal_events.NodeJournalEvent.NODE_PROCESS_STARTUP_SUCCEEDED, + journal_events.NodeJournalEvent.NODE_PROCESS_STARTUP_FAILED, + journal_events.NodeJournalEvent.WALLET_SETUP_SUCCEEDED, + journal_events.NodeJournalEvent.WALLET_SETUP_FAILED, + journal_events.NodeJournalEvent.EXTERNAL_INTERFACE_CONNECTED, + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_SUCCEEDED, + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_FAILED, + journal_events.NodeJournalEvent.ACCOUNT_VALIDATED, + journal_events.NodeJournalEvent.ACCOUNT_VALIDATION_FAILED, + journal_events.NodeJournalEvent.STRATEGY_CREATE_SUCCEEDED, + journal_events.NodeJournalEvent.STRATEGY_CREATE_FAILED, + journal_events.NodeJournalEvent.STRATEGY_EDIT_SUCCEEDED, + journal_events.NodeJournalEvent.STRATEGY_EDIT_FAILED, + journal_events.NodeJournalEvent.AUTOMATION_CREATE_ATTEMPT, + journal_events.NodeJournalEvent.AUTOMATION_CREATE_FAILED, + journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED, +) + +FUNNEL_STEP_RANK = {event: index for index, event in enumerate(FUNNEL_STEP_ORDER)} + +JOURNEY_MILESTONE_LABELS: dict[journal_events.NodeJournalEvent, str] = { + journal_events.NodeJournalEvent.WALLET_SETUP_SUCCEEDED: "wallet_setup", + journal_events.NodeJournalEvent.EXTERNAL_INTERFACE_CONNECTED: "external_connect", + journal_events.NodeJournalEvent.ACCOUNT_VALIDATED: "account_validated", + journal_events.NodeJournalEvent.STRATEGY_CREATE_SUCCEEDED: "strategy_create", + journal_events.NodeJournalEvent.STRATEGY_EDIT_SUCCEEDED: "strategy_edit", + journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED: "first_automation", +} + +JOURNEY_MILESTONE_LABEL_ORDER: tuple[str, ...] = tuple( + JOURNEY_MILESTONE_LABELS[event] + for event in FUNNEL_STEP_ORDER + if event in JOURNEY_MILESTONE_LABELS +) + +JOURNEY_SUCCESS_EVENTS = frozenset({ + journal_events.NodeJournalEvent.WALLET_SETUP_SUCCEEDED, + journal_events.NodeJournalEvent.EXTERNAL_INTERFACE_CONNECTED, + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_SUCCEEDED, + journal_events.NodeJournalEvent.ACCOUNT_VALIDATED, + journal_events.NodeJournalEvent.STRATEGY_CREATE_SUCCEEDED, + journal_events.NodeJournalEvent.STRATEGY_EDIT_SUCCEEDED, + journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED, + journal_events.NodeJournalEvent.AUTOMATION_CREATE_ATTEMPT, +}) diff --git a/octobot/community/node_journal/journal.py b/octobot/community/node_journal/journal.py new file mode 100644 index 0000000000..51985c37c2 --- /dev/null +++ b/octobot/community/node_journal/journal.py @@ -0,0 +1,211 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +"""Core write/read pipeline for the node journal.""" + +import logging +import time +import typing + +import octobot.constants as octobot_constants +import octobot_commons.os_util as os_util + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.state as journal_state +import octobot.community.node_journal.store as journal_store + +logger = logging.getLogger(__name__) + +def is_journal_enabled() -> bool: + return os_util.parse_boolean_environment_var( + journal_constants.JOURNAL_ENABLED_ENV_VAR, + journal_constants.JOURNAL_ENABLED_DEFAULT, + ) + + +def run_journal_operation( + operation_name: str, + operation: typing.Callable[[], typing.Any], + *, + default: typing.Any, +) -> typing.Any: + try: + return operation() + except Exception as exc: + logger.exception("Journal %s failed: %s", operation_name, exc) + return default + + +def sanitize_attribute_value(value: typing.Any) -> typing.Any: + if isinstance(value, bool): + return value + if isinstance(value, (int, float, str)): + return value + if isinstance(value, list): + return value + return str(value) + + +def sanitize_event_attributes( + event: journal_events.NodeJournalEvent, + attributes: journal_models.JournalEventAttributes, +) -> journal_models.JournalEventAttributes: + sanitized_attributes = {} + for field_name, value in attributes.to_dict().items(): + if value is None: + continue + sanitized_attributes[field_name] = sanitize_attribute_value(value) + return journal_models.JournalEventAttributes.from_dict(sanitized_attributes) + + +def record( + event: journal_events.NodeJournalEvent | str, + *, + attributes: journal_models.JournalEventAttributes | dict | None = None, + timestamp: float | None = None, +) -> journal_models.JournalEventLine: + return run_journal_operation( + "record", + lambda: _record(event, attributes=attributes, timestamp=timestamp), + default=_build_disabled_event_stub_for_input(event, attributes), + ) + + +def read_events() -> list[journal_models.JournalEventLine]: + if not is_journal_enabled(): + return [] + return journal_store.get_store().read_all_events() + + +def initialize_for_config(config) -> None: + if not is_journal_enabled(): + return + journal_state.bind_config(config) + journal_state.load_persisted_state(config) + + +def record_failure( + event: journal_events.NodeJournalEvent, + *, + error: BaseException | None = None, + error_message: str | None = None, + error_category: str | None = None, + attributes: journal_models.JournalEventAttributes | dict | None = None, +) -> journal_models.JournalEventLine: + failure_attributes = journal_models.JournalEventAttributes.merge( + _coerce_attributes(attributes), + {}, + ) + if error_category is None and error is not None: + failure_attributes.error_category = error.__class__.__name__ + elif error_category is not None: + failure_attributes.error_category = error_category + if error_message is not None: + failure_attributes.error_message = error_message + return record(event, attributes=failure_attributes) + + +def _record( + event: journal_events.NodeJournalEvent | str, + *, + attributes: journal_models.JournalEventAttributes | dict | None, + timestamp: float | None, +) -> journal_models.JournalEventLine: + parsed_event, raw_event_name = journal_events.coerce_node_journal_event(event) + coerced_attributes = _merge_raw_event_name(_coerce_attributes(attributes), raw_event_name) + if parsed_event == journal_events.NodeJournalEvent.UNKNOWN: + logger.error("Unknown journal event: %s", raw_event_name or event) + return _build_disabled_event_stub(parsed_event, coerced_attributes, recorded=False) + if not is_journal_enabled(): + return _build_disabled_event_stub(parsed_event, coerced_attributes) + sanitized_attributes = sanitize_event_attributes(parsed_event, coerced_attributes) + if ( + parsed_event in journal_events.FAILURE_EVENTS + and sanitized_attributes.error_category is None + ): + logger.error("%s requires error_category", parsed_event.value) + return _build_disabled_event_stub(parsed_event, sanitized_attributes, recorded=False) + emit_timestamp = time.time() if timestamp is None else timestamp + persisted_state = journal_state.load_persisted_state() + event_line = journal_models.JournalEventLine( + event=parsed_event, + timestamp=emit_timestamp, + session_id=journal_state.get_session_id(), + install_id=persisted_state.install_id, + app_version=octobot_constants.LONG_VERSION, + distribution=journal_constants.DISTRIBUTION_NODE, + onboarding_complete=persisted_state.onboarding_complete, + attributes=sanitized_attributes, + ) + is_onboarding_segment = not persisted_state.onboarding_complete + journal_store.get_store().append(event_line, is_onboarding_segment=is_onboarding_segment) + if parsed_event == journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED: + journal_state.mark_first_automation_started(emit_timestamp) + return event_line + + +def _coerce_attributes( + attributes: journal_models.JournalEventAttributes | dict | None, +) -> journal_models.JournalEventAttributes: + if attributes is None: + return journal_models.JournalEventAttributes() + if isinstance(attributes, journal_models.JournalEventAttributes): + return attributes + return journal_models.JournalEventAttributes.from_dict(attributes) + + +def _merge_raw_event_name( + attributes: journal_models.JournalEventAttributes, + raw_event_name: str | None, +) -> journal_models.JournalEventAttributes: + if raw_event_name is None: + return attributes + return journal_models.JournalEventAttributes.merge( + attributes, + {"raw_event_name": raw_event_name}, + ) + + +def _build_disabled_event_stub_for_input( + event: journal_events.NodeJournalEvent | str, + attributes: journal_models.JournalEventAttributes | dict | None, + *, + recorded: bool = False, +) -> journal_models.JournalEventLine: + parsed_event, raw_event_name = journal_events.coerce_node_journal_event(event) + coerced_attributes = _merge_raw_event_name(_coerce_attributes(attributes), raw_event_name) + return _build_disabled_event_stub(parsed_event, coerced_attributes, recorded=recorded) + + +def _build_disabled_event_stub( + parsed_event: journal_events.NodeJournalEvent, + attributes: journal_models.JournalEventAttributes, + *, + recorded: bool = False, +) -> journal_models.JournalEventLine: + return journal_models.JournalEventLine( + event=parsed_event, + timestamp=time.time(), + session_id="", + install_id="", + app_version=octobot_constants.LONG_VERSION, + distribution=journal_constants.DISTRIBUTION_NODE, + onboarding_complete=False, + attributes=attributes, + recorded=recorded, + ) diff --git a/octobot/community/node_journal/journey_summary.py b/octobot/community/node_journal/journey_summary.py new file mode 100644 index 0000000000..4bd2b82b20 --- /dev/null +++ b/octobot/community/node_journal/journey_summary.py @@ -0,0 +1,206 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import collections + +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.events_metadata as journal_events_metadata +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.state as journal_state + +def build_journey_summary(events: list[journal_models.JournalEventLine]) -> journal_models.JourneySummary: + persisted_state = journal_state.load_persisted_state() + install_start = persisted_state.onboarding_started_at + parsed_events = _parse_events(events) + onboarding_complete = persisted_state.onboarding_complete or any( + event == journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED for event, _ in parsed_events + ) + external_stats = _get_external_connect_stats(parsed_events) + return journal_models.JourneySummary( + onboarding_complete=onboarding_complete, + furthest_step_reached=_get_furthest_step(parsed_events), + last_successful_step=_get_last_successful_step(parsed_events), + first_failure=_get_first_failure(parsed_events), + retry_counts=_get_retry_counts(parsed_events), + step_durations_seconds=_get_step_durations(parsed_events, install_start), + step_deltas_seconds=_get_step_deltas(_get_step_durations(parsed_events, install_start)), + external_connect_count=external_stats["external_connect_count"], + first_external_connect_at=external_stats["first_external_connect_at"], + last_external_connect_at=external_stats["last_external_connect_at"], + longest_connect_gap_seconds=external_stats["longest_connect_gap_seconds"], + ui_blocking_issues_count=_get_ui_blocking_issues_count(parsed_events), + ) + + +def _parse_events( + events: list[journal_models.JournalEventLine], +) -> list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]]: + parsed_events = [] + for event_line in events: + if event_line.event == journal_events.NodeJournalEvent.UNKNOWN: + parsed_events.append((None, event_line)) + continue + parsed_events.append((event_line.event, event_line)) + return parsed_events + + +def _get_furthest_step( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> str | None: + best_rank = -1 + best_event_name = None + for event, _ in parsed_events: + if event is None: + continue + rank = journal_events_metadata.FUNNEL_STEP_RANK.get(event) + if rank is not None and rank >= best_rank: + best_rank = rank + best_event_name = event.value + return best_event_name + + +def _get_last_successful_step( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> str | None: + last_success = None + for event, _ in parsed_events: + if event in journal_events_metadata.JOURNEY_SUCCESS_EVENTS: + last_success = event.value + return last_success + + +def _get_first_failure( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> journal_models.FirstFailureInfo | None: + for event, event_line in parsed_events: + if event is None or event not in journal_events.FAILURE_EVENTS: + continue + attributes = event_line.attributes + return journal_models.FirstFailureInfo( + event=event.value, + timestamp=event_line.timestamp, + error_category=attributes.error_category, + error_message=attributes.error_message, + ) + return None + + +def _get_ui_blocking_issues_count( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> int: + blocking_issue_count = 0 + for event, _ in parsed_events: + if event is not None and event in journal_events.UI_BLOCKING_JOURNAL_EVENTS: + blocking_issue_count += 1 + return blocking_issue_count + + +def _get_retry_counts( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> dict[str, int]: + retry_counts: dict[str, int] = collections.Counter() + for event, _ in parsed_events: + if event is not None and event in journal_events.FAILURE_EVENTS: + retry_counts[event.value] += 1 + return dict(retry_counts) + + +def _get_step_durations( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], + install_start: float | None, +) -> dict[str, float]: + if install_start is None: + return {} + step_durations: dict[str, float] = {} + for event, event_line in parsed_events: + label = journal_events_metadata.JOURNEY_MILESTONE_LABELS.get(event) if event is not None else None + if label is None: + continue + event_timestamp = float(event_line.timestamp) + step_durations[label] = round(max(0.0, event_timestamp - install_start), 3) + return step_durations + + +def _get_step_deltas(step_durations_seconds: dict[str, float]) -> dict[str, float]: + ordered_labels = journal_events_metadata.JOURNEY_MILESTONE_LABEL_ORDER + deltas: dict[str, float] = {} + previous_label = None + previous_duration = None + for label in ordered_labels: + current_duration = step_durations_seconds.get(label) + if current_duration is None: + continue + if previous_label is not None and previous_duration is not None: + delta_key = f"{previous_label}_to_{label}" + deltas[delta_key] = round(max(0.0, current_duration - previous_duration), 3) + previous_label = label + previous_duration = current_duration + return deltas + + +def _get_external_connect_stats( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> dict: + connect_events = [ + event_line + for event, event_line in parsed_events + if event == journal_events.NodeJournalEvent.EXTERNAL_INTERFACE_CONNECTED + ] + if not connect_events: + return { + "external_connect_count": 0, + "first_external_connect_at": None, + "last_external_connect_at": None, + "longest_connect_gap_seconds": None, + } + timestamps = [float(event_line.timestamp) for event_line in connect_events] + prior_gaps = [ + float(event_line.attributes.prior_gap_seconds) + for event_line in connect_events + if event_line.attributes.prior_gap_seconds is not None + ] + return { + "external_connect_count": len(connect_events), + "first_external_connect_at": min(timestamps), + "last_external_connect_at": max(timestamps), + "longest_connect_gap_seconds": max(prior_gaps) if prior_gaps else None, + } + + +def build_upload_envelope( + events: list[journal_models.JournalEventLine], + *, + app_version: str, + note: str | None = None, + ui_error_name: str | None = None, + ui_error_route: str | None = None, +) -> journal_models.UploadEnvelope: + persisted_state = journal_state.load_persisted_state() + journey_summary = build_journey_summary(events) + return journal_models.UploadEnvelope( + install_id=persisted_state.install_id, + app_version=app_version, + onboarding_started_at=persisted_state.onboarding_started_at, + onboarding_complete=journey_summary.onboarding_complete, + journey_summary=journey_summary, + events=events, + uploaded=False, + ready=True, + event_count=len(events), + note=note, + ui_error_name=ui_error_name, + ui_error_route=ui_error_route, + ) diff --git a/octobot/community/node_journal/lifecycle.py b/octobot/community/node_journal/lifecycle.py new file mode 100644 index 0000000000..d68f3ddc47 --- /dev/null +++ b/octobot/community/node_journal/lifecycle.py @@ -0,0 +1,154 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import dataclasses +import typing + +import octobot_commons.configuration as configuration + +import octobot.configuration_manager as configuration_manager +import octobot.enums as octobot_enums +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot.community.authentication as community_authentication +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.recording as journal_recording +import octobot.community.node_journal.state as journal_state + + +@dataclasses.dataclass(frozen=True) +class ExistingConfigSnapshot: + wallet_configured: bool + account_count: int + reconciled: bool + + +_reconcile_completed = False + + +def build_existing_config_snapshot() -> ExistingConfigSnapshot: + auth = community_authentication.CommunityAuthentication.instance() + wallet_configured = auth is not None and auth.is_node_wallet_configured() + account_count = 0 + if wallet_configured: + wallet_user_ids = collection_providers.AccountProvider.instance().list_collectable_wallet_ids() + for wallet_id in wallet_user_ids: + account_count += len(collection_providers.AccountProvider.instance().list_accounts(wallet_id)) + has_existing_config = wallet_configured or account_count > 0 + return ExistingConfigSnapshot( + wallet_configured=wallet_configured, + account_count=account_count, + reconciled=has_existing_config, + ) + + +def initialize_journal(config: configuration.Configuration) -> None: + if not journal_module.is_journal_enabled(): + return + distribution = configuration_manager.get_distribution(config.config) + if distribution is not octobot_enums.OctoBotDistribution.NODE: + return + journal_module.initialize_for_config(config) + snapshot = build_existing_config_snapshot() + if snapshot.reconciled: + journal_recording.record_existing_config_detected( + wallet_configured=snapshot.wallet_configured, + account_count=snapshot.account_count, + ) + + +def record_node_startup_succeeded(config: configuration.Configuration) -> None: + journal_module.initialize_for_config(config) + snapshot = build_existing_config_snapshot() + journal_recording.record_process_startup_succeeded( + wallet_configured=snapshot.wallet_configured, + new_install=not snapshot.reconciled, + reconciled=snapshot.reconciled, + ) + + +def record_node_startup_failed( + error: BaseException, + *, + startup_phase: journal_enums.JournalStartupPhase, + force_exit: bool, + config: typing.Optional[configuration.Configuration] = None, +) -> None: + resolved_config = config if config is not None else journal_state.get_config() + if resolved_config is not None: + journal_module.initialize_for_config(resolved_config) + snapshot = build_existing_config_snapshot() + journal_recording.record_process_startup_failed( + startup_phase=startup_phase, + error=error, + force_exit=force_exit, + wallet_configured=snapshot.wallet_configured, + new_install=not snapshot.reconciled, + reconciled=snapshot.reconciled, + ) + + +def _distinct_automation_ids_from_states( + automation_states: list[protocol_models.AutomationState], +) -> list[str]: + return sorted({ + automation_state.id + for automation_state in automation_states + if automation_state.id + }) + + +async def complete_reconcile_automations( + config: typing.Optional[configuration.Configuration] = None, +) -> None: + global _reconcile_completed + if not journal_module.is_journal_enabled(): + return + if _reconcile_completed: + return + resolved_config = config if config is not None else journal_state.get_config() + if resolved_config is not None: + journal_module.initialize_for_config(resolved_config) + snapshot = build_existing_config_snapshot() + if not snapshot.reconciled: + return + # Lazy import: octobot_node.scheduler pulls octobot_flow → octobot.community (circular at import time). + import octobot_node.scheduler as scheduler_module + import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader + + if not scheduler_module.is_initialized(): + return + + automation_ids: set[str] = set() + running_automation_count = 0 + for wallet_id in collection_providers.AccountProvider.instance().list_collectable_wallet_ids(): + automation_states = await automation_states_loader.load_protocol_automation_states( + wallet_id, + statuses=None, + ) + automation_ids.update(_distinct_automation_ids_from_states(automation_states)) + running_automation_count += sum( + 1 + for automation_state in automation_states + if automation_state.status == protocol_models.WorkflowStatus.RUNNING + ) + journal_recording.record_reconcile_completed( + automation_count=len(automation_ids), + running_automation_count=running_automation_count, + ) + _reconcile_completed = True diff --git a/octobot/community/node_journal/models.py b/octobot/community/node_journal/models.py new file mode 100644 index 0000000000..90d155275f --- /dev/null +++ b/octobot/community/node_journal/models.py @@ -0,0 +1,211 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import dataclasses + +import octobot_commons.dataclasses as commons_dataclasses + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events + + +@dataclasses.dataclass +class _NodeJournalMinimizableDataclass(commons_dataclasses.MinimizableDataclass): + def to_dict(self, include_default_values: bool = False) -> dict: + return super().to_dict(include_default_values=include_default_values) + + +@dataclasses.dataclass +class JournalEventAttributes(_NodeJournalMinimizableDataclass): + account_count: int | None = None + account_id: str | None = None + account_ids: list[str] | None = None + account_state: str | None = None + action_type: str | None = None + automation_count: int | None = None + automation_id: str | None = None + running_automation_count: int | None = None + backend: str | None = None + cancel_orders: bool | None = None + collection: str | None = None + configured: bool | None = None + configuration_type: str | None = None + connection_sequence: int | None = None + duration_since_install_start: float | None = None + error_category: str | None = None + error_message: str | None = None + error_origin: str | None = None + error_status: str | None = None + exchange_name: str | None = None + failure_reason: str | None = None + flow_subtype: str | None = None + force_exit: bool | None = None + http_status: int | None = None + init_phase: str | None = None + is_reconnect: bool | None = None + is_simulated: bool | None = None + new_install: bool | None = None + node_type: str | None = None + octobot_kind: str | None = None + operation: str | None = None + prior_gap_seconds: float | None = None + provider: str | None = None + reconciled: bool | None = None + recovery_action: str | None = None + refreshed_count: int | None = None + retriable: bool | None = None + raw_event_name: str | None = None + setup_method: str | None = None + source: str | None = None + startup_phase: str | None = None + strategy_id: str | None = None + sync_session_id: str | None = None + user_action_id: str | None = None + wallet_configured: bool | None = None + + @classmethod + def merge(cls, base: "JournalEventAttributes | None", overrides: dict | None) -> "JournalEventAttributes": + merged = dataclasses.asdict(base) if base is not None else {} + if overrides: + known_fields = cls.get_field_names() + for raw_key, value in overrides.items(): + if raw_key in known_fields: + merged[raw_key] = value + return cls(**merged) + + +@dataclasses.dataclass +class JournalEventLine(_NodeJournalMinimizableDataclass): + event: journal_events.NodeJournalEvent + timestamp: float + session_id: str + install_id: str + app_version: str + distribution: str + onboarding_complete: bool + attributes: JournalEventAttributes + recorded: bool = True + + def to_dict(self, include_default_values: bool = False) -> dict: + serialized = super().to_dict(include_default_values=include_default_values) + event_line_field = journal_enums.JournalEventLineField + serialized[event_line_field.EVENT.value] = self.event.value + if self.attributes is not None: + serialized[event_line_field.ATTRIBUTES.value] = self.attributes.to_dict( + include_default_values=include_default_values, + ) + else: + serialized[event_line_field.ATTRIBUTES.value] = {} + if not self.recorded: + serialized[event_line_field.RECORDED.value] = False + elif not include_default_values and serialized.get(event_line_field.RECORDED.value) is True: + serialized.pop(event_line_field.RECORDED.value, None) + return serialized + + def to_storage_dict(self) -> dict: + event_line_field = journal_enums.JournalEventLineField + serialized = { + event_line_field.EVENT.value: self.event.value, + event_line_field.TIMESTAMP.value: self.timestamp, + } + if self.attributes is not None: + attributes_dict = self.attributes.to_dict() + if attributes_dict: + serialized[event_line_field.ATTRIBUTES.value] = attributes_dict + if not self.recorded: + serialized[event_line_field.RECORDED.value] = False + return serialized + + @classmethod + def from_dict(cls, event_line: dict) -> "JournalEventLine": + event_line_field = journal_enums.JournalEventLineField + parsed_event, coerced_raw_event_name = journal_events.coerce_node_journal_event( + event_line[event_line_field.EVENT.value], + ) + attributes = JournalEventAttributes.from_dict(event_line.get(event_line_field.ATTRIBUTES.value)) + if attributes is None: + attributes = JournalEventAttributes() + if coerced_raw_event_name is not None: + attributes = JournalEventAttributes.merge( + attributes, + {"raw_event_name": coerced_raw_event_name}, + ) + return cls( + event=parsed_event, + timestamp=float(event_line[event_line_field.TIMESTAMP.value]), + session_id=str(event_line.get(event_line_field.SESSION_ID.value, "")), + install_id=str(event_line.get(event_line_field.INSTALL_ID.value, "")), + app_version=str(event_line.get(event_line_field.APP_VERSION.value, "")), + distribution=str(event_line.get(event_line_field.DISTRIBUTION.value, "")), + onboarding_complete=bool(event_line.get(event_line_field.ONBOARDING_COMPLETE.value, False)), + attributes=attributes, + recorded=event_line.get(event_line_field.RECORDED.value, True), + ) + + +@dataclasses.dataclass +class FirstFailureInfo(_NodeJournalMinimizableDataclass): + event: str + timestamp: float | None + error_category: str | None + error_message: str | None + + @classmethod + def from_dict(cls, payload: dict | None) -> "FirstFailureInfo | None": + if payload is None: + return None + return super().from_dict(payload) + + +@dataclasses.dataclass +class JourneySummary(_NodeJournalMinimizableDataclass): + onboarding_complete: bool + furthest_step_reached: str | None + last_successful_step: str | None + first_failure: FirstFailureInfo | None + retry_counts: dict[str, int] + step_durations_seconds: dict[str, float] + step_deltas_seconds: dict[str, float] + external_connect_count: int + first_external_connect_at: float | None + last_external_connect_at: float | None + longest_connect_gap_seconds: float | None + ui_blocking_issues_count: int = 0 + + +@dataclasses.dataclass +class UploadEnvelope(_NodeJournalMinimizableDataclass): + install_id: str + app_version: str + onboarding_started_at: float | None + onboarding_complete: bool + journey_summary: JourneySummary + events: list[JournalEventLine] + uploaded: bool + ready: bool + event_count: int + note: str | None = None + ui_error_name: str | None = None + ui_error_route: str | None = None + + def to_dict(self, include_default_values: bool = False) -> dict: + serialized = super().to_dict(include_default_values=include_default_values) + serialized["events"] = [event_line.to_storage_dict() for event_line in self.events] + serialized["journey_summary"] = self.journey_summary.to_dict( + include_default_values=include_default_values, + ) + return serialized diff --git a/octobot/community/node_journal/recording/__init__.py b/octobot/community/node_journal/recording/__init__.py new file mode 100644 index 0000000000..c51123004f --- /dev/null +++ b/octobot/community/node_journal/recording/__init__.py @@ -0,0 +1,99 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot.community.node_journal.recording.accounts as accounts_module +import octobot.community.node_journal.recording.automations as automations_module +import octobot.community.node_journal.recording.process as process_module +import octobot.community.node_journal.recording.strategies as strategies_module +import octobot.community.node_journal.recording.sync as sync_module +import octobot.community.node_journal.recording.user_actions as user_actions_module +import octobot.community.node_journal.recording.wallet as wallet_module + +record_account_auth_create_succeeded = accounts_module.record_account_auth_create_succeeded +record_account_auth_deleted = accounts_module.record_account_auth_deleted +record_account_deleted = accounts_module.record_account_deleted +record_account_edit_succeeded = accounts_module.record_account_edit_succeeded +record_account_validated = accounts_module.record_account_validated +record_account_validated_from_account = accounts_module.record_account_validated_from_account +record_account_validation_failed = accounts_module.record_account_validation_failed +record_accounts_refreshed = accounts_module.record_accounts_refreshed + +record_automation_restarted = automations_module.record_automation_restarted +record_automation_run_errored = automations_module.record_automation_run_errored +record_automation_started = automations_module.record_automation_started +record_automation_stopped = automations_module.record_automation_stopped +record_first_automation_started = automations_module.record_first_automation_started +record_new_automation_created = automations_module.record_new_automation_created +record_new_automation_created_from_strategy = automations_module.record_new_automation_created_from_strategy + +record_existing_config_detected = process_module.record_existing_config_detected +record_process_startup_failed = process_module.record_process_startup_failed +record_process_startup_succeeded = process_module.record_process_startup_succeeded +record_reconcile_completed = process_module.record_reconcile_completed +record_scheduler_init_failed = process_module.record_scheduler_init_failed + +record_strategy_create_succeeded = strategies_module.record_strategy_create_succeeded +record_strategy_edit_succeeded = strategies_module.record_strategy_edit_succeeded + +on_user_data_pull_succeeded = sync_module.on_user_data_pull_succeeded +record_sync_read_failed = sync_module.record_sync_read_failed +record_sync_storage_event = sync_module.record_sync_storage_event +reset_sync_tracker_after_node_startup = sync_module.reset_sync_tracker_after_node_startup + +record_executor_failure = user_actions_module.record_executor_failure +record_external_action_failed = user_actions_module.record_external_action_failed +record_external_action_received = user_actions_module.record_external_action_received + +record_wallet_operation_failed = wallet_module.record_wallet_operation_failed +record_wallet_setup_attempt = wallet_module.record_wallet_setup_attempt +record_wallet_setup_failed = wallet_module.record_wallet_setup_failed +record_wallet_setup_succeeded = wallet_module.record_wallet_setup_succeeded + +__all__ = [ + "on_user_data_pull_succeeded", + "record_account_auth_create_succeeded", + "record_account_auth_deleted", + "record_account_deleted", + "record_account_edit_succeeded", + "record_account_validated", + "record_account_validated_from_account", + "record_account_validation_failed", + "record_accounts_refreshed", + "record_automation_restarted", + "record_automation_run_errored", + "record_automation_started", + "record_automation_stopped", + "record_executor_failure", + "record_existing_config_detected", + "record_external_action_failed", + "record_external_action_received", + "record_first_automation_started", + "record_new_automation_created", + "record_new_automation_created_from_strategy", + "record_process_startup_failed", + "record_process_startup_succeeded", + "record_reconcile_completed", + "record_scheduler_init_failed", + "record_strategy_create_succeeded", + "record_strategy_edit_succeeded", + "record_sync_read_failed", + "record_sync_storage_event", + "record_wallet_operation_failed", + "record_wallet_setup_attempt", + "record_wallet_setup_failed", + "record_wallet_setup_succeeded", + "reset_sync_tracker_after_node_startup", +] diff --git a/octobot/community/node_journal/recording/_utils.py b/octobot/community/node_journal/recording/_utils.py new file mode 100644 index 0000000000..11187f0df8 --- /dev/null +++ b/octobot/community/node_journal/recording/_utils.py @@ -0,0 +1,21 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import enum + + +def enum_value(value: enum.StrEnum) -> str: + return value.value if isinstance(value, enum.StrEnum) else str(value) diff --git a/octobot/community/node_journal/recording/accounts.py b/octobot/community/node_journal/recording/accounts.py new file mode 100644 index 0000000000..72f5fc2e56 --- /dev/null +++ b/octobot/community/node_journal/recording/accounts.py @@ -0,0 +1,145 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models + +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording.classify as journal_classify + + +def record_account_validated_from_account( + checked_account: protocol_models.Account, + user_id: str, + *, + user_action_id: str | None = None, +) -> None: + if checked_account.state is None or checked_account.state.status != protocol_models.AccountStatus.VALID: + return + exchange_name = journal_classify.resolve_account_exchange_name(checked_account, user_id) + record_account_validated( + account_id=checked_account.id or "", + is_simulated=checked_account.is_simulated, + exchange_name=exchange_name or "", + user_action_id=user_action_id, + ) + + +def record_account_validated( + *, + account_id: str = "", + is_simulated: bool, + exchange_name: str | None, + user_action_id: str | None = None, + account_count: int | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes( + account_id=account_id or None, + is_simulated=is_simulated, + exchange_name=exchange_name, + user_action_id=user_action_id, + account_count=account_count, + ) + journal_module.record(journal_events.NodeJournalEvent.ACCOUNT_VALIDATED, attributes=attributes) + + +def record_account_validation_failed( + *, + is_simulated: bool, + exchange_name: str | None, + error: BaseException, + user_action_id: str | None = None, +) -> None: + journal_module.record_failure( + journal_events.NodeJournalEvent.ACCOUNT_VALIDATION_FAILED, + error=error, + attributes=journal_models.JournalEventAttributes( + is_simulated=is_simulated, + exchange_name=exchange_name, + user_action_id=user_action_id, + ), + ) + + +def record_account_auth_create_succeeded( + *, + exchange_name: str | None, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_SUCCEEDED, + attributes=journal_models.JournalEventAttributes( + exchange_name=exchange_name, + user_action_id=user_action_id, + ), + ) + + +def record_account_edit_succeeded( + *, + account_id: str, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.ACCOUNT_EDIT_SUCCEEDED, + attributes=journal_models.JournalEventAttributes( + account_id=account_id, + user_action_id=user_action_id, + ), + ) + + +def record_account_deleted( + *, + account_id: str, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.ACCOUNT_DELETED, + attributes=journal_models.JournalEventAttributes( + account_id=account_id, + user_action_id=user_action_id, + ), + ) + + +def record_account_auth_deleted( + *, + exchange_name: str | None, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.ACCOUNT_AUTH_DELETED, + attributes=journal_models.JournalEventAttributes( + exchange_name=exchange_name, + user_action_id=user_action_id, + ), + ) + + +def record_accounts_refreshed( + *, + account_ids: list[str], + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.ACCOUNTS_REFRESHED, + attributes=journal_models.JournalEventAttributes( + account_ids=list(account_ids), + user_action_id=user_action_id, + ), + ) diff --git a/octobot/community/node_journal/recording/automations.py b/octobot/community/node_journal/recording/automations.py new file mode 100644 index 0000000000..c42f92008f --- /dev/null +++ b/octobot/community/node_journal/recording/automations.py @@ -0,0 +1,162 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models + +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording._utils as recording_utils +import octobot.community.node_journal.recording.classify as journal_classify +import octobot.community.node_journal.state as journal_state + + +def record_new_automation_created_from_strategy( + automation_id: str, + strategy: protocol_models.Strategy, + *, + user_action_id: str | None = None, + source: str | None = None, +) -> None: + persisted_state = journal_state.load_persisted_state() + if automation_id in persisted_state.tracked_automation_ids: + return + octobot_kind, flow_subtype = journal_classify.classify_octobot_kind(strategy) + is_first_automation = persisted_state.first_automation_started_at is None + persisted_state.tracked_automation_ids.append(automation_id) + journal_state.save_persisted_state(persisted_state) + record_new_automation_created( + automation_id=automation_id, + octobot_kind=octobot_kind.value, + flow_subtype=flow_subtype, + user_action_id=user_action_id, + source=source, + is_first_automation=is_first_automation, + ) + + +def record_first_automation_started( + *, + automation_id: str, + octobot_kind: journal_enums.OctobotKind, + flow_subtype: str | None, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED, + attributes=journal_models.JournalEventAttributes( + automation_id=automation_id, + octobot_kind=recording_utils.enum_value(octobot_kind), + flow_subtype=flow_subtype, + user_action_id=user_action_id, + duration_since_install_start=journal_state.duration_since_install_start(), + ), + ) + + +def record_automation_started( + *, + automation_id: str, + octobot_kind: journal_enums.OctobotKind, + flow_subtype: str | None, + user_action_id: str | None = None, + source: str | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes( + automation_id=automation_id, + octobot_kind=recording_utils.enum_value(octobot_kind), + flow_subtype=flow_subtype, + user_action_id=user_action_id, + source=source, + ) + journal_module.record(journal_events.NodeJournalEvent.AUTOMATION_STARTED, attributes=attributes) + + +def record_new_automation_created( + *, + automation_id: str, + octobot_kind: journal_enums.OctobotKind, + flow_subtype: str | None, + user_action_id: str | None = None, + source: str | None = None, + is_first_automation: bool, +) -> None: + if is_first_automation: + record_first_automation_started( + automation_id=automation_id, + octobot_kind=octobot_kind, + flow_subtype=flow_subtype, + user_action_id=user_action_id, + ) + return + record_automation_started( + automation_id=automation_id, + octobot_kind=octobot_kind, + flow_subtype=flow_subtype, + user_action_id=user_action_id, + source=source, + ) + + +def record_automation_run_errored( + *, + automation_id: str, + error_status: str, + error_origin: str, + error: BaseException, + retriable: bool, +) -> None: + journal_module.record_failure( + journal_events.NodeJournalEvent.AUTOMATION_RUN_ERRORED, + error=error, + attributes=journal_models.JournalEventAttributes( + automation_id=automation_id, + error_status=error_status, + error_origin=error_origin, + retriable=retriable, + ), + ) + + +def record_automation_stopped( + *, + automation_id: str, + cancel_orders: bool, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.AUTOMATION_STOPPED, + attributes=journal_models.JournalEventAttributes( + automation_id=automation_id, + cancel_orders=cancel_orders, + user_action_id=user_action_id, + ), + ) + + +def record_automation_restarted( + *, + automation_id: str, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.AUTOMATION_RESTARTED, + attributes=journal_models.JournalEventAttributes( + automation_id=automation_id, + user_action_id=user_action_id, + ), + ) diff --git a/octobot/community/node_journal/recording/classify.py b/octobot/community/node_journal/recording/classify.py new file mode 100644 index 0000000000..6d6652bf0b --- /dev/null +++ b/octobot/community/node_journal/recording/classify.py @@ -0,0 +1,97 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping + +import octobot.constants as constants +import octobot_sync.sync.collection_backend.errors as collection_errors + +import octobot.community.node_journal.enums as journal_enums + + +def classify_octobot_kind( + strategy: protocol_models.Strategy, +) -> tuple[journal_enums.OctobotKind, str | None]: + configuration_wrapper = strategy.configuration + if configuration_wrapper is None or configuration_wrapper.actual_instance is None: + return journal_enums.OctobotKind.MANUAL, None + inner_configuration = configuration_wrapper.actual_instance + if isinstance(inner_configuration, protocol_models.GenericProcessConfiguration): + return journal_enums.OctobotKind.MANUAL, None + if isinstance(inner_configuration, protocol_models.MarketMakingConfiguration): + return journal_enums.OctobotKind.MARKET_MAKING, None + if isinstance(inner_configuration, protocol_models.CopyConfiguration): + return journal_enums.OctobotKind.FLOW, journal_enums.FlowSubtype.COPY.value + if isinstance(inner_configuration, protocol_models.SignalBotConfiguration): + return journal_enums.OctobotKind.FLOW, journal_enums.FlowSubtype.SIGNAL_BOT.value + if isinstance(inner_configuration, protocol_models.GenericWorkflowConfiguration): + return journal_enums.OctobotKind.FLOW, journal_enums.FlowSubtype.AI_AGENTS.value + if isinstance(inner_configuration, protocol_models.TradingTentaclesConfiguration): + tentacle_name = inner_configuration.name or "" + if not tentacle_name: + return journal_enums.OctobotKind.FLOW, journal_enums.FlowSubtype.OTHER.value + return journal_enums.OctobotKind.FLOW, tentacle_name + return journal_enums.OctobotKind.FLOW, journal_enums.FlowSubtype.OTHER.value + + +def configuration_type_from_strategy(strategy: protocol_models.Strategy) -> journal_enums.ConfigurationType: + configuration_wrapper = strategy.configuration + if configuration_wrapper is None or configuration_wrapper.actual_instance is None: + return journal_enums.ConfigurationType.GENERIC_PROCESS + inner_configuration = configuration_wrapper.actual_instance + if isinstance(inner_configuration, protocol_models.GenericProcessConfiguration): + return journal_enums.ConfigurationType.GENERIC_PROCESS + if isinstance(inner_configuration, protocol_models.MarketMakingConfiguration): + return journal_enums.ConfigurationType.MARKET_MAKING + if isinstance(inner_configuration, protocol_models.CopyConfiguration): + return journal_enums.ConfigurationType.COPY + if isinstance(inner_configuration, protocol_models.SignalBotConfiguration): + return journal_enums.ConfigurationType.SIGNAL_BOT + if isinstance(inner_configuration, protocol_models.GenericWorkflowConfiguration): + return journal_enums.ConfigurationType.GENERIC_WORKFLOW + if isinstance(inner_configuration, protocol_models.TradingTentaclesConfiguration): + return journal_enums.ConfigurationType.TRADING_TENTACLES + return journal_enums.ConfigurationType.GENERIC_PROCESS + + +def resolve_account_exchange_name( + checked_account: protocol_models.Account, + user_id: str, +) -> str: + specifics = checked_account.specifics + if specifics is None or specifics.actual_instance is None: + return constants.METRICS_GENERIC_EXCHANGE_NAME + exchange_account = specifics.actual_instance + if not isinstance(exchange_account, protocol_models.ExchangeAccount): + return constants.METRICS_GENERIC_EXCHANGE_NAME + config_ids = exchange_account.exchange_config_ids or [] + if not config_ids: + return constants.METRICS_GENERIC_EXCHANGE_NAME + import octobot_sync.sync.collection_providers as collection_providers + try: + exchange_config = collection_providers.AccountProvider.instance().get_exchange_config( + user_id, + config_ids[0], + ) + except collection_errors.CollectionNoDataError: + return constants.METRICS_GENERIC_EXCHANGE_NAME + exchange_type = protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get( + protocol_models.TradingType.SPOT + ) + if exchange_type is None: + return exchange_config.exchange + return exchange_config.exchange diff --git a/octobot/community/node_journal/recording/process.py b/octobot/community/node_journal/recording/process.py new file mode 100644 index 0000000000..66bebe8da7 --- /dev/null +++ b/octobot/community/node_journal/recording/process.py @@ -0,0 +1,111 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording._utils as recording_utils +import octobot.community.node_journal.recording.sync as sync_module +import octobot.community.node_journal.state as journal_state + + +def record_process_startup_succeeded( + *, + wallet_configured: bool, + new_install: bool, + reconciled: bool, +) -> None: + sync_module.reset_sync_tracker_after_node_startup() + journal_module.record( + journal_events.NodeJournalEvent.NODE_PROCESS_STARTUP_SUCCEEDED, + attributes=journal_models.JournalEventAttributes( + wallet_configured=wallet_configured, + new_install=new_install, + reconciled=reconciled, + duration_since_install_start=journal_state.duration_since_install_start(), + ), + ) + + +def record_process_startup_failed( + *, + startup_phase: journal_enums.JournalStartupPhase, + error: BaseException, + force_exit: bool, + wallet_configured: bool | None = None, + new_install: bool | None = None, + reconciled: bool | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes( + startup_phase=recording_utils.enum_value(startup_phase), + force_exit=force_exit, + ) + if wallet_configured is not None: + attributes.wallet_configured = wallet_configured + if new_install is not None: + attributes.new_install = new_install + if reconciled is not None: + attributes.reconciled = reconciled + journal_module.record_failure( + journal_events.NodeJournalEvent.NODE_PROCESS_STARTUP_FAILED, + error=error, + attributes=attributes, + ) + + +def record_existing_config_detected( + *, + wallet_configured: bool, + account_count: int, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.EXISTING_CONFIG_DETECTED, + attributes=journal_models.JournalEventAttributes( + wallet_configured=wallet_configured, + account_count=account_count, + ), + ) + + +def record_reconcile_completed( + *, + automation_count: int, + running_automation_count: int, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.RECONCILE_COMPLETED, + attributes=journal_models.JournalEventAttributes( + automation_count=automation_count, + running_automation_count=running_automation_count, + ), + ) + + +def record_scheduler_init_failed( + *, + init_phase: journal_enums.JournalInitPhase, + backend: journal_enums.JournalSchedulerBackend, + error: BaseException, +) -> None: + journal_module.record_failure( + journal_events.NodeJournalEvent.SCHEDULER_INIT_FAILED, + error=error, + attributes=journal_models.JournalEventAttributes( + init_phase=recording_utils.enum_value(init_phase), + backend=recording_utils.enum_value(backend), + ), + ) diff --git a/octobot/community/node_journal/recording/strategies.py b/octobot/community/node_journal/recording/strategies.py new file mode 100644 index 0000000000..3857c251c3 --- /dev/null +++ b/octobot/community/node_journal/recording/strategies.py @@ -0,0 +1,64 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models + +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording._utils as recording_utils +import octobot.community.node_journal.recording.classify as journal_classify + + +def record_strategy_create_succeeded( + *, + strategy_id: str, + strategy: protocol_models.Strategy | None = None, + configuration_type: journal_enums.ConfigurationType | str | None = None, + user_action_id: str | None = None, +) -> None: + resolved_configuration_type = configuration_type + if strategy is not None: + resolved_configuration_type = journal_classify.configuration_type_from_strategy(strategy) + journal_module.record( + journal_events.NodeJournalEvent.STRATEGY_CREATE_SUCCEEDED, + attributes=journal_models.JournalEventAttributes( + strategy_id=strategy_id, + configuration_type=recording_utils.enum_value(resolved_configuration_type), + user_action_id=user_action_id, + ), + ) + + +def record_strategy_edit_succeeded( + *, + strategy_id: str, + strategy: protocol_models.Strategy | None = None, + configuration_type: journal_enums.ConfigurationType | str | None = None, + user_action_id: str | None = None, +) -> None: + resolved_configuration_type = configuration_type + if strategy is not None: + resolved_configuration_type = journal_classify.configuration_type_from_strategy(strategy) + journal_module.record( + journal_events.NodeJournalEvent.STRATEGY_EDIT_SUCCEEDED, + attributes=journal_models.JournalEventAttributes( + strategy_id=strategy_id, + configuration_type=recording_utils.enum_value(resolved_configuration_type), + user_action_id=user_action_id, + ), + ) diff --git a/octobot/community/node_journal/recording/sync.py b/octobot/community/node_journal/recording/sync.py new file mode 100644 index 0000000000..0b20eed569 --- /dev/null +++ b/octobot/community/node_journal/recording/sync.py @@ -0,0 +1,107 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import time +import uuid + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording._utils as recording_utils +import octobot.community.node_journal.state as journal_state + + +_tracker_reset_after_startup = False + + +def reset_sync_tracker_after_node_startup() -> None: + global _tracker_reset_after_startup + _tracker_reset_after_startup = True + + +def on_user_data_pull_succeeded(*, sync_user_id: str, collection: str) -> None: + global _tracker_reset_after_startup + if collection != "user-data": + return + now = time.time() + persisted_state = journal_state.load_persisted_state() + is_first_ever = persisted_state.last_user_data_pull_at is None + prior_gap_seconds = None + is_reconnect = False + if _tracker_reset_after_startup and not is_first_ever: + is_reconnect = True + prior_gap_seconds = round(now - persisted_state.last_user_data_pull_at, 3) + elif persisted_state.last_user_data_pull_at is not None: + gap_seconds = now - persisted_state.last_user_data_pull_at + if gap_seconds > journal_constants.SYNC_SESSION_GAP_SECONDS: + is_reconnect = True + prior_gap_seconds = round(gap_seconds, 3) + if is_first_ever or is_reconnect or _tracker_reset_after_startup: + persisted_state.connection_sequence += 1 + attributes = journal_models.JournalEventAttributes( + collection=collection, + sync_session_id=str(uuid.uuid4()), + connection_sequence=persisted_state.connection_sequence, + is_reconnect=not is_first_ever, + duration_since_install_start=journal_state.duration_since_install_start(now), + prior_gap_seconds=prior_gap_seconds, + ) + journal_module.record( + journal_events.NodeJournalEvent.EXTERNAL_INTERFACE_CONNECTED, + attributes=attributes, + ) + persisted_state.last_user_data_pull_at = now + journal_state.save_persisted_state(persisted_state) + _tracker_reset_after_startup = False + + +def record_sync_read_failed( + *, + collection: str, + failure_reason: journal_enums.SyncReadFailureReason, + error: BaseException | None = None, + error_message: str | None = None, +) -> None: + journal_module.record_failure( + journal_events.NodeJournalEvent.SYNC_READ_FAILED, + error=error, + error_message=error_message, + attributes=journal_models.JournalEventAttributes( + collection=collection, + failure_reason=recording_utils.enum_value(failure_reason), + ), + ) + + +def record_sync_storage_event( + event: journal_events.NodeJournalEvent, + *, + collection: str, + provider: journal_enums.SyncStorageProvider, + error: BaseException | None = None, + recovery_action: journal_enums.SyncStorageRecoveryAction | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes( + collection=collection, + provider=recording_utils.enum_value(provider), + recovery_action=recording_utils.enum_value(recovery_action) if recovery_action is not None else None, + ) + if error is None: + journal_module.record(event, attributes=attributes) + return + journal_module.record_failure(event, error=error, attributes=attributes) diff --git a/octobot/community/node_journal/recording/user_actions.py b/octobot/community/node_journal/recording/user_actions.py new file mode 100644 index 0000000000..ad32d7dcad --- /dev/null +++ b/octobot/community/node_journal/recording/user_actions.py @@ -0,0 +1,152 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models + +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording.accounts as accounts_module + + +ACTION_EVENT_MAP: dict[ + protocol_models.UserActionType, + tuple[journal_events.NodeJournalEvent, journal_events.NodeJournalEvent | None], +] = { + protocol_models.UserActionType.ACCOUNT_AUTH_CREATE: ( + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_ATTEMPT, + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_FAILED, + ), + protocol_models.UserActionType.ACCOUNT_CREATE: ( + journal_events.NodeJournalEvent.ACCOUNT_CREATE_ATTEMPT, + None, + ), + protocol_models.UserActionType.ACCOUNT_EDIT: ( + journal_events.NodeJournalEvent.ACCOUNT_EDIT_ATTEMPT, + journal_events.NodeJournalEvent.ACCOUNT_EDIT_FAILED, + ), + protocol_models.UserActionType.STRATEGY_CREATE: ( + journal_events.NodeJournalEvent.STRATEGY_CREATE_ATTEMPT, + journal_events.NodeJournalEvent.STRATEGY_CREATE_FAILED, + ), + protocol_models.UserActionType.STRATEGY_EDIT: ( + journal_events.NodeJournalEvent.STRATEGY_EDIT_ATTEMPT, + journal_events.NodeJournalEvent.STRATEGY_EDIT_FAILED, + ), + protocol_models.UserActionType.AUTOMATION_CREATE: ( + journal_events.NodeJournalEvent.AUTOMATION_CREATE_ATTEMPT, + journal_events.NodeJournalEvent.AUTOMATION_CREATE_FAILED, + ), + protocol_models.UserActionType.AUTOMATION_EDIT: ( + journal_events.NodeJournalEvent.AUTOMATION_EDIT_ATTEMPT, + journal_events.NodeJournalEvent.AUTOMATION_EDIT_FAILED, + ), +} + + +def record_external_action_received( + user_action: protocol_models.UserAction, + *, + source: str, +) -> None: + action_type = _resolve_user_action_type(user_action) + journal_module.record( + journal_events.NodeJournalEvent.EXTERNAL_ACTION_RECEIVED, + attributes=journal_models.JournalEventAttributes( + source=source, + action_type=action_type.value if action_type is not None else "unknown", + user_action_id=user_action.id, + ), + ) + _record_action_attempt(user_action, source=source) + + +def record_external_action_failed( + user_action: protocol_models.UserAction, + *, + source: str, + error: BaseException, +) -> None: + action_type = _resolve_user_action_type(user_action) + journal_module.record_failure( + journal_events.NodeJournalEvent.EXTERNAL_ACTION_FAILED, + error=error, + attributes=journal_models.JournalEventAttributes( + source=source, + action_type=action_type.value if action_type is not None else "unknown", + user_action_id=user_action.id, + ), + ) + _record_typed_action_failure(user_action, error=error) + + +def record_executor_failure( + user_action: protocol_models.UserAction, + *, + error: BaseException, +) -> None: + _record_typed_action_failure(user_action, error=error) + + +def _record_action_attempt(user_action: protocol_models.UserAction, *, source: str) -> None: + action_type = _resolve_user_action_type(user_action) + if action_type is None: + return + attempt_event, _failure_event = ACTION_EVENT_MAP.get(action_type, (None, None)) + if attempt_event is None: + return + journal_module.record( + attempt_event, + attributes=journal_models.JournalEventAttributes( + user_action_id=user_action.id, + source=source, + ), + ) + + +def _record_typed_action_failure( + user_action: protocol_models.UserAction, + *, + error: BaseException, +) -> None: + action_type = _resolve_user_action_type(user_action) + if action_type is None: + return + _attempt_event, failure_event = ACTION_EVENT_MAP.get(action_type, (None, None)) + if action_type == protocol_models.UserActionType.ACCOUNT_CREATE: + accounts_module.record_account_validation_failed( + is_simulated=False, + exchange_name=None, + error=error, + user_action_id=user_action.id, + ) + return + if failure_event is None: + return + journal_module.record_failure( + failure_event, + error=error, + attributes=journal_models.JournalEventAttributes(user_action_id=user_action.id), + ) + + +def _resolve_user_action_type( + user_action: protocol_models.UserAction, +) -> protocol_models.UserActionType | None: + configuration = user_action.configuration + if configuration is None or configuration.actual_instance is None: + return None + return configuration.actual_instance.action_type diff --git a/octobot/community/node_journal/recording/wallet.py b/octobot/community/node_journal/recording/wallet.py new file mode 100644 index 0000000000..1889df8cee --- /dev/null +++ b/octobot/community/node_journal/recording/wallet.py @@ -0,0 +1,80 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording._utils as recording_utils + + +def record_wallet_setup_attempt( + *, + node_type: str, + setup_method: journal_enums.WalletSetupMethod, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.WALLET_SETUP_ATTEMPT, + attributes=journal_models.JournalEventAttributes( + node_type=node_type, + setup_method=recording_utils.enum_value(setup_method), + ), + ) + + +def record_wallet_setup_succeeded() -> None: + journal_module.record( + journal_events.NodeJournalEvent.WALLET_SETUP_SUCCEEDED, + attributes=journal_models.JournalEventAttributes(configured=True), + ) + + +def record_wallet_setup_failed( + *, + http_status: int, + failure_reason: journal_enums.WalletSetupFailureReason, + error: BaseException | None = None, + error_message: str | None = None, + setup_method: journal_enums.WalletSetupMethod | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes( + http_status=http_status, + failure_reason=recording_utils.enum_value(failure_reason), + ) + if setup_method is not None: + attributes.setup_method = recording_utils.enum_value(setup_method) + journal_module.record_failure( + journal_events.NodeJournalEvent.WALLET_SETUP_FAILED, + error=error, + error_message=error_message, + attributes=attributes, + ) + + +def record_wallet_operation_failed( + *, + operation: journal_enums.WalletOperation, + error: BaseException, + http_status: int | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes(operation=recording_utils.enum_value(operation)) + if http_status is not None: + attributes.http_status = http_status + journal_module.record_failure( + journal_events.NodeJournalEvent.WALLET_OPERATION_FAILED, + error=error, + attributes=attributes, + ) diff --git a/octobot/community/node_journal/recording_context.py b/octobot/community/node_journal/recording_context.py new file mode 100644 index 0000000000..87d64561dc --- /dev/null +++ b/octobot/community/node_journal/recording_context.py @@ -0,0 +1,132 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import contextlib +import typing + +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.recording as journal_recording + + +class scheduler_init_phase(contextlib.AbstractContextManager): + def __init__( + self, + *, + init_phase: journal_enums.JournalInitPhase, + backend: journal_enums.JournalSchedulerBackend, + on_failure: typing.Callable[..., None], + ) -> None: + self._init_phase = init_phase + self._backend = backend + self._on_failure = on_failure + + def __exit__(self, exc_type, exc, traceback) -> bool: + del traceback + if exc is not None and exc_type is not None: + self._on_failure( + init_phase=self._init_phase, + backend=self._backend, + error=exc, + ) + return False + + +class sync_read_operation(contextlib.AbstractContextManager): + def __init__( + self, + *, + resolve_collection: typing.Callable[[], str], + resolve_failure_reason: typing.Callable[[BaseException], str], + ) -> None: + self._resolve_collection = resolve_collection + self._resolve_failure_reason = resolve_failure_reason + + def __exit__(self, exc_type, exc, traceback) -> bool: + del traceback + if exc is not None and exc_type is not None: + journal_recording.record_sync_read_failed( + collection=self._resolve_collection(), + failure_reason=self._resolve_failure_reason(exc), + error=exc, + ) + return False + + +class sync_storage_error(contextlib.AbstractContextManager): + def __init__( + self, + *, + event: journal_events.NodeJournalEvent, + collection: str, + provider: journal_enums.SyncStorageProvider = journal_enums.SyncStorageProvider.LOCAL, + recovery_action: journal_enums.SyncStorageRecoveryAction | None = None, + ) -> None: + self._event = event + self._collection = collection + self._provider = provider + self._recovery_action = recovery_action + + def __exit__(self, exc_type, exc, traceback) -> bool: + del traceback + if exc is not None and exc_type is not None: + journal_recording.record_sync_storage_event( + self._event, + collection=self._collection, + provider=self._provider, + error=exc, + recovery_action=self._recovery_action, + ) + return False + + +def raise_wallet_setup_http_error( + *, + http_status: int, + failure_reason: journal_enums.WalletSetupFailureReason, + setup_method: journal_enums.WalletSetupMethod, + detail: str, + error: BaseException | None = None, + error_message: str | None = None, +) -> typing.NoReturn: + journal_recording.record_wallet_setup_failed( + http_status=http_status, + failure_reason=failure_reason, + error=error, + error_message=error_message, + setup_method=setup_method, + ) + from fastapi import HTTPException + raise HTTPException(status_code=http_status, detail=detail) from error + + +def node_api_startup_failure( + *, + error: BaseException, + startup_phase: journal_enums.JournalStartupPhase, + force_exit: bool, + config, + scheduler_init_failure_was_recorded: typing.Callable[[], bool], + record_node_startup_failed: typing.Callable[..., None], +) -> None: + if scheduler_init_failure_was_recorded(): + return + record_node_startup_failed( + error, + startup_phase=startup_phase, + force_exit=force_exit, + config=config, + ) \ No newline at end of file diff --git a/octobot/community/node_journal/state.py b/octobot/community/node_journal/state.py new file mode 100644 index 0000000000..b9b76e6208 --- /dev/null +++ b/octobot/community/node_journal/state.py @@ -0,0 +1,225 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import dataclasses +import logging +import os +import time +import typing +import uuid + +import octobot_commons.configuration as configuration +import octobot_commons.json_util as json_util +import octobot_commons.user_root_folder_provider as user_root_folder_provider + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.journal as journal_module + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class JournalPersistedState: + install_id: str + onboarding_started_at: float | None = None + first_automation_started_at: float | None = None + connection_sequence: int = 0 + last_user_data_pull_at: float | None = None + onboarding_complete: bool = False + tracked_automation_ids: list[str] = dataclasses.field(default_factory=list) + + +_session_id: str | None = None +_persisted_state: JournalPersistedState | None = None +_config: configuration.Configuration | None = None + + +def _get_journal_section(config: configuration.Configuration) -> dict: + journal_section = config.config.setdefault(journal_constants.CONFIG_JOURNAL_SECTION, {}) + if not isinstance(journal_section, dict): + logger.error( + "%s must be a mapping in config", + journal_constants.CONFIG_JOURNAL_SECTION, + ) + return {} + return journal_section + + +def bind_config(config: configuration.Configuration | None) -> None: + global _config, _persisted_state + _config = config + _persisted_state = None + + +def get_config() -> configuration.Configuration | None: + return _config + + +def get_session_id() -> str: + global _session_id + if _session_id is None: + _session_id = str(uuid.uuid4()) + return _session_id + + +def reset_session_id() -> str: + global _session_id + _session_id = str(uuid.uuid4()) + return _session_id + + +def get_journal_directory() -> str: + user_root = user_root_folder_provider.get_user_root_folder() + return os.path.join(user_root, journal_constants.JOURNAL_DIR_NAME) + + +def _is_valid_manifest(manifest: typing.Any) -> bool: + manifest_field = journal_enums.JournalManifestField + if not isinstance(manifest, dict): + return False + if manifest.get(manifest_field.SCHEMA.value) != journal_constants.JOURNAL_SCHEMA_VERSION: + return False + install_id = manifest.get(manifest_field.INSTALL_ID.value) + return isinstance(install_id, str) and bool(install_id) + + +def ensure_journal_manifest(journal_directory: str | None = None) -> dict: + journal_dir = journal_directory or get_journal_directory() + os.makedirs(journal_dir, exist_ok=True) + manifest_path = os.path.join(journal_dir, journal_constants.MANIFEST_FILE_NAME) + manifest = None + if os.path.isfile(manifest_path): + manifest = json_util.read_file(manifest_path, raise_errors=False, on_error_value={}) + if manifest is not None and _is_valid_manifest(manifest): + return manifest + logger.warning("Recreating journal manifest at %s", manifest_path) + persisted_state = load_persisted_state() + manifest_field = journal_enums.JournalManifestField + recreated_manifest = { + manifest_field.SCHEMA.value: journal_constants.JOURNAL_SCHEMA_VERSION, + manifest_field.INSTALL_ID.value: persisted_state.install_id, + } + json_util.safe_dump(recreated_manifest, manifest_path) + return recreated_manifest + + +def load_persisted_state(config: configuration.Configuration | None = None) -> JournalPersistedState: + return journal_module.run_journal_operation( + "load_persisted_state", + lambda: _load_persisted_state(config), + default=_default_persisted_state(), + ) + + +def save_persisted_state(state: JournalPersistedState, config: configuration.Configuration | None = None) -> None: + journal_module.run_journal_operation( + "save_persisted_state", + lambda: _save_persisted_state(state, config), + default=None, + ) + + +def mark_first_automation_started(now: float | None = None) -> None: + journal_module.run_journal_operation( + "mark_first_automation_started", + lambda: _mark_first_automation_started(now), + default=None, + ) + + +def _default_persisted_state() -> JournalPersistedState: + global _persisted_state + if _persisted_state is not None: + return _persisted_state + _persisted_state = JournalPersistedState(install_id=str(uuid.uuid4())) + return _persisted_state + + +def _load_persisted_state(config: configuration.Configuration | None) -> JournalPersistedState: + global _persisted_state + resolved_config = config if config is not None else _config + if resolved_config is None: + return _default_persisted_state() + if _persisted_state is not None and config is None: + return _persisted_state + journal_section = _get_journal_section(resolved_config) + stored_install_id = journal_section.get(journal_constants.CONFIG_INSTALL_ID) + if not stored_install_id: + stored_install_id = str(uuid.uuid4()) + journal_section[journal_constants.CONFIG_INSTALL_ID] = stored_install_id + if journal_section.get(journal_constants.CONFIG_ONBOARDING_STARTED_AT) is None: + journal_section[journal_constants.CONFIG_ONBOARDING_STARTED_AT] = time.time() + resolved_config.save() + _persisted_state = JournalPersistedState( + install_id=str(stored_install_id), + onboarding_started_at=_optional_float(journal_section.get(journal_constants.CONFIG_ONBOARDING_STARTED_AT)), + first_automation_started_at=_optional_float( + journal_section.get(journal_constants.CONFIG_FIRST_AUTOMATION_STARTED_AT) + ), + connection_sequence=int(journal_section.get(journal_constants.CONFIG_CONNECTION_SEQUENCE, 0) or 0), + last_user_data_pull_at=_optional_float(journal_section.get(journal_constants.CONFIG_LAST_USER_DATA_PULL_AT)), + onboarding_complete=journal_section.get(journal_constants.CONFIG_FIRST_AUTOMATION_STARTED_AT) is not None, + tracked_automation_ids=list(journal_section.get(journal_constants.CONFIG_TRACKED_AUTOMATION_IDS, []) or []), + ) + return _persisted_state + + +def _save_persisted_state( + state: JournalPersistedState, + config: configuration.Configuration | None = None, +) -> None: + global _persisted_state + resolved_config = config if config is not None else _config + if resolved_config is None: + _persisted_state = state + return + journal_section = _get_journal_section(resolved_config) + journal_section[journal_constants.CONFIG_INSTALL_ID] = state.install_id + if state.onboarding_started_at is not None: + journal_section[journal_constants.CONFIG_ONBOARDING_STARTED_AT] = state.onboarding_started_at + if state.first_automation_started_at is not None: + journal_section[journal_constants.CONFIG_FIRST_AUTOMATION_STARTED_AT] = state.first_automation_started_at + journal_section[journal_constants.CONFIG_CONNECTION_SEQUENCE] = state.connection_sequence + if state.last_user_data_pull_at is not None: + journal_section[journal_constants.CONFIG_LAST_USER_DATA_PULL_AT] = state.last_user_data_pull_at + journal_section[journal_constants.CONFIG_TRACKED_AUTOMATION_IDS] = list(state.tracked_automation_ids) + resolved_config.save() + _persisted_state = state + + +def _mark_first_automation_started(now: float | None) -> None: + state = _load_persisted_state(None) + if state.first_automation_started_at is not None: + return + emit_now = time.time() if now is None else now + state.first_automation_started_at = emit_now + state.onboarding_complete = True + _save_persisted_state(state) + + +def duration_since_install_start(now: float | None = None) -> float | None: + persisted_state = load_persisted_state() + if persisted_state.onboarding_started_at is None: + return None + emit_now = time.time() if now is None else now + return round(max(0.0, emit_now - persisted_state.onboarding_started_at), 3) + + +def _optional_float(value: typing.Any) -> float | None: + if value is None: + return None + return float(value) diff --git a/octobot/community/node_journal/storage_hydration.py b/octobot/community/node_journal/storage_hydration.py new file mode 100644 index 0000000000..221f211549 --- /dev/null +++ b/octobot/community/node_journal/storage_hydration.py @@ -0,0 +1,87 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import logging + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.state as journal_state + +logger = logging.getLogger(__name__) + + +def _derive_onboarding_complete( + event_timestamp: float, + persisted_state: journal_state.JournalPersistedState, +) -> bool: + cutoff = persisted_state.first_automation_started_at + if cutoff is None: + return False + return event_timestamp > cutoff + + +def _log_missing_hydration_field(field_name: str, *, missing_data_warnings: set[str] | None) -> None: + if missing_data_warnings is not None and field_name in missing_data_warnings: + return + logger.warning("Journal event hydration missing %s", field_name) + if missing_data_warnings is not None: + missing_data_warnings.add(field_name) + + +def hydrate_storage_event( + raw: dict, + ctx: dict, + manifest: dict | None, + persisted_state: journal_state.JournalPersistedState, + *, + missing_data_warnings: set[str] | None = None, +) -> journal_models.JournalEventLine: + event_line_field = journal_enums.JournalEventLineField + manifest_field = journal_enums.JournalManifestField + context_field = journal_enums.JournalStorageContextField + + install_id = "" + if isinstance(manifest, dict) and manifest.get(manifest_field.INSTALL_ID.value): + install_id = str(manifest[manifest_field.INSTALL_ID.value]) + else: + _log_missing_hydration_field(manifest_field.INSTALL_ID.value, missing_data_warnings=missing_data_warnings) + + session_id = "" + if ctx.get(context_field.SESSION_ID.value): + session_id = str(ctx[context_field.SESSION_ID.value]) + else: + _log_missing_hydration_field(context_field.SESSION_ID.value, missing_data_warnings=missing_data_warnings) + + app_version = "" + if ctx.get(context_field.APP_VERSION.value): + app_version = str(ctx[context_field.APP_VERSION.value]) + else: + _log_missing_hydration_field(context_field.APP_VERSION.value, missing_data_warnings=missing_data_warnings) + + event_timestamp = float(raw[event_line_field.TIMESTAMP.value]) + expanded_line = { + event_line_field.EVENT.value: raw[event_line_field.EVENT.value], + event_line_field.TIMESTAMP.value: event_timestamp, + event_line_field.SESSION_ID.value: session_id, + event_line_field.INSTALL_ID.value: install_id, + event_line_field.APP_VERSION.value: app_version, + event_line_field.DISTRIBUTION.value: journal_constants.DISTRIBUTION_NODE, + event_line_field.ONBOARDING_COMPLETE.value: _derive_onboarding_complete(event_timestamp, persisted_state), + event_line_field.ATTRIBUTES.value: raw.get(event_line_field.ATTRIBUTES.value), + event_line_field.RECORDED.value: raw.get(event_line_field.RECORDED.value, True), + } + return journal_models.JournalEventLine.from_dict(expanded_line) diff --git a/octobot/community/node_journal/store.py b/octobot/community/node_journal/store.py new file mode 100644 index 0000000000..3d58ba7f4b --- /dev/null +++ b/octobot/community/node_journal/store.py @@ -0,0 +1,321 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import dataclasses +import json +import logging +import os +import threading +import typing + +import octobot.constants as octobot_constants + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.storage_hydration as journal_storage_hydration +import octobot.community.node_journal.state as journal_state + +logger = logging.getLogger(__name__) + + +def run_journal_store_operation( + operation_name: str, + operation: typing.Callable[[], typing.Any], + *, + default: typing.Any, +) -> typing.Any: + try: + return operation() + except Exception as exc: + logger.exception("Journal %s failed: %s", operation_name, exc) + return default + + +@dataclasses.dataclass +class _FileWriteState: + last_ctx: dict | None = None + + +class JournalStore: + def __init__( + self, + *, + max_events: int = journal_constants.JOURNAL_MAX_EVENTS, + journal_directory: str | None = None, + ) -> None: + self._max_events = max_events + self._journal_directory = journal_directory or journal_state.get_journal_directory() + self._events_path = os.path.join(self._journal_directory, journal_constants.EVENTS_FILE_NAME) + self._onboarding_path = os.path.join( + self._journal_directory, journal_constants.ONBOARDING_SEGMENT_FILE_NAME + ) + self._lock = threading.Lock() + self._file_write_states: dict[str, _FileWriteState] = {} + os.makedirs(self._journal_directory, exist_ok=True) + journal_state.ensure_journal_manifest(self._journal_directory) + + def append(self, event_line: journal_models.JournalEventLine, *, is_onboarding_segment: bool) -> None: + run_journal_store_operation( + "store.append", + lambda: self._append(event_line, is_onboarding_segment=is_onboarding_segment), + default=None, + ) + + def read_all_events(self) -> list[journal_models.JournalEventLine]: + return run_journal_store_operation( + "store.read_all_events", + self._read_all_events, + default=[], + ) + + def _append(self, event_line: journal_models.JournalEventLine, *, is_onboarding_segment: bool) -> None: + journal_state.ensure_journal_manifest(self._journal_directory) + with self._lock: + if is_onboarding_segment: + self._append_to_file(self._onboarding_path, event_line) + else: + self._append_to_file(self._events_path, event_line) + self._enforce_cap() + + def _read_all_events(self) -> list[journal_models.JournalEventLine]: + manifest = journal_state.ensure_journal_manifest(self._journal_directory) + with self._lock: + pinned_onboarding = self._parse_jsonl_file(self._onboarding_path, manifest=manifest) + main_events = self._parse_jsonl_file(self._events_path, manifest=manifest) + if not pinned_onboarding: + return main_events + merged_by_key = {} + for event_line in pinned_onboarding + main_events: + merged_by_key[self._event_dedup_key(event_line)] = event_line + return sorted(merged_by_key.values(), key=lambda line: line.timestamp) + + def _enforce_cap(self) -> None: + manifest = journal_state.ensure_journal_manifest(self._journal_directory) + main_events = self._parse_jsonl_file(self._events_path, manifest=manifest) + if len(main_events) <= self._max_events: + return + persisted_state = journal_state.load_persisted_state() + onboarding_cutoff = persisted_state.first_automation_started_at + protected_events = [] + evictable_events = [] + for event_line in main_events: + if onboarding_cutoff is None or event_line.timestamp <= onboarding_cutoff: + protected_events.append(event_line) + else: + evictable_events.append(event_line) + overflow = len(main_events) - self._max_events + if overflow <= 0: + return + if len(evictable_events) >= overflow: + kept_evictable = evictable_events[overflow:] + else: + kept_evictable = [] + protected_overflow = overflow - len(evictable_events) + protected_events = protected_events[protected_overflow:] + kept_events = protected_events + kept_evictable + self._write_jsonl_file(self._events_path, kept_events) + + def _get_write_state(self, path: str) -> _FileWriteState: + if path not in self._file_write_states: + self._file_write_states[path] = _FileWriteState() + return self._file_write_states[path] + + def _current_ctx(self) -> dict[str, str]: + context_field = journal_enums.JournalStorageContextField + return { + context_field.SESSION_ID.value: journal_state.get_session_id(), + context_field.APP_VERSION.value: octobot_constants.LONG_VERSION, + } + + def _should_write_ctx(self, path: str, current_ctx: dict[str, str], write_state: _FileWriteState) -> bool: + context_field = journal_enums.JournalStorageContextField + if not os.path.isfile(path) or os.path.getsize(path) == 0: + return True + if write_state.last_ctx is None: + return True + return ( + write_state.last_ctx.get(context_field.SESSION_ID.value) != current_ctx[context_field.SESSION_ID.value] + or write_state.last_ctx.get(context_field.APP_VERSION.value) != current_ctx[context_field.APP_VERSION.value] + ) + + def _append_to_file(self, path: str, event_line: journal_models.JournalEventLine) -> None: + current_ctx = self._current_ctx() + write_state = self._get_write_state(path) + lines_to_write = [] + if self._should_write_ctx(path, current_ctx, write_state): + lines_to_write.append(self._serialize_ctx_line(current_ctx)) + lines_to_write.append(json.dumps(event_line.to_storage_dict(), separators=(",", ":"), sort_keys=True)) + with open(path, "a", encoding="utf-8") as jsonl_file: + for serialized_line in lines_to_write: + jsonl_file.write(serialized_line + "\n") + write_state.last_ctx = current_ctx + + @staticmethod + def _serialize_ctx_line(ctx: dict[str, str]) -> str: + return json.dumps( + {journal_constants.STORAGE_CTX_KEY: ctx}, + separators=(",", ":"), + sort_keys=True, + ) + + @staticmethod + def _event_dedup_key(event_line: journal_models.JournalEventLine) -> tuple: + return ( + event_line.timestamp, + event_line.event.value, + event_line.session_id, + json.dumps(event_line.attributes.to_dict(), sort_keys=True), + ) + + def _parse_jsonl_file( + self, + path: str, + *, + manifest: dict, + ) -> list[journal_models.JournalEventLine]: + if not os.path.isfile(path): + return [] + persisted_state = journal_state.load_persisted_state() + parsed_lines = [] + running_ctx: dict = {} + missing_data_warnings: set[str] = set() + with open(path, encoding="utf-8") as jsonl_file: + for line_number, raw_line in enumerate(jsonl_file, start=1): + stripped_line = raw_line.strip() + if not stripped_line: + continue + try: + parsed_line = json.loads(stripped_line) + except json.JSONDecodeError as exc: + logger.warning( + "Skipping invalid journal line %s:%s (%s: %s)", + path, + line_number, + type(exc).__name__, + exc, + ) + continue + if journal_constants.STORAGE_CTX_KEY in parsed_line: + ctx_payload = parsed_line[journal_constants.STORAGE_CTX_KEY] + if isinstance(ctx_payload, dict): + running_ctx = ctx_payload + else: + logger.warning( + "Skipping invalid journal context line %s:%s", + path, + line_number, + ) + continue + if journal_enums.JournalEventLineField.EVENT.value not in parsed_line: + logger.warning( + "Skipping unrecognized journal line %s:%s", + path, + line_number, + ) + continue + try: + parsed_lines.append( + journal_storage_hydration.hydrate_storage_event( + parsed_line, + running_ctx, + manifest, + persisted_state, + missing_data_warnings=missing_data_warnings, + ) + ) + except Exception as exc: + logger.warning( + "Skipping invalid journal line %s:%s (%s: %s)", + path, + line_number, + type(exc).__name__, + exc, + ) + return parsed_lines + + def _write_jsonl_file(self, path: str, events: list[journal_models.JournalEventLine]) -> None: + write_state = self._get_write_state(path) + if not events: + with open(path, "w", encoding="utf-8"): + pass + write_state.last_ctx = None + return + + grouped_lines: list[str] = [] + current_group_key: tuple[str, str] | None = None + current_group_events: list[journal_models.JournalEventLine] = [] + for event_line in events: + group_key = (event_line.session_id, event_line.app_version) + if group_key != current_group_key: + if current_group_events and current_group_key is not None: + grouped_lines.extend( + self._serialize_event_group(current_group_key, current_group_events) + ) + current_group_key = group_key + current_group_events = [event_line] + else: + current_group_events.append(event_line) + if current_group_events and current_group_key is not None: + grouped_lines.extend(self._serialize_event_group(current_group_key, current_group_events)) + + with open(path, "w", encoding="utf-8") as jsonl_file: + jsonl_file.write("\n".join(grouped_lines) + "\n") + + if current_group_key is not None: + last_session_id, last_app_version = current_group_key + context_field = journal_enums.JournalStorageContextField + write_state.last_ctx = { + context_field.SESSION_ID.value: last_session_id, + context_field.APP_VERSION.value: last_app_version, + } + + def _serialize_event_group( + self, + group_key: tuple[str, str], + group_events: list[journal_models.JournalEventLine], + ) -> list[str]: + session_id, app_version = group_key + context_field = journal_enums.JournalStorageContextField + serialized_lines = [ + self._serialize_ctx_line({ + context_field.SESSION_ID.value: session_id, + context_field.APP_VERSION.value: app_version, + }), + ] + for event_line in group_events: + serialized_lines.append( + json.dumps(event_line.to_storage_dict(), separators=(",", ":"), sort_keys=True) + ) + return serialized_lines + + +_default_store: JournalStore | None = None + + +def get_store(*, max_events: int | None = None) -> JournalStore: + global _default_store + if _default_store is None or (max_events is not None and _default_store._max_events != max_events): + kwargs = {} + if max_events is not None: + kwargs["max_events"] = max_events + _default_store = JournalStore(**kwargs) + return _default_store + + +def reset_default_store() -> None: + global _default_store + _default_store = None diff --git a/octobot/community/supabase_backend/community_supabase_client.py b/octobot/community/supabase_backend/community_supabase_client.py index 3c5a3c0358..5f38c3a3bf 100644 --- a/octobot/community/supabase_backend/community_supabase_client.py +++ b/octobot/community/supabase_backend/community_supabase_client.py @@ -54,7 +54,7 @@ _INTERNAL_LOGGERS = [ "httpx", "asyncio", - "httpcore.http11", "httpcore.connection", + "httpcore.http11", "httpcore.http2", "httpcore.connection", ] # disable httpx info logs as it logs every request commons_logging.set_logging_level(_INTERNAL_LOGGERS, logging.WARNING) diff --git a/octobot/community/wallet_backend/__init__.py b/octobot/community/wallet_backend/__init__.py index 4216a560d9..8b6f73990e 100644 --- a/octobot/community/wallet_backend/__init__.py +++ b/octobot/community/wallet_backend/__init__.py @@ -31,6 +31,8 @@ CannotRemoveAdminWalletError, InvalidPrivateKeyError, PassphraseTooShortError, + WalletProofMismatchError, + WalletStorageReadOnlyError, ) from octobot.community.wallet_backend import wallet_storage from octobot.community.wallet_backend.wallet_storage import ( @@ -54,6 +56,8 @@ "CannotRemoveAdminWalletError", "InvalidPrivateKeyError", "PassphraseTooShortError", + "WalletProofMismatchError", + "WalletStorageReadOnlyError", "WalletStorage", "ConfigJsonWalletStorage", "DedicatedFileWalletStorage", diff --git a/octobot/community/wallet_backend/community_wallet.py b/octobot/community/wallet_backend/community_wallet.py index 141e2f83d9..fe11f578be 100644 --- a/octobot/community/wallet_backend/community_wallet.py +++ b/octobot/community/wallet_backend/community_wallet.py @@ -32,7 +32,10 @@ InvalidPrivateKeyError, PassphraseTooShortError, WalletAlreadyExistsError, + WalletError, WalletNotFoundError, + WalletProofMismatchError, + WalletStorageReadOnlyError, ) from octobot.community.wallet_backend.wallet_storage import ( WalletStorage, @@ -43,6 +46,16 @@ _PBKDF2_ALG = "sha256" +def _record_wallet_operation_failure(*, operation: str, error: WalletError) -> None: + if isinstance( + error, + (InvalidPrivateKeyError, WalletAlreadyExistsError, InvalidPassphraseError), + ): + return + import octobot.community.node_journal as node_journal + node_journal.record_wallet_operation_failed(operation=operation, error=error) + + @dataclasses.dataclass class WalletInfo(commons_dataclasses.FlexibleDataclass): address: str = "" @@ -137,7 +150,9 @@ def import_wallet( try: address = sync_chain.address_from_evm_key(private_key) except Exception as err: - raise InvalidPrivateKeyError(f"Invalid EVM private key: {err}") from err + wallet_error = InvalidPrivateKeyError(f"Invalid EVM private key: {err}") + _record_wallet_operation_failure(operation="import", error=wallet_error) + raise wallet_error from err return self._add_wallet_entry(private_key, address, name, passphrase, is_admin) def import_wallet_from_seed( @@ -150,7 +165,9 @@ def import_wallet_from_seed( try: wallet = sync_chain.wallet_from_mnemonic(seed.strip()) except Exception as err: - raise InvalidPrivateKeyError(f"Invalid seed phrase: {err}") from err + wallet_error = InvalidPrivateKeyError(f"Invalid seed phrase: {err}") + _record_wallet_operation_failure(operation="import", error=wallet_error) + raise wallet_error from err return self._add_wallet_entry(wallet.private_key, wallet.address, name, passphrase, is_admin, seed=seed.strip()) def _add_wallet_entry( @@ -163,14 +180,20 @@ def _add_wallet_entry( seed: typing.Optional[str] = None, ) -> sync_chain.Wallet: if len(passphrase) < 8: - raise PassphraseTooShortError("Passphrase must be at least 8 characters") + wallet_error = PassphraseTooShortError("Passphrase must be at least 8 characters") + _record_wallet_operation_failure(operation="create", error=wallet_error) + raise wallet_error normalized = address.lower() with self._wallet_lock: node_wallets = self._get_node_wallets_list() if any(e.address == normalized for e in node_wallets): - raise WalletAlreadyExistsError(f"Wallet {address} already exists") + wallet_error = WalletAlreadyExistsError(f"Wallet {address} already exists") + _record_wallet_operation_failure(operation="import", error=wallet_error) + raise wallet_error if is_admin and any(e.is_admin for e in node_wallets): - raise AdminWalletAlreadyExistsError("An admin wallet already exists") + wallet_error = AdminWalletAlreadyExistsError("An admin wallet already exists") + _record_wallet_operation_failure(operation="create", error=wallet_error) + raise wallet_error entry = WalletEntry( address=normalized, name=name or None, @@ -191,9 +214,13 @@ def authenticate(self, address: str, passphrase: str) -> WalletInfo: """ entry = self._find_wallet_entry(address) if entry is None: - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error if not _verify_passphrase_hash(passphrase, entry.passphrase_hash): - raise InvalidPassphraseError("Invalid passphrase") + wallet_error = InvalidPassphraseError("Invalid passphrase") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error return WalletInfo(is_admin=entry.is_admin, name=entry.name, address=entry.address) def verify_wallet_passphrase(self, address: str, passphrase: str) -> bool: @@ -206,26 +233,42 @@ def verify_wallet_passphrase(self, address: str, passphrase: str) -> bool: def decrypt_wallet_by_address(self, address: str, passphrase: str) -> sync_chain.Wallet: entry = self._find_wallet_entry(address) if entry is None: - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error if not _verify_passphrase_hash(passphrase, entry.passphrase_hash): - raise InvalidPassphraseError("Invalid passphrase") + wallet_error = InvalidPassphraseError("Invalid passphrase") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error return self._wallet_from_entry(entry) def decrypt_wallet_entry_by_address(self, address: str, passphrase: str) -> WalletEntry: entry = self._find_wallet_entry(address) if entry is None: - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error if not _verify_passphrase_hash(passphrase, entry.passphrase_hash): - raise InvalidPassphraseError("Invalid passphrase") + wallet_error = InvalidPassphraseError("Invalid passphrase") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error return entry def get_wallet_for_bot(self, address: str) -> sync_chain.Wallet: """Return wallet without passphrase verification — for bot auto-unlock at startup.""" entry = self._find_wallet_entry(address) if entry is None: - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="lookup", error=wallet_error) + raise wallet_error return self._wallet_from_entry(entry) + def has_wallet_for_user_id(self, user_id: str) -> bool: + for entry in self._get_node_wallets_list(): + if sync_auth.derive_user_id(entry.private_key) == user_id: + return True + return False + def get_wallet_by_user_id(self, user_id: str) -> sync_chain.Wallet: """Return the wallet whose derived Starfish ``user_id`` matches *user_id*. @@ -239,19 +282,27 @@ def get_wallet_by_user_id(self, user_id: str) -> sync_chain.Wallet: for entry in self._get_node_wallets_list(): if sync_auth.derive_user_id(entry.private_key) == user_id: return self._wallet_from_entry(entry) - raise WalletNotFoundError(f"Wallet not found for user_id: {user_id}") + wallet_error = WalletNotFoundError(f"Wallet not found for user_id: {user_id}") + _record_wallet_operation_failure(operation="lookup", error=wallet_error) + raise wallet_error def remove_wallet(self, address: str) -> None: normalized = address.lower() with self._wallet_lock: node_wallets = self._get_node_wallets_list() if len(node_wallets) <= 1: - raise CannotRemoveLastWalletError("Cannot remove the last wallet") + wallet_error = CannotRemoveLastWalletError("Cannot remove the last wallet") + _record_wallet_operation_failure(operation="delete", error=wallet_error) + raise wallet_error entry = next((e for e in node_wallets if e.address == normalized), None) if entry is None: - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="delete", error=wallet_error) + raise wallet_error if entry.is_admin: - raise CannotRemoveAdminWalletError("Cannot remove the admin wallet") + wallet_error = CannotRemoveAdminWalletError("Cannot remove the admin wallet") + _record_wallet_operation_failure(operation="delete", error=wallet_error) + raise wallet_error self._save_node_wallets_list([e for e in node_wallets if e.address != normalized]) def rename_wallet(self, address: str, name: typing.Optional[str]) -> None: @@ -263,7 +314,9 @@ def rename_wallet(self, address: str, name: typing.Optional[str]) -> None: entry.name = name or None self._save_node_wallets_list(node_wallets) return - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="rename", error=wallet_error) + raise wallet_error def is_admin_wallet(self, address: str) -> bool: entry = self._find_wallet_entry(address) @@ -272,3 +325,72 @@ def is_admin_wallet(self, address: str) -> bool: def get_wallet_name(self, address: str) -> typing.Optional[str]: entry = self._find_wallet_entry(address) return entry.name if entry else None + + def recover_passphrase_from_ownership_proof( + self, + address: str, + new_passphrase: str, + seed: typing.Optional[str] = None, + private_key: typing.Optional[str] = None, + ) -> None: + """Replace passphrase_hash after proving ownership via BIP39 seed or hex private key.""" + if len(new_passphrase) < 8: + raise PassphraseTooShortError("Passphrase must be at least 8 characters") + + seed_value = seed.strip() if seed else "" + key_value = private_key.strip() if private_key else "" + if bool(seed_value) == bool(key_value): + raise InvalidPrivateKeyError( + "Provide exactly one of seed phrase or private key" + ) + + normalized_target = address.lower() + if self._find_wallet_entry(address) is None: + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="recover_passphrase", error=wallet_error) + raise wallet_error + + try: + if seed_value: + derived = sync_chain.wallet_from_mnemonic(seed_value) + else: + derived_address = sync_chain.address_from_evm_key(key_value) + derived = sync_chain.Wallet(private_key=key_value, address=derived_address) + except Exception as err: + raise InvalidPrivateKeyError("Invalid seed phrase or private key") from err + + if derived.address.lower() != normalized_target: + raise WalletProofMismatchError( + "Seed phrase or private key does not match this wallet" + ) + + new_hash = _hash_passphrase(new_passphrase) + with self._wallet_lock: + node_wallets = self._get_node_wallets_list() + updated: list[WalletEntry] = [] + found = False + for entry in node_wallets: + if entry.address == normalized_target: + found = True + updated.append( + WalletEntry( + address=entry.address, + name=entry.name, + is_admin=entry.is_admin, + private_key=entry.private_key, + passphrase_hash=new_hash, + seed=entry.seed, + ) + ) + else: + updated.append(entry) + if not found: + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="recover_passphrase", error=wallet_error) + raise wallet_error + try: + self._save_node_wallets_list(updated) + except NotImplementedError as err: + raise WalletStorageReadOnlyError( + "Wallet storage is read-only; passphrase cannot be changed on this node" + ) from err diff --git a/octobot/community/wallet_backend/errors.py b/octobot/community/wallet_backend/errors.py index c523c4fbde..a9f0371348 100644 --- a/octobot/community/wallet_backend/errors.py +++ b/octobot/community/wallet_backend/errors.py @@ -49,3 +49,11 @@ class InvalidPrivateKeyError(WalletError): class PassphraseTooShortError(WalletError): pass + + +class WalletProofMismatchError(WalletError): + pass + + +class WalletStorageReadOnlyError(WalletError): + pass diff --git a/octobot/config/config_schema.json b/octobot/config/config_schema.json index a4aefdff21..17e8fd1297 100644 --- a/octobot/config/config_schema.json +++ b/octobot/config/config_schema.json @@ -47,6 +47,32 @@ } } }, + "journal": { + "type": "object", + "properties": { + "install_id": { + "type": "string" + }, + "onboarding_started_at": { + "type": ["number", "null"] + }, + "first_automation_started_at": { + "type": ["number", "null"] + }, + "connection_sequence": { + "type": "integer" + }, + "last_user_data_pull_at": { + "type": ["number", "null"] + }, + "tracked_automation_ids": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "backtesting": { "type": "object", "properties": { @@ -74,17 +100,6 @@ } } }, - "metrics": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - }, - "activity_bot_id": { - "type": "string" - } - } - }, "community-token": { "type": "string" }, @@ -138,7 +153,5 @@ "services", "notification" ], - "additionalProperties": { - "type": "string" - } + "additionalProperties": true } diff --git a/octobot/constants.py b/octobot/constants.py index d85952686a..ac048e845b 100644 --- a/octobot/constants.py +++ b/octobot/constants.py @@ -95,18 +95,6 @@ CLICKHOUSE_USERNAME = os.getenv("CLICKHOUSE_USERNAME") CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD") -ICEBERG_CATALOG_NAMESPACE = os.getenv("ICEBERG_CATALOG_NAMESPACE") -ICEBERG_CATALOG_NAME = os.getenv("ICEBERG_CATALOG_NAME") -ICEBERG_CATALOG_WAREHOUSE = os.getenv("ICEBERG_CATALOG_WAREHOUSE") -ICEBERG_OHLCV_HISTORY_TABLE = os.getenv("ICEBERG_OHLCV_HISTORY_TABLE") -ICEBERG_CATALOG_URI = os.getenv("ICEBERG_CATALOG_URI") -ICEBERG_CATALOG_TOKEN = os.getenv("ICEBERG_CATALOG_TOKEN") -ICEBERG_S3_ACCESS_KEY = os.getenv("ICEBERG_S3_ACCESS_KEY") -ICEBERG_S3_SECRET_KEY = os.getenv("ICEBERG_S3_SECRET_KEY") -ICEBERG_S3_REGION = os.getenv("ICEBERG_S3_REGION") -ICEBERG_S3_ENDPOINT = os.getenv("ICEBERG_S3_ENDPOINT") -CREATE_ICEBERG_DB_IF_MISSING = os_util.parse_boolean_environment_var("CREATE_ICEBERG_DB_IF_MISSING", "false") - OCTOBOT_MARKET_MAKING_URL = os.getenv("OCTOBOT_MARKET_MAKING_URL", "https://market-making.octobot.cloud") # sync server @@ -116,8 +104,7 @@ SYNC_NAMESPACE=os.getenv("SYNC_NAMESPACE", "dk") SYNC_CHAIN_ID = os.getenv("SYNC_CHAIN_ID", "evm:8453") -ERROR_TRACKER_DSN = os.getenv("ERROR_TRACKER_DSN") # has priority over ACTIVITY_TRACKER_DSN if set -ACTIVITY_TRACKER_DSN = os.getenv("ACTIVITY_TRACKER_DSN", "https://c2d3481420c3d1d5f3017c5508e7802f@o4508733888331776.ingest.de.sentry.io/4508733892132944") +ERROR_TRACKER_DSN = os.getenv("ERROR_TRACKER_DSN") # Sentry error reporting only; unset disables Sentry CONFIG_COMMUNITY = "community" CONFIG_COMMUNITY_BOT_ID = "bot_id" @@ -246,6 +233,9 @@ os.getenv("CLOUD_FIRST_METRICS_UPDATE_TIME", 5) ) +# Metrics — onboarding funnel +METRICS_GENERIC_EXCHANGE_NAME = "generic" + # config types keys CONFIG_KEY = "config" TENTACLES_SETUP_CONFIG_KEY = "tentacles_setup" diff --git a/octobot/enums.py b/octobot/enums.py index a4dd8911fd..6acc483660 100644 --- a/octobot/enums.py +++ b/octobot/enums.py @@ -25,8 +25,7 @@ class CommunityFeedType(enum.Enum): class CommunityHistoricalBackendType(enum.Enum): Clickhouse = "Clickhouse" - Iceberg = "Iceberg" - DEFAULT = Iceberg + DEFAULT = Clickhouse class WalletStorageBackend(str, enum.Enum): diff --git a/octobot/octobot.py b/octobot/octobot.py index ce2e44ed67..93340f4fc9 100644 --- a/octobot/octobot.py +++ b/octobot/octobot.py @@ -36,6 +36,8 @@ import octobot.logger as logger import octobot.community as community +import octobot.community.community_bot_stats as community_bot_stats +import octobot.community.node_journal.lifecycle as journal_startup import octobot.constants as constants import octobot.enums as enums import octobot.configuration_manager as configuration_manager @@ -78,9 +80,6 @@ def __init__(self, config: configuration.Configuration, community_authenticator= # unique aiohttp session: to be initialized from getter in a task self._aiohttp_session = None - # community if enabled - self.activity_metrics = None - # use edited config in community authentication community_config = self.get_edited_config(constants.CONFIG_KEY, dict_only=False) self.community_auth = community_authenticator or community.CommunityAuthentication.create(community_config) @@ -95,6 +94,8 @@ def __init__(self, config: configuration.Configuration, community_authenticator= # octobot_api to request the current instance self.octobot_api = octobot_api.OctoBotAPI(self) + self.community_bot_stats = community_bot_stats.CommunityBotStats(self.octobot_api) + # Logger self.logger = logging.get_logger(self.__class__.__name__) @@ -166,6 +167,10 @@ async def _post_initialize(self): self.automation = automation.Automation(self.bot_id, self.tentacles_setup_config) self._init_metadata_run_task = asyncio.create_task(self._store_run_metadata_when_available()) await self._init_profile_synchronizer() + if configuration_manager.get_distribution(self.config) is enums.OctoBotDistribution.NODE: + journal_startup.record_node_startup_succeeded( + self.get_edited_config(constants.CONFIG_KEY, dict_only=False) + ) async def _wait_for_run_data_init(self, exchange_managers, timeout): for exchange_manager in exchange_managers: @@ -243,9 +248,7 @@ async def _start_tools_tasks(self): await self.task_manager.start_tools_tasks() def _init_community(self): - self.activity_metrics = community.ActivityMetrics(self.octobot_api) - distribution = configuration_manager.get_distribution(self.config) - self.activity_metrics.setup_activity_tracking(distribution) + self.community_bot_stats.task_enabled = True async def _ensure_clock(self): if trading_api.is_trader_enabled_in_config(self.config) and constants.ENABLE_CLOCK_SYNCH: diff --git a/octobot/octobot_api.py b/octobot/octobot_api.py index 493842b9c6..7503ba4d9a 100644 --- a/octobot/octobot_api.py +++ b/octobot/octobot_api.py @@ -91,9 +91,6 @@ def get_aiohttp_session(self) -> object: def get_automation(self) -> automation.Automation: return self._octobot.automation - def get_activity_metrics(self): - return self._octobot.activity_metrics - def get_interface(self, interface_class): for interface in self._octobot.interface_producer.interfaces: if isinstance(interface, interface_class): diff --git a/octobot/task_manager.py b/octobot/task_manager.py index f921fb66ea..6d31110651 100644 --- a/octobot/task_manager.py +++ b/octobot/task_manager.py @@ -61,8 +61,8 @@ def init_async_loop(self): async def start_tools_tasks(self): task_list = [] - if self.octobot.activity_metrics: - task_list.append(self.octobot.activity_metrics.start_community_task()) + if self.octobot.community_bot_stats.task_enabled: + task_list.append(self.octobot.community_bot_stats.start_community_task()) self.octobot.async_loop = self.async_loop self.ready = True @@ -116,9 +116,8 @@ async def stop_timeout(timeout): self._process_bot_state_dump_task.cancel() self._process_bot_state_dump_task = None - # close community session - if self.octobot.activity_metrics: - stop_coroutines.append(self.octobot.activity_metrics.stop_task()) + if self.octobot.community_bot_stats.task_enabled: + stop_coroutines.append(self.octobot.community_bot_stats.stop_task()) async def _await_timeouted_gather(tasks): # await this gather to be sure to complete each stop call or timeout diff --git a/package-lock.json b/package-lock.json index 7639e11819..d5e5028e24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1690,7 +1690,7 @@ "version": "0.8.1", "license": "LGPL-3.0", "dependencies": { - "@drakkar.software/octobot-protocol": "^0.8.0", + "@drakkar.software/octobot-protocol": "^0.9.0", "@drakkar.software/starfish-client": "3.0.0-alpha.72", "@drakkar.software/starfish-identities": "3.0.0-alpha.72", "@drakkar.software/starfish-keyring": "3.0.0-alpha.72", @@ -1711,7 +1711,7 @@ }, "packages/protocol/octobot_protocol_ts": { "name": "@drakkar.software/octobot-protocol", - "version": "0.8.0", + "version": "0.9.0", "license": "GPL-3.0" } } diff --git a/packages/agents/AGENTS.md b/packages/agents/AGENTS.md new file mode 100644 index 0000000000..cc11a6f7e1 --- /dev/null +++ b/packages/agents/AGENTS.md @@ -0,0 +1,31 @@ +# Agents: agents + +## Role + +Agent automation primitives (OctoBot agents package) used by newer automation flows. + +## Owns + +- `packages/agents/octobot_agents/` (layout per repo) +- `packages/agents/tests/` + +## Public surface + +- `octobot_agents` modules referenced by node/automation features + +## May depend on + +- `octobot_commons` and packages declared in agents `requirements` + +## Do not + +- Couple directly to tentacle UI without going through documented APIs + +## Tests + +- **cwd:** `packages/agents` +- **Example:** `pytest tests -n auto --dist loadfile` + +## Last reviewed + +- 2026-09-16 diff --git a/packages/async_channel/AGENTS.md b/packages/async_channel/AGENTS.md new file mode 100644 index 0000000000..5a52215f32 --- /dev/null +++ b/packages/async_channel/AGENTS.md @@ -0,0 +1,31 @@ +# Agents: async_channel + +## Role + +Async consumer/producer channels and queues between OctoBot subsystems. + +## Owns + +- `packages/async_channel/octobot_async_channel/` +- `packages/async_channel/tests/` + +## Public surface + +- `octobot_async_channel` channel primitives + +## May depend on + +- Standard library + small deps only + +## Do not + +- Pull in trading or tentacles + +## Tests + +- **cwd:** `packages/async_channel` +- **Example:** `pytest tests -n auto --dist loadfile` + +## Last reviewed + +- 2026-09-16 diff --git a/packages/backtesting/AGENTS.md b/packages/backtesting/AGENTS.md new file mode 100644 index 0000000000..5c429b4e09 --- /dev/null +++ b/packages/backtesting/AGENTS.md @@ -0,0 +1,31 @@ +# Agents: backtesting + +## Role + +Historical simulation and backtesting engine over trading + evaluators data. + +## Owns + +- `packages/backtesting/octobot_backtesting/` +- `packages/backtesting/tests/` + +## Public surface + +- `octobot_backtesting` APIs used by tentacles and CLI + +## May depend on + +- `octobot_commons`, `octobot_trading`, `octobot_evaluators` (as existing imports) + +## Do not + +- Change exchange live-trading connectors for backtest-only shortcuts without tests + +## Tests + +- **cwd:** `packages/backtesting` +- **Example:** `pytest tests -n auto --dist loadfile` + +## Last reviewed + +- 2026-09-16 diff --git a/packages/binary/AGENTS.md b/packages/binary/AGENTS.md new file mode 100644 index 0000000000..a9842e4012 --- /dev/null +++ b/packages/binary/AGENTS.md @@ -0,0 +1,43 @@ +# Agents: binary + +## Role + +Builds the single-file PyInstaller executables of OctoBot (Linux, Windows, macOS) that CI publishes as workflow artifacts and release assets. Holds the build scripts and the helpers that list modules and fetch data for PyInstaller. It does not contain application logic. + +## Owns + +- `packages/binary/build_scripts/` (per-platform build entry points) +- `packages/binary/scripts/` (`python_file_lister.py`, `fetch_nltk_data.py`) +- `packages/binary/requirements.txt` +- `packages/binary/README.md` +- `packages/binary/BINARY_TESTING_INSTRUCTIONS.md` + +## Public surface + +- CI calls `build_scripts/unix.sh` and `build_scripts/windows.ps1`, which use `bin/start.spec` at the repo root. +- Output artifacts: `OctoBot_linux_x64`, `OctoBot_windows_x64.exe`, `OctoBot_macos_arm64`. + +## May depend on + +- The repo-root `bin/` PyInstaller spec and hooks, and the packages the binary bundles (`octobot`, `packages/*`) + +## Do not + +- Put application logic here, it belongs in `octobot/` or `packages/*` +- Commit built binaries, `user/`, `logs/` or `tentacles/` data +- Bypass tentacles signature verification (`ALLOW_UNSIGNED_TENTACLES`) when testing a binary + +## Tests + +- Quick check of a built binary: `packages/binary/scripts/smoke_test.sh ` (curl only, exit code 0 or 1). +- Full testing of a built binary (download from CI, `TENTACLES_URL_TAG` rules, run, UI and API checks): follow [BINARY_TESTING_INSTRUCTIONS.md](BINARY_TESTING_INSTRUCTIONS.md). Builds from branch `bin_factory` need `TENTACLES_URL_TAG=latest`, other builds do not. +- The instructions also cover seeded QA against the binary (`tools/agent_seed`, skill **agent-seed**), debug API user actions, the scheduler DB, the node journal, the REST spec and restart recovery. Use `pkill -INT -x OctoBot_x64` to stop it, never `pkill -f`. +- No pytest suite under this package. + +## Related human doc + +- [README.md](README.md) + +## Last reviewed + +- 2026-09-29 diff --git a/packages/binary/BINARY_TESTING_INSTRUCTIONS.md b/packages/binary/BINARY_TESTING_INSTRUCTIONS.md new file mode 100644 index 0000000000..01736c51fb --- /dev/null +++ b/packages/binary/BINARY_TESTING_INSTRUCTIONS.md @@ -0,0 +1,236 @@ +# Binary testing instructions + +How to download a PyInstaller build of OctoBot from CI, run it, and check that it works. Written from a real run of the Linux x64 build on a headless Linux machine (build from branch `bin_factory`, OctoBot `3.0.0-beta3`, Python 3.13). + +## 1. Get the binary + +Every CI run of the `OctoBot-CI` workflow (`.github/workflows/main.yml`) uploads one artifact per platform: + +| Artifact name | Platform | +|---------------|----------| +| `OctoBot_linux_x64` | Linux x86_64 | +| `OctoBot_windows_x64.exe` | Windows x64 | +| `OctoBot_macos_arm64` | macOS Apple silicon | +| `octobot-wheel` | Python wheel (not a binary) | + +Find the run under the repository Actions tab. Artifacts are kept for 90 days. + +Download with a GitHub token (the artifact URL needs authentication, a browser download also works): + +```bash +RUN_ARTIFACT_ID=10993723554 # id from the artifact URL +curl -sS -L -o artifact.zip \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/Drakkar-Software/OctoBot/actions/artifacts/$RUN_ARTIFACT_ID/zip +``` + +The API answers with a redirect to Azure blob storage (`*.blob.core.windows.net`). On a machine with an egress allowlist, that host must be allowed or the download fails with a 403 on the proxy tunnel. + +Check the download against the `digest` shown for the artifact in the run page or in the API listing, then unpack: + +```bash +sha256sum artifact.zip +unzip artifact.zip # the Linux zip contains a single file: OctoBot_x64 +chmod +x OctoBot_x64 +``` + +The file is a single executable of about 150 MB. Nothing else has to be installed, Python is bundled. + +## 2. Choose the tentacles tag + +On first start the binary downloads and installs the default tentacles for its own version. Which package it asks for depends on `TENTACLES_URL_TAG`. + +- **Build from branch `bin_factory`:** always set `export TENTACLES_URL_TAG=latest` before running. These builds carry an unreleased version (for example `3.0.0-beta3`) that has no tentacles package, so the default request returns 404 and the bot cannot install its default profiles. +- **Any other build (a release, or a branch whose version has published tentacles):** do not set the variable. The default is the binary's own version. + +You can check whether a version has tentacles with `curl -I https://tentacles.octobot.online/officials/packages/full/base//any_platform.zip` (200 means yes). + +The variable is only needed while tentacles are being installed. Once the `tentacles/` folder exists, restarts work without it. + +## 3. Run it + +Always start from a new empty directory. The binary writes `user/`, `logs/` and `tentacles/` into the current directory and never asks before doing so. + +```bash +mkdir octobot-test && cd octobot-test +export TENTACLES_URL_TAG=latest # bin_factory builds only, see section 2 +./path/to/OctoBot_x64 > octobot.log 2>&1 & +``` + +Expected behavior: + +- The binary starts in **node mode** (`Using node OctoBot distribution`). The Node UI (React, `node_web_interface`) is the interface. The classic Flask web interface is disabled (`Web interface disabled`). +- The Node API and UI listen on **port 8000**, on all interfaces. The UI is at `http://127.0.0.1:8000/app`. +- Nothing listens on port 5001 in node mode. Builds that include `dev` commit `6eea4ff` log `WebInterface configured to use port 5001 if started.` Older builds, such as the `bin_factory` build this guide was first written against, log `Interface successfully initialized and accessible at: http://127.0.0.1:5001` instead, which is misleading here: ignore that line. +- Start-up takes a few seconds once tentacles are installed. The first start is longer because it downloads and installs them (about 12 MB). + +Wait for readiness instead of sleeping: + +```bash +until curl -sf -o /dev/null http://127.0.0.1:8000/app; do sleep 2; done && echo ready +``` + +Stop it with `pkill -INT -x OctoBot_x64` (or Ctrl+C in the foreground). It exits in about 2 seconds. The log line `forcing immediate process exit without stop_tasks()` is the normal handling of an interrupt, not a crash. + +### When the machine cannot reach the tentacles host directly + +The tentacles downloader uses its own HTTP client, which does not read `HTTPS_PROXY`. On a machine where outbound traffic only works through a proxy (for example a cloud sandbox), the first start fails with: + +``` +Failed to download file at url : https://tentacles.octobot.online/... (status: 403, text: Host not in allowlist ...) +Missing default profiles. OctoBot can't start without a valid default profile configuration. +``` + +`curl` does use the proxy, so fetch the package yourself and install it from the local file with the binary's own command. Keep the `.signature` file next to the zip, the install refuses unsigned packages: + +```bash +BASE=https://tentacles.octobot.online/officials/packages/full/base/latest +curl -sS -o tentacles.zip $BASE/any_platform.zip +curl -sS -o tentacles.zip.signature $BASE/any_platform.zip.signature +TENTACLES_URL_TAG=latest ./path/to/OctoBot_x64 tentacles --install --all --force --location "$PWD/tentacles.zip" +``` + +Then start the bot as above. Do not set `ALLOW_UNSIGNED_TENTACLES` to get around a signature error, that turns off the check that protects the install. + +## 4. Test it + +Run these in order on a fresh directory. Everything below was seen passing on the `bin_factory` Linux x64 build, except where a finding says otherwise. + +### 4.0 Automated smoke test (curl only) + +`scripts/smoke_test.sh` runs sections 3, 4.1, 4.2, 4.5 and the setup and debug API checks of 4.6 and 4.8 in one go. It needs only bash, curl, grep and sed, so it runs on a plain machine with the binary and nothing else. + +```bash +TENTACLES_URL_TAG=latest packages/binary/scripts/smoke_test.sh ./OctoBot_x64 # bin_factory build: keep the variable, other builds: drop it +``` + +It boots the binary in a temporary empty directory, checks the HTTP surface, creates a throwaway wallet through the setup API, checks the debug API with that wallet, scans the log for `Traceback` and unexpected `ERROR` lines, stops the node gracefully, restarts it without `TENTACLES_URL_TAG` and checks the wallet survived. It exits 0 when everything passes and 1 otherwise (the work dir is then kept and its path printed). Set `SMOKE_TENTACLES_ZIP` to a local signed tentacles package (section 3) when the machine cannot reach the tentacles host, `SMOKE_TIMEOUT` to change the 180 s start-up wait, and pass a second argument to choose the work dir. It stops the node by PID and never by process name pattern. + +The manual sections below cover what a script cannot: the UI, the wizard, and the seeded automations. + +### 4.1 Boot + +- [ ] Process stays up and the log reaches `Uvicorn running on http://0.0.0.0:8000`. +- [ ] `grep -E "ERROR|Traceback" octobot.log` shows nothing except the known message below. +- [ ] The scheduler starts: `Scheduler: initialize_scheduler completed`, and the recurring workflows (`global_view_refresh`, `portfolio_history`, `dbos_cleanup`) log success. +- [ ] Tentacles are installed: `tentacles/` exists with `Agent`, `Automation`, `Backtesting`, `Evaluator`, `Meta`, `Services`, `Trading` and `profiles/`. + +Known, harmless log line on machines where the tentacles host is only reachable through a proxy: `Error when checking ssl certificates: fetching .../metadata.yaml returned 403. Considering certificates as valid.` This is a certificate pre-check that is allowed to fail. + +### 4.2 HTTP surface + +| Request | Expected on a fresh install | +|---------|-----------------------------| +| `GET /` | 307 redirect to `/app` | +| `GET /app` | 200, HTML titled `OctoBot Node` | +| `GET /app/assets/index-*.js` and `.css` (names are in the HTML) | 200 | +| `GET /api/v1/wallets/` | 200, `[]` | +| `GET /api/v1/nodes/config` | 503, `{"detail":{"code":"auth_node_not_configured",...}}` (no wallet yet) | + +### 4.3 Node UI + +Open `http://127.0.0.1:8000/app` in a browser. On a fresh install: + +- [ ] It redirects to `/app/setup/welcome` and shows the "Welcome to OctoBot Node" card with a **Get started** button. +- [ ] `/app/debug` also redirects to setup until a wallet exists. Once a wallet exists, a new browser session must enter the passphrase before using the app or `/app/debug`. +- [ ] **Get started** opens `/app/setup`, "Step 1 / 5, Set up your wallet". Entering a passphrase and confirming it, then **Generate wallet**, moves to "Step 2 / 5, Save your seed phrase" and shows 12 words. +- [ ] `GET /api/v1/wallets/` now returns one wallet. + +For a headless check, drive it with Playwright and the Chromium already installed on the machine (`PLAYWRIGHT_BROWSERS_PATH`, do not download a browser). Collect `console` errors and failed requests. In a sandbox with a TLS-intercepting proxy, Google Fonts requests fail with `ERR_CERT_AUTHORITY_INVALID`. That is the only expected failure, everything served by the bot itself must return below 400. + +### 4.4 UI copy + +Per `.cursor/skills/end-user-ui/SKILL.md`, user-visible text must use entry-level trading vocabulary, stay short, and contain no em dash. Skim each screen you open for these. + +### 4.5 Persistence and restart + +- [ ] Stop the bot, start it again in the same directory **without** `TENTACLES_URL_TAG`. It boots (tentacles already installed) and `GET /api/v1/wallets/` still returns the wallet created earlier. + +### 4.6 Seeded QA (agent-seed) + +The demo fixtures from `tools/agent_seed` (see its [README](../../tools/agent_seed/README.md) and skill **agent-seed**) work unchanged against the binary. This is the way to test a logged-in node with accounts, strategies and a running automation, and it is the documented QA path for the Node UI. + +Only the seed and bootstrap commands need the source checkout: they are Python code that has to run from the repo root in a Python 3.13 environment that has all the repo requirement files loaded: root `requirements.txt`, `full_requirements.txt` and `extra_requirements.txt`, then every `packages/*/requirements.txt` and every `packages/*/full_requirements.txt`. The `full_requirements.txt` files matter: `jsonschema`, `aiosqlite`, `psutil` and others are only declared there. Add `pytest` and its plugins as well if you want to run `tools/tests`. The binary itself needs nothing besides the tentacles from section 3, so use a scratch environment outside the repo and do not commit anything from it. + +Use the directory where the binary already installed its tentacles (call it `$RUN`). Two paths differ from the source-tree flow: + +- `OCTOBOT_AGENT_SEED_MASTER_USER_ROOT=$RUN/user` points the seed at the reference tentacles config and profiles the binary created. +- `PYTHONPATH` must list the repo root, every `packages/*` folder except `tentacles` and `binary`, and `$RUN` itself so that `import tentacles` finds the installed tentacles. + +```bash +export OCTOBOT_AGENT_SEED_MASTER_USER_ROOT=$RUN/user +python -m tools.agent_seed seed --user-folder $RUN/user/agent-seed # run from the repo root, prints nothing on success + +cd $RUN +export EXIT_BEFORE_TENTACLES_AUTO_REINSTALL=true +export SCHEDULER_SQLITE_FILE=$RUN/user/agent-seed/tasks.db +./path/to/OctoBot_x64 --master --user-folder user/agent-seed & # same as `seed-agent.sh start`, with the binary + +python -m tools.agent_seed bootstrap --base-url http://127.0.0.1:8000 # from the repo root, once the node listens: starts the grid automation +python -m tools.agent_seed bootstrap --scenario index --scenario completed +python -m tools.agent_seed bootstrap --scenario lifecycle # see the finding in 4.10, fails on this build +``` + +`bootstrap` takes repeatable `--scenario` options, all idempotent (a scenario already in its target state does nothing): + +| Scenario | Result | +|----------|--------| +| `grid` (default) | Grid automation `...0001` on Seed kraken A is `running` | +| `index` | Index automation `...0002` (BTC, ETH, SOL) on Seed kraken B is `running` | +| `completed` | Index automation `...0003` on Seed kraken B is created, then stopped, so it shows as completed | +| `lifecycle` | Stops and restarts the grid automation, then checks it is `running` again with its name | +| `all` | The four above in that order | + +There is no seeded "errored" automation on purpose: a failing automation keeps retrying and stays `running`, and it only becomes `failed` after the scheduler exhausts its recovery attempts, so it cannot be produced quickly and reliably from user actions. + +Checks, all seen passing on the `bin_factory` Linux x64 build: + +- [ ] `GET /api/v1/wallets/` lists the demo wallet `0x70997970c51812dc3a010c7d01b50e0d17dc79c8`, name `demo`. +- [ ] `GET /api/v1/debug/` without credentials returns 401. With HTTP Basic `wallet:demodemo` it returns 200 with accounts **Seed kraken A** (1000 USDC) and **Seed kraken B** (500 USDC), one exchange config, two strategies. +- [ ] `/app` redirects to `/app/login` ("Unlock your node"). The passphrase `demodemo` opens `/app/octobots`. `/app/debug` shows the debug view with the seeded counts and no browser errors. +- [ ] `bootstrap` exits 0. The automation `a0000000-0000-4000-8000-000000000001` is `running`, the create user action is `completed`, and the log shows the simulated trader placing 3 buy and 3 sell BTC/USDC limit orders. +- [ ] `/app/octobots` shows one active OctoBot as Running. +- [ ] After `--scenario index --scenario completed`: automation `...0002` is `running` with the name `Agent seed BTC/ETH/SOL index`, and `...0003` is `completed` with the name `Agent seed stopped index`. `/app/octobots` counts 2 active and 1 completed. Running the same command again changes nothing. + +### 4.7 User actions through the debug API + +`POST /api/v1/debug/` (HTTP Basic, body is a `UserAction` as JSON) returns 204 when the action is accepted. Poll `GET /api/v1/debug/` until the action in `user_actions` is `completed` or `failed`. Build payloads with `tools.agent_seed.protocol.builders` and send them with `json.loads(user_action.to_json())`. + +- [ ] Creating a **live** (not simulated) account returns 403 with `Demo agent-seed wallet cannot create live exchange accounts or automations`. This is the demo wallet guard, not a bug. +- [ ] Creating a **simulated** account returns 204 and completes. The account list grows. +- [ ] `automation_stop` (with `cancel_orders`) returns 204, completes, and the automation status becomes `completed`. +- [ ] `automation_restart` returns 204, completes, and the automation is `running` again. + +### 4.8 Node internals (packages/node, node_journal, node_api_interface) + +- [ ] **Scheduler:** `user/agent-seed/tasks.db` (SQLite, set by `SCHEDULER_SQLITE_FILE`) has successful `execute_user_action`, `execute_automation`, `global_view_refresh`, `portfolio_history_collection` and `dbos_cleanup` workflows. +- [ ] **Journal (record-only):** `/node_journal/events.jsonl` and `onboarding_segment.jsonl` contain the events for what you did (`external_action_received`, `account_create_attempt`, `automation_stopped`, `automation_restarted`, `first_automation_started`, `node_process_startup_succeeded`). Nothing in the app should read them back. +- [ ] **REST spec:** `GET /api/v1/openapi.json` returns 200 (about 100 KB). `/docs` and `/redoc` return 200. `/openapi.json` returns 404, that is expected. +- [ ] **Encryption:** debug routes answer 404 when node-side encryption is on. The seeded demo expects it off. + +### 4.9 Restart recovery + +- [ ] With the automation running, stop the bot (`pkill -INT -x OctoBot_x64`) and start it again with the same command. The log shows `Recovering 1 workflows from application version octobot_node_v1`, the automation is `running` again with its 6 orders, accounts and strategies are unchanged, and there is no `ERROR` or `Traceback`. + +### 4.10 Known findings on this build + +These were seen on the `bin_factory` build. They do not block boot or the checks above, and none is caused by the binary packaging itself. + +- `bootstrap --scenario lifecycle` fails on this build with `AutomationNameLostError` and exit code 1. That is the finding below, the check is doing its job. It also means `grid` cannot be re-run afterwards on the same node, because the grid check finds its automation by name. +- After `automation_restart`, the automation `metadata.name` is empty, so the Node UI titles the card `OctoBot a00000` instead of `Agent seed BTC/USDC grid`. The name is correct until the restart. The restart executor rebuilds the task from the latest terminal workflow (`user_actions_executor/automation/restart_automation.py`). + +## 5. Clean up + +Stop the bot by exact process name. Do not use `pkill -f OctoBot_x64`, it also matches the shell that runs the command and kills it. + +```bash +pkill -INT -x OctoBot_x64 +cd .. && rm -rf octobot-test +``` + +The wallet created during testing is a throwaway. Never reuse a test seed phrase or passphrase for real funds. + +## 6. What to report + +State the artifact (run id, branch, commit, artifact id and sha256), whether `TENTACLES_URL_TAG` was set, the result of each check in section 4, and any `ERROR` or `Traceback` lines other than the known one. Attach the relevant part of `octobot.log` for anything that failed. diff --git a/packages/binary/README.md b/packages/binary/README.md index a5c924c1db..4f037590ac 100644 --- a/packages/binary/README.md +++ b/packages/binary/README.md @@ -21,11 +21,10 @@ $ ./OctoBot_linux 2. Install OctoBot requirements `python -m pip install -r octobot/requirements.txt` 3. Install pyinstaller >= 4.0 pip install `https://github.com/pyinstaller/pyinstaller/archive/develop.zip` 4. List OctoBot modules files for pyinstaller discovery `python scripts/python_file_lister.py "bin/octobot_packages_files.txt" ` -5. Add pyinstaller required imports `python scripts/insert_imports.py octobot/octobot/cli.py` -6. Copy bin folder into the octobot folder -7. Go into the octobot folder -8. Compile the OctoBot project `python setup.py build_ext --inplace` -9. Call pyinstaller `pyinstaller bin\start.spec` -10. Binary should be available in the dist folder +5. Copy bin folder into the octobot folder +6. Go into the octobot folder +7. Compile the OctoBot project `python setup.py build_ext --inplace` +8. Call pyinstaller `pyinstaller bin\start.spec` +9. Binary should be available in the dist folder More information on [OctoBot wiki page](https://github.com/Drakkar-Software/OctoBot/wiki/Installation). diff --git a/packages/binary/build_scripts/unix.sh b/packages/binary/build_scripts/unix.sh index 9f001f1277..119137ea6e 100644 --- a/packages/binary/build_scripts/unix.sh +++ b/packages/binary/build_scripts/unix.sh @@ -1,7 +1,6 @@ #!/bin/bash python3 -m pip freeze python3 packages/binary/scripts/python_file_lister.py bin/octobot_packages_files.txt -python3 packages/binary/scripts/insert_imports.py octobot/cli.py python3 packages/binary/scripts/fetch_nltk_data.py words $NLTK_DATA python3 -m PyInstaller bin/start.spec --workpath installer ./dist/OctoBot --version diff --git a/packages/binary/build_scripts/windows.ps1 b/packages/binary/build_scripts/windows.ps1 index 3f103fcb74..eff7b038a4 100644 --- a/packages/binary/build_scripts/windows.ps1 +++ b/packages/binary/build_scripts/windows.ps1 @@ -1,6 +1,5 @@ python -m pip freeze python packages/binary/scripts/python_file_lister.py bin/octobot_packages_files.txt -python packages/binary/scripts/insert_imports.py octobot/cli.py python packages/binary/scripts/fetch_nltk_data.py words $env:NLTK_DATA python -m PyInstaller bin/start.spec --workpath installer Move-Item dist\OctoBot.exe OctoBot_windows.exe diff --git a/packages/binary/requirements.txt b/packages/binary/requirements.txt index 8908fbb60f..67da26caee 100644 --- a/packages/binary/requirements.txt +++ b/packages/binary/requirements.txt @@ -1,2 +1,2 @@ -pyinstaller==6.18.0 +pyinstaller==6.22.3 nltk diff --git a/packages/binary/scripts/insert_imports.py b/packages/binary/scripts/insert_imports.py deleted file mode 100644 index a32965d11c..0000000000 --- a/packages/binary/scripts/insert_imports.py +++ /dev/null @@ -1,16 +0,0 @@ -import sys - - -TO_INSERT_LINES = [ - "from engineio.async_drivers import gevent" # required for pyinstaller to add gevent and be able to use it later. -] - - -def insert_imports(target_file): - with open(target_file, "a") as file_append: - file_append.writelines([f"{line}\n" for line in TO_INSERT_LINES]) - print(f"{len(TO_INSERT_LINES)} lines appended into {target_file}") - - -if __name__ == "__main__": - insert_imports(sys.argv[1]) diff --git a/packages/binary/scripts/smoke_test.sh b/packages/binary/scripts/smoke_test.sh new file mode 100755 index 0000000000..76de24cb0e --- /dev/null +++ b/packages/binary/scripts/smoke_test.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# Smoke test for a CI-built OctoBot binary. Needs only bash, curl, grep and sed (no Python, no repo checkout). +# +# Usage: smoke_test.sh [work-dir] +# +# Boots the binary in an empty work dir, checks the HTTP surface and the setup API, creates a throwaway +# wallet, stops it gracefully, restarts it and checks the wallet is still there. +# Exit code is 0 when every check passes, 1 otherwise. See packages/binary/BINARY_TESTING_INSTRUCTIONS.md. +# +# Environment: +# TENTACLES_URL_TAG set to "latest" for builds from branch bin_factory (unreleased version), leave unset otherwise +# SMOKE_TENTACLES_ZIP local tentacles zip (with its .signature file next to it) to install before the first +# start, for machines where the binary cannot reach the tentacles host directly +# SMOKE_TIMEOUT seconds to wait for the node to listen (default 180) +# SMOKE_KEEP set to 1 to keep the work dir +set -uo pipefail + +BINARY="${1:-}" +if [[ -z "${BINARY}" || ! -x "${BINARY}" ]]; then + echo "Usage: $0 [work-dir]" >&2 + exit 2 +fi +BINARY="$(cd "$(dirname "${BINARY}")" && pwd)/$(basename "${BINARY}")" + +BASE_URL="http://127.0.0.1:8000" +SMOKE_TIMEOUT="${SMOKE_TIMEOUT:-180}" +PASSPHRASE="smoke-test-passphrase" +KNOWN_ERROR_PATTERN="Error when checking ssl certificates" +STOP_TIMEOUT=30 + +USER_WORK_DIR="${2:-}" +WORK_DIR="${USER_WORK_DIR:-$(mktemp -d -t octobot-smoke.XXXXXX)}" +mkdir -p "${WORK_DIR}" +WORK_DIR="$(cd "${WORK_DIR}" && pwd)" +BODY_FILE="${WORK_DIR}/.smoke_body" +PID="" +FAILED=0 +PASSED=0 + +pass() { PASSED=$((PASSED + 1)); echo " ok $1"; } +fail() { FAILED=$((FAILED + 1)); echo " FAIL $1"; } + +# Args: description, expected, actual +expect_equal() { + if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1 (expected '$2', got '$3')"; fi +} + +# Args: description, pattern (grep -E), file +expect_body_matches() { + if grep -Eiq "$2" "$3"; then pass "$1"; else fail "$1 (pattern '$2' not found in: $(head -c 200 "$3"))"; fi +} + +# Prints the HTTP status. Response body goes to BODY_FILE. Extra curl args follow the url. +http() { + local url="$1" + shift + local code + code="$(curl -sS --noproxy '*' -m 15 -o "${BODY_FILE}" -w '%{http_code}' "$@" "${url}" 2>/dev/null)" || true + echo "${code:-000}" +} + +start_node() { + (cd "${WORK_DIR}" && exec "${BINARY}" >>"${WORK_DIR}/octobot.log" 2>&1) & + PID=$! +} + +wait_ready() { + local waited=0 + while (( waited < SMOKE_TIMEOUT )); do + if ! kill -0 "${PID}" 2>/dev/null; then return 1; fi + if [[ "$(http "${BASE_URL}/app")" == "200" ]]; then return 0; fi + sleep 2 + waited=$((waited + 2)) + done + return 1 +} + +# Interrupt by PID (never pkill -f: that pattern also matches the shell running this script). +stop_node() { + [[ -n "${PID}" ]] || return 0 + kill -INT "${PID}" 2>/dev/null || true + local waited=0 + while kill -0 "${PID}" 2>/dev/null && (( waited < STOP_TIMEOUT )); do + sleep 1 + waited=$((waited + 1)) + done + if kill -0 "${PID}" 2>/dev/null; then + kill -KILL "${PID}" 2>/dev/null || true + PID="" + return 1 + fi + wait "${PID}" 2>/dev/null || true + PID="" + return 0 +} + +cleanup() { + stop_node >/dev/null 2>&1 || true + if [[ "${SMOKE_KEEP:-0}" != "1" && -z "${USER_WORK_DIR}" ]]; then rm -rf "${WORK_DIR}"; fi +} +trap 'cleanup' EXIT + +echo "OctoBot binary smoke test" +echo " binary: ${BINARY}" +echo " work dir: ${WORK_DIR}" +echo " tentacles tag: ${TENTACLES_URL_TAG:-}" + +if [[ "$(http "${BASE_URL}/app")" != "000" ]]; then + echo "Something already answers on ${BASE_URL}. Stop it first." >&2 + exit 2 +fi + +echo "1. Version" +version_output="$("${BINARY}" --version 2>/dev/null | tail -1)" +if [[ -n "${version_output}" ]]; then pass "--version prints '${version_output}'"; else fail "--version printed nothing"; fi + +if [[ -n "${SMOKE_TENTACLES_ZIP:-}" ]]; then + echo "2. Install tentacles from ${SMOKE_TENTACLES_ZIP}" + if (cd "${WORK_DIR}" && "${BINARY}" tentacles --install --all --force --location "${SMOKE_TENTACLES_ZIP}" >"${WORK_DIR}/tentacles_install.log" 2>&1); then + pass "tentacles installed from local package" + else + fail "tentacles install failed (see ${WORK_DIR}/tentacles_install.log)" + fi +fi + +echo "3. First start" +start_node +if wait_ready; then + pass "node listens on ${BASE_URL}" +else + fail "node did not become ready within ${SMOKE_TIMEOUT}s (or exited)" + echo "---- last log lines ----" + tail -20 "${WORK_DIR}/octobot.log" | sed 's/\x1b\[[0-9;]*m//g' + echo "Result: ${PASSED} passed, ${FAILED} failed" + SMOKE_KEEP=1 + exit 1 +fi + +echo "4. HTTP surface" +expect_equal "GET / redirects" "307" "$(http "${BASE_URL}/")" +code="$(http "${BASE_URL}/app")" +expect_equal "GET /app" "200" "${code}" +expect_body_matches "/app is the Node UI" "OctoBot Node" "${BODY_FILE}" +assets="$(grep -Eo '/app/assets/[^"]+\.(js|css)' "${BODY_FILE}" | sort -u)" +if [[ -z "${assets}" ]]; then fail "no js/css assets referenced by /app"; fi +for asset in ${assets}; do + expect_equal "GET ${asset}" "200" "$(http "${BASE_URL}${asset}")" +done +expect_equal "GET /api/v1/openapi.json" "200" "$(http "${BASE_URL}/api/v1/openapi.json")" +expect_equal "GET /docs" "200" "$(http "${BASE_URL}/docs")" + +echo "5. Setup API on a fresh node" +expect_equal "GET /api/v1/setup/status" "200" "$(http "${BASE_URL}/api/v1/setup/status")" +expect_body_matches "node is not configured yet" '"configured": ?false' "${BODY_FILE}" +expect_equal "GET /api/v1/wallets/" "200" "$(http "${BASE_URL}/api/v1/wallets/")" +expect_body_matches "no wallet yet" '^\[\]$' "${BODY_FILE}" +expect_equal "GET /api/v1/nodes/config before setup" "503" "$(http "${BASE_URL}/api/v1/nodes/config")" +expect_body_matches "typed not-configured error" "auth_node_not_configured" "${BODY_FILE}" + +echo "6. Create a throwaway wallet" +code="$(http "${BASE_URL}/api/v1/setup/init" -X POST -H 'Content-Type: application/json' \ + -d "{\"passphrase\":\"${PASSPHRASE}\",\"node_type\":\"standalone\"}")" +expect_equal "POST /api/v1/setup/init" "200" "${code}" +address="$(grep -Eo '0x[0-9a-fA-F]{40}' "${BODY_FILE}" | head -1)" +if [[ -n "${address}" ]]; then pass "wallet address returned (${address})"; else fail "no wallet address in the setup response"; fi +http "${BASE_URL}/api/v1/setup/status" >/dev/null +expect_body_matches "node is configured" '"configured": ?true' "${BODY_FILE}" +http "${BASE_URL}/api/v1/wallets/" >/dev/null +expect_body_matches "wallet is listed" "${address:-0xMISSING}" "${BODY_FILE}" + +echo "7. Debug API" +expect_equal "GET /api/v1/debug/ without credentials" "401" "$(http "${BASE_URL}/api/v1/debug/")" +code="$(http "${BASE_URL}/api/v1/debug/" -u "${address}:${PASSPHRASE}")" +expect_equal "GET /api/v1/debug/ with the wallet passphrase" "200" "${code}" +expect_body_matches "debug state has automations" '"automations"' "${BODY_FILE}" + +echo "8. Log" +clean_log="$(sed 's/\x1b\[[0-9;]*m//g' "${WORK_DIR}/octobot.log")" +if grep -q "Traceback" <<<"${clean_log}"; then fail "Traceback in the log"; else pass "no Traceback in the log"; fi +unexpected_errors="$(grep -E ' ERROR ' <<<"${clean_log}" | grep -v "${KNOWN_ERROR_PATTERN}" || true)" +if [[ -z "${unexpected_errors}" ]]; then + pass "no ERROR lines besides the known ssl pre-check" +else + fail "unexpected ERROR lines:" + echo "${unexpected_errors}" | cut -c1-220 | head -5 +fi + +echo "9. Graceful stop" +if stop_node; then pass "stopped within ${STOP_TIMEOUT}s"; else fail "did not stop within ${STOP_TIMEOUT}s, killed"; fi + +echo "10. Restart without TENTACLES_URL_TAG" +unset TENTACLES_URL_TAG +start_node +if wait_ready; then + pass "node listens again" + http "${BASE_URL}/api/v1/wallets/" >/dev/null + expect_body_matches "wallet survived the restart" "${address:-0xMISSING}" "${BODY_FILE}" +else + fail "node did not come back after restart" +fi +stop_node >/dev/null 2>&1 || true + +echo +echo "Result: ${PASSED} passed, ${FAILED} failed" +if (( FAILED > 0 )); then + echo "Work dir kept for inspection: ${WORK_DIR}" + SMOKE_KEEP=1 + exit 1 +fi +exit 0 diff --git a/packages/client/octobot_client_ts/package.json b/packages/client/octobot_client_ts/package.json index f61ae110b1..01bb3c1ce0 100644 --- a/packages/client/octobot_client_ts/package.json +++ b/packages/client/octobot_client_ts/package.json @@ -37,7 +37,7 @@ "dist" ], "dependencies": { - "@drakkar.software/octobot-protocol": "^0.8.0", + "@drakkar.software/octobot-protocol": "^0.9.0", "@drakkar.software/starfish-client": "3.0.0-alpha.72", "@drakkar.software/starfish-identities": "3.0.0-alpha.72", "@drakkar.software/starfish-keyring": "3.0.0-alpha.72", diff --git a/packages/commons/AGENTS.md b/packages/commons/AGENTS.md new file mode 100644 index 0000000000..adbe67e640 --- /dev/null +++ b/packages/commons/AGENTS.md @@ -0,0 +1,36 @@ +# Agents: commons + +## Role + +Shared utilities: logging, configuration helpers, databases, time, enums, and cross-cutting types used by most packages. + +## Owns + +- `packages/commons/octobot_commons/` +- `packages/commons/tests/` + +## Public surface + +- `octobot_commons.*` — prefer stable helpers here over duplicating in trading/node + +## May depend on + +- Minimal third-party stack; avoid importing `octobot_trading` or app layer + +## Do not + +- Import from `packages/trading`, `octobot` app, or tentacles +- Add trading-specific logic (belongs in trading) + +## Tests + +- **cwd:** `packages/commons` +- **Example:** `pytest tests -n auto --dist loadfile` + +## Related human doc + +- `docs/content/developers/packages/commons.md` (if present) + +## Last reviewed + +- 2026-09-16 diff --git a/packages/commons/octobot_commons/asyncio_tools.py b/packages/commons/octobot_commons/asyncio_tools.py index abd45b243f..0bb3eb9b1a 100644 --- a/packages/commons/octobot_commons/asyncio_tools.py +++ b/packages/commons/octobot_commons/asyncio_tools.py @@ -27,6 +27,17 @@ _BACKGROUND_FINGERPRINT_ASYNC_EXECUTOR = None +def is_on_async_loop(loop: asyncio.AbstractEventLoop) -> bool: + """ + True when the current thread is running asyncio on the given loop. + False when there is no running loop (e.g. sync WSGI worker thread) or another loop is running. + """ + try: + return asyncio.get_running_loop() is loop + except RuntimeError: + return False + + def run_background_fingerprint_async_executor(fingerprint: str, coroutine) -> bool: """ Run a coroutine in the background async executor diff --git a/packages/commons/octobot_commons/configuration/configuration.py b/packages/commons/octobot_commons/configuration/configuration.py index bf42dce383..9ff94471a6 100644 --- a/packages/commons/octobot_commons/configuration/configuration.py +++ b/packages/commons/octobot_commons/configuration/configuration.py @@ -16,7 +16,6 @@ # License along with this library. import os import functools -import copy import typing import octobot_commons.logging as logging @@ -28,6 +27,7 @@ import octobot_commons.configuration.config_file_manager as config_file_manager import octobot_commons.configuration.config_operations as config_operations import octobot_commons.user_root_folder_provider as user_root_folder_provider +import octobot_commons.copy_util as copy_util class Configuration: @@ -90,7 +90,7 @@ def read( should_raise=should_raise, fill_missing_fields=fill_missing_fields, ) - self.config = copy.deepcopy(self._read_config) + self.config = copy_util.deepcopy(self._read_config, "read config") if activate_profile: self.load_profiles_if_possible_and_necessary() @@ -144,8 +144,10 @@ def remove_profile(self, profile_id: str) -> None: def _generate_config_from_user_config_and_profile(self): for profile_managed_element in self.profile.FULLY_MANAGED_ELEMENTS: - self.config[profile_managed_element] = copy.deepcopy( - self.profile.config[profile_managed_element] + self.config[profile_managed_element] = copy_util.deepcopy( + self.profile.config[profile_managed_element], + f"profile element {profile_managed_element!r}", + str(profile_managed_element), ) for partially_managed_element in self.profile.PARTIALLY_MANAGED_ELEMENTS: self.profile.merge_partially_managed_element_into_config( @@ -331,17 +333,6 @@ def get_tentacles_setup_config_for_package_operations(self): ) return self.get_active_tentacles_setup_config() - def get_metrics_enabled(self) -> bool: - """ - Check if metrics are enabled - :return: True if metrics are enabled - """ - return bool( - self.config.get(commons_constants.CONFIG_METRICS, {}).get( - commons_constants.CONFIG_ENABLED_OPTION, True - ) - ) - def accepted_terms(self) -> bool: """ Check if terms has been accepted @@ -505,7 +496,7 @@ def refresh_sync_profiles(self) -> None: self.profile = refreshed_profile def _get_config_without_profile_elements(self) -> dict: - filtered_config = copy.deepcopy(self.config) + filtered_config = copy_util.deepcopy(self.config, "save config filter") # do not include profile fully managed elements into filtered config for profile_managed_element in profiles.Profile.FULLY_MANAGED_ELEMENTS: filtered_config.pop(profile_managed_element, None) diff --git a/packages/commons/octobot_commons/configuration/historical_configuration.py b/packages/commons/octobot_commons/configuration/historical_configuration.py index ea2e6125f2..0443524823 100644 --- a/packages/commons/octobot_commons/configuration/historical_configuration.py +++ b/packages/commons/octobot_commons/configuration/historical_configuration.py @@ -72,10 +72,7 @@ def get_historical_tentacle_configs( for config_start_time_and_config in master_config[ constants.CONFIG_HISTORICAL_CONFIGURATION ] - if ( - config_start_time_and_config[0] >= from_time - and config_start_time_and_config[0] <= to_time - ) + if from_time <= config_start_time_and_config[0] <= to_time ] except KeyError: raise KeyError( diff --git a/packages/commons/octobot_commons/constants.py b/packages/commons/octobot_commons/constants.py index a5cfbe8323..eb7d393317 100644 --- a/packages/commons/octobot_commons/constants.py +++ b/packages/commons/octobot_commons/constants.py @@ -220,9 +220,6 @@ def parse_boolean_environment_var(env_key: str, default_value: str) -> bool: DEFAULT_DISTRIBUTION = "default" CONFIG_DISTRIBUTION = "distribution" -# metrics -CONFIG_METRICS = "metrics" -CONFIG_METRICS_ACTIVITY_BOT_ID = "activity_bot_id" TIMER_BEFORE_METRICS_REGISTRATION_SECONDS = 600 TIMER_BETWEEN_METRICS_UPTIME_UPDATE = float( os.getenv("TIMER_BETWEEN_METRICS_UPTIME_UPDATE", str(3600 * 4)) diff --git a/packages/commons/octobot_commons/copy_util.py b/packages/commons/octobot_commons/copy_util.py new file mode 100644 index 0000000000..4290064e48 --- /dev/null +++ b/packages/commons/octobot_commons/copy_util.py @@ -0,0 +1,78 @@ +# pylint: disable=broad-exception-caught +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import copy +import typing + +import octobot_commons.errors as errors + +_ROOT_FIELD_PATH = "" +_CopyValue = typing.TypeVar("_CopyValue") + + +def _build_field_path(parent_path: str, segment: str) -> str: + if not parent_path: + return segment + return f"{parent_path}.{segment}" + + +def _format_field_path(field_path: str) -> str: + return field_path or _ROOT_FIELD_PATH + + +def _find_deepcopy_failure(value, field_path: str = "") -> tuple[str, object]: + if isinstance(value, dict): + for key, nested_value in value.items(): + nested_path = _build_field_path(field_path, str(key)) + try: + copy.deepcopy(nested_value) + except Exception: + return _find_deepcopy_failure(nested_value, nested_path) + elif isinstance(value, list): + for index, nested_value in enumerate(value): + nested_path = f"{field_path}[{index}]" + try: + copy.deepcopy(nested_value) + except Exception: + return _find_deepcopy_failure(nested_value, nested_path) + elif isinstance(value, tuple): + for index, nested_value in enumerate(value): + nested_path = f"{field_path}[{index}]" + try: + copy.deepcopy(nested_value) + except Exception: + return _find_deepcopy_failure(nested_value, nested_path) + return field_path, value + + +def deepcopy(value: _CopyValue, context: str = "", root_path: str = "") -> _CopyValue: + """ + Deepcopy a value and raise CopyError with field path context on failure. + :param value: the value to deepcopy + :param context: caller context included in the error message + :param root_path: optional prefix for nested field paths + :return: the deepcopied value + """ + try: + return copy.deepcopy(value) + except Exception as err: + failing_path, failing_value = _find_deepcopy_failure(value, root_path) + formatted_path = _format_field_path(failing_path) + context_message = f" during {context}" if context else "" + raise errors.CopyError( + f"Cannot deepcopy{context_message} at field {formatted_path!r}: " + f"{type(failing_value).__name__}={failing_value!r}" + ) from err diff --git a/packages/commons/octobot_commons/errors.py b/packages/commons/octobot_commons/errors.py index 132356d8d8..3cc86c6e18 100644 --- a/packages/commons/octobot_commons/errors.py +++ b/packages/commons/octobot_commons/errors.py @@ -27,6 +27,12 @@ class RemoteConfigError(ConfigError): """ +class CopyError(Exception): + """ + Copy related Exception + """ + + class NoProfileError(Exception): """ Profile related Exception: raised when the current profile can't be found and default profile can't be loaded diff --git a/packages/commons/octobot_commons/in_process_rate_limit.py b/packages/commons/octobot_commons/in_process_rate_limit.py new file mode 100644 index 0000000000..100af106d3 --- /dev/null +++ b/packages/commons/octobot_commons/in_process_rate_limit.py @@ -0,0 +1,143 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import dataclasses +import threading +import time +import typing + +# RAM-only counters in this process. Not shared across workers, pods, or nodes. + + +@dataclasses.dataclass(frozen=True) +class FailureWindowPolicy: + """One failure-count window for a named request dimension (e.g. client IP).""" + + name: str + max_failures: int + window_seconds: float + normalize_key: typing.Callable[[str], str] = lambda key: key + + +@dataclasses.dataclass +class _FailureBucket: + failure_count: int = 0 + window_start: float = 0.0 + + +class InProcessFailureRateLimiter: + """In-process, multi-dimension failure rate limiter (fixed window per key).""" + + def __init__(self, policies: tuple[FailureWindowPolicy, ...]) -> None: + if not policies: + raise ValueError("At least one FailureWindowPolicy is required") + self._policies = policies + self._lock = threading.Lock() + self._stores: dict[str, dict[str, _FailureBucket]] = { + policy.name: {} for policy in policies + } + + def reset_all(self) -> None: + """Clear all failure counters for every policy dimension.""" + with self._lock: + for store in self._stores.values(): + store.clear() + + def _policy_value(self, policy: FailureWindowPolicy, **dimensions: str) -> str: + try: + raw = dimensions[policy.name] + except KeyError: + raise KeyError( + f"Missing rate-limit dimension '{policy.name}'" + ) from None + return policy.normalize_key(raw) + + def _is_limited_for_policy( + self, + policy: FailureWindowPolicy, + key: str, + now: float, + ) -> bool: + store = self._stores[policy.name] + bucket = store.get(key) + if bucket is None or now - bucket.window_start >= policy.window_seconds: + store[key] = _FailureBucket(failure_count=0, window_start=now) + return False + return bucket.failure_count >= policy.max_failures + + def _remaining_seconds_for_policy( + self, + policy: FailureWindowPolicy, + key: str, + now: float, + ) -> float: + """Read-only: seconds until this policy's window ends, or 0 if not limited.""" + store = self._stores[policy.name] + bucket = store.get(key) + if bucket is None: + return 0.0 + if now - bucket.window_start >= policy.window_seconds: + return 0.0 + if bucket.failure_count < policy.max_failures: + return 0.0 + remaining = bucket.window_start + policy.window_seconds - now + if remaining <= 0.0: + return 0.0 + return remaining + + def retry_after_seconds(self, **dimensions: str) -> float: + """Monotonic seconds until the strictest active limit expires; 0 if not limited.""" + now = time.monotonic() + max_remaining = 0.0 + with self._lock: + for policy in self._policies: + key = self._policy_value(policy, **dimensions) + remaining = self._remaining_seconds_for_policy(policy, key, now) + max_remaining = max(max_remaining, remaining) + return max_remaining + + def is_rate_limited(self, **dimensions: str) -> bool: + """Return True when any policy dimension has reached its failure budget.""" + now = time.monotonic() + with self._lock: + for policy in self._policies: + key = self._policy_value(policy, **dimensions) + if self._is_limited_for_policy(policy, key, now): + return True + return False + + def record_failure(self, **dimensions: str) -> None: + """Increment failure counts for all policy dimensions.""" + now = time.monotonic() + with self._lock: + for policy in self._policies: + key = self._policy_value(policy, **dimensions) + store = self._stores[policy.name] + bucket = store.get(key) + if ( + bucket is None + or now - bucket.window_start >= policy.window_seconds + ): + bucket = _FailureBucket(failure_count=0, window_start=now) + store[key] = bucket + bucket.failure_count += 1 + + def record_success(self, **dimensions: str) -> None: + """Clear failure counters for the given dimension keys.""" + with self._lock: + for policy in self._policies: + key = self._policy_value(policy, **dimensions) + self._stores[policy.name].pop(key, None) diff --git a/packages/commons/octobot_commons/symbols/__init__.py b/packages/commons/octobot_commons/symbols/__init__.py index 7f349f0ce5..9d4247e60e 100644 --- a/packages/commons/octobot_commons/symbols/__init__.py +++ b/packages/commons/octobot_commons/symbols/__init__.py @@ -22,7 +22,9 @@ merge_currencies, convert_symbol, is_symbol, + get_bare_ticker_from_coin_with_maybe_network, is_usd_like_coin, + is_same_coin, is_usd_like_to_usd_like_pair, get_most_common_usd_like_symbol, trading_type_from_symbol, @@ -42,7 +44,9 @@ "merge_currencies", "convert_symbol", "is_symbol", + "get_bare_ticker_from_coin_with_maybe_network", "is_usd_like_coin", + "is_same_coin", "is_usd_like_to_usd_like_pair", "get_most_common_usd_like_symbol", "trading_type_from_symbol", diff --git a/packages/commons/octobot_commons/symbols/symbol.py b/packages/commons/octobot_commons/symbols/symbol.py index c6f6e47f93..88f77389a6 100644 --- a/packages/commons/octobot_commons/symbols/symbol.py +++ b/packages/commons/octobot_commons/symbols/symbol.py @@ -23,39 +23,51 @@ # pylint: disable=R0902,R0913 class Symbol: - # base / quote : settlement-identifier-strike price-type - # Inspired from CCXT https://docs.ccxt.com/#/README?id=contract-naming-conventions: - # // - # // base asset or currency - # // ↓ - # // ↓ quote asset or currency - # // ↓ ↓ - # // ↓ ↓ settlement asset or currency - # // ↓ ↓ ↓ - # // ↓ ↓ ↓ identifier (settlement date) - # // ↓ ↓ ↓ ↓ - # // ↓ ↓ ↓ ↓ strike price - # // ↓ ↓ ↓ ↓ ↓ - # // ↓ ↓ ↓ ↓ ↓ type, put (P) or call (C) - # // ↓ ↓ ↓ ↓ ↓ ↓ - # 'BTC/USDT:BTC-211225-60000-P' // BTC/USDT put option contract strike price 60000 USDT settled in BTC (inverse) - # on 2021-12-25 - # 'ETH/USDT:USDT-211225-40000-C' // BTC/USDT call option contract strike price 40000 USDT settled in USDT (linear, - # vanilla) on 2021-12-25 - # 'ETH/USDT:ETH-210625-5000-P' // ETH/USDT put option contract strike price 5000 USDT settled in ETH (inverse) - # on 2021-06-25 - # 'ETH/USDT:USDT-210625-5000-C' // ETH/USDT call option contract strike price 5000 USDT settled in USDT (linear, - # vanilla) on 2021-06-25 - # Local OctoBot addition (supported by the OctoBot ccxt fork) - # Uses https://docs.ccxt.com/#/README?id=unified-networks for network names. - # Optional network and DEX suffix (in uppercase, always at the end of the symbol): - # 'BTC/USDT@SOL!RAYDIUM' // BTC/USDT spot pair on solana network and raydium dex - # 'WETH/USDT:USDT-211225-40000-C@ETH!UNISWAP' // call option on ethereum network and uniswap dex - # 'WETH/USDT:USDT-211225-40000-C@ETH!*' // call option on ethereum network and any (most liquid) dex - # Note: network and dex names cannot contain '!', which separates network from dex after '@'. - # Ticker-wise per-leg network: - # 'USDT@ETH' // USDT on the ETH network (portfolio asset key) - # 'ETH@ETH/USDT@BNB' // spot pair with per-leg networks + """ + base / quote : settlement-identifier-strike price-type + Inspired from CCXT https://docs.ccxt.com/#/README?id=contract-naming-conventions: + base asset or currency + ↓ + ↓ quote asset or currency + ↓ ↓ + ↓ ↓ settlement asset or currency + ↓ ↓ ↓ + ↓ ↓ ↓ identifier (settlement date) + ↓ ↓ ↓ ↓ + ↓ ↓ ↓ ↓ strike price + ↓ ↓ ↓ ↓ ↓ + ↓ ↓ ↓ ↓ ↓ type, put (P) or call (C) + ↓ ↓ ↓ ↓ ↓ ↓ + 'BTC/USDT:BTC-211225-60000-P' // BTC/USDT put option contract strike price 60000 USDT settled in BTC (inverse) + on 2021-12-25 + 'ETH/USDT:USDT-211225-40000-C' // BTC/USDT call option contract strike price 40000 USDT settled in USDT (linear, + vanilla) on 2021-12-25 + 'ETH/USDT:ETH-210625-5000-P' // ETH/USDT put option contract strike price 5000 USDT settled in ETH (inverse) + on 2021-06-25 + 'ETH/USDT:USDT-210625-5000-C' // ETH/USDT call option contract strike price 5000 USDT settled in USDT (linear, + vanilla) on 2021-06-25 + + Local OctoBot addition (supported by the OctoBot ccxt fork) + @NETWORK and !DEX + Uses https://docs.ccxt.com/#/README?id=unified-networks for network names. + Optional network and DEX suffix (in uppercase, always at the end of the symbol): + 'BTC/USDT@SOL!RAYDIUM' // BTC/USDT spot pair on solana network and raydium dex + 'WETH/USDT:USDT-211225-40000-C@ETH!UNISWAP' // call option on ethereum network and uniswap dex + 'WETH/USDT:USDT-211225-40000-C@ETH!*' // call option on ethereum network and any (most liquid) dex + Note: network and dex names cannot contain '!', which separates network from dex after '@'. + + Optional ticker-wise per-leg @NETWORK: + 'USDT@ETH' // USDT on the ETH network (portfolio asset key) + 'ETH@ETH/USDT@BNB' // spot pair with per-leg networks + 'BTC@BTC/USDT@ETH' // base=BTC@BTC, quote=USDT@ETH; bare tickers via base_asset_ticker()/quote_asset_ticker() + After parse, .base and .quote: + - Plain spot and pair-level network (BTC/USDT, BTC/USDT@SOL): bare currency tickers. + - Ticker-wise pairs (BTC@BTC/USDT@ETH): network-qualified legs for + reference_market, holdings, fee currency, and config currency sets. + When bare currency is required (USD-like checks, CCXT fetch legs, bridge routing, etc.): + base_asset_ticker() / quote_asset_ticker(). + At each such call site: # Bare asset ticker: — .base/.quote are network-qualified on + """ def __init__( self, @@ -107,12 +119,18 @@ def parse_symbol(self, symbol_str: str): self.network_separator, ): base_leg, quote_leg = symbol_str.split(self.market_separator, 1) - self.base, self.base_network = parse_currency_network_leg( + base_currency, self.base_network = parse_currency_network_leg( base_leg, self.network_separator, self.dex_separator ) - self.quote, self.quote_network = parse_currency_network_leg( + quote_currency, self.quote_network = parse_currency_network_leg( quote_leg, self.network_separator, self.dex_separator ) + self.base = _format_network_qualified_leg( + base_currency, self.base_network, self.network_separator + ) + self.quote = _format_network_qualified_leg( + quote_currency, self.quote_network, self.network_separator + ) self.network = None self.dex = None self.settlement_asset = self.identifier = self.strike_price = "" @@ -143,10 +161,30 @@ def parse_symbol(self, symbol_str: str): def base_and_quote(self) -> typing.Tuple[str, str]: """ - return a tuple made of this symbol's base and quote assets + Return this symbol's base and quote legs (qualified on ticker-wise pairs). """ return self.base, self.quote + def base_asset_ticker(self) -> str: + """ + Bare currency ticker for the base leg (e.g. ``BTC`` on ``BTC@BTC/USDT@ETH``). + On non-ticker-wise symbols, equals ``base``. + """ + if self.has_ticker_wise_networks(): + return self.base.rsplit(self.network_separator, 1)[0] + return self.base + + def quote_asset_ticker(self) -> typing.Optional[str]: + """ + Bare currency ticker for the quote leg (e.g. ``USDT`` on ``BTC@BTC/USDT@ETH``). + On non-ticker-wise symbols, equals ``quote``. + """ + if self.quote is None: + return None + if self.has_ticker_wise_networks(): + return self.quote.rsplit(self.network_separator, 1)[0] + return self.quote + def merged_str_symbol( self, market_separator: str = octobot_commons.MARKET_SEPARATOR, @@ -158,11 +196,8 @@ def merged_str_symbol( """ return the base/quote representation of this symbol. includes settlement asset if set """ - if self.base_network and self.quote_network: - return ( - f"{self.base}{network_separator}{self.base_network}" - f"{market_separator}{self.quote}{network_separator}{self.quote_network}" - ) + if self.has_ticker_wise_networks(): + return f"{self.base}{market_separator}{self.quote}" if self.quote is None and self.network: return f"{self.base}{network_separator}{self.network}" merged_symbol = f"{self.base}{market_separator}{self.quote}" @@ -321,6 +356,14 @@ def __repr__(self): return str(self) +def _format_network_qualified_leg( + currency: str, + network: str, + network_separator: str = octobot_commons.NETWORK_SEPARATOR, +) -> str: + return f"{currency}{network_separator}{network}" + + def extract_network_and_dex( symbol_str: str, network_separator: str = octobot_commons.NETWORK_SEPARATOR, diff --git a/packages/commons/octobot_commons/symbols/symbol_util.py b/packages/commons/octobot_commons/symbols/symbol_util.py index d968210b22..709a237e16 100644 --- a/packages/commons/octobot_commons/symbols/symbol_util.py +++ b/packages/commons/octobot_commons/symbols/symbol_util.py @@ -127,11 +127,31 @@ def convert_symbol( return symbol.replace(symbol_separator, new_symbol_separator) +def get_bare_ticker_from_coin_with_maybe_network(coin: str) -> str: + """ + Bare currency ticker from a portfolio asset key (``USDT`` or ``USDT@ETH``). + Not for trading pair strings; use ``base_asset_ticker()`` / ``quote_asset_ticker()`` on symbols. + """ + if not coin: + return "" + if octobot_commons.NETWORK_SEPARATOR in coin: + return coin.rsplit(octobot_commons.NETWORK_SEPARATOR, 1)[0] + return coin + + def is_usd_like_coin(coin: str) -> bool: """ - :return: True if the given coin is a USD-like coin + :return: True if the given coin's bare ticker is a USD-like coin + """ + asset_ticker = get_bare_ticker_from_coin_with_maybe_network(coin) + return bool(asset_ticker) and asset_ticker in constants.USD_LIKE_COINS + + +def is_same_coin(coin_a: str, coin_b: str) -> bool: + """ + :return: True when both coins share the same bare ticker (e.g. ``USDT`` and ``USDT@ETH``) """ - return coin in constants.USD_LIKE_COINS + return get_bare_ticker_from_coin_with_maybe_network(coin_a) == get_bare_ticker_from_coin_with_maybe_network(coin_b) def is_usd_like_to_usd_like_pair(symbol: str) -> bool: @@ -140,12 +160,12 @@ def is_usd_like_to_usd_like_pair(symbol: str) -> bool: """ if not is_symbol(symbol): return False - base_asset, quote_asset = parse_symbol(symbol).base_and_quote() + parsed_symbol = parse_symbol(symbol) return ( - bool(base_asset) - and bool(quote_asset) - and is_usd_like_coin(base_asset) - and is_usd_like_coin(quote_asset) + bool(parsed_symbol.base) + and bool(parsed_symbol.quote) + and is_usd_like_coin(parsed_symbol.base) + and is_usd_like_coin(parsed_symbol.quote) ) diff --git a/packages/commons/octobot_commons/tests/dsl_test_keyword_support.py b/packages/commons/octobot_commons/tests/dsl_test_keyword_support.py new file mode 100644 index 0000000000..3afb28ee26 --- /dev/null +++ b/packages/commons/octobot_commons/tests/dsl_test_keyword_support.py @@ -0,0 +1,52 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import octobot_commons.dsl_interpreter as dsl_interpreter +import octobot_commons.enums as commons_enums + + +class TestKeywordMixin: + """ + Catalog defaults for module-level test Operator subclasses. + + pytest-xdist can import test modules before /api/v1/dsl/keywords tests; incomplete + operators used to make operator_docs_to_dsl_keyword() raise and return HTTP 500. + Inherit this mixin first on test operators that only implement get_name() / compute(). + """ + + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value + DESCRIPTION = "Test-only DSL operator." + EXAMPLE = "" + + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return dsl_interpreter.Operator.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Test operator result", + ) + + +class TestKeywordOperator(TestKeywordMixin, dsl_interpreter.Operator): + """Minimal test operator base with catalog-safe defaults.""" + + __test__ = False + + @staticmethod + def get_name() -> str: + raise NotImplementedError("get_name is not implemented") + + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: + raise NotImplementedError("compute is not implemented") diff --git a/packages/commons/requirements.txt b/packages/commons/requirements.txt index 415c95e56f..c3252f5bd0 100644 --- a/packages/commons/requirements.txt +++ b/packages/commons/requirements.txt @@ -1,5 +1,5 @@ # Setup requirements -numpy==2.4.2 +numpy # Commons requirements cryptography diff --git a/packages/commons/standard.rc b/packages/commons/standard.rc index 978ec7d18b..2b575e5fb2 100644 --- a/packages/commons/standard.rc +++ b/packages/commons/standard.rc @@ -57,6 +57,7 @@ confidence= # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". disable=too-few-public-methods, + too-many-public-methods, missing-module-docstring, missing-class-docstring, raise-missing-from, diff --git a/packages/commons/tests/configuration/test_configuration.py b/packages/commons/tests/configuration/test_configuration.py index 2ec9cafcc0..688f2a1315 100644 --- a/packages/commons/tests/configuration/test_configuration.py +++ b/packages/commons/tests/configuration/test_configuration.py @@ -17,11 +17,13 @@ import shutil import json import copy +import threading import pytest import mock import octobot_commons.errors as errors import octobot_commons.json_util import octobot_commons.configuration as configuration +import octobot_commons.configuration.config_file_manager as config_file_manager_module import octobot_commons.profiles as profiles import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.profiles.profile_data as profile_data_module @@ -447,27 +449,6 @@ def test_master_reference_path_uses_sync_data_root_for_child_process(self, confi assert config._get_master_reference_tentacles_config_file_path() == expected_path -def test_get_metrics_enabled(config): - config.config = {} - assert config.get_metrics_enabled() is True - config.config = { - constants.CONFIG_METRICS: {} - } - assert config.get_metrics_enabled() is True - config.config = { - constants.CONFIG_METRICS: { - constants.CONFIG_ENABLED_OPTION: True - } - } - assert config.get_metrics_enabled() is True - config.config = { - constants.CONFIG_METRICS: { - constants.CONFIG_ENABLED_OPTION: False - } - } - assert config.get_metrics_enabled() is False - - def test_accepted_terms(config): config.config = {} assert config.accepted_terms() is False @@ -768,6 +749,40 @@ def test_get_config_without_profile_elements(config): } +class TestCopyUtilOnSaveFilter: + def test_raises_copy_error_when_config_contains_non_deepcopyable_value(self, config): + config.config = {"metrics": {"runtime": threading.Lock()}} + with pytest.raises(errors.CopyError) as raised_error: + config._get_config_without_profile_elements() + assert "metrics.runtime" in str(raised_error.value) + + +class TestCopyUtilOnRead: + def test_raises_copy_error_when_read_config_contains_non_deepcopyable_value(self, config): + loaded_config = {"metrics": {"runtime": threading.Lock()}} + with mock.patch.object( + config_file_manager_module, + "load", + mock.Mock(return_value=loaded_config), + ): + with pytest.raises(errors.CopyError) as raised_error: + config.read(activate_profile=False) + assert "metrics.runtime" in str(raised_error.value) + + +class TestCopyUtilOnProfileMerge: + def test_raises_copy_error_when_profile_managed_element_contains_non_deepcopyable_value( + self, config + ): + config.profile = _load_test_profile(config) + managed_element = profiles.Profile.FULLY_MANAGED_ELEMENTS[0] + config.profile.config[managed_element] = {"runtime": threading.Lock()} + config.config = {} + with pytest.raises(errors.CopyError) as raised_error: + config._generate_config_from_user_config_and_profile() + assert str(managed_element) in str(raised_error.value) + + class TestConfigurationReadonlyProfileOverlay: def _write_readonly_master_profile( self, diff --git a/packages/commons/tests/dsl_interpreter/__init__.py b/packages/commons/tests/dsl_interpreter/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/commons/tests/dsl_interpreter/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py b/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py index f8a5eb80be..003e2ba543 100644 --- a/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py +++ b/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py @@ -22,14 +22,17 @@ import octobot_commons.errors as commons_errors import octobot_commons.process_util as process_util +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support -class _BareOperator(dsl_interpreter_operator.Operator): + +class _BareOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter_operator.Operator): @staticmethod def get_name() -> str: return "bare_process_bound_test" class _BoundOperator( + dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter_operator.Operator, process_bound_operator_mixin.ProcessBoundOperatorMixin, ): diff --git a/packages/commons/tests/dsl_interpreter/operators/test_re_callable_operator_mixin.py b/packages/commons/tests/dsl_interpreter/operators/test_re_callable_operator_mixin.py index 45d3fe31e7..39f5cdeb20 100644 --- a/packages/commons/tests/dsl_interpreter/operators/test_re_callable_operator_mixin.py +++ b/packages/commons/tests/dsl_interpreter/operators/test_re_callable_operator_mixin.py @@ -21,6 +21,8 @@ import octobot_commons.dsl_interpreter.operator_parameter as operator_parameter import octobot_commons.dsl_interpreter.operators.re_callable_operator_mixin as re_callable_operator_mixin +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support + class TestReCallingOperatorResult: def test_is_re_calling_operator_result_with_reset_to_id(self): @@ -100,7 +102,11 @@ def get_name() -> str: -class _ReCreateScriptTestOperator(dsl_interpreter.Operator, dsl_interpreter.ReCallableOperatorMixin): +class _ReCreateScriptTestOperator( + dsl_test_keyword_support.TestKeywordMixin, + dsl_interpreter.Operator, + dsl_interpreter.ReCallableOperatorMixin, +): """Operator with parameters so re_create_script can call resove_operator_params.""" @staticmethod diff --git a/packages/commons/tests/dsl_interpreter/test_compute_expression_with_result.py b/packages/commons/tests/dsl_interpreter/test_compute_expression_with_result.py index 1a95102c10..dbb4b949f3 100644 --- a/packages/commons/tests/dsl_interpreter/test_compute_expression_with_result.py +++ b/packages/commons/tests/dsl_interpreter/test_compute_expression_with_result.py @@ -20,6 +20,8 @@ import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.errors as commons_errors +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support + class TestDSLCallResult: def test_error_message_defaults_to_none(self): @@ -33,7 +35,7 @@ def test_success_has_no_error_fields(self): assert call_result.error_message is None -class ErrorRaisingTestOperator(dsl_interpreter.CallOperator): +class ErrorRaisingTestOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator): @staticmethod def get_name() -> str: return "test_error" @@ -43,7 +45,7 @@ def compute(self): raise commons_errors.ErrorStatementEncountered(*params) -class SumPlusFortyTwoOperator(dsl_interpreter.NaryOperator): +class SumPlusFortyTwoOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.NaryOperator): def __init__(self, *parameters: dsl_interpreter.OperatorParameterType, **kwargs: typing.Any): super().__init__(*parameters, **kwargs) self.added_value = 42 diff --git a/packages/commons/tests/dsl_interpreter/test_custom_operators.py b/packages/commons/tests/dsl_interpreter/test_custom_operators.py index a4f5592a98..20b7c7c96d 100644 --- a/packages/commons/tests/dsl_interpreter/test_custom_operators.py +++ b/packages/commons/tests/dsl_interpreter/test_custom_operators.py @@ -23,12 +23,14 @@ import octobot_commons.constants as commons_constants import octobot_commons.errors as commons_errors +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support + async def get_x_value_async() -> int: return 666 -class SumPlusXOperatorWithoutInit(dsl_interpreter.NaryOperator): +class SumPlusXOperatorWithoutInit(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.NaryOperator): def __init__(self, *parameters: dsl_interpreter.OperatorParameterType, **kwargs: typing.Any): super().__init__(*parameters, **kwargs) self.x_value = 42 @@ -42,7 +44,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return sum(computed_parameters) + self.x_value -class SumPlusXOperatorWithPreCompute(dsl_interpreter.NaryOperator): +class SumPlusXOperatorWithPreCompute(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.NaryOperator): def __init__(self, *parameters: dsl_interpreter.OperatorParameterType, **kwargs: typing.Any): super().__init__(*parameters, **kwargs) self.x_value = 42 @@ -67,7 +69,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return sum(computed_parameters) + self.x_value -class TimeFrameToSecondsOperator(dsl_interpreter.CallOperator): +class TimeFrameToSecondsOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator): MIN_PARAMS = 1 MAX_PARAMS = 1 @@ -83,7 +85,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return commons_enums.TimeFramesMinutes[commons_enums.TimeFrames(computed_parameters[0])] * commons_constants.MINUTE_TO_SECONDS -class AddOperator(dsl_interpreter.BinaryOperator): +class AddOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.BinaryOperator): @staticmethod def get_name() -> str: return ast.Add.__name__ @@ -93,7 +95,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return left + right -class SubOperator(dsl_interpreter.BinaryOperator): +class SubOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.BinaryOperator): @staticmethod def get_name() -> str: return ast.Sub.__name__ @@ -103,7 +105,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return left - right -class LtOperator(dsl_interpreter.CompareOperator): +class LtOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CompareOperator): @staticmethod def get_name() -> str: return ast.Lt.__name__ @@ -113,7 +115,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return left < right -class LtEOperator(dsl_interpreter.CompareOperator): +class LtEOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CompareOperator): @staticmethod def get_name() -> str: return ast.LtE.__name__ @@ -123,7 +125,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return left <= right -class GtOperator(dsl_interpreter.CompareOperator): +class GtOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CompareOperator): @staticmethod def get_name() -> str: return ast.Gt.__name__ @@ -133,7 +135,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return left > right -class GtEOperator(dsl_interpreter.CompareOperator): +class GtEOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CompareOperator): @staticmethod def get_name() -> str: return ast.GtE.__name__ @@ -143,7 +145,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return left >= right -class EqOperator(dsl_interpreter.CompareOperator): +class EqOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CompareOperator): @staticmethod def get_name() -> str: return ast.Eq.__name__ @@ -153,7 +155,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return left == right -class NotEqOperator(dsl_interpreter.CompareOperator): +class NotEqOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CompareOperator): @staticmethod def get_name() -> str: return ast.NotEq.__name__ @@ -163,7 +165,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return left != right -class IsOperator(dsl_interpreter.CompareOperator): +class IsOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CompareOperator): @staticmethod def get_name() -> str: return ast.Is.__name__ @@ -173,7 +175,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return left is right -class IsNotOperator(dsl_interpreter.CompareOperator): +class IsNotOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CompareOperator): @staticmethod def get_name() -> str: return ast.IsNot.__name__ @@ -183,7 +185,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return left is not right -class AndOperator(dsl_interpreter.NaryOperator): +class AndOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.NaryOperator): MIN_PARAMS = 1 @staticmethod @@ -194,7 +196,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return all(self.get_computed_parameters()) -class OrOperator(dsl_interpreter.NaryOperator): +class OrOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.NaryOperator): MIN_PARAMS = 1 @staticmethod @@ -204,7 +206,7 @@ def get_name() -> str: def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return any(self.get_computed_parameters()) -class Add2Operator(dsl_interpreter.CallOperator): +class Add2Operator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator): @staticmethod def get_name() -> str: return "add2" @@ -220,7 +222,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: left, right = self.get_computed_left_and_right_parameters() return left + right -class PreComputeSumOperator(dsl_interpreter.PreComputingCallOperator): +class PreComputeSumOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.PreComputingCallOperator): @staticmethod def get_name() -> str: return "pre_compute_sum" @@ -238,7 +240,7 @@ async def pre_compute(self) -> None: self.value = value_by_parameter["a"] + value_by_parameter["b"] -class CallWithDefaultParametersOperator(dsl_interpreter.CallOperator): +class CallWithDefaultParametersOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator): @staticmethod def get_name() -> str: return "call_with_default_parameters" @@ -262,7 +264,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: ) -class ParamMerger(dsl_interpreter.CallOperator): +class ParamMerger(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator): @staticmethod def get_name() -> str: return "param_merger" @@ -279,7 +281,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return str(value_by_parameter) -class NestedDictSumOperator(dsl_interpreter.CallOperator): +class NestedDictSumOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator): @staticmethod def get_name() -> str: return "nested_dict_sum" @@ -462,7 +464,7 @@ def test_get_input_value_by_parameter(): ParamMerger(p1=1, p2=2, extra=99, another=1).get_input_value_by_parameter() -class OperatorWithName(dsl_interpreter.Operator): +class OperatorWithName(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.Operator): NAME = "custom_name" DESCRIPTION = "A custom operator with NAME set" EXAMPLE = "custom_name(1, 2)" @@ -475,7 +477,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return sum(self.get_computed_parameters()) -class OperatorWithoutName(dsl_interpreter.Operator): +class OperatorWithoutName(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.Operator): DESCRIPTION = "An operator without NAME, uses get_name()" EXAMPLE = "fallback_name(5)" @@ -487,7 +489,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return sum(self.get_computed_parameters()) -class OperatorWithParameters(dsl_interpreter.Operator): +class OperatorWithParameters(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.Operator): NAME = "param_op" DESCRIPTION = "Operator with parameters" EXAMPLE = "param_op(1, 2)" @@ -507,7 +509,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return sum(self.get_computed_parameters()) -class OperatorWithoutParameters(dsl_interpreter.Operator): +class OperatorWithoutParameters(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.Operator): NAME = "no_param_op" DESCRIPTION = "Operator without parameters" EXAMPLE = "no_param_op()" @@ -520,7 +522,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return 42 -class OperatorWithCustomLibrary(dsl_interpreter.Operator): +class OperatorWithCustomLibrary(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.Operator): NAME = "custom_lib_op" DESCRIPTION = "Operator with custom library" EXAMPLE = "custom_lib_op()" @@ -537,8 +539,11 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return 42 -class OperatorWithEmptyFields(dsl_interpreter.Operator): - # NAME, DESCRIPTION, EXAMPLE all empty/default +class OperatorWithEmptyFields(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.Operator): + # NAME, DESCRIPTION, EXAMPLE empty for get_docs(); mixin still supplies catalog fields. + DESCRIPTION = "" + EXAMPLE = "" + @staticmethod def get_name() -> str: return "empty_fields_op" diff --git a/packages/commons/tests/dsl_interpreter/test_dictionnaries.py b/packages/commons/tests/dsl_interpreter/test_dictionnaries.py index 37332d1a4f..f3bb67e3ad 100644 --- a/packages/commons/tests/dsl_interpreter/test_dictionnaries.py +++ b/packages/commons/tests/dsl_interpreter/test_dictionnaries.py @@ -18,10 +18,10 @@ import octobot_commons.constants import octobot_commons.dsl_interpreter +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support - -class BinOperator1(octobot_commons.dsl_interpreter.BinaryOperator): +class BinOperator1(dsl_test_keyword_support.TestKeywordMixin, octobot_commons.dsl_interpreter.BinaryOperator): @staticmethod def get_name() -> str: return "b1" @@ -30,7 +30,7 @@ def compute(self) -> octobot_commons.dsl_interpreter.ComputedOperatorParameterTy return 0 -class BinOperator2(octobot_commons.dsl_interpreter.BinaryOperator): +class BinOperator2(dsl_test_keyword_support.TestKeywordMixin, octobot_commons.dsl_interpreter.BinaryOperator): @staticmethod def get_name() -> str: return "b2" @@ -39,7 +39,7 @@ def compute(self) -> octobot_commons.dsl_interpreter.ComputedOperatorParameterTy return 0 -class BinOperator3(octobot_commons.dsl_interpreter.BinaryOperator): +class BinOperator3(dsl_test_keyword_support.TestKeywordMixin, octobot_commons.dsl_interpreter.BinaryOperator): @staticmethod def get_name() -> str: return "b3" @@ -48,7 +48,7 @@ def compute(self) -> octobot_commons.dsl_interpreter.ComputedOperatorParameterTy return 0 -class UnaryOperator1(octobot_commons.dsl_interpreter.UnaryOperator): +class UnaryOperator1(dsl_test_keyword_support.TestKeywordMixin, octobot_commons.dsl_interpreter.UnaryOperator): @staticmethod def get_name() -> str: return "u1" @@ -57,7 +57,7 @@ def compute(self) -> octobot_commons.dsl_interpreter.ComputedOperatorParameterTy return 0 -class UnaryOperator2(octobot_commons.dsl_interpreter.UnaryOperator): +class UnaryOperator2(dsl_test_keyword_support.TestKeywordMixin, octobot_commons.dsl_interpreter.UnaryOperator): @staticmethod def get_name() -> str: return "u2" @@ -117,7 +117,7 @@ def test_get_all_operators_unknown_library(): def test_clear_get_all_operators_cache(): def create_new_operator(): - class NewOperator(octobot_commons.dsl_interpreter.Operator): + class NewOperator(dsl_test_keyword_support.TestKeywordMixin, octobot_commons.dsl_interpreter.Operator): @staticmethod def get_name() -> str: return "new_operator" diff --git a/packages/commons/tests/dsl_interpreter/test_interpreter.py b/packages/commons/tests/dsl_interpreter/test_interpreter.py index d037a04335..92e864b2d0 100644 --- a/packages/commons/tests/dsl_interpreter/test_interpreter.py +++ b/packages/commons/tests/dsl_interpreter/test_interpreter.py @@ -22,13 +22,15 @@ import octobot_commons.constants as commons_constants import ast +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support + @dataclasses.dataclass class ChannelDependency(dsl_interpreter.InterpreterDependency): channel_name: str -class SumPlusXOperatorWithoutInit(dsl_interpreter.NaryOperator): +class SumPlusXOperatorWithoutInit(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.NaryOperator): def __init__(self, *parameters: dsl_interpreter.OperatorParameterType, **kwargs: typing.Any): super().__init__(*parameters, **kwargs) self.x_value = 42 @@ -42,7 +44,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return sum(computed_parameters) + self.x_value -class TimeFrameToSecondsOperator(dsl_interpreter.CallOperator): +class TimeFrameToSecondsOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator): def __init__(self, operand: dsl_interpreter.OperatorParameterType, **kwargs: typing.Any): super().__init__(operand, **kwargs) @@ -60,7 +62,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return commons_enums.TimeFramesMinutes[commons_enums.TimeFrames(computed_parameters[0])] * commons_constants.MINUTE_TO_SECONDS -class AddOperator(dsl_interpreter.BinaryOperator): +class AddOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.BinaryOperator): @staticmethod def get_name() -> str: return ast.Add.__name__ diff --git a/packages/commons/tests/dsl_interpreter/test_nested_interpretation.py b/packages/commons/tests/dsl_interpreter/test_nested_interpretation.py index f430ceb35d..533d7d3a7b 100644 --- a/packages/commons/tests/dsl_interpreter/test_nested_interpretation.py +++ b/packages/commons/tests/dsl_interpreter/test_nested_interpretation.py @@ -20,8 +20,10 @@ import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.errors +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support -class AddOperator(dsl_interpreter.BinaryOperator): + +class AddOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.BinaryOperator): @staticmethod def get_name() -> str: return ast.Add.__name__ @@ -32,6 +34,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: class NestedEchoOperator( + dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator, dsl_interpreter.NestedInterpretationMixin, ): diff --git a/packages/commons/tests/symbols/test_symbol.py b/packages/commons/tests/symbols/test_symbol.py index 736a133ed1..e3735f779f 100644 --- a/packages/commons/tests/symbols/test_symbol.py +++ b/packages/commons/tests/symbols/test_symbol.py @@ -248,10 +248,12 @@ def test_network_qualified_asset(self): def test_ticker_wise_spot_pair(self): symbol = octobot_commons.symbols.Symbol("ETH@ETH/USDT@BNB") - assert symbol.base == "ETH" + assert symbol.base == "ETH@ETH" assert symbol.base_network == "ETH" - assert symbol.quote == "USDT" + assert symbol.quote == "USDT@BNB" assert symbol.quote_network == "BNB" + assert symbol.base_asset_ticker() == "ETH" + assert symbol.quote_asset_ticker() == "USDT" assert symbol.network is None assert symbol.dex is None assert symbol.has_ticker_wise_networks() is True @@ -261,9 +263,32 @@ def test_ticker_wise_spot_pair(self): def test_ticker_wise_coinrabbit_market_symbol(self): symbol = octobot_commons.symbols.Symbol("BTC@BTC/USDT@ETH") - assert symbol.base == "BTC" + assert symbol.base == "BTC@BTC" assert symbol.base_network == "BTC" - assert symbol.quote == "USDT" + assert symbol.quote == "USDT@ETH" assert symbol.quote_network == "ETH" + assert symbol.base_asset_ticker() == "BTC" + assert symbol.quote_asset_ticker() == "USDT" assert symbol.has_ticker_wise_networks() is True assert symbol.merged_str_symbol() == "BTC@BTC/USDT@ETH" + + def test_ticker_wise_merged_str_round_trip(self): + for symbol_str in ("BTC@BTC/USDT@ETH", "ETH@ETH/USDT@BNB"): + assert octobot_commons.symbols.Symbol(symbol_str).merged_str_symbol() == symbol_str + + +class TestSymbolAssetTickers: + def test_spot_asset_tickers_match_base_and_quote(self): + symbol = octobot_commons.symbols.Symbol("BTC/USDT") + assert symbol.base_asset_ticker() == "BTC" + assert symbol.quote_asset_ticker() == "USDT" + + def test_pair_level_network_asset_tickers_match_base_and_quote(self): + symbol = octobot_commons.symbols.Symbol("BTC/USDT@SOL") + assert symbol.base_asset_ticker() == "BTC" + assert symbol.quote_asset_ticker() == "USDT" + + def test_network_qualified_single_asset_base_asset_ticker(self): + symbol = octobot_commons.symbols.Symbol("USDT@ETH") + assert symbol.base_asset_ticker() == "USDT" + assert symbol.quote_asset_ticker() is None diff --git a/packages/commons/tests/symbols/test_symbol_util.py b/packages/commons/tests/symbols/test_symbol_util.py index a11b8c444b..a8213488e7 100644 --- a/packages/commons/tests/symbols/test_symbol_util.py +++ b/packages/commons/tests/symbols/test_symbol_util.py @@ -25,6 +25,25 @@ def test_merge_symbol(): assert octobot_commons.symbols.merge_symbol("BTC/USDT:USDT") == "BTCUSDT_USDT" +def test_merge_currencies_ticker_wise_network_legs(): + merged = octobot_commons.symbols.merge_currencies("BTC@BTC", "USDT@ETH") + assert merged == "BTC@BTC/USDT@ETH" + parsed = octobot_commons.symbols.parse_symbol(merged) + assert parsed.has_ticker_wise_networks() is True + assert parsed.base == "BTC@BTC" + assert parsed.quote == "USDT@ETH" + assert parsed.base_and_quote() == ("BTC@BTC", "USDT@ETH") + assert parsed.base_asset_ticker() == "BTC" + assert parsed.quote_asset_ticker() == "USDT" + + +def test_base_and_quote_spot_uses_bare_legs(): + parsed = octobot_commons.symbols.parse_symbol("BTC/USDT") + assert parsed.base_and_quote() == ("BTC", "USDT") + assert parsed.base_asset_ticker() == "BTC" + assert parsed.quote_asset_ticker() == "USDT" + + def test_merge_currencies(): assert octobot_commons.symbols.merge_currencies("BTC", "USDT") == "BTC/USDT" assert octobot_commons.symbols.merge_currencies("BTC", "USDT", "BTC") == "BTC/USDT:BTC" @@ -144,6 +163,30 @@ def test_mixed_symbol_types_raise_ambiguous_error(self): octobot_commons.symbols.trading_type_from_traded_symbols(["BTC/USDT", "BTC/USDT:USDT"]) +class TestIsUsdLikeCoin: + def test_bare_usd_like_tickers(self): + assert octobot_commons.symbols.is_usd_like_coin("USDT") + assert octobot_commons.symbols.is_usd_like_coin("USDC") + + def test_network_qualified_usd_like_key(self): + assert octobot_commons.symbols.is_usd_like_coin("USDT@ETH") + + def test_non_usd_like_tickers(self): + assert not octobot_commons.symbols.is_usd_like_coin("BTC") + assert not octobot_commons.symbols.is_usd_like_coin("BTC@BTC") + + def test_empty_coin_is_not_usd_like(self): + assert not octobot_commons.symbols.is_usd_like_coin("") + + def test_malformed_at_only_coin(self): + assert not octobot_commons.symbols.is_usd_like_coin("@") + assert octobot_commons.symbols.is_same_coin("@", "@") + + def test_is_same_coin_matches_bare_and_qualified(self): + assert octobot_commons.symbols.is_same_coin("USDT", "USDT@ETH") + assert not octobot_commons.symbols.is_same_coin("USDT@ETH", "USDC@ETH") + + class TestIsUsdLikeToUsdLikePair: def test_returns_true_for_usdc_usd(self): assert octobot_commons.symbols.is_usd_like_to_usd_like_pair("USDC/USD") @@ -157,6 +200,10 @@ def test_returns_false_for_btc_usdt(self): def test_returns_false_for_usdc_eur(self): assert not octobot_commons.symbols.is_usd_like_to_usd_like_pair("USDC/EUR") + def test_ticker_wise_qualified_legs(self): + assert octobot_commons.symbols.is_usd_like_to_usd_like_pair("USDC@ETH/USDT@BNB") + assert not octobot_commons.symbols.is_usd_like_to_usd_like_pair("BTC@BTC/USDT@ETH") + class TestMergeSymbolNetworkDex: def test_strips_network_and_dex_suffix(self): diff --git a/packages/commons/tests/test_asyncio_tools.py b/packages/commons/tests/test_asyncio_tools.py index 91494eac06..5d3a559c2f 100644 --- a/packages/commons/tests/test_asyncio_tools.py +++ b/packages/commons/tests/test_asyncio_tools.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. import asyncio +import threading + import mock import pytest @@ -23,6 +25,23 @@ pytestmark = pytest.mark.asyncio +class TestIsOnAsyncLoop: + async def test_returns_true_when_running_on_given_loop(self): + assert asyncio_tools.is_on_async_loop(asyncio.get_running_loop()) + + async def test_returns_false_from_thread_without_loop(self): + bot_loop = asyncio.get_running_loop() + thread_result = {} + + def worker(): + thread_result["is_on_loop"] = asyncio_tools.is_on_async_loop(bot_loop) + + thread = threading.Thread(target=worker) + thread.start() + thread.join() + assert thread_result["is_on_loop"] is False + + async def test_without_error_container(): # will not propagate exception asyncio.get_event_loop().call_soon(_exception_raiser) diff --git a/packages/commons/tests/test_copy_util.py b/packages/commons/tests/test_copy_util.py new file mode 100644 index 0000000000..042b7bfbb6 --- /dev/null +++ b/packages/commons/tests/test_copy_util.py @@ -0,0 +1,67 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import threading + +import pytest + +import octobot_commons.copy_util as copy_util +import octobot_commons.errors as errors + + +class TestDeepcopy: + def test_deepcopies_plain_nested_dict(self): + source_config = {"metrics": {"count": 1}, "enabled": True} + copied_config = copy_util.deepcopy(source_config) + assert copied_config == source_config + assert copied_config is not source_config + assert copied_config["metrics"] is not source_config["metrics"] + + def test_raises_copy_error_for_nested_dict_failure(self): + non_deepcopyable_value = threading.Lock() + source_config = {"metrics": {"runtime": non_deepcopyable_value}} + with pytest.raises(errors.CopyError) as raised_error: + copy_util.deepcopy(source_config, "test context") + error_message = str(raised_error.value) + assert "metrics.runtime" in error_message + assert "lock=" in error_message + assert "test context" in error_message + + def test_raises_copy_error_for_list_index_path(self): + non_deepcopyable_value = threading.Lock() + source_config = {"items": [1, non_deepcopyable_value, 3]} + with pytest.raises(errors.CopyError) as raised_error: + copy_util.deepcopy(source_config) + assert "items[1]" in str(raised_error.value) + assert "lock=" in str(raised_error.value) + + def test_raises_copy_error_with_root_path_prefix(self): + non_deepcopyable_value = threading.Lock() + source_config = {"binance": {"enabled": non_deepcopyable_value}} + with pytest.raises(errors.CopyError) as raised_error: + copy_util.deepcopy( + source_config, + "profile element", + "crypto-currencies", + ) + error_message = str(raised_error.value) + assert "crypto-currencies.binance.enabled" in error_message + assert "lock=" in error_message + + def test_chains_original_exception_as_cause(self): + non_deepcopyable_value = threading.Lock() + with pytest.raises(errors.CopyError) as raised_error: + copy_util.deepcopy(non_deepcopyable_value) + assert isinstance(raised_error.value.__cause__, TypeError) diff --git a/packages/commons/tests/test_in_process_rate_limit.py b/packages/commons/tests/test_in_process_rate_limit.py new file mode 100644 index 0000000000..ff5ff683f0 --- /dev/null +++ b/packages/commons/tests/test_in_process_rate_limit.py @@ -0,0 +1,220 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. + +import mock +import pytest + +import octobot_commons.in_process_rate_limit as in_process_rate_limit + +_MONOTONIC_PATCH = "octobot_commons.in_process_rate_limit.time.monotonic" + + +def _client_ip_policy(**overrides): + policy_kwargs = { + "name": "client_ip", + "max_failures": 3, + "window_seconds": 60.0, + } + policy_kwargs.update(overrides) + return in_process_rate_limit.FailureWindowPolicy(**policy_kwargs) + + +def _address_policy(**overrides): + policy_kwargs = { + "name": "address", + "max_failures": 3, + "window_seconds": 60.0, + "normalize_key": str.lower, + } + policy_kwargs.update(overrides) + return in_process_rate_limit.FailureWindowPolicy(**policy_kwargs) + + +def _limiter(*policies): + return in_process_rate_limit.InProcessFailureRateLimiter(policies) + + +def _dual_policy_limiter(): + return _limiter( + _client_ip_policy(max_failures=2), + _address_policy(max_failures=4), + ) + + +class TestFailureWindowPolicyNormalizeKey: + def test_normalizes_keys_per_policy(self): + limiter = _limiter( + _address_policy(max_failures=2, window_seconds=60), + ) + limiter.record_failure(address="0xAbC") + limiter.record_failure(address="0xabc") + assert limiter.is_rate_limited(address="0xABC") is True + + +class TestInProcessFailureRateLimiterInit: + def test_raises_when_no_policies(self): + with pytest.raises(ValueError): + in_process_rate_limit.InProcessFailureRateLimiter(()) + + +class TestInProcessFailureRateLimiterMissingDimension: + def test_is_rate_limited_raises_key_error(self): + limiter = _limiter(_client_ip_policy()) + with pytest.raises(KeyError): + limiter.is_rate_limited() + + def test_record_failure_raises_key_error(self): + limiter = _limiter(_client_ip_policy()) + with pytest.raises(KeyError): + limiter.record_failure() + + def test_record_success_raises_key_error(self): + limiter = _limiter(_client_ip_policy()) + with pytest.raises(KeyError): + limiter.record_success() + + def test_retry_after_seconds_raises_key_error(self): + limiter = _limiter(_client_ip_policy()) + with pytest.raises(KeyError): + limiter.retry_after_seconds() + + +class TestInProcessFailureRateLimiterRetryAfterSeconds: + def test_zero_when_not_limited(self): + limiter = _limiter(_client_ip_policy(max_failures=3)) + assert limiter.retry_after_seconds(client_ip="1.2.3.4") == 0.0 + limiter.record_failure(client_ip="1.2.3.4") + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.retry_after_seconds(client_ip="1.2.3.4") == 0.0 + + def test_decay_over_window(self): + limiter = _limiter(_client_ip_policy(max_failures=2, window_seconds=10)) + with mock.patch(_MONOTONIC_PATCH, return_value=0.0): + limiter.record_failure(client_ip="1.2.3.4") + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.retry_after_seconds(client_ip="1.2.3.4") == pytest.approx(10.0) + with mock.patch(_MONOTONIC_PATCH, return_value=5.0): + assert limiter.retry_after_seconds(client_ip="1.2.3.4") == pytest.approx(5.0) + with mock.patch(_MONOTONIC_PATCH, return_value=11.0): + assert limiter.retry_after_seconds(client_ip="1.2.3.4") == 0.0 + + def test_multi_policy_returns_max_remaining(self): + short_policy = _client_ip_policy(max_failures=2, window_seconds=10) + long_policy = _address_policy(max_failures=2, window_seconds=100) + limiter = _limiter(short_policy, long_policy) + with mock.patch(_MONOTONIC_PATCH, return_value=0.0): + limiter.record_failure(client_ip="1.2.3.4", address="0xaaa") + limiter.record_failure(client_ip="1.2.3.4", address="0xaaa") + remaining = limiter.retry_after_seconds(client_ip="1.2.3.4", address="0xaaa") + assert remaining == pytest.approx(100.0) + + def test_does_not_reset_buckets(self): + limiter = _limiter(_client_ip_policy(max_failures=2, window_seconds=10)) + with mock.patch(_MONOTONIC_PATCH, return_value=0.0): + limiter.record_failure(client_ip="1.2.3.4") + limiter.record_failure(client_ip="1.2.3.4") + limiter.retry_after_seconds(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is True + + +class TestInProcessFailureRateLimiterFailureBudget: + def test_not_limited_below_max_failures(self): + limiter = _limiter(_client_ip_policy(max_failures=3)) + limiter.record_failure(client_ip="1.2.3.4") + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is False + + def test_limited_at_max_failures(self): + limiter = _limiter(_client_ip_policy(max_failures=3)) + for _ in range(3): + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is True + + def test_is_rate_limited_does_not_increment(self): + limiter = _limiter(_client_ip_policy(max_failures=3)) + for _ in range(3): + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is True + assert limiter.is_rate_limited(client_ip="1.2.3.4") is True + + +class TestInProcessFailureRateLimiterMultiPolicy: + def test_limited_when_client_ip_exceeds_budget(self): + limiter = _dual_policy_limiter() + limiter.record_failure(client_ip="1.2.3.4", address="0xaaa") + limiter.record_failure(client_ip="1.2.3.4", address="0xbbb") + assert limiter.is_rate_limited(client_ip="1.2.3.4", address="0xccc") is True + + def test_limited_when_address_exceeds_budget(self): + limiter = _dual_policy_limiter() + address = "0xdef" + for index in range(4): + limiter.record_failure(client_ip=f"10.0.0.{index}", address=address) + assert limiter.is_rate_limited(client_ip="10.0.0.99", address=address) is True + + def test_record_failure_updates_all_policy_stores(self): + limiter = _dual_policy_limiter() + limiter.record_failure(client_ip="1.2.3.4", address="0xaaa") + limiter.record_failure(client_ip="1.2.3.4", address="0xbbb") + assert limiter.is_rate_limited(client_ip="1.2.3.4", address="0xaaa") is True + assert limiter.is_rate_limited(client_ip="9.9.9.9", address="0xaaa") is False + + +class TestInProcessFailureRateLimiterRecordSuccess: + def test_clears_buckets_for_given_keys(self): + limiter = _limiter(_client_ip_policy(max_failures=3)) + limiter.record_failure(client_ip="1.2.3.4") + limiter.record_failure(client_ip="1.2.3.4") + limiter.record_success(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is False + for _ in range(3): + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is True + + def test_success_on_one_address_does_not_clear_other_address(self): + limiter = _limiter(_address_policy(max_failures=3)) + limiter.record_failure(address="0xaaa") + limiter.record_failure(address="0xaaa") + limiter.record_failure(address="0xbbb") + limiter.record_failure(address="0xbbb") + limiter.record_success(address="0xaaa") + assert limiter.is_rate_limited(address="0xaaa") is False + assert limiter.is_rate_limited(address="0xbbb") is False + limiter.record_failure(address="0xbbb") + assert limiter.is_rate_limited(address="0xbbb") is True + + +class TestInProcessFailureRateLimiterResetAll: + def test_clears_every_policy_and_key(self): + limiter = _limiter(_client_ip_policy(max_failures=2)) + limiter.record_failure(client_ip="1.2.3.4") + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is True + limiter.reset_all() + assert limiter.is_rate_limited(client_ip="1.2.3.4") is False + + +class TestInProcessFailureRateLimiterWindowExpiry: + def test_is_rate_limited_opens_new_window_after_expiry(self): + limiter = _limiter(_client_ip_policy(max_failures=2, window_seconds=10)) + with mock.patch(_MONOTONIC_PATCH, return_value=0.0): + limiter.record_failure(client_ip="1.2.3.4") + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is True + with mock.patch(_MONOTONIC_PATCH, return_value=11.0): + assert limiter.is_rate_limited(client_ip="1.2.3.4") is False + + def test_record_failure_starts_new_window_after_expiry(self): + limiter = _limiter(_client_ip_policy(max_failures=3, window_seconds=10)) + with mock.patch(_MONOTONIC_PATCH, return_value=0.0): + for _ in range(3): + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is True + with mock.patch(_MONOTONIC_PATCH, return_value=11.0): + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is False diff --git a/packages/copy/AGENTS.md b/packages/copy/AGENTS.md new file mode 100644 index 0000000000..264f0fea24 --- /dev/null +++ b/packages/copy/AGENTS.md @@ -0,0 +1,33 @@ +# Agents: copy + +## Role + +Copy trading / mirror trading features between accounts and followers. + +## Owns + +- `packages/copy/octobot_copy/` (per repo layout) +- `packages/copy/tests/` + +## Public surface + +- Copy trading APIs consumed by tentacles and node + +## May depend on + +- `octobot_commons`, `octobot_trading` + +## Do not + +- Bypass trading order safeguards + +## Tests + +- **cwd:** repo root +- **PYTHONPATH:** `PYTHONPATH=.:$PYTHONPATH` +- **Tentacles:** required +- **Example:** `pytest packages/copy/tests -n auto --dist loadfile` + +## Last reviewed + +- 2026-09-16 diff --git a/packages/copy/octobot_copy/rebalancing/rebalancer/rebalancer.py b/packages/copy/octobot_copy/rebalancing/rebalancer/rebalancer.py index 9ebb9de50a..639e2cfee9 100644 --- a/packages/copy/octobot_copy/rebalancing/rebalancer/rebalancer.py +++ b/packages/copy/octobot_copy/rebalancing/rebalancer/rebalancer.py @@ -415,7 +415,7 @@ def _get_rebalance_details_keys_for_side(self, side: trading_enums.TradeOrderSid def _get_symbol_and_base_asset(self, coin_or_symbol: str) -> tuple[str, str]: if symbol_util.is_symbol(coin_or_symbol): - return coin_or_symbol, symbol_util.parse_symbol(coin_or_symbol).base # type: ignore + return coin_or_symbol, symbol_util.parse_symbol(coin_or_symbol).base # type: ignore return symbol_util.merge_currencies(coin_or_symbol, self._exchange_interface.portfolio.reference_market), coin_or_symbol def _get_logger(self): diff --git a/packages/copy/tests/orders_mirroring/test_orders_synchronizer_ticker_wise.py b/packages/copy/tests/orders_mirroring/test_orders_synchronizer_ticker_wise.py new file mode 100644 index 0000000000..694661fc1c --- /dev/null +++ b/packages/copy/tests/orders_mirroring/test_orders_synchronizer_ticker_wise.py @@ -0,0 +1,59 @@ +import decimal +import mock +import time + +import octobot_protocol.models as protocol_models +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_copy.constants as copy_constants +import octobot_copy.entities as copy_entities +import octobot_copy.orders_mirroring.orders_synchronizer as orders_synchronizer_import + +TICKER_WISE_SYMBOL = "BTC@BTC/USDT@ETH" + + +class TestScaleMirroredOrderQuantityTickerWise: + def test_buy_scales_using_qualified_quote(self): + synchronizer = orders_synchronizer_import.OrdersSynchronizer.__new__( + orders_synchronizer_import.OrdersSynchronizer + ) + synchronizer._reference_account = mock.Mock() + synchronizer._exchange_interface = mock.Mock() + synchronizer._exchange_interface.portfolio.get_currency_portfolio_total = mock.Mock( + return_value=decimal.Decimal("200") + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.copy_entities.copied_asset_total_by_name", + mock.Mock(return_value={"USDT@ETH": decimal.Decimal("100")}), + ): + order = mock.Mock(quantity="10") + scaled = synchronizer._scale_mirrored_order_quantity( + order, TICKER_WISE_SYMBOL, trading_enums.TradeOrderSide.BUY + ) + assert scaled == decimal.Decimal("20") + + +class TestReferencePairLegShareNetworkQualified: + def test_uses_qualified_leg_ratio_keys(self): + reference = protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=time.time(), + copied_assets=[ + protocol_models.CopiedAsset( + name="BTC@BTC", total=1.0, available=1.0, ratio=0.25 + ), + protocol_models.CopiedAsset( + name="USDT@ETH", total=1.0, available=1.0, ratio=0.5 + ), + ], + orders=[], + ) + synchronizer = orders_synchronizer_import.OrdersSynchronizer( + reference, + mock.MagicMock(), + copy_entities.AccountCopySettings(), + ) + expected = decimal.Decimal("0.25") / ( + decimal.Decimal("0.25") + decimal.Decimal("0.5") + ) + assert synchronizer._reference_pair_leg_share(TICKER_WISE_SYMBOL) == expected diff --git a/packages/copy/tests/python/orders_mirroring/test_mirrored_order_replication_failure_util.py b/packages/copy/tests/python/orders_mirroring/test_mirrored_order_replication_failure_util.py index 41470e0265..cac8a66b59 100644 --- a/packages/copy/tests/python/orders_mirroring/test_mirrored_order_replication_failure_util.py +++ b/packages/copy/tests/python/orders_mirroring/test_mirrored_order_replication_failure_util.py @@ -343,3 +343,76 @@ def test_sell_uses_base_currency_totals(self): assert context["copier_total"] == decimal.Decimal("0.5") assert context["scaled_quantity"] is None exchange_interface.portfolio.get_currency_portfolio_total.assert_called_once_with("ETH") + + +TICKER_WISE_SYMBOL = "BTC@BTC/USDT@ETH" + + +class TestMirrorScaleFailureContextNetworkQualified: + def test_buy_uses_qualified_quote(self): + order = _reference_limit_order(symbol=TICKER_WISE_SYMBOL, quantity=0.002) + reference_account = protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=time.time(), + copied_assets=[ + protocol_models.CopiedAsset( + name="USDT@ETH", total=500.0, available=500.0, ratio=0.5 + ), + protocol_models.CopiedAsset( + name="BTC@BTC", total=1.0, available=1.0, ratio=0.5 + ), + ], + orders=[], + ) + exchange_interface = mock.MagicMock() + exchange_interface.portfolio.get_currency_portfolio_total = mock.Mock( + return_value=decimal.Decimal("169") + ) + + context = mirrored_order_replication_failure_util.mirror_scale_failure_context( + order, + TICKER_WISE_SYMBOL, + trading_enums.TradeOrderSide.BUY, + decimal.Decimal("0.001"), + reference_account, + exchange_interface, + ) + + assert context["scale_currency"] == "USDT@ETH" + exchange_interface.portfolio.get_currency_portfolio_total.assert_called_once_with("USDT@ETH") + + def test_sell_uses_qualified_base(self): + order = _reference_limit_order( + symbol=TICKER_WISE_SYMBOL, + quantity=0.003, + side=protocol_models.Side.SELL, + ) + reference_account = protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=time.time(), + copied_assets=[ + protocol_models.CopiedAsset( + name="BTC@BTC", total=2.0, available=2.0, ratio=0.5 + ), + protocol_models.CopiedAsset( + name="USDT@ETH", total=100.0, available=100.0, ratio=0.5 + ), + ], + orders=[], + ) + exchange_interface = mock.MagicMock() + exchange_interface.portfolio.get_currency_portfolio_total = mock.Mock( + return_value=decimal.Decimal("0.5") + ) + + context = mirrored_order_replication_failure_util.mirror_scale_failure_context( + order, + TICKER_WISE_SYMBOL, + trading_enums.TradeOrderSide.SELL, + None, + reference_account, + exchange_interface, + ) + + assert context["scale_currency"] == "BTC@BTC" + exchange_interface.portfolio.get_currency_portfolio_total.assert_called_once_with("BTC@BTC") diff --git a/packages/copy/tests/rebalancing/test_rebalancer_network_qualified_portfolio_assets.py b/packages/copy/tests/rebalancing/test_rebalancer_network_qualified_portfolio_assets.py new file mode 100644 index 0000000000..6f60cb4dfb --- /dev/null +++ b/packages/copy/tests/rebalancing/test_rebalancer_network_qualified_portfolio_assets.py @@ -0,0 +1,42 @@ +import decimal + +import mock +import pytest + +import octobot_copy.enums as copy_enums +import octobot_trading.enums as trading_enums +import octobot_trading.errors as trading_errors +import octobot_copy.rebalancing.rebalancer.spot_rebalancer as spot_rebalancer + +TICKER_WISE_SYMBOL = "BTC@BTC/USDT@ETH" + + +def _remove_only_details(remove_asset: str) -> dict: + return { + copy_enums.RebalanceDetails.REMOVE.value: {remove_asset: decimal.Decimal("1")}, + copy_enums.RebalanceDetails.BUY_MORE.value: {}, + copy_enums.RebalanceDetails.ADD.value: {}, + copy_enums.RebalanceDetails.SWAP.value: {}, + } + + +class TestRebalancerSoldCoinsNetworkQualified: + @pytest.mark.asyncio + async def test_removed_asset_matches_sell_order_qualified_base(self): + rebalancer = spot_rebalancer.SpotRebalancer(mock.Mock(), mock.Mock(), {}) + details = _remove_only_details("BTC@BTC") + removed_orders = [ + mock.Mock(symbol=TICKER_WISE_SYMBOL, side=trading_enums.TradeOrderSide.SELL) + ] + await rebalancer._validate_sold_removed_assets(details, removed_orders) + + @pytest.mark.asyncio + async def test_removed_asset_mismatch_raises(self): + rebalancer = spot_rebalancer.SpotRebalancer(mock.Mock(), mock.Mock(), {}) + rebalancer._get_logger = mock.Mock(return_value=mock.Mock()) + details = _remove_only_details("BTC@BTC") + removed_orders = [ + mock.Mock(symbol=TICKER_WISE_SYMBOL, side=trading_enums.TradeOrderSide.BUY) + ] + with pytest.raises(trading_errors.MissingMinimalExchangeTradeVolume): + await rebalancer._validate_sold_removed_assets(details, removed_orders) diff --git a/packages/evaluators/AGENTS.md b/packages/evaluators/AGENTS.md new file mode 100644 index 0000000000..67a9f7a527 --- /dev/null +++ b/packages/evaluators/AGENTS.md @@ -0,0 +1,32 @@ +# Agents: evaluators + +## Role + +TA evaluators, matrices, and signals feeding trading decisions and tentacles. + +## Owns + +- `packages/evaluators/octobot_evaluators/` +- `packages/evaluators/tests/` + +## Public surface + +- `octobot_evaluators` evaluator classes and matrix APIs + +## May depend on + +- `octobot_commons` + +## Do not + +- Import trading exchange connectors directly for new features (use trading APIs) +- Edit installed tentacles instead of evaluator tentacles under `packages/tentacles` + +## Tests + +- **cwd:** `packages/evaluators` +- **Example:** `pytest tests -n auto --dist loadfile` + +## Last reviewed + +- 2026-09-16 diff --git a/packages/evaluators/octobot_evaluators/evaluators/evaluator_dsl_factory.py b/packages/evaluators/octobot_evaluators/evaluators/evaluator_dsl_factory.py index e69b924ada..e286f1eb57 100644 --- a/packages/evaluators/octobot_evaluators/evaluators/evaluator_dsl_factory.py +++ b/packages/evaluators/octobot_evaluators/evaluators/evaluator_dsl_factory.py @@ -113,7 +113,8 @@ def _get_cryptocurrency_from_symbol( return cryptocurrency if symbol is None: return None - return symbols_util.parse_symbol(symbol).base + # Bare asset ticker: evaluator ticker id — .base/.quote are network-qualified on ticker-wise pairs; use .base/.quote for portfolio[...] / reference_market. + return symbols_util.parse_symbol(symbol).base_asset_ticker() class EvaluatorOperator( diff --git a/packages/evaluators/requirements.txt b/packages/evaluators/requirements.txt index 42ac857bf6..26b0863b62 100644 --- a/packages/evaluators/requirements.txt +++ b/packages/evaluators/requirements.txt @@ -1,5 +1,5 @@ # Setup requirements -numpy==2.4.2 +numpy==2.4.2 # sync with octobot-tulipy # Drakkar-Software requirements OctoBot-Tulipy>=0.4.11b13 diff --git a/packages/flow/AGENTS.md b/packages/flow/AGENTS.md new file mode 100644 index 0000000000..e4886248a6 --- /dev/null +++ b/packages/flow/AGENTS.md @@ -0,0 +1,48 @@ +# Agents: flow + +## Role + +`octobot_flow` is a **stateless automation runner**: `AutomationState` in, updated state out. **Node** (or serverless callers) persist scheduling, encryption, and DBOS; flow does not. + +- **Execution:** action DAG (`actions_dag`, `actions_dag_parser`); ready actions run through **`DSLExecutor`** (commons DSL plus **tentacle-registered operators**). **Priority actions** run before the normal cycle (e.g. `apply_configuration` on first run). `wait()` can reset dependents via `ReCallingOperatorResult`. +- **Exchange per job:** `ExchangeContextMixin` builds and tears down `ExchangeManager` with storage disabled; simulated repositories without credentials; live OHLCV when needed; **explicit** portfolio sync after actions—not trading-mode automatic sync. +- **Job entry points:** `automation_job`, `automation_runner_job`, `portfolio_history_job`, `global_view_account_job`, `exchange_account_job`—invoked from node’s `octobot_flow_client`, not from DBOS code in this package. +- **Not here:** DBOS, task encryption, REST routes, tentacle pack/install, or long-lived scheduler state. + +## Owns + +- `packages/flow/octobot_flow/` +- `packages/flow/tests/` + +## Public surface + +- `octobot_flow.entities` — `AutomationState`, accounts, actions +- `octobot_flow.jobs` — automation, portfolio history, global view jobs +- `octobot_flow.parsers` — DAG and automation state readers +- `octobot_flow.logic.dsl` — `DSLExecutor` and execution context +- `octobot_flow.repositories` — exchange and community data access for jobs + +## May depend on + +- `octobot_commons`, `octobot_trading` (per-job `ExchangeManager`), imports used by jobs today + +## Do not + +- Import DBOS or node scheduler modules into flow +- Break wire/protocol enums or action contracts without updating **node** and **tentacles** API layers +- Assume flow runs without importable **tentacles** (operators register via `octobot_flow.environment.initialize_environment`) + +## Tests + +- **cwd:** repo root +- **PYTHONPATH:** `PYTHONPATH=.:$PYTHONPATH` +- **Tentacles:** required +- **Example:** `pytest packages/flow/tests -n auto --dist loadfile` + +## Related human doc + +- `docs/content/developers/packages/flow.md` + +## Last reviewed + +- 2026-09-16 diff --git a/packages/flow/octobot_flow/entities/actions/actions_dag.py b/packages/flow/octobot_flow/entities/actions/actions_dag.py index cb806ccdc8..56c7524adf 100644 --- a/packages/flow/octobot_flow/entities/actions/actions_dag.py +++ b/packages/flow/octobot_flow/entities/actions/actions_dag.py @@ -4,6 +4,7 @@ import octobot_flow.entities.actions.action_details as action_details import octobot_flow.entities.actions.actions_dependencies as actions_dependencies +import octobot_flow.enums import octobot_flow.errors @@ -54,6 +55,14 @@ def get_pending_actions(self) -> list[action_details.AbstractActionDetails]: if not action.is_completed() ] + def has_failed_action(self) -> bool: + """True if any action completed with a non-success error_status (not NO_ERROR).""" + no_error = octobot_flow.enums.ActionErrorStatus.NO_ERROR.value + return any( + action.is_completed() and action.error_status is not no_error + for action in self.actions + ) + def reset_to(self, action_id: str): """ Reset the action identified by action_id and all DAG actions that depend diff --git a/packages/flow/octobot_flow/jobs/portfolio_history_job.py b/packages/flow/octobot_flow/jobs/portfolio_history_job.py index ce1d879216..77589ff388 100644 --- a/packages/flow/octobot_flow/jobs/portfolio_history_job.py +++ b/packages/flow/octobot_flow/jobs/portfolio_history_job.py @@ -112,9 +112,6 @@ async def _fetch_and_persist( tentacles_setup_config, price_fallback=None, ) as exchange_manager: - await trades_repository_module.TradesRepository.ensure_temporary_trades_channel( - exchange_manager, - ) fetched_exchange_data = octobot_flow.entities.FetchedExchangeData() trades_repo = trades_repository_module.TradesRepository( exchange_manager, [], fetched_exchange_data @@ -153,28 +150,37 @@ async def _fetch_and_persist( self.data_root, ) fetched_trades_count = 0 - trades = await trades_repo.fetch_trades_paginated( - discovered_symbols, - existing_config_symbols=existing_config_symbols, - exchange_name=exchange_config.exchange, - account_id=account.id, - exchange_config_id=exchange_config.id, - exchange_config_name=exchange_config.name, - symbol_since_ms=symbol_since_ms or None, - ) - fetched_trades_count = len(trades) - trades, dropped_trade_symbols = _filter_trades_on_live_markets( - exchange_manager, - trades, - ) - if dropped_trade_symbols: + dropped_trade_symbols: set[str] = set() + if exchange_api.exchange_uses_network_qualified_markets(exchange_manager): logger.info( - "Dropped %d trades on delisted/unknown markets for %s account %s: %s", - fetched_trades_count - len(trades), + "Skipping historical trade fetch for %s account %s: exchange markets use network-qualified symbols", exchange_config.exchange, account.id, - ", ".join(sorted(dropped_trade_symbols)), ) + trades = [] + else: + trades = await trades_repo.fetch_trades_paginated( + discovered_symbols, + existing_config_symbols=existing_config_symbols, + exchange_name=exchange_config.exchange, + account_id=account.id, + exchange_config_id=exchange_config.id, + exchange_config_name=exchange_config.name, + symbol_since_ms=symbol_since_ms or None, + ) + fetched_trades_count = len(trades) + trades, dropped_trade_symbols = _filter_trades_on_live_markets( + exchange_manager, + trades, + ) + if dropped_trade_symbols: + logger.info( + "Dropped %d trades on delisted/unknown markets for %s account %s: %s", + fetched_trades_count - len(trades), + exchange_config.exchange, + account.id, + ", ".join(sorted(dropped_trade_symbols)), + ) live_discovered_symbols = _filter_symbols_on_live_markets( exchange_manager, @@ -374,7 +380,7 @@ def _reference_market_from_account_assets( if account.assets: for assets_for_trading_type in account.assets: for asset in assets_for_trading_type.assets or []: - if asset.symbol not in commons_constants.USD_LIKE_COINS: + if not symbol_util.is_usd_like_coin(asset.symbol): continue asset_total = float(asset.total or 0) if asset_total <= min_holdings_threshold: @@ -419,21 +425,27 @@ def _derive_price_symbols( transactions: list[dict], reference_market: str, ) -> list[str]: - """Build the set of symbols whose daily prices should be cached.""" + """ + Build the set of symbols whose daily prices should be cached. + @NETWORK from pairs are ignored so that ETH/USDT and ETH@ETH/USDT@ETH share the same cache. + """ base_assets: set[str] = set() for trading_symbol in trade_symbols: if not symbol_util.is_symbol(trading_symbol): continue - base_currency, _quote_currency = symbol_util.parse_symbol(trading_symbol).base_and_quote() - if symbol_util.is_usd_like_coin(base_currency): + # Bare asset ticker: merged valuation pair for daily cache — .base/.quote are network-qualified on ticker-wise pairs; use .base/.quote for portfolio[...] / reference_market. + parsed_trading_symbol = symbol_util.parse_symbol(trading_symbol) + base_currency = parsed_trading_symbol.base_asset_ticker() + _quote_currency = parsed_trading_symbol.quote_asset_ticker() + if symbol_util.is_usd_like_coin(parsed_trading_symbol.base): continue - if base_currency and base_currency != reference_market: + if base_currency and not symbol_util.is_same_coin(base_currency, reference_market): base_assets.add(base_currency) for transaction in transactions: transaction_currency = transaction.get("currency") if ( not transaction_currency - or transaction_currency == reference_market + or symbol_util.is_same_coin(transaction_currency, reference_market) or symbol_util.is_usd_like_coin(transaction_currency) ): continue @@ -452,7 +464,10 @@ def _derive_price_symbols( def _is_valid_trading_symbol(symbol: str) -> bool: if not symbol_util.is_symbol(symbol): return False - base_currency, quote_currency = symbol_util.parse_symbol(symbol).base_and_quote() - if symbol_util.is_usd_like_coin(base_currency): + # Bare asset ticker: validate merged valuation symbol legs — .base/.quote are network-qualified on ticker-wise pairs; use .base/.quote for portfolio[...] / reference_market. + parsed_symbol = symbol_util.parse_symbol(symbol) + base_currency = parsed_symbol.base_asset_ticker() + quote_currency = parsed_symbol.quote_asset_ticker() + if symbol_util.is_usd_like_coin(parsed_symbol.base): return False return bool(base_currency) and bool(quote_currency) and base_currency != quote_currency diff --git a/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py b/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py index 5eb320266c..1d39c7c9a1 100644 --- a/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py +++ b/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py @@ -1,7 +1,9 @@ import typing +import octobot_commons import octobot_commons.profiles.profile_data as profile_data_import import octobot_commons.constants +import octobot_commons.symbols as commons_symbols import octobot_protocol.models as protocol_models import octobot_trading.enums as trading_enums import octobot_trading.api.exchange as exchange_api @@ -84,7 +86,8 @@ def infer_reference_market( ): return octobot_commons.constants.DEFAULT_REFERENCE_MARKET if crypto_currencies: - return octobot_commons.symbols.parse_symbol(crypto_currencies[0].trading_pairs[0]).quote # type: ignore + # Ref market uses qualified .quote on ticker-wise pairs (e.g. USDT@ETH). + return octobot_commons.symbols.parse_symbol(crypto_currencies[0].trading_pairs[0]).quote # type: ignore elif exchange_account_details: if exchange_account_details.portfolio.unit: # portfolio unit can be used to define the reference market @@ -95,8 +98,14 @@ def infer_reference_market( ) return octobot_commons.constants.DEFAULT_REFERENCE_MARKET + def _get_crypto_currencies(symbols: set[str]) -> list[profile_data_import.CryptoCurrencyData]: + trading_pairs_by_bare_base: dict[str, list[str]] = {} + for symbol in symbols: + parsed_symbol = commons_symbols.parse_symbol(symbol) + base_bare_ticker = parsed_symbol.base_asset_ticker() + trading_pairs_by_bare_base.setdefault(base_bare_ticker, []).append(symbol) return [ - profile_data_import.CryptoCurrencyData(trading_pairs=[symbol], name=symbol) - for symbol in symbols + profile_data_import.CryptoCurrencyData(trading_pairs=trading_pairs, name=base_bare_ticker) + for base_bare_ticker, trading_pairs in sorted(trading_pairs_by_bare_base.items()) ] \ No newline at end of file diff --git a/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py b/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py index 490ae295cd..aa29bf0ce2 100644 --- a/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py +++ b/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py @@ -12,9 +12,6 @@ import octobot_flow.entities import octobot_flow.logic.exchange.simulator.simulated_order_fill_detector as simulated_order_fill_detector_module import octobot_flow.repositories.exchange.exchange_repository_factory as exchange_repository_factory_module -import octobot_flow.repositories.exchange.orders_repository as orders_repository_module -import octobot_flow.repositories.exchange.portfolio_repository as portfolio_repository_module -import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module def _get_logger() -> octobot_commons_logging.BotLogger: @@ -52,8 +49,6 @@ async def refresh_exchange_account( fetch_open_orders: bool = True, open_order_symbols: list[str] | None = None, ) -> octobot_flow.entities.ExchangeAccountRefreshResult: - await tickers_repository_module.TickersRepository.ensure_temporary_ticker_channel(exchange_manager) - # Step: fetch balance and open orders from the exchange (real accounts only). tickers: dict[str, dict] | None = None repository_factory = None @@ -76,9 +71,6 @@ async def refresh_exchange_account( ticker_close_by_symbol, ) else: - await portfolio_repository_module.PortfolioRepository.ensure_temporary_balance_channel(exchange_manager) - if fetch_open_orders: - await orders_repository_module.OrdersRepository.ensure_temporary_orders_channel(exchange_manager) repository_factory = _create_exchange_repository_factory(exchange_manager) portfolio_repository = repository_factory.get_portfolio_repository() await portfolio_repository.fetch_and_apply_portfolio() diff --git a/packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py b/packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py index ce241a7f57..1cefd1741a 100644 --- a/packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py +++ b/packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py @@ -33,20 +33,23 @@ async def update_daily_prices( ) for reference_symbol in symbols: - parsed = _parse_base_quote(reference_symbol) + parsed_reference = symbol_util.parse_symbol(reference_symbol) + parsed = _parse_bare_base_quote(reference_symbol) if parsed is None: continue - base_asset, quote = parsed - if base_asset in commons_constants.USD_LIKE_COINS: + fetch_bare_base, fetch_bare_quote = parsed + # Qualified base leg keys SOURCES (e.g. BTC@BTC); CCXT fetch uses bare tickers from _parse_bare_base_quote only. + sources_cache_key = parsed_reference.base + if symbol_util.is_usd_like_coin(fetch_bare_base): continue - sticky_fetch_symbol = trading_api.get_daily_close_source(daily_prices, base_asset) + sticky_fetch_symbol = trading_api.get_daily_close_source(daily_prices, sources_cache_key) if _is_daily_cache_up_to_date(daily_prices, reference_symbol): continue since_ms = _compute_fetch_since_ms(daily_prices, reference_symbol) candidates = _build_fetch_candidates( - exchange_manager, base_asset, quote, sticky_fetch_symbol, + exchange_manager, fetch_bare_base, fetch_bare_quote, sticky_fetch_symbol, ) fetch_result = await _fetch_daily_candles( @@ -64,7 +67,7 @@ async def update_daily_prices( if sticky_fetch_symbol and fetch_symbol != sticky_fetch_symbol: logger.info( "Migrating daily closes for %s on %s from %s to %s", - base_asset, + sources_cache_key, exchange_name, sticky_fetch_symbol, fetch_symbol, @@ -95,14 +98,17 @@ async def update_daily_prices( exchange_name, exchange_type, sandboxed, fetch_symbol, closes_by_timestamp, data_root, ) await trading_api.set_daily_close_source( - exchange_name, exchange_type, sandboxed, base_asset, fetch_symbol, data_root, + exchange_name, exchange_type, sandboxed, sources_cache_key, fetch_symbol, data_root, ) - trading_api.set_daily_close_source_in_memory(daily_prices, base_asset, fetch_symbol) + trading_api.set_daily_close_source_in_memory(daily_prices, sources_cache_key, fetch_symbol) trading_api.merge_daily_prices_in_memory(daily_prices, fetch_symbol, closes_by_timestamp) -def _parse_base_quote(symbol: str) -> tuple[str, str] | None: - base_asset, quote = symbol_util.parse_symbol(symbol).base_and_quote() +def _parse_bare_base_quote(symbol: str) -> tuple[str, str] | None: + # Bare tickers for exchange fetch (base_asset_ticker / quote_asset_ticker), not qualified .base/.quote legs. + parsed_symbol = symbol_util.parse_symbol(symbol) + base_asset = parsed_symbol.base_asset_ticker() + quote = parsed_symbol.quote_asset_ticker() if not base_asset or not quote or base_asset == quote: return None return base_asset, quote @@ -119,18 +125,18 @@ def _ordered_usd_like_quotes(quote: str) -> list[str]: return quotes -def _is_reversed_fetch_symbol(base_asset: str, fetch_symbol: str) -> bool: - parsed = _parse_base_quote(fetch_symbol) +def _is_reversed_fetch_symbol(bare_base_asset: str, fetch_symbol: str) -> bool: + parsed = _parse_bare_base_quote(fetch_symbol) if parsed is None: return False - fetch_base, _fetch_quote = parsed - return fetch_base != base_asset + fetch_bare_base, _fetch_bare_quote = parsed + return fetch_bare_base != bare_base_asset def _build_fetch_candidates( exchange_manager, - base_asset: str, - quote: str, + bare_base_asset: str, + bare_quote: str, sticky_fetch_symbol: str | None, ) -> list[tuple[str, bool]]: candidates: list[tuple[str, bool]] = [] @@ -145,18 +151,18 @@ def add_candidate(fetch_symbol: str | None, is_reversed: bool) -> None: if sticky_fetch_symbol: add_candidate( sticky_fetch_symbol, - _is_reversed_fetch_symbol(base_asset, sticky_fetch_symbol), + _is_reversed_fetch_symbol(bare_base_asset, sticky_fetch_symbol), ) direct_symbol, is_reversed = exchange_util.get_associated_symbol( - exchange_manager, base_asset, quote, + exchange_manager, bare_base_asset, bare_quote, ) add_candidate(direct_symbol, is_reversed) - if quote in commons_constants.USD_LIKE_COINS: - for alt_quote in _ordered_usd_like_quotes(quote): + if symbol_util.is_usd_like_coin(bare_quote): + for alt_quote in _ordered_usd_like_quotes(bare_quote): alt_symbol, alt_is_reversed = exchange_util.get_associated_symbol( - exchange_manager, base_asset, alt_quote, + exchange_manager, bare_base_asset, alt_quote, ) add_candidate(alt_symbol, alt_is_reversed) diff --git a/packages/flow/octobot_flow/logic/portfolio_history/trade_fetch_cursors.py b/packages/flow/octobot_flow/logic/portfolio_history/trade_fetch_cursors.py index 083a171acf..13fa31bb81 100644 --- a/packages/flow/octobot_flow/logic/portfolio_history/trade_fetch_cursors.py +++ b/packages/flow/octobot_flow/logic/portfolio_history/trade_fetch_cursors.py @@ -48,10 +48,11 @@ def resolve_daily_cache_symbol( return trade_symbol if not symbol_util.is_symbol(trade_symbol): return None - base_asset, _quote_asset = symbol_util.parse_symbol(trade_symbol).base_and_quote() - if not base_asset: + parsed_trade_symbol = symbol_util.parse_symbol(trade_symbol) + sources_cache_key = parsed_trade_symbol.base + if not sources_cache_key: return None - sticky_fetch_symbol = trading_api.get_daily_close_source(daily_prices, base_asset) + sticky_fetch_symbol = trading_api.get_daily_close_source(daily_prices, sources_cache_key) if sticky_fetch_symbol is None: return None if trading_api.get_latest_daily_price_timestamp(daily_prices, sticky_fetch_symbol) is None: diff --git a/packages/flow/octobot_flow/logic/portfolio_history/trade_symbols_discovery.py b/packages/flow/octobot_flow/logic/portfolio_history/trade_symbols_discovery.py index 1dad618fa7..22dfbaf05c 100644 --- a/packages/flow/octobot_flow/logic/portfolio_history/trade_symbols_discovery.py +++ b/packages/flow/octobot_flow/logic/portfolio_history/trade_symbols_discovery.py @@ -162,7 +162,11 @@ def _add_market_pair_for_currency( currency: str | None, reference_market: str, ) -> None: - if not currency or currency == reference_market or currency in commons_constants.USD_LIKE_COINS: + if ( + not currency + or symbol_util.is_same_coin(currency, reference_market) + or symbol_util.is_usd_like_coin(currency) + ): return direct_symbol, _is_reversed_symbol = exchange_util_module.get_associated_symbol( exchange_manager, diff --git a/packages/flow/octobot_flow/parsers/actions_dag_parser.py b/packages/flow/octobot_flow/parsers/actions_dag_parser.py index 703754d5b3..22b9b1701e 100644 --- a/packages/flow/octobot_flow/parsers/actions_dag_parser.py +++ b/packages/flow/octobot_flow/parsers/actions_dag_parser.py @@ -117,6 +117,7 @@ class ActionsDAGParserParams(octobot_commons.dataclasses.MinimizableDataclass): BLOCKCHAIN_BALANCE_ASSET: typing.Optional[str] = None BLOCKCHAIN_BALANCE: typing.Optional[str] = None LOOP_INTERVAL: typing.Optional[float] = None + LOOP_INTERVAL_MAX: typing.Optional[float] = None LOOP_TIMEOUT: typing.Optional[float] = None LOOP_MAX_ATTEMPTS: typing.Optional[int] = None CONTENT: typing.Optional[dict] = None @@ -136,10 +137,41 @@ def validate(self): if self.EXCHANGE_TO and self.EXCHANGE_FROM: if self.EXCHANGE_TO != self.EXCHANGE_FROM: raise octobot_flow.errors.InvalidAutomationActionError("EXCHANGE_TO and EXCHANGE_FROM must be the same") + self._normalize_loop_params() + + def _normalize_loop_params(self) -> None: + if ( + self.LOOP_INTERVAL is None + and self.LOOP_INTERVAL_MAX is None + and self.LOOP_TIMEOUT is None + and self.LOOP_MAX_ATTEMPTS is None + ): + return + try: + if self.LOOP_INTERVAL is not None: + self.LOOP_INTERVAL = float(self.LOOP_INTERVAL) + if self.LOOP_INTERVAL_MAX is not None: + self.LOOP_INTERVAL_MAX = float(self.LOOP_INTERVAL_MAX) + if self.LOOP_TIMEOUT is not None: + self.LOOP_TIMEOUT = float(self.LOOP_TIMEOUT) + if self.LOOP_MAX_ATTEMPTS is not None: + self.LOOP_MAX_ATTEMPTS = int(self.LOOP_MAX_ATTEMPTS) + except (TypeError, ValueError): + raise octobot_flow.errors.InvalidAutomationActionError( + f"LOOP_INTERVAL {self.LOOP_INTERVAL} and LOOP_INTERVAL_MAX {self.LOOP_INTERVAL_MAX} must be numbers" + ) + if ( + self.LOOP_INTERVAL is not None + and self.LOOP_INTERVAL_MAX is not None + and self.LOOP_INTERVAL_MAX < self.LOOP_INTERVAL + ): + raise octobot_flow.errors.InvalidAutomationActionError( + f"LOOP_INTERVAL_MAX {self.LOOP_INTERVAL_MAX} must be greater than or equal to LOOP_INTERVAL {self.LOOP_INTERVAL} when set" + ) def get_reference_market(self) -> typing.Optional[str]: if self.ORDER_SYMBOL: - parsed_symbol = octobot_commons.symbols.parse_symbol(self.ORDER_SYMBOL) + parsed_symbol = _parse_order_symbol(self.ORDER_SYMBOL) return parsed_symbol.quote return None @@ -430,8 +462,8 @@ def _create_generic_action( case ActionType.WAIT.value: return self._create_wait_action(index) case _: - raise ValueError( - f"Unknown action: {action}" + raise octobot_flow.errors.InvalidAutomationActionError( + f"Unknown action: {action!r}" ) def _create_order_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: @@ -441,13 +473,20 @@ def _create_order_action(self, index: int) -> octobot_flow.entities.AbstractActi ["ORDER_SYMBOL", "ORDER_AMOUNT", "ORDER_TYPE"], "trade", ) - parsed_symbol = octobot_commons.symbols.parse_symbol(self.params.ORDER_SYMBOL) + parsed_symbol = _parse_order_symbol(self.params.ORDER_SYMBOL) if self.params.ORDER_SIDE: signal = self.params.ORDER_SIDE.lower() - elif parsed_symbol.base == self.params.BLOCKCHAIN_FROM_ASSET and parsed_symbol.quote == self.params.BLOCKCHAIN_TO_ASSET: # type: ignore + # Bare asset ticker: DAG blockchain leg params are bare tickers — .base/.quote are network-qualified on ticker-wise pairs; use .base/.quote for portfolio[...] / reference_market. + elif ( + parsed_symbol.base_asset_ticker() == self.params.BLOCKCHAIN_FROM_ASSET + and parsed_symbol.quote_asset_ticker() == self.params.BLOCKCHAIN_TO_ASSET + ): # type: ignore # sell the first blockchain asset to get the second one signal = tv_trading_mode.SELL_SIGNAL - elif parsed_symbol.base == self.params.BLOCKCHAIN_TO_ASSET and parsed_symbol.quote == self.params.BLOCKCHAIN_FROM_ASSET: # type: ignore + elif ( + parsed_symbol.base_asset_ticker() == self.params.BLOCKCHAIN_TO_ASSET + and parsed_symbol.quote_asset_ticker() == self.params.BLOCKCHAIN_FROM_ASSET + ): # type: ignore # buy the second blockchain asset to get the first one signal = tv_trading_mode.BUY_SIGNAL else: @@ -617,18 +656,28 @@ def _create_transfer_action( dataclasses.asdict(transfer_details), ) - def _get_loop_params(self) -> tuple[typing.Optional[float], typing.Optional[float], int]: - loop_interval, loop_timeout, loop_max_attempts = ( - self.params.LOOP_INTERVAL, self.params.LOOP_TIMEOUT, self.params.LOOP_MAX_ATTEMPTS - ) - if not loop_interval: + def _get_loop_params( + self, + ) -> tuple[float, float, typing.Optional[float], typing.Optional[int]]: + if not self.params.LOOP_INTERVAL: raise octobot_flow.errors.InvalidAutomationActionError( "LOOP_INTERVAL must be provided for the loop_until action" ) - return loop_interval, loop_timeout, loop_max_attempts # type: ignore + loop_interval = self.params.LOOP_INTERVAL + max_interval = ( + self.params.LOOP_INTERVAL_MAX + if self.params.LOOP_INTERVAL_MAX is not None + else loop_interval + ) + return loop_interval, max_interval, self.params.LOOP_TIMEOUT, self.params.LOOP_MAX_ATTEMPTS + + def _format_loop_until_interval_args(self, min_interval: float, max_interval: float) -> str: + if max_interval > min_interval: + return f"{min_interval}, max_retry_interval={max_interval}" + return f"{min_interval}" def _create_loop_until_order_closed_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: - loop_interval, loop_timeout, loop_max_attempts = self._get_loop_params() + loop_interval, loop_interval_max, loop_timeout, loop_max_attempts = self._get_loop_params() self._ensure_params( ["ORDER_EXCHANGE_ID", "ORDER_SYMBOL"], "loop_until_order_closed", @@ -644,9 +693,10 @@ def _create_loop_until_order_closed_action(self, index: int) -> octobot_flow.ent f"\"get({commons_constants.LOCAL_VALUE_PLACEHOLDER}, 'status', '{trading_enums.OrderStatus.OPEN.value}') " f"!= '{trading_enums.OrderStatus.OPEN.value}'\")" ) + interval_args = self._format_loop_until_interval_args(loop_interval, loop_interval_max) dsl_script = ( f"loop_until({selector}, " - f"{loop_interval}, timeout={loop_timeout}, max_attempts={loop_max_attempts}, " + f"{interval_args}, timeout={loop_timeout}, max_attempts={loop_max_attempts}, " f"return_remaining_time=True)" ) action_id = f"action_loop_until_order_closed_{index}" @@ -655,7 +705,7 @@ def _create_loop_until_order_closed_action(self, index: int) -> octobot_flow.ent def _create_loop_until_blockchain_balance_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: tradingview_signal_to_dsl_translator = _tradingview_signal_to_dsl_translator() - loop_interval, loop_timeout, loop_max_attempts = self._get_loop_params() + loop_interval, loop_interval_max, loop_timeout, loop_max_attempts = self._get_loop_params() amount, asset = self.params.BLOCKCHAIN_BALANCE_AMOUNT, self.params.BLOCKCHAIN_BALANCE_ASSET if not amount or not asset: raise octobot_flow.errors.InvalidAutomationActionError( @@ -668,9 +718,10 @@ def _create_loop_until_blockchain_balance_action(self, index: int) -> octobot_fl wallet_params, {"asset": asset}, ) + interval_args = self._format_loop_until_interval_args(loop_interval, loop_interval_max) dsl_script = ( f"loop_until(value_if({wallet_check}, ' >= {float(amount)}'), " - f"{loop_interval}, timeout={loop_timeout}, max_attempts={loop_max_attempts}, " + f"{interval_args}, timeout={loop_timeout}, max_attempts={loop_max_attempts}, " f"return_remaining_time=True)" ) action_id = f"action_loop_until_blockchain_balance_{index}" @@ -846,6 +897,15 @@ def create_configured_action_details( ) +def _parse_order_symbol(symbol_str: str) -> octobot_commons.symbols.Symbol: + try: + return octobot_commons.symbols.parse_symbol(symbol_str) + except (AttributeError, ValueError, TypeError) as error: + raise octobot_flow.errors.InvalidAutomationActionError( + f"Invalid order symbol: {symbol_str!r}" + ) from error + + def _parse_dependency_param_value( value: str, ) -> typing.Optional[tuple[str, tuple[str, ...]]]: diff --git a/packages/flow/octobot_flow/repositories/exchange/channel_producer_ensure.py b/packages/flow/octobot_flow/repositories/exchange/channel_producer_ensure.py new file mode 100644 index 0000000000..8b3d06f7af --- /dev/null +++ b/packages/flow/octobot_flow/repositories/exchange/channel_producer_ensure.py @@ -0,0 +1,36 @@ +import octobot_trading.exchange_channel as exchange_channel_module +import octobot_trading.exchanges as trading_exchanges + + +def exchange_has_registered_channels(exchange_manager) -> bool: + try: + registered_channels = exchange_channel_module.get_exchange_channels(exchange_manager.id) + except KeyError: + return False + return bool(registered_channels) + + +def channel_has_producer(exchange_manager, channel_name: str) -> bool: + try: + channel = exchange_channel_module.get_chan(channel_name, exchange_manager.id) + except KeyError: + return False + return bool(channel.get_producers()) + + +async def ensure_temporary_channel_producer( + exchange_manager, + channel_name: str, + updater_class: type, + *, + create_all_channels_if_missing: bool = True, +) -> None: + if create_all_channels_if_missing and not exchange_has_registered_channels(exchange_manager): + await trading_exchanges.create_exchange_channels(exchange_manager) + if channel_has_producer(exchange_manager, channel_name): + return + await trading_exchanges.create_producers( + exchange_manager, + [updater_class], + start_producers=False, + ) diff --git a/packages/flow/octobot_flow/repositories/exchange/ohlcv_repository.py b/packages/flow/octobot_flow/repositories/exchange/ohlcv_repository.py index 6e25269eca..bcb101b7bc 100644 --- a/packages/flow/octobot_flow/repositories/exchange/ohlcv_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/ohlcv_repository.py @@ -2,6 +2,7 @@ import octobot_commons.enums as common_enums import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import +import octobot_flow.repositories.exchange.channel_producer_ensure as channel_producer_ensure_module import octobot_trading.exchange_data import octobot_trading.exchanges.util.exchange_data as exchange_data_import import octobot_trading.constants @@ -9,9 +10,18 @@ class OhlcvRepository(base_exchange_repository_import.BaseExchangeRepository): + @classmethod + async def ensure_temporary_ohlcv_channel(cls, exchange_manager) -> None: + await channel_producer_ensure_module.ensure_temporary_channel_producer( + exchange_manager, + octobot_trading.constants.OHLCV_CHANNEL, + octobot_trading.exchange_data.OHLCVUpdater, + ) + async def fetch_ohlcv( self, symbol: str, time_frame: str, limit: int, tickers: dict[str, dict[str, typing.Any]] ) -> exchange_data_import.MarketDetails: + await self.ensure_temporary_ohlcv_channel(self.exchange_manager) updater = typing.cast( octobot_trading.exchange_data.OHLCVUpdater, self.get_channel_updater(octobot_trading.constants.OHLCV_CHANNEL) @@ -20,4 +30,3 @@ async def fetch_ohlcv( symbol, common_enums.TimeFrames(time_frame), limit, allow_cache=True, tickers_backup=tickers ) return exchange_data_import.MarketDetails.from_ohlcvs(symbol, time_frame, ohlcvs) - diff --git a/packages/flow/octobot_flow/repositories/exchange/orders_repository.py b/packages/flow/octobot_flow/repositories/exchange/orders_repository.py index 95b51e4381..86d9027737 100644 --- a/packages/flow/octobot_flow/repositories/exchange/orders_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/orders_repository.py @@ -1,9 +1,9 @@ import typing import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import +import octobot_flow.repositories.exchange.channel_producer_ensure as channel_producer_ensure_module import octobot_trading.constants as trading_constants import octobot_trading.enums as trading_enums -import octobot_trading.exchanges as trading_exchanges import octobot_trading.storage as orders_storage import octobot_trading.personal_data as trading_personal_data @@ -12,10 +12,10 @@ class OrdersRepository(base_exchange_repository_import.BaseExchangeRepository): @classmethod async def ensure_temporary_orders_channel(cls, exchange_manager) -> None: - await trading_exchanges.create_producers( + await channel_producer_ensure_module.ensure_temporary_channel_producer( exchange_manager, - [trading_personal_data.OrdersUpdater], - start_producers=False, + trading_constants.ORDERS_CHANNEL, + trading_personal_data.OrdersUpdater, ) async def fetch_open_orders( @@ -23,6 +23,7 @@ async def fetch_open_orders( ) -> list[dict]: if not symbols: return [] + await self.ensure_temporary_orders_channel(self.exchange_manager) updater = typing.cast( trading_personal_data.OrdersUpdater, self.get_channel_updater(trading_constants.ORDERS_CHANNEL) @@ -52,4 +53,4 @@ def update_enriched_orders( self.exchange_manager ) for order in updated_orders - ] \ No newline at end of file + ] diff --git a/packages/flow/octobot_flow/repositories/exchange/portfolio_repository.py b/packages/flow/octobot_flow/repositories/exchange/portfolio_repository.py index acb2e39b2e..3fe2e0c3a1 100644 --- a/packages/flow/octobot_flow/repositories/exchange/portfolio_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/portfolio_repository.py @@ -2,9 +2,9 @@ import typing import octobot_trading.api as trading_api -import octobot_trading.exchanges as trading_exchanges import octobot_trading.personal_data as personal_data import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import +import octobot_flow.repositories.exchange.channel_producer_ensure as channel_producer_ensure_module import octobot_trading.constants as trading_constants import octobot_trading.personal_data as trading_personal_data @@ -13,14 +13,14 @@ class PortfolioRepository(base_exchange_repository_import.BaseExchangeRepository @classmethod async def ensure_temporary_balance_channel(cls, exchange_manager) -> None: - await trading_exchanges.create_producers( + await channel_producer_ensure_module.ensure_temporary_channel_producer( exchange_manager, - [trading_personal_data.BalanceUpdater], - start_producers=False, + trading_constants.BALANCE_CHANNEL, + trading_personal_data.BalanceUpdater, ) async def fetch_portfolio(self) -> dict[str, dict[str, decimal.Decimal]]: - + await self.ensure_temporary_balance_channel(self.exchange_manager) updater = typing.cast( trading_personal_data.BalanceUpdater, self.get_channel_updater(trading_constants.BALANCE_CHANNEL) @@ -31,6 +31,7 @@ async def fetch_portfolio(self) -> dict[str, dict[str, decimal.Decimal]]: ) # type: ignore async def fetch_and_apply_portfolio(self) -> dict[str, dict[str, decimal.Decimal]]: + await self.ensure_temporary_balance_channel(self.exchange_manager) updater = typing.cast( trading_personal_data.BalanceUpdater, self.get_channel_updater(trading_constants.BALANCE_CHANNEL) diff --git a/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py b/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py index 9a58b3855b..2ae165e7bf 100644 --- a/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py @@ -2,11 +2,11 @@ import octobot_trading.exchange_data import octobot_trading.enums as trading_enums -import octobot_trading.exchanges as trading_exchanges import octobot_trading.exchanges.util.exchange_data as exchange_data_import import octobot_flow.entities import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import +import octobot_flow.repositories.exchange.channel_producer_ensure as channel_producer_ensure_module import octobot_trading.constants as trading_constants @@ -14,14 +14,14 @@ class TickersRepository(base_exchange_repository_import.BaseExchangeRepository): @classmethod async def ensure_temporary_ticker_channel(cls, exchange_manager) -> None: - await trading_exchanges.create_exchange_channels(exchange_manager) - await trading_exchanges.create_producers( + await channel_producer_ensure_module.ensure_temporary_channel_producer( exchange_manager, - [octobot_trading.exchange_data.TickerUpdater], - start_producers=False, + trading_constants.TICKER_CHANNEL, + octobot_trading.exchange_data.TickerUpdater, ) async def fetch_tickers(self, symbols: typing.Optional[list[str]]) -> dict[str, dict]: + await self.ensure_temporary_ticker_channel(self.exchange_manager) updater = typing.cast( octobot_trading.exchange_data.TickerUpdater, self.get_channel_updater(trading_constants.TICKER_CHANNEL) diff --git a/packages/flow/octobot_flow/repositories/exchange/trades_repository.py b/packages/flow/octobot_flow/repositories/exchange/trades_repository.py index 611c5f950a..3ceb9e8efb 100644 --- a/packages/flow/octobot_flow/repositories/exchange/trades_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/trades_repository.py @@ -4,11 +4,11 @@ import octobot_commons.logging as commons_logging import octobot_trading.constants as trading_constants import octobot_trading.enums as trading_enums -import octobot_trading.exchanges as trading_exchanges import octobot_trading.personal_data as trading_personal_data import octobot_trading.personal_data.trades.trades_util as trades_util_module import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import +import octobot_flow.repositories.exchange.channel_producer_ensure as channel_producer_ensure_module logger = commons_logging.get_logger("PortfolioHistoryJob") @@ -140,11 +140,10 @@ class TradesRepository(base_exchange_repository_import.BaseExchangeRepository): @classmethod async def ensure_temporary_trades_channel(cls, exchange_manager) -> None: - await trading_exchanges.create_exchange_channels(exchange_manager) - await trading_exchanges.create_producers( + await channel_producer_ensure_module.ensure_temporary_channel_producer( exchange_manager, - [trading_personal_data.TradesUpdater], - start_producers=False, + trading_constants.TRADES_CHANNEL, + trading_personal_data.TradesUpdater, ) async def fetch_trades(self, symbols: list[str]) -> list[dict]: @@ -209,6 +208,7 @@ async def _fetch_raw_trades( since: int | None = None, exhaust_history: bool = False, ) -> list[dict]: + await self.ensure_temporary_trades_channel(self.exchange_manager) fetch_kwargs: dict = {} if since is not None: fetch_kwargs["since"] = since diff --git a/packages/flow/tests/conftest.py b/packages/flow/tests/conftest.py index eb3d4dd107..bb75b42bfa 100644 --- a/packages/flow/tests/conftest.py +++ b/packages/flow/tests/conftest.py @@ -2,11 +2,26 @@ import dotenv import os +import sys + dotenv.load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), ".env")) import mock import pytest +_OCTOBOT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +_TESTS_ROOT = os.path.join(_OCTOBOT_ROOT, "tests") +if _TESTS_ROOT not in sys.path: + sys.path.insert(0, _TESTS_ROOT) + +from test_utils.journal_test_support import disabled_node_journal_environment + + +@pytest.fixture(autouse=True) +def disable_node_journal(): + with disabled_node_journal_environment(): + yield + @pytest.fixture(autouse=True) def _disable_auto_open_in_web_browser(): diff --git a/packages/flow/tests/entities/actions/test_actions_dag.py b/packages/flow/tests/entities/actions/test_actions_dag.py index 6805c83fc4..3709f2f605 100644 --- a/packages/flow/tests/entities/actions/test_actions_dag.py +++ b/packages/flow/tests/entities/actions/test_actions_dag.py @@ -107,6 +107,30 @@ def test_resets_dsl_action_and_its_dependents(self): assert strategy_action.executed_at is None +class TestHasFailedAction: + def test_true_when_completed_action_has_error_status(self): + failed_action = _configured_action("action_init") + failed_action.complete( + error_status=flow_enums.ActionErrorStatus.BLOCKCHAIN_WALLET_ERROR.value, + ) + dag = actions_dag.ActionsDAG(actions=[failed_action]) + + assert dag.has_failed_action() is True + + def test_false_when_all_completed_without_error(self): + success_action = _configured_action("action_init") + success_action.complete(result={"done": True}) + dag = actions_dag.ActionsDAG(actions=[success_action]) + + assert dag.has_failed_action() is False + + def test_false_when_only_pending_actions(self): + pending_action = _configured_action("action_init") + dag = actions_dag.ActionsDAG(actions=[pending_action]) + + assert dag.has_failed_action() is False + + class TestResolveDslScripts: def test_delegates_to_dependencies_resolver_for_dsl_actions_only(self): init_action = _configured_action("action_init") diff --git a/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py b/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py index e02c73c54c..63bc2dd8f6 100644 --- a/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py +++ b/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py @@ -19,7 +19,6 @@ import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module import octobot_sync.constants as sync_constants -from tests.logic.global_view.portfolio_test_util import patch_temporary_exchange_channel_ensure from tests.logic.global_view.portfolio_test_util import wire_portfolio_pipeline from tests.logic.global_view.portfolio_test_util import wire_repository_factory @@ -139,11 +138,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "exchange_manager_from_exchange_data", fake_exchange_manager, ), - patch_temporary_exchange_channel_ensure() as ( - ensure_ticker_channel_mock, - ensure_balance_channel_mock, - ensure_orders_channel_mock, - ), mock.patch.object( personal_data, "refresh_portfolio_valuation", @@ -154,11 +148,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "_fetch_tickers", mock.AsyncMock(return_value={}), ), - mock.patch.object( - exchange_account_refresh_module.tickers_repository_module.TickersRepository, - "fetch_tickers", - mock.AsyncMock(return_value={}), - ), mock.patch.object( global_view_account_job_module.tentacles_manager_api, "get_full_tentacles_setup_config", @@ -184,9 +173,6 @@ async def fake_exchange_manager(*_args, **_kwargs): context, ).run() - ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) - ensure_balance_channel_mock.assert_not_awaited() - ensure_orders_channel_mock.assert_not_awaited() portfolio_manager.apply_forced_portfolio.assert_called_once() assert refresh_result.updated_account.assets asset_symbols = { @@ -223,11 +209,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "exchange_manager_from_exchange_data", fake_exchange_manager, ), - patch_temporary_exchange_channel_ensure() as ( - ensure_ticker_channel_mock, - ensure_balance_channel_mock, - ensure_orders_channel_mock, - ), mock.patch.object( personal_data, "refresh_portfolio_valuation", @@ -263,9 +244,6 @@ async def fake_exchange_manager(*_args, **_kwargs): context, ).run() - ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) - ensure_balance_channel_mock.assert_awaited_once_with(exchange_manager) - ensure_orders_channel_mock.assert_awaited_once_with(exchange_manager) orders_repository.fetch_open_orders.assert_awaited_once_with(["BTC/USDT", "ETH/USDT"]) assert refresh_result.updated_account.assets assert refresh_result.changed_order_ids == {"gone-order-1"} diff --git a/packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py b/packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py index ef13a05deb..e3eaf50e24 100644 --- a/packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py +++ b/packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py @@ -349,11 +349,6 @@ async def fake_exchange_manager(exchange_data, *_args, **_kwargs): "get_full_tentacles_setup_config", return_value=mock.Mock(), ), - mock.patch.object( - portfolio_history_job_module.trades_repository_module.TradesRepository, - "ensure_temporary_trades_channel", - trades_repository_test_util.ensure_trades_channel, - ), mock.patch.object( daily_price_cache_updater_module.exchange_util, "get_historical_ohlcv", diff --git a/packages/flow/tests/functionnal_tests/trading_modes_actions/simulator/coinrabbit_ticker_wise_test_util.py b/packages/flow/tests/functionnal_tests/trading_modes_actions/simulator/coinrabbit_ticker_wise_test_util.py new file mode 100644 index 0000000000..cbb1ee6f32 --- /dev/null +++ b/packages/flow/tests/functionnal_tests/trading_modes_actions/simulator/coinrabbit_ticker_wise_test_util.py @@ -0,0 +1,214 @@ +""" +CoinRabbit ticker-wise symbol functional test helpers. + +Portfolio keys: reference USDT@ETH; after index rebalance on CoinRabbit, BTC@BTC is acquired +(CoinRabbit lists BTC@BTC/USDT@ only — no ETH@ETH/USDT@ETH market; index still +configures ETH@ETH in the basket alongside BTC@BTC). +""" +import contextlib +import decimal +import time +import typing + +import mock + +import octobot_commons.asyncio_tools as asyncio_tools +import octobot_commons.enums as common_enums +import octobot_commons.constants as commons_constants +import octobot_commons.symbols as commons_symbols +import octobot_commons.tests.test_config as test_config +import octobot_trading.api as trading_api +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges as trading_exchanges +import octobot_trading.exchanges.util.exchange_data as exchange_data +import octobot_trading.personal_data as trading_personal_data +import octobot_trading.util.test_tools.exchanges_test_tools as exchanges_test_tools +import octobot_flow.repositories.exchange as exchange_repositories + +EXCHANGE_INTERNAL_NAME = "coinrabbit" +REFERENCE_ASSET = "USDT@ETH" +INDEX_COINS = ("ETH@ETH", "BTC@BTC") +# CoinRabbit exposes BTC@BTC/USDT@ pairs only (no ETH@ETH/USDT@ETH). +REQUIRED_MARKET_SYMBOLS = ("BTC@BTC/USDT@ETH",) +TRADED_PAIR = "BTC@BTC/USDT@ETH" +# USDT@ETH may be omitted from the dump when fully allocated to the index leg. +EXPECTED_PORTFOLIO_KEYS_AFTER_REBALANCE = ("BTC@BTC",) + +INITIAL_REFERENCE_HOLDING = 1000.0 +MOCK_CLOSE_BY_SYMBOL: dict[str, float] = { + "BTC@BTC/USDT@ETH": 100_000.0, +} + + +@contextlib.asynccontextmanager +async def coinrabbit_exchange_manager_context(): + config = test_config.load_test_config() + if EXCHANGE_INTERNAL_NAME not in config[commons_constants.CONFIG_EXCHANGES]: + config[commons_constants.CONFIG_EXCHANGES][EXCHANGE_INTERNAL_NAME] = {} + exchange_manager = trading_exchanges.ExchangeManager(config, EXCHANGE_INTERNAL_NAME) + exchange_manager.exchange_only = True + await exchange_manager.initialize(exchange_config_by_exchange=None) + try: + yield exchange_manager + finally: + await exchange_manager.stop() + trading_api.cancel_ccxt_throttle_task() + await asyncio_tools.wait_asyncio_next_cycle() + + +async def assert_coinrabbit_markets_expose_symbols( + exchange_manager: trading_exchanges.ExchangeManager, + required_symbols: typing.Sequence[str] = REQUIRED_MARKET_SYMBOLS, # noqa: UP006 +) -> None: + available = set(trading_api.get_all_available_symbols(exchange_manager)) + for symbol in required_symbols: + assert symbol in available, ( + f"{symbol!r} missing from get_all_available_symbols; sample: {sorted(available)[:12]}" + ) + market_status = exchange_manager.exchange.get_market_status(symbol, with_fixer=False) + info = market_status.get(trading_enums.ExchangeConstantsMarketStatusColumns.INFO.value, {}) + assert info.get("base_network"), f"market info must preserve base_network for {symbol}" + assert info.get("quote_network"), f"market info must preserve quote_network for {symbol}" + parsed = commons_symbols.parse_symbol(TRADED_PAIR) + assert parsed.has_ticker_wise_networks() is True + + +def tickers_repository_fetch_tickers_close_override( + close_by_symbol: dict[str, float], +): + close_col = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + + async def patched_fetch_tickers(self, symbols): + if symbols == []: + return {} + if isinstance(symbols, list): + return { + symbol: {close_col: close_by_symbol[symbol]} + for symbol in symbols + if symbol in close_by_symbol + } + return { + symbol: {close_col: close_value} + for symbol, close_value in close_by_symbol.items() + } + + return patched_fetch_tickers + + +def _mock_tickers_for_symbols( + symbols: typing.Optional[list[str]], + close_by_symbol: dict[str, float], +) -> dict[str, dict]: + close_col = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + if symbols: + return { + symbol: {close_col: close_by_symbol[symbol]} + for symbol in symbols + if symbol in close_by_symbol + } + return {symbol: {close_col: close} for symbol, close in close_by_symbol.items()} + + +def _fetch_ohlcv_side_effect_for_close_prices(close_by_symbol: dict[str, float]): + async def patched_fetch_ohlcv( + symbol: str, + time_frame: str, + limit: int, + _tickers: dict[str, dict[str, typing.Any]], + ): + time_frame_seconds = common_enums.TimeFramesMinutes[common_enums.TimeFrames(time_frame)] * 60 + close_price = float(close_by_symbol.get(symbol, next(iter(close_by_symbol.values())))) + candle_count = max(int(limit or 1), 1) + local_time = time.time() + current_candle_open_time = local_time - (local_time % time_frame_seconds) + first_candle_open_time = current_candle_open_time - (candle_count - 1) * time_frame_seconds + times = [float(first_candle_open_time + index * time_frame_seconds) for index in range(candle_count)] + closes = [close_price] * candle_count + return exchange_data.MarketDetails( + symbol=symbol, + time_frame=time_frame, + close=closes, + open=closes, + high=closes, + low=closes, + volume=[0.0] * candle_count, + time=times, + ) + + return patched_fetch_ohlcv + + +@contextlib.contextmanager +def patch_coinrabbit_ticker_closes(close_by_symbol: dict[str, float] | None = None): + prices = close_by_symbol or MOCK_CLOSE_BY_SYMBOL + patched_fetch_tickers_impl = tickers_repository_fetch_tickers_close_override(prices) + patched_fetch_ohlcv_impl = _fetch_ohlcv_side_effect_for_close_prices(prices) + + async def tracked_fetch_tickers(self, symbols): + return await patched_fetch_tickers_impl(self, symbols) + + async def tracked_fetch_all_tickers(self, symbols): + return _mock_tickers_for_symbols(symbols, prices) + + async def tracked_fetch_ohlcv(self, symbol, time_frame, limit, tickers): + return await patched_fetch_ohlcv_impl(symbol, time_frame, limit, tickers) + + import octobot_trading.exchange_data.ticker.channel.ticker_updater as ticker_updater_module + + close_col = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + orig_get_all_tickers = exchanges_test_tools.get_all_currencies_price_ticker + orig_get_price_ticker = exchanges_test_tools.get_price_ticker + orig_get_up_to_date_price = trading_personal_data.get_up_to_date_price + + async def patched_get_all_currencies_price_ticker(exchange_manager, **kwargs): + tickers = await orig_get_all_tickers(exchange_manager, **kwargs) + for symbol, close_value in prices.items(): + if symbol in tickers: + tickers[symbol] = {**tickers[symbol], close_col: close_value} + else: + tickers[symbol] = {close_col: close_value} + return tickers + + async def patched_get_price_ticker(exchange_manager, symbol: str, **kwargs): + if symbol in prices: + return {close_col: prices[symbol]} + return await orig_get_price_ticker(exchange_manager, symbol, **kwargs) + + async def patched_get_up_to_date_price(exchange_manager, symbol: str, **kwargs): + if symbol in prices: + return decimal.Decimal(str(prices[symbol])) + return await orig_get_up_to_date_price(exchange_manager, symbol, **kwargs) + + with ( + mock.patch.object( + exchange_repositories.TickersRepository, + "fetch_tickers", + new=tracked_fetch_tickers, + ), + mock.patch.object( + exchange_repositories.OhlcvRepository, + "fetch_ohlcv", + new=tracked_fetch_ohlcv, + ), + mock.patch.object( + ticker_updater_module.TickerUpdater, + "fetch_all_tickers", + new=tracked_fetch_all_tickers, + ), + mock.patch.object( + exchanges_test_tools, + "get_all_currencies_price_ticker", + new=patched_get_all_currencies_price_ticker, + ), + mock.patch.object( + exchanges_test_tools, + "get_price_ticker", + new=patched_get_price_ticker, + ), + mock.patch.object( + trading_personal_data, + "get_up_to_date_price", + new=patched_get_up_to_date_price, + ), + ): + yield diff --git a/packages/flow/tests/functionnal_tests/trading_modes_actions/simulator/test_coinrabbit_index_trading_mode_ticker_wise.py b/packages/flow/tests/functionnal_tests/trading_modes_actions/simulator/test_coinrabbit_index_trading_mode_ticker_wise.py new file mode 100644 index 0000000000..d560193ed5 --- /dev/null +++ b/packages/flow/tests/functionnal_tests/trading_modes_actions/simulator/test_coinrabbit_index_trading_mode_ticker_wise.py @@ -0,0 +1,199 @@ +import contextlib +import json + +import mock +import pytest + +import octobot_copy.enums as rebalancer_enums +import octobot_flow.entities +import octobot_flow.enums +import octobot_flow.jobs +import octobot_flow.repositories.community as community_repository_module + +import tests.functionnal_tests as functionnal_tests +from tests.functionnal_tests import ( + automation_state_dict, + empty_copy_exchange_account_action, + resolved_actions, + set_emit_signals_metadata, + trading_signal_emission_patches, +) +from tests.functionnal_tests.trading_modes_actions.simulator import coinrabbit_ticker_wise_test_util as coinrabbit_util + + +def _index_content(): + return [ + {rebalancer_enums.DistributionKeys.NAME: coin, rebalancer_enums.DistributionKeys.VALUE: 1} + for coin in coinrabbit_util.INDEX_COINS + ] + + +def _init_action() -> dict: + return { + "id": "action_init", + "action": octobot_flow.enums.ActionType.APPLY_CONFIGURATION.value, + "config": { + "automation": { + "metadata": { + "automation_id": "automation_coinrabbit_ticker_wise", + }, + "exchange_account_elements": { + "portfolio": { + "content": { + coinrabbit_util.REFERENCE_ASSET: { + "available": coinrabbit_util.INITIAL_REFERENCE_HOLDING, + "total": coinrabbit_util.INITIAL_REFERENCE_HOLDING, + }, + }, + }, + }, + }, + "exchange_account_details": { + "exchange_details": { + "internal_name": coinrabbit_util.EXCHANGE_INTERNAL_NAME, + }, + "auth_details": {}, + "portfolio": { + "unit": coinrabbit_util.REFERENCE_ASSET, + }, + }, + }, + } + + +def _index_trading_mode_action(dependency_action: dict) -> dict: + return { + "id": "action_1", + "dsl_script": ( + f"index_trading_mode(index_content={json.dumps(_index_content())}, " + f"rebalance_trigger_min_percent=5)" + ), + "dependencies": [{"action_id": dependency_action["id"]}], + } + + +@pytest.mark.asyncio +async def test_coinrabbit_markets_expose_ticker_wise_symbols(skip_on_exchange_proxy_error): + async with coinrabbit_util.coinrabbit_exchange_manager_context() as exchange_manager: + await coinrabbit_util.assert_coinrabbit_markets_expose_symbols(exchange_manager) + + +@pytest.mark.asyncio +async def test_coinrabbit_index_trading_mode_ticker_wise_rebalance(skip_on_exchange_proxy_error): + init_action = _init_action() + all_actions = [init_action, _index_trading_mode_action(init_action)] + automation_state = automation_state_dict(resolved_actions(all_actions)) + initial_reference = coinrabbit_util.INITIAL_REFERENCE_HOLDING + + async with octobot_flow.jobs.AutomationJob(automation_state, [], [], {}) as automation_job: + await automation_job.run() + after_init_dump = automation_job.dump() + portfolio_after_init = ( + after_init_dump["automation"]["exchange_account_elements"]["portfolio"]["content"] + ) + assert portfolio_after_init[coinrabbit_util.REFERENCE_ASSET]["total"] == initial_reference + + with coinrabbit_util.patch_coinrabbit_ticker_closes(): + async with octobot_flow.jobs.AutomationJob(after_init_dump, [], [], {}) as automation_job: + await automation_job.run() + after_index_dump = automation_job.dump() + + for action in after_index_dump["automation"]["actions_dag"]["actions"]: + if action.get("id") == "action_1": + assert action.get("error_status") == octobot_flow.enums.ActionErrorStatus.NO_ERROR.value + break + else: + pytest.fail("action_1 not found") + + portfolio_content = ( + after_index_dump["automation"]["exchange_account_elements"]["portfolio"]["content"] + ) + for key in coinrabbit_util.EXPECTED_PORTFOLIO_KEYS_AFTER_REBALANCE: + assert key in portfolio_content, f"missing portfolio key {key!r}: {portfolio_content.keys()}" + assert portfolio_content["BTC@BTC"]["total"] > 0 + reference_total = portfolio_content.get(coinrabbit_util.REFERENCE_ASSET, {}).get("total", 0) + assert reference_total < initial_reference + + +def _copier_init_action() -> dict: + action = _init_action() + action["config"]["automation"]["metadata"]["automation_id"] = "automation_coinrabbit_ticker_wise_copier" + return action + + +@pytest.mark.asyncio +async def test_coinrabbit_index_signals_copied_by_copy_trading_bot(skip_on_exchange_proxy_error): + """ + Index leader on CoinRabbit emits a trading signal after ticker-wise rebalance; + a separate copier automation fetches that signal and rebalances to match. + """ + leader_init = _init_action() + leader_actions = [leader_init, _index_trading_mode_action(leader_init)] + leader_state = automation_state_dict(resolved_actions(leader_actions)) + leader_state["automation"]["metadata"]["strategy_id"] = functionnal_tests.FUNCTIONAL_TEST_COPY_STRATEGY_ID + set_emit_signals_metadata(leader_state, True) + initial_reference = coinrabbit_util.INITIAL_REFERENCE_HOLDING + + with trading_signal_emission_patches(True) as insert_trading_signal_mock: + async with octobot_flow.jobs.AutomationJob(leader_state, [], [], {}) as automation_job: + await automation_job.run() + after_leader_init = automation_job.dump() + + with coinrabbit_util.patch_coinrabbit_ticker_closes(): + async with octobot_flow.jobs.AutomationJob(after_leader_init, [], [], {}) as automation_job: + await automation_job.run() + + insert_trading_signal_mock.assert_awaited_once() + emitted_signal = insert_trading_signal_mock.await_args.args[0] + assert emitted_signal.strategy_id == functionnal_tests.FUNCTIONAL_TEST_COPY_STRATEGY_ID + + fetch_trading_signals_mock = mock.AsyncMock(return_value=[emitted_signal]) + copier_init = _copier_init_action() + copy_action = empty_copy_exchange_account_action() + copier_actions = [copier_init, copy_action] + copier_state = automation_state_dict(resolved_actions(copier_actions)) + + @contextlib.asynccontextmanager + async def _fake_maybe_authenticator(_self): + yield mock.MagicMock() + + with contextlib.ExitStack() as patch_stack: + patch_stack.enter_context(coinrabbit_util.patch_coinrabbit_ticker_closes()) + patch_stack.enter_context( + mock.patch.object( + octobot_flow.jobs.AutomationJob, + "_maybe_authenticator", + _fake_maybe_authenticator, + ) + ) + patch_stack.enter_context( + mock.patch.object( + community_repository_module.TradingSignalsRepository, + "fetch_trading_signals", + fetch_trading_signals_mock, + ) + ) + copier_auth = octobot_flow.entities.UserAuthentication( + wallet_address="simulator-coinrabbit-copy-functional-test-wallet", + ) + async with octobot_flow.jobs.AutomationJob( + copier_state, [], [], copier_auth + ) as automation_job: + await automation_job.run() + after_copier_init = automation_job.dump() + + async with octobot_flow.jobs.AutomationJob( + after_copier_init, [], [], copier_auth + ) as automation_job: + await automation_job.run() + after_copy_dump = automation_job.dump() + + fetch_trading_signals_mock.assert_awaited_once() + portfolio_content = ( + after_copy_dump["automation"]["exchange_account_elements"]["portfolio"]["content"] + ) + for key in coinrabbit_util.EXPECTED_PORTFOLIO_KEYS_AFTER_REBALANCE: + assert key in portfolio_content, f"missing portfolio key {key!r}: {portfolio_content.keys()}" + assert portfolio_content["BTC@BTC"]["total"] > 0 + reference_total = portfolio_content.get(coinrabbit_util.REFERENCE_ASSET, {}).get("total", 0) + assert reference_total < initial_reference diff --git a/packages/flow/tests/jobs/test_portfolio_history_job.py b/packages/flow/tests/jobs/test_portfolio_history_job.py index e4ca1b1f05..9606561e53 100644 --- a/packages/flow/tests/jobs/test_portfolio_history_job.py +++ b/packages/flow/tests/jobs/test_portfolio_history_job.py @@ -97,7 +97,6 @@ async def test_account_history_provider_not_called( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) @@ -112,9 +111,6 @@ async def test_account_history_provider_not_called( assert results[0].duration_seconds is not None assert results[0].duration_seconds >= 0 assert results[0].price_symbols_count >= 0 - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel.assert_awaited_once_with( - mock_exchange_manager, - ) # merge_and_persist_trading_history should be called, but not AccountHistoryProvider mock_merge.merge_and_persist_trading_history.assert_called_once() @@ -154,7 +150,6 @@ async def test_passes_currencies_with_balance_to_tx_repo( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) @@ -203,7 +198,6 @@ async def test_skips_tx_fetch_when_no_balance( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) @@ -250,7 +244,6 @@ async def test_fetch_trades_uses_context_trade_symbols( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) @@ -296,7 +289,6 @@ async def test_empty_trade_symbols_skips_trade_fetch( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) @@ -311,6 +303,54 @@ async def test_empty_trade_symbols_skips_trade_fetch( assert results[0].trade_symbols_count == 0 +class TestSkipTradeFetchForNetworkQualifiedMarkets: + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_skips_fetch_when_markets_use_network_qualified_symbols( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + import octobot_flow.logic.portfolio_history.trade_symbols_discovery as trade_symbols_discovery + + account = _make_account("acc1") + context = _make_context(account, trade_symbols=["BTC@BTC/USDT@ETH"]) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["BTC@BTC/USDT@ETH"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC@BTC/USDT@ETH"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + await job.run() + + fetch_trades_paginated_mock.assert_not_awaited() + mock_daily_cache.update_daily_prices.assert_awaited_once() + + class TestRunParallelExchangeAccounts: @pytest.mark.asyncio @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") @@ -345,7 +385,6 @@ async def test_multiple_accounts_run_in_parallel( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) @@ -372,6 +411,18 @@ def test_uses_dominant_usd_like_holding(self): assert reference_market == "USDC" + def test_uses_dominant_usd_like_network_qualified_holding(self): + account = mock.MagicMock() + usdt_on_eth = mock.MagicMock(symbol="USDT@ETH", total=500.0) + account.assets = [mock.MagicMock(assets=[usdt_on_eth])] + + reference_market = portfolio_history_job_module._reference_market_from_account_assets( + account, + "kraken", + ) + + assert reference_market == "USDT@ETH" + def test_falls_back_to_exchange_default_when_no_usd_like_holdings(self): account = mock.MagicMock() account.assets = [] @@ -495,6 +546,18 @@ def test_skips_usd_like_stablecoin_from_transactions(self, _mock_reference_marke ) assert symbols == [] + @mock.patch( + "octobot_trading.api.exchange.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_skips_transaction_matching_reference_market_ticker(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + [], + [{"currency": "USDT"}], + "USDT@ETH", + ) + assert symbols == [] + class TestFilterTradesOnLiveMarkets: def test_keeps_trades_on_live_markets(self): @@ -602,7 +665,6 @@ async def test_drops_delisted_trades_and_logs_symbols( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock( return_value=[ {"symbol": "BTC/USDC"}, @@ -675,7 +737,6 @@ async def test_discovers_portfolio_symbols_beyond_config_seeds( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) @@ -724,7 +785,6 @@ async def test_fetches_deposits_before_trades( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() call_order: list[str] = [] async def fetch_deposits(**_kwargs): @@ -790,7 +850,6 @@ async def test_logs_new_symbol_fetch_at_info( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.side_effect = ( lambda exchange_manager, *_args, **_kwargs: ( trades_repository_module.TradesRepository(exchange_manager, [], mock.MagicMock()) @@ -841,7 +900,6 @@ async def test_does_not_log_new_symbol_for_configured_symbol( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.side_effect = ( lambda exchange_manager, *_args, **_kwargs: ( trades_repository_module.TradesRepository(exchange_manager, [], mock.MagicMock()) @@ -897,7 +955,6 @@ async def test_persists_only_trade_confirmed_symbols( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock( return_value=[{"symbol": "ALGO/USDC"}], ) @@ -947,7 +1004,6 @@ async def test_logs_added_symbols_at_info( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) @@ -998,7 +1054,6 @@ async def test_trade_symbols_count_reflects_discovered_set( mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) @@ -1104,7 +1159,6 @@ async def test_load_daily_prices_before_fetch_and_passes_symbol_since_ms(self): mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) @@ -1157,7 +1211,6 @@ async def test_empty_account_trading_passes_no_symbol_since_ms(self): mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) @@ -1218,7 +1271,6 @@ async def test_persisted_trades_without_daily_cache_use_full_fetch(self, mock_lo mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) @@ -1272,7 +1324,6 @@ async def test_collection_no_data_error_passes_no_symbol_since_ms(self): mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) @@ -1330,7 +1381,6 @@ async def test_logs_incremental_and_full_symbol_counts(self, mock_logger): mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) @@ -1400,7 +1450,6 @@ async def test_usd_like_pair_uses_global_cache_for_symbol_since_ms(self): mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) @@ -1462,7 +1511,6 @@ async def test_mixed_symbols_log_global_cursor_and_no_candle_fallback(self, mock mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( return_value=False ) - mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) diff --git a/packages/flow/tests/logic/configuration/test_profile_data_factory.py b/packages/flow/tests/logic/configuration/test_profile_data_factory.py index 0231d86f68..b9a25117b1 100644 --- a/packages/flow/tests/logic/configuration/test_profile_data_factory.py +++ b/packages/flow/tests/logic/configuration/test_profile_data_factory.py @@ -113,7 +113,76 @@ def test_enables_simulator_for_simulated_account(self): assert profile_data.trader_simulator.enabled is True +class TestGetCryptoCurrenciesFromSymbols: + def test_groups_trading_pairs_under_bare_base_ticker(self): + exchange_account_details = exchange_account_details_module.ExchangeAccountDetails( + exchange_details=profile_data_module.ExchangeData(internal_name="coinrabbit"), + auth_details=exchange_data_module.ExchangeAuthDetails(), + ) + profile_data = profile_data_factory_module.create_profile_data( + exchange_account_details, + automation_id="automation-ticker-wise", + symbols={ + "BTC@BTC/USDT@ETH", + "ETH@ETH/USDT@ETH", + }, + as_simulator=True, + ) + crypto_by_name = {currency.name: currency.trading_pairs for currency in profile_data.crypto_currencies} + assert crypto_by_name == { + "BTC": ["BTC@BTC/USDT@ETH"], + "ETH": ["ETH@ETH/USDT@ETH"], + } + assert profile_data.trading.reference_market == "USDT@ETH" + + def test_spot_pair_reference_market_uses_bare_quote(self): + exchange_account_details = exchange_account_details_module.ExchangeAccountDetails( + exchange_details=profile_data_module.ExchangeData(internal_name="binanceus"), + auth_details=exchange_data_module.ExchangeAuthDetails(), + ) + profile_data = profile_data_factory_module.create_profile_data( + exchange_account_details, + automation_id="automation-spot", + symbols={"BTC/USDT"}, + as_simulator=True, + ) + assert profile_data.trading.reference_market == "USDT" + + class TestInferReferenceMarket: + def test_uses_qualified_quote_from_crypto_when_pairs_present(self): + exchange_account_details = exchange_account_details_module.ExchangeAccountDetails( + exchange_details=profile_data_module.ExchangeData(internal_name="coinrabbit"), + auth_details=exchange_data_module.ExchangeAuthDetails(), + portfolio=exchange_account_details_module.ExchangeAccountPortfolio(unit="USDT@ETH"), + ) + crypto_currencies = [ + profile_data_module.CryptoCurrencyData( + name="BTC", + trading_pairs=["BTC@BTC/USDT@ETH"], + ) + ] + assert ( + profile_data_factory_module.infer_reference_market(exchange_account_details, crypto_currencies) + == "USDT@ETH" + ) + + def test_uses_bare_quote_from_spot_trading_pair(self): + exchange_account_details = exchange_account_details_module.ExchangeAccountDetails( + exchange_details=profile_data_module.ExchangeData(internal_name="binanceus"), + auth_details=exchange_data_module.ExchangeAuthDetails(), + ) + crypto_currencies = [ + profile_data_module.CryptoCurrencyData( + name="BTC", + trading_pairs=["BTC/USDT"], + ) + ] + assert ( + profile_data_factory_module.infer_reference_market(exchange_account_details, crypto_currencies) + == "USDT" + ) + def test_returns_default_reference_market_when_internal_name_missing(self): exchange_account_details = exchange_account_details_module.ExchangeAccountDetails( exchange_details=profile_data_module.ExchangeData(), @@ -123,3 +192,30 @@ def test_returns_default_reference_market_when_internal_name_missing(self): profile_data_factory_module.infer_reference_market(exchange_account_details, []) == commons_constants.DEFAULT_REFERENCE_MARKET ) + + def test_infers_quote_from_first_trading_pair_when_portfolio_unit_missing(self): + exchange_account_details = exchange_account_details_module.ExchangeAccountDetails( + exchange_details=profile_data_module.ExchangeData(internal_name="coinrabbit"), + auth_details=exchange_data_module.ExchangeAuthDetails(), + ) + crypto_currencies = [ + profile_data_module.CryptoCurrencyData( + name="BTC@BTC", + trading_pairs=["BTC@BTC/USDT@ETH"], + ) + ] + assert ( + profile_data_factory_module.infer_reference_market(exchange_account_details, crypto_currencies) + == "USDT@ETH" + ) + + def test_uses_portfolio_unit_when_crypto_currencies_empty(self): + exchange_account_details = exchange_account_details_module.ExchangeAccountDetails( + exchange_details=profile_data_module.ExchangeData(internal_name="coinrabbit"), + auth_details=exchange_data_module.ExchangeAuthDetails(), + portfolio=exchange_account_details_module.ExchangeAccountPortfolio(unit="USDT@ETH"), + ) + assert ( + profile_data_factory_module.infer_reference_market(exchange_account_details, []) + == "USDT@ETH" + ) diff --git a/packages/flow/tests/logic/dsl/test_dsl_actions_util.py b/packages/flow/tests/logic/dsl/test_dsl_actions_util.py index 900e264081..9951e6a35f 100644 --- a/packages/flow/tests/logic/dsl/test_dsl_actions_util.py +++ b/packages/flow/tests/logic/dsl/test_dsl_actions_util.py @@ -7,8 +7,11 @@ import octobot_flow.entities import octobot_flow.logic.dsl.dsl_actions_util +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support + class _RecallableTestOperator( + dsl_test_keyword_support.TestKeywordMixin, octobot_commons.dsl_interpreter.Operator, octobot_commons.dsl_interpreter.ReCallableOperatorMixin, ): @@ -17,7 +20,10 @@ def get_name(cls) -> str: return "test_recallable_operator" -class _NonRecallableTestOperator(octobot_commons.dsl_interpreter.Operator): +class _NonRecallableTestOperator( + dsl_test_keyword_support.TestKeywordMixin, + octobot_commons.dsl_interpreter.Operator, +): @classmethod def get_name(cls) -> str: return "test_non_recallable_operator" diff --git a/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py b/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py index 1dd788fc65..c94e78d064 100644 --- a/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py +++ b/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py @@ -33,6 +33,9 @@ import octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder as simulated_portfolio_seeder_module import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module +import octobot_flow.repositories.exchange.orders_repository as orders_repository_module +import octobot_flow.repositories.exchange.portfolio_repository as portfolio_repository_module +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module from tests.logic.global_view.portfolio_test_util import patch_temporary_exchange_channel_ensure from tests.logic.global_view.portfolio_test_util import wire_portfolio_pipeline @@ -964,3 +967,70 @@ async def test_fetches_order_and_valuation_symbols_once(self): assert set(fetched_symbols) == {"BTC/USDT", "ETH/USDT"} +class TestRefreshExchangeAccountGlobalViewShapedChannels: + @pytest.mark.asyncio + async def test_cold_manager_real_repositories_create_channels_before_portfolio_fetch(self): + balance_content = _portfolio_content() + exchange_manager = mock.Mock() + exchange_manager.id = "global-view-cold-exchange" + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline(exchange_manager, {}) + balance_updater = mock.Mock() + balance_updater.fetch_portfolio = mock.AsyncMock(return_value={ + "USDT": {"total": 1000.0, "free": 1000.0}, + "BTC": {"total": 0.1, "free": 0.1}, + }) + tickers_updater = mock.Mock() + tickers_updater.fetch_all_tickers = mock.AsyncMock(return_value={}) + channels_registered = {"value": False} + + async def register_channels_stub(_exchange_manager): + channels_registered["value"] = True + + with ( + mock.patch.object( + portfolio_repository_module.channel_producer_ensure_module, + "exchange_has_registered_channels", + side_effect=lambda _exchange_manager: channels_registered["value"], + ), + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(side_effect=register_channels_stub), + ) as create_exchange_channels_mock, + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ), + mock.patch.object( + portfolio_repository_module.PortfolioRepository, + "get_channel_updater", + return_value=balance_updater, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "get_channel_updater", + return_value=tickers_updater, + ), + mock.patch.object( + trading_api, + "get_portfolio", + return_value=balance_content, + ), + mock.patch.object( + personal_data, + "refresh_portfolio_valuation", + mock.Mock(), + ), + ): + refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + fetch_open_orders=False, + ) + + create_exchange_channels_mock.assert_awaited_once_with(exchange_manager) + balance_updater.fetch_portfolio.assert_awaited_once() + assert refresh_result.assets + diff --git a/packages/flow/tests/logic/global_view/test_global_view_account_job.py b/packages/flow/tests/logic/global_view/test_global_view_account_job.py index e23c4fcfab..7a551d0859 100644 --- a/packages/flow/tests/logic/global_view/test_global_view_account_job.py +++ b/packages/flow/tests/logic/global_view/test_global_view_account_job.py @@ -19,7 +19,6 @@ import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module import octobot_sync.constants as sync_constants -from tests.logic.global_view.portfolio_test_util import patch_temporary_exchange_channel_ensure from tests.logic.global_view.portfolio_test_util import wire_portfolio_pipeline from tests.logic.global_view.portfolio_test_util import wire_repository_factory @@ -134,11 +133,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "exchange_manager_from_exchange_data", fake_exchange_manager, ), - patch_temporary_exchange_channel_ensure() as ( - ensure_ticker_channel_mock, - ensure_balance_channel_mock, - ensure_orders_channel_mock, - ), mock.patch.object( personal_data, "refresh_portfolio_valuation", @@ -171,9 +165,6 @@ async def fake_exchange_manager(*_args, **_kwargs): context, ).run() - ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) - ensure_balance_channel_mock.assert_awaited_once_with(exchange_manager) - ensure_orders_channel_mock.assert_not_awaited() portfolio_repository.fetch_and_apply_portfolio.assert_awaited_once() orders_repository.fetch_open_orders.assert_not_called() persist_mock.assert_called_once() @@ -214,11 +205,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "exchange_manager_from_exchange_data", fake_exchange_manager, ), - patch_temporary_exchange_channel_ensure() as ( - ensure_ticker_channel_mock, - ensure_balance_channel_mock, - ensure_orders_channel_mock, - ), mock.patch.object( personal_data, "refresh_portfolio_valuation", @@ -252,9 +238,6 @@ async def fake_exchange_manager(*_args, **_kwargs): ).run() orders_repository.fetch_open_orders.assert_awaited_once_with(["BTC/USDT", "ETH/USDT"]) - ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) - ensure_balance_channel_mock.assert_awaited_once_with(exchange_manager) - ensure_orders_channel_mock.assert_awaited_once_with(exchange_manager) assert persist_mock.call_args.kwargs["persist_open_orders"] is False assert refresh_result.changed_order_ids == {"gone-order-1"} @@ -297,11 +280,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "exchange_manager_from_exchange_data", fake_exchange_manager, ), - patch_temporary_exchange_channel_ensure() as ( - ensure_ticker_channel_mock, - ensure_balance_channel_mock, - ensure_orders_channel_mock, - ), mock.patch.object( personal_data, "refresh_portfolio_valuation", @@ -335,9 +313,6 @@ async def fake_exchange_manager(*_args, **_kwargs): ).run() orders_repository.fetch_open_orders.assert_awaited_once_with(["BTC/USDT", "ETH/USDT"]) - ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) - ensure_balance_channel_mock.assert_awaited_once_with(exchange_manager) - ensure_orders_channel_mock.assert_awaited_once_with(exchange_manager) assert persist_mock.call_args.kwargs["persist_open_orders"] is True assert refresh_result.changed_order_ids == {"gone-order-1"} @@ -378,11 +353,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "exchange_manager_from_exchange_data", fake_exchange_manager, ), - patch_temporary_exchange_channel_ensure() as ( - ensure_ticker_channel_mock, - ensure_balance_channel_mock, - ensure_orders_channel_mock, - ), mock.patch.object( personal_data, "refresh_portfolio_valuation", @@ -432,9 +402,6 @@ async def fake_exchange_manager(*_args, **_kwargs): context, ).run() - ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) - ensure_balance_channel_mock.assert_not_awaited() - ensure_orders_channel_mock.assert_not_awaited() create_factory_mock.assert_not_called() assert persist_mock.call_args.kwargs["persist_open_orders"] is True assert refresh_result.changed_order_ids == {"filled-order"} diff --git a/packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py b/packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py index 5f6d9c3fd4..da6421a382 100644 --- a/packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py +++ b/packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py @@ -9,6 +9,9 @@ import octobot_commons.constants as commons_constants import octobot_flow.constants as flow_constants import octobot_flow.logic.portfolio_history.daily_price_cache_updater as daily_price_cache_updater_module +import octobot_flow.logic.portfolio_history.trade_fetch_cursors as trade_fetch_cursors + +TICKER_WISE_REFERENCE_SYMBOL = "BTC@BTC/USDT@ETH" def _empty_daily_prices(): @@ -63,6 +66,16 @@ def mock_empty_historical_ohlcv(): yield +class TestParseBareBaseQuote: + def test_spot_symbol_returns_bare_base_and_quote(self): + assert daily_price_cache_updater_module._parse_bare_base_quote("BTC/USDT") == ("BTC", "USDT") + + def test_ticker_wise_symbol_returns_bare_base_and_quote(self): + assert daily_price_cache_updater_module._parse_bare_base_quote( + TICKER_WISE_REFERENCE_SYMBOL, + ) == ("BTC", "USDT") + + class TestComputeFetchTimeRangeMs: def test_incremental_uses_since_ms_as_start(self): since_ms = 1_700_000_000_000 @@ -179,6 +192,24 @@ async def test_fetches_and_merges_closes(self, tmp_path): assert result[trading_enums.DailyPricesCacheKeys.SOURCES]["BTC"] == "BTC/USDT" assert "since" not in exchange_manager.exchange.get_symbol_prices.call_args[1] + @pytest.mark.asyncio + async def test_ticker_wise_reference_symbol_keys_sources_by_qualified_base(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + day_one = _utc_day_start(1) + exchange_manager.exchange.get_symbol_prices.return_value = [ + _sample_candle(day_one, 40500), + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, [TICKER_WISE_REFERENCE_SYMBOL], data_root, + ) + + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result[trading_enums.DailyPricesCacheKeys.SOURCES]["BTC@BTC"] == "BTC/USDT" + assert trade_fetch_cursors.resolve_daily_cache_symbol( + result, TICKER_WISE_REFERENCE_SYMBOL, + ) == "BTC/USDT" + @pytest.mark.asyncio async def test_empty_cache_fetches_without_since(self, tmp_path): exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) @@ -276,7 +307,8 @@ async def test_stale_cache_with_long_history_uses_newest_not_oldest(self, tmp_pa assert exchange_manager.exchange.get_symbol_prices.call_args[1]["since"] != 1000 * 1000 @pytest.mark.asyncio - async def test_skips_usd_like_stablecoin_symbol(self, tmp_path): + async def test_skips_when_bare_base_is_usd_like(self, tmp_path): + """Skip daily fetch when the bare base asset is USD-like, not when only the quote is.""" exchange_manager = _exchange_manager_with_symbols(["USDC/USDT"]) data_root = str(tmp_path) await daily_price_cache_updater_module.update_daily_prices( diff --git a/packages/flow/tests/logic/portfolio_history/test_trade_fetch_cursors.py b/packages/flow/tests/logic/portfolio_history/test_trade_fetch_cursors.py index 58f704de88..74250e2009 100644 --- a/packages/flow/tests/logic/portfolio_history/test_trade_fetch_cursors.py +++ b/packages/flow/tests/logic/portfolio_history/test_trade_fetch_cursors.py @@ -1,339 +1,41 @@ -import datetime - import octobot_commons.constants as commons_constants -import octobot_protocol.models as protocol_models import octobot_trading.enums as trading_enums +import octobot_trading.exchange_data.prices.daily_prices_cache_types as daily_prices_cache_types import octobot_flow.constants as flow_constants -import octobot_flow.logic.portfolio_history.trade_fetch_cursors as trade_fetch_cursors_module - -_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) - - -def _empty_daily_prices(): - return { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - - -def _make_trade(symbol: str) -> protocol_models.Trade: - return protocol_models.Trade( - id=f"trade-{symbol}", - trade_id=f"trade-{symbol}", - type=protocol_models.OrderType.LIMIT, - symbol=symbol, - side=protocol_models.Side.BUY, - quantity=1.0, - price=1.0, - status=protocol_models.OrderStatus.FILLED, - executed_at=_TEST_TIMESTAMP, - ) +import octobot_flow.logic.portfolio_history.trade_fetch_cursors as trade_fetch_cursors -def _make_account_trading(symbols: list[str] | None) -> protocol_models.AccountTrading: - if symbols is None: - return protocol_models.AccountTrading( - updated_at=_TEST_TIMESTAMP, - trades=None, - ) - return protocol_models.AccountTrading( - updated_at=_TEST_TIMESTAMP, - trades=[_make_trade(symbol) for symbol in symbols], - ) - +TICKER_WISE_TRADE_SYMBOL = "BTC@BTC/USDT@ETH" -class TestSymbolsWithPersistedTrades: - def test_returns_empty_when_account_trading_is_none(self): - assert trade_fetch_cursors_module.symbols_with_persisted_trades(None) == set() - def test_returns_empty_when_trades_is_none(self): - account_trading = _make_account_trading(None) - assert trade_fetch_cursors_module.symbols_with_persisted_trades(account_trading) == set() +def _daily_prices_with_sticky_base() -> daily_prices_cache_types.DailyPricesCache: + daily_prices: daily_prices_cache_types.DailyPricesCache = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: { + "BTC/USDT": {1_700_000_000: 50000.0}, + }, + trading_enums.DailyPricesCacheKeys.SOURCES: { + "BTC@BTC": "BTC/USDT", + }, + } + return daily_prices - def test_returns_single_symbol(self): - account_trading = _make_account_trading(["BTC/USDT"]) - assert trade_fetch_cursors_module.symbols_with_persisted_trades(account_trading) == {"BTC/USDT"} - def test_returns_multiple_symbols(self): - account_trading = _make_account_trading(["BTC/USDT", "ETH/USDT"]) - assert trade_fetch_cursors_module.symbols_with_persisted_trades(account_trading) == { - "BTC/USDT", - "ETH/USDT", - } +class TestResolveDailyCacheSymbolTickerWise: + def test_uses_qualified_base_for_sticky_lookup(self): + daily_prices = _daily_prices_with_sticky_base() + resolved = trade_fetch_cursors.resolve_daily_cache_symbol(daily_prices, TICKER_WISE_TRADE_SYMBOL) + assert resolved == "BTC/USDT" - def test_ignores_trades_without_symbol(self): - account_trading = protocol_models.AccountTrading( - updated_at=_TEST_TIMESTAMP, - trades=[ - _make_trade("BTC/USDT"), - protocol_models.Trade( - id="trade-missing-symbol", - trade_id="trade-missing-symbol", - type=protocol_models.OrderType.LIMIT, - symbol="", - side=protocol_models.Side.BUY, - quantity=1.0, - price=1.0, - status=protocol_models.OrderStatus.FILLED, - executed_at=_TEST_TIMESTAMP, - ), - ], - ) - assert trade_fetch_cursors_module.symbols_with_persisted_trades(account_trading) == {"BTC/USDT"} - -def _expected_since_ms(day_timestamp: int) -> int: - return int( - ( - day_timestamp - - flow_constants.PORTFOLIO_HISTORY_TRADE_FETCH_SINCE_LOOKBACK_DAYS +class TestComputeTradeFetchSinceMsTickerWise: + def test_returns_since_ms_when_sticky_resolves(self): + daily_prices = _daily_prices_with_sticky_base() + since_ms = trade_fetch_cursors.compute_trade_fetch_since_ms(daily_prices, TICKER_WISE_TRADE_SYMBOL) + assert since_ms is not None + lookback_ms = int( + flow_constants.PORTFOLIO_HISTORY_TRADE_FETCH_SINCE_LOOKBACK_DAYS * commons_constants.DAYS_TO_SECONDS - ) - * 1000 - ) - - -class TestGetGlobalLatestDailyPriceTimestamp: - def test_returns_none_for_empty_cache(self): - assert trade_fetch_cursors_module.get_global_latest_daily_price_timestamp( - _empty_daily_prices(), - ) is None - - def test_returns_max_newest_timestamp_across_symbols(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: { - "BTC/USDT": {"1000": 42000.0, "2000": 43000.0}, - "ETH/USDT": {"1500": 2500.0, "3000": 2600.0}, - }, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - assert trade_fetch_cursors_module.get_global_latest_daily_price_timestamp( - daily_prices, - ) == 3000.0 - - -class TestUsesGlobalDailyPriceCursor: - def test_returns_true_for_usd_like_pair_with_global_cache(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"2000": 43000.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - assert trade_fetch_cursors_module.uses_global_daily_price_cursor( - daily_prices, - "USDC/USDT", - ) - - def test_returns_false_when_direct_cache_exists(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"USDC/USDT": {"2000": 1.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - assert not trade_fetch_cursors_module.uses_global_daily_price_cursor( - daily_prices, - "USDC/USDT", - ) - - def test_returns_false_for_usdc_eur_with_global_cache(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"2000": 43000.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - assert not trade_fetch_cursors_module.uses_global_daily_price_cursor( - daily_prices, - "USDC/EUR", - ) - - def test_returns_false_for_usd_like_pair_with_empty_cache(self): - assert not trade_fetch_cursors_module.uses_global_daily_price_cursor( - _empty_daily_prices(), - "USDC/USD", - ) - - -class TestResolveDailyCacheSymbol: - def test_returns_trade_symbol_when_direct_cache_hit(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"1700000000": 42000.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - assert trade_fetch_cursors_module.resolve_daily_cache_symbol( - daily_prices, - "BTC/USDT", - ) == "BTC/USDT" - - def test_returns_sticky_source_when_trade_symbol_missing(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"KNC/USDC": {"1700000000": 1.2}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {"KNC": "KNC/USDC"}, - } - assert trade_fetch_cursors_module.resolve_daily_cache_symbol( - daily_prices, - "KNC/USD", - ) == "KNC/USDC" - - def test_returns_none_when_no_cache_entry(self): - daily_prices = _empty_daily_prices() - assert trade_fetch_cursors_module.resolve_daily_cache_symbol( - daily_prices, - "BTC/USDT", - ) is None - - -class TestComputeTradeFetchSinceMs: - def test_returns_none_when_no_candle(self): - daily_prices = _empty_daily_prices() - assert trade_fetch_cursors_module.compute_trade_fetch_since_ms( - daily_prices, - "BTC/USDT", - ) is None - - def test_uses_newest_minus_two_days(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"1000": 42000.0, "2000": 43000.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - since_ms = trade_fetch_cursors_module.compute_trade_fetch_since_ms( - daily_prices, - "BTC/USDT", - ) - expected_since_ms = int( - ( - 2000 - - flow_constants.PORTFOLIO_HISTORY_TRADE_FETCH_SINCE_LOOKBACK_DAYS - * commons_constants.DAYS_TO_SECONDS - ) - * 1000 - ) - assert since_ms == expected_since_ms - - def test_uses_resolved_cache_symbol(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"KNC/USDC": {"3000": 1.1}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {"KNC": "KNC/USDC"}, - } - since_ms = trade_fetch_cursors_module.compute_trade_fetch_since_ms( - daily_prices, - "KNC/USD", - ) - expected_since_ms = int( - ( - 3000 - - flow_constants.PORTFOLIO_HISTORY_TRADE_FETCH_SINCE_LOOKBACK_DAYS - * commons_constants.DAYS_TO_SECONDS - ) * 1000 ) - assert since_ms == expected_since_ms - - def test_uses_global_cache_for_usd_like_pair(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"4000": 42000.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - since_ms = trade_fetch_cursors_module.compute_trade_fetch_since_ms( - daily_prices, - "USDC/USDT", - ) - assert since_ms == _expected_since_ms(4000) - - def test_returns_none_for_usd_like_pair_with_empty_cache(self): - assert trade_fetch_cursors_module.compute_trade_fetch_since_ms( - _empty_daily_prices(), - "USDC/USDT", - ) is None - - def test_returns_none_for_usdc_eur_with_only_global_cache(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"4000": 42000.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - assert trade_fetch_cursors_module.compute_trade_fetch_since_ms( - daily_prices, - "USDC/EUR", - ) is None - - -class TestBuildSymbolSinceMs: - def test_omits_symbol_without_persisted_trades(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"2000": 43000.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - assert trade_fetch_cursors_module.build_symbol_since_ms( - ["BTC/USDT"], - _make_account_trading([]), - daily_prices, - ) == {} - - def test_includes_symbol_with_persisted_trades_and_candle(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"2000": 43000.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - symbol_since_ms = trade_fetch_cursors_module.build_symbol_since_ms( - ["BTC/USDT"], - _make_account_trading(["BTC/USDT"]), - daily_prices, - ) - expected_since_ms = int( - ( - 2000 - - flow_constants.PORTFOLIO_HISTORY_TRADE_FETCH_SINCE_LOOKBACK_DAYS - * commons_constants.DAYS_TO_SECONDS - ) - * 1000 - ) - assert symbol_since_ms == {"BTC/USDT": expected_since_ms} - - def test_omits_symbol_with_persisted_trades_but_no_candle(self): - assert trade_fetch_cursors_module.build_symbol_since_ms( - ["BTC/USDT"], - _make_account_trading(["BTC/USDT"]), - _empty_daily_prices(), - ) == {} - - def test_only_includes_discovered_symbols(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: { - "BTC/USDT": {"2000": 43000.0}, - "ETH/USDT": {"2000": 2500.0}, - }, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - symbol_since_ms = trade_fetch_cursors_module.build_symbol_since_ms( - ["BTC/USDT"], - _make_account_trading(["BTC/USDT", "ETH/USDT"]), - daily_prices, - ) - assert set(symbol_since_ms) == {"BTC/USDT"} - - def test_returns_empty_map_when_account_trading_is_none(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"2000": 43000.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - assert trade_fetch_cursors_module.build_symbol_since_ms( - ["BTC/USDT"], - None, - daily_prices, - ) == {} - - def test_includes_usd_like_pair_using_global_cache(self): - daily_prices = { - trading_enums.DailyPricesCacheKeys.SYMBOLS: {"ETH/USDT": {"5000": 2500.0}}, - trading_enums.DailyPricesCacheKeys.SOURCES: {}, - } - symbol_since_ms = trade_fetch_cursors_module.build_symbol_since_ms( - ["USDC/USD"], - _make_account_trading(["USDC/USD"]), - daily_prices, - ) - assert symbol_since_ms == {"USDC/USD": _expected_since_ms(5000)} - - def test_omits_usd_like_pair_when_cache_is_empty(self): - assert trade_fetch_cursors_module.build_symbol_since_ms( - ["USDC/USDT"], - _make_account_trading(["USDC/USDT"]), - _empty_daily_prices(), - ) == {} + assert since_ms <= 1_700_000_000_000 - lookback_ms + 1000 diff --git a/packages/flow/tests/logic/portfolio_history/test_trade_symbols_discovery.py b/packages/flow/tests/logic/portfolio_history/test_trade_symbols_discovery.py index df250262ee..e71db78550 100644 --- a/packages/flow/tests/logic/portfolio_history/test_trade_symbols_discovery.py +++ b/packages/flow/tests/logic/portfolio_history/test_trade_symbols_discovery.py @@ -251,6 +251,42 @@ def test_skips_usd_like_and_reference_market_assets(self): ) assert result == ["SOL/USDC"] + def test_skips_reference_market_when_tickers_match_qualified_key(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets(set()) + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ) as get_associated_symbol_mock: + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=["SOL/USDC"], + account=_make_account({"USDT": 402.0}), + account_trading=None, + fresh_transactions=[], + reference_market="USDT@ETH", + ) + get_associated_symbol_mock.assert_not_called() + assert result == ["SOL/USDC"] + + def test_skips_usd_like_network_qualified_holding(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets(set()) + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ) as get_associated_symbol_mock: + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({"USDT@ETH": 402.0}), + account_trading=None, + fresh_transactions=[], + reference_market="BTC", + ) + get_associated_symbol_mock.assert_not_called() + assert result == [] + def test_excludes_pair_when_quote_not_held(self): exchange_manager, get_associated_symbol = _exchange_manager_with_markets({"ALGO"}) with mock.patch.object( diff --git a/packages/flow/tests/parsers/test_actions_dag_parser.py b/packages/flow/tests/parsers/test_actions_dag_parser.py index 4de2c13c54..09faa1adae 100644 --- a/packages/flow/tests/parsers/test_actions_dag_parser.py +++ b/packages/flow/tests/parsers/test_actions_dag_parser.py @@ -198,3 +198,111 @@ def test_cancel_action_details(self): assert action_details.dsl_script.startswith("cancel_order(") assert "BTC/USDC" in action_details.dsl_script + +class TestParseOrderSymbol: + _invalid_order_symbol = "invalid:symbol" + + def test_get_reference_market_raises_invalid_automation_action_error(self): + params = actions_dag_parser.ActionsDAGParserParams.from_dict({ + "ORDER_SYMBOL": self._invalid_order_symbol, + }) + with pytest.raises(octobot_flow.errors.InvalidAutomationActionError) as raised: + params.get_reference_market() + assert self._invalid_order_symbol in str(raised.value) + assert isinstance(raised.value, octobot_flow.errors.ConfigurationError) + assert isinstance(raised.value.__cause__, (AttributeError, ValueError)) + + +class TestActionsDAGParserParseInvalidOrderSymbol: + _invalid_order_symbol = "invalid:symbol" + + def _trade_params_with_invalid_symbol(self) -> dict: + return { + "ACTIONS": ["trade"], + "EXCHANGE_FROM": "binanceus", + "ORDER_SYMBOL": self._invalid_order_symbol, + "ORDER_AMOUNT": 1, + "ORDER_TYPE": "market", + "ORDER_SIDE": "BUY", + "SIMULATED_PORTFOLIO": {"BTC": 1}, + } + + def test_parse_raises_invalid_automation_action_error(self): + with pytest.raises(octobot_flow.errors.InvalidAutomationActionError) as raised: + actions_dag_parser.ActionsDAGParser( + self._trade_params_with_invalid_symbol() + ).parse() + assert self._invalid_order_symbol in str(raised.value) + + +class TestCreateGenericActionUnknownAction: + def test_unknown_action_raises_invalid_automation_action_error(self): + unknown_action = "not_a_real_action" + with pytest.raises(octobot_flow.errors.InvalidAutomationActionError) as raised: + actions_dag_parser.ActionsDAGParser({"ACTIONS": [unknown_action]}).parse() + assert unknown_action in str(raised.value) + + +class TestParseLoopUntilOrderClosed: + _exchange = "binanceus" + + def _loop_until_order_closed_params(self, **overrides) -> dict: + params = { + "ACTIONS": [ + actions_dag_parser.ActionType.TRADE.value, + actions_dag_parser.ActionType.LOOP_UNTIL_ORDER_CLOSED.value, + ], + "EXCHANGE_FROM": self._exchange, + "ORDER_SYMBOL": "ETH/BTC", + "ORDER_AMOUNT": 1, + "ORDER_TYPE": "market", + "ORDER_SIDE": "BUY", + "SIMULATED_PORTFOLIO": {"BTC": 1}, + "ORDER_EXCHANGE_ID": "order-123", + "LOOP_INTERVAL": 50, + "LOOP_TIMEOUT": 100, + "LOOP_MAX_ATTEMPTS": 5, + } + params.update(overrides) + return params + + def _loop_action_dsl(self, params: dict) -> str: + actions_dag = actions_dag_parser.ActionsDAGParser(params).parse() + loop_action = next( + action for action in actions_dag.actions if action.id == "action_loop_until_order_closed_2" + ) + assert isinstance(loop_action, octobot_flow.entities.DSLScriptActionDetails) + return loop_action.dsl_script + + def test_emits_max_retry_interval_when_loop_interval_max_set(self): + dsl_script = self._loop_action_dsl( + self._loop_until_order_closed_params(LOOP_INTERVAL_MAX=70) + ) + assert "max_retry_interval=70" in dsl_script + assert "return_remaining_time=True" in dsl_script + assert ", 50.0, max_retry_interval=70.0," in dsl_script + + def test_emits_max_retry_interval_when_loop_interval_max_is_string(self): + # JSON / task templates pass numeric params as strings; lexicographic "30" > "5" is False. + dsl_script = self._loop_action_dsl( + self._loop_until_order_closed_params(LOOP_INTERVAL="5", LOOP_INTERVAL_MAX="30") + ) + assert ", 5.0, max_retry_interval=30.0," in dsl_script + + def test_emits_max_retry_interval_when_template_style_string_intervals(self): + dsl_script = self._loop_action_dsl( + self._loop_until_order_closed_params(LOOP_INTERVAL="30", LOOP_INTERVAL_MAX="120") + ) + assert ", 30.0, max_retry_interval=120.0," in dsl_script + + def test_omits_max_retry_interval_without_loop_interval_max(self): + dsl_script = self._loop_action_dsl(self._loop_until_order_closed_params()) + assert "max_retry_interval" not in dsl_script + assert ", 50.0, timeout=" in dsl_script + + def test_loop_interval_max_less_than_loop_interval_raises(self): + with pytest.raises(octobot_flow.errors.InvalidAutomationActionError, match="LOOP_INTERVAL_MAX"): + actions_dag_parser.ActionsDAGParserParams.from_dict( + self._loop_until_order_closed_params(LOOP_INTERVAL=50, LOOP_INTERVAL_MAX=10) + ) + diff --git a/packages/flow/tests/repositories/exchange/test_channel_producer_ensure.py b/packages/flow/tests/repositories/exchange/test_channel_producer_ensure.py new file mode 100644 index 0000000000..81ae22da4b --- /dev/null +++ b/packages/flow/tests/repositories/exchange/test_channel_producer_ensure.py @@ -0,0 +1,166 @@ +# Drakkar-Software OctoBot-Flow + +import mock +import pytest + +import octobot_trading.personal_data as trading_personal_data + +import octobot_flow.repositories.exchange.channel_producer_ensure as channel_producer_ensure_module + +pytestmark = pytest.mark.asyncio + + +class TestEnsureTemporaryChannelProducer: + async def test_creates_channels_and_producer_when_no_channels(self): + exchange_manager = mock.Mock() + with ( + mock.patch.object( + channel_producer_ensure_module, + "exchange_has_registered_channels", + return_value=False, + ), + mock.patch.object( + channel_producer_ensure_module, + "channel_has_producer", + return_value=False, + ), + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(), + ) as create_exchange_channels_mock, + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ) as create_producers_mock, + ): + await channel_producer_ensure_module.ensure_temporary_channel_producer( + exchange_manager, + "Trades", + trading_personal_data.TradesUpdater, + ) + + create_exchange_channels_mock.assert_awaited_once_with(exchange_manager) + create_producers_mock.assert_awaited_once_with( + exchange_manager, + [trading_personal_data.TradesUpdater], + start_producers=False, + ) + + async def test_no_op_when_producer_already_registered(self): + exchange_manager = mock.Mock() + with ( + mock.patch.object( + channel_producer_ensure_module, + "exchange_has_registered_channels", + return_value=True, + ), + mock.patch.object( + channel_producer_ensure_module, + "channel_has_producer", + return_value=True, + ), + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(), + ) as create_exchange_channels_mock, + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ) as create_producers_mock, + ): + await channel_producer_ensure_module.ensure_temporary_channel_producer( + exchange_manager, + "Trades", + trading_personal_data.TradesUpdater, + ) + + create_exchange_channels_mock.assert_not_called() + create_producers_mock.assert_not_called() + + async def test_registers_producer_only_when_channels_exist(self): + exchange_manager = mock.Mock() + with ( + mock.patch.object( + channel_producer_ensure_module, + "exchange_has_registered_channels", + return_value=True, + ), + mock.patch.object( + channel_producer_ensure_module, + "channel_has_producer", + return_value=False, + ), + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(), + ) as create_exchange_channels_mock, + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ) as create_producers_mock, + ): + await channel_producer_ensure_module.ensure_temporary_channel_producer( + exchange_manager, + "Trades", + trading_personal_data.TradesUpdater, + ) + + create_exchange_channels_mock.assert_not_called() + create_producers_mock.assert_awaited_once_with( + exchange_manager, + [trading_personal_data.TradesUpdater], + start_producers=False, + ) + + async def test_create_all_channels_if_missing_false_only_adds_producer_when_channels_exist(self): + exchange_manager = mock.Mock() + with ( + mock.patch.object( + channel_producer_ensure_module, + "exchange_has_registered_channels", + return_value=True, + ), + mock.patch.object( + channel_producer_ensure_module, + "channel_has_producer", + return_value=False, + ), + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(), + ) as create_exchange_channels_mock, + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ) as create_producers_mock, + ): + await channel_producer_ensure_module.ensure_temporary_channel_producer( + exchange_manager, + "Orders", + trading_personal_data.OrdersUpdater, + create_all_channels_if_missing=False, + ) + + create_exchange_channels_mock.assert_not_called() + create_producers_mock.assert_awaited_once_with( + exchange_manager, + [trading_personal_data.OrdersUpdater], + start_producers=False, + ) + + +class TestChannelHasProducer: + async def test_returns_false_when_channel_missing(self): + exchange_manager = mock.Mock() + exchange_manager.id = "missing-channel-exchange" + with mock.patch.object( + channel_producer_ensure_module.exchange_channel_module, + "get_chan", + side_effect=KeyError("Channel Balance not found"), + ): + has_producer = channel_producer_ensure_module.channel_has_producer( + exchange_manager, + "Balance", + ) + + assert has_producer is False diff --git a/packages/flow/tests/repositories/exchange/test_exchange_repository_fetch_wiring.py b/packages/flow/tests/repositories/exchange/test_exchange_repository_fetch_wiring.py new file mode 100644 index 0000000000..cfcb2c18b7 --- /dev/null +++ b/packages/flow/tests/repositories/exchange/test_exchange_repository_fetch_wiring.py @@ -0,0 +1,216 @@ +# Drakkar-Software OctoBot-Flow + +import mock +import pytest + +import octobot_flow.entities +import octobot_flow.repositories.exchange.ohlcv_repository as ohlcv_repository_module +import octobot_flow.repositories.exchange.orders_repository as orders_repository_module +import octobot_flow.repositories.exchange.portfolio_repository as portfolio_repository_module +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module +import octobot_flow.repositories.exchange.trades_repository as trades_repository_module + +pytestmark = pytest.mark.asyncio + + +def _fetched_exchange_data(): + return octobot_flow.entities.FetchedExchangeData() + + +class TestFetchWiringDependencyShapedManager: + async def test_fetch_methods_no_op_ensure_when_producer_already_present(self): + exchange_manager = mock.Mock() + exchange_manager.id = "exchange-manager-1" + with ( + mock.patch.object( + trades_repository_module.channel_producer_ensure_module, + "exchange_has_registered_channels", + return_value=True, + ), + mock.patch.object( + trades_repository_module.channel_producer_ensure_module, + "channel_has_producer", + return_value=True, + ), + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(), + ) as create_exchange_channels_mock, + mock.patch.object( + trades_repository_module.TradesRepository, + "_get_trades_updater", + return_value=mock.AsyncMock(fetch_trades=mock.AsyncMock(return_value=[])), + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "get_channel_updater", + return_value=mock.AsyncMock(fetch_all_tickers=mock.AsyncMock(return_value={})), + ), + mock.patch.object( + orders_repository_module.OrdersRepository, + "get_channel_updater", + return_value=mock.AsyncMock(fetch_open_orders=mock.AsyncMock(return_value=[])), + ), + mock.patch.object( + portfolio_repository_module.PortfolioRepository, + "get_channel_updater", + return_value=mock.AsyncMock(fetch_portfolio=mock.AsyncMock(return_value={})), + ), + mock.patch.object( + ohlcv_repository_module.OhlcvRepository, + "get_channel_updater", + return_value=mock.AsyncMock(fetch_ohlcv=mock.AsyncMock(return_value=[])), + ), + mock.patch.object( + portfolio_repository_module.trading_api, + "get_portfolio", + return_value={}, + ), + mock.patch.object( + portfolio_repository_module.personal_data, + "filter_empty_values", + side_effect=lambda portfolio: portfolio, + ), + mock.patch.object( + portfolio_repository_module.personal_data, + "from_raw_to_formatted_portfolio", + return_value={}, + ), + mock.patch.object( + portfolio_repository_module.personal_data, + "parse_decimal_portfolio", + return_value={}, + ), + ): + trades_repo = trades_repository_module.TradesRepository( + exchange_manager, [], _fetched_exchange_data(), + ) + tickers_repo = tickers_repository_module.TickersRepository( + exchange_manager, [], _fetched_exchange_data(), + ) + orders_repo = orders_repository_module.OrdersRepository( + exchange_manager, [], _fetched_exchange_data(), + ) + portfolio_repo = portfolio_repository_module.PortfolioRepository( + exchange_manager, [], _fetched_exchange_data(), + ) + ohlcv_repo = ohlcv_repository_module.OhlcvRepository( + exchange_manager, [], _fetched_exchange_data(), + ) + exchange_manager.exchange_personal_data = mock.Mock() + exchange_manager.exchange_personal_data.handle_portfolio_update = mock.AsyncMock(return_value=True) + + await trades_repo.fetch_trades(["BTC/USDT"]) + await tickers_repo.fetch_tickers(["BTC/USDT"]) + await orders_repo.fetch_open_orders(["BTC/USDT"]) + await portfolio_repo.fetch_portfolio() + await ohlcv_repo.fetch_ohlcv("BTC/USDT", "1h", 10, {}) + + create_exchange_channels_mock.assert_not_called() + + +class TestFetchWiringDagShapedManager: + async def test_fetch_trades_registers_producer_when_channels_exist_without_one(self): + exchange_manager = mock.Mock() + exchange_manager.id = "exchange-manager-2" + trades_updater = mock.AsyncMock(fetch_trades=mock.AsyncMock(return_value=[])) + with ( + mock.patch.object( + trades_repository_module.channel_producer_ensure_module, + "exchange_has_registered_channels", + return_value=True, + ), + mock.patch.object( + trades_repository_module.channel_producer_ensure_module, + "channel_has_producer", + side_effect=[False, True], + ), + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ) as create_producers_mock, + mock.patch.object( + trades_repository_module.TradesRepository, + "_get_trades_updater", + return_value=trades_updater, + ), + ): + trades_repo = trades_repository_module.TradesRepository( + exchange_manager, [], _fetched_exchange_data(), + ) + await trades_repo.fetch_trades(["BTC/USDT"]) + + create_producers_mock.assert_awaited_once() + trades_updater.fetch_trades.assert_awaited_once_with(["BTC/USDT"]) + + +class TestFetchWiringGlobalViewShapedManager: + async def test_portfolio_and_orders_fetch_create_channels_on_cold_manager(self): + exchange_manager = mock.Mock() + exchange_manager.id = "global-view-cold-exchange" + channels_registered = {"value": False} + + def exchange_has_registered_channels(_exchange_manager): + return channels_registered["value"] + + async def register_channels_stub(_exchange_manager): + channels_registered["value"] = True + + balance_updater = mock.Mock() + balance_updater.fetch_portfolio = mock.AsyncMock(return_value={"USDT": {"total": 1.0, "free": 1.0}}) + orders_updater = mock.Mock() + orders_updater.fetch_open_orders = mock.AsyncMock(return_value=[]) + with ( + mock.patch.object( + portfolio_repository_module.channel_producer_ensure_module, + "exchange_has_registered_channels", + side_effect=exchange_has_registered_channels, + ), + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(side_effect=register_channels_stub), + ) as create_exchange_channels_mock, + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ), + mock.patch.object( + portfolio_repository_module.PortfolioRepository, + "get_channel_updater", + return_value=balance_updater, + ), + mock.patch.object( + orders_repository_module.OrdersRepository, + "get_channel_updater", + return_value=orders_updater, + ), + mock.patch.object( + portfolio_repository_module.personal_data, + "filter_empty_values", + side_effect=lambda portfolio: portfolio, + ), + mock.patch.object( + portfolio_repository_module.personal_data, + "parse_decimal_portfolio", + return_value={}, + ), + mock.patch.object( + portfolio_repository_module.trading_api, + "get_portfolio", + return_value={}, + ), + ): + exchange_manager.exchange_personal_data = mock.Mock() + exchange_manager.exchange_personal_data.handle_portfolio_update = mock.AsyncMock(return_value=True) + portfolio_repo = portfolio_repository_module.PortfolioRepository( + exchange_manager, [], _fetched_exchange_data(), + ) + orders_repo = orders_repository_module.OrdersRepository( + exchange_manager, [], _fetched_exchange_data(), + ) + await portfolio_repo.fetch_and_apply_portfolio() + await orders_repo.fetch_open_orders(["BTC/USDT"]) + + create_exchange_channels_mock.assert_awaited_once_with(exchange_manager) + balance_updater.fetch_portfolio.assert_awaited_once() + orders_updater.fetch_open_orders.assert_awaited_once_with(["BTC/USDT"]) diff --git a/packages/flow/tests/repositories/exchange/test_ohlcv_repository.py b/packages/flow/tests/repositories/exchange/test_ohlcv_repository.py new file mode 100644 index 0000000000..244886b16f --- /dev/null +++ b/packages/flow/tests/repositories/exchange/test_ohlcv_repository.py @@ -0,0 +1,27 @@ +# Drakkar-Software OctoBot-Flow + +import mock +import pytest + +import octobot_trading.exchange_data as exchange_data_module + +import octobot_flow.repositories.exchange.ohlcv_repository as ohlcv_repository_module + +pytestmark = pytest.mark.asyncio + + +class TestOhlcvRepositoryEnsureTemporaryOhlcvChannel: + async def test_delegates_to_channel_producer_ensure(self): + exchange_manager = mock.Mock() + with mock.patch.object( + ohlcv_repository_module.channel_producer_ensure_module, + "ensure_temporary_channel_producer", + mock.AsyncMock(), + ) as ensure_mock: + await ohlcv_repository_module.OhlcvRepository.ensure_temporary_ohlcv_channel(exchange_manager) + + ensure_mock.assert_awaited_once_with( + exchange_manager, + ohlcv_repository_module.octobot_trading.constants.OHLCV_CHANNEL, + exchange_data_module.OHLCVUpdater, + ) diff --git a/packages/flow/tests/repositories/exchange/test_orders_repository.py b/packages/flow/tests/repositories/exchange/test_orders_repository.py index 97b3d9373c..4ae0e4b191 100644 --- a/packages/flow/tests/repositories/exchange/test_orders_repository.py +++ b/packages/flow/tests/repositories/exchange/test_orders_repository.py @@ -22,18 +22,19 @@ def _open_order_dict(exchange_id: str, symbol: str) -> dict: class TestOrdersRepositoryEnsureTemporaryOrdersChannel: - async def test_creates_orders_producer_only(self): + async def test_delegates_to_channel_producer_ensure(self): exchange_manager = mock.Mock() - with mock.patch( - "octobot_trading.exchanges.create_producers", + with mock.patch.object( + orders_repository_module.channel_producer_ensure_module, + "ensure_temporary_channel_producer", mock.AsyncMock(), - ) as create_producers_mock: + ) as ensure_mock: await orders_repository_module.OrdersRepository.ensure_temporary_orders_channel(exchange_manager) - create_producers_mock.assert_awaited_once_with( + ensure_mock.assert_awaited_once_with( exchange_manager, - [trading_personal_data.OrdersUpdater], - start_producers=False, + trading_constants.ORDERS_CHANNEL, + trading_personal_data.OrdersUpdater, ) @@ -52,6 +53,11 @@ async def test_returns_parsed_orders_from_updater(self): fetched_exchange_data=octobot_flow.entities.FetchedExchangeData(), ) with ( + mock.patch.object( + orders_repository_module.OrdersRepository, + "ensure_temporary_orders_channel", + mock.AsyncMock(), + ), mock.patch.object( repository, "get_channel_updater", @@ -81,5 +87,4 @@ async def test_returns_empty_list_when_no_symbols(self): known_automations=[], fetched_exchange_data=octobot_flow.entities.FetchedExchangeData(), ) - open_orders = await repository.fetch_open_orders([]) - assert open_orders == [] + assert await repository.fetch_open_orders([]) == [] diff --git a/packages/flow/tests/repositories/exchange/test_portfolio_repository.py b/packages/flow/tests/repositories/exchange/test_portfolio_repository.py index d9467c9928..d3ea47fb9d 100644 --- a/packages/flow/tests/repositories/exchange/test_portfolio_repository.py +++ b/packages/flow/tests/repositories/exchange/test_portfolio_repository.py @@ -16,18 +16,19 @@ class TestPortfolioRepositoryEnsureTemporaryBalanceChannel: - async def test_creates_balance_producer_only(self): + async def test_delegates_to_channel_producer_ensure(self): exchange_manager = mock.Mock() - with mock.patch( - "octobot_trading.exchanges.create_producers", + with mock.patch.object( + portfolio_repository_module.channel_producer_ensure_module, + "ensure_temporary_channel_producer", mock.AsyncMock(), - ) as create_producers_mock: + ) as ensure_mock: await portfolio_repository_module.PortfolioRepository.ensure_temporary_balance_channel(exchange_manager) - create_producers_mock.assert_awaited_once_with( + ensure_mock.assert_awaited_once_with( exchange_manager, - [trading_personal_data.BalanceUpdater], - start_producers=False, + trading_constants.BALANCE_CHANNEL, + trading_personal_data.BalanceUpdater, ) @@ -55,6 +56,11 @@ async def test_applies_decimal_balance_to_portfolio_manager(self): fetched_exchange_data=octobot_flow.entities.FetchedExchangeData(), ) with ( + mock.patch.object( + portfolio_repository_module.PortfolioRepository, + "ensure_temporary_balance_channel", + mock.AsyncMock(), + ), mock.patch.object( repository, "get_channel_updater", diff --git a/packages/flow/tests/repositories/exchange/test_tickers_repository.py b/packages/flow/tests/repositories/exchange/test_tickers_repository.py index 57cb90ae69..4295482e07 100644 --- a/packages/flow/tests/repositories/exchange/test_tickers_repository.py +++ b/packages/flow/tests/repositories/exchange/test_tickers_repository.py @@ -4,6 +4,7 @@ import pytest import octobot_trading.exchange_data as exchange_data_module +import octobot_trading.constants as trading_constants import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module @@ -11,23 +12,17 @@ class TestTickersRepositoryEnsureTemporaryTickerChannel: - async def test_creates_channels_and_ticker_producer_only(self): + async def test_delegates_to_channel_producer_ensure(self): exchange_manager = mock.Mock() - with ( - mock.patch( - "octobot_trading.exchanges.create_exchange_channels", - mock.AsyncMock(), - ) as create_exchange_channels_mock, - mock.patch( - "octobot_trading.exchanges.create_producers", - mock.AsyncMock(), - ) as create_producers_mock, - ): + with mock.patch.object( + tickers_repository_module.channel_producer_ensure_module, + "ensure_temporary_channel_producer", + mock.AsyncMock(), + ) as ensure_mock: await tickers_repository_module.TickersRepository.ensure_temporary_ticker_channel(exchange_manager) - create_exchange_channels_mock.assert_awaited_once_with(exchange_manager) - create_producers_mock.assert_awaited_once_with( + ensure_mock.assert_awaited_once_with( exchange_manager, - [exchange_data_module.TickerUpdater], - start_producers=False, + trading_constants.TICKER_CHANNEL, + exchange_data_module.TickerUpdater, ) diff --git a/packages/flow/tests/repositories/exchange/test_trades_repository.py b/packages/flow/tests/repositories/exchange/test_trades_repository.py index 260a09e55a..067a75491a 100644 --- a/packages/flow/tests/repositories/exchange/test_trades_repository.py +++ b/packages/flow/tests/repositories/exchange/test_trades_repository.py @@ -6,6 +6,19 @@ import octobot_trading.personal_data as trading_personal_data +@pytest.fixture(autouse=True) +def _patch_trades_channel_ensure(request): + if request.cls is TestTradesRepositoryEnsureTemporaryTradesChannel: + yield + return + with mock.patch.object( + trades_repository_module.TradesRepository, + "ensure_temporary_trades_channel", + mock.AsyncMock(), + ): + yield + + def _make_repo(exchange_manager): fetched_data = mock.MagicMock() return trades_repository_module.TradesRepository(exchange_manager, [], fetched_data) @@ -299,25 +312,19 @@ async def test_bulk_fetch_logs_skipped_delisted_symbols(self, caplog): class TestTradesRepositoryEnsureTemporaryTradesChannel: @pytest.mark.asyncio - async def test_creates_channels_and_trades_producer_only(self): + async def test_delegates_to_channel_producer_ensure(self): exchange_manager = mock.Mock() - with ( - mock.patch( - "octobot_trading.exchanges.create_exchange_channels", - mock.AsyncMock(), - ) as create_exchange_channels_mock, - mock.patch( - "octobot_trading.exchanges.create_producers", - mock.AsyncMock(), - ) as create_producers_mock, - ): + with mock.patch.object( + trades_repository_module.channel_producer_ensure_module, + "ensure_temporary_channel_producer", + mock.AsyncMock(), + ) as ensure_mock: await trades_repository_module.TradesRepository.ensure_temporary_trades_channel(exchange_manager) - create_exchange_channels_mock.assert_awaited_once_with(exchange_manager) - create_producers_mock.assert_awaited_once_with( + ensure_mock.assert_awaited_once_with( exchange_manager, - [trading_personal_data.TradesUpdater], - start_producers=False, + trades_repository_module.trading_constants.TRADES_CHANNEL, + trading_personal_data.TradesUpdater, ) diff --git a/packages/node/AGENTS.md b/packages/node/AGENTS.md new file mode 100644 index 0000000000..35f7c18ba0 --- /dev/null +++ b/packages/node/AGENTS.md @@ -0,0 +1,63 @@ +# Agents: node + +## Role + +`octobot_node` is **durable orchestration** for OctoBot Node: FastAPI, DBOS scheduling, and protocol APIs. It runs work; **`octobot_flow`** executes automation logic statelessly inside DBOS steps. + +- **Service:** master and/or consumer roles; **SQLite** DBOS for single-node dev, CI, and cloud agents; **PostgreSQL** only for multi-node—do not assume shared Postgres in cloud unless the user configures it. +- **User actions (how clients drive the node):** Web UI (`node_api_interface`), sync, and other callers submit **`protocol_models.UserAction`**. `execute_user_action` journals the request and triggers **`user_action_workflow`**. Executors under `scheduler/user_actions/user_actions_executor/` implement accounts, account auth, automations (create/stop/restart/signal), strategies, historical data refresh, and related commands—users manage the node **without editing flow DAGs** directly. +- **Two user-action paths:** (1) **User-action workflow** — discrete API commands as their own DBOS workflow. (2) **In-flight automation** — DBOS messages on topic `"user_actions"` while an automation runs (overrides between DAG iterations). +- **Scheduled work:** automation iterations as child DBOS workflows (`execute_iteration` as a DBOS step); portfolio history, global view, and cleanup under `scheduler/workflows/`. +- **Bridge to flow:** `octobot_flow_client` runs `octobot_flow` jobs inside steps; optional `encrypted_task` wrapper—flow stays encryption-agnostic. +- **Not here:** DAG action bodies, ccxt plumbing, evaluator tentacles. Tentacle HTTP routes live in **`packages/tentacles`** but must delegate to node protocol and scheduler APIs. +- **OpenAPI:** Node **REST** spec is owned by tentacles (`node_api_interface` → `node_web_interface/openapi.json` — do not hand-edit); shared **wire** types live in **`packages/protocol/openapi.json`** (`npm run generate:all`). See [`../tentacles/AGENTS.md`](../tentacles/AGENTS.md#node-rest-openapi-node_api_interface--node_web_interface) and [`../protocol/AGENTS.md`](../protocol/AGENTS.md). + +## Owns + +- `packages/node/octobot_node/` +- `packages/node/tests/` +- Optional Rust crates under `packages/node/crates/` (clippy in CI) + +## Public surface + +- `octobot_node.protocol` — `user_actions`, accounts, automations, debug +- `octobot_node.scheduler.tasks` — workflow triggers +- `octobot_node.scheduler.workflows` — including `user_action_workflow` +- `octobot_node.scheduler.user_actions` — executors and factories +- `octobot_node.constants` — `SCHEDULER_APPLICATION_VERSION`, scheduler executor id + +## May depend on + +- `octobot_commons`, `octobot_trading`, `octobot_agents`, `octobot_flow`, protocol/sync as imported today + +## Do not + +- Add new user-facing operations only in tentacle routes—extend protocol, executor, and **node** tests for new action types +- Change `actions_dag` or `AutomationJob` semantics without matching **flow** changes +- Assume shared **PostgreSQL** multi-node DBOS in cloud unless configured (default dev/CI and cloud agents use **SQLite** DBOS locally) +- Construct `dbos.Queue(...)` or enqueue scheduler workflows via the `dbos.DBOS` class—use `octobot_node.enums.SchedulerQueues`, [`octobot_node/scheduler/queues.py`](octobot_node/scheduler/queues.py) after launch, and `SCHEDULER.INSTANCE.enqueue_workflow_async` (see `octobot_node/scheduler/tasks.py`) + +## Agent seed (demo wallet / Cloud QA) + +Only when driving the **live Node web UI** or debug view — not for routine `pytest` in this package. + +- Operator docs: [`tools/agent_seed/README.md`](../../tools/agent_seed/README.md) and skill **agent-seed** (`.cursor/skills/agent-seed/SKILL.md`). +- Fixtures: `python -m tools.agent_seed seed` or `bash .cursor/seed-agent.sh seed|bootstrap|start|--full|--clear` (after `.cursor/agent-seed.env`). +- After seed: login passphrase; full **`/app`** UI for QA; **`/app/debug`** to submit user actions and inspect automations. +- Runtime sandbox: `packages/node/octobot_node/agent_seed/` (`is_demo_agent_seed_user`, demo wallet action guards). + +## Tests + +- **cwd:** repo root +- **PYTHONPATH:** `PYTHONPATH=.:$PYTHONPATH` +- **Tentacles:** required (CI installs tentacles) +- **DBOS:** SQLite-backed scheduler tests are OK in Cursor Cloud (same as local dev) +- **Example:** `pytest packages/node/tests -n auto --dist loadfile` + +## Related human doc + +- `docs/content/developers/packages/node.md` + +## Last reviewed + +- 2026-09-22 diff --git a/packages/node/octobot_node/agent_seed/__init__.py b/packages/node/octobot_node/agent_seed/__init__.py new file mode 100644 index 0000000000..4aa6973d1e --- /dev/null +++ b/packages/node/octobot_node/agent_seed/__init__.py @@ -0,0 +1,4 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +"""Agent-seed demo wallet fixtures for local and Cloud agent QA (not production).""" diff --git a/packages/node/octobot_node/agent_seed/constants.py b/packages/node/octobot_node/agent_seed/constants.py new file mode 100644 index 0000000000..c6016a24a4 --- /dev/null +++ b/packages/node/octobot_node/agent_seed/constants.py @@ -0,0 +1,54 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# Demo-only agent seed fixture identifiers. Not for production wallets or real funds. +# Private key material lives only in tools/agent_seed/secrets.py (insecure by design). + +DEMO_AGENT_SEED_WALLET_DISPLAY_NAME = "demo" + +# Anvil / Hardhat account #1 (public test vector; matches tools.agent_seed.secrets). +DEMO_AGENT_SEED_WALLET_EVM_ADDRESS = "0x70997970c51812dc3a010c7d01b50e0d17dc79c8" + +# Starfish user_id for the demo agent-seed wallet (derived from demo insecure key at fixture design time). +DEMO_AGENT_SEED_USER_ID = "49f2d71b4bba99febdca5c0da6e4b60b" + +DEMO_AGENT_SEED_STRATEGY_VERSION = "1.0.0" +DEMO_AGENT_SEED_MARKER_VERSION = "1" + +DEMO_AGENT_SEED_EXCHANGE_CONFIG_ID = "agent-seed-kraken-sim-exchange-config" +DEMO_AGENT_SEED_ACCOUNT_GRID_ID = "agent-seed-kraken-sim-account-grid" +DEMO_AGENT_SEED_ACCOUNT_INDEX_IDLE_ID = "agent-seed-kraken-sim-account-index-idle" +DEMO_AGENT_SEED_STRATEGY_GRID_ID = "agent-seed-strategy-grid-btc-usdc" +DEMO_AGENT_SEED_STRATEGY_INDEX_ID = "agent-seed-strategy-index-btc-eth-sol" +# Stable agent-seed grid automation parent id (Node create-automation requires canonical lowercase UUID). +DEMO_AGENT_SEED_AUTOMATION_GRID_ID = "a0000000-0000-4000-8000-000000000001" +# Index automation (runs on the idle account) and index automation that bootstrap stops (completed state in the UI). +DEMO_AGENT_SEED_AUTOMATION_INDEX_ID = "a0000000-0000-4000-8000-000000000002" +DEMO_AGENT_SEED_AUTOMATION_COMPLETED_ID = "a0000000-0000-4000-8000-000000000003" + +DEMO_AGENT_SEED_GRID_SYMBOL = "BTC/USDC" +DEMO_AGENT_SEED_GRID_SPREAD = 2000.0 +DEMO_AGENT_SEED_GRID_INCREMENT = 500.0 +DEMO_AGENT_SEED_GRID_BUY_COUNT = 3 +DEMO_AGENT_SEED_GRID_SELL_COUNT = 3 + +DEMO_AGENT_SEED_INDEX_REBALANCE_TRIGGER_MIN_PERCENT = 10.0 + +DEMO_AGENT_SEED_ACCOUNT_GRID_USDC = 1000.0 +DEMO_AGENT_SEED_ACCOUNT_INDEX_IDLE_USDC = 500.0 +DEMO_AGENT_SEED_ACCOUNT_GRID_DISPLAY_NAME = "Seed kraken A" +DEMO_AGENT_SEED_ACCOUNT_INDEX_IDLE_DISPLAY_NAME = "Seed kraken B" + +DEMO_AGENT_SEED_EXCHANGE_INTERNAL_NAME = "kraken" + +DEMO_AGENT_SEED_GRID_AUTOMATION_DISPLAY_NAME = "Agent seed BTC/USDC grid" +DEMO_AGENT_SEED_INDEX_AUTOMATION_DISPLAY_NAME = "Agent seed BTC/ETH/SOL index" +DEMO_AGENT_SEED_COMPLETED_AUTOMATION_DISPLAY_NAME = "Agent seed stopped index" + +DEMO_AGENT_SEED_FORBIDDEN_ACTION_DETAIL = ( + "Demo agent-seed wallet cannot create live exchange accounts or automations" +) + +DEMO_AGENT_SEED_CLEAR_LOCKED_SQLITE_MESSAGE_PREFIX = ( + "Agent seed clear failed: stop the OctoBot node first — tasks.db is locked" +) diff --git a/packages/node/octobot_node/agent_seed/demo_wallet.py b/packages/node/octobot_node/agent_seed/demo_wallet.py new file mode 100644 index 0000000000..e84a601fa2 --- /dev/null +++ b/packages/node/octobot_node/agent_seed/demo_wallet.py @@ -0,0 +1,117 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# Demo-only agent seed wallet sandbox rules for the debug user-action API. +# Not for production; pairs with tools/agent_seed insecure dev credentials. + +import typing + +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot.community.authentication as community_authentication + +import octobot_node.agent_seed.constants as agent_seed_constants + + +class DemoAgentSeedUserActionForbiddenError(Exception): + """Raised when a demo agent-seed wallet attempts a disallowed user action.""" + + +def _normalize_user_id(user_id: str) -> str: + return user_id.strip().lower() + + +def _wallet_display_name_for_user_id(user_id: str) -> typing.Optional[str]: + try: + wallet = community_authentication.CommunityAuthentication.instance().get_wallet_by_user_id( + user_id, + ) + except Exception: + return None + address = wallet.address.lower() + return community_authentication.CommunityAuthentication.instance().get_wallet_name(address) + + +def is_demo_agent_seed_user(user_id: str) -> bool: + if _normalize_user_id(user_id) != _normalize_user_id( + agent_seed_constants.DEMO_AGENT_SEED_USER_ID, + ): + return False + display_name = _wallet_display_name_for_user_id(user_id) + if display_name is None: + return False + return display_name.strip().lower() == ( + agent_seed_constants.DEMO_AGENT_SEED_WALLET_DISPLAY_NAME.strip().lower() + ) + + +def _forbidden() -> None: + raise DemoAgentSeedUserActionForbiddenError( + agent_seed_constants.DEMO_AGENT_SEED_FORBIDDEN_ACTION_DETAIL, + ) + + +def _account_is_simulated(user_id: str, account_id: str) -> bool: + account = collection_providers.AccountProvider.instance().get_item(user_id, account_id) + return bool(account.is_simulated) + + +def _automation_account_ids( + automation_configuration: protocol_models.AutomationConfiguration, +) -> list[str]: + account_ids: list[str] = [] + for account_reference in automation_configuration.accounts or []: + if account_reference.id: + account_ids.append(account_reference.id) + return account_ids + + +def _validate_automation_accounts_simulated(user_id: str, account_ids: list[str]) -> None: + for account_id in account_ids: + if not _account_is_simulated(user_id, account_id): + _forbidden() + + +def validate_demo_agent_seed_user_action( + user_id: str, + user_action: protocol_models.UserAction, +) -> None: + if not is_demo_agent_seed_user(user_id): + return + wrapper = user_action.configuration + if wrapper is None or wrapper.actual_instance is None: + return + payload = wrapper.actual_instance + action_type = getattr(payload, "action_type", None) + + if action_type in ( + protocol_models.UserActionType.ACCOUNT_AUTH_CREATE, + protocol_models.UserActionType.ACCOUNT_AUTH_EDIT, + protocol_models.UserActionType.EXCHANGE_CONFIG_CREATE, + protocol_models.UserActionType.EXCHANGE_CONFIG_EDIT, + ): + _forbidden() + + if isinstance(payload, protocol_models.CreateAccountConfiguration): + if not payload.configuration.is_simulated: + _forbidden() + return + + if isinstance(payload, protocol_models.EditAccountConfiguration): + if payload.configuration.is_simulated is False: + _forbidden() + return + + if isinstance(payload, protocol_models.CreateAutomationConfiguration): + _validate_automation_accounts_simulated( + user_id, + _automation_account_ids(payload.configuration), + ) + return + + if isinstance(payload, protocol_models.EditAutomationConfiguration): + _validate_automation_accounts_simulated( + user_id, + _automation_account_ids(payload.configuration), + ) diff --git a/packages/node/octobot_node/constants.py b/packages/node/octobot_node/constants.py index 5167c8ebf7..6b8f366ea8 100644 --- a/packages/node/octobot_node/constants.py +++ b/packages/node/octobot_node/constants.py @@ -33,6 +33,9 @@ AUTOMATION_LOG_FILE_TRIM_LINES_FRACTION = 0.2 PARENT_WORKFLOW_ID_LENGTH = 36 # length of a UUID4 +# Max tasks returned per GET /api/v1/tasks/ page (UI fetches page=1 at this limit). +TASKS_LIST_MAX_PAGE_LIMIT = 500 + # default to 19 retry after 1, 2.5, 4.75, 8.125, 13.188, ... 2953.784 seconds (total of 4430 seconds) AUTOMATION_WORKFLOW_RETRY_INTERVAL_SECONDS = float(os.getenv("AUTOMATION_WORKFLOW_RETRY_INTERVAL_SECONDS", 1.0)) AUTOMATION_WORKFLOW_MAX_ITERATION_RETRIES = int(os.getenv("AUTOMATION_WORKFLOW_MAX_ITERATION_RETRIES", 19)) @@ -116,4 +119,10 @@ ALWAYS_ENSURE_SCHEDULER_APPLICATION_VERSION = bool(( os.getenv("ALWAYS_ENSURE_SCHEDULER_APPLICATION_VERSION") or "false" ).lower() == "true") - +# DBOS cron schedules (dbos cleanup, global view refresh, portfolio history). +# Env: ENABLE_SCHEDULED_WORKFLOWS (default on unless set to "false"). +# When false: on scheduler init, existing schedules are deleted and not re-registered (no new cron triggers). +# Does not cancel workflows already enqueued or running. Read at import; restart required to apply. +ENABLE_SCHEDULED_WORKFLOWS = bool(( + os.getenv("ENABLE_SCHEDULED_WORKFLOWS") or "true" +).lower() != "false") diff --git a/packages/node/octobot_node/enums.py b/packages/node/octobot_node/enums.py index 9a37624917..33d706106f 100644 --- a/packages/node/octobot_node/enums.py +++ b/packages/node/octobot_node/enums.py @@ -43,6 +43,19 @@ class SchedulerQueues(enum.Enum): PORTFOLIO_HISTORY_QUEUE = "portfolio_history_queue" +class SchedulerWorkflowNames(enum.Enum): + EXECUTE_AUTOMATION = "execute_automation" + EXECUTE_USER_ACTION = "execute_user_action" + GLOBAL_VIEW_REFRESH = "global_view_refresh" + PORTFOLIO_HISTORY_COLLECTION = "portfolio_history_collection" + DBOS_CLEANUP = "dbos_cleanup" + + +class UserActionSource(enum.Enum): + SYNC = "sync" + DEBUG_API = "debug_api" + + class SignalPriorityActionPayloadKeys(enum.Enum): ACTIONS = "actions" SIGNAL = "signal" diff --git a/packages/node/octobot_node/models.py b/packages/node/octobot_node/models.py index dd637788d1..8b32c88421 100644 --- a/packages/node/octobot_node/models.py +++ b/packages/node/octobot_node/models.py @@ -41,6 +41,7 @@ class TaskStatus(str, Enum): RUNNING = "running" COMPLETED = "completed" FAILED = "failed" + CANCELLED = "cancelled" class TaskType(str, Enum): diff --git a/packages/node/octobot_node/protocol/automations.py b/packages/node/octobot_node/protocol/automations.py index d305b69b7b..b85971f6f3 100644 --- a/packages/node/octobot_node/protocol/automations.py +++ b/packages/node/octobot_node/protocol/automations.py @@ -58,10 +58,13 @@ def to_protocol_automations_state( workflow_error=source.workflow_error, )) except Exception as exc: + # cloud be InvalidAutomationActionError when dag parsing fails task = source.task - content_preview = (task.content or "")[:80] if isinstance(task.content, str) else repr(task.content) - logger.warning( - f"Skipping malformed automation task id={task.id!r}: {exc} " + content_preview = octobot_commons_logging.get_private_minimized_message_if_necessary( + (task.content or "")[:80] if isinstance(task.content, str) else repr(task.content) + ) + logger.debug( + f"Skipping malformed automation task id={task.id!r}: {exc} ({exc.__class__.__name__}) " f"(content_metadata={task.content_metadata!r}, content_preview={content_preview!r})" ) return states diff --git a/packages/node/octobot_node/protocol/user_actions.py b/packages/node/octobot_node/protocol/user_actions.py index 355a3147d0..68585d94a2 100644 --- a/packages/node/octobot_node/protocol/user_actions.py +++ b/packages/node/octobot_node/protocol/user_actions.py @@ -14,13 +14,23 @@ # You should have received a copy of the GNU General Public License along # with OctoBot. If not, see . import octobot_protocol.models as protocol_models + +import octobot.community.node_journal as node_journal + +import octobot_node.enums as octobot_node_enums import octobot_node.scheduler.tasks as scheduler_tasks async def execute_user_action( user_action: protocol_models.UserAction, user_id: str, + *, + source: octobot_node_enums.UserActionSource = octobot_node_enums.UserActionSource.SYNC, ) -> None: + node_journal.record_external_action_received( + user_action, + source=source.value, + ) await scheduler_tasks.trigger_user_action_workflow( user_action, user_id ) diff --git a/packages/node/octobot_node/scheduler/__init__.py b/packages/node/octobot_node/scheduler/__init__.py index 58d2eb595a..41a52fcaba 100644 --- a/packages/node/octobot_node/scheduler/__init__.py +++ b/packages/node/octobot_node/scheduler/__init__.py @@ -16,16 +16,32 @@ import logging +import octobot_node.config import octobot_node.constants import octobot_node.scheduler.scheduler as scheduler_lib -import octobot_node.scheduler.workflows +import octobot_node.scheduler.workflows as scheduler_workflows import octobot_node.scheduler.workflows_version_migration as workflows_version_migration -scheduler_logger = logging.getLogger(__name__) +import octobot.community.node_journal as node_journal +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.recording_context as journal_recording_context + +scheduler_logger = logging.getLogger("scheduler.init") SCHEDULER: scheduler_lib.Scheduler = scheduler_lib.Scheduler() _shutdown_done = False +_scheduler_init_failure_recorded = False + + +def scheduler_init_failure_was_recorded() -> bool: + return _scheduler_init_failure_recorded + + +def _record_scheduler_init_failed(**kwargs) -> None: + global _scheduler_init_failure_recorded + _scheduler_init_failure_recorded = True + node_journal.record_scheduler_init_failed(**kwargs) def is_enabled() -> bool: @@ -36,20 +52,60 @@ def is_initialized() -> bool: return SCHEDULER.is_initialized() +def _scheduler_backend() -> journal_enums.JournalSchedulerBackend: + if octobot_node.config.settings.SCHEDULER_POSTGRES_URL: + return journal_enums.JournalSchedulerBackend.POSTGRES + return journal_enums.JournalSchedulerBackend.SQLITE + + async def initialize_scheduler(): - global _shutdown_done + global _shutdown_done, _scheduler_init_failure_recorded _shutdown_done = False + _scheduler_init_failure_recorded = False scheduler_logger.info("Initializing scheduler") - SCHEDULER.create() - octobot_node.scheduler.workflows.register_workflows() + backend = _scheduler_backend() + with journal_recording_context.scheduler_init_phase( + init_phase=journal_enums.JournalInitPhase.DBOS_CREATE, + backend=backend, + on_failure=_record_scheduler_init_failed, + ): + SCHEDULER.create() + with journal_recording_context.scheduler_init_phase( + init_phase=journal_enums.JournalInitPhase.REGISTER_WORKFLOWS, + backend=backend, + on_failure=_record_scheduler_init_failed, + ): + scheduler_workflows.register_workflows() if octobot_node.constants.ALWAYS_ENSURE_SCHEDULER_APPLICATION_VERSION: - workflows_version_migration.migrate_stranded_workflow_versions( - target_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, - ) + with journal_recording_context.scheduler_init_phase( + init_phase=journal_enums.JournalInitPhase.VERSION_MIGRATION, + backend=backend, + on_failure=_record_scheduler_init_failed, + ): + workflows_version_migration.migrate_stranded_workflow_versions( + target_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, + ) + import octobot_node.scheduler.queues as scheduler_queues import octobot_node.scheduler.schedules as schedules - SCHEDULER.start() + with journal_recording_context.scheduler_init_phase( + init_phase=journal_enums.JournalInitPhase.DBOS_LAUNCH, + backend=backend, + on_failure=_record_scheduler_init_failed, + ): + SCHEDULER.start() + scheduler_logger.info("Scheduler: registering DBOS queues") + await scheduler_queues.register_scheduler_queues_async() + scheduler_logger.info("Scheduler: DBOS queues registered") # apply_schedules requires DBOS launch (sys_db); must run after start(). - await schedules.register_schedules(SCHEDULER) + with journal_recording_context.scheduler_init_phase( + init_phase=journal_enums.JournalInitPhase.REGISTER_SCHEDULES, + backend=backend, + on_failure=_record_scheduler_init_failed, + ): + scheduler_logger.info("Scheduler: applying schedules") + await schedules.register_schedules(SCHEDULER) + scheduler_logger.info("Scheduler: schedules applied") + scheduler_logger.info("Scheduler: initialize_scheduler completed") async def shutdown_scheduler_and_trading_signal_channel() -> None: diff --git a/packages/node/octobot_node/scheduler/api.py b/packages/node/octobot_node/scheduler/api.py index 4bb71e96eb..bd0c2b9df3 100644 --- a/packages/node/octobot_node/scheduler/api.py +++ b/packages/node/octobot_node/scheduler/api.py @@ -24,6 +24,7 @@ import octobot_node.config import octobot_node.constants +import octobot_node.enums import octobot_node.models import octobot_node.scheduler import octobot_node.scheduler.workflows_util as workflows_util @@ -79,34 +80,26 @@ async def get_task_metrics( if not scheduler.INSTANCE: return {"pending": 0, "scheduled": 0, "results": 0} try: + wallet_scoped_metrics = user_id is not None pending_statuses, result_statuses = await asyncio.gather( - scheduler.INSTANCE.list_workflows_async(status=[ - dbos.WorkflowStatusString.ENQUEUED.value, - dbos.WorkflowStatusString.PENDING.value, - ], load_output=False), - scheduler.INSTANCE.list_workflows_async(status=[ - dbos.WorkflowStatusString.SUCCESS.value, - dbos.WorkflowStatusString.ERROR.value, - ], load_output=False), - ) - if user_id is not None: - automation_queue = octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value - pending_only_automation = [ - row for row in (pending_statuses or []) if row.queue_name == automation_queue - ] - result_only_automation = [ - row for row in (result_statuses or []) if row.queue_name == automation_queue - ] - pending_statuses = workflows_util.filter_by_wallet( - pending_only_automation, + workflows_util.list_scheduler_workflows_async( + scheduler.INSTANCE, + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + None, user_id, - octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE, - ) - result_statuses = workflows_util.filter_by_wallet( - result_only_automation, + load_output=False, + load_input=wallet_scoped_metrics, + queues_only=True, + ), + workflows_util.list_scheduler_workflows_async( + scheduler.INSTANCE, + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + list(workflows_util.DBOS_TERMINAL_WORKFLOW_STATUSES), user_id, - octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE, - ) + load_output=False, + load_input=wallet_scoped_metrics, + ), + ) return { "pending": len(pending_statuses or []), "scheduled": 0, @@ -130,6 +123,35 @@ def _get_active_execution( return dated[-1] if dated else (executions[-1] if executions else None) +def _task_recency_sort_key(task: octobot_node.models.Task) -> tuple[int, str, str]: + active_execution = _get_active_execution(task.executions or []) + if active_execution is None: + return (0, "", task.id or "") + sort_at = active_execution.completed_at or active_execution.scheduled_at + if sort_at is None: + return (0, "", task.id or "") + return (1, sort_at.isoformat(), task.id or "") + + +def sort_tasks_by_recency( + tasks: list[octobot_node.models.Task], +) -> list[octobot_node.models.Task]: + return sorted(tasks, key=_task_recency_sort_key, reverse=True) + + +def paginate_tasks( + tasks: list[octobot_node.models.Task], + page: int, + limit: int, +) -> list[octobot_node.models.Task]: + page = max(1, page) + max_limit = octobot_node.constants.TASKS_LIST_MAX_PAGE_LIMIT + limit = max(1, min(limit, max_limit)) + sorted_tasks = sort_tasks_by_recency(tasks) + start_idx = (page - 1) * limit + return sorted_tasks[start_idx:start_idx + limit] + + def _build_tasks_from_executions( executions: list[octobot_node.models.Execution], ) -> list[octobot_node.models.Task]: @@ -284,6 +306,8 @@ async def get_task_result(task_id: str): except Exception as error: result_data = {"error": str(error)} return {"status": "completed", "data": result_data} + if wf_status == "CANCELLED": + return {"status": "cancelled", "data": None} except Exception as error: logger.debug(f"Workflow {task_id} not yet complete: {error}") return {"status": "pending or running"} diff --git a/packages/node/octobot_node/scheduler/automations/automation_states_loader.py b/packages/node/octobot_node/scheduler/automations/automation_states_loader.py index c0f0327f1c..bc33fd6b6c 100644 --- a/packages/node/octobot_node/scheduler/automations/automation_states_loader.py +++ b/packages/node/octobot_node/scheduler/automations/automation_states_loader.py @@ -12,7 +12,9 @@ import octobot_flow.entities as flow_entities import octobot_flow.parsers.automation_state_reader as automation_state_reader_module +import octobot_node.enums as octobot_node_enums import octobot_node.models as node_models +import octobot_node.scheduler as scheduler_module import octobot_node.scheduler.automations.octobot_flow_client as octobot_flow_client import octobot_node.scheduler.task_context as task_context_module import octobot_node.scheduler.workflows_util as workflows_util_module @@ -96,9 +98,20 @@ def patch_task_content_degraded_state( async def get_automation_workflow_status(automation_id: str) -> dbos_lib.WorkflowStatus: - for workflow_status in await dbos_lib.DBOS.list_workflows_async(status=[ - dbos_lib.WorkflowStatusString.PENDING.value, dbos_lib.WorkflowStatusString.ENQUEUED.value - ]): + if not scheduler_module.is_initialized(): + raise RuntimeError("Scheduler is not initialized") + pending_workflows = await workflows_util_module.list_scheduler_workflows_async( + scheduler_module.SCHEDULER.INSTANCE, + octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + [ + dbos_lib.WorkflowStatusString.PENDING, + dbos_lib.WorkflowStatusString.ENQUEUED, + ], + None, + load_output=False, + load_input=True, + ) + for workflow_status in pending_workflows: if get_automation_id(workflow_status) == automation_id: return workflow_status raise ValueError(f"No automation workflow found for automation_id: {automation_id}") @@ -145,7 +158,6 @@ async def load_automation_state_sources( load_output: bool = True, ) -> list["automations_protocol.AutomationStateSource"]: import octobot_node.protocol.automations as automations_protocol - import octobot_node.scheduler as scheduler_module scheduler = scheduler_module.SCHEDULER workflows = await scheduler._get_latest_workflow_for_each_automation( diff --git a/packages/node/octobot_node/scheduler/automations/octobot_flow_client.py b/packages/node/octobot_node/scheduler/automations/octobot_flow_client.py index 17e2036e73..8613455ace 100644 --- a/packages/node/octobot_node/scheduler/automations/octobot_flow_client.py +++ b/packages/node/octobot_node/scheduler/automations/octobot_flow_client.py @@ -156,8 +156,8 @@ async def run(self) -> None: ) self.after_execution_state = automation_job.automation_state post_execution_state_dump = automation_job.dump() - next_actions_description, has_next_actions = self.get_next_actions_description(post_execution_state_dump) self.result.processed_actions = executed_actions + next_actions_description, has_next_actions = self.get_next_actions_description(post_execution_state_dump) self.result.next_actions_description = next_actions_description self.result.has_next_actions = has_next_actions self.result.actions_dag = automation_job.automation_state.automation.actions_dag @@ -173,6 +173,11 @@ def get_next_actions_description( ) has_next_actions = bool(automation.actions_dag.get_executable_actions()) if not has_next_actions and (pending_actions := automation.actions_dag.get_pending_actions()): + # After an upstream failure, dependents stay pending (deps require NO_ERROR). + # Finish the iteration with has_next_actions=False so WorkflowDAGDependenciesError + # does not mask the real error. + if automation.actions_dag.has_failed_action(): + return next_actions_description, False raise errors.WorkflowDAGDependenciesError( f"Automation {automation.metadata.automation_id}: actions DAG dependencies issue: " f"no executable actions while there are still " diff --git a/packages/node/octobot_node/scheduler/internal_trading_signals.py b/packages/node/octobot_node/scheduler/internal_trading_signals.py index e02743838c..2ed61b4ec7 100644 --- a/packages/node/octobot_node/scheduler/internal_trading_signals.py +++ b/packages/node/octobot_node/scheduler/internal_trading_signals.py @@ -1,10 +1,11 @@ -import dbos - import octobot_commons.logging import octobot_flow.entities import octobot_flow.repositories.community as trading_signals_channel +import octobot_node.enums as octobot_node_enums +import octobot_node.scheduler as scheduler import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader import octobot_node.scheduler.tasks as tasks +import octobot_node.scheduler.workflows_util as workflows_util_module async def _on_internal_trading_signal(trading_signal: octobot_flow.entities.TradingSignal) -> None: @@ -27,19 +28,35 @@ async def send_internal_trading_signal(trading_signal: octobot_flow.entities.Tra """ await trading_signals_channel.send_internal_trading_signal(trading_signal) + +def workflow_row_matches_copier_trading_signal( + pending_workflow_status, + trading_signal: octobot_flow.entities.TradingSignal, +) -> bool: + copied_strategy_ids = automation_states_loader.get_automation_copied_strategy_ids(pending_workflow_status) + return trading_signal.strategy_id in copied_strategy_ids + + +async def list_pending_copier_automation_workflow_statuses(): + return await workflows_util_module.list_scheduler_workflows_async( + scheduler.SCHEDULER.INSTANCE, + octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + None, + None, + load_output=False, + load_input=True, + queues_only=True, + ) + + async def _trigger_copier_automation(trading_signal: octobot_flow.entities.TradingSignal) -> None: """ Triggers copier automations with the given trading signal. Automations are triggered one by one to avoid concurrent executions. """ - import octobot_node.scheduler as scheduler - pending_workflow_statuses = await scheduler.SCHEDULER.INSTANCE.list_workflows_async( - status=[dbos.WorkflowStatusString.ENQUEUED.value, dbos.WorkflowStatusString.PENDING.value] - ) + pending_workflow_statuses = await list_pending_copier_automation_workflow_statuses() for pending_workflow_status in pending_workflow_statuses: - if ( - trading_signal.strategy_id in automation_states_loader.get_automation_copied_strategy_ids(pending_workflow_status) - ): + if workflow_row_matches_copier_trading_signal(pending_workflow_status, trading_signal): octobot_commons.logging.get_logger("internal_trading_signals").info( f"Triggering copier automation {pending_workflow_status.workflow_id} with trading signal {trading_signal.strategy_id}" ) diff --git a/packages/node/octobot_node/scheduler/queues.py b/packages/node/octobot_node/scheduler/queues.py new file mode 100644 index 0000000000..eb3501a87f --- /dev/null +++ b/packages/node/octobot_node/scheduler/queues.py @@ -0,0 +1,51 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . +import typing + +import octobot_node.enums + +_SCHEDULER_QUEUE_SPECS: tuple[ + tuple[octobot_node.enums.SchedulerQueues, dict[str, typing.Any]], + ..., +] = ( + (octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE, {}), + (octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE, {}), + ( + octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE, + {"global_concurrency": 1}, + ), + ( + octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE, + {"global_concurrency": 1}, + ), + ( + octobot_node.enums.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE, + {"global_concurrency": 1}, + ), +) + + +async def register_scheduler_queues_async() -> None: + """Register DBOS scheduler queues after launch.""" + import octobot_node.scheduler as scheduler_module + + if scheduler_module.SCHEDULER.INSTANCE is None: + raise RuntimeError("Scheduler not initialized") + for queue, register_kwargs in _SCHEDULER_QUEUE_SPECS: + await scheduler_module.SCHEDULER.INSTANCE.register_queue_async( + queue.value, + **register_kwargs, + ) diff --git a/packages/node/octobot_node/scheduler/scheduler.py b/packages/node/octobot_node/scheduler/scheduler.py index 01b5475070..ee40a20b7c 100644 --- a/packages/node/octobot_node/scheduler/scheduler.py +++ b/packages/node/octobot_node/scheduler/scheduler.py @@ -63,17 +63,6 @@ def _sanitize(result: typing.Any) -> typing.Any: class Scheduler: INSTANCE: dbos.DBOS = None # type: ignore - AUTOMATION_WORKFLOW_QUEUE: dbos.Queue = None # type: ignore - USER_ACTION_QUEUE: dbos.Queue = None # type: ignore - DBOS_CLEANUP_QUEUE: dbos.Queue = None # type: ignore - GLOBAL_VIEW_QUEUE: dbos.Queue = None # type: ignore - PORTFOLIO_HISTORY_QUEUE: dbos.Queue = None # type: ignore - - @staticmethod - def _wallet_filter_queue(queue_names: typing.Optional[list[str]]) -> octobot_node.enums.SchedulerQueues: - if queue_names == [octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE.value]: - return octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE - return octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE def __init__(self): self.logger = logging.getLogger(self.__class__.__name__) @@ -138,7 +127,6 @@ def is_initialized(self) -> bool: def start(self): if self.INSTANCE: - self.create_queues() self.logger.info("Starting scheduler") self.INSTANCE.launch() self.logger.info("Scheduler started") @@ -151,28 +139,6 @@ def stop(self) -> None: self.INSTANCE.destroy() self.logger.info("Scheduler stopped") Scheduler.INSTANCE = None - Scheduler.AUTOMATION_WORKFLOW_QUEUE = None - Scheduler.USER_ACTION_QUEUE = None - Scheduler.DBOS_CLEANUP_QUEUE = None - Scheduler.GLOBAL_VIEW_QUEUE = None - Scheduler.PORTFOLIO_HISTORY_QUEUE = None - - def create_queues(self): - self.AUTOMATION_WORKFLOW_QUEUE = dbos.Queue(name=octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value) - self.USER_ACTION_QUEUE = dbos.Queue(name=octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE.value) - self.DBOS_CLEANUP_QUEUE = dbos.Queue( - name=octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE.value, - # only one cleanup workflow can run at a time - concurrency=1, - ) - self.GLOBAL_VIEW_QUEUE = dbos.Queue( - name=octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value, - concurrency=1, - ) - self.PORTFOLIO_HISTORY_QUEUE = dbos.Queue( - name=octobot_node.enums.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value, - concurrency=1, - ) async def get_periodic_tasks(self, user_id: typing.Optional[str] = None) -> list[octobot_node.models.Execution]: """DBOS scheduled workflows are not easily introspectable; return empty list.""" @@ -185,15 +151,14 @@ async def get_pending_tasks(self, user_id: typing.Optional[str] = None) -> list[ try: pending_workflow_statuses = await self._list_workflows( user_id, - [ - dbos.WorkflowStatusString.ENQUEUED, dbos.WorkflowStatusString.PENDING - ], - queue_names=[octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], - load_output=False + None, + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + load_output=False, + load_input=True, + queues_only=True, ) for pending_workflow_status in pending_workflow_statuses: try: - task = workflows_util.get_automation_input_task(pending_workflow_status) if reader := automation_states_loader.get_automation_state_reader(pending_workflow_status): next_step = ", ".join([ action.get_summary() @@ -212,53 +177,72 @@ async def get_pending_tasks(self, user_id: typing.Optional[str] = None) -> list[ async def _list_workflows( self, - user_id: typing.Optional[str], + user_id: typing.Optional[str], statuses: typing.Optional[list[dbos.WorkflowStatusString]], - queue_names: typing.Optional[list[str]], - load_output: bool + workflow_name: octobot_node.enums.SchedulerWorkflowNames, + load_output: bool, + *, + load_input: bool = False, + sort_desc: typing.Optional[bool] = None, + limit: typing.Optional[int] = None, + workflow_id_prefix: typing.Optional[str | list[str]] = None, + queues_only: bool = False, ) -> list[dbos.WorkflowStatus]: - workflows = await self.INSTANCE.list_workflows_async( - status=[status.value for status in statuses] if statuses else None, - queue_name=queue_names, - load_output=load_output + if not self.INSTANCE: + return [] + return await workflows_util.list_scheduler_workflows_async( + self.INSTANCE, + workflow_name, + statuses, + user_id, + load_output=load_output, + load_input=load_input, + sort_desc=sort_desc, + limit=limit, + workflow_id_prefix=workflow_id_prefix, + queues_only=queues_only, ) - if user_id: - workflows = workflows_util.filter_by_wallet( - workflows, - user_id, - self._wallet_filter_queue(queue_names), - ) - return workflows async def _get_parent_and_children_automation_workflows( self, user_id: typing.Optional[str], workflow_ids: list[str], - statuses: list[dbos.WorkflowStatusString], + statuses: typing.Optional[list[dbos.WorkflowStatusString]], load_output: bool = False, + *, + queues_only: bool = False, ) -> list[dbos.WorkflowStatus]: - all_workflows = await self._list_workflows( - user_id, statuses, [octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], load_output - ) - parent_workflow_ids = set( + parent_workflow_ids = list(dict.fromkeys( workflows_util.normalize_parent_automation_id(workflow_id) for workflow_id in workflow_ids + )) + if not parent_workflow_ids: + return [] + list_statuses = None if queues_only else statuses + return await self._list_workflows( + user_id, + list_statuses, + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + load_output, + workflow_id_prefix=parent_workflow_ids, + queues_only=queues_only, ) - return [ - workflow - for workflow in all_workflows - if workflows_util.normalize_parent_automation_id(workflow.workflow_id) in parent_workflow_ids - ] async def get_parent_and_children_automation_workflow_ids( self, wallet_address: typing.Optional[str], workflow_ids: list[str], - statuses: list[dbos.WorkflowStatusString], - load_output: bool = False + statuses: typing.Optional[list[dbos.WorkflowStatusString]], + load_output: bool = False, + *, + queues_only: bool = False, ) -> list[str]: matching_workflows = await self._get_parent_and_children_automation_workflows( - wallet_address, workflow_ids, statuses, load_output + wallet_address, + workflow_ids, + statuses, + load_output, + queues_only=queues_only, ) return [workflow.workflow_id for workflow in matching_workflows] @@ -273,8 +257,10 @@ async def _get_user_action_workflow_ids( return [] user_action_id_set = set(user_action_ids) matching_workflows = await self._list_workflows( - user_id, statuses, - [octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE.value], load_output + user_id, + statuses, + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_USER_ACTION, + load_output, ) matched_workflow_ids: list[str] = [] for workflow in matching_workflows: @@ -298,11 +284,9 @@ async def resolve_active_automation_workflow_ids_for_parent_id( matching_workflows = await self._get_parent_and_children_automation_workflows( user_id, [parent_id], - [ - dbos.WorkflowStatusString.ENQUEUED, - dbos.WorkflowStatusString.PENDING, - ], + None, load_output=False, + queues_only=True, ) if not matching_workflows: return [] @@ -322,11 +306,9 @@ async def resolve_automation_owner_user_id( matching_workflows = await self._get_parent_and_children_automation_workflows( None, [parent_id], - [ - dbos.WorkflowStatusString.ENQUEUED, - dbos.WorkflowStatusString.PENDING, - ], + None, load_output=False, + queues_only=True, ) if not matching_workflows: return None @@ -395,7 +377,11 @@ async def _get_latest_workflow_for_each_automation( load_output: bool = False, ) -> list[dbos.WorkflowStatus]: workflows = await self._list_workflows( - user_id, statuses, [octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], load_output=load_output + user_id, + statuses, + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + load_output, + load_input=True, ) by_parent = workflows_util.get_workflows_by_parent_id(workflows) return [ @@ -408,9 +394,8 @@ async def cancel_workflows(self, workflow_ids: list[str]) -> list[str]: to_cancel = await self.get_parent_and_children_automation_workflow_ids( None, workflow_ids, - [ - dbos.WorkflowStatusString.ENQUEUED, dbos.WorkflowStatusString.PENDING - ] + None, + queues_only=True, ) self.logger.info(f"Cancelling {len(to_cancel)} workflows {to_cancel}") await self.INSTANCE.cancel_workflows_async(to_cancel) @@ -437,60 +422,251 @@ async def get_scheduled_tasks(self, user_id: typing.Optional[str] = None) -> lis """DBOS has no direct 'scheduled for later' queue; return empty list.""" return [] + async def _list_terminal_automation_workflows( + self, + user_id: typing.Optional[str], + *, + load_output: bool, + ) -> list[dbos.WorkflowStatus]: + return await self._list_workflows( + user_id, + list(workflows_util.DBOS_TERMINAL_WORKFLOW_STATUSES), + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + load_output, + load_input=True, + ) + + @staticmethod + def _automation_input_wallet_and_encryption( + latest_input_task: typing.Optional[octobot_node.models.Task], + ) -> tuple[typing.Optional[str], bool]: + if latest_input_task is None: + return None, False + return latest_input_task.user_id, bool(latest_input_task.content_metadata) + + @staticmethod + def _automation_execution_display_name( + latest_input_task: typing.Optional[octobot_node.models.Task], + workflow_id: str, + *, + is_latest_child: bool, + ) -> typing.Optional[str]: + if not is_latest_child: + return None + if latest_input_task and latest_input_task.name: + return latest_input_task.name + return str(workflow_id) + + def _executions_for_automation_group( + self, + group: list[dbos.WorkflowStatus], + latest: dbos.WorkflowStatus, + latest_input_task: typing.Optional[octobot_node.models.Task], + ) -> list[octobot_node.models.Execution]: + """ + Build tasks API Execution rows for one parent automation's terminal children. + + When the latest child is CANCELLED, return only that row (hide prior SUCCESS/ERROR), + matching workflows_util.resolve_automation_result_for_group export semantics. + """ + if latest.status == dbos.WorkflowStatusString.CANCELLED.value: + return [ + self._execution_from_automation_child_row( + latest, + latest_input_task=latest_input_task, + is_latest_child=True, + ) + ] + group_executions: list[octobot_node.models.Execution] = [] + for terminal_workflow_status in group: + if terminal_workflow_status.status not in ( + dbos.WorkflowStatusString.SUCCESS.value, + dbos.WorkflowStatusString.ERROR.value, + ): + continue + is_latest_child = terminal_workflow_status.workflow_id == latest.workflow_id + group_executions.append( + self._execution_from_automation_child_row( + terminal_workflow_status, + latest_input_task=latest_input_task, + is_latest_child=is_latest_child, + ) + ) + return group_executions + + def _execution_from_automation_child_row( + self, + workflow_status: dbos.WorkflowStatus, + *, + latest_input_task: typing.Optional[octobot_node.models.Task], + is_latest_child: bool, + ) -> octobot_node.models.Execution: + """ + Map one terminal child workflow row to a tasks API Execution. + + Wallet, display name, and base encryption come from the parent's latest child input + (`latest_input_task`), including for older SUCCESS/ERROR rows in the same group. + Row status, errors, and output-derived encryption come from this child's DBOS row; + output parsing applies only when `is_latest_child` and status is SUCCESS. + """ + parent_user_id, parent_is_encrypted = self._automation_input_wallet_and_encryption( + latest_input_task + ) + workflow_identifier = str(workflow_status.workflow_id) + display_name = self._automation_execution_display_name( + latest_input_task, + workflow_identifier, + is_latest_child=is_latest_child, + ) + + error_message: typing.Optional[str] = None + if workflow_status.status == dbos.WorkflowStatusString.CANCELLED.value: + if not is_latest_child: + raise ValueError( + f"CANCELLED workflow row must be latest child: {workflow_identifier!r}" + ) + return octobot_node.models.Execution( + id=workflow_identifier, + name=display_name, + description="Cancelled", + status=octobot_node.models.TaskStatus.CANCELLED, + is_encrypted=parent_is_encrypted, + result="", + result_metadata="", + scheduled_at=workflow_status.created_at, + completed_at=workflow_status.updated_at, + error=None, + error_message=None, + user_id=parent_user_id, + ) + if workflow_status.status == dbos.WorkflowStatusString.SUCCESS.value: + if is_latest_child: + output_error = None + if workflow_status.output: + try: + raw_output = workflow_status.output + if isinstance(raw_output, str): + raw_output = json.loads(raw_output) + output = workflow_params.AutomationWorkflowOutput.from_dict(raw_output) + output_error = output.error + error_message = output.error_message + except Exception as parse_err: + self.logger.warning( + f"Failed to parse output for workflow {workflow_identifier}: {parse_err}" + ) + if output_error: + row_status = octobot_node.models.TaskStatus.FAILED + description = "ERROR" + error = output_error + else: + row_status = octobot_node.models.TaskStatus.COMPLETED + description = "Completed" + error = None + error_message = None + if workflow_status.output: + parsed_output = workflows_util.parse_automation_workflow_output(workflow_status) + row_is_encrypted = bool( + parsed_output.state_metadata if parsed_output else None + ) or parent_is_encrypted + else: + row_is_encrypted = parent_is_encrypted + else: + row_status = octobot_node.models.TaskStatus.COMPLETED + description = "Completed" + error = None + error_message = None + row_is_encrypted = parent_is_encrypted + elif workflow_status.status == dbos.WorkflowStatusString.ERROR.value: + row_status = octobot_node.models.TaskStatus.FAILED + description = "ERROR" + error = ( + str(workflow_status.error) + if workflow_status.error + else "Execution failed" + ) + row_is_encrypted = parent_is_encrypted + else: + raise ValueError(f"Unexpected terminal workflow status for execution row: {workflow_status.status!r}") + + return octobot_node.models.Execution( + id=workflow_identifier, + name=display_name, + description=description, + status=row_status, + is_encrypted=row_is_encrypted, + result="", + result_metadata="", + scheduled_at=workflow_status.created_at, + completed_at=workflow_status.updated_at, + error=error, + error_message=error_message, + user_id=parent_user_id, + ) + async def get_results(self, user_id: typing.Optional[str] = None) -> list[octobot_node.models.Execution]: + """ + List terminal automation workflow executions for the tasks API. + + Uses a cheap metadata scan, then hydrates each parent's latest child + (input and output) in one request. + """ if not self.INSTANCE: return [] executions: list[octobot_node.models.Execution] = [] try: - completed_workflow_statuses = await self._list_workflows(user_id, [ - dbos.WorkflowStatusString.SUCCESS, dbos.WorkflowStatusString.ERROR - ], [octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], load_output=True) - for completed_workflow_status in completed_workflow_statuses: + # Step 1 — Terminal workflows without input/output payloads. + metadata_rows = await workflows_util.list_scheduler_workflows_async( + self.INSTANCE, + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + list(workflows_util.DBOS_TERMINAL_WORKFLOW_STATUSES), + None, + load_output=False, + load_input=False, + ) + # Step 2 — Group by parent automation id; latest child per group. + by_parent = workflows_util.get_workflows_by_parent_id(metadata_rows) + if not by_parent: + return executions + + latest_by_parent: dict[str, dbos.WorkflowStatus] = { + parent_id: workflows_util.get_latest_workflow(group) + for parent_id, group in by_parent.items() + } + all_metadata_rows: list[dbos.WorkflowStatus] = [] + for group in by_parent.values(): + all_metadata_rows.extend(group) + + # Step 3 — Hydrate each parent's latest child (input and output) in one request. + latest_workflow_ids = [ + latest_workflow.workflow_id for latest_workflow in latest_by_parent.values() + ] + if latest_workflow_ids: + await workflows_util.hydrate_scheduler_workflows_async( + self.INSTANCE, + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + all_metadata_rows, + latest_workflow_ids, + load_input=True, + load_output=True, + ) + + # Step 4 — Wallet filter and build Execution rows per parent group. + for parent_id, group in list(by_parent.items()): + latest = latest_by_parent[parent_id] try: - task = workflows_util.get_automation_input_task(completed_workflow_status) - error_message = None - if completed_workflow_status.status == dbos.WorkflowStatusString.SUCCESS.value: - output_error = None - if completed_workflow_status.output: - try: - output = workflow_params.AutomationWorkflowOutput.from_dict( - json.loads(completed_workflow_status.output) - ) - output_error = output.error - error_message = output.error_message - except Exception as parse_err: - self.logger.warning( - f"Failed to parse output for workflow {completed_workflow_status.workflow_id}: {parse_err}" - ) - if output_error: - status = octobot_node.models.TaskStatus.FAILED - description = "ERROR" - error = output_error - else: - status = octobot_node.models.TaskStatus.COMPLETED - description = "Completed" - error = None - error_message = None - else: - status = octobot_node.models.TaskStatus.FAILED - description = "ERROR" - error = str(completed_workflow_status.error) if completed_workflow_status.error else "Execution failed" - executions.append(octobot_node.models.Execution( - id=completed_workflow_status.workflow_id, - name=task.name if task else completed_workflow_status.workflow_id, - description=description, - status=status, - is_encrypted=bool(task.content_metadata) if task else False, - result="", - result_metadata="", - scheduled_at=completed_workflow_status.created_at, - completed_at=completed_workflow_status.updated_at, - error=error, - error_message=error_message, - user_id=task.user_id if task else None, - )) + latest_input_task = workflows_util.get_automation_input_task(latest) + if ( + user_id is not None + and latest_input_task is not None + and latest_input_task.user_id + and latest_input_task.user_id != user_id + ): + continue + executions.extend( + self._executions_for_automation_group(group, latest, latest_input_task) + ) except Exception as e: - self.logger.exception(e, True, f"Failed to process result workflow {completed_workflow_status.workflow_id}: {e}") + self.logger.exception(e, True, f"Failed to process result workflow group: {e}") except Exception as e: self.logger.warning(f"Failed to list result workflows: {e}") return executions @@ -524,8 +700,9 @@ def _build_export_result_from_status( output, result_task = self._parse_output_and_task_from_workflow_output(workflow_status) except Exception as e: self.logger.warning(f"Failed to parse output for workflow {workflow_status.workflow_id}: {e}") - output = workflow_params.AutomationWorkflowOutput() - if not output.state: + return {"result": "", "result_metadata": ""} + # Latest run: export persisted output.state when present, else workflow input task content. + if not output.state and not result_task.content: return {"result": "", "result_metadata": ""} with task_context.encrypted_task(result_task): if (result_task.content == output.state and output.state_metadata @@ -533,10 +710,12 @@ def _build_export_result_from_status( raise encryption.EncryptionTaskError("Internal state decryption silently failed") user_rsa_key = user_rsa_public_key or octobot_node.config.settings.TASKS_USER_RSA_PUBLIC_KEY if not user_rsa_key or not octobot_node.config.settings.TASKS_SERVER_ECDSA_PRIVATE_KEY: + # Node-side decrypt only; return plaintext state (typical dev / tests). return { "result": result_task.content, # type: ignore "result_metadata": "", } + # Re-encrypt for the requesting user's RSA key (export to client). result, metadata = encryption.encrypt_task_result( result_task.content, rsa_public_key=user_rsa_key, @@ -550,12 +729,18 @@ async def get_workflows_export_results( user_id: typing.Optional[str], user_rsa_public_key: typing.Optional[str] = None, ) -> dict[str, dict[str, str]]: + """ + Batch-export automation state for parent task IDs. + + Per task_id the value is one of: + - ``{"error": "not found" | "forbidden" | ...}`` — request/lookup failure. + - ``{"result", "result_metadata"}`` — decrypted or re-encrypted state (may be empty strings). + - ``{"result": "", "result_metadata": "", "error": "..."}`` — chosen row is ERROR with no + persisted output.state; empty result fields mean nothing to export, ``error`` is the run failure. + """ if not self.INSTANCE: return {} - completed = await self._list_workflows(None, [ - dbos.WorkflowStatusString.SUCCESS, - dbos.WorkflowStatusString.ERROR, - ], [octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], load_output=True) + completed = await self._list_terminal_automation_workflows(None, load_output=True) by_parent = workflows_util.get_workflows_by_parent_id(completed) out: dict[str, dict[str, str]] = {} @@ -572,24 +757,24 @@ async def get_workflows_export_results( if user_id is not None and (task is None or task.user_id != user_id): out[task_id] = {"error": "forbidden"} continue - result_workflows = [ - w for w in group - if w.status == dbos.WorkflowStatusString.SUCCESS.value and w.output - ] - chosen = workflows_util.get_latest_workflow(result_workflows) if result_workflows else None - if chosen is None: - error_ws = [w for w in group if w.status == dbos.WorkflowStatusString.ERROR.value] - if error_ws: - err = error_ws[-1].error - out[task_id] = { - "result": "", "result_metadata": "", - "error": str(err) if err else "Execution failed", - } - else: - out[task_id] = {"result": "", "result_metadata": ""} - continue + export_workflow = workflows_util.get_latest_workflow(group) user_rsa = user_rsa_public_key.encode("utf-8") if user_rsa_public_key else None - out[task_id] = self._build_export_result_from_status(chosen, user_rsa) + built = self._build_export_result_from_status(export_workflow, user_rsa) + if built.get("result") or built.get("result_metadata"): + # SUCCESS/ERROR with output.state, or CANCELLED with input task content. + out[task_id] = built + continue + if export_workflow.status == dbos.WorkflowStatusString.ERROR.value: + # ERROR without persisted state: same empty payload as above, plus workflow error. + err = export_workflow.error + out[task_id] = { + "result": "", + "result_metadata": "", + "error": str(err) if err else "Execution failed", + } + else: + # CANCELLED (or other) with no input content — legitimately empty export. + out[task_id] = built except Exception as e: self.logger.warning(f"Failed to export result for {task_id}: {e}") out[task_id] = {"error": str(e)} @@ -757,13 +942,13 @@ async def list_user_actions( if not self.INSTANCE: return [] _ = active_only # Reserved for API; DBOS fetches always use explicit non-terminal / terminal status sets. - user_action_queue_name = octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE.value loaded: list[tuple[tuple[int, str, str], protocol_models.UserAction]] = [] input_workflows = await self._list_workflows( user_id, list(workflows_util.get_user_action_input_workflow_statuses()), - [user_action_queue_name], + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_USER_ACTION, load_output=False, + load_input=True, ) for workflow_status in input_workflows: user_action_row = self._user_action_from_workflow_inputs(workflow_status, terminal=False) @@ -771,9 +956,10 @@ async def list_user_actions( loaded.append((sort_key, user_action_row)) terminal_workflows = await self._list_workflows( user_id, - list(workflows_util.get_user_action_terminal_workflow_statuses()), - [user_action_queue_name], + list(workflows_util.DBOS_TERMINAL_WORKFLOW_STATUSES), + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_USER_ACTION, load_output=True, + load_input=True, ) for workflow_status in terminal_workflows: user_action_row = self._user_action_from_terminal_workflow(workflow_status) diff --git a/packages/node/octobot_node/scheduler/schedules.py b/packages/node/octobot_node/scheduler/schedules.py index 485afcc1cf..dea341c54b 100644 --- a/packages/node/octobot_node/scheduler/schedules.py +++ b/packages/node/octobot_node/scheduler/schedules.py @@ -339,13 +339,40 @@ async def _ensure_schedule( await _maybe_catch_up_schedule_once_on_startup(schedule_name, schedule_input) -async def register_schedules(scheduler: scheduler_module.Scheduler) -> None: - schedule_inputs: list[dbos.ScheduleInput] = [ +def scheduled_workflows_enabled() -> bool: + return constants.ENABLE_SCHEDULED_WORKFLOWS + + +def get_registered_schedule_inputs() -> list[dbos.ScheduleInput]: + return [ dbos_cleanup_workflow.get_schedule_input(), global_view_workflow.get_schedule_input(), portfolio_history_workflow.get_schedule_input(), ] - for schedule_input in schedule_inputs: + + +async def _remove_registered_schedule(schedule_name: str) -> None: + logger = _get_logger() + existing_schedule = await dbos.DBOS.get_schedule_async(schedule_name) + if existing_schedule is None: + return + logger.info( + "Scheduled workflows disabled; removing schedule %s so no future cron runs are registered", + schedule_name, + ) + await dbos.DBOS.delete_schedule_async(schedule_name) + + +async def _remove_all_registered_schedules() -> None: + for schedule_input in get_registered_schedule_inputs(): + await _remove_registered_schedule(schedule_input["schedule_name"]) + + +async def register_schedules(scheduler: scheduler_module.Scheduler) -> None: + if not scheduled_workflows_enabled(): + await _remove_all_registered_schedules() + return + for schedule_input in get_registered_schedule_inputs(): await _ensure_schedule(scheduler, schedule_input) @@ -353,6 +380,8 @@ async def update_cleanup_schedule_cron( scheduler: scheduler_module.Scheduler, cron: str, ) -> dict[str, typing.Any]: + if not scheduled_workflows_enabled(): + return {"changed": False, "cron": cron} existing_schedule = await dbos.DBOS.get_schedule_async(dbos_cleanup_workflow.SCHEDULE_NAME) if existing_schedule is not None and existing_schedule["schedule"] == cron: return {"changed": False, "cron": cron} diff --git a/packages/node/octobot_node/scheduler/tasks.py b/packages/node/octobot_node/scheduler/tasks.py index e27bc8ea4d..115e20691c 100644 --- a/packages/node/octobot_node/scheduler/tasks.py +++ b/packages/node/octobot_node/scheduler/tasks.py @@ -37,11 +37,12 @@ async def trigger_user_action_workflow( if not octobot_node.scheduler.is_initialized(): raise RuntimeError("Scheduler is not initialized") import octobot_node.scheduler.workflows.user_action_workflow as user_action_workflow - handle = await octobot_node.scheduler.SCHEDULER.USER_ACTION_QUEUE.enqueue_async( + handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.enqueue_workflow_async( + octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE.value, user_action_workflow.UserActionWorkflow.execute_user_action, inputs=params.UserActionWorkflowInputs( user_id=user_id, user_action=user_action, - ).to_dict(include_default_values=False) + ).to_dict(include_default_values=False), ) return handle.workflow_id @@ -57,7 +58,8 @@ async def trigger_portfolio_history_collection( workflow_context = None if collection_params is not None: workflow_context = collection_params.to_dict(include_default_values=False) - handle = await octobot_node.scheduler.SCHEDULER.PORTFOLIO_HISTORY_QUEUE.enqueue_async( + handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.enqueue_workflow_async( + octobot_node.enums.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value, portfolio_history_workflow.PortfolioHistoryWorkflow.portfolio_history_collection, scheduled_time, workflow_context, @@ -78,14 +80,16 @@ async def trigger_task( inputs = params.AutomationWorkflowInputs(task=task).to_dict(include_default_values=False) if target_workflow_id: with octobot_node.scheduler.SCHEDULER.SetWorkflowID(target_workflow_id): - handle = await octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE.enqueue_async( + handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.enqueue_workflow_async( + octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value, automation_workflow.AutomationWorkflow.execute_automation, - inputs=inputs + inputs=inputs, ) else: - handle = await octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE.enqueue_async( + handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.enqueue_workflow_async( + octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value, automation_workflow.AutomationWorkflow.execute_automation, - inputs=inputs + inputs=inputs, ) else: raise ValueError(f"Unsupported task type: {task.type}") diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py index 3245162126..85bb648ed9 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py @@ -24,6 +24,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater +import octobot.community.node_journal as node_journal + def _get_create_account_payload( user_action: protocol_models.UserAction, @@ -66,3 +68,8 @@ async def _do_execute( ), ) self._mark_user_action_completed(user_action) + node_journal.record_account_validated_from_account( + checked_account, + self._user_id, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/delete_account.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/delete_account.py index ece68d5934..363f3c555f 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/delete_account.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/delete_account.py @@ -20,6 +20,8 @@ import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor +import octobot.community.node_journal as node_journal + def _get_delete_account_payload( user_action: protocol_models.UserAction, @@ -43,8 +45,13 @@ async def _do_execute( user_action: protocol_models.UserAction, ) -> None: delete_payload = _get_delete_account_payload(user_action) - collection_providers.AccountProvider.instance().delete_item( + account_provider = collection_providers.AccountProvider.instance() + account_provider.delete_item( self._user_id, delete_payload.id, ) self._mark_user_action_completed(user_action) + node_journal.record_account_deleted( + account_id=delete_payload.id or "", + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/edit_account.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/edit_account.py index 693a36e5ea..862e61d841 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/edit_account.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/edit_account.py @@ -21,6 +21,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater +import octobot.community.node_journal as node_journal + def _get_edit_account_payload( user_action: protocol_models.UserAction, @@ -61,3 +63,7 @@ async def _do_execute( checked_account, ) self._mark_user_action_completed(user_action) + node_journal.record_account_edit_succeeded( + account_id=checked_account.id or "", + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/refresh_accounts.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/refresh_accounts.py index df4b206429..03c6f7cb4c 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/refresh_accounts.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/refresh_accounts.py @@ -21,6 +21,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater +import octobot.community.node_journal as node_journal + def _get_refresh_accounts_payload( user_action: protocol_models.UserAction, @@ -45,11 +47,18 @@ async def _do_execute( ) -> None: refresh_payload = _get_refresh_accounts_payload(user_action) account_provider = collection_providers.AccountProvider.instance() - account_ids_to_refresh = refresh_payload.account_ids or [ - account.id for account in account_provider.list_items(self._user_id) - ] + if refresh_payload.account_ids: + account_ids_to_refresh = refresh_payload.account_ids + cached_all_accounts = None + else: + cached_all_accounts = account_provider.list_items(self._user_id) + account_ids_to_refresh = [account.id for account in cached_all_accounts] for account_id in account_ids_to_refresh: account = account_provider.get_item(self._user_id, account_id) checked_account = await account_state_updater.update_account_state(account, self._user_id) account_provider.update_item(self._user_id, checked_account) self._mark_user_action_completed(user_action) + node_journal.record_accounts_refreshed( + account_ids=list(account_ids_to_refresh), + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/create_account_auth.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/create_account_auth.py index 2e442234ba..f7dc2c163c 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/create_account_auth.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/create_account_auth.py @@ -20,6 +20,8 @@ import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.account_auth.account_auth_user_action_executor as account_auth_user_action_executor +import octobot.community.node_journal as node_journal + def _get_create_account_auth_payload( user_action: protocol_models.UserAction, @@ -51,3 +53,7 @@ async def _do_execute( account_auth_user_action_executor.with_updated_at(create_payload.configuration), ) self._mark_user_action_completed(user_action) + node_journal.record_account_auth_create_succeeded( + exchange_name=None, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/delete_account_auth.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/delete_account_auth.py index 4f51d704b0..4955735c1e 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/delete_account_auth.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/delete_account_auth.py @@ -20,6 +20,8 @@ import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.account_auth.account_auth_user_action_executor as account_auth_user_action_executor +import octobot.community.node_journal as node_journal + def _get_delete_account_auth_payload( user_action: protocol_models.UserAction, @@ -51,3 +53,7 @@ async def _do_execute( delete_payload.id, ) self._mark_user_action_completed(user_action) + node_journal.record_account_auth_deleted( + exchange_name=None, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py index e8b645487a..b6d34e9539 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py @@ -26,6 +26,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.automation.automation_user_action_executor as automation_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.util.action_details_factory as action_details_factory +import octobot.community.node_journal as node_journal + import octobot_sync.sync.collection_backend.errors as collection_errors import octobot_sync.sync.collection_providers as collection_providers @@ -117,14 +119,24 @@ async def _do_execute( user_action: protocol_models.UserAction, ) -> None: actions = self._create_automation_actions(user_action) - task = await self._create_automation_task(user_action, actions) + automation_id, task = await self._create_automation_task(user_action, actions) self.post_actions.to_create_automation_task = task self._mark_user_action_completed( user_action, created_automation_id=task.id ) + automation_configuration = self._get_automation_configuration(user_action) + stored_strategy = _load_strategy_for_automation( + self._user_id, + automation_configuration.strategy, + ) + node_journal.record_new_automation_created_from_strategy( + automation_id, + stored_strategy, + user_action_id=user_action.id, + ) - async def _create_automation_task(self, user_action, actions: list[flow_entities.AbstractActionDetails]) -> models.Task: + async def _create_automation_task(self, user_action, actions: list[flow_entities.AbstractActionDetails]) -> tuple[str, models.Task]: automation_configuration = self._get_automation_configuration(user_action) if automation_configuration.id: _validate_automation_configuration_id(automation_configuration.id) @@ -137,7 +149,7 @@ async def _create_automation_task(self, user_action, actions: list[flow_entities } if automation_configuration.id: task_fields["id"] = automation_configuration.id - return models.Task(**task_fields) + return automation_id, models.Task(**task_fields) def _get_automation_configuration(self, user_action: protocol_models.UserAction) -> protocol_models.AutomationConfiguration: create_payload = _get_create_automation_payload(user_action) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py index 2e3aefb58a..a2f191615b 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py @@ -29,6 +29,8 @@ import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader import octobot_node.scheduler.workflows_util as workflows_util +import octobot.community.node_journal as node_journal + def _get_restart_automation_payload( user_action: protocol_models.UserAction, @@ -166,3 +168,7 @@ async def _do_execute( user_action, created_automation_id=parent_automation_id, ) + node_journal.record_automation_restarted( + automation_id=parent_automation_id, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/stop_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/stop_automation.py index 26c7625263..ac3de708ee 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/stop_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/stop_automation.py @@ -21,6 +21,8 @@ import octobot_node.scheduler as scheduler_module import octobot_node.scheduler.tasks as scheduler_tasks +import octobot.community.node_journal as node_journal + def _get_stop_automation_payload( user_action: protocol_models.UserAction, @@ -71,3 +73,8 @@ async def _do_execute( actions, ) self._mark_user_action_completed(user_action) + node_journal.record_automation_stopped( + automation_id=stop_payload.id, + cancel_orders=bool(stop_payload.cancel_orders), + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/base_user_action_executor.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/base_user_action_executor.py index df92edadb5..3cc286460e 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/base_user_action_executor.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/base_user_action_executor.py @@ -21,6 +21,8 @@ import octobot_node.scheduler.user_actions.user_action_post_actions as user_action_post_actions +import octobot.community.node_journal as node_journal + class UserActionExecutor(abc.ABC): """ @@ -92,6 +94,10 @@ async def execute( f"User action execution failed: {user_action.id}: {exc} ({exc.__class__.__name__})" ) self._apply_execution_failure(user_action, exc) + node_journal.record_executor_failure( + user_action, + error=exc, + ) raise finally: await self.after_execute(user_action) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py index 792fda9cc6..6750a0e937 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py @@ -21,6 +21,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as strategy_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_profile_validation as strategy_profile_validation +import octobot.community.node_journal as node_journal + def _get_create_strategy_payload( user_action: protocol_models.UserAction, @@ -55,3 +57,8 @@ async def _do_execute( create_payload.configuration, ) self._mark_user_action_completed(user_action) + node_journal.record_strategy_create_succeeded( + strategy_id=create_payload.configuration.id, + strategy=create_payload.configuration, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py index 1e0e0765b2..04fee35649 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py @@ -21,6 +21,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as strategy_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_profile_validation as strategy_profile_validation +import octobot.community.node_journal as node_journal + def _get_edit_strategy_payload( user_action: protocol_models.UserAction, @@ -63,3 +65,8 @@ async def _do_execute( edit_payload.configuration, ) self._mark_user_action_completed(user_action) + node_journal.record_strategy_edit_succeeded( + strategy_id=edit_payload.configuration.id, + strategy=edit_payload.configuration, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py index 0e38c25160..8b5cd7e33b 100644 --- a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py @@ -43,10 +43,9 @@ import octobot_node.scheduler.automations.signal_execution_result_util as signal_execution_result_util import octobot_node.errors as errors -from octobot_node.scheduler import SCHEDULER # avoid circular import - -WORKFLOW_NAME = "execute_automation" +import octobot.community.node_journal as node_journal +from octobot_node.scheduler import SCHEDULER # avoid circular import @dataclasses.dataclass class _IterationExecutionState: @@ -63,7 +62,7 @@ class AutomationWorkflow: # Always use dict as input to parse minimizable dataclasses and facilitate data format updates @staticmethod - @SCHEDULER.INSTANCE.workflow(name=WORKFLOW_NAME) + @SCHEDULER.INSTANCE.workflow(name=octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value) async def execute_automation(inputs: dict) -> typing.Optional[str]: """ Automation workflow runner: @@ -73,7 +72,9 @@ async def execute_automation(inputs: dict) -> typing.Optional[str]: 4. Either: A. Reschedule the next iteration as a child workflow to avoid growing the workflow forever. B. Complete the workflow and stop the automation. - 5. If completed, return tthe updated task.content (the automation state) as workflow output + 5. If completed, return the updated task.content (the automation state) as workflow output. + On unexpected error, output state is best-effort: latest iteration description if any, + else workflow input task.content from the previous iteration. """ output: typing.Optional[params.AutomationWorkflowOutput] = None iteration_result = None @@ -104,8 +105,9 @@ async def execute_automation(inputs: dict) -> typing.Optional[str]: AutomationWorkflow.get_logger(parsed_inputs).info( f"Automation stopped (remaining steps: {iteration_result.progress_status.remaining_steps})" ) - final_state = iteration_result.next_iteration_description - final_state_metadata = iteration_result.next_iteration_description_metadata + final_state, final_state_metadata = AutomationWorkflow._best_effort_workflow_output_state( + parsed_inputs, iteration_result + ) final_error = iteration_result.progress_status.error if final_state is not None or final_error is not None: output = params.AutomationWorkflowOutput( @@ -115,31 +117,61 @@ async def execute_automation(inputs: dict) -> typing.Optional[str]: error_message=iteration_result.progress_status.error_message, ) except Exception as err: - workflow_logger = ( - AutomationWorkflow.get_logger(parsed_inputs) - if parsed_inputs is not None - else octobot_commons.logging.get_logger(AutomationWorkflow.__name__) - ) - workflow_logger.exception( + AutomationWorkflow.get_logger(parsed_inputs).exception( err, True, f"Interrupted workflow: unexpected critical error: {err} ({err.__class__.__name__})" ) - await AutomationWorkflow._send_signal_execution_result_safe( + output = await AutomationWorkflow._finalize_interrupted_workflow_output( parsed_inputs, actions_update, - [], - iteration_error=AutomationWorkflow._get_failed_error_status(err), - iteration_error_message=str(err), - ) - output = params.AutomationWorkflowOutput( - # use available iteration result when possible (might be the one of the previous iteration) - state=iteration_result.next_iteration_description if iteration_result else None, - state_metadata=iteration_result.next_iteration_description_metadata if iteration_result else None, - # keep track of the failed iteration - error=AutomationWorkflow._get_failed_error_status(err), + iteration_result, + error_status=AutomationWorkflow._get_failed_error_status(err), error_message=str(err), ) return json.dumps(output.to_dict(include_default_values=False)) if output else None + @staticmethod + def _best_effort_workflow_output_state( + parsed_inputs: typing.Optional[params.AutomationWorkflowInputs], + iteration_result: typing.Optional[params.AutomationWorkflowIterationResult], + ) -> tuple[typing.Optional[str], typing.Optional[str]]: + if ( + iteration_result is not None + and iteration_result.next_iteration_description is not None + ): + return ( + iteration_result.next_iteration_description, + iteration_result.next_iteration_description_metadata, + ) + if parsed_inputs is not None and parsed_inputs.task.content: + return parsed_inputs.task.content, parsed_inputs.task.content_metadata + return None, None + + @staticmethod + async def _finalize_interrupted_workflow_output( + parsed_inputs: typing.Optional[params.AutomationWorkflowInputs], + actions_update: typing.Optional[dict], + iteration_result: typing.Optional[params.AutomationWorkflowIterationResult], + *, + error_status: str, + error_message: str, + ) -> params.AutomationWorkflowOutput: + await AutomationWorkflow._send_signal_execution_result_safe( + parsed_inputs, + actions_update, + [], + iteration_error=error_status, + iteration_error_message=error_message, + ) + state, state_metadata = AutomationWorkflow._best_effort_workflow_output_state( + parsed_inputs, iteration_result + ) + return params.AutomationWorkflowOutput( + state=state, + state_metadata=state_metadata, + error=error_status, + error_message=error_message, + ) + @staticmethod def _should_retry(error: BaseException) -> bool: return not isinstance(error, ( @@ -158,6 +190,7 @@ def _should_retry(error: BaseException) -> bool: max_attempts=constants.AUTOMATION_WORKFLOW_MAX_ITERATION_RETRIES, backoff_rate=constants.AUTOMATION_WORKFLOW_BACKOFF_RATE, should_retry=_should_retry, + preemptible=True, # cancelling workflow will cancel the iteration ) async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) -> dict: """ @@ -431,6 +464,17 @@ def _finalize_postponed_iteration( f"Iteration postponed ({iteration_state.execution_error}: {iteration_state.execution_error_message}), " f"retry scheduled in {retry_delay_seconds:.0f} seconds" ) + if iteration_state.execution_error is not None: + AutomationWorkflow._record_automation_run_errored( + parsed_inputs, + error_status=iteration_state.execution_error, + error_origin="postponed_iteration", + error=Exception( + iteration_state.execution_error_message + or iteration_state.execution_error + ), + retriable=True, + ) @staticmethod async def _send_signal_execution_result_safe( @@ -449,12 +493,7 @@ async def _send_signal_execution_result_safe( iteration_error_message=iteration_error_message, ) except Exception: - workflow_logger = ( - AutomationWorkflow.get_logger(parsed_inputs) - if parsed_inputs is not None - else octobot_commons.logging.get_logger(AutomationWorkflow.__name__) - ) - workflow_logger.exception( + AutomationWorkflow.get_logger(parsed_inputs).exception( "Failed to send signal execution result callback", exc_info=True, ) @@ -636,9 +675,10 @@ async def _schedule_next_iteration( ) next_workflow_id = AutomationWorkflow._get_next_child_workflow_id() with SCHEDULER.SetWorkflowID(next_workflow_id): - await SCHEDULER.AUTOMATION_WORKFLOW_QUEUE.enqueue_async( + await SCHEDULER.INSTANCE.enqueue_workflow_async( + octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value, AutomationWorkflow.execute_automation, - inputs=next_iteration_inputs + inputs=next_iteration_inputs, ) @staticmethod @@ -683,6 +723,13 @@ def _should_continue_workflow( f"Automation stopped: unrecoverable iteration error: {progress_status.error}. " f"Iteration's last step: {progress_status.latest_step}" ) + AutomationWorkflow._record_automation_run_errored( + parsed_inputs, + error_status=progress_status.error, + error_origin="terminal_iteration", + error=Exception(progress_status.error_message or progress_status.error), + retriable=False, + ) return stop_on_error elif progress_status.should_stop: AutomationWorkflow.get_logger(parsed_inputs).info( @@ -695,6 +742,34 @@ def _should_continue_workflow( def _get_actions_summary(actions: list["octobot_flow.entities.AbstractActionDetails"], minimal: bool = False) -> str: return ", ".join([action.get_summary(minimal=minimal) for action in actions]) if actions else "" + @staticmethod + def _resolve_automation_id(parsed_inputs: params.AutomationWorkflowInputs) -> str | None: + try: + automation_state = automation_states_loader.get_automation_dict(parsed_inputs.task.content) + return automation_state.get("automation", {}).get("metadata", {}).get("automation_id") + except ValueError: + return None + + @staticmethod + def _record_automation_run_errored( + parsed_inputs: params.AutomationWorkflowInputs, + *, + error_status: str, + error_origin: str, + error: BaseException, + retriable: bool, + ) -> None: + automation_id = AutomationWorkflow._resolve_automation_id(parsed_inputs) + if automation_id is None: + return + node_journal.record_automation_run_errored( + automation_id=automation_id, + error_status=error_status, + error_origin=error_origin, + error=error, + retriable=retriable, + ) + @staticmethod def _get_failed_error_status(error: Exception) -> str: if isinstance(error, errors.WorkflowActionExecutionError): @@ -706,10 +781,14 @@ def _get_failed_error_status(error: Exception) -> str: return octobot_flow.enums.AutomationWorkflowErrorStatus.EXCEPTION_DURING_ITERATION.value @staticmethod - def get_logger(parsed_inputs: params.AutomationWorkflowInputs) -> octobot_commons.logging.BotLogger: - return octobot_commons.logging.get_logger( - parsed_inputs.task.name or AutomationWorkflow.__name__ - ) + def get_logger( + parsed_inputs: typing.Optional[params.AutomationWorkflowInputs], + ) -> octobot_commons.logging.BotLogger: + if parsed_inputs is not None: + return octobot_commons.logging.get_logger( + parsed_inputs.task.name or AutomationWorkflow.__name__ + ) + return octobot_commons.logging.get_logger(AutomationWorkflow.__name__) @staticmethod def _get_postponed_iteration_error_status_and_delay(error: Exception) -> tuple[ diff --git a/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py b/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py index 7cd212926b..458927ffeb 100644 --- a/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py @@ -24,7 +24,6 @@ from octobot_node.scheduler import SCHEDULER # avoid circular import -WORKFLOW_NAME = "dbos_cleanup" SCHEDULE_NAME = "dbos_cleanup_daily" SCHEDULE_CRON = "0 0 * * *" # daily at midnight UTC @@ -32,7 +31,7 @@ @SCHEDULER.INSTANCE.dbos_class() class DbosCleanupWorkflow: @staticmethod - @SCHEDULER.INSTANCE.workflow(name=WORKFLOW_NAME) + @SCHEDULER.INSTANCE.workflow(name=octobot_node.enums.SchedulerWorkflowNames.DBOS_CLEANUP.value) async def dbos_cleanup( scheduled_time: datetime.datetime, context: typing.Any, diff --git a/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py b/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py index e8cead645c..89fdc0db00 100644 --- a/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py @@ -33,7 +33,6 @@ from octobot_node.scheduler import SCHEDULER # avoid circular import -WORKFLOW_NAME = "global_view_refresh" SCHEDULE_NAME = "global_view_refresh_every_5m" SCHEDULE_CRON = "*/5 * * * *" # every 5 minutes @@ -86,7 +85,7 @@ def _log_successful_account_refresh( @SCHEDULER.INSTANCE.dbos_class() class GlobalViewRefreshWorkflow: @staticmethod - @SCHEDULER.INSTANCE.workflow(name=WORKFLOW_NAME) + @SCHEDULER.INSTANCE.workflow(name=octobot_node.enums.SchedulerWorkflowNames.GLOBAL_VIEW_REFRESH.value) async def global_view_refresh( scheduled_time: datetime.datetime, context: typing.Any, diff --git a/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py b/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py index 3c0e06494a..1da0766c5c 100644 --- a/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py @@ -11,7 +11,6 @@ import octobot_node.enums from octobot_node.scheduler import SCHEDULER -WORKFLOW_NAME = "portfolio_history_collection" SCHEDULE_NAME = "portfolio_history_daily_3am" SCHEDULE_CRON = "0 3 * * *" # 3:00 AM every day @@ -39,7 +38,7 @@ def _parse_collection_params( @SCHEDULER.INSTANCE.dbos_class() class PortfolioHistoryWorkflow: @staticmethod - @SCHEDULER.INSTANCE.workflow(name=WORKFLOW_NAME) + @SCHEDULER.INSTANCE.workflow(name=octobot_node.enums.SchedulerWorkflowNames.PORTFOLIO_HISTORY_COLLECTION.value) async def portfolio_history_collection( scheduled_time: datetime.datetime, context: typing.Any, diff --git a/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py b/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py index c9e41d038d..de30b9bd7e 100644 --- a/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py @@ -74,7 +74,7 @@ async def _recv_signal_execution_result( @SCHEDULER.INSTANCE.dbos_class() class UserActionWorkflow: @staticmethod - @SCHEDULER.INSTANCE.workflow(name="execute_user_action") + @SCHEDULER.INSTANCE.workflow(name=node_enums.SchedulerWorkflowNames.EXECUTE_USER_ACTION.value) async def execute_user_action(inputs: dict) -> dict: dispatch_output = await UserActionWorkflow._dispatch_user_action(inputs) parsed_dispatch_output = params.UserActionExecutionResult.from_dict(dispatch_output) diff --git a/packages/node/octobot_node/scheduler/workflows_retention.py b/packages/node/octobot_node/scheduler/workflows_retention.py index 7e8251fe86..631b4a1f1e 100644 --- a/packages/node/octobot_node/scheduler/workflows_retention.py +++ b/packages/node/octobot_node/scheduler/workflows_retention.py @@ -67,7 +67,7 @@ _TERMINAL_WORKFLOW_STATUS_VALUES = frozenset( workflow_status.value - for workflow_status in workflows_util.get_user_action_terminal_workflow_statuses() + for workflow_status in workflows_util.DBOS_TERMINAL_WORKFLOW_STATUSES ) def should_skip_retention_cleanup_on_this_node() -> bool: @@ -240,31 +240,37 @@ async def cleanup_outdated_automation_executions( retention_seconds, database_size_bytes, ) - import octobot_node.scheduler.workflows.automation_workflow as automation_workflow - import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow - import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow - import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow - automation_workflows = await scheduler.INSTANCE.list_workflows_async( - name=automation_workflow.WORKFLOW_NAME, - queue_name=[octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], - load_input=False, + automation_workflows = await workflows_util.list_scheduler_workflows_async( + scheduler.INSTANCE, + octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + None, + None, load_output=False, - ) - cleanup_workflows = await scheduler.INSTANCE.list_workflows_async( - name=dbos_cleanup_workflow.WORKFLOW_NAME, load_input=False, - load_output=False, ) - global_view_workflows = await scheduler.INSTANCE.list_workflows_async( - name=global_view_workflow.WORKFLOW_NAME, - queue_name=[octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value], - load_input=False, + cleanup_workflows = await workflows_util.list_scheduler_workflows_async( + scheduler.INSTANCE, + octobot_node.enums.SchedulerWorkflowNames.DBOS_CLEANUP, + None, + None, load_output=False, + load_input=False, ) - portfolio_history_workflows = await scheduler.INSTANCE.list_workflows_async( - name=portfolio_history_workflow.WORKFLOW_NAME, + global_view_workflows = await workflows_util.list_scheduler_workflows_async( + scheduler.INSTANCE, + octobot_node.enums.SchedulerWorkflowNames.GLOBAL_VIEW_REFRESH, + None, + None, + load_output=False, load_input=False, + ) + portfolio_history_workflows = await workflows_util.list_scheduler_workflows_async( + scheduler.INSTANCE, + octobot_node.enums.SchedulerWorkflowNames.PORTFOLIO_HISTORY_COLLECTION, + None, + None, load_output=False, + load_input=False, ) deletions_by_automation = get_outdated_automation_execution_deletions( automation_workflows, @@ -369,14 +375,15 @@ async def should_skip_retention_cleanup_for_scheduled_time( ) -> bool: if not scheduler.is_initialized(): return True - import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow - cleanup_workflows = await scheduler.INSTANCE.list_workflows_async( - name=dbos_cleanup_workflow.WORKFLOW_NAME, - status=[dbos.WorkflowStatusString.SUCCESS.value], + cleanup_workflows = await workflows_util.list_scheduler_workflows_async( + scheduler.INSTANCE, + octobot_node.enums.SchedulerWorkflowNames.DBOS_CLEANUP, + [dbos.WorkflowStatusString.SUCCESS], + None, + load_output=False, + load_input=False, sort_desc=True, limit=1, - load_input=False, - load_output=False, ) latest_timestamp_ms = _get_latest_completed_cleanup_timestamp_ms(cleanup_workflows) if latest_timestamp_ms == 0: @@ -392,8 +399,7 @@ def _get_sqlite_database_size_bytes() -> int | None: _get_logger().exception( missing_file_error, True, - "Scheduler sqlite file not found: %s", - sqlite_path, + f"Scheduler sqlite file not found: {sqlite_path}", ) return None total_size_bytes = sqlite_path.stat().st_size diff --git a/packages/node/octobot_node/scheduler/workflows_util.py b/packages/node/octobot_node/scheduler/workflows_util.py index 0b95fd57cc..506b4ff045 100644 --- a/packages/node/octobot_node/scheduler/workflows_util.py +++ b/packages/node/octobot_node/scheduler/workflows_util.py @@ -27,7 +27,9 @@ logger = octobot_commons.logging.get_logger("octobot_node.scheduler.workflows_util") -_USER_ACTION_TERMINAL_WORKFLOW_STATUSES = ( +_SCHEDULER_HYDRATE_WORKFLOW_IDS_CHUNK_SIZE = 200 + +DBOS_TERMINAL_WORKFLOW_STATUSES = ( dbos_lib.WorkflowStatusString.SUCCESS, dbos_lib.WorkflowStatusString.ERROR, dbos_lib.WorkflowStatusString.CANCELLED, @@ -36,7 +38,7 @@ _USER_ACTION_INPUT_WORKFLOW_STATUSES = tuple( workflow_status for workflow_status in dbos_lib.WorkflowStatusString - if workflow_status not in _USER_ACTION_TERMINAL_WORKFLOW_STATUSES + if workflow_status not in DBOS_TERMINAL_WORKFLOW_STATUSES ) @@ -44,8 +46,100 @@ def get_user_action_input_workflow_statuses() -> tuple[dbos_lib.WorkflowStatusSt return _USER_ACTION_INPUT_WORKFLOW_STATUSES -def get_user_action_terminal_workflow_statuses() -> tuple[dbos_lib.WorkflowStatusString, ...]: - return _USER_ACTION_TERMINAL_WORKFLOW_STATUSES +def wallet_filter_queue_for_workflow( + workflow_name: octobot_node_enums.SchedulerWorkflowNames, +) -> typing.Optional[octobot_node_enums.SchedulerQueues]: + if workflow_name == octobot_node_enums.SchedulerWorkflowNames.EXECUTE_USER_ACTION: + return octobot_node_enums.SchedulerQueues.USER_ACTION_QUEUE + if workflow_name == octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION: + return octobot_node_enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE + return None + + +async def list_scheduler_workflows_async( + dbos_instance, + workflow_name: octobot_node_enums.SchedulerWorkflowNames, + statuses: typing.Optional[list[dbos_lib.WorkflowStatusString]], + user_id: typing.Optional[str], + *, + load_output: bool, + load_input: bool = False, + sort_desc: typing.Optional[bool] = None, + limit: typing.Optional[int] = None, + workflow_ids: typing.Optional[list[str]] = None, + workflow_id_prefix: typing.Optional[str | list[str]] = None, + queues_only: bool = False, +) -> list[dbos_lib.WorkflowStatus]: + """ + List DBOS workflows by registered workflow name (not enqueue queue). + + DBOS omits ``workflow_status.input`` / ``.output`` unless ``load_input`` / ``load_output`` are + true. Defaults stay false for cheap scans (retention, counts); callers that parse task content + must pass ``load_input=True`` (and output when needed). Prefer listing metadata first and + bulk-hydrating with ``workflow_ids=`` when only a subset needs payloads. + + When ``queues_only=True``, DBOS lists queued workflows only. Pending-only callers pass + ``statuses=None``. If both ``queues_only=True`` and ``statuses`` are set, both are forwarded + to DBOS (``queues_only`` and mapped ``status``). + """ + list_kwargs: dict[str, typing.Any] = { + "name": workflow_name.value, + "load_output": load_output, + "load_input": load_input, + } + if queues_only: + list_kwargs["queues_only"] = True + if statuses is not None: + list_kwargs["status"] = [status.value for status in statuses] + if sort_desc is not None: + list_kwargs["sort_desc"] = sort_desc + if limit is not None: + list_kwargs["limit"] = limit + if workflow_ids is not None: + list_kwargs["workflow_ids"] = workflow_ids + if workflow_id_prefix is not None: + list_kwargs["workflow_id_prefix"] = workflow_id_prefix + workflows = await dbos_instance.list_workflows_async(**list_kwargs) + wallet_queue = wallet_filter_queue_for_workflow(workflow_name) + if user_id is not None and wallet_queue is not None: + workflows = filter_by_wallet(workflows, user_id, wallet_queue) + return workflows + + +async def hydrate_scheduler_workflows_async( + dbos_instance, + workflow_name: octobot_node_enums.SchedulerWorkflowNames, + workflow_rows: list[dbos_lib.WorkflowStatus], + workflow_ids: list[str], + *, + load_input: bool, + load_output: bool, +) -> None: + """Merge DBOS input/output payloads onto existing metadata rows (in place).""" + if not workflow_ids or not workflow_rows: + return + rows_by_id = {workflow_row.workflow_id: workflow_row for workflow_row in workflow_rows} + ids_to_fetch = [workflow_id for workflow_id in workflow_ids if workflow_id in rows_by_id] + chunk_size = _SCHEDULER_HYDRATE_WORKFLOW_IDS_CHUNK_SIZE + for chunk_start in range(0, len(ids_to_fetch), chunk_size): + chunk_ids = ids_to_fetch[chunk_start : chunk_start + chunk_size] + hydrated_rows = await list_scheduler_workflows_async( + dbos_instance, + workflow_name, + None, + None, + load_output=load_output, + load_input=load_input, + workflow_ids=chunk_ids, + ) + for hydrated_row in hydrated_rows: + metadata_row = rows_by_id.get(hydrated_row.workflow_id) + if metadata_row is None: + continue + if load_input: + metadata_row.input = hydrated_row.input + if load_output: + metadata_row.output = hydrated_row.output @dataclasses.dataclass @@ -246,6 +340,41 @@ def get_workflows_by_parent_id( return by_parent +def resolve_automation_result_for_group( + group: list[dbos_lib.WorkflowStatus], +) -> typing.Optional[dbos_lib.WorkflowStatus]: + """ + Pick the terminal child workflow row used for batch export of one parent automation. + + - Latest child is CANCELLED → that row (state from workflow input, not prior children). + - Else latest SUCCESS child that has DBOS ``output`` (may not be the chronologically latest child). + - Else latest ERROR child by child index. + - Else ``None`` (no row with exportable or error-signalling terminal status). + """ + if not group: + return None + latest = get_latest_workflow(group) + if latest.status == dbos_lib.WorkflowStatusString.CANCELLED.value: + return latest + # Prefer SUCCESS iterations that persisted workflow output over newer failures without output. + success_with_output = [ + workflow_row + for workflow_row in group + if workflow_row.status == dbos_lib.WorkflowStatusString.SUCCESS.value and workflow_row.output + ] + if success_with_output: + return get_latest_workflow(success_with_output) + # No success output: surface the latest ERROR child (export may return error-only payload). + error_rows = [ + workflow_row + for workflow_row in group + if workflow_row.status == dbos_lib.WorkflowStatusString.ERROR.value + ] + if error_rows: + return sorted(error_rows, key=_automation_child_workflow_sort_key)[-1] + return None + + def parse_automation_workflow_output( workflow_status: dbos_lib.WorkflowStatus, ) -> typing.Optional[params.AutomationWorkflowOutput]: diff --git a/packages/node/requirements.txt b/packages/node/requirements.txt index e495a6f636..e2a8998f29 100644 --- a/packages/node/requirements.txt +++ b/packages/node/requirements.txt @@ -1,5 +1,7 @@ -fastapi[standard]==0.135.1 +fastapi==0.141.0 # avoid bundling fastapi CLI with fastapi[standard] +uvicorn # version locked in packages/services +starlette # version locked in packages/services passlib[bcrypt]==1.7.4 pydantic -dbos==2.27.0 +dbos==3.0.0 diff --git a/packages/node/tests/agent_seed/test_demo_wallet.py b/packages/node/tests/agent_seed/test_demo_wallet.py new file mode 100644 index 0000000000..5d048379ed --- /dev/null +++ b/packages/node/tests/agent_seed/test_demo_wallet.py @@ -0,0 +1,248 @@ +# Demo agent-seed wallet guard unit tests. + +import datetime +import uuid + +import mock +import pytest + +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_node.agent_seed.constants as demo_agent_seed_constants +import octobot_node.agent_seed.demo_wallet as demo_agent_seed_wallet + + +def _demo_user_id() -> str: + return demo_agent_seed_constants.DEMO_AGENT_SEED_USER_ID + + +def _user_action_with_configuration( + configuration: protocol_models.UserActionConfiguration, +) -> protocol_models.UserAction: + return protocol_models.UserAction( + id=f"ua-demo-guard-{uuid.uuid4()}", + configuration=configuration, + ) + + +def _minimal_exchange_account() -> protocol_models.ExchangeAccount: + return protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="remote", + exchange_config_ids=[demo_agent_seed_constants.DEMO_AGENT_SEED_EXCHANGE_CONFIG_ID], + ) + + +def _minimal_account(is_simulated: bool) -> protocol_models.Account: + return protocol_models.Account( + id="acct-guard-test", + name="guard-test", + is_simulated=is_simulated, + created_at=datetime.datetime(2026, 6, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 6, 1, tzinfo=datetime.UTC), + specifics=protocol_models.AccountSpecifics(actual_instance=_minimal_exchange_account()), + ) + + +@mock.patch.object(collection_providers.AccountProvider, "instance") +@mock.patch.object(demo_agent_seed_wallet.community_authentication.CommunityAuthentication, "instance") +class TestValidateDemoAgentSeedUserActionBlocked: + def _setup_demo_wallet_auth(self, auth_mock: mock.Mock) -> None: + wallet_mock = mock.Mock(address=demo_agent_seed_constants.DEMO_AGENT_SEED_WALLET_EVM_ADDRESS) + auth_mock.return_value.get_wallet_by_user_id.return_value = wallet_mock + auth_mock.return_value.get_wallet_name.return_value = ( + demo_agent_seed_constants.DEMO_AGENT_SEED_WALLET_DISPLAY_NAME + ) + + def test_blocks_account_auth_create(self, auth_mock, account_provider_mock): + self._setup_demo_wallet_auth(auth_mock) + payload = protocol_models.CreateAccountAuthConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_AUTH_CREATE, + configuration=protocol_models.AccountAuthentication( + id="auth-1", + api_key="key", + api_secret="secret", + ), + ) + user_action = _user_action_with_configuration( + protocol_models.UserActionConfiguration(payload), + ) + with pytest.raises(demo_agent_seed_wallet.DemoAgentSeedUserActionForbiddenError): + demo_agent_seed_wallet.validate_demo_agent_seed_user_action( + _demo_user_id(), + user_action, + ) + + def test_blocks_account_auth_edit(self, auth_mock, account_provider_mock): + self._setup_demo_wallet_auth(auth_mock) + payload = protocol_models.EditAccountAuthConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_AUTH_EDIT, + id="auth-1", + configuration=protocol_models.AccountAuthentication( + id="auth-1", + api_key="key", + api_secret="secret", + ), + ) + user_action = _user_action_with_configuration( + protocol_models.UserActionConfiguration(payload), + ) + with pytest.raises(demo_agent_seed_wallet.DemoAgentSeedUserActionForbiddenError): + demo_agent_seed_wallet.validate_demo_agent_seed_user_action( + _demo_user_id(), + user_action, + ) + + def test_blocks_exchange_config_create(self, auth_mock, account_provider_mock): + self._setup_demo_wallet_auth(auth_mock) + payload = protocol_models.CreateExchangeConfigConfiguration( + action_type=protocol_models.UserActionType.EXCHANGE_CONFIG_CREATE, + configuration=agent_seed_exchange_config(), + ) + user_action = _user_action_with_configuration( + protocol_models.UserActionConfiguration(payload), + ) + with pytest.raises(demo_agent_seed_wallet.DemoAgentSeedUserActionForbiddenError): + demo_agent_seed_wallet.validate_demo_agent_seed_user_action( + _demo_user_id(), + user_action, + ) + + def test_blocks_exchange_config_edit(self, auth_mock, account_provider_mock): + self._setup_demo_wallet_auth(auth_mock) + payload = protocol_models.EditExchangeConfigConfiguration( + action_type=protocol_models.UserActionType.EXCHANGE_CONFIG_EDIT, + id="extra-exchange-config", + configuration=agent_seed_exchange_config(), + ) + user_action = _user_action_with_configuration( + protocol_models.UserActionConfiguration(payload), + ) + with pytest.raises(demo_agent_seed_wallet.DemoAgentSeedUserActionForbiddenError): + demo_agent_seed_wallet.validate_demo_agent_seed_user_action( + _demo_user_id(), + user_action, + ) + + def test_blocks_live_account_create(self, auth_mock, account_provider_mock): + self._setup_demo_wallet_auth(auth_mock) + payload = protocol_models.CreateAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_CREATE, + configuration=_minimal_account(is_simulated=False), + ) + user_action = _user_action_with_configuration( + protocol_models.UserActionConfiguration(payload), + ) + with pytest.raises(demo_agent_seed_wallet.DemoAgentSeedUserActionForbiddenError): + demo_agent_seed_wallet.validate_demo_agent_seed_user_action( + _demo_user_id(), + user_action, + ) + + def test_blocks_account_edit_setting_live(self, auth_mock, account_provider_mock): + self._setup_demo_wallet_auth(auth_mock) + payload = protocol_models.EditAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_EDIT, + id="acct-guard-test", + configuration=_minimal_account(is_simulated=False), + ) + user_action = _user_action_with_configuration( + protocol_models.UserActionConfiguration(payload), + ) + with pytest.raises(demo_agent_seed_wallet.DemoAgentSeedUserActionForbiddenError): + demo_agent_seed_wallet.validate_demo_agent_seed_user_action( + _demo_user_id(), + user_action, + ) + + def test_blocks_automation_create_with_live_account(self, auth_mock, account_provider_mock): + self._setup_demo_wallet_auth(auth_mock) + account_provider_mock.return_value.get_item.return_value = _minimal_account( + is_simulated=False, + ) + payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=protocol_models.AutomationConfiguration( + id="automation-live", + name="live automation", + created_at=datetime.datetime(2026, 6, 1, tzinfo=datetime.UTC), + strategy=protocol_models.StrategyReference( + id=demo_agent_seed_constants.DEMO_AGENT_SEED_STRATEGY_GRID_ID, + version=demo_agent_seed_constants.DEMO_AGENT_SEED_STRATEGY_VERSION, + ), + accounts=[protocol_models.AccountReference(id="acct-guard-test")], + ), + ) + user_action = _user_action_with_configuration( + protocol_models.UserActionConfiguration(payload), + ) + with pytest.raises(demo_agent_seed_wallet.DemoAgentSeedUserActionForbiddenError): + demo_agent_seed_wallet.validate_demo_agent_seed_user_action( + _demo_user_id(), + user_action, + ) + + +def agent_seed_exchange_config() -> protocol_models.ExchangeConfig: + return protocol_models.ExchangeConfig( + id="extra-exchange-config", + name="extra", + exchange="binance", + sandboxed=False, + ) + + +@mock.patch.object(collection_providers.AccountProvider, "instance") +@mock.patch.object(demo_agent_seed_wallet.community_authentication.CommunityAuthentication, "instance") +class TestValidateDemoAgentSeedUserActionAllowed: + def _setup_demo_wallet_auth(self, auth_mock: mock.Mock) -> None: + wallet_mock = mock.Mock(address=demo_agent_seed_constants.DEMO_AGENT_SEED_WALLET_EVM_ADDRESS) + auth_mock.return_value.get_wallet_by_user_id.return_value = wallet_mock + auth_mock.return_value.get_wallet_name.return_value = ( + demo_agent_seed_constants.DEMO_AGENT_SEED_WALLET_DISPLAY_NAME + ) + + def test_allows_simulated_automation_create(self, auth_mock, account_provider_mock): + self._setup_demo_wallet_auth(auth_mock) + account_provider_mock.return_value.get_item.return_value = _minimal_account( + is_simulated=True, + ) + payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=protocol_models.AutomationConfiguration( + id=demo_agent_seed_constants.DEMO_AGENT_SEED_AUTOMATION_GRID_ID, + name=demo_agent_seed_constants.DEMO_AGENT_SEED_GRID_AUTOMATION_DISPLAY_NAME, + created_at=datetime.datetime(2026, 6, 1, tzinfo=datetime.UTC), + strategy=protocol_models.StrategyReference( + id=demo_agent_seed_constants.DEMO_AGENT_SEED_STRATEGY_GRID_ID, + version=demo_agent_seed_constants.DEMO_AGENT_SEED_STRATEGY_VERSION, + ), + accounts=[ + protocol_models.AccountReference( + id=demo_agent_seed_constants.DEMO_AGENT_SEED_ACCOUNT_GRID_ID, + ), + ], + ), + ) + user_action = _user_action_with_configuration( + protocol_models.UserActionConfiguration(payload), + ) + demo_agent_seed_wallet.validate_demo_agent_seed_user_action( + _demo_user_id(), + user_action, + ) + + def test_allows_automation_stop(self, auth_mock, account_provider_mock): + self._setup_demo_wallet_auth(auth_mock) + payload = protocol_models.StopAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_STOP, + id=demo_agent_seed_constants.DEMO_AGENT_SEED_AUTOMATION_GRID_ID, + ) + user_action = _user_action_with_configuration( + protocol_models.UserActionConfiguration(payload), + ) + demo_agent_seed_wallet.validate_demo_agent_seed_user_action( + _demo_user_id(), + user_action, + ) diff --git a/packages/node/tests/conftest.py b/packages/node/tests/conftest.py index b9c2767756..05b43aad56 100644 --- a/packages/node/tests/conftest.py +++ b/packages/node/tests/conftest.py @@ -1,8 +1,15 @@ import contextlib -import os - import mock +import os import pytest +import sys + +_OCTOBOT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +_TESTS_ROOT = os.path.join(_OCTOBOT_ROOT, "tests") +if _TESTS_ROOT not in sys.path: + sys.path.insert(0, _TESTS_ROOT) + +from test_utils.journal_test_support import disabled_node_journal_environment _TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS = 2 _TESTS_RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS = 30.0 @@ -29,6 +36,12 @@ def mocked_local_user_configuration(): yield +@pytest.fixture(autouse=True) +def disable_node_journal(): + with disabled_node_journal_environment(): + yield + + @pytest.fixture(autouse=True) def _mock_local_user_configuration(): with mocked_local_user_configuration(): diff --git a/packages/node/tests/functional_tests/test_account_auth_CRUD_operations.py b/packages/node/tests/functional_tests/test_account_auth_CRUD_operations.py index ab80a3af5d..4ddd8e9697 100644 --- a/packages/node/tests/functional_tests/test_account_auth_CRUD_operations.py +++ b/packages/node/tests/functional_tests/test_account_auth_CRUD_operations.py @@ -41,8 +41,7 @@ def temp_dbos_scheduler_account_auth_crud( """Mirrors ``tests.scheduler.temp_dbos_scheduler``; requests the patch fixture so registration sees capped retries.""" with tempfile.NamedTemporaryFile() as temp_file: dbos_runtime = scheduler_tests.init_scheduler(temp_file.name) - dbos_runtime.reset_system_database() - dbos_runtime.launch() + scheduler_tests.reset_launch_and_register_scheduler_queues(dbos_runtime) try: yield octobot_node.scheduler.SCHEDULER finally: diff --git a/packages/node/tests/functional_tests/test_accounts_CRUD_operations.py b/packages/node/tests/functional_tests/test_accounts_CRUD_operations.py index dabb42905b..821916c3e4 100644 --- a/packages/node/tests/functional_tests/test_accounts_CRUD_operations.py +++ b/packages/node/tests/functional_tests/test_accounts_CRUD_operations.py @@ -55,8 +55,7 @@ def temp_dbos_scheduler_account_crud( """Mirrors ``tests.scheduler.temp_dbos_scheduler``; requests the patch fixture so registration sees capped retries.""" with tempfile.NamedTemporaryFile() as temp_file: dbos_runtime = scheduler_tests.init_scheduler(temp_file.name) - dbos_runtime.reset_system_database() - dbos_runtime.launch() + scheduler_tests.reset_launch_and_register_scheduler_queues(dbos_runtime) try: yield octobot_node.scheduler.SCHEDULER finally: diff --git a/packages/node/tests/functional_tests/test_exchange_configs_CRUD_operations.py b/packages/node/tests/functional_tests/test_exchange_configs_CRUD_operations.py index 3f25e9ab82..2a5b005a09 100644 --- a/packages/node/tests/functional_tests/test_exchange_configs_CRUD_operations.py +++ b/packages/node/tests/functional_tests/test_exchange_configs_CRUD_operations.py @@ -47,8 +47,7 @@ def temp_dbos_scheduler_exchange_config_crud( """Mirrors ``tests.scheduler.temp_dbos_scheduler``; requests the patch fixture so registration sees capped retries.""" with tempfile.NamedTemporaryFile() as temp_file: dbos_runtime = scheduler_tests.init_scheduler(temp_file.name) - dbos_runtime.reset_system_database() - dbos_runtime.launch() + scheduler_tests.reset_launch_and_register_scheduler_queues(dbos_runtime) try: yield octobot_node.scheduler.SCHEDULER finally: diff --git a/packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py b/packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py index b6f316a922..6ffa2a977f 100644 --- a/packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py +++ b/packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py @@ -131,6 +131,15 @@ def _functional_seed_strategy_for_outdated_reference_test(_wallet_address, store _T_POLL_SECONDS, ) ) + fresh_delivery_task = asyncio.create_task( + outdated_reference_workflow_module.deliver_fresh_trading_signal_after_outdated_skip( + temp_dbos_scheduler, + caplog, + automation_id, + fresh_signal, + _T_POLL_SECONDS, + ) + ) # Step 1 — Enqueue AUTOMATION_CREATE for copy follower; expect COMPLETED create result. try: @@ -153,17 +162,8 @@ def _functional_seed_strategy_for_outdated_reference_test(_wallet_address, store expected_workflow_id=None, ) - # Step 2 — Stale signal delivered while copy workflow was pending; expect outdated skip log. - stale_skip_deadline = time.monotonic() + _T_POLL_SECONDS - while time.monotonic() < stale_skip_deadline: - if outdated_reference_workflow_module.caplog_contains_outdated_skip(caplog): - break - await asyncio.sleep(workflow_common_module.DEFAULT_GRID_WORKFLOW_POLL_INTERVAL_SECONDS) - else: - pytest.fail( - "Timed out waiting for outdated reference account skip log " - f"({outdated_reference_workflow_module.OUTDATED_SKIP_LOG_SUBSTRING!r})" - ) + # Step 2 — Stale skip + fresh delivery race the next child recv; fresh task started above. + await fresh_delivery_task await outdated_reference_workflow_module.poll_state_reader_until( temp_dbos_scheduler, @@ -173,14 +173,6 @@ def _functional_seed_strategy_for_outdated_reference_test(_wallet_address, store _T_POLL_SECONDS, "stale sell limit not mirrored", ) - - # Step 3 — Send fresh TradingSignal after child workflow is pending again; expect mirror at fresh price. - await outdated_reference_workflow_module.deliver_trading_signal_when_pending( - temp_dbos_scheduler, - automation_id, - fresh_signal, - _T_POLL_SECONDS, - ) fresh_reader = await outdated_reference_workflow_module.poll_state_reader_until( temp_dbos_scheduler, automation_id, diff --git a/packages/node/tests/functional_tests/test_schedule_startup_backfill_workflow.py b/packages/node/tests/functional_tests/test_schedule_startup_backfill_workflow.py index a8d93342c7..a984cb827c 100644 --- a/packages/node/tests/functional_tests/test_schedule_startup_backfill_workflow.py +++ b/packages/node/tests/functional_tests/test_schedule_startup_backfill_workflow.py @@ -93,16 +93,8 @@ def _seed_stale_latest_application_version(stale_version_name: str) -> None: ) -def _get_or_create_registry_queue(queue_name: str, **queue_options) -> dbos.Queue: - # destroy_launched_dbos() keeps the global registry; reuse an existing Queue - # declaration when this fixture runs more than once on the same xdist worker. - import dbos._dbos as dbos_internals - - registry = dbos_internals._get_or_create_dbos_registry() - existing_queue = registry.queue_info_map.get(queue_name) - if existing_queue is not None: - return existing_queue - return dbos.Queue(name=queue_name, **queue_options) +def _register_blank_backfill_queue() -> None: + dbos.DBOS.register_queue(_BLANK_QUEUE_NAME, global_concurrency=1) @pytest.fixture @@ -114,10 +106,10 @@ def temp_dbos_scheduler_backfill(): temp_file.name, _TEST_APP_VERSION, ) - _get_or_create_registry_queue(_BLANK_QUEUE_NAME, concurrency=1) blank_workflow_fn = _register_blank_backfill_workflow() dbos_runtime.reset_system_database() dbos_runtime.launch() + _register_blank_backfill_queue() try: yield blank_workflow_fn finally: diff --git a/packages/node/tests/functional_tests/test_signal_automation_user_action.py b/packages/node/tests/functional_tests/test_signal_automation_user_action.py index 80672e2bd3..9f42e856c0 100644 --- a/packages/node/tests/functional_tests/test_signal_automation_user_action.py +++ b/packages/node/tests/functional_tests/test_signal_automation_user_action.py @@ -33,8 +33,7 @@ def temp_dbos_scheduler_signal_automation_user_action( ): with tempfile.NamedTemporaryFile() as temp_file: dbos_runtime = scheduler_tests.init_scheduler(temp_file.name) - dbos_runtime.reset_system_database() - dbos_runtime.launch() + scheduler_tests.reset_launch_and_register_scheduler_queues(dbos_runtime) try: yield octobot_node.scheduler.SCHEDULER finally: diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_dca_coinrabbit_ticker_wise.py b/packages/node/tests/functional_tests/test_start_check_and_stop_dca_coinrabbit_ticker_wise.py new file mode 100644 index 0000000000..a12ef5e601 --- /dev/null +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_dca_coinrabbit_ticker_wise.py @@ -0,0 +1,306 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import asyncio +import pathlib +import sys +import time +import typing + +import dbos +import mock +import pytest + +import octobot_protocol.models as octobot_protocol_models +import octobot_trading.enums as trading_enums_module +import octobot_trading.errors as trading_errors_module + +import octobot.community.authentication as community_authentication_module +import octobot_flow.repositories.exchange as octobot_flow_repositories_exchange_module +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module + +from tests.scheduler import temp_dbos_scheduler + +from .util import authenticator_mocks as authenticator_mocks_module +from .util import coinrabbit_dca_workflow as coinrabbit_dca_util +from .util import exchange_account_elements_access as exchange_account_elements_access_module +from .util import price_mocks as price_mocks_module +from .util import user_action_assertions as user_action_assertions_module +from .util import workflow_common as workflow_common_module + +_FLOW_TESTS_ROOT = pathlib.Path(__file__).resolve().parents[3] / "flow" / "tests" +if str(_FLOW_TESTS_ROOT) not in sys.path: + sys.path.insert(0, str(_FLOW_TESTS_ROOT)) + +from functionnal_tests.trading_modes_actions.simulator import ( # noqa: E402 + coinrabbit_ticker_wise_test_util as coinrabbit_price_util, +) + +_T_ENQUEUE_SECONDS = 30.0 +_T_DCA_BUY_SECONDS = 180.0 +_T_STOP_SEND_SECONDS = 15.0 +_T_STOP_COMPLETE_SECONDS = 30.0 + +_DCA_ACCOUNT_ID = "functional_coinrabbit_dca_ticker_wise_account" +_DCA_AUTOMATION_DISPLAY_NAME = "test_coinrabbit_dca_ticker_wise_automation" + + +@pytest.fixture +def skip_on_exchange_proxy_error(request): + class _SkipOnExchangeProxyErrorPlugin: + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport(self, item, call): + outcome = yield + report = outcome.get_result() + if item is not request.node: + return + if call.when != "call" or call.excinfo is None: + return + if not isinstance(call.excinfo.value, trading_errors_module.ExchangeProxyError): + return + skip_reason = f"CoinRabbit exchange proxy unavailable: {call.excinfo.value}" + report.outcome = "skipped" + line_number = item.location[1] if item.location else 0 + report.longrepr = (str(item.path), line_number, f"Skipped: {skip_reason}") + + plugin = _SkipOnExchangeProxyErrorPlugin() + request.config.pluginmanager.register(plugin) + yield + request.config.pluginmanager.unregister(plugin) + + +def _portfolio_reference_total(elements: typing.Any) -> float: + content = exchange_account_elements_access_module.portfolio_content_from_elements(elements) + row = content.get(coinrabbit_dca_util.REFERENCE_MARKET, {}) + if not row: + return 0.0 + return exchange_account_elements_access_module.portfolio_row_scalar( + row, + "total", + ) + + +def _portfolio_btc_total(elements: typing.Any) -> float: + content = exchange_account_elements_access_module.portfolio_content_from_elements(elements) + row = content.get("BTC@BTC", {}) + if not row: + return 0.0 + return exchange_account_elements_access_module.portfolio_row_scalar( + row, + "total", + ) + + +def _has_filled_market_buy_on_traded_symbol(elements: typing.Any) -> bool: + symbol_key = trading_enums_module.ExchangeConstantsOrderColumns.SYMBOL.value + side_key = trading_enums_module.ExchangeConstantsOrderColumns.SIDE.value + type_key = trading_enums_module.ExchangeConstantsOrderColumns.TYPE.value + status_key = trading_enums_module.ExchangeConstantsOrderColumns.STATUS.value + for trade_row in exchange_account_elements_access_module.trades_from_elements(elements): + payload = exchange_account_elements_access_module.order_storage_payload(trade_row) + if payload.get(symbol_key) != coinrabbit_dca_util.TRADED_SYMBOL: + continue + if payload.get(side_key) == trading_enums_module.TradeOrderSide.BUY.value: + return True + for order_row in exchange_account_elements_access_module.open_orders_from_elements(elements): + payload = exchange_account_elements_access_module.order_storage_payload(order_row) + if payload.get(symbol_key) != coinrabbit_dca_util.TRADED_SYMBOL: + continue + if payload.get(side_key) != trading_enums_module.TradeOrderSide.BUY.value: + continue + order_type = payload.get(type_key) + if order_type in ( + trading_enums_module.TradeOrderType.BUY_MARKET.value, + trading_enums_module.TraderOrderType.BUY_MARKET.value, + ): + return True + status = payload.get(status_key) + if status in ( + trading_enums_module.OrderStatus.FILLED.value, + trading_enums_module.OrderStatus.CLOSED.value, + ): + return True + return False + + +class TestCoinrabbitTickerWiseDCADbosIntegration: + @pytest.mark.asyncio + async def test_create_dca_buy_on_ticker_wise_pair_then_stop( + self, + temp_dbos_scheduler, + skip_on_exchange_proxy_error, + ): + close_by_symbol = coinrabbit_dca_util.default_close_prices_by_symbol() + patched_fetch_tickers = coinrabbit_dca_util.tickers_repository_fetch_tickers_close_override( + lambda symbol: close_by_symbol[symbol], + ) + patched_fetch_ohlcv = price_mocks_module.fetch_ohlcv_side_effect_for_close_prices( + lambda symbol: close_by_symbol[symbol], + ) + user_id = workflow_common_module.SIMULATOR_GRID_TEST_COMMUNITY_USER_ID + initial_reference = coinrabbit_dca_util.INITIAL_REFERENCE_HOLDING + protocol_account = coinrabbit_dca_util.protocol_account_for_coinrabbit_functional( + account_id=_DCA_ACCOUNT_ID, + reference_total=initial_reference, + ) + create_user_action = coinrabbit_dca_util.build_create_dca_user_action( + account_id=_DCA_ACCOUNT_ID, + name=_DCA_AUTOMATION_DISPLAY_NAME, + ) + authentication_instance = authenticator_mocks_module.build_community_authentication( + workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, + workflow_common_module.SIMULATOR_GRID_TEST_WALLET_PASSPHRASE, + ) + + with ( + mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ), + coinrabbit_price_util.patch_coinrabbit_ticker_closes(close_by_symbol), + mock.patch.object( + octobot_flow_repositories_exchange_module.TickersRepository, + "fetch_tickers", + new=patched_fetch_tickers, + ), + mock.patch.object( + octobot_flow_repositories_exchange_module.OhlcvRepository, + "fetch_ohlcv", + side_effect=patched_fetch_ohlcv, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=mock.Mock( + get_item=mock.Mock(return_value=protocol_account), + get_exchange_config=mock.Mock( + return_value=coinrabbit_dca_util.protocol_exchange_config_for_coinrabbit_functional(), + ), + ), + ), + mock.patch( + "octobot_sync.sync.collection_providers.StrategyProvider.instance", + return_value=mock.Mock( + get_item=mock.Mock( + return_value=coinrabbit_dca_util.seeded_dca_strategy_for_coinrabbit_wallet( + stored_strategy_id=coinrabbit_dca_util.COINRABBIT_DCA_DEFAULT_STRATEGY_ID, + ), + ), + ), + ), + ): + workflow_common_module.seed_empty_account_trading_state(user_id, _DCA_ACCOUNT_ID) + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + create_user_action, + user_id, + ), + timeout=_T_ENQUEUE_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action timed out enqueueing automation workflow") from exc + + await user_action_assertions_module.assert_user_action_selector_completed_automation_create( + user_id=user_id, + user_action_id=create_user_action.id, + expected_workflow_id=None, + ) + + metadata_automation_id = user_action_assertions_module.resolve_create_automation_metadata_id( + create_user_action, + ) + parent_automation_id = await user_action_assertions_module.get_created_automation_id_from_user_action( + user_action_id=create_user_action.id, + user_id=user_id, + ) + + async def _enqueue_forced_trigger(user_action_id: str) -> None: + signal_user_action = workflow_common_module.build_forced_trigger_signal_user_action( + automation_id=parent_automation_id, + user_action_id=user_action_id, + ) + await workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + signal_user_action, + user_id, + ) + await user_action_assertions_module.assert_user_action_selector_completed_automation_signal( + user_id=user_id, + user_action_id=signal_user_action.id, + ) + + await _enqueue_forced_trigger("ua-signal-coinrabbit-dca-initial") + + reference_after_init = initial_reference + buy_deadline = time.monotonic() + _T_DCA_BUY_SECONDS + elements_after_buy: typing.Any = None + last_seen_elements: typing.Any = None + forced_trigger_counter = 0 + while time.monotonic() < buy_deadline: + for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: + continue + reader = automation_states_loader_module.get_automation_state_reader(workflow_row) + if reader is None: + continue + candidate_elements = reader.state.automation.exchange_account_elements + last_seen_elements = candidate_elements + btc_total = _portfolio_btc_total(candidate_elements) + reference_total = _portfolio_reference_total(candidate_elements) + if btc_total > 0 and reference_total < reference_after_init: + elements_after_buy = candidate_elements + break + if elements_after_buy is not None: + break + forced_trigger_counter += 1 + if forced_trigger_counter % 20 == 0: + await _enqueue_forced_trigger(f"ua-signal-coinrabbit-dca-retry-{forced_trigger_counter}") + await asyncio.sleep(workflow_common_module.DEFAULT_WORKFLOW_POLL_INTERVAL_SECONDS) + else: + last_portfolio = exchange_account_elements_access_module.portfolio_content_from_elements( + last_seen_elements + ) + pytest.fail( + "Timed out waiting for DCA market buy on " + f"{coinrabbit_dca_util.TRADED_SYMBOL} with portfolio delta; " + f"last portfolio keys={list(last_portfolio.keys()) if last_portfolio else 'n/a'}" + ) + + assert _has_filled_market_buy_on_traded_symbol(elements_after_buy) or _portfolio_btc_total( + elements_after_buy + ) > 0, ( + "expected a filled/closed market buy or buy trade on " + f"{coinrabbit_dca_util.TRADED_SYMBOL}" + ) + + stop_user_action = workflow_common_module.build_stop_user_action( + automation_id=parent_automation_id, + user_action_id="ua-stop-coinrabbit-dca-functional", + ) + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + stop_user_action, + user_id, + ), + timeout=_T_STOP_SEND_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action stop timed out") from exc + + await user_action_assertions_module.assert_user_action_selector_completed_automation_stop( + user_id=user_id, + user_action_id=stop_user_action.id, + ) + + final_output_text = await workflow_common_module.wait_for_stop_success_output( + temp_dbos_scheduler, + metadata_automation_id, + _T_STOP_COMPLETE_SECONDS, + ) + assert final_output_text is not None + parsed_final = workflow_common_module.parse_automation_workflow_output(final_output_text) + assert parsed_final.error is None diff --git a/packages/node/tests/functional_tests/test_strategies_CRUD_operations.py b/packages/node/tests/functional_tests/test_strategies_CRUD_operations.py index db4a2f1497..2ee3a4a500 100644 --- a/packages/node/tests/functional_tests/test_strategies_CRUD_operations.py +++ b/packages/node/tests/functional_tests/test_strategies_CRUD_operations.py @@ -41,8 +41,7 @@ def temp_dbos_scheduler_strategy_crud( """Mirrors ``tests.scheduler.temp_dbos_scheduler``; requests the patch fixture so registration sees capped retries.""" with tempfile.NamedTemporaryFile() as temp_file: dbos_runtime = scheduler_tests.init_scheduler(temp_file.name) - dbos_runtime.reset_system_database() - dbos_runtime.launch() + scheduler_tests.reset_launch_and_register_scheduler_queues(dbos_runtime) try: yield octobot_node.scheduler.SCHEDULER finally: diff --git a/packages/node/tests/functional_tests/util/coinrabbit_dca_workflow.py b/packages/node/tests/functional_tests/util/coinrabbit_dca_workflow.py new file mode 100644 index 0000000000..9706a2b387 --- /dev/null +++ b/packages/node/tests/functional_tests/util/coinrabbit_dca_workflow.py @@ -0,0 +1,126 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +"""CoinRabbit ticker-wise DCA helpers for DBOS functional tests.""" + +from __future__ import annotations + +import typing + +import octobot_protocol.models as protocol_models_module +import tentacles.Trading.Mode.dca_trading_mode.dca_trading as dca_trading + +from tests.scheduler.user_actions.user_actions_executor.util import trading_tentacles_test_utils + +from . import dca_workflow as dca_workflow_module +from . import price_mocks as price_mocks_module +from . import workflow_common as workflow_common_module + +EXCHANGE_INTERNAL_NAME = "coinrabbit" +REFERENCE_MARKET = "USDT@ETH" +TRADED_SYMBOL = "BTC@BTC/USDT@ETH" +FIXED_TRADED_SYMBOL_CLOSE = 100_000.0 +INITIAL_REFERENCE_HOLDING = 1000.0 + +COINRABBIT_DCA_DEFAULT_STRATEGY_ID = "coinrabbit_dca_ticker_wise_functional_strategy" +COINRABBIT_FUNCTIONAL_EXCHANGE_CONFIG_ID = "functional-test-coinrabbit-exchange-config-id" + + +def default_close_prices_by_symbol() -> dict[str, float]: + return {TRADED_SYMBOL: FIXED_TRADED_SYMBOL_CLOSE} + + +def dca_configuration_for_coinrabbit_ticker_wise() -> protocol_models_module.TradingTentaclesConfiguration: + return trading_tentacles_test_utils.trading_tentacles_configuration( + name=dca_trading.DCATradingMode.get_name(), + config=trading_tentacles_test_utils.dca_tentacle_config( + **{ + dca_trading.DCATradingModeProducer.TRIGGER_MODE: ( + dca_trading.TriggerMode.ALWAYS_TRIGGER_LONG.value + ), + dca_trading.DCATradingMode.TRADING_PAIRS: [TRADED_SYMBOL], + dca_trading.DCATradingModeConsumer.USE_MARKET_ENTRY_ORDERS: True, + } + ), + ) + + +def protocol_exchange_config_for_coinrabbit_functional() -> protocol_models_module.ExchangeConfig: + return protocol_models_module.ExchangeConfig( + id=COINRABBIT_FUNCTIONAL_EXCHANGE_CONFIG_ID, + name="coinrabbit-main", + exchange=EXCHANGE_INTERNAL_NAME, + sandboxed=False, + ) + + +def protocol_account_for_coinrabbit_functional( + *, + account_id: str, + reference_total: float = INITIAL_REFERENCE_HOLDING, + account_name: str = "CoinRabbit ticker-wise functional DCA account", +) -> protocol_models_module.Account: + return protocol_models_module.Account( + id=account_id, + name=account_name, + is_simulated=True, + created_at=workflow_common_module._FUNCTIONAL_PROTOCOL_ACCOUNT_TS, + updated_at=workflow_common_module._FUNCTIONAL_PROTOCOL_ACCOUNT_TS, + assets=[ + protocol_models_module.DetailedAssetsForTradingType( + trading_type=protocol_models_module.TradingType.SPOT, + assets=[ + protocol_models_module.DetailedAsset( + symbol=REFERENCE_MARKET, + total=reference_total, + available=reference_total, + ) + ], + ) + ], + specifics=protocol_models_module.AccountSpecifics( + actual_instance=protocol_models_module.ExchangeAccount( + account_type=protocol_models_module.AccountType.EXCHANGE, + remote_account_id=account_id, + exchange_config_ids=[COINRABBIT_FUNCTIONAL_EXCHANGE_CONFIG_ID], + ), + ), + ) + + +def seeded_dca_strategy_for_coinrabbit_wallet( + *, + stored_strategy_id: str, +) -> protocol_models_module.Strategy: + return protocol_models_module.Strategy( + id=stored_strategy_id, + version=workflow_common_module.SIMULATOR_FUNCTIONAL_STRATEGY_VERSION, + name="CoinRabbit ticker-wise DCA automation strategy", + reference_market=REFERENCE_MARKET, + configuration=protocol_models_module.StrategyConfiguration( + dca_configuration_for_coinrabbit_ticker_wise(), + ), + ) + + +def build_create_dca_user_action( + *, + account_id: str, + name: str, + strategy_id: str | None = None, + automation_id: str | None = None, +) -> protocol_models_module.UserAction: + return dca_workflow_module.build_create_dca_user_action( + account_id=account_id, + name=name, + strategy_id=strategy_id or COINRABBIT_DCA_DEFAULT_STRATEGY_ID, + automation_id=automation_id, + ) + + +def tickers_repository_fetch_tickers_close_override( + get_close_price_for_symbol: typing.Callable[[str], typing.Union[int, float]], +): + return price_mocks_module.tickers_repository_fetch_tickers_close_override( + get_close_price_for_symbol, + traded_symbols=[TRADED_SYMBOL], + ) diff --git a/packages/node/tests/functional_tests/util/global_view_workflow.py b/packages/node/tests/functional_tests/util/global_view_workflow.py index feed486b04..f75e29728e 100644 --- a/packages/node/tests/functional_tests/util/global_view_workflow.py +++ b/packages/node/tests/functional_tests/util/global_view_workflow.py @@ -215,11 +215,13 @@ def build_running_automation_state( async def enqueue_and_await_global_view_refresh() -> dict[str, typing.Any]: - import octobot_node.scheduler + import dbos + import octobot_node.enums import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module scheduled_time = datetime.datetime.now(datetime.UTC) - workflow_handle = await octobot_node.scheduler.SCHEDULER.GLOBAL_VIEW_QUEUE.enqueue_async( + workflow_handle = await dbos.DBOS.enqueue_workflow_async( + octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value, global_view_workflow_module.GlobalViewRefreshWorkflow.global_view_refresh, scheduled_time, None, diff --git a/packages/node/tests/functional_tests/util/outdated_reference_workflow.py b/packages/node/tests/functional_tests/util/outdated_reference_workflow.py index 0225f8bdf7..f36a3d4682 100644 --- a/packages/node/tests/functional_tests/util/outdated_reference_workflow.py +++ b/packages/node/tests/functional_tests/util/outdated_reference_workflow.py @@ -9,7 +9,6 @@ import time import typing -import dbos import pytest import octobot_commons.timestamp_util as timestamp_util @@ -17,6 +16,7 @@ import octobot_flow.entities as octobot_flow_entities import octobot_node.scheduler.internal_trading_signals as internal_trading_signals_module import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module +import octobot_node.scheduler.workflows_util as workflows_util_module import octobot_protocol.models as protocol_models import octobot_trading.enums as trading_enums @@ -104,6 +104,49 @@ def caplog_contains_outdated_skip(caplog) -> bool: ) +def _pending_copier_ready_for_automation( + pending_rows: typing.Iterable[typing.Any], + automation_id: str, + trading_signal: octobot_flow_entities.TradingSignal, +) -> bool: + return any( + automation_states_loader_module.get_automation_id(workflow_row) == automation_id + and internal_trading_signals_module.workflow_row_matches_copier_trading_signal( + workflow_row, trading_signal + ) + for workflow_row in pending_rows + ) + + +def _pending_latest_copier_ready_for_automation( + pending_rows: typing.Iterable[typing.Any], + automation_id: str, + trading_signal: octobot_flow_entities.TradingSignal, + *, + min_child_workflow_index: int = 1, +) -> bool: + """ + After outdated skip, the recv-blocking child is the latest enqueued iteration; + readiness on an older queued child re-triggers the wrong workflow row. + """ + automation_pending_rows = [ + workflow_row + for workflow_row in pending_rows + if automation_states_loader_module.get_automation_id(workflow_row) == automation_id + ] + if not automation_pending_rows: + return False + latest_pending_child = workflows_util_module.get_latest_child_workflow(automation_pending_rows) + child_workflow_index = workflows_util_module.parse_automation_child_workflow_index( + latest_pending_child.workflow_id + ) + if child_workflow_index < min_child_workflow_index: + return False + return internal_trading_signals_module.workflow_row_matches_copier_trading_signal( + latest_pending_child, trading_signal + ) + + async def deliver_trading_signal_when_pending( scheduler: typing.Any, automation_id: str, @@ -113,18 +156,8 @@ async def deliver_trading_signal_when_pending( poll_interval = 0.05 poll_deadline = time.monotonic() + deadline_seconds while time.monotonic() < poll_deadline: - pending_rows = await scheduler.INSTANCE.list_workflows_async( - status=[ - dbos.WorkflowStatusString.ENQUEUED.value, - dbos.WorkflowStatusString.PENDING.value, - ], - ) - for workflow_row in pending_rows: - if automation_states_loader_module.get_automation_id(workflow_row) != automation_id: - continue - copied_strategy_ids = automation_states_loader_module.get_automation_copied_strategy_ids(workflow_row) - if trading_signal.strategy_id not in copied_strategy_ids: - continue + pending_rows = await internal_trading_signals_module.list_pending_copier_automation_workflow_statuses() + if _pending_copier_ready_for_automation(pending_rows, automation_id, trading_signal): await internal_trading_signals_module.send_internal_trading_signal(trading_signal) return await asyncio.sleep(poll_interval) @@ -134,6 +167,45 @@ async def deliver_trading_signal_when_pending( ) +async def deliver_fresh_trading_signal_after_outdated_skip( + scheduler: typing.Any, + caplog, + automation_id: str, + trading_signal: octobot_flow_entities.TradingSignal, + deadline_seconds: float, +) -> None: + """ + Wait for the outdated-reference skip log, then deliver the signal while the automation + child workflow is still pending on recv (before a signal-less DAG iteration stops it). + """ + poll_interval = 0.05 + poll_deadline = time.monotonic() + deadline_seconds + while time.monotonic() < poll_deadline: + if caplog_contains_outdated_skip(caplog): + break + await asyncio.sleep(poll_interval) + else: + pytest.fail( + "Timed out waiting for outdated reference account skip log " + f"({OUTDATED_SKIP_LOG_SUBSTRING!r}) before fresh signal delivery" + ) + while time.monotonic() < poll_deadline: + pending_rows = await internal_trading_signals_module.list_pending_copier_automation_workflow_statuses() + if _pending_latest_copier_ready_for_automation( + pending_rows, + automation_id, + trading_signal, + min_child_workflow_index=2, + ): + await internal_trading_signals_module.send_internal_trading_signal(trading_signal) + return + await asyncio.sleep(poll_interval) + pytest.fail( + f"Timed out delivering fresh trading signal for automation_id={automation_id!r} " + f"strategy_id={trading_signal.strategy_id!r} after outdated skip" + ) + + def sell_limit_prices_from_reader(state_reader: typing.Any) -> list[decimal.Decimal]: return exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( state_reader.state.automation.exchange_account_elements, diff --git a/packages/node/tests/functional_tests/util/workflow_common.py b/packages/node/tests/functional_tests/util/workflow_common.py index ed91db9102..17a7300596 100644 --- a/packages/node/tests/functional_tests/util/workflow_common.py +++ b/packages/node/tests/functional_tests/util/workflow_common.py @@ -339,11 +339,13 @@ async def _list_matching_automation_workflow_rows( scheduler: typing.Any, automation_id: str, ) -> list[dbos.WorkflowStatus]: + # Third argument is SchedulerWorkflowNames (DBOS workflow name), not enqueue queue. workflow_rows = await scheduler._list_workflows( None, None, - [node_enums_module.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], + node_enums_module.SchedulerWorkflowNames.EXECUTE_AUTOMATION, load_output=False, + load_input=True, ) return [ workflow_row diff --git a/packages/node/tests/protocol/test_dsl.py b/packages/node/tests/protocol/test_dsl.py index e431dad2ba..0355049e2c 100644 --- a/packages/node/tests/protocol/test_dsl.py +++ b/packages/node/tests/protocol/test_dsl.py @@ -16,7 +16,9 @@ import pytest +import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.enums as commons_enums +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support import octobot_commons.dsl_interpreter.operator_docs as dsl_interpreter_operator_docs import octobot_commons.dsl_interpreter.operator_parameter as dsl_interpreter_operator_parameter import octobot_node.protocol.dsl as dsl_protocol @@ -254,6 +256,55 @@ def test_raises_when_parameter_type_invalid(self): ) ) + def test_minimal_operator_without_catalog_metadata_fails_conversion(self): + # Nested only: module-level incomplete operators are what broke GET /api/v1/dsl/keywords (HTTP 500). + class _IncompleteTestKeyword(dsl_interpreter.Operator): + @staticmethod + def get_name() -> str: + return "incomplete_test_keyword" + + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: + return None + + # Same checks as production conversion; TestKeywordMixin supplies CATEGORY and return_values. + with pytest.raises(ValueError, match="CATEGORY|return_values"): + dsl_protocol.operator_docs_to_dsl_keyword(_IncompleteTestKeyword.get_docs()) + + +# Module-level subclass: registered when pytest imports this file (same as xdist loading a test module early). +# Without TestKeywordMixin, operator_docs_to_dsl_keyword() would raise and the keywords route would return 500. +class _PollutingIntrospectionTestKeyword( + dsl_test_keyword_support.TestKeywordMixin, + dsl_interpreter.Operator, +): + @staticmethod + def get_name() -> str: + return "polluting_introspection_test_keyword" + + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: + return None + + +class TestDslKeywordsCatalogWithRegisteredTestOperator: + """ + Regression for CI 500 on GET /api/v1/dsl/keywords when test operators are on the worker. + + Catalog assembly uses flow DSLExecutor (needs tentacles), so these tests live in node, not commons. + _PollutingIntrospectionTestKeyword is already registered before any test method runs. + """ + + def test_operator_docs_to_dsl_keyword_succeeds(self): + dsl_protocol.operator_docs_to_dsl_keyword(_PollutingIntrospectionTestKeyword.get_docs()) + + def test_get_dsl_keywords_state_does_not_raise(self): + # Entry point used by the keywords API; must succeed with the polluter in get_all_operators(). + dsl_protocol.get_dsl_keywords_state() + + def test_all_operator_docs_convert_to_dsl_keywords(self): + # Every registered operator on this worker, not only _PollutingIntrospectionTestKeyword. + for operator_docs in dsl_protocol.list_dsl_operator_docs(): + dsl_protocol.operator_docs_to_dsl_keyword(operator_docs) + class TestGetDslKeywordsState: def test_returns_full_catalog(self): diff --git a/packages/node/tests/protocol/test_user_actions.py b/packages/node/tests/protocol/test_user_actions.py index 9eea42f274..21a96df024 100644 --- a/packages/node/tests/protocol/test_user_actions.py +++ b/packages/node/tests/protocol/test_user_actions.py @@ -10,6 +10,7 @@ import octobot_protocol.models as protocol_models +import octobot_node.enums as octobot_node_enums import octobot_node.protocol.user_actions as user_actions_module _TEST_WALLET_ADDRESS = "0x742d35Cc6634C0532925a3b844Bc9e7595f0bEb" @@ -34,10 +35,55 @@ async def test_calls_trigger_user_action_workflow_with_wallet_and_payload(self): user_actions_module.scheduler_tasks, "trigger_user_action_workflow", new_callable=mock.AsyncMock, - ) as trigger_workflow_mock: + ) as trigger_workflow_mock, mock.patch.object( + user_actions_module.node_journal, + "record_external_action_received", + ): await user_actions_module.execute_user_action(user_action_payload, _TEST_WALLET_ADDRESS) trigger_workflow_mock.assert_awaited_once_with(user_action_payload, _TEST_WALLET_ADDRESS) + @pytest.mark.asyncio + async def test_sync_source_emits_entry_onboarding_milestones(self): + user_action_payload = _minimal_user_action(action_identifier="ua-entry-check") + with mock.patch.object( + user_actions_module.node_journal, + "record_external_action_received", + ) as entry_mock, mock.patch.object( + user_actions_module.scheduler_tasks, + "trigger_user_action_workflow", + new_callable=mock.AsyncMock, + ): + await user_actions_module.execute_user_action( + user_action_payload, + _TEST_WALLET_ADDRESS, + source=octobot_node_enums.UserActionSource.SYNC, + ) + entry_mock.assert_called_once_with( + user_action_payload, + source=octobot_node_enums.UserActionSource.SYNC.value, + ) + + @pytest.mark.asyncio + async def test_debug_api_emits_entry_onboarding_milestones_with_source(self): + user_action_payload = _minimal_user_action(action_identifier="ua-debug-check") + with mock.patch.object( + user_actions_module.node_journal, + "record_external_action_received", + ) as entry_mock, mock.patch.object( + user_actions_module.scheduler_tasks, + "trigger_user_action_workflow", + new_callable=mock.AsyncMock, + ): + await user_actions_module.execute_user_action( + user_action_payload, + _TEST_WALLET_ADDRESS, + source=octobot_node_enums.UserActionSource.DEBUG_API, + ) + entry_mock.assert_called_once_with( + user_action_payload, + source=octobot_node_enums.UserActionSource.DEBUG_API.value, + ) + @pytest.mark.asyncio async def test_surfaces_scheduler_not_initialized_as_runtime_error(self): user_action_payload = _minimal_user_action(action_identifier="ua-not-init") diff --git a/packages/node/tests/scheduler/__init__.py b/packages/node/tests/scheduler/__init__.py index e8830819ab..f041fb3100 100644 --- a/packages/node/tests/scheduler/__init__.py +++ b/packages/node/tests/scheduler/__init__.py @@ -78,29 +78,6 @@ def _reset_asyncio_default_executor() -> None: loop._default_executor = None -def _ensure_scheduler_queues() -> None: - # destroy_launched_dbos() clears Scheduler queue handles; when destroy_registry=False - # the global registry still owns the Queue declarations and re-creating them fails. - scheduler = octobot_node.scheduler.SCHEDULER - if scheduler.AUTOMATION_WORKFLOW_QUEUE is not None: - return - import dbos._dbos as dbos_internals - import octobot_node.enums as octobot_node_enums_module - registry = dbos_internals._get_or_create_dbos_registry() - queue_bindings = { - octobot_node_enums_module.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value: "AUTOMATION_WORKFLOW_QUEUE", - octobot_node_enums_module.SchedulerQueues.USER_ACTION_QUEUE.value: "USER_ACTION_QUEUE", - octobot_node_enums_module.SchedulerQueues.DBOS_CLEANUP_QUEUE.value: "DBOS_CLEANUP_QUEUE", - octobot_node_enums_module.SchedulerQueues.GLOBAL_VIEW_QUEUE.value: "GLOBAL_VIEW_QUEUE", - octobot_node_enums_module.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value: "PORTFOLIO_HISTORY_QUEUE", - } - if all(queue_name in registry.queue_info_map for queue_name in queue_bindings): - for queue_name, scheduler_attribute in queue_bindings.items(): - setattr(scheduler, scheduler_attribute, registry.queue_info_map[queue_name]) - return - scheduler.create_queues() - - def destroy_launched_dbos(*, destroy_registry: bool = False) -> None: """ Tear down the DBOS singleton so the next test can reset the system database. @@ -114,11 +91,6 @@ def destroy_launched_dbos(*, destroy_registry: bool = False) -> None: # rebinds registry.dbos on the next launch. dbos.DBOS.destroy(workflow_completion_timeout_sec=0, destroy_registry=destroy_registry) octobot_node.scheduler.SCHEDULER.INSTANCE = None - octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE = None - octobot_node.scheduler.SCHEDULER.USER_ACTION_QUEUE = None - octobot_node.scheduler.SCHEDULER.DBOS_CLEANUP_QUEUE = None - octobot_node.scheduler.SCHEDULER.GLOBAL_VIEW_QUEUE = None - octobot_node.scheduler.SCHEDULER.PORTFOLIO_HISTORY_QUEUE = None def init_scheduler(db_file_name: str, application_version: str | None = None): @@ -136,7 +108,6 @@ def init_scheduler(db_file_name: str, application_version: str | None = None): } if application_version is not None: config["application_version"] = application_version - _ensure_scheduler_queues() dbos.DBOS(config=config) octobot_node.scheduler.SCHEDULER.INSTANCE = dbos.DBOS octobot_node.scheduler.workflows.register_workflows() @@ -147,6 +118,21 @@ def init_scheduler_with_app_version(db_file_name: str, application_version: str) return init_scheduler(db_file_name, application_version=application_version) +def register_scheduler_queues() -> None: + """Sync queue registration for pytest fixtures (requires ``DBOS.launch()`` first).""" + import octobot_node.scheduler.queues as scheduler_queues_module + + for queue, register_kwargs in scheduler_queues_module._SCHEDULER_QUEUE_SPECS: + dbos.DBOS.register_queue(queue.value, **register_kwargs) + + +def reset_launch_and_register_scheduler_queues(dbos_runtime) -> None: + """Prepare a test DBOS runtime for enqueue/consume after ``init_scheduler``.""" + dbos_runtime.reset_system_database() + dbos_runtime.launch() + register_scheduler_queues() + + @pytest.fixture() def temp_dbos_scheduler(): # from https://docs.dbos.dev/python/tutorials/testing @@ -154,8 +140,7 @@ def temp_dbos_scheduler(): with tempfile.NamedTemporaryFile() as temp_file: destroy_launched_dbos() dbos_runtime = init_scheduler(temp_file.name) - dbos_runtime.reset_system_database() - dbos_runtime.launch() + reset_launch_and_register_scheduler_queues(dbos_runtime) try: yield octobot_node.scheduler.SCHEDULER finally: @@ -165,6 +150,5 @@ def temp_dbos_scheduler(): def init_and_destroy_scheduler(db_file_name: str): destroy_launched_dbos() dbos_runtime = init_scheduler(db_file_name) - dbos_runtime.reset_system_database() - dbos_runtime.launch() + reset_launch_and_register_scheduler_queues(dbos_runtime) destroy_launched_dbos() diff --git a/packages/node/tests/scheduler/automations/test_automation_states_loader.py b/packages/node/tests/scheduler/automations/test_automation_states_loader.py index a50653d98b..1efd96bb49 100644 --- a/packages/node/tests/scheduler/automations/test_automation_states_loader.py +++ b/packages/node/tests/scheduler/automations/test_automation_states_loader.py @@ -11,8 +11,10 @@ import octobot_protocol.models as protocol_models import octobot_node.models as node_models import octobot_node.protocol.automations as automations_protocol +import octobot_node.enums as octobot_node_enums import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module import octobot_node.scheduler.workflows.params as workflow_params +import octobot_node.scheduler.workflows_util as workflows_util_module _PARENT_WORKFLOW_ID = "741ce171-dac9-40be-83dc-b443c0eaf0e2" @@ -400,23 +402,35 @@ async def test_returns_matching_pending_workflow(self): automation_id="automation-pending", ), ) - with mock.patch.object( - dbos.DBOS, - "list_workflows_async", - new=mock.AsyncMock(return_value=[workflow_status]), + list_mock = mock.AsyncMock(return_value=[workflow_status]) + import octobot_node.scheduler as scheduler_module + with ( + mock.patch.object(scheduler_module, "is_initialized", return_value=True), + mock.patch.object( + workflows_util_module, + "list_scheduler_workflows_async", + list_mock, + ), ): resolved_workflow = await automation_states_loader_module.get_automation_workflow_status( "automation-pending", ) assert resolved_workflow is workflow_status + list_mock.assert_awaited_once() + assert list_mock.await_args.args[1] == octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION + assert list_mock.await_args.kwargs.get("load_input") is True @pytest.mark.asyncio async def test_raises_when_no_workflow_matches(self): - with mock.patch.object( - dbos.DBOS, - "list_workflows_async", - new=mock.AsyncMock(return_value=[]), + import octobot_node.scheduler as scheduler_module + with ( + mock.patch.object(scheduler_module, "is_initialized", return_value=True), + mock.patch.object( + workflows_util_module, + "list_scheduler_workflows_async", + new=mock.AsyncMock(return_value=[]), + ), ): with pytest.raises(ValueError, match="No automation workflow found"): await automation_states_loader_module.get_automation_workflow_status("missing-automation") diff --git a/packages/node/tests/scheduler/automations/test_octobot_flow_client.py b/packages/node/tests/scheduler/automations/test_octobot_flow_client.py index b37d8ab4fe..f038e7eee9 100644 --- a/packages/node/tests/scheduler/automations/test_octobot_flow_client.py +++ b/packages/node/tests/scheduler/automations/test_octobot_flow_client.py @@ -298,6 +298,16 @@ def trade_and_loop_until_order_closed(market_order_action): return all +@pytest.fixture +def trade_and_loop_until_order_closed_with_jitter(trade_and_loop_until_order_closed): + return { + "params": { + **trade_and_loop_until_order_closed["params"], + "LOOP_INTERVAL_MAX": 70, + } + } + + @pytest.fixture def multiple_action_bundle_with_wait(deposit_action, market_order_action, withdraw_action): all = { @@ -363,6 +373,29 @@ def get_deposit_and_withdrawal_details(actions: list["octobot_flow.entities.Abst return list_util.flatten_list(withdrawal_lists) if withdrawal_lists else [] +async def _loop_until_order_closed_dsl_after_trade(job_config: dict) -> str: + job = octobot_flow_client.OctoBotActionsJob( + job_config, [], [], octobot_flow_client.OctoBotActionsJobResult() + ) + await job.run() + next_actions_description = job.result.next_actions_description + assert next_actions_description is not None + job2 = octobot_flow_client.OctoBotActionsJob( + next_actions_description.to_dict(include_default_values=False), [], [], + octobot_flow_client.OctoBotActionsJobResult(), + ) + await job2.run() + next_actions_description = job2.result.next_actions_description + assert next_actions_description is not None + parsed_state = octobot_flow.entities.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], octobot_flow.entities.DSLScriptActionDetails) + loop_dsl = next_actions[0].dsl_script + assert loop_dsl is not None + return loop_dsl + + class TestOctoBotActionsJob: def setup_method(self): @@ -894,6 +927,16 @@ async def test_run_multiple_actions_bundle_no_wait(self, multiple_actions_bundle # created a buy order but not executed: locked BTC in portfolio assert post_deposit_portfolio["BTC"][common_constants.PORTFOLIO_AVAILABLE] < post_deposit_portfolio["BTC"][common_constants.PORTFOLIO_TOTAL] + async def test_trade_and_loop_until_order_closed_jitter_emits_max_retry_interval( + self, trade_and_loop_until_order_closed_with_jitter + ): + loop_dsl = await _loop_until_order_closed_dsl_after_trade( + trade_and_loop_until_order_closed_with_jitter + ) + assert loop_dsl.startswith("loop_until(") + assert "max_retry_interval=70" in loop_dsl + assert "3.0, max_retry_interval=70.0, timeout=10.0, max_attempts=4, return_remaining_time=True)" in loop_dsl + async def test_run_trade_and_loop_until_order_closed(self, trade_and_loop_until_order_closed): # Step 1 — Apply automation config (ACTIONS: trade, loop_until_order_closed). # The only runnable action is init/APPLY_CONFIGURATION; portfolio is seeded (e.g. BTC for the later market buy). @@ -955,7 +998,8 @@ async def test_run_trade_and_loop_until_order_closed(self, trade_and_loop_until_ assert loop_dsl.startswith("loop_until(") assert "fetch_order" in loop_dsl assert f"!= '{trading_enums.OrderStatus.OPEN.value}'" in loop_dsl - assert "3, timeout=10, max_attempts=4, return_remaining_time=True)" in loop_dsl + assert "3.0, timeout=10.0, max_attempts=4, return_remaining_time=True)" in loop_dsl + assert "max_retry_interval" not in loop_dsl job3 = octobot_flow_client.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False), [], [], octobot_flow_client.OctoBotActionsJobResult(), @@ -1209,7 +1253,7 @@ def _create_order_instance_with_address_from(*args, **kwargs): assert next_actions[0].dsl_script.startswith("loop_until(") assert "blockchain_wallet_balance" in next_actions[0].dsl_script assert "123_balance_address" in next_actions[0].dsl_script - assert "3, timeout=10, max_attempts=4, return_remaining_time=True)" in next_actions[0].dsl_script + assert "3.0, timeout=10.0, max_attempts=4, return_remaining_time=True)" in next_actions[0].dsl_script job5 = octobot_flow_client.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False), [], [], octobot_flow_client.OctoBotActionsJobResult(), @@ -1663,3 +1707,73 @@ async def test_logs_executed_actions_not_input_priority_actions(self): mock_logger.info.assert_called_once_with( f"Running automation actions: {[executed_action]}" ) + + +class TestGetNextActionsDescription: + pytestmark = [] + + @staticmethod + def _job_with_automation_state(automation_state: "octobot_flow.entities.AutomationState"): + job = octobot_flow_client.OctoBotActionsJob( + {"state": {}}, + [], + [], + octobot_flow_client.OctoBotActionsJobResult(), + ) + job.after_execution_state = automation_state + return job + + @staticmethod + def _automation_state_with_dag(actions_dag: "octobot_flow.entities.ActionsDAG"): + return octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation_test"), + actions_dag=actions_dag, + ) + ) + + def test_returns_has_next_actions_false_when_upstream_action_failed(self): + if message := misses_required_octobot_flow_client_import(): + pytest.skip(reason=message) + init_action = octobot_flow.entities.ConfiguredActionDetails(id="action_init") + init_action.complete( + error_status=octobot_flow.enums.ActionErrorStatus.BLOCKCHAIN_WALLET_ERROR.value, + error_message="connection timed out", + ) + trade_action = octobot_flow.entities.DSLScriptActionDetails( + id="action_trade", + dsl_script="trade()", + dependencies=[{"action_id": "action_init"}], + ) + automation_state = self._automation_state_with_dag( + octobot_flow.entities.ActionsDAG(actions=[init_action, trade_action]) + ) + job = self._job_with_automation_state(automation_state) + post_execution_state = automation_state.to_dict(include_default_values=False) + next_description, has_next_actions = job.get_next_actions_description(post_execution_state) + assert has_next_actions is False + assert next_description is not None + assert next_description.state == post_execution_state + + def test_raises_workflow_dag_dependencies_error_on_circular_pending_deps(self): + if message := misses_required_octobot_flow_client_import(): + pytest.skip(reason=message) + import octobot_node.errors as node_errors + + action_c = octobot_flow.entities.DSLScriptActionDetails( + id="action_c", + dsl_script="dma_evaluator()", + dependencies=[{"action_id": "action_d"}], + ) + action_d = octobot_flow.entities.DSLScriptActionDetails( + id="action_d", + dsl_script="dma_evaluator()", + dependencies=[{"action_id": "action_c"}], + ) + automation_state = self._automation_state_with_dag( + octobot_flow.entities.ActionsDAG(actions=[action_c, action_d]) + ) + job = self._job_with_automation_state(automation_state) + post_execution_state = automation_state.to_dict(include_default_values=False) + with pytest.raises(node_errors.WorkflowDAGDependenciesError): + job.get_next_actions_description(post_execution_state) diff --git a/packages/node/tests/scheduler/test_api.py b/packages/node/tests/scheduler/test_api.py index ce3a95eb6f..99f4a3ac50 100644 --- a/packages/node/tests/scheduler/test_api.py +++ b/packages/node/tests/scheduler/test_api.py @@ -14,12 +14,15 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. +import datetime import json import pytest import mock import dbos import octobot_protocol.models as protocol_models from octobot_node.models import Execution, Task, TaskStatus +import octobot_node.constants + from octobot_node.scheduler.api import ( get_node_status, get_task_metrics, @@ -28,6 +31,8 @@ get_tasks_export_results, await_workflow_result_from_id, retrieve_workflow_handle, + paginate_tasks, + sort_tasks_by_recency, ) from tests.scheduler import temp_dbos_scheduler @@ -45,6 +50,7 @@ def _make_wf_status(workflow_id: str, status: str, user_id: str = "0xaaa") -> mo wf = mock.Mock() wf.workflow_id = workflow_id wf.status = status + wf.name = octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value wf.queue_name = octobot_node_enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value wf.input = { "args": [inputs.to_dict(include_default_values=False)], @@ -167,8 +173,8 @@ async def test_get_task_metrics_success(self) -> None: mock_instance = mock.AsyncMock() - def list_side_effect(status=None, **kwargs): - if dbos.WorkflowStatusString.PENDING.value in (status or []): + def list_side_effect(status=None, queues_only=False, **kwargs): + if queues_only: return pending_wf return result_wf @@ -182,8 +188,16 @@ def list_side_effect(status=None, **kwargs): assert result["pending"] == 5 assert result["scheduled"] == 0 assert result["results"] == 10 - for call in mock_instance.list_workflows_async.call_args_list: + import octobot_node.enums as octobot_node_enums + pending_call, results_call = mock_instance.list_workflows_async.call_args_list + assert pending_call.kwargs.get("queues_only") is True + assert "status" not in pending_call.kwargs + assert results_call.kwargs.get("queues_only") is not True + assert results_call.kwargs.get("status") is not None + for call in (pending_call, results_call): assert call.kwargs.get("load_output") is False + assert call.kwargs.get("name") == octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value + assert "queue_name" not in call.kwargs @pytest.mark.asyncio async def test_get_task_metrics_wallet_scoped(self) -> None: @@ -206,8 +220,8 @@ async def test_get_task_metrics_wallet_scoped(self) -> None: mock_instance = mock.AsyncMock() - def list_side_effect(status=None, **kwargs): - if dbos.WorkflowStatusString.PENDING.value in (status or []): + def list_side_effect(status=None, queues_only=False, **kwargs): + if queues_only: return pending_wf return result_wf @@ -236,7 +250,8 @@ async def test_get_task_metrics_counts_legacy_and_unparseable_for_every_wallet(s unparseable_wf = mock.Mock() unparseable_wf.workflow_id = "dddddddd-dddd-dddd-dddd-ddddddddddd2" unparseable_wf.status = dbos.WorkflowStatusString.SUCCESS.value - unparseable_wf.queue_name = octobot_node_enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value + unparseable_wf.name = octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value + unparseable_wf.queue_name = None unparseable_wf.input = {"args": [], "kwargs": {}} # nothing parseable unparseable_wf.created_at = None unparseable_wf.updated_at = None @@ -245,8 +260,8 @@ async def test_get_task_metrics_counts_legacy_and_unparseable_for_every_wallet(s mock_instance = mock.AsyncMock() - def list_side_effect(status=None, **kwargs): - if dbos.WorkflowStatusString.PENDING.value in (status or []): + def list_side_effect(status=None, queues_only=False, **kwargs): + if queues_only: return [] return [legacy_wf, unparseable_wf, other_wf] @@ -260,6 +275,34 @@ def list_side_effect(status=None, **kwargs): # legacy + unparseable kept (2), explicit other_wallet dropped (1) assert result["results"] == 2 + @pytest.mark.asyncio + async def test_get_task_metrics_counts_cancelled_automation_with_null_queue(self) -> None: + import octobot_node.enums as octobot_node_enums + + my_wallet = "0xmine" + cancelled_wf = _make_wf_status( + "cccccccc-cccc-cccc-cccc-ccccccccccc0", + dbos.WorkflowStatusString.CANCELLED.value, + user_id=my_wallet, + ) + cancelled_wf.queue_name = None + + mock_instance = mock.AsyncMock() + + def list_side_effect(status=None, queues_only=False, **kwargs): + if queues_only: + return [] + return [cancelled_wf] + + mock_instance.list_workflows_async = mock.AsyncMock(side_effect=list_side_effect) + mock_scheduler = mock.Mock() + mock_scheduler.INSTANCE = mock_instance + + with mock.patch("octobot_node.scheduler.SCHEDULER", mock_scheduler): + result = await get_task_metrics(user_id=my_wallet) + + assert result["results"] == 1 + @pytest.mark.asyncio async def test_get_task_metrics_uninitialized_scheduler(self) -> None: """Test task metrics when scheduler is not initialized.""" @@ -301,8 +344,8 @@ async def test_get_task_metrics_no_periodic_tasks(self) -> None: mock_instance = mock.AsyncMock() - def list_side_effect(status=None, **kwargs): - if dbos.WorkflowStatusString.PENDING.value in (status or []): + def list_side_effect(status=None, queues_only=False, **kwargs): + if queues_only: return pending_wf return result_wf @@ -559,6 +602,151 @@ async def test_get_all_tasks_skips_automation_states_when_all_tasks_completed(se assert tasks[0].metadata is None +def _task_with_execution_dates( + task_id: str, + *, + completed_at: datetime.datetime | None = None, + scheduled_at: datetime.datetime | None = None, + status: TaskStatus = TaskStatus.COMPLETED, +) -> Task: + return Task( + id=task_id, + executions=[ + Execution( + id=task_id, + status=status, + completed_at=completed_at, + scheduled_at=scheduled_at, + ) + ], + ) + + +class TestSortTasksByRecency: + def test_orders_by_completed_at_newest_first(self) -> None: + oldest_id = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + middle_id = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + newest_id = "cccccccc-cccc-cccc-cccc-cccccccccccc" + tasks = [ + _task_with_execution_dates( + oldest_id, + completed_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), + ), + _task_with_execution_dates( + newest_id, + completed_at=datetime.datetime(2024, 3, 1, tzinfo=datetime.timezone.utc), + ), + _task_with_execution_dates( + middle_id, + completed_at=datetime.datetime(2024, 2, 1, tzinfo=datetime.timezone.utc), + ), + ] + sorted_tasks = sort_tasks_by_recency(tasks) + assert [task.id for task in sorted_tasks] == [newest_id, middle_id, oldest_id] + + def test_uses_scheduled_at_when_completed_at_missing(self) -> None: + later_id = "dddddddd-dddd-dddd-dddd-dddddddddddd" + earlier_id = "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee" + tasks = [ + _task_with_execution_dates( + earlier_id, + status=TaskStatus.SCHEDULED, + scheduled_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), + ), + _task_with_execution_dates( + later_id, + status=TaskStatus.SCHEDULED, + scheduled_at=datetime.datetime(2024, 6, 1, tzinfo=datetime.timezone.utc), + ), + ] + sorted_tasks = sort_tasks_by_recency(tasks) + assert [task.id for task in sorted_tasks] == [later_id, earlier_id] + + def test_tasks_without_sortable_date_sort_last(self) -> None: + dated_id = "ffffffff-ffff-ffff-ffff-ffffffffffff" + undated_id = "00000000-0000-0000-0000-000000000099" + tasks = [ + Task(id=undated_id, executions=[Execution(id=undated_id, status=TaskStatus.PENDING)]), + _task_with_execution_dates( + dated_id, + completed_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), + ), + ] + sorted_tasks = sort_tasks_by_recency(tasks) + assert sorted_tasks[0].id == dated_id + assert sorted_tasks[1].id == undated_id + + +class TestPaginateTasks: + def test_page_one_returns_newest_after_sort(self) -> None: + tasks = [ + _task_with_execution_dates( + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + completed_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), + ), + _task_with_execution_dates( + "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + completed_at=datetime.datetime(2024, 3, 1, tzinfo=datetime.timezone.utc), + ), + _task_with_execution_dates( + "cccccccc-cccc-cccc-cccc-cccccccccccc", + completed_at=datetime.datetime(2024, 2, 1, tzinfo=datetime.timezone.utc), + ), + ] + page = paginate_tasks(tasks, page=1, limit=2) + assert [task.id for task in page] == [ + "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + "cccccccc-cccc-cccc-cccc-cccccccccccc", + ] + + def test_page_two_returns_remainder(self) -> None: + tasks = [ + _task_with_execution_dates( + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + completed_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), + ), + _task_with_execution_dates( + "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + completed_at=datetime.datetime(2024, 3, 1, tzinfo=datetime.timezone.utc), + ), + _task_with_execution_dates( + "cccccccc-cccc-cccc-cccc-cccccccccccc", + completed_at=datetime.datetime(2024, 2, 1, tzinfo=datetime.timezone.utc), + ), + ] + page = paginate_tasks(tasks, page=2, limit=2) + assert [task.id for task in page] == ["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"] + + def test_non_positive_page_treated_as_first_page(self) -> None: + tasks = [ + _task_with_execution_dates( + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + completed_at=datetime.datetime(2024, 2, 1, tzinfo=datetime.timezone.utc), + ), + _task_with_execution_dates( + "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + completed_at=datetime.datetime(2024, 3, 1, tzinfo=datetime.timezone.utc), + ), + ] + page_zero = paginate_tasks(tasks, page=0, limit=1) + page_negative = paginate_tasks(tasks, page=-3, limit=1) + assert page_zero[0].id == "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + assert page_negative[0].id == "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + + def test_limit_above_max_clamped(self) -> None: + task_ids = [f"{index:08d}-0000-0000-0000-000000000000" for index in range(600)] + tasks = [ + _task_with_execution_dates( + task_id, + completed_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc) + + datetime.timedelta(seconds=task_index), + ) + for task_index, task_id in enumerate(task_ids) + ] + page = paginate_tasks(tasks, page=1, limit=999_999) + assert len(page) == octobot_node.constants.TASKS_LIST_MAX_PAGE_LIMIT + + class TestGetTasksExportResults: """Tests for get_tasks_export_results function (batch decrypt-on-demand).""" @@ -745,6 +933,26 @@ async def test_get_task_result_running(self) -> None: assert result["status"] == "pending or running" + @pytest.mark.asyncio + async def test_get_task_result_cancelled_returns_no_data(self) -> None: + task_id = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa_2" + + mock_handle = mock.AsyncMock() + mock_handle.get_status = mock.AsyncMock(return_value=mock.Mock(status="CANCELLED")) + mock_handle.get_result = mock.AsyncMock() + + mock_instance = mock.AsyncMock() + mock_instance.retrieve_workflow_async = mock.AsyncMock(return_value=mock_handle) + + mock_scheduler = mock.Mock() + mock_scheduler.INSTANCE = mock_instance + + with mock.patch("octobot_node.scheduler.SCHEDULER", mock_scheduler): + result = await get_task_result(task_id) + + assert result == {"status": "cancelled", "data": None} + mock_handle.get_result.assert_not_called() + class TestAwaitWorkflowResultFromId: @pytest.mark.asyncio diff --git a/packages/node/tests/scheduler/test_internal_trading_signals.py b/packages/node/tests/scheduler/test_internal_trading_signals.py index 14a42b9f78..6fe453b27e 100644 --- a/packages/node/tests/scheduler/test_internal_trading_signals.py +++ b/packages/node/tests/scheduler/test_internal_trading_signals.py @@ -1,6 +1,7 @@ import asyncio import time +import dbos import mock import pytest @@ -48,6 +49,76 @@ async def test_get_or_create_after_shutdown_creates_new_channel(): await trading_signals_channel.shutdown_internal_trading_signal_channel() +@pytest.mark.asyncio +async def test_trigger_copier_automation_lists_pending_automation_workflows_by_name(): + pending_workflow_status = mock.Mock(workflow_id="automation-wf-1") + list_pending_mock = mock.AsyncMock(return_value=[pending_workflow_status]) + signal = flow_entities.TradingSignal( + account=protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=time.time(), + copied_assets=[], + ), + strategy_id="test-strategy-id", + ) + with ( + mock.patch.object( + internal_trading_signals, + "list_pending_copier_automation_workflow_statuses", + list_pending_mock, + ), + mock.patch( + "octobot_node.scheduler.automations.automation_states_loader.get_automation_copied_strategy_ids", + return_value=set(), + ), + mock.patch( + "octobot_node.scheduler.tasks.trigger_copier_automation", + mock.AsyncMock(), + ) as trigger_copier_automation_mock, + ): + await internal_trading_signals._trigger_copier_automation(signal) + + list_pending_mock.assert_awaited_once() + trigger_copier_automation_mock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_trigger_copier_automation_triggers_all_matching_pending_children(): + pending_child_one = mock.Mock(workflow_id="automation-wf-1") + pending_child_two = mock.Mock(workflow_id="automation-wf-2") + list_pending_mock = mock.AsyncMock(return_value=[pending_child_one, pending_child_two]) + signal = flow_entities.TradingSignal( + account=protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=time.time(), + copied_assets=[], + ), + strategy_id="test-strategy-id", + ) + with ( + mock.patch.object( + internal_trading_signals, + "list_pending_copier_automation_workflow_statuses", + list_pending_mock, + ), + mock.patch( + "octobot_node.scheduler.automations.automation_states_loader.get_automation_copied_strategy_ids", + return_value={"test-strategy-id"}, + ), + mock.patch( + "octobot_node.scheduler.tasks.trigger_copier_automation", + mock.AsyncMock(), + ) as trigger_copier_automation_mock, + ): + await internal_trading_signals._trigger_copier_automation(signal) + + assert trigger_copier_automation_mock.await_count == 2 + triggered_workflow_ids = [ + call.args[0] for call in trigger_copier_automation_mock.await_args_list + ] + assert triggered_workflow_ids == ["automation-wf-1", "automation-wf-2"] + + @pytest.mark.asyncio async def test_insert_trading_signal_completes_without_error_after_subscribe(): async_channel_channels.del_chan(_channel_name()) diff --git a/packages/node/tests/scheduler/test_queues.py b/packages/node/tests/scheduler/test_queues.py new file mode 100644 index 0000000000..7d50b02749 --- /dev/null +++ b/packages/node/tests/scheduler/test_queues.py @@ -0,0 +1,75 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import mock +import pytest + +import octobot_node.enums +import octobot_node.scheduler +import octobot_node.scheduler.queues as scheduler_queues_module + +import tests.scheduler as scheduler_test_util + + +def _expected_register_calls() -> list[tuple[str, dict]]: + return [ + (octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value, {}), + (octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE.value, {}), + ( + octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE.value, + {"global_concurrency": 1}, + ), + ( + octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value, + {"global_concurrency": 1}, + ), + ( + octobot_node.enums.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value, + {"global_concurrency": 1}, + ), + ] + + +def _assert_register_queue_calls(register_mock: mock.Mock) -> None: + assert register_mock.call_count == 5 + expected_calls = _expected_register_calls() + for call_index, (expected_name, expected_kwargs) in enumerate(expected_calls): + call = register_mock.call_args_list[call_index] + assert call.args[0] == expected_name + assert call.kwargs == expected_kwargs + + +class TestSchedulerQueueSpecs: + def test_scheduler_queue_specs_count(self): + assert len(scheduler_queues_module._SCHEDULER_QUEUE_SPECS) == 5 + + +class TestRegisterSchedulerQueues: + def test_registers_all_scheduler_queues_in_order(self): + with mock.patch.object( + scheduler_test_util.dbos.DBOS, + "register_queue", + ) as register_queue_mock: + scheduler_test_util.register_scheduler_queues() + _assert_register_queue_calls(register_queue_mock) + + +class TestRegisterSchedulerQueuesAsync: + @pytest.mark.asyncio + async def test_registers_all_scheduler_queues_in_order(self): + mock_dbos_instance = mock.Mock() + mock_dbos_instance.register_queue_async = mock.AsyncMock() + previous_instance = octobot_node.scheduler.SCHEDULER.INSTANCE + octobot_node.scheduler.SCHEDULER.INSTANCE = mock_dbos_instance + try: + await scheduler_queues_module.register_scheduler_queues_async() + finally: + octobot_node.scheduler.SCHEDULER.INSTANCE = previous_instance + _assert_register_queue_calls(mock_dbos_instance.register_queue_async) diff --git a/packages/node/tests/scheduler/test_scheduler.py b/packages/node/tests/scheduler/test_scheduler.py index fb3592488b..edbaf3f138 100644 --- a/packages/node/tests/scheduler/test_scheduler.py +++ b/packages/node/tests/scheduler/test_scheduler.py @@ -34,6 +34,8 @@ PARENT_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" CHILD_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa-iter-1" +CHILD_INDEX_1_ID = f"{PARENT_ID}_1" +CHILD_INDEX_2_ID = f"{PARENT_ID}_2" def _build_mock_workflow_status(task: octobot_node.models.Task, encrypted_state: str, state_metadata: str, workflow_id: str = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa") -> mock.Mock: @@ -89,6 +91,46 @@ def _make_scheduler_with_mock_instance() -> tuple[scheduler_module.Scheduler, mo return sched, sched.INSTANCE +def _metadata_row_from_hydrated(full_row: mock.Mock) -> mock.Mock: + metadata_row = mock.Mock(spec=dbos.WorkflowStatus) + metadata_row.workflow_id = full_row.workflow_id + metadata_row.status = full_row.status + metadata_row.name = getattr(full_row, "name", None) + metadata_row.error = getattr(full_row, "error", None) + metadata_row.created_at = full_row.created_at + metadata_row.updated_at = full_row.updated_at + metadata_row.queue_name = getattr(full_row, "queue_name", None) + metadata_row.input = None + metadata_row.output = None + return metadata_row + + +def _install_get_results_dbos_list_side_effect( + mock_instance: mock.AsyncMock, + full_rows: list[mock.Mock], +) -> list[mock.Mock]: + hydrated_by_id = {row.workflow_id: row for row in full_rows} + metadata_rows = [_metadata_row_from_hydrated(row) for row in full_rows] + + async def list_workflows_side_effect(**kwargs): + if kwargs.get("workflow_ids"): + batch: list[mock.Mock] = [] + for workflow_id in kwargs["workflow_ids"]: + full_row = hydrated_by_id.get(workflow_id) + if full_row is None: + continue + payload_row = mock.Mock(spec=dbos.WorkflowStatus) + payload_row.workflow_id = workflow_id + payload_row.input = full_row.input if kwargs.get("load_input") else None + payload_row.output = full_row.output if kwargs.get("load_output") else None + batch.append(payload_row) + return batch + return metadata_rows + + mock_instance.list_workflows_async = mock.AsyncMock(side_effect=list_workflows_side_effect) + return metadata_rows + + _DELETE_WORKFLOW_STATUSES = workflows_retention._TERMINAL_DELETE_WORKFLOW_STATUSES @@ -145,7 +187,7 @@ async def test_get_results_returns_empty_result_no_crypto(self): ws = _build_mock_workflow_status(task, "encrypted_state", "state_meta") sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws]) + _install_get_results_dbos_list_side_effect(mock_instance, [ws]) with mock.patch("octobot_node.scheduler.encryption.encrypt_task_result") as mock_crypto: executions = await sched.get_results() @@ -176,7 +218,7 @@ async def test_get_results_sets_is_encrypted_from_content_metadata(self): ws_plain = _build_mock_workflow_status(plain_task, None, None, workflow_id=plain_task.id) sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws_enc, ws_plain]) + _install_get_results_dbos_list_side_effect(mock_instance, [ws_enc, ws_plain]) executions = await sched.get_results() @@ -206,7 +248,7 @@ async def test_get_results_dbos_error_workflow_sets_error(self): ws.updated_at = None sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws]) + _install_get_results_dbos_list_side_effect(mock_instance, [ws]) executions = await sched.get_results() @@ -237,7 +279,7 @@ async def test_get_results_dbos_error_no_exception_fallback(self): ws.updated_at = None sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws]) + _install_get_results_dbos_list_side_effect(mock_instance, [ws]) executions = await sched.get_results() @@ -266,7 +308,7 @@ async def test_get_results_success_with_output_error_marks_failed(self): ws.updated_at = None sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws]) + _install_get_results_dbos_list_side_effect(mock_instance, [ws]) executions = await sched.get_results() @@ -299,7 +341,7 @@ async def test_get_results_success_with_malformed_output_falls_back_to_completed ws.updated_at = None sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws]) + _install_get_results_dbos_list_side_effect(mock_instance, [ws]) executions = await sched.get_results() @@ -322,7 +364,7 @@ async def test_get_results_wallet_filter_keeps_legacy_task_without_wallet_addres ws = _build_mock_workflow_status(legacy_task, None, None, workflow_id=legacy_task.id) sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws]) + _install_get_results_dbos_list_side_effect(mock_instance, [ws]) executions = await sched.get_results(user_id="0xcaller") @@ -345,7 +387,7 @@ async def test_get_results_wallet_filter_keeps_workflow_with_unparseable_input(s ws.updated_at = None sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws]) + _install_get_results_dbos_list_side_effect(mock_instance, [ws]) executions = await sched.get_results(user_id="0xcaller") @@ -365,12 +407,203 @@ async def test_get_results_wallet_filter_drops_task_with_explicit_other_wallet(s ws = _build_mock_workflow_status(other_task, None, None, workflow_id=other_task.id) sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws]) + _install_get_results_dbos_list_side_effect(mock_instance, [ws]) executions = await sched.get_results(user_id="0xcaller") assert executions == [] + @pytest.mark.asyncio + async def test_get_results_cancelled_latest_has_empty_result(self): + task = octobot_node.models.Task( + id=PARENT_ID, + name="cancelled-automation", + content="input-state", + type="execute_actions", + ) + prior_ws = _build_mock_workflow_status(task, "prior_plain_state", None, workflow_id=CHILD_INDEX_1_ID) + prior_ws.updated_at = 10 + cancelled_ws = _build_mock_workflow_status_no_output(task, workflow_id=CHILD_INDEX_2_ID) + cancelled_ws.status = dbos.WorkflowStatusString.CANCELLED.value + cancelled_ws.name = octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value + cancelled_ws.queue_name = None + cancelled_ws.updated_at = 20 + + sched, mock_instance = _make_scheduler_with_mock_instance() + _install_get_results_dbos_list_side_effect(mock_instance, [prior_ws, cancelled_ws]) + + executions = await sched.get_results() + metadata_call = mock_instance.list_workflows_async.await_args_list[0] + list_kwargs = metadata_call.kwargs + assert list_kwargs["name"] == octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value + assert list_kwargs["load_input"] is False + assert list_kwargs["load_output"] is False + assert "queue_name" not in list_kwargs + + assert len(executions) == 1 + assert executions[0].id == CHILD_INDEX_2_ID + assert executions[0].status == octobot_node.models.TaskStatus.CANCELLED + assert executions[0].result == "" + assert executions[0].result_metadata == "" + + def test_executions_for_automation_group_cancelled_hides_prior_success(self): + task = octobot_node.models.Task( + id=PARENT_ID, + name="cancelled-automation", + content="input-state", + type="execute_actions", + ) + prior_ws = _build_mock_workflow_status(task, "prior_plain_state", None, workflow_id=CHILD_INDEX_1_ID) + prior_ws.updated_at = 10 + cancelled_ws = _build_mock_workflow_status_no_output(task, workflow_id=CHILD_INDEX_2_ID) + cancelled_ws.status = dbos.WorkflowStatusString.CANCELLED.value + cancelled_ws.name = octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value + cancelled_ws.queue_name = None + cancelled_ws.updated_at = 20 + + sched, _mock_instance = _make_scheduler_with_mock_instance() + group = [prior_ws, cancelled_ws] + latest_input_task = workflows_util.get_automation_input_task(cancelled_ws) + + group_executions = sched._executions_for_automation_group( + group, cancelled_ws, latest_input_task + ) + + assert len(group_executions) == 1 + assert group_executions[0].id == CHILD_INDEX_2_ID + assert group_executions[0].status == octobot_node.models.TaskStatus.CANCELLED + + @pytest.mark.asyncio + async def test_get_results_first_iteration_cancelled_has_empty_result(self): + task = octobot_node.models.Task( + id=PARENT_ID, + name="cancelled-first-iteration", + content="input-state", + type="execute_actions", + ) + cancelled_ws = _build_mock_workflow_status_no_output(task, workflow_id=PARENT_ID) + cancelled_ws.status = dbos.WorkflowStatusString.CANCELLED.value + cancelled_ws.name = octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value + cancelled_ws.queue_name = None + + sched, mock_instance = _make_scheduler_with_mock_instance() + _install_get_results_dbos_list_side_effect(mock_instance, [cancelled_ws]) + + executions = await sched.get_results() + + assert len(executions) == 1 + assert executions[0].status == octobot_node.models.TaskStatus.CANCELLED + assert executions[0].result == "" + + @pytest.mark.asyncio + async def test_get_results_lists_metadata_then_hydrates_latest(self): + task = octobot_node.models.Task( + id=PARENT_ID, + name="hydrate-sequence-task", + content=None, + type="execute_actions", + ) + full_row = _build_mock_workflow_status(task, "state", "meta", workflow_id=PARENT_ID) + sched, mock_instance = _make_scheduler_with_mock_instance() + _install_get_results_dbos_list_side_effect(mock_instance, [full_row]) + + await sched.get_results() + + assert mock_instance.list_workflows_async.await_count >= 2 + metadata_kwargs = mock_instance.list_workflows_async.await_args_list[0].kwargs + assert metadata_kwargs["load_input"] is False + assert metadata_kwargs["load_output"] is False + hydrate_calls = [ + call.kwargs + for call in mock_instance.list_workflows_async.await_args_list[1:] + if call.kwargs.get("workflow_ids") + ] + assert len(hydrate_calls) == 1 + assert PARENT_ID in hydrate_calls[0]["workflow_ids"] + assert hydrate_calls[0]["load_input"] is True + assert hydrate_calls[0]["load_output"] is True + + @pytest.mark.asyncio + async def test_get_results_hydrates_latest_with_input_and_output(self): + task = octobot_node.models.Task( + id=PARENT_ID, + name="error-latest-task", + content=None, + type="execute_actions", + ) + full_row = _build_mock_workflow_status_error( + task, RuntimeError("failed"), workflow_id=PARENT_ID + ) + sched, mock_instance = _make_scheduler_with_mock_instance() + _install_get_results_dbos_list_side_effect(mock_instance, [full_row]) + + await sched.get_results() + + hydrate_calls = [ + call.kwargs + for call in mock_instance.list_workflows_async.await_args_list[1:] + if call.kwargs.get("workflow_ids") + ] + assert len(hydrate_calls) == 1 + assert PARENT_ID in hydrate_calls[0]["workflow_ids"] + assert hydrate_calls[0]["load_input"] is True + assert hydrate_calls[0]["load_output"] is True + + @pytest.mark.asyncio + async def test_get_results_historical_success_and_error_latest(self): + task = octobot_node.models.Task( + id=PARENT_ID, + name="multi-child-task", + content=None, + type="execute_actions", + ) + prior_success = _build_mock_workflow_status(task, "old-state", None, workflow_id=CHILD_INDEX_1_ID) + prior_success.updated_at = 10 + latest_error = _build_mock_workflow_status_error( + task, RuntimeError("latest failure"), workflow_id=CHILD_INDEX_2_ID + ) + + sched, mock_instance = _make_scheduler_with_mock_instance() + _install_get_results_dbos_list_side_effect(mock_instance, [prior_success, latest_error]) + + executions = await sched.get_results() + + assert len(executions) == 2 + by_id = {execution.id: execution for execution in executions} + assert by_id[CHILD_INDEX_1_ID].status == octobot_node.models.TaskStatus.COMPLETED + assert by_id[CHILD_INDEX_2_ID].status == octobot_node.models.TaskStatus.FAILED + assert "latest failure" in by_id[CHILD_INDEX_2_ID].error + + @pytest.mark.asyncio + async def test_get_results_historical_success_output_error_not_failed(self): + task = octobot_node.models.Task( + id=PARENT_ID, + name="tradeoff-task", + content=None, + type="execute_actions", + ) + stale_output = params.AutomationWorkflowOutput(error="stale graceful failure") + stale_success = mock.Mock(spec=dbos.WorkflowStatus) + stale_success.workflow_id = CHILD_INDEX_1_ID + stale_success.status = dbos.WorkflowStatusString.SUCCESS.value + stale_success.output = json.dumps(stale_output.to_dict()) + stale_success.input = {"args": [params.AutomationWorkflowInputs(task=task).to_dict()], "kwargs": {}} + stale_success.created_at = None + stale_success.updated_at = 10 + stale_success.error = None + + latest_success = _build_mock_workflow_status(task, "clean-state", None, workflow_id=CHILD_INDEX_2_ID) + latest_success.updated_at = 20 + + sched, mock_instance = _make_scheduler_with_mock_instance() + _install_get_results_dbos_list_side_effect(mock_instance, [stale_success, latest_success]) + + executions = await sched.get_results() + + by_id = {execution.id: execution for execution in executions} + assert by_id[CHILD_INDEX_1_ID].status == octobot_node.models.TaskStatus.COMPLETED + assert by_id[CHILD_INDEX_2_ID].status == octobot_node.models.TaskStatus.COMPLETED + class TestGetWorkflowsExportResults: @@ -386,7 +619,7 @@ def _make_task(self, user_id: str = "wallet-a") -> octobot_node.models.Task: @pytest.mark.asyncio async def test_picks_child_with_output_for_multi_iteration_parent(self): - """For multi-iteration task: parent output=None, child has actual output → child is used.""" + """Latest child run with persisted output.state is exported.""" task = self._make_task() parent_ws = _build_mock_workflow_status_no_output(task, workflow_id=PARENT_ID) child_output = params.AutomationWorkflowOutput(state="encrypted_state", state_metadata=None) @@ -430,12 +663,13 @@ def _fake_encrypted_task(result_task): async def test_skips_crypto_when_encryption_disabled(self): """When node-side encryption is off, state is returned as-is without crypto.""" task = self._make_task() - child_ws = _build_mock_workflow_status(task, "plain_state", None, workflow_id=CHILD_ID) - child_ws.updated_at = 10 - parent_ws = _build_mock_workflow_status_no_output(task, workflow_id=PARENT_ID) + prior_ws = _build_mock_workflow_status_no_output(task, workflow_id=CHILD_INDEX_1_ID) + prior_ws.updated_at = 10 + child_ws = _build_mock_workflow_status(task, "plain_state", None, workflow_id=CHILD_INDEX_2_ID) + child_ws.updated_at = 20 sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[parent_ws, child_ws]) + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[prior_ws, child_ws]) with mock.patch.object( type(octobot_node.config.settings), @@ -451,26 +685,37 @@ async def test_skips_crypto_when_encryption_disabled(self): mock_encrypt.assert_not_called() @pytest.mark.asyncio - async def test_returns_empty_when_no_workflow_has_output(self): - """All workflows in group have output=None → returns empty result, no exception.""" + async def test_returns_latest_run_input_when_no_workflow_has_output(self): + """Latest run has output=None → export workflow input task content.""" task = self._make_task() parent_ws = _build_mock_workflow_status_no_output(task, workflow_id=PARENT_ID) sched, mock_instance = _make_scheduler_with_mock_instance() mock_instance.list_workflows_async = mock.AsyncMock(return_value=[parent_ws]) - with mock.patch( + with mock.patch.object( + type(octobot_node.config.settings), + "is_node_side_encryption_enabled", + new_callable=mock.PropertyMock, + return_value=False, + ), mock.patch( "octobot_node.scheduler.encryption.encrypt_task_result" ) as mock_encrypt: result = await sched.get_workflows_export_results([PARENT_ID], None) - assert result[PARENT_ID] == {"result": "", "result_metadata": ""} + assert result[PARENT_ID] == {"result": "encrypted_content", "result_metadata": ""} mock_encrypt.assert_not_called() @pytest.mark.asyncio - async def test_returns_error_for_all_failed_group(self): - """All workflows ERROR → error message surfaced in response.""" - task = self._make_task() + async def test_returns_error_when_latest_error_has_no_exportable_content(self): + """Latest ERROR without output or input content → error message in response.""" + task = octobot_node.models.Task( + id=PARENT_ID, + name="test-task", + content=None, + type="execute_actions", + user_id="wallet-a", + ) error_ws = _build_mock_workflow_status_error(task, RuntimeError("boom"), workflow_id=PARENT_ID) sched, mock_instance = _make_scheduler_with_mock_instance() @@ -603,6 +848,74 @@ async def test_request_key_encrypts_plaintext_imported_task(self): call_kwargs = mock_encrypt.call_args assert call_kwargs[1]["rsa_public_key"] == user_key_pem.encode("utf-8") + @pytest.mark.asyncio + async def test_export_cancelled_uses_workflow_input_not_prior_output(self): + task = self._make_task() + prior_ws = _build_mock_workflow_status(task, "prior_export_state", None, workflow_id=CHILD_INDEX_1_ID) + prior_ws.updated_at = 10 + cancelled_ws = _build_mock_workflow_status_no_output(task, workflow_id=CHILD_INDEX_2_ID) + cancelled_ws.status = dbos.WorkflowStatusString.CANCELLED.value + cancelled_ws.updated_at = 20 + + sched, mock_instance = _make_scheduler_with_mock_instance() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[prior_ws, cancelled_ws]) + + with mock.patch.object( + type(octobot_node.config.settings), + "is_node_side_encryption_enabled", + new_callable=mock.PropertyMock, + return_value=False, + ), mock.patch( + "octobot_node.scheduler.encryption.encrypt_task_result" + ) as mock_encrypt: + result = await sched.get_workflows_export_results([PARENT_ID], None) + + assert result[PARENT_ID] == {"result": "encrypted_content", "result_metadata": ""} + mock_encrypt.assert_not_called() + + @pytest.mark.asyncio + async def test_export_latest_error_uses_input_not_prior_success_output(self): + task = self._make_task() + prior_ws = _build_mock_workflow_status(task, "prior_export_state", None, workflow_id=CHILD_INDEX_1_ID) + prior_ws.updated_at = 10 + error_ws = _build_mock_workflow_status_error(task, RuntimeError("boom"), workflow_id=CHILD_INDEX_2_ID) + error_ws.updated_at = 20 + + sched, mock_instance = _make_scheduler_with_mock_instance() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[prior_ws, error_ws]) + + with mock.patch.object( + type(octobot_node.config.settings), + "is_node_side_encryption_enabled", + new_callable=mock.PropertyMock, + return_value=False, + ), mock.patch( + "octobot_node.scheduler.encryption.encrypt_task_result" + ) as mock_encrypt: + result = await sched.get_workflows_export_results([PARENT_ID], None) + + assert result[PARENT_ID] == {"result": "encrypted_content", "result_metadata": ""} + mock_encrypt.assert_not_called() + + @pytest.mark.asyncio + async def test_export_cancelled_without_input_content_returns_empty(self): + task = octobot_node.models.Task( + id=PARENT_ID, + name="test-task", + content=None, + type="execute_actions", + user_id="wallet-a", + ) + cancelled_ws = _build_mock_workflow_status_no_output(task, workflow_id=PARENT_ID) + cancelled_ws.status = dbos.WorkflowStatusString.CANCELLED.value + + sched, mock_instance = _make_scheduler_with_mock_instance() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[cancelled_ws]) + + result = await sched.get_workflows_export_results([PARENT_ID], None) + + assert result[PARENT_ID] == {"result": "", "result_metadata": ""} + class TestSchedulerGetPendingTasks: @@ -883,11 +1196,14 @@ async def test_list_user_actions_uses_explicit_non_terminal_and_terminal_status_ status.value for status in workflows_util.get_user_action_input_workflow_statuses() } expected_terminal_status_values = { - status.value for status in workflows_util.get_user_action_terminal_workflow_statuses() + status.value for status in workflows_util.DBOS_TERMINAL_WORKFLOW_STATUSES } assert input_status_values == expected_input_status_values assert terminal_status_values == expected_terminal_status_values assert "SUCCESS" not in input_status_values + for call in mock_instance.list_workflows_async.await_args_list: + assert call.kwargs["name"] == octobot_node.enums.SchedulerWorkflowNames.EXECUTE_USER_ACTION.value + assert "queue_name" not in call.kwargs @pytest.mark.asyncio async def test_active_only_false_terminal_unparseable_appears_once(self): @@ -1046,10 +1362,29 @@ async def test_returns_none_when_no_matching_workflow(self): assert result is None +class TestGetLatestWorkflowForEachAutomation: + @pytest.mark.asyncio + async def test_list_workflows_loads_input_for_automation_state_paths(self): + scheduler = scheduler_module.Scheduler() + scheduler.INSTANCE = mock.Mock() + with mock.patch.object( + scheduler, + "_list_workflows", + mock.AsyncMock(return_value=[]), + ) as list_workflows_mock: + await scheduler._get_latest_workflow_for_each_automation( + "0xwallet", + None, + load_output=False, + ) + list_workflows_mock.assert_awaited_once() + assert list_workflows_mock.await_args.kwargs.get("load_input") is True + + class TestResolveLatestTerminalAutomationWorkflowForParentId: @pytest.mark.asyncio async def test_returns_latest_terminal_when_only_input_state_is_usable(self): - from tests.scheduler.test_workflows_util_automation_state import ( + from tests.scheduler.test_workflows_util import ( _automation_task_content, _child_workflow_id, _workflow_status_with_automation_task, @@ -1129,3 +1464,93 @@ async def test_delegates_to_workflows_retention_helpers(self): mock_instance, merged_workflow_ids, ) + + +class TestGetParentAndChildrenAutomationWorkflowsDbosKwargs: + @pytest.mark.asyncio + async def test_pending_passes_queues_only_true_prefix_omit_status(self): + sched, mock_instance = _make_scheduler_with_mock_instance() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[]) + await sched._get_parent_and_children_automation_workflows( + "0xw1", + [PARENT_ID], + None, + load_output=False, + queues_only=True, + ) + mock_instance.list_workflows_async.assert_awaited_once() + call_kwargs = mock_instance.list_workflows_async.await_args.kwargs + assert call_kwargs.get("queues_only") is True + assert call_kwargs.get("workflow_id_prefix") == [PARENT_ID] + assert "status" not in call_kwargs + + @pytest.mark.asyncio + async def test_terminal_passes_queues_only_false_prefix_and_status(self): + sched, mock_instance = _make_scheduler_with_mock_instance() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[]) + terminal_statuses = [ + dbos.WorkflowStatusString.SUCCESS, + dbos.WorkflowStatusString.ERROR, + ] + await sched._get_parent_and_children_automation_workflows( + "0xw1", + [PARENT_ID], + terminal_statuses, + load_output=True, + queues_only=False, + ) + mock_instance.list_workflows_async.assert_awaited_once() + call_kwargs = mock_instance.list_workflows_async.await_args.kwargs + assert call_kwargs.get("queues_only") is not True + assert call_kwargs.get("workflow_id_prefix") == [PARENT_ID] + assert call_kwargs.get("status") == [status.value for status in terminal_statuses] + + @pytest.mark.asyncio + async def test_multi_parent_prefix_list(self): + sched, mock_instance = _make_scheduler_with_mock_instance() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[]) + other_parent = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + await sched._get_parent_and_children_automation_workflows( + None, + [PARENT_ID, other_parent, PARENT_ID], + None, + queues_only=True, + ) + call_kwargs = mock_instance.list_workflows_async.await_args.kwargs + assert call_kwargs.get("workflow_id_prefix") == [PARENT_ID, other_parent] + + +class TestGetPendingTasksDbosKwargs: + @pytest.mark.asyncio + async def test_list_workflows_uses_queues_only_without_status(self): + sched, mock_instance = _make_scheduler_with_mock_instance() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[]) + with mock.patch( + "octobot_node.scheduler.automations.automation_states_loader.get_automation_state_reader", + return_value=None, + ): + await sched.get_pending_tasks() + mock_instance.list_workflows_async.assert_awaited_once() + call_kwargs = mock_instance.list_workflows_async.await_args.kwargs + assert call_kwargs.get("queues_only") is True + assert "status" not in call_kwargs + + +class TestCancelWorkflowsQueuesOnly: + @pytest.mark.asyncio + async def test_cancel_passes_queues_only_true_to_get_parent_and_children(self): + sched, mock_instance = _make_scheduler_with_mock_instance() + mock_instance.cancel_workflows_async = mock.AsyncMock() + with mock.patch.object( + sched, + "get_parent_and_children_automation_workflow_ids", + new_callable=mock.AsyncMock, + return_value=["wf-1"], + ) as get_ids_mock: + await sched.cancel_workflows([PARENT_ID]) + get_ids_mock.assert_awaited_once_with( + None, + [PARENT_ID], + None, + queues_only=True, + ) diff --git a/packages/node/tests/scheduler/test_schedules.py b/packages/node/tests/scheduler/test_schedules.py index 108c177553..5592c69952 100644 --- a/packages/node/tests/scheduler/test_schedules.py +++ b/packages/node/tests/scheduler/test_schedules.py @@ -1136,6 +1136,164 @@ async def test_skips_when_flag_disabled(self, portfolio_history_workflow_module, backfill_to_thread_mock.assert_not_awaited() +class TestScheduledWorkflowsDisabled: + pytestmark = pytest.mark.asyncio + + @pytest.fixture + def dbos_cleanup_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module_loaded + + yield dbos_cleanup_workflow_module_loaded + + @pytest.fixture + def global_view_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module_loaded + + yield global_view_workflow_module_loaded + + @pytest.fixture + def portfolio_history_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module_loaded + + yield portfolio_history_workflow_module_loaded + + async def test_register_schedules_when_disabled_removes_existing_schedules( + self, + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + with mock.patch( + "octobot_node.scheduler.schedules.scheduled_workflows_enabled", + return_value=False, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + cleanup_existing=_matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ), + global_view_existing=_matching_existing_global_view_schedule( + global_view_workflow_module, + ), + portfolio_history_existing=_matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ), + ), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.delete_schedule_async", + new_callable=mock.AsyncMock, + ) as delete_schedule_mock, mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", + new_callable=mock.AsyncMock, + ) as create_schedule_mock, mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ) as apply_schedules_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + assert delete_schedule_mock.await_count == 3 + delete_schedule_mock.assert_any_await(dbos_cleanup_workflow_module.SCHEDULE_NAME) + delete_schedule_mock.assert_any_await(global_view_workflow_module.SCHEDULE_NAME) + delete_schedule_mock.assert_any_await(portfolio_history_workflow_module.SCHEDULE_NAME) + create_schedule_mock.assert_not_awaited() + apply_schedules_mock.assert_not_awaited() + + async def test_register_schedules_when_disabled_skips_delete_when_schedule_missing( + self, + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + with mock.patch( + "octobot_node.scheduler.schedules.scheduled_workflows_enabled", + return_value=False, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + cleanup_existing=None, + global_view_existing=None, + portfolio_history_existing=None, + ), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.delete_schedule_async", + new_callable=mock.AsyncMock, + ) as delete_schedule_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + delete_schedule_mock.assert_not_awaited() + + async def test_register_schedules_when_disabled_does_not_run_startup_backfill_or_catch_up( + self, + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + with mock.patch( + "octobot_node.scheduler.schedules.scheduled_workflows_enabled", + return_value=False, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=None, + ), mock.patch( + "octobot_node.scheduler.schedules._maybe_backfill_schedule_on_startup", + new_callable=mock.AsyncMock, + ) as backfill_mock, mock.patch( + "octobot_node.scheduler.schedules._maybe_catch_up_schedule_once_on_startup", + new_callable=mock.AsyncMock, + ) as catch_up_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + backfill_mock.assert_not_awaited() + catch_up_mock.assert_not_awaited() + + async def test_update_cleanup_schedule_cron_when_disabled( + self, + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + desired_cron = "0 */6 * * *" + + with mock.patch( + "octobot_node.scheduler.schedules.scheduled_workflows_enabled", + return_value=False, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + ) as get_schedule_mock, mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ) as apply_schedules_mock: + result = await schedules_module.update_cleanup_schedule_cron( + temp_dbos_scheduler, + desired_cron, + ) + + assert result == {"changed": False, "cron": desired_cron} + get_schedule_mock.assert_not_awaited() + apply_schedules_mock.assert_not_awaited() + + class TestUpdateCleanupScheduleCron: pytestmark = pytest.mark.asyncio diff --git a/packages/node/tests/scheduler/test_shutdown.py b/packages/node/tests/scheduler/test_shutdown.py index 2183bc23d3..fe8b606e03 100644 --- a/packages/node/tests/scheduler/test_shutdown.py +++ b/packages/node/tests/scheduler/test_shutdown.py @@ -48,6 +48,9 @@ async def test_initialize_scheduler_resets_shutdown_guard(self): def track_start() -> None: init_call_order.append("start") + async def track_register_queues(*args, **kwargs) -> None: + init_call_order.append("register_queues") + async def track_register_schedules(*args, **kwargs) -> None: init_call_order.append("register_schedules") @@ -62,18 +65,23 @@ async def track_register_schedules(*args, **kwargs) -> None: "start", side_effect=track_start, ): - with mock.patch("octobot_node.scheduler.workflows.register_workflows"): - with mock.patch.object( - schedules_module, - "register_schedules", - side_effect=track_register_schedules, - ): - scheduler_module._shutdown_done = True - await scheduler_module.initialize_scheduler() + with mock.patch( + "octobot_node.scheduler.queues.register_scheduler_queues_async", + side_effect=track_register_queues, + ): + with mock.patch("octobot_node.scheduler.workflows.register_workflows"): + with mock.patch.object( + schedules_module, + "register_schedules", + side_effect=track_register_schedules, + ): + scheduler_module._shutdown_done = True + await scheduler_module.initialize_scheduler() finally: scheduler_module.SCHEDULER.INSTANCE = previous_instance assert scheduler_module._shutdown_done is False assert init_call_order == [ "start", + "register_queues", "register_schedules", ] diff --git a/packages/node/tests/scheduler/test_tasks.py b/packages/node/tests/scheduler/test_tasks.py index 763823ed48..d515572fa7 100644 --- a/packages/node/tests/scheduler/test_tasks.py +++ b/packages/node/tests/scheduler/test_tasks.py @@ -69,7 +69,9 @@ async def test_trigger_all_task_types(self, schedule_task, temp_dbos_scheduler): for task_type in octobot_node.models.TaskType: schedule_task.type = task_type.value with mock.patch.object( - temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE, "enqueue_async", mock.AsyncMock() + temp_dbos_scheduler.INSTANCE, + "enqueue_workflow_async", + mock.AsyncMock(), ) as mock_enqueue_async: mock_handle = mock.Mock() mock_handle.workflow_id = expected_workflow_id @@ -77,14 +79,18 @@ async def test_trigger_all_task_types(self, schedule_task, temp_dbos_scheduler): result = await octobot_node.scheduler.tasks.trigger_task(schedule_task) assert result == expected_workflow_id mock_enqueue_async.assert_called_once() - call_kwargs = mock_enqueue_async.call_args[1] + call_args = mock_enqueue_async.call_args + assert call_args[0][0] == node_enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value + call_kwargs = call_args[1] assert "inputs" in call_kwargs assert len(call_kwargs["inputs"]) == 1 inputs = call_kwargs["inputs"] assert inputs["task"] == schedule_task.model_dump(exclude_defaults=True) with pytest.raises(ValueError, match="Unsupported task type"): with mock.patch.object( - temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE, "enqueue_async", mock.AsyncMock() + temp_dbos_scheduler.INSTANCE, + "enqueue_workflow_async", + mock.AsyncMock(), ) as mock_enqueue_async: schedule_task.type = "invalid_type" await octobot_node.scheduler.tasks.trigger_task(schedule_task) @@ -105,25 +111,18 @@ async def test_trigger_task_execute_actions_with_target_workflow_id_enqueues_aut schedule_task.type = octobot_node.models.TaskType.EXECUTE_ACTIONS.value mock_handle = mock.Mock() mock_handle.workflow_id = expected_workflow_id - with ( - mock.patch.object( - temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE, - "enqueue_async", - mock.AsyncMock(return_value=mock_handle), - ) as mock_enqueue_automation, - mock.patch.object( - temp_dbos_scheduler.USER_ACTION_QUEUE, - "enqueue_async", - mock.AsyncMock(), - ) as mock_enqueue_user_action, - ): + with mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "enqueue_workflow_async", + mock.AsyncMock(return_value=mock_handle), + ) as mock_enqueue_automation: result = await octobot_node.scheduler.tasks.trigger_task( schedule_task, target_workflow_id=target_workflow_id, ) assert result == expected_workflow_id mock_enqueue_automation.assert_awaited_once() - mock_enqueue_user_action.assert_not_called() + assert mock_enqueue_automation.call_args[0][0] == node_enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value call_kwargs = mock_enqueue_automation.call_args[1] assert call_kwargs["inputs"]["task"] == schedule_task.model_dump(exclude_defaults=True) @@ -160,8 +159,8 @@ async def test_enqueues_execute_user_action_workflow_with_encoded_inputs(self, t test_wallet_address = "0xaaabbbbbccccddddeeeeffff00002222" expected_workflow_handle_identifier = "user-action-workflow-test-id" with mock.patch.object( - temp_dbos_scheduler.USER_ACTION_QUEUE, - "enqueue_async", + temp_dbos_scheduler.INSTANCE, + "enqueue_workflow_async", mock.AsyncMock(), ) as mock_enqueue_async_operation: import octobot_node.scheduler.workflows.user_action_workflow as user_action_workflow_module_loaded @@ -178,8 +177,9 @@ async def test_enqueues_execute_user_action_workflow_with_encoded_inputs(self, t assert enqueue_function_result == expected_workflow_handle_identifier mock_enqueue_async_operation.assert_awaited_once() positional_workflow_targets, enqueue_keyword_arguments = mock_enqueue_async_operation.call_args + assert positional_workflow_targets[0] == node_enums.SchedulerQueues.USER_ACTION_QUEUE.value assert ( - positional_workflow_targets[0] + positional_workflow_targets[1] is user_action_workflow_module_loaded.UserActionWorkflow.execute_user_action ) assert list(enqueue_keyword_arguments) == ["inputs"] @@ -214,8 +214,8 @@ async def test_enqueues_portfolio_history_collection_on_portfolio_history_queue( return_value=scheduled_time, ), mock.patch.object( - temp_dbos_scheduler.PORTFOLIO_HISTORY_QUEUE, - "enqueue_async", + temp_dbos_scheduler.INSTANCE, + "enqueue_workflow_async", mock.AsyncMock(), ) as mock_enqueue_async_operation, ): @@ -230,12 +230,13 @@ async def test_enqueues_portfolio_history_collection_on_portfolio_history_queue( assert enqueue_function_result == expected_workflow_id mock_enqueue_async_operation.assert_awaited_once() positional_workflow_targets, enqueue_keyword_arguments = mock_enqueue_async_operation.call_args + assert positional_workflow_targets[0] == node_enums.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value assert ( - positional_workflow_targets[0] + positional_workflow_targets[1] is portfolio_history_workflow_module_loaded.PortfolioHistoryWorkflow.portfolio_history_collection ) - assert positional_workflow_targets[1] == scheduled_time - assert positional_workflow_targets[2] == collection_params.to_dict(include_default_values=False) + assert positional_workflow_targets[2] == scheduled_time + assert positional_workflow_targets[3] == collection_params.to_dict(include_default_values=False) assert enqueue_keyword_arguments == {} diff --git a/packages/node/tests/scheduler/test_tasks_recovery.py b/packages/node/tests/scheduler/test_tasks_recovery.py index 482f49cfed..d222e1eb10 100644 --- a/packages/node/tests/scheduler/test_tasks_recovery.py +++ b/packages/node/tests/scheduler/test_tasks_recovery.py @@ -28,7 +28,7 @@ import tests.scheduler as scheduler_test_util -QUEUE = dbos.Queue(name="test_queue") +_TEST_QUEUE_NAME = "test_tasks_recovery_queue" WF_TO_CREATE = 10 WF_SLEEP_TIME = 1.5 # note: reducing this value wont speed up the test @@ -66,12 +66,17 @@ async def sleeper_workflow(identifier: float) -> float: logging.info(f"Launching DBOS instance 1 ...") octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + await dbos.DBOS.register_queue_async(_TEST_QUEUE_NAME) logging.info(f"DBOS instance 1 launched") # 1. simple execution t0 = time.time() for i in range(WF_TO_CREATE): - await QUEUE.enqueue_async(Sleeper.sleeper_workflow, i) + await dbos.DBOS.enqueue_workflow_async( + _TEST_QUEUE_NAME, + Sleeper.sleeper_workflow, + i, + ) wfs = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async( status=["ENQUEUED", "PENDING"] ) @@ -94,7 +99,11 @@ async def sleeper_workflow(identifier: float) -> float: # 2. enqueue 10 more and restart for i in range(WF_TO_CREATE): - await QUEUE.enqueue_async(Sleeper.sleeper_workflow, i) + await dbos.DBOS.enqueue_workflow_async( + _TEST_QUEUE_NAME, + Sleeper.sleeper_workflow, + i, + ) logging.info(f"Destroying DBOS instance 1 ...") octobot_node.scheduler.SCHEDULER.INSTANCE.destroy() logging.info(f"DBOS instance 1 destroyed") @@ -103,6 +112,7 @@ async def sleeper_workflow(identifier: float) -> float: logging.info(f"Launching DBOS instance 2 ...") await _init_dbos_scheduler(temp_file.name) octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + await dbos.DBOS.register_queue_async(_TEST_QUEUE_NAME) logging.info(f"DBOS instance 2 launched") all_wfs = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async() assert len(all_wfs) == WF_TO_CREATE * 2 @@ -112,7 +122,11 @@ async def sleeper_workflow(identifier: float) -> float: assert len(pending_wfs) == WF_TO_CREATE # enqueue a second batch of workflows for i in range(WF_TO_CREATE, WF_TO_CREATE*2): - await QUEUE.enqueue_async(Sleeper.sleeper_workflow, i) + await dbos.DBOS.enqueue_workflow_async( + _TEST_QUEUE_NAME, + Sleeper.sleeper_workflow, + i, + ) # Only ENQUEUED/PENDING: part 1 workflows are already SUCCESS (same inputs 0..9) # and must not be awaited again or get_result would duplicate those ids here. wfs_to_finish = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async( @@ -147,6 +161,7 @@ async def sleeper_workflow(identifier: float) -> float: logging.info(f"Launching DBOS instance 3 ...") await _init_dbos_scheduler(temp_file.name) octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + await dbos.DBOS.register_queue_async(_TEST_QUEUE_NAME) logging.info(f"DBOS instance 3 launched") # all 30 worflows are now historized pending_wfs = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async( diff --git a/packages/node/tests/scheduler/test_workflows_retention.py b/packages/node/tests/scheduler/test_workflows_retention.py index 1d9350c90b..9290e88ce4 100644 --- a/packages/node/tests/scheduler/test_workflows_retention.py +++ b/packages/node/tests/scheduler/test_workflows_retention.py @@ -310,10 +310,10 @@ async def test_merges_automation_and_user_action_ids(self): user_action_workflow = _build_user_action_workflow_with_output("ua-delete", "wf-ua-delete") async def list_workflows_side_effect(**kwargs): - queue_name = kwargs.get("queue_name") - if queue_name == [octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value]: + workflow_name = kwargs.get("name") + if workflow_name == octobot_node.enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value: return [automation_workflow] - if queue_name == [octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE.value]: + if workflow_name == octobot_node.enums.SchedulerWorkflowNames.EXECUTE_USER_ACTION.value: return [user_action_workflow] return [] @@ -521,13 +521,14 @@ async def test_returns_false_when_cleanup_never_ran(self, temp_dbos_scheduler): assert result is False mock_instance.list_workflows_async.assert_awaited_once_with( - name="dbos_cleanup", + name=octobot_node.enums.SchedulerWorkflowNames.DBOS_CLEANUP.value, status=[dbos.WorkflowStatusString.SUCCESS.value], sort_desc=True, limit=1, load_input=False, load_output=False, ) + assert "queue_name" not in mock_instance.list_workflows_async.await_args.kwargs @pytest.mark.asyncio async def test_returns_true_when_latest_cleanup_is_newer_than_scheduled_time(self, temp_dbos_scheduler): diff --git a/packages/node/tests/scheduler/test_workflows_util.py b/packages/node/tests/scheduler/test_workflows_util.py new file mode 100644 index 0000000000..a1e9113690 --- /dev/null +++ b/packages/node/tests/scheduler/test_workflows_util.py @@ -0,0 +1,522 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import json + +import dbos +import mock +import pytest + +import octobot_node.enums as octobot_node_enums +import octobot_node.models as node_models +import octobot_node.scheduler.workflows.params as workflow_params +import octobot_node.scheduler.workflows_util as workflows_util +import octobot_protocol.models as protocol_models + + +class TestListSchedulerWorkflowsAsync: + @pytest.mark.asyncio + async def test_lists_by_workflow_name_without_queue_name(self): + mock_dbos = mock.AsyncMock() + mock_dbos.list_workflows_async = mock.AsyncMock(return_value=[]) + await workflows_util.list_scheduler_workflows_async( + mock_dbos, + octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + [dbos.WorkflowStatusString.PENDING], + None, + load_output=True, + ) + mock_dbos.list_workflows_async.assert_awaited_once_with( + name=octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value, + status=[dbos.WorkflowStatusString.PENDING.value], + load_output=True, + load_input=False, + ) + assert "queue_name" not in mock_dbos.list_workflows_async.await_args.kwargs + + @pytest.mark.asyncio + async def test_applies_wallet_filter_for_automation_when_user_id_set(self): + task = node_models.Task( + name="t", + content=None, + type="execute_actions", + user_id="0xmine", + ) + inputs = workflow_params.AutomationWorkflowInputs(task=task, execution_time=0) + matching_row = mock.Mock(spec=dbos.WorkflowStatus) + matching_row.workflow_id = "wf-mine" + matching_row.input = {"args": [inputs.to_dict()], "kwargs": {}} + other_task = node_models.Task( + name="t2", + content=None, + type="execute_actions", + user_id="0xother", + ) + other_inputs = workflow_params.AutomationWorkflowInputs(task=other_task, execution_time=0) + other_row = mock.Mock(spec=dbos.WorkflowStatus) + other_row.workflow_id = "wf-other" + other_row.input = {"args": [other_inputs.to_dict()], "kwargs": {}} + + mock_dbos = mock.AsyncMock() + mock_dbos.list_workflows_async = mock.AsyncMock(return_value=[matching_row, other_row]) + listed = await workflows_util.list_scheduler_workflows_async( + mock_dbos, + octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + [dbos.WorkflowStatusString.PENDING], + "0xmine", + load_output=False, + ) + assert listed == [matching_row] + + +class TestListSchedulerWorkflowsAsyncDbosKwargs: + @pytest.mark.asyncio + async def test_forwards_workflow_id_prefix(self): + mock_dbos = mock.AsyncMock() + mock_dbos.list_workflows_async = mock.AsyncMock(return_value=[]) + parent_id = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + await workflows_util.list_scheduler_workflows_async( + mock_dbos, + octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + None, + None, + load_output=False, + workflow_id_prefix=parent_id, + queues_only=True, + ) + mock_dbos.list_workflows_async.assert_awaited_once_with( + name=octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value, + load_output=False, + load_input=False, + workflow_id_prefix=parent_id, + queues_only=True, + ) + assert "status" not in mock_dbos.list_workflows_async.await_args.kwargs + + @pytest.mark.asyncio + async def test_queues_only_without_statuses_omits_status_kwarg(self): + mock_dbos = mock.AsyncMock() + mock_dbos.list_workflows_async = mock.AsyncMock(return_value=[]) + await workflows_util.list_scheduler_workflows_async( + mock_dbos, + octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + None, + None, + load_output=False, + queues_only=True, + ) + call_kwargs = mock_dbos.list_workflows_async.await_args.kwargs + assert call_kwargs.get("queues_only") is True + assert "status" not in call_kwargs + + @pytest.mark.asyncio + async def test_queues_only_with_statuses_passes_both(self): + mock_dbos = mock.AsyncMock() + mock_dbos.list_workflows_async = mock.AsyncMock(return_value=[]) + await workflows_util.list_scheduler_workflows_async( + mock_dbos, + octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + [dbos.WorkflowStatusString.PENDING], + None, + load_output=False, + queues_only=True, + ) + mock_dbos.list_workflows_async.assert_awaited_once_with( + name=octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value, + load_output=False, + load_input=False, + queues_only=True, + status=[dbos.WorkflowStatusString.PENDING.value], + ) + + @pytest.mark.asyncio + async def test_non_queues_only_passes_status(self): + mock_dbos = mock.AsyncMock() + mock_dbos.list_workflows_async = mock.AsyncMock(return_value=[]) + await workflows_util.list_scheduler_workflows_async( + mock_dbos, + octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + [dbos.WorkflowStatusString.SUCCESS], + None, + load_output=True, + ) + mock_dbos.list_workflows_async.assert_awaited_once_with( + name=octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION.value, + status=[dbos.WorkflowStatusString.SUCCESS.value], + load_output=True, + load_input=False, + ) + assert "queues_only" not in mock_dbos.list_workflows_async.await_args.kwargs + + +class TestHydrateSchedulerWorkflowsAsync: + @pytest.mark.asyncio + async def test_merges_input_and_output_onto_metadata_rows(self): + metadata_row = mock.Mock(spec=dbos.WorkflowStatus) + metadata_row.workflow_id = "wf-1" + metadata_row.input = None + metadata_row.output = None + + hydrated_row = mock.Mock(spec=dbos.WorkflowStatus) + hydrated_row.workflow_id = "wf-1" + hydrated_row.input = {"args": [], "kwargs": {}} + hydrated_row.output = '{"state": "x"}' + + mock_dbos = mock.AsyncMock() + + async def list_workflows_side_effect(**kwargs): + if kwargs.get("workflow_ids") == ["wf-1"]: + if kwargs.get("load_input"): + row = mock.Mock(spec=dbos.WorkflowStatus) + row.workflow_id = "wf-1" + row.input = hydrated_row.input + row.output = None + return [row] + if kwargs.get("load_output"): + row = mock.Mock(spec=dbos.WorkflowStatus) + row.workflow_id = "wf-1" + row.input = None + row.output = hydrated_row.output + return [row] + return [] + + mock_dbos.list_workflows_async = mock.AsyncMock(side_effect=list_workflows_side_effect) + + await workflows_util.hydrate_scheduler_workflows_async( + mock_dbos, + octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + [metadata_row], + ["wf-1"], + load_input=True, + load_output=False, + ) + await workflows_util.hydrate_scheduler_workflows_async( + mock_dbos, + octobot_node_enums.SchedulerWorkflowNames.EXECUTE_AUTOMATION, + [metadata_row], + ["wf-1"], + load_input=False, + load_output=True, + ) + + assert metadata_row.input == hydrated_row.input + assert metadata_row.output == hydrated_row.output + + +_AUTOMATION_STATE_KEY = "state" + +_PARENT_WORKFLOW_ID = "741ce171-dac9-40be-83dc-b443c0eaf0e2" + + +def _child_workflow_id(child_index: int) -> str: + if child_index == 0: + return _PARENT_WORKFLOW_ID + return f"{_PARENT_WORKFLOW_ID}_{child_index}" + + +def _workflow_status_row( + *, + workflow_id: str, + updated_at: int = 0, + status: str = dbos.WorkflowStatusString.ENQUEUED.value, +) -> mock.Mock: + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = workflow_id + workflow_status.updated_at = updated_at + workflow_status.status = status + return workflow_status + + +def _automation_task_content(*, automation_name: str) -> str: + return json.dumps( + { + _AUTOMATION_STATE_KEY: { + "automation": { + "metadata": { + "automation_id": "automation_1", + "name": automation_name, + }, + "actions_dag": {"actions": []}, + "execution": {}, + }, + }, + } + ) + + +def _workflow_status_with_automation_task( + *, + status: str, + input_content: str, + output_content: str | None = None, +) -> mock.Mock: + task = node_models.Task( + name="automation-task", + content=input_content, + type=node_models.TaskType.EXECUTE_ACTIONS.value, + ) + encoded_inputs = workflow_params.AutomationWorkflowInputs(task=task).to_dict( + include_default_values=False + ) + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = "parent-workflow-id_1" + workflow_status.status = status + workflow_status.input = {"args": [encoded_inputs], "kwargs": {}} + if output_content is None: + workflow_status.output = None + else: + workflow_status.output = json.dumps( + workflow_params.AutomationWorkflowOutput(state=output_content).to_dict( + include_default_values=False + ) + ) + return workflow_status + + +class TestNormalizeParentAutomationId: + def test_parent_workflow_id_unchanged(self): + assert workflows_util.normalize_parent_automation_id(_PARENT_WORKFLOW_ID) == _PARENT_WORKFLOW_ID + + def test_child_workflow_id_truncated_to_parent(self): + child_id = _child_workflow_id(5) + assert workflows_util.normalize_parent_automation_id(child_id) == _PARENT_WORKFLOW_ID + + +class TestBuildNextChildAutomationWorkflowId: + def test_parent_workflow_id_maps_to_first_child(self): + assert ( + workflows_util.build_next_child_automation_workflow_id(_PARENT_WORKFLOW_ID) + == _child_workflow_id(1) + ) + + def test_child_workflow_id_increments_suffix(self): + assert ( + workflows_util.build_next_child_automation_workflow_id(_child_workflow_id(2)) + == _child_workflow_id(3) + ) + + def test_invalid_suffix_raises_value_error(self): + invalid_child_id = f"{_PARENT_WORKFLOW_ID}-4-4" + with pytest.raises(ValueError, match="Invalid child workflow suffix format"): + workflows_util.build_next_child_automation_workflow_id(invalid_child_id) + + +class TestParseAutomationChildWorkflowIndex: + def test_parent_workflow_id_maps_to_zero(self): + assert workflows_util.parse_automation_child_workflow_index(_PARENT_WORKFLOW_ID) == 0 + + def test_underscore_suffix_maps_to_child_index(self): + assert workflows_util.parse_automation_child_workflow_index(_child_workflow_id(1)) == 1 + assert workflows_util.parse_automation_child_workflow_index(_child_workflow_id(12)) == 12 + + def test_hyphen_suffix_maps_to_child_index(self): + hyphen_child_id = f"{_PARENT_WORKFLOW_ID}-3" + assert workflows_util.parse_automation_child_workflow_index(hyphen_child_id) == 3 + + def test_non_numeric_suffix_raises_value_error(self): + with pytest.raises(ValueError, match="Invalid child workflow suffix format"): + workflows_util.parse_automation_child_workflow_index(f"{_PARENT_WORKFLOW_ID}_abc") + + def test_multi_segment_hyphen_suffix_raises_value_error(self): + with pytest.raises(ValueError, match="Invalid child workflow suffix format"): + workflows_util.parse_automation_child_workflow_index(f"{_PARENT_WORKFLOW_ID}-4-4") + + def test_invalid_suffix_format_raises_value_error(self): + with pytest.raises(ValueError, match="Invalid child workflow suffix format"): + workflows_util.parse_automation_child_workflow_index(f"{_PARENT_WORKFLOW_ID}abc") + + +class TestGetLatestChildWorkflow: + def test_picks_highest_child_suffix_over_newer_updated_at(self): + finished_child = _workflow_status_row( + workflow_id=_child_workflow_id(56), + updated_at=200, + status=dbos.WorkflowStatusString.SUCCESS.value, + ) + waiting_child = _workflow_status_row( + workflow_id=_child_workflow_id(57), + updated_at=100, + status=dbos.WorkflowStatusString.ENQUEUED.value, + ) + chosen = workflows_util.get_latest_child_workflow([finished_child, waiting_child]) + assert chosen.workflow_id == _child_workflow_id(57) + + def test_picks_child_over_parent(self): + parent_workflow = _workflow_status_row(workflow_id=_child_workflow_id(0), updated_at=50) + child_workflow = _workflow_status_row(workflow_id=_child_workflow_id(1), updated_at=10) + chosen = workflows_util.get_latest_child_workflow([parent_workflow, child_workflow]) + assert chosen.workflow_id == _child_workflow_id(1) + + def test_single_element_list_returns_that_workflow(self): + only_workflow = _workflow_status_row(workflow_id=_child_workflow_id(4), updated_at=1) + chosen = workflows_util.get_latest_child_workflow([only_workflow]) + assert chosen is only_workflow + + def test_equal_suffix_tie_breaks_on_updated_at(self): + older_duplicate = _workflow_status_row(workflow_id=_child_workflow_id(5), updated_at=10) + newer_duplicate = _workflow_status_row(workflow_id=_child_workflow_id(5), updated_at=20) + chosen = workflows_util.get_latest_child_workflow([older_duplicate, newer_duplicate]) + assert chosen is newer_duplicate + + def test_unparseable_suffix_does_not_win_over_valid_child(self): + unrelated_workflow = _workflow_status_row( + workflow_id=f"{_PARENT_WORKFLOW_ID}-4-4", + updated_at=999, + ) + valid_child = _workflow_status_row( + workflow_id=_child_workflow_id(57), + updated_at=1, + ) + chosen = workflows_util.get_latest_child_workflow([unrelated_workflow, valid_child]) + assert chosen.workflow_id == _child_workflow_id(57) + + +class TestGetLatestWorkflow: + def test_delegates_to_latest_child_workflow(self): + parent_workflow = _workflow_status_row(workflow_id=_child_workflow_id(0), updated_at=100) + child_workflow = _workflow_status_row(workflow_id=_child_workflow_id(2), updated_at=1) + chosen = workflows_util.get_latest_workflow([parent_workflow, child_workflow]) + assert chosen.workflow_id == _child_workflow_id(2) + + +class TestGetResolvedAutomationTask: + def test_pending_workflow_uses_input_task_content(self): + input_content = _automation_task_content(automation_name="from-input") + output_content = _automation_task_content(automation_name="from-output") + workflow_status = _workflow_status_with_automation_task( + status=dbos.WorkflowStatusString.PENDING.value, + input_content=input_content, + output_content=output_content, + ) + + resolved_task = workflows_util.get_resolved_automation_task(workflow_status) + + assert resolved_task is not None + assert resolved_task.content == input_content + + def test_success_workflow_uses_output_state(self): + input_content = _automation_task_content(automation_name="from-input") + output_content = _automation_task_content(automation_name="from-output") + workflow_status = _workflow_status_with_automation_task( + status=dbos.WorkflowStatusString.SUCCESS.value, + input_content=input_content, + output_content=output_content, + ) + + resolved_task = workflows_util.get_resolved_automation_task(workflow_status) + + assert resolved_task is not None + assert resolved_task.content == output_content + + def test_error_workflow_uses_output_state(self): + input_content = _automation_task_content(automation_name="from-input") + output_content = _automation_task_content(automation_name="from-output") + workflow_status = _workflow_status_with_automation_task( + status=dbos.WorkflowStatusString.ERROR.value, + input_content=input_content, + output_content=output_content, + ) + + resolved_task = workflows_util.get_resolved_automation_task(workflow_status) + + assert resolved_task is not None + assert resolved_task.content == output_content + + def test_error_workflow_falls_back_to_input_when_output_state_missing(self): + input_content = _automation_task_content(automation_name="from-input") + workflow_status = _workflow_status_with_automation_task( + status=dbos.WorkflowStatusString.ERROR.value, + input_content=input_content, + output_content=None, + ) + workflow_status.output = json.dumps( + workflow_params.AutomationWorkflowOutput(state=None).to_dict( + include_default_values=False + ) + ) + + resolved_task = workflows_util.get_resolved_automation_task(workflow_status) + + assert resolved_task is not None + assert resolved_task.content == input_content + + +class TestResolveAutomationResultForGroup: + def test_cancelled_latest_chooses_cancelled_child(self): + prior_child = _workflow_status_row(workflow_id=_child_workflow_id(1), updated_at=10) + cancelled_child = _workflow_status_row( + workflow_id=_child_workflow_id(2), + updated_at=20, + status=dbos.WorkflowStatusString.CANCELLED.value, + ) + chosen = workflows_util.resolve_automation_result_for_group([prior_child, cancelled_child]) + assert chosen is cancelled_child + + def test_latest_success_with_output_chooses_that_child(self): + success_child = _workflow_status_row( + workflow_id=_child_workflow_id(1), + updated_at=10, + status=dbos.WorkflowStatusString.SUCCESS.value, + ) + success_child.output = json.dumps( + workflow_params.AutomationWorkflowOutput(state="state-1").to_dict( + include_default_values=False + ) + ) + chosen = workflows_util.resolve_automation_result_for_group([success_child]) + assert chosen is success_child + + +class TestResolveUserActionWorkflowInputs: + def test_unwraps_dbos_kwargs_inputs_key(self): + user_action = protocol_models.UserAction(id="ua-kwargs", configuration=None) + encoded = workflow_params.UserActionWorkflowInputs( + user_id="0xkwargs", + user_action=user_action, + ).to_dict(include_default_values=False) + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = "wf-kwargs" + workflow_status.input = {"args": [], "kwargs": {"inputs": encoded}} + + resolved = workflows_util.resolve_user_action_workflow_inputs(workflow_status) + + assert resolved.inputs is not None + assert resolved.inputs.user_id == "0xkwargs" + assert resolved.inputs.user_action.id == "ua-kwargs" + + def test_unwraps_portable_json_named_args_inputs(self): + user_action = protocol_models.UserAction(id="ua-portable", configuration=None) + encoded = workflow_params.UserActionWorkflowInputs( + user_id="0xportable", + user_action=user_action, + ).to_dict(include_default_values=False) + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = "wf-portable" + workflow_status.input = { + "args": [ + { + "positionalArgs": [], + "namedArgs": {"inputs": encoded}, + }, + ], + "kwargs": {}, + } + + resolved = workflows_util.resolve_user_action_workflow_inputs(workflow_status) + + assert resolved.inputs is not None + assert resolved.inputs.user_id == "0xportable" + assert resolved.inputs.user_action.id == "ua-portable" + + def test_empty_wrappers_report_no_inputs_not_missing_fields(self): + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = "wf-empty" + workflow_status.input = { + "args": [{"positionalArgs": [], "namedArgs": {}}], + "kwargs": {}, + } + + resolved = workflows_util.resolve_user_action_workflow_inputs(workflow_status) + + assert resolved.inputs is None + assert resolved.parse_error == "no user-action workflow inputs found" diff --git a/packages/node/tests/scheduler/test_workflows_util_automation_state.py b/packages/node/tests/scheduler/test_workflows_util_automation_state.py deleted file mode 100644 index f8d892f9b6..0000000000 --- a/packages/node/tests/scheduler/test_workflows_util_automation_state.py +++ /dev/null @@ -1,249 +0,0 @@ -# Drakkar-Software OctoBot-Node -# Copyright (c) 2025 Drakkar-Software, All rights reserved. - -import json - -import dbos -import mock -import pytest -import octobot_node.models as node_models -import octobot_node.scheduler.workflows.params as workflow_params -import octobot_node.scheduler.workflows_util as workflows_util - -_AUTOMATION_STATE_KEY = "state" - -_PARENT_WORKFLOW_ID = "741ce171-dac9-40be-83dc-b443c0eaf0e2" - - -def _child_workflow_id(child_index: int) -> str: - if child_index == 0: - return _PARENT_WORKFLOW_ID - return f"{_PARENT_WORKFLOW_ID}_{child_index}" - - -def _workflow_status_row( - *, - workflow_id: str, - updated_at: int = 0, - status: str = dbos.WorkflowStatusString.ENQUEUED.value, -) -> mock.Mock: - workflow_status = mock.Mock(spec=dbos.WorkflowStatus) - workflow_status.workflow_id = workflow_id - workflow_status.updated_at = updated_at - workflow_status.status = status - return workflow_status - - -def _automation_task_content(*, automation_name: str) -> str: - return json.dumps( - { - _AUTOMATION_STATE_KEY: { - "automation": { - "metadata": { - "automation_id": "automation_1", - "name": automation_name, - }, - "actions_dag": {"actions": []}, - "execution": {}, - }, - }, - } - ) - - -def _workflow_status_with_automation_task( - *, - status: str, - input_content: str, - output_content: str | None = None, -) -> mock.Mock: - task = node_models.Task( - name="automation-task", - content=input_content, - type=node_models.TaskType.EXECUTE_ACTIONS.value, - ) - encoded_inputs = workflow_params.AutomationWorkflowInputs(task=task).to_dict( - include_default_values=False - ) - workflow_status = mock.Mock(spec=dbos.WorkflowStatus) - workflow_status.workflow_id = "parent-workflow-id_1" - workflow_status.status = status - workflow_status.input = {"args": [encoded_inputs], "kwargs": {}} - if output_content is None: - workflow_status.output = None - else: - workflow_status.output = json.dumps( - workflow_params.AutomationWorkflowOutput(state=output_content).to_dict( - include_default_values=False - ) - ) - return workflow_status - - -class TestNormalizeParentAutomationId: - def test_parent_workflow_id_unchanged(self): - assert workflows_util.normalize_parent_automation_id(_PARENT_WORKFLOW_ID) == _PARENT_WORKFLOW_ID - - def test_child_workflow_id_truncated_to_parent(self): - child_id = _child_workflow_id(5) - assert workflows_util.normalize_parent_automation_id(child_id) == _PARENT_WORKFLOW_ID - - -class TestBuildNextChildAutomationWorkflowId: - def test_parent_workflow_id_maps_to_first_child(self): - assert ( - workflows_util.build_next_child_automation_workflow_id(_PARENT_WORKFLOW_ID) - == _child_workflow_id(1) - ) - - def test_child_workflow_id_increments_suffix(self): - assert ( - workflows_util.build_next_child_automation_workflow_id(_child_workflow_id(2)) - == _child_workflow_id(3) - ) - - def test_invalid_suffix_raises_value_error(self): - invalid_child_id = f"{_PARENT_WORKFLOW_ID}-4-4" - with pytest.raises(ValueError, match="Invalid child workflow suffix format"): - workflows_util.build_next_child_automation_workflow_id(invalid_child_id) - - -class TestParseAutomationChildWorkflowIndex: - def test_parent_workflow_id_maps_to_zero(self): - assert workflows_util.parse_automation_child_workflow_index(_PARENT_WORKFLOW_ID) == 0 - - def test_underscore_suffix_maps_to_child_index(self): - assert workflows_util.parse_automation_child_workflow_index(_child_workflow_id(1)) == 1 - assert workflows_util.parse_automation_child_workflow_index(_child_workflow_id(12)) == 12 - - def test_hyphen_suffix_maps_to_child_index(self): - hyphen_child_id = f"{_PARENT_WORKFLOW_ID}-3" - assert workflows_util.parse_automation_child_workflow_index(hyphen_child_id) == 3 - - def test_non_numeric_suffix_raises_value_error(self): - with pytest.raises(ValueError, match="Invalid child workflow suffix format"): - workflows_util.parse_automation_child_workflow_index(f"{_PARENT_WORKFLOW_ID}_abc") - - def test_multi_segment_hyphen_suffix_raises_value_error(self): - with pytest.raises(ValueError, match="Invalid child workflow suffix format"): - workflows_util.parse_automation_child_workflow_index(f"{_PARENT_WORKFLOW_ID}-4-4") - - def test_invalid_suffix_format_raises_value_error(self): - with pytest.raises(ValueError, match="Invalid child workflow suffix format"): - workflows_util.parse_automation_child_workflow_index(f"{_PARENT_WORKFLOW_ID}abc") - - -class TestGetLatestChildWorkflow: - def test_picks_highest_child_suffix_over_newer_updated_at(self): - finished_child = _workflow_status_row( - workflow_id=_child_workflow_id(56), - updated_at=200, - status=dbos.WorkflowStatusString.SUCCESS.value, - ) - waiting_child = _workflow_status_row( - workflow_id=_child_workflow_id(57), - updated_at=100, - status=dbos.WorkflowStatusString.ENQUEUED.value, - ) - chosen = workflows_util.get_latest_child_workflow([finished_child, waiting_child]) - assert chosen.workflow_id == _child_workflow_id(57) - - def test_picks_child_over_parent(self): - parent_workflow = _workflow_status_row(workflow_id=_child_workflow_id(0), updated_at=50) - child_workflow = _workflow_status_row(workflow_id=_child_workflow_id(1), updated_at=10) - chosen = workflows_util.get_latest_child_workflow([parent_workflow, child_workflow]) - assert chosen.workflow_id == _child_workflow_id(1) - - def test_single_element_list_returns_that_workflow(self): - only_workflow = _workflow_status_row(workflow_id=_child_workflow_id(4), updated_at=1) - chosen = workflows_util.get_latest_child_workflow([only_workflow]) - assert chosen is only_workflow - - def test_equal_suffix_tie_breaks_on_updated_at(self): - older_duplicate = _workflow_status_row(workflow_id=_child_workflow_id(5), updated_at=10) - newer_duplicate = _workflow_status_row(workflow_id=_child_workflow_id(5), updated_at=20) - chosen = workflows_util.get_latest_child_workflow([older_duplicate, newer_duplicate]) - assert chosen is newer_duplicate - - def test_unparseable_suffix_does_not_win_over_valid_child(self): - unrelated_workflow = _workflow_status_row( - workflow_id=f"{_PARENT_WORKFLOW_ID}-4-4", - updated_at=999, - ) - valid_child = _workflow_status_row( - workflow_id=_child_workflow_id(57), - updated_at=1, - ) - chosen = workflows_util.get_latest_child_workflow([unrelated_workflow, valid_child]) - assert chosen.workflow_id == _child_workflow_id(57) - - -class TestGetLatestWorkflow: - def test_delegates_to_latest_child_workflow(self): - parent_workflow = _workflow_status_row(workflow_id=_child_workflow_id(0), updated_at=100) - child_workflow = _workflow_status_row(workflow_id=_child_workflow_id(2), updated_at=1) - chosen = workflows_util.get_latest_workflow([parent_workflow, child_workflow]) - assert chosen.workflow_id == _child_workflow_id(2) - - -class TestGetResolvedAutomationTask: - def test_pending_workflow_uses_input_task_content(self): - input_content = _automation_task_content(automation_name="from-input") - output_content = _automation_task_content(automation_name="from-output") - workflow_status = _workflow_status_with_automation_task( - status=dbos.WorkflowStatusString.PENDING.value, - input_content=input_content, - output_content=output_content, - ) - - resolved_task = workflows_util.get_resolved_automation_task(workflow_status) - - assert resolved_task is not None - assert resolved_task.content == input_content - - def test_success_workflow_uses_output_state(self): - input_content = _automation_task_content(automation_name="from-input") - output_content = _automation_task_content(automation_name="from-output") - workflow_status = _workflow_status_with_automation_task( - status=dbos.WorkflowStatusString.SUCCESS.value, - input_content=input_content, - output_content=output_content, - ) - - resolved_task = workflows_util.get_resolved_automation_task(workflow_status) - - assert resolved_task is not None - assert resolved_task.content == output_content - - def test_error_workflow_uses_output_state(self): - input_content = _automation_task_content(automation_name="from-input") - output_content = _automation_task_content(automation_name="from-output") - workflow_status = _workflow_status_with_automation_task( - status=dbos.WorkflowStatusString.ERROR.value, - input_content=input_content, - output_content=output_content, - ) - - resolved_task = workflows_util.get_resolved_automation_task(workflow_status) - - assert resolved_task is not None - assert resolved_task.content == output_content - - def test_error_workflow_falls_back_to_input_when_output_state_missing(self): - input_content = _automation_task_content(automation_name="from-input") - workflow_status = _workflow_status_with_automation_task( - status=dbos.WorkflowStatusString.ERROR.value, - input_content=input_content, - output_content=None, - ) - workflow_status.output = json.dumps( - workflow_params.AutomationWorkflowOutput(state=None).to_dict( - include_default_values=False - ) - ) - - resolved_task = workflows_util.get_resolved_automation_task(workflow_status) - - assert resolved_task is not None - assert resolved_task.content == input_content - diff --git a/packages/node/tests/scheduler/test_workflows_util_user_action_inputs.py b/packages/node/tests/scheduler/test_workflows_util_user_action_inputs.py deleted file mode 100644 index 5f21fa4aaf..0000000000 --- a/packages/node/tests/scheduler/test_workflows_util_user_action_inputs.py +++ /dev/null @@ -1,65 +0,0 @@ -# Drakkar-Software OctoBot-Node -# Copyright (c) Drakkar-Software, All rights reserved. - -import mock -import pytest -import dbos - -import octobot_protocol.models as protocol_models -import octobot_node.scheduler.workflows.params as params -import octobot_node.scheduler.workflows_util as workflows_util - - -class TestResolveUserActionWorkflowInputs: - def test_unwraps_dbos_kwargs_inputs_key(self): - user_action = protocol_models.UserAction(id="ua-kwargs", configuration=None) - encoded = params.UserActionWorkflowInputs( - user_id="0xkwargs", - user_action=user_action, - ).to_dict(include_default_values=False) - workflow_status = mock.Mock(spec=dbos.WorkflowStatus) - workflow_status.workflow_id = "wf-kwargs" - workflow_status.input = {"args": [], "kwargs": {"inputs": encoded}} - - resolved = workflows_util.resolve_user_action_workflow_inputs(workflow_status) - - assert resolved.inputs is not None - assert resolved.inputs.user_id == "0xkwargs" - assert resolved.inputs.user_action.id == "ua-kwargs" - - def test_unwraps_portable_json_named_args_inputs(self): - user_action = protocol_models.UserAction(id="ua-portable", configuration=None) - encoded = params.UserActionWorkflowInputs( - user_id="0xportable", - user_action=user_action, - ).to_dict(include_default_values=False) - workflow_status = mock.Mock(spec=dbos.WorkflowStatus) - workflow_status.workflow_id = "wf-portable" - workflow_status.input = { - "args": [ - { - "positionalArgs": [], - "namedArgs": {"inputs": encoded}, - }, - ], - "kwargs": {}, - } - - resolved = workflows_util.resolve_user_action_workflow_inputs(workflow_status) - - assert resolved.inputs is not None - assert resolved.inputs.user_id == "0xportable" - assert resolved.inputs.user_action.id == "ua-portable" - - def test_empty_wrappers_report_no_inputs_not_missing_fields(self): - workflow_status = mock.Mock(spec=dbos.WorkflowStatus) - workflow_status.workflow_id = "wf-empty" - workflow_status.input = { - "args": [{"positionalArgs": [], "namedArgs": {}}], - "kwargs": {}, - } - - resolved = workflows_util.resolve_user_action_workflow_inputs(workflow_status) - - assert resolved.inputs is None - assert resolved.parse_error == "no user-action workflow inputs found" diff --git a/packages/node/tests/scheduler/test_workflows_version_migration.py b/packages/node/tests/scheduler/test_workflows_version_migration.py index 565397febe..3bbedb78bd 100644 --- a/packages/node/tests/scheduler/test_workflows_version_migration.py +++ b/packages/node/tests/scheduler/test_workflows_version_migration.py @@ -27,7 +27,7 @@ import tests.scheduler as scheduler_test_util _OLD_APPLICATION_VERSION = "3.0.0-beta0" -_TEST_QUEUE = dbos.Queue(name="version_migration_test_queue") +_TEST_QUEUE_NAME = "version_migration_test_queue" _WORKFLOW_SLEEP_SECONDS = 1.5 @@ -59,12 +59,16 @@ def test_retags_enqueued_workflows_to_target_version(self): ) dbos.DBOS.reset_system_database() dbos.DBOS.launch() + dbos.DBOS.register_queue(_TEST_QUEUE_NAME) @octobot_node.scheduler.SCHEDULER.INSTANCE.workflow() async def stranded_workflow() -> str: return "done" - workflow_handle = _TEST_QUEUE.enqueue(stranded_workflow) + workflow_handle = dbos.DBOS.enqueue_workflow( + _TEST_QUEUE_NAME, + stranded_workflow, + ) stranded_workflow_id = workflow_handle.get_workflow_id() octobot_node.scheduler.SCHEDULER.INSTANCE.destroy() @@ -90,6 +94,7 @@ async def stranded_workflow() -> str: application_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, ) octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + dbos.DBOS.register_queue(_TEST_QUEUE_NAME) migrated_status = dbos.DBOS.get_workflow_status(stranded_workflow_id) assert migrated_status is not None @@ -117,7 +122,11 @@ async def sleeper_workflow() -> str: dbos.DBOS.reset_system_database() dbos.DBOS.launch() - await _TEST_QUEUE.enqueue_async(sleeper_workflow) + await dbos.DBOS.register_queue_async(_TEST_QUEUE_NAME) + await dbos.DBOS.enqueue_workflow_async( + _TEST_QUEUE_NAME, + sleeper_workflow, + ) enqueued_workflows = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async( status=[dbos.WorkflowStatusString.ENQUEUED.value], @@ -146,6 +155,7 @@ async def sleeper_workflow() -> str: application_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, ) octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + await dbos.DBOS.register_queue_async(_TEST_QUEUE_NAME) recovery_handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async( stranded_workflow_id, @@ -176,7 +186,11 @@ async def sleeper_workflow() -> str: dbos.DBOS.reset_system_database() dbos.DBOS.launch() - await _TEST_QUEUE.enqueue_async(sleeper_workflow) + await dbos.DBOS.register_queue_async(_TEST_QUEUE_NAME) + await dbos.DBOS.enqueue_workflow_async( + _TEST_QUEUE_NAME, + sleeper_workflow, + ) pending_workflows = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async( status=[ @@ -209,6 +223,7 @@ async def sleeper_workflow() -> str: application_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, ) octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + await dbos.DBOS.register_queue_async(_TEST_QUEUE_NAME) recovery_handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async( stranded_workflow_id, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py index 5a1926e327..3989067e92 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py @@ -229,3 +229,105 @@ async def test_fails_when_authentication_details_missing_for_live_account(self): expect_error_details=True, expected_error_message=protocol_models.AccountActionResultErrorMessage.ACCOUNT_AUTHENTICATION_DETAILS_NOT_FOUND, ) + + +class TestCreateAccountActionExecutorRecordAccountValidated: + @pytest.mark.asyncio + async def test_emits_account_validated_via_node_journal(self): + account_model = account_executor_test_utils.minimal_exchange_account( + account_id="validated-acc", + is_simulated=False, + ) + validated_account = account_model.model_copy( + update={ + "state": protocol_models.AccountState( + status=protocol_models.AccountStatus.VALID, + ), + }, + ) + inner = protocol_models.CreateAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_CREATE, + configuration=account_model, + ) + user_action = protocol_models.UserAction( + id="ua-validated", + configuration=account_executor_test_utils.wrap_configuration(inner), + ) + provider_mock = mock.Mock() + trading_provider_mock = mock.Mock() + metrics_config = mock.Mock() + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountTradingProvider.instance", + return_value=trading_provider_mock, + ), + mock.patch.object( + account_state_updater_module, + "update_account_state", + new=mock.AsyncMock(return_value=validated_account), + ), + mock.patch( + "octobot.community.node_journal.record_account_validated_from_account", + ) as record_account_validated_mock, + ): + executor = create_account_executor.CreateAccountActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + await executor.execute(user_action) + record_account_validated_mock.assert_called_once_with( + validated_account, + account_executor_test_utils.WALLET_ADDRESS, + user_action_id="ua-validated", + ) + + @pytest.mark.asyncio + async def test_invalid_real_emits_no_account_validated(self): + account_model = account_executor_test_utils.minimal_exchange_account( + account_id="invalid-acc", + is_simulated=False, + ) + invalid_account = account_model.model_copy( + update={ + "state": protocol_models.AccountState( + status=protocol_models.AccountStatus.INVALID, + ), + }, + ) + inner = protocol_models.CreateAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_CREATE, + configuration=account_model, + ) + user_action = protocol_models.UserAction( + id="ua-invalid", + configuration=account_executor_test_utils.wrap_configuration(inner), + ) + provider_mock = mock.Mock() + trading_provider_mock = mock.Mock() + metrics_config = mock.Mock() + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountTradingProvider.instance", + return_value=trading_provider_mock, + ), + mock.patch.object( + account_state_updater_module, + "update_account_state", + new=mock.AsyncMock(return_value=invalid_account), + ), + mock.patch( + "octobot.community.node_journal.record_account_validated_from_account", + ) as record_account_validated_mock, + ): + executor = create_account_executor.CreateAccountActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + await executor.execute(user_action) + record_account_validated_mock.assert_called_once() diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_delete_account.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_delete_account.py index 1ec943317f..f3d3aafd9b 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_delete_account.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_delete_account.py @@ -36,12 +36,23 @@ async def test_calls_provider_delete_with_wallet_and_id(self): ) user_action = protocol_models.UserAction(id="ua-del", configuration=account_executor_test_utils.wrap_configuration(inner)) provider_mock = mock.Mock() - with mock.patch( - "octobot_sync.sync.collection_providers.AccountProvider.instance", - return_value=provider_mock, + deleted_account = account_executor_test_utils.minimal_exchange_account(account_id="del-1") + provider_mock.get_item.return_value = deleted_account + provider_mock.list_items.return_value = [] + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account.delete_account.node_journal.record_account_deleted", + ) as record_account_deleted_mock, ): executor = delete_account_executor.DeleteAccountActionExecutor(account_executor_test_utils.WALLET_ADDRESS) await executor.execute(user_action) + record_account_deleted_mock.assert_called_once() + assert record_account_deleted_mock.call_args.kwargs["account_id"] == "del-1" + assert record_account_deleted_mock.call_args.kwargs["user_action_id"] == "ua-del" provider_mock.delete_item.assert_called_once_with(account_executor_test_utils.WALLET_ADDRESS, "del-1") provider_assertions.assert_user_action_terminal_state( user_action=user_action, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_edit_account.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_edit_account.py index a85b9fac70..a7c5909b3a 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_edit_account.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_edit_account.py @@ -48,9 +48,15 @@ async def test_calls_provider_update_with_wallet_and_account(self): "update_account_state", new=mock.AsyncMock(return_value=account_model), ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account.edit_account.node_journal.record_account_edit_succeeded", + ) as record_account_edit_succeeded_mock, ): executor = edit_account_executor.EditAccountActionExecutor(account_executor_test_utils.WALLET_ADDRESS) await executor.execute(user_action) + record_account_edit_succeeded_mock.assert_called_once() + assert record_account_edit_succeeded_mock.call_args.kwargs["account_id"] == "edit-acc" + assert record_account_edit_succeeded_mock.call_args.kwargs["user_action_id"] == "ua-edit" provider_mock.update_item.assert_called_once_with(account_executor_test_utils.WALLET_ADDRESS, account_model) provider_assertions.assert_user_action_terminal_state( user_action=user_action, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_refresh_accounts.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_refresh_accounts.py index 821e9b89de..78df5920a1 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_refresh_accounts.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_refresh_accounts.py @@ -65,9 +65,16 @@ async def test_updates_all_accounts_when_ids_are_not_provided(self): "update_account_state", new=mock.AsyncMock(side_effect=[checked_first_account, checked_second_account]), ) as check_mock, + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account.refresh_accounts.node_journal.record_accounts_refreshed", + ) as record_accounts_refreshed_mock, ): executor = refresh_accounts_executor.RefreshAccountsActionExecutor(account_executor_test_utils.WALLET_ADDRESS) await executor.execute(user_action) + record_accounts_refreshed_mock.assert_called_once_with( + account_ids=["acc-1", "acc-2"], + user_action_id="ua-refresh-all", + ) provider_mock.list_items.assert_called_once_with(account_executor_test_utils.WALLET_ADDRESS) assert provider_mock.get_item.call_count == 2 provider_mock.update_item.assert_has_calls( @@ -102,6 +109,7 @@ async def test_updates_only_requested_accounts(self): user_action = protocol_models.UserAction(id="ua-refresh-one", configuration=account_executor_test_utils.wrap_configuration(refresh_inner)) provider_mock = mock.Mock() provider_mock.get_item.return_value = account_model + provider_mock.list_items.return_value = [account_model] with ( mock.patch( "octobot_sync.sync.collection_providers.AccountProvider.instance", diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_create_account_auth.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_create_account_auth.py index d83ead7805..16fe080afb 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_create_account_auth.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_create_account_auth.py @@ -56,11 +56,18 @@ async def test_calls_provider_create_with_wallet_and_authentication(self): "octobot_node.scheduler.user_actions.user_actions_executor.account_auth.account_auth_user_action_executor.timestamp_util.utc_now_datetime", return_value=_FIXED_TIMESTAMP, ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account_auth.create_account_auth.node_journal.record_account_auth_create_succeeded", + ) as record_account_auth_create_succeeded_mock, ): executor = create_account_auth_executor.CreateAccountAuthActionExecutor( account_auth_executor_test_utils.WALLET_ADDRESS, ) await executor.execute(user_action) + record_account_auth_create_succeeded_mock.assert_called_once_with( + exchange_name=None, + user_action_id="ua-create-auth", + ) provider_mock.create_item.assert_called_once_with( account_auth_executor_test_utils.WALLET_ADDRESS, authentication_model.model_copy(update={"updated_at": _FIXED_TIMESTAMP}), diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_delete_account_auth.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_delete_account_auth.py index 6d320d9ff8..718fb9ac42 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_delete_account_auth.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_delete_account_auth.py @@ -39,14 +39,23 @@ async def test_calls_provider_delete_with_wallet_and_id(self): configuration=account_auth_executor_test_utils.wrap_configuration(inner), ) provider_mock = mock.Mock() - with mock.patch( - "octobot_sync.sync.collection_providers.AccountAuthenticationProvider.instance", - return_value=provider_mock, + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountAuthenticationProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account_auth.delete_account_auth.node_journal.record_account_auth_deleted", + ) as record_account_auth_deleted_mock, ): executor = delete_account_auth_executor.DeleteAccountAuthActionExecutor( account_auth_executor_test_utils.WALLET_ADDRESS, ) await executor.execute(user_action) + record_account_auth_deleted_mock.assert_called_once_with( + exchange_name=None, + user_action_id="ua-del-auth", + ) provider_mock.delete_item.assert_called_once_with( account_auth_executor_test_utils.WALLET_ADDRESS, "del-auth-1", diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py index ebe2fc7165..21d2041a99 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py @@ -1180,7 +1180,7 @@ def test_create_automation_task_wraps_actions_in_state_envelope(self): strategy_mock.return_value.get_item.return_value = stored actions = executor._create_automation_actions(user_action) - task = asyncio.run(executor._create_automation_task(user_action, actions)) + resolved_automation_id, task = asyncio.run(executor._create_automation_task(user_action, actions)) assert task.name == automation_name _assert_task_content_matches_actions( task=task, @@ -1308,7 +1308,8 @@ def test_configuration_id_used_for_automation_identity(self): protocol_account=account, strategy_reference=strat_ref, ) - task = asyncio.run(executor._create_automation_task(user_action, actions)) + resolved_automation_id, task = asyncio.run(executor._create_automation_task(user_action, actions)) + assert resolved_automation_id == configuration_automation_id assert task.id == configuration_automation_id _assert_task_content_matches_actions( task=task, @@ -1381,7 +1382,7 @@ def test_missing_configuration_id_keeps_legacy_identity_split(self): protocol_account=_minimal_exchange_account(account_id="acc-1"), strategy_reference=strat_ref, ) - task = asyncio.run(executor._create_automation_task(user_action, actions)) + _resolved_automation_id, task = asyncio.run(executor._create_automation_task(user_action, actions)) assert task.id != user_action.id _assert_task_content_matches_actions( task=task, @@ -1454,3 +1455,37 @@ async def test_execute_fails_with_invalid_configuration_id(self): expect_error_details=True, expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_AUTOMATION_ID, ) + + +class TestCreateAutomationActionExecutorRecordNewAutomationCreated: + @pytest.mark.asyncio + async def test_calls_record_new_automation_created_on_success(self): + idx = trading_tentacles_test_utils.index_trading_configuration( + coins=[("BTC", 1.0)], + rebalance_trigger_min_percent=5.0, + ) + strat_ref = _default_strategy_reference() + create_payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=_automation_configuration( + name="metrics-automation", + strategy_reference=strat_ref, + account_id="acc-1", + ), + ) + user_action = _user_action_with_context(action_id="ua-metrics", payload=create_payload) + executor = create_automation_executor.CreateAutomationActionExecutor(_TEST_WALLET_ADDRESS) + stored = _stored_strategy_matching_reference(strat_ref, idx) + with mock.patch(_ACCOUNT_PROVIDER_INSTANCE_PATCH) as account_mock, mock.patch( + _STRATEGY_PROVIDER_INSTANCE_PATCH, + ) as strategy_mock, mock.patch( + "octobot.community.node_journal.record_new_automation_created_from_strategy", + ) as record_new_automation_mock: + _stub_account_provider(account_mock, _minimal_exchange_account(account_id="acc-1")) + strategy_mock.return_value.get_item.return_value = stored + await executor.execute(user_action) + record_new_automation_mock.assert_called_once_with( + user_action.id, + stored, + user_action_id="ua-metrics", + ) diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py index a43143c00a..55db5e1f23 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py @@ -167,9 +167,16 @@ async def test_execute_enqueues_task_from_latest_output(self): new_callable=mock.AsyncMock, return_value=terminal_workflow, ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.node_journal.record_automation_restarted", + ) as record_automation_restarted_mock, ): await executor.execute(user_action) + record_automation_restarted_mock.assert_called_once_with( + automation_id=_PARENT_AUTOMATION_ID, + user_action_id="ua-restart-1", + ) scheduled_task = executor.post_actions.to_create_automation_task assert scheduled_task is not None assert scheduled_task.id == f"{_PARENT_AUTOMATION_ID}_1" diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py index 034d75a03f..e576dffca5 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py @@ -60,11 +60,9 @@ async def test_returns_latest_pending_child_workflow_id(self): inner_mock.assert_awaited_once_with( _TEST_WALLET_ADDRESS, [parent_id], - [ - dbos.WorkflowStatusString.ENQUEUED, - dbos.WorkflowStatusString.PENDING, - ], + None, load_output=False, + queues_only=True, ) @@ -85,6 +83,9 @@ async def test_execute_sends_stop_actions_to_active_automation(self): "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_tasks.send_actions_to_active_automation", new_callable=mock.AsyncMock, ) as send_actions_mock, + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.node_journal.record_automation_stopped", + ) as record_automation_stopped_mock, ): await executor.execute(user_action) @@ -98,6 +99,11 @@ async def test_execute_sends_stop_actions_to_active_automation(self): } ], ) + record_automation_stopped_mock.assert_called_once_with( + automation_id="00000000-0000-4000-8000-000000000001", + cancel_orders=False, + user_action_id="ua-stop-1", + ) provider_assertions.assert_user_action_terminal_state( user_action=user_action, expected_status=protocol_models.UserActionStatus.COMPLETED, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_create_strategy.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_create_strategy.py index 95e5fc8859..9c2d2d3e10 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_create_strategy.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_create_strategy.py @@ -40,14 +40,24 @@ async def test_calls_provider_create_with_wallet_and_strategy(self): configuration=strategy_executor_test_utils.wrap_configuration(inner), ) provider_mock = mock.Mock() - with mock.patch( - "octobot_sync.sync.collection_providers.StrategyProvider.instance", - return_value=provider_mock, + with ( + mock.patch( + "octobot_sync.sync.collection_providers.StrategyProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.strategy.create_strategy.node_journal.record_strategy_create_succeeded", + ) as record_strategy_create_succeeded_mock, ): executor = create_strategy_executor.CreateStrategyActionExecutor( strategy_executor_test_utils.WALLET_ADDRESS, ) await executor.execute(user_action) + record_strategy_create_succeeded_mock.assert_called_once_with( + strategy_id="new-strategy", + strategy=strategy_model, + user_action_id="ua-create-strategy", + ) provider_mock.create_item.assert_called_once_with( strategy_executor_test_utils.WALLET_ADDRESS, strategy_model, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_edit_strategy.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_edit_strategy.py index 653a36594f..13377e8eea 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_edit_strategy.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_edit_strategy.py @@ -44,14 +44,24 @@ async def test_calls_provider_update_with_wallet_and_strategy(self): configuration=strategy_executor_test_utils.wrap_configuration(inner), ) provider_mock = mock.Mock() - with mock.patch( - "octobot_sync.sync.collection_providers.StrategyProvider.instance", - return_value=provider_mock, + with ( + mock.patch( + "octobot_sync.sync.collection_providers.StrategyProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.strategy.edit_strategy.node_journal.record_strategy_edit_succeeded", + ) as record_strategy_edit_succeeded_mock, ): executor = edit_strategy_executor.EditStrategyActionExecutor( strategy_executor_test_utils.WALLET_ADDRESS, ) await executor.execute(user_action) + record_strategy_edit_succeeded_mock.assert_called_once_with( + strategy_id="edit-strategy", + strategy=strategy_model, + user_action_id="ua-edit-strategy", + ) provider_mock.update_item.assert_called_once_with( strategy_executor_test_utils.WALLET_ADDRESS, strategy_model, diff --git a/packages/node/tests/scheduler/workflows/test_automation_workflow.py b/packages/node/tests/scheduler/workflows/test_automation_workflow.py index 29d40ab56d..1193bf79a6 100644 --- a/packages/node/tests/scheduler/workflows/test_automation_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_automation_workflow.py @@ -75,12 +75,10 @@ def import_automation_workflow(): importlib.import_module("octobot_node.scheduler.workflows.automation_workflow") AUTOMATION_WORKFLOW_IMPORTED = True # init_and_destroy_scheduler() tears down DBOS after registering workflows, leaving - # INSTANCE/queues as None. Unit tests patch recv_async/enqueue_async on these objects; + # INSTANCE as None. Unit tests patch recv_async / enqueue_workflow_async on SCHEDULER.INSTANCE; # provide mocks so patch.object has a real target (re-apply after other fixtures tear down). if octobot_node.scheduler.SCHEDULER.INSTANCE is None: octobot_node.scheduler.SCHEDULER.INSTANCE = mock.Mock() - if octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE is None: - octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE = mock.Mock() def _automation_state_dict(actions: list[dict[str, typing.Any]]) -> dict[str, typing.Any]: @@ -316,14 +314,12 @@ async def _run_execute_automation_until_iteration_retries_exhausted( "execute_iteration", max_attempts, ) - assert workflow_result == json.dumps( - params.AutomationWorkflowOutput( - error=octobot_flow.enums.AutomationWorkflowErrorStatus.EXCEPTION_DURING_ITERATION.value, - error_message=expected_error_message, - ).to_dict(include_default_values=False) + assert workflow_result == _expected_automation_workflow_envelope_json( + "{}", + error=octobot_flow.enums.AutomationWorkflowErrorStatus.EXCEPTION_DURING_ITERATION.value, + error_message=expected_error_message, ) parsed_output = _parse_automation_workflow_output(workflow_result) - assert parsed_output.state is None assert ( parsed_output.error == octobot_flow.enums.AutomationWorkflowErrorStatus.EXCEPTION_DURING_ITERATION.value @@ -433,6 +429,90 @@ def iteration_result(): has_next_actions=True, ) + +class TestBestEffortWorkflowOutputState: + """``_best_effort_workflow_output_state`` picks state for interrupted workflow output. + + Priority: in-flight iteration snapshot (``next_iteration_description``) over workflow + input task content; empty/missing inputs yield ``(None, None)``. + """ + + @pytest.mark.parametrize( + "iteration_description,task_content,task_metadata,expected_state,expected_metadata", + [ + # No inputs and no iteration progress → nothing to persist. + (None, None, None, None, None), + # Empty task content is treated as absent (falsy), same as no task. + (None, "", None, None, None), + # Input-only: fall back to task content when iteration has no description. + (None, "input-state", "input-meta", "input-state", "input-meta"), + # Iteration description wins over input task when both are present. + ( + "iteration-state", + "input-state", + "input-meta", + "iteration-state", + "iteration-meta", + ), + # Non-None but empty iteration description is still preferred over input. + ( + "", + "input-state", + None, + "", + None, + ), + ], + ) + def test_best_effort_workflow_output_state( + self, + import_automation_workflow, + iteration_description, + task_content, + task_metadata, + expected_state, + expected_metadata, + ): + automation_workflow = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + # task_content=None → no parsed_inputs (first param row only). + parsed_inputs = None + if task_content is not None: + task = octobot_node.models.Task( + name="test_task", + content=task_content, + content_metadata=task_metadata, + type=octobot_node.models.TaskType.EXECUTE_ACTIONS.value, + ) + parsed_inputs = params.AutomationWorkflowInputs(task=task, execution_time=0) + # Build iteration_result when the row exercises iteration and/or input paths. + iteration_result = None + if iteration_description is not None or task_content is not None: + iteration_result = params.AutomationWorkflowIterationResult( + progress_status=params.ProgressStatus( + latest_step="step", + next_step=None, + next_step_at=None, + remaining_steps=0, + error=None, + should_stop=False, + ), + next_iteration_description=iteration_description, + next_iteration_description_metadata="iteration-meta" if iteration_description else None, + has_next_actions=False, + ) + state, metadata = automation_workflow._best_effort_workflow_output_state( + parsed_inputs, iteration_result + ) + assert state == expected_state + # Metadata follows the same source as state (iteration meta vs task content_metadata). + if iteration_description is not None: + assert metadata == ( + "iteration-meta" if iteration_description else None + ) + else: + assert metadata == expected_metadata + + def required_imports(func): @functools.wraps(func) async def wrapper(*args, **kwargs): @@ -482,7 +562,9 @@ async def test_execute_automation( octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_automation, inputs=inputs, ) - assert await handle.get_result() is None # next_iteration_description.next_actions_description is None + assert await handle.get_result() == _expected_automation_workflow_envelope_json( + parsed_inputs.task.content + ) # no iteration description; best-effort falls back to input task.content mock_wait.assert_awaited_once_with(parsed_inputs, 0) mock_iteration.assert_called_once_with(inputs, None) mock_should_continue.assert_called_once() @@ -663,14 +745,12 @@ async def test_execute_automation( inputs=inputs, ) workflow_result = await handle.get_result() - assert workflow_result == json.dumps( - params.AutomationWorkflowOutput( - error=expected_error_status, - error_message=expected_error_message, - ).to_dict(include_default_values=False) + assert workflow_result == _expected_automation_workflow_envelope_json( + parsed_inputs.task.content, + error=expected_error_status, + error_message=expected_error_message, ) parsed_output = _parse_automation_workflow_output(workflow_result) - assert parsed_output.state is None assert parsed_output.error == expected_error_status assert parsed_output.error_message == expected_error_message assert run_mock.await_count == expected_run_await_count @@ -1919,6 +1999,116 @@ async def test_execute_automation_sends_signal_execution_result_on_crash( ) +class TestExecuteAutomationBestEffortOutputState: + @pytest.mark.asyncio + @required_imports + async def test_preserves_input_state_on_terminal_iteration_error_without_description( + self, + import_automation_workflow, + temp_dbos_scheduler, + ): + task_content = json.dumps({"state": _automation_state_dict([])}) + task = octobot_node.models.Task( + name="terminal_iteration_error_test", + content=task_content, + type=octobot_node.models.TaskType.EXECUTE_ACTIONS.value, + ) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict( + include_default_values=False + ) + inputs["task"] = task.model_dump(exclude_defaults=True) + terminal_error = octobot_flow.enums.ActionErrorStatus.NO_TRADING_SIGNAL.value + iteration_result = params.AutomationWorkflowIterationResult( + progress_status=params.ProgressStatus( + latest_step="action_1", + next_step=None, + next_step_at=None, + remaining_steps=0, + error=terminal_error, + error_message="no signal", + should_stop=False, + ), + next_iteration_description=None, + has_next_actions=False, + ) + mock_wait = mock.AsyncMock(return_value=None) + mock_iteration = mock.AsyncMock( + return_value=iteration_result.to_dict(include_default_values=False) + ) + mock_process = mock.AsyncMock() + + with mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_wait_and_trigger_on_actions_update", + mock_wait, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "execute_iteration", + mock_iteration, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_process_pending_priority_actions_and_reschedule", + mock_process, + ): + handle = await temp_dbos_scheduler.INSTANCE.start_workflow_async( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_automation, + inputs=inputs, + ) + workflow_result = await handle.get_result() + + parsed_output = _parse_automation_workflow_output(workflow_result) + assert parsed_output.state == task_content + assert parsed_output.error == terminal_error + assert parsed_output.error_message == "no signal" + mock_process.assert_not_called() + + @pytest.mark.asyncio + @required_imports + async def test_preserves_input_state_on_exception_when_iteration_has_no_description( + self, + import_automation_workflow, + temp_dbos_scheduler, + ): + task_content = json.dumps({"state": _automation_state_dict([])}) + task = octobot_node.models.Task( + name="exception_preserves_state_test", + content=task_content, + type=octobot_node.models.TaskType.EXECUTE_ACTIONS.value, + ) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict( + include_default_values=False + ) + inputs["task"] = task.model_dump(exclude_defaults=True) + workflow_error = ValueError("iteration exploded") + mock_wait = mock.AsyncMock(return_value=None) + mock_iteration = mock.AsyncMock(side_effect=workflow_error) + + with mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_wait_and_trigger_on_actions_update", + mock_wait, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "execute_iteration", + mock_iteration, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_process_pending_priority_actions_and_reschedule", + mock.AsyncMock(), + ): + handle = await temp_dbos_scheduler.INSTANCE.start_workflow_async( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_automation, + inputs=inputs, + ) + workflow_result = await handle.get_result() + + parsed_output = _parse_automation_workflow_output(workflow_result) + assert parsed_output.state == task_content + assert parsed_output.error == ( + octobot_flow.enums.AutomationWorkflowErrorStatus.EXCEPTION_DURING_ITERATION.value + ) + assert parsed_output.error_message == str(workflow_error) + class TestExecuteIterationOutdatedReferenceAccountError: @pytest.mark.asyncio @required_imports @@ -2516,8 +2706,8 @@ async def test_schedule_next_iteration_enqueues_workflow(self, import_automation "SetWorkflowID", mock_set_workflow_id, ), mock.patch.object( - octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE, - "enqueue_async", + octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.INSTANCE, + "enqueue_workflow_async", mock_enqueue, ): await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._schedule_next_iteration( @@ -2526,7 +2716,8 @@ async def test_schedule_next_iteration_enqueues_workflow(self, import_automation mock_enqueue.assert_called_once() mock_set_workflow_id.assert_called_once_with(f"{parent_workflow_id}_1") call_args = mock_enqueue.call_args - assert call_args[0][0] == octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_automation + assert call_args[0][0] == octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value + assert call_args[0][1] == octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_automation assert "inputs" in call_args[1] @pytest.mark.asyncio @@ -2544,8 +2735,8 @@ async def test_schedule_next_iteration_increments_existing_child_workflow_id( "SetWorkflowID", mock_set_workflow_id, ), mock.patch.object( - octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE, - "enqueue_async", + octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.INSTANCE, + "enqueue_workflow_async", mock_enqueue, ): await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._schedule_next_iteration( @@ -2720,9 +2911,10 @@ async def test_execute_automation_reschedules_after_failed_request_via_dbos_step inputs["task"] = task.model_dump(exclude_defaults=True) fixed_now = 1000.0 recv_path = "octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.INSTANCE.recv_async" - real_enqueue_async = temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE.enqueue_async - enqueue_mock = mock.AsyncMock(wraps=real_enqueue_async) - automation_wf = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + automation_workflow_module = octobot_node.scheduler.workflows.automation_workflow + real_enqueue_automation = automation_workflow_module.SCHEDULER.INSTANCE.enqueue_workflow_async + enqueue_mock = mock.AsyncMock(wraps=real_enqueue_automation) + automation_wf = automation_workflow_module.AutomationWorkflow with mock.patch(recv_path, mock.AsyncMock(return_value=None)), mock.patch( "octobot_node.scheduler.workflows.automation_workflow.time.time", return_value=fixed_now, @@ -2734,8 +2926,8 @@ async def test_execute_automation_reschedules_after_failed_request_via_dbos_step octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, "persist_account_trading_from_iteration_state", ), mock.patch.object( - octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE, - "enqueue_async", + automation_workflow_module.SCHEDULER.INSTANCE, + "enqueue_workflow_async", enqueue_mock, ), mock.patch.object( automation_wf, @@ -2801,9 +2993,10 @@ async def test_execute_automation_reschedules_after_postponed_iteration_with_pri inputs["task"] = task.model_dump(exclude_defaults=True) fixed_now = 1000.0 recv_path = "octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.INSTANCE.recv_async" - real_enqueue_async = temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE.enqueue_async - enqueue_mock = mock.AsyncMock(wraps=real_enqueue_async) - automation_wf = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + automation_workflow_module = octobot_node.scheduler.workflows.automation_workflow + real_enqueue_automation = automation_workflow_module.SCHEDULER.INSTANCE.enqueue_workflow_async + enqueue_mock = mock.AsyncMock(wraps=real_enqueue_automation) + automation_wf = automation_workflow_module.AutomationWorkflow with mock.patch( recv_path, mock.AsyncMock(side_effect=[None, trading_signal_envelope, None]), @@ -2818,8 +3011,8 @@ async def test_execute_automation_reschedules_after_postponed_iteration_with_pri octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, "persist_account_trading_from_iteration_state", ), mock.patch.object( - octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE, - "enqueue_async", + automation_workflow_module.SCHEDULER.INSTANCE, + "enqueue_workflow_async", enqueue_mock, ), mock.patch.object( automation_wf, @@ -3057,6 +3250,44 @@ def test_returns_unchanged_executed_step_when_no_processed_actions( assert executed_step == "no action executed" assert iteration_state.execution_error is None + # When get_next_actions_description returns has_next_actions=False after an upstream + # failure (instead of WorkflowDAGDependenciesError), the workflow must still copy + # processed_actions errors into iteration_state; locks blockchain_wallet_error (e.g. wallet RPC timeout). + def test_copies_blockchain_wallet_error_into_iteration_state( + self, import_automation_workflow, parsed_inputs + ): + automation_workflow_module = octobot_node.scheduler.workflows.automation_workflow + mock_action = mock.Mock() + mock_action.error_status = octobot_flow.enums.ActionErrorStatus.BLOCKCHAIN_WALLET_ERROR.value + mock_action.error_message = ( + "Timed out while waiting for an idle wallet RPC process in the configured port range" + ) + mock_action.id = "blockchain_wallet_init_2" + mock_action.get_summary.return_value = "blockchain wallet init" + result = octobot_flow_client.OctoBotActionsJobResult() + result.processed_actions = [mock_action] + iteration_state = automation_workflow_module._IterationExecutionState() + with mock.patch.object( + automation_workflow_module.AutomationWorkflow, + "get_logger", + return_value=mock.Mock(), + ): + with mock.patch.object( + automation_workflow_module.AutomationWorkflow, + "_get_actions_summary", + return_value="executed summary", + ): + automation_workflow_module.AutomationWorkflow._apply_processed_action_errors( + parsed_inputs, + result, + iteration_state, + "no action executed", + ) + assert iteration_state.execution_error == ( + octobot_flow.enums.ActionErrorStatus.BLOCKCHAIN_WALLET_ERROR.value + ) + assert iteration_state.execution_error_message == mock_action.error_message + class TestSendSignalExecutionResultSafe: @pytest.mark.asyncio @@ -3156,6 +3387,13 @@ def test_get_logger_uses_class_name_when_task_name_none(self, import_automation_ octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.get_logger(parsed_inputs) mock_get_logger.assert_called_once_with(octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.__name__) + def test_get_logger_uses_class_name_when_parsed_inputs_none(self, import_automation_workflow): + with mock.patch("octobot_commons.logging.get_logger", mock.Mock()) as mock_get_logger: + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.get_logger(None) + mock_get_logger.assert_called_once_with( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.__name__ + ) + class TestExecuteAutomationIntegration: def setup_method(self): @@ -3227,7 +3465,8 @@ async def test_execute_automation_full_workflow_three_iterations( recv_path = "octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.INSTANCE.recv_async" with mock.patch(recv_path, mock.AsyncMock(return_value=[])): - await temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE.enqueue_async( + await temp_dbos_scheduler.INSTANCE.enqueue_workflow_async( + octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value, octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_automation, inputs=inputs, ) @@ -3422,6 +3661,15 @@ async def wait_forever_in_iteration(*args, **kwargs): assert cancelled_status is not None assert cancelled_status.status == dbos.WorkflowStatusString.CANCELLED.value + executions = await octobot_node.scheduler.SCHEDULER.get_results() + cancelled_executions = [ + execution + for execution in executions + if execution.status == octobot_node.models.TaskStatus.CANCELLED + ] + assert len(cancelled_executions) == 1 + assert cancelled_executions[0].id == automation_workflow_id + @pytest.mark.asyncio @required_imports async def test_cancel_workflow_async_during_iteration_retries_stops_further_job_attempts( @@ -3939,8 +4187,8 @@ def _encrypted_description_raw_json(template: octobot_flow_client.OctoBotActions "OctoBotActionsJob", mock_octobot_actions_job_class_schedule, ), mock.patch.object( - octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE, - "enqueue_async", + octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.INSTANCE, + "enqueue_workflow_async", enqueue_mock, ): schedule_handle = await temp_dbos_scheduler.INSTANCE.start_workflow_async( @@ -3950,8 +4198,8 @@ def _encrypted_description_raw_json(template: octobot_flow_client.OctoBotActions await schedule_handle.get_result() enqueue_mock.assert_called_once() - assert len(enqueue_mock.call_args.args) == 1 # function to call - assert len(enqueue_mock.call_args.kwargs) == 1 # inputs + assert len(enqueue_mock.call_args.args) == 2 # queue name and workflow function + assert len(enqueue_mock.call_args.kwargs) == 1 # inputs enqueued_inputs = enqueue_mock.call_args.kwargs["inputs"] schedule_plaintext_state = _encrypted_description_raw_json(schedule_result_template) assert enqueued_inputs["task"]["content"] != schedule_plaintext_state @@ -4002,7 +4250,9 @@ async def test_execute_automation_always_calls_initial_wait_when_execution_time_ octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_automation, inputs=inputs, ) - assert await handle.get_result() is None + assert await handle.get_result() == _expected_automation_workflow_envelope_json( + parsed_inputs.task.content + ) mock_wait.assert_awaited_once_with(parsed_inputs, 0) mock_iteration.assert_awaited_once_with(inputs, None) diff --git a/packages/protocol/AGENTS.md b/packages/protocol/AGENTS.md new file mode 100644 index 0000000000..37f45460ec --- /dev/null +++ b/packages/protocol/AGENTS.md @@ -0,0 +1,42 @@ +# Agents: protocol + +## Role + +Shared protocol messages and schemas between node, clients, and services. + +## Owns + +- `packages/protocol/octobot_protocol/` (per repo layout) +- `packages/protocol/test/` (note CI uses `pytest test`) + +## Public surface + +- Protocol models and serialization used by node and clients + +## May depend on + +- `octobot_commons` + +## Do not + +- Embed HTTP route handlers (belongs in node/services/tentacles) +- Hand-edit generated trees (`octobot_protocol/`, `octobot_protocol_ts/`, root `test/test_*.py`) — regenerate from `openapi.json` + +## OpenAPI and codegen + +- **Source of truth:** `openapi.json` in this package — **schemas only** (no HTTP paths). Edit manually when changing wire types. +- **cwd:** `packages/protocol` for all commands below. +- **One-time:** user runs `npm install` (agents do not install per repo policy). +- **After every `openapi.json` change:** run **`npm run generate:all`** (Python + TypeScript codegen). +- **Compatibility PRs** (same PR as the spec change): `npm run generate:all` → `python -m scripts.build_openapi_schema_manifest` → `npm run generate:fixtures` → `pytest test` (or `npm run test:pytest`). Breaking changes, promote, legacy wire — follow [`README.md`](README.md) § “Backwards compatibility workflow”. +- **Not** `node_web_interface/openapi.json` (Node REST, generated from FastAPI). See [`../tentacles/AGENTS.md`](../tentacles/AGENTS.md). + +## Tests + +- **cwd:** `packages/protocol` +- **Example:** `pytest test` +- After spec changes: `generate:all` and compat steps above before merging. + +## Last reviewed + +- 2026-09-18 diff --git a/packages/protocol/octobot_protocol_ts/package.json b/packages/protocol/octobot_protocol_ts/package.json index 0160354d8d..0df7f500d2 100644 --- a/packages/protocol/octobot_protocol_ts/package.json +++ b/packages/protocol/octobot_protocol_ts/package.json @@ -1,6 +1,6 @@ { "name": "@drakkar.software/octobot-protocol", - "version": "0.8.0", + "version": "0.9.0", "description": "TypeScript types generated from OctoBot OpenAPI schema", "repository": { "type": "git", diff --git a/packages/services/AGENTS.md b/packages/services/AGENTS.md new file mode 100644 index 0000000000..c36ddcd986 --- /dev/null +++ b/packages/services/AGENTS.md @@ -0,0 +1,32 @@ +# Agents: services + +## Role + +Notifications, interfaces, and service abstractions (web, telegram, etc.) consumed by the bot and tentacles. + +## Owns + +- `packages/services/octobot_services/` +- `packages/services/tests/` + +## Public surface + +- `octobot_services` service base classes and interface hooks + +## May depend on + +- `octobot_commons` + +## Do not + +- Embed trading execution logic +- Confuse with `packages/tentacles/Services` tentacle implementations (those extend services) + +## Tests + +- **cwd:** `packages/services` +- **Example:** `pytest tests -n auto --dist loadfile` + +## Last reviewed + +- 2026-09-16 diff --git a/packages/services/full_requirements.txt b/packages/services/full_requirements.txt index 66cbf5bace..4aab503f74 100644 --- a/packages/services/full_requirements.txt +++ b/packages/services/full_requirements.txt @@ -1,42 +1,38 @@ # Services # Reddit -asyncpraw==7.8.1 +asyncpraw==8.0.3 # Telegram -python-telegram-bot==22.6 # update alongside supabase for httpx requirement -telethon==1.42.0 +python-telegram-bot==22.8 # update alongside supabase for httpx requirement # Twitter (associated tentacles are disabled as starting from feb 9 2023, API is now paid only). # see https://twitter.com/TwitterDev/status/1621026986784337922 # Python-Twitter==3.5 # Google simplifiedpytrends>=1.1.2 # Ngrok -pyngrok==7.5.0 +pyngrok==7.5.0 # migration to 8 hasn't been tested yet (impossible to dl/install bin) # Web ## http server -flask==3.1.2 -werkzeug == 3.1.5 +flask==3.1.3 +werkzeug == 3.1.8 # Flask templates jinja2==3.1.6 -## flask minification -flask-compress==1.23 -## flask cache -flask-caching==2.3.1 ## flask user authentication management flask-login==0.6.3 ## flask CORS management -flask-cors==6.0.2 +flask-cors==6.0.5 ## user form validators -WTForms==3.2.1 -Flask-WTF==1.2.2 -## websockets -### used by the webhook service and flask-socketio for the web interface -gevent==25.9.1 -### used by flask-socketio with gevent (listed here because multiple libs are usable, force this one) -gevent-websocket==0.10.1 -flask-socketio==5.6.0 +WTForms==3.2.2 +Flask-WTF==1.3.0 +## websockets / ASGI (web interface composite app + isolated webhook Flask server; uvicorn pin matches node package) +# ASGI HTTP server: binds host/port, runs the Starlette (or WsgiToAsgi) app (web_interface/web.py, webhook_service) +uvicorn==0.53.0 +# WSGI→ASGI bridge: Flask (sync WSGI) behind uvicorn (web_interface/asgi/wsgi_to_asgi.py, webhook) +asgiref==3.12.1 +# ASGI router: WebSocket namespaces + Mount("/") for Flask on one port (web_interface/websockets/server/asgi_composite.py) +starlette==1.7.0 # AI -openai==2.20.0 -mcp==1.26.0 +openai==2.20.0 # todo later migrate to 3 which requires HTTPX2 +mcp==1.27.2 # Coingecko coingecko-openapi-client==1.4.0 # Analysis tools diff --git a/packages/services/octobot_services/constants.py b/packages/services/octobot_services/constants.py index 2afd61bf98..9b04f762ad 100644 --- a/packages/services/octobot_services/constants.py +++ b/packages/services/octobot_services/constants.py @@ -88,7 +88,7 @@ NODE_REDIS_URL = "node-redis-url" BACKEND_CORS_ALLOWED_ORIGINS = "backend-cors-allowed-origins" NODE_EXTERNAL_HOST = "node-external-host" -ENV_NODE_SQLITE_FILE = "ENV_NODE_SQLITE_FILE" +ENV_NODE_SQLITE_FILE = "SCHEDULER_SQLITE_FILE" # aligned with node config ENV_NODE_POSTGRES_URL = "ENV_NODE_POSTGRES_URL" ENV_NODE_EXTERNAL_HOST = "NODE_EXTERNAL_HOST" diff --git a/packages/services/octobot_services/managers/interface_manager.py b/packages/services/octobot_services/managers/interface_manager.py index d8a50eec9e..b61db972fc 100644 --- a/packages/services/octobot_services/managers/interface_manager.py +++ b/packages/services/octobot_services/managers/interface_manager.py @@ -16,10 +16,22 @@ import octobot_commons.logging as logging +def _interface_uses_background_thread(interface) -> bool: + return interface.__class__.__name__ == "NodeApiInterface" + + async def start_interfaces(interfaces: list): + logger = logging.get_logger(__name__) started_interfaces = [] for interface in interfaces: - if await interface.start(): + interface_name = interface.get_name() + background_thread = _interface_uses_background_thread(interface) + if background_thread: + logger.info("Starting %s (threaded=True)", interface_name) + started = await interface.start() + if background_thread: + logger.info("Started %s (spawned=%s)", interface_name, started) + if started: started_interfaces.append(interface) return started_interfaces diff --git a/packages/services/octobot_services/services/service_factory.py b/packages/services/octobot_services/services/service_factory.py index 2bde86b9b6..e44e0c8b02 100644 --- a/packages/services/octobot_services/services/service_factory.py +++ b/packages/services/octobot_services/services/service_factory.py @@ -18,6 +18,8 @@ import octobot_commons.logging as logging +import octobot.configuration_manager as configuration_manager +import octobot.enums as octobot_enums import octobot_services.constants as constants import octobot_services.services as services @@ -85,6 +87,22 @@ async def _perform_checkup(self, service) -> bool: self.logger.warning(f"{service.get_name()} initial checkup failed.") except Exception as e: self.logger.exception(e, True, f"{service.get_name()} preparation produced the following error: {e}") + if ( + configuration_manager.get_distribution(self.config) is octobot_enums.OctoBotDistribution.NODE + and service.get_type() == constants.CONFIG_NODE_API + ): + import octobot_node.scheduler as scheduler_module + import octobot.community.node_journal.enums as journal_enums + import octobot.community.node_journal.recording_context as journal_recording_context + import octobot.community.node_journal.lifecycle as journal_startup + journal_recording_context.node_api_startup_failure( + error=e, + startup_phase=journal_enums.JournalStartupPhase.NODE_API_START, + force_exit=False, + config=self.config, + scheduler_init_failure_was_recorded=scheduler_module.scheduler_init_failure_was_recorded, + record_node_startup_failed=journal_startup.record_node_startup_failed, + ) return False @staticmethod diff --git a/packages/services/tests/conftest.py b/packages/services/tests/conftest.py new file mode 100644 index 0000000000..2507677251 --- /dev/null +++ b/packages/services/tests/conftest.py @@ -0,0 +1,33 @@ +# Drakkar-Software OctoBot-Services +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import os +import sys + +import pytest + +_OCTOBOT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +_TESTS_ROOT = os.path.join(_OCTOBOT_ROOT, "tests") +if _TESTS_ROOT not in sys.path: + sys.path.insert(0, _TESTS_ROOT) + +from test_utils.journal_test_support import disabled_node_journal_environment + + +@pytest.fixture(autouse=True) +def disable_node_journal(): + with disabled_node_journal_environment(): + yield diff --git a/packages/services/tests/test_imports.py b/packages/services/tests/test_imports.py index e5fdc18979..ddaabbaefa 100644 --- a/packages/services/tests/test_imports.py +++ b/packages/services/tests/test_imports.py @@ -23,11 +23,9 @@ def test_web_imports(): import flask - import flask_caching - import flask_compress - import flask_socketio - import gevent - import geventwebsocket + import uvicorn + import asgiref.wsgi + import starlette import flask_login import wtforms import flask_wtf @@ -35,7 +33,6 @@ def test_web_imports(): def test_telegram_imports(): import telegram - import telethon def test_openai_imports(): diff --git a/packages/sync/AGENTS.md b/packages/sync/AGENTS.md new file mode 100644 index 0000000000..5f4c2a06fa --- /dev/null +++ b/packages/sync/AGENTS.md @@ -0,0 +1,32 @@ +# Agents: sync + +## Role + +Cloud sync, backup, and replication helpers for OctoBot profiles and node state. + +## Owns + +- `packages/sync/octobot_sync/` (per repo layout) +- `packages/sync/tests/` +- Rust backend optional (`pytest --backend=rust` in CI when crates exist) + +## Public surface + +- Sync client/server modules imported by node and services + +## May depend on + +- `octobot_commons`, protocol as needed + +## Do not + +- Store credentials in repo; use config patterns documented for sync + +## Tests + +- **cwd:** `packages/sync` +- **Example:** `pytest tests -n auto --dist loadfile` + +## Last reviewed + +- 2026-09-16 diff --git a/packages/sync/octobot_sync/app.py b/packages/sync/octobot_sync/app.py index d3495939af..7779813e9c 100644 --- a/packages/sync/octobot_sync/app.py +++ b/packages/sync/octobot_sync/app.py @@ -31,6 +31,8 @@ import octobot_sync.constants as constants import octobot_sync.sync as sync +import octobot.community.node_journal as node_journal + _VERSION_MARKER = f"/{constants.STARFISH_SERVER_MAJOR_VERSION}/" @@ -121,9 +123,23 @@ def _build_role_resolver(is_allowed_user_id: Callable[[str], bool] | None): return resolver async def gated_resolver(request): - result = await resolver(request) + try: + result = await resolver(request) + except CapAuthError as err: + node_journal.record_sync_read_failed( + collection="user-data", + failure_reason="cap_auth", + error=err, + ) + raise if result.identity and not is_allowed_user_id(result.identity): - raise CapAuthError(403, "user not allowed") + auth_error = CapAuthError(403, "user not allowed") + node_journal.record_sync_read_failed( + collection="user-data", + failure_reason="cap_auth", + error=auth_error, + ) + raise auth_error return result return gated_resolver diff --git a/packages/sync/octobot_sync/server.py b/packages/sync/octobot_sync/server.py index ac6a5ead4b..83ca71d71e 100644 --- a/packages/sync/octobot_sync/server.py +++ b/packages/sync/octobot_sync/server.py @@ -38,6 +38,10 @@ import octobot_sync.crypto as sync_crypto import octobot_sync.enums as enums import octobot_sync.errors as errors +import octobot_sync.sync.collection_backend.errors as collection_errors + +import octobot.community.node_journal as node_journal +import octobot.community.node_journal.recording_context as journal_recording_context # Re-exported for callers (e.g. node_api) that build the userId allowlist from # the node's own wallet keys — must use the same derivation as the client. @@ -46,6 +50,7 @@ import octobot_protocol.models as protocol_models import octobot_node.protocol.user_actions as user_actions_protocol +import octobot_node.enums as octobot_node_enums import octobot_node.protocol.user_data as user_data_protocol import octobot_node.protocol.debug as debug_protocol import octobot_node.protocol.accounts as accounts_protocol @@ -147,7 +152,11 @@ async def _user_actions_after_write(event: WriteEvent) -> None: if action is None: return try: - await user_actions_protocol.execute_user_action(action, identity) + await user_actions_protocol.execute_user_action( + action, + identity, + source=octobot_node_enums.UserActionSource.SYNC, + ) except Exception as exc: _get_logger().exception( exc, True, f"Unexpected error executing user action: {action.id}: {exc}" @@ -175,85 +184,112 @@ def _get_opaque_store() -> FilesystemObjectStore: return _opaque_store +def _sync_read_failure_reason(error: BaseException) -> str: + if isinstance(error, errors.OctobotSyncWalletNotFoundError): + return "wallet_not_found" + if isinstance(error, errors.OctobotSyncIdentityMissingError): + return "identity_missing" + if isinstance(error, collection_errors.CollectionStorageError): + return "storage_error" + return "other" + + +def _sync_read_failure_collection(context: StoreContext | None) -> str: + if context and context.collection: + return context.collection + return "unknown" + + async def get_data(key: str, context: StoreContext | None = None) -> str | None: # called when client pulls - collection = _get_collection(context) - plaintext = None - already_encrypted_payload = None - match collection: - case enums.Collections.USER_DATA.value: - user_data_state = await user_data_protocol.get_user_data_state( - _get_identity(context) - ) - plaintext = user_data_state.to_json() - case enums.Collections.USER_ACCOUNTS.value: - encrypted_blob = accounts_protocol.get_accounts_state_encrypted( - _get_identity(context) - ) - already_encrypted_payload = json.dumps(encrypted_blob) - case enums.Collections.USER_ACCOUNTS_AUTH.value: - encrypted_blob = accounts_auth_protocol.get_accounts_authentication_state_encrypted( - _get_identity(context) - ) - already_encrypted_payload = json.dumps(encrypted_blob) - case enums.Collections.USER_ACCOUNTS_TRADING.value: - encrypted_blob = accounts_trading_protocol.get_account_trading_state_encrypted( - _get_identity(context), - _get_account_id(context), - ) - already_encrypted_payload = json.dumps(encrypted_blob) - case enums.Collections.USER_ACCOUNTS_HISTORY.value: - history_state = await accounts_history_protocol.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( - _get_identity(context), - _get_account_id(context), - ) - plaintext = history_state.to_json() - case enums.Collections.USER_ACCOUNTS_HISTORY_AGGREGATED_REAL.value: - history_state = await accounts_history_protocol.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( - _get_identity(context), - is_simulated=False, - ) - plaintext = history_state.to_json() - case enums.Collections.USER_ACCOUNTS_HISTORY_AGGREGATED_SIMULATED.value: - history_state = await accounts_history_protocol.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( - _get_identity(context), - is_simulated=True, - ) - plaintext = history_state.to_json() - case enums.TemporaryCollections.TEMP_USER_STRATEGIES.value: - encrypted_blob = strategies_protocol.get_strategies_state_encrypted( - _get_identity(context) - ) - already_encrypted_payload = json.dumps(encrypted_blob) - case enums.Collections.USER_ACTIONS.value: - # reading user actions should always return an empty list - actions_state = protocol_models.UserActionsState( - version=sync_constants.USER_ACTIONS_STATE_VERSION, - user_actions=[] - ) - plaintext = actions_state.to_json() - case enums.Collections.DEBUG.value: - debug_state = await debug_protocol.get_debug_state( - _get_identity(context) + with journal_recording_context.sync_read_operation( + resolve_collection=lambda: _sync_read_failure_collection(context), + resolve_failure_reason=_sync_read_failure_reason, + ): + collection = _get_collection(context) + plaintext = None + already_encrypted_payload = None + match collection: + case enums.Collections.USER_DATA.value: + user_data_state = await user_data_protocol.get_user_data_state( + _get_identity(context) + ) + plaintext = user_data_state.to_json() + case enums.Collections.USER_ACCOUNTS.value: + encrypted_blob = accounts_protocol.get_accounts_state_encrypted( + _get_identity(context) + ) + already_encrypted_payload = json.dumps(encrypted_blob) + case enums.Collections.USER_ACCOUNTS_AUTH.value: + encrypted_blob = accounts_auth_protocol.get_accounts_authentication_state_encrypted( + _get_identity(context) + ) + already_encrypted_payload = json.dumps(encrypted_blob) + case enums.Collections.USER_ACCOUNTS_TRADING.value: + encrypted_blob = accounts_trading_protocol.get_account_trading_state_encrypted( + _get_identity(context), + _get_account_id(context), + ) + already_encrypted_payload = json.dumps(encrypted_blob) + case enums.Collections.USER_ACCOUNTS_HISTORY.value: + history_state = await accounts_history_protocol.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + _get_identity(context), + _get_account_id(context), + ) + plaintext = history_state.to_json() + case enums.Collections.USER_ACCOUNTS_HISTORY_AGGREGATED_REAL.value: + history_state = await accounts_history_protocol.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + _get_identity(context), + is_simulated=False, + ) + plaintext = history_state.to_json() + case enums.Collections.USER_ACCOUNTS_HISTORY_AGGREGATED_SIMULATED.value: + history_state = await accounts_history_protocol.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + _get_identity(context), + is_simulated=True, + ) + plaintext = history_state.to_json() + case enums.TemporaryCollections.TEMP_USER_STRATEGIES.value: + encrypted_blob = strategies_protocol.get_strategies_state_encrypted( + _get_identity(context) + ) + already_encrypted_payload = json.dumps(encrypted_blob) + case enums.Collections.USER_ACTIONS.value: + # reading user actions should always return an empty list + actions_state = protocol_models.UserActionsState( + version=sync_constants.USER_ACTIONS_STATE_VERSION, + user_actions=[] + ) + plaintext = actions_state.to_json() + case enums.Collections.DEBUG.value: + debug_state = await debug_protocol.get_debug_state( + _get_identity(context) + ) + plaintext = debug_state.to_json() + case _: + # Opaque storage: collections with no protocol bridge are persisted + # as client-encrypted ciphertext and the node never decrypts them. + ciphertext = await _get_opaque_store().get_string(key) + if ciphertext is None: + return None + # Stored bytes are already the client's ciphertext — hash them + # directly so it stays stable until the next push overwrites it. + return _wrap_as_stored_document(ciphertext, ciphertext) + if already_encrypted_payload is not None: + # Pre-encrypted payload (USER_ACCOUNTS): hash the encrypted JSON itself — + # it is deterministic (read from disk) so the hash stays stable. + result = _wrap_as_stored_document(already_encrypted_payload, already_encrypted_payload) + elif plaintext is None: + return None + else: + encrypted = _encrypt(plaintext, _get_identity(context), collection) + result = _wrap_as_stored_document(encrypted, plaintext) + if collection == enums.Collections.USER_DATA.value: + node_journal.on_user_data_pull_succeeded( + sync_user_id=_get_identity(context), + collection=collection, ) - plaintext = debug_state.to_json() - case _: - # Opaque storage: collections with no protocol bridge are persisted - # as client-encrypted ciphertext and the node never decrypts them. - ciphertext = await _get_opaque_store().get_string(key) - if ciphertext is None: - return None - # Stored bytes are already the client's ciphertext — hash them - # directly so it stays stable until the next push overwrites it. - return _wrap_as_stored_document(ciphertext, ciphertext) - if already_encrypted_payload is not None: - # Pre-encrypted payload (USER_ACCOUNTS): hash the encrypted JSON itself — - # it is deterministic (read from disk) so the hash stays stable. - return _wrap_as_stored_document(already_encrypted_payload, already_encrypted_payload) - if plaintext is None: - return None - encrypted = _encrypt(plaintext, _get_identity(context), collection) - return _wrap_as_stored_document(encrypted, plaintext) + return result async def put_data(key: str, body: str, context: StoreContext | None = None) -> None: # Opaque storage: persist the client ciphertext as-is. The node never diff --git a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py index 1ddd09d511..f22424caca 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py +++ b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py @@ -32,6 +32,10 @@ import octobot_sync.sync.collection_backend.state_model as state_model import octobot_sync.sync.collection_backend.tolerant_state_loading as tolerant_state_loading +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.recording_context as journal_recording_context + _MISSING_FILE_CHECKSUM = "" @@ -125,11 +129,21 @@ def _decrypt( ] = None, ) -> state_model.StateModel: try: - plaintext_bytes = sync_crypto.decrypt_blob_dict_to_bytes( - blob, - wallet_private_key, - self.collection, - ) + with journal_recording_context.sync_storage_error( + event=journal_events.NodeJournalEvent.SYNC_STORAGE_FORMAT_ERROR, + collection=self.collection, + provider=journal_enums.SyncStorageProvider.LOCAL, + ): + with journal_recording_context.sync_storage_error( + event=journal_events.NodeJournalEvent.SYNC_STORAGE_DECRYPT_FAILED, + collection=self.collection, + provider=journal_enums.SyncStorageProvider.LOCAL, + ): + plaintext_bytes = sync_crypto.decrypt_blob_dict_to_bytes( + blob, + wallet_private_key, + self.collection, + ) except sync_errors.OctobotSyncCryptoFormatError as err: raise collection_errors.CollectionFileFormatError( f"{self.collection} blob: {err}" @@ -140,15 +154,20 @@ def _decrypt( ) from err try: - if strict: - decrypted_state = state_model.from_json(plaintext_bytes.decode("utf-8")) - else: - decrypted_state = tolerant_state_loading.TolerantStateLoader( - state_model, - collection=self.collection, - model_sanitizers=model_sanitizers, - model_fallbacks=model_fallbacks, - ).from_json(plaintext_bytes.decode("utf-8")) + with journal_recording_context.sync_storage_error( + event=journal_events.NodeJournalEvent.SYNC_STORAGE_FORMAT_ERROR, + collection=self.collection, + provider=journal_enums.SyncStorageProvider.LOCAL, + ): + if strict: + decrypted_state = state_model.from_json(plaintext_bytes.decode("utf-8")) + else: + decrypted_state = tolerant_state_loading.TolerantStateLoader( + state_model, + collection=self.collection, + model_sanitizers=model_sanitizers, + model_fallbacks=model_fallbacks, + ).from_json(plaintext_bytes.decode("utf-8")) except Exception as err: raise collection_errors.CollectionFileFormatError( f"Decrypted {self.collection} payload is not valid JSON: {err}" @@ -201,9 +220,15 @@ def _read_blob(self, storage_key: str) -> dict[str, typing.Any]: with open(path, "r", encoding="utf-8") as handle: raw = json.load(handle) if not isinstance(raw, dict): - raise collection_errors.CollectionFileFormatError( + format_error = collection_errors.CollectionFileFormatError( f"{self.collection} file must contain an encrypted blob object" ) + with journal_recording_context.sync_storage_error( + event=journal_events.NodeJournalEvent.SYNC_STORAGE_FORMAT_ERROR, + collection=self.collection, + provider=journal_enums.SyncStorageProvider.LOCAL, + ): + raise format_error return raw def load_items_encrypted(self, storage_key: str) -> dict[str, str]: diff --git a/packages/sync/octobot_sync/sync/collection_backend/tolerant_state_loading.py b/packages/sync/octobot_sync/sync/collection_backend/tolerant_state_loading.py index 109872ceaf..5851317d29 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/tolerant_state_loading.py +++ b/packages/sync/octobot_sync/sync/collection_backend/tolerant_state_loading.py @@ -21,6 +21,9 @@ import octobot_sync.sync.collection_backend.state_model as state_model +import octobot.community.node_journal as node_journal +import octobot.community.node_journal.events as journal_events + logger = commons_logging.get_logger("TolerantStateLoading") @@ -50,6 +53,14 @@ def __init__( self.model_sanitizers = model_sanitizers or {} self.model_fallbacks = model_fallbacks or {} + def _record_schema_recovery(self, recovery_action: str) -> None: + node_journal.record_sync_storage_event( + journal_events.NodeJournalEvent.SYNC_STORAGE_SCHEMA_RECOVERY, + collection=self.collection, + provider="local", + recovery_action=recovery_action, + ) + def from_json(self, json_str: str) -> state_model.StateModel: if self.state_class is None: raise ValueError("state_class is required for from_json") @@ -98,9 +109,11 @@ def from_dict( parsed_state_dict, ) try: - return _parse_model_strict(self.state_class, sanitized_state_dict) + healed_state = _parse_model_strict(self.state_class, sanitized_state_dict) except Exception as retry_error: raise retry_error from strict_error + self._record_schema_recovery("sanitize_state") + return healed_state def model_from_dict_lenient( self, @@ -118,6 +131,7 @@ def model_from_dict_lenient( context, type(raw_dict).__name__, ) + self._record_schema_recovery("skip_item") return None raise ValueError( f"Expected dict for {model_class.__name__} in {context}, " @@ -133,13 +147,16 @@ def model_from_dict_lenient( return _parse_model_strict(model_class, sanitized_dict) except Exception as retry_error: try: - return self._rebuild_model_from_sanitized_dict( + rebuilt_model = self._rebuild_model_from_sanitized_dict( model_class, sanitized_dict, context=context, ) except Exception as rebuild_error: retry_error = rebuild_error + else: + self._record_schema_recovery("rebuild_item") + return rebuilt_model fallback_factory = self.model_fallbacks.get(model_class) if fallback_factory is not None: logger.warning( @@ -149,6 +166,7 @@ def model_from_dict_lenient( context, retry_error, ) + self._record_schema_recovery("fallback_item") return fallback_factory() if allow_skip: logger.warning( @@ -158,6 +176,7 @@ def model_from_dict_lenient( context, retry_error, ) + self._record_schema_recovery("skip_item") return None raise retry_error from strict_error diff --git a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py index 5cb37641eb..58c6a2e8bf 100644 --- a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py +++ b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py @@ -18,7 +18,6 @@ import typing import octobot.community.authentication as community_authentication -import octobot.community.wallet_backend.errors as wallet_backend_errors import octobot_commons.logging as commons_logging import octobot_commons.singleton.singleton_class as singleton_class import octobot_sync.constants as sync_constants @@ -139,13 +138,11 @@ def list_collectable_wallet_ids(self) -> list[str]: community_auth = community_authentication.CommunityAuthentication.instance() collectable_wallet_ids = [] for wallet_id in self.list_registered_wallet_ids(): - try: - community_auth.get_wallet_by_user_id(wallet_id) - except wallet_backend_errors.WalletNotFoundError: + if community_auth.has_wallet_for_user_id(wallet_id): + collectable_wallet_ids.append(wallet_id) + else: logger.debug( "Skipping wallet %s: not registered locally", wallet_id, ) - continue - collectable_wallet_ids.append(wallet_id) return collectable_wallet_ids diff --git a/packages/sync/tests/conftest.py b/packages/sync/tests/conftest.py index 6c1a4297f5..ebced54a9a 100644 --- a/packages/sync/tests/conftest.py +++ b/packages/sync/tests/conftest.py @@ -16,6 +16,52 @@ """Shared test fixtures.""" +import os +import sys + +import pytest + +_OCTOBOT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +_TESTS_ROOT = os.path.join(_OCTOBOT_ROOT, "tests") +if _TESTS_ROOT not in sys.path: + sys.path.insert(0, _TESTS_ROOT) + +from test_utils.journal_integration_fixtures import ( + isolated_journal_environment_context, + journal_persisted_state, + journal_user_root, + surface_journal_errors_context, +) +from test_utils.journal_test_support import ( + disabled_node_journal_environment, + enabled_node_journal_environment, +) + + +def _is_sync_journal_integration_test(request) -> bool: + return request.node.path.name.endswith("_journal.py") + + +@pytest.fixture(autouse=True) +def disable_node_journal(request): + if _is_sync_journal_integration_test(request): + yield + return + with disabled_node_journal_environment(): + yield + + +@pytest.fixture(autouse=True) +def sync_journal_integration_autouse(request): + if not _is_sync_journal_integration_test(request): + yield + return + journal_user_root = request.getfixturevalue("journal_user_root") + with enabled_node_journal_environment(): + with isolated_journal_environment_context(str(journal_user_root)): + with surface_journal_errors_context(): + yield + class MemoryObjectStore: """Minimal AbstractObjectStore for testing.""" diff --git a/packages/sync/tests/sync/collection_providers/test_user_account_provider.py b/packages/sync/tests/sync/collection_providers/test_user_account_provider.py index beaa56c092..b109c60c5d 100644 --- a/packages/sync/tests/sync/collection_providers/test_user_account_provider.py +++ b/packages/sync/tests/sync/collection_providers/test_user_account_provider.py @@ -17,9 +17,10 @@ import mock import octobot.community.authentication as community_authentication -import octobot.community.wallet_backend.errors as wallet_backend_errors import octobot_sync.sync.collection_providers.user_account_provider as account_provider_module +_JOURNAL_PATCH = "octobot.community.node_journal.record_wallet_operation_failed" + class TestListCollectableWalletIds: def test_returns_only_wallets_registered_locally(self): @@ -29,11 +30,7 @@ def test_returns_only_wallets_registered_locally(self): "wallet-missing", ] community_auth = mock.Mock() - community_auth.get_wallet_by_user_id.side_effect = lambda wallet_id: ( - mock.Mock() - if wallet_id == "wallet-known" - else (_ for _ in ()).throw(wallet_backend_errors.WalletNotFoundError("missing")) - ) + community_auth.has_wallet_for_user_id.side_effect = lambda wallet_id: wallet_id == "wallet-known" with mock.patch.object( community_authentication.CommunityAuthentication, "instance", @@ -42,6 +39,23 @@ def test_returns_only_wallets_registered_locally(self): result = account_provider_module.AccountProvider.list_collectable_wallet_ids(provider) assert result == ["wallet-known"] + def test_skips_orphan_wallet_ids_without_journaling(self): + provider = mock.Mock(spec=account_provider_module.AccountProvider) + provider.list_registered_wallet_ids.return_value = [ + "wallet-known", + "wallet-missing", + ] + community_auth = mock.Mock() + community_auth.has_wallet_for_user_id.side_effect = lambda wallet_id: wallet_id == "wallet-known" + with mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=community_auth, + ), mock.patch(_JOURNAL_PATCH) as record_mock: + result = account_provider_module.AccountProvider.list_collectable_wallet_ids(provider) + assert result == ["wallet-known"] + record_mock.assert_not_called() + def test_returns_empty_when_no_registered_wallets(self): provider = mock.Mock(spec=account_provider_module.AccountProvider) provider.list_registered_wallet_ids.return_value = [] diff --git a/packages/sync/tests/test_server.py b/packages/sync/tests/test_server.py index cfbd70d3e3..f3a8da044a 100644 --- a/packages/sync/tests/test_server.py +++ b/packages/sync/tests/test_server.py @@ -11,6 +11,8 @@ import octobot_sync.enums as enums import octobot_sync.errors as errors +import octobot_node.enums as octobot_node_enums + from starfish_server.storage.base import StoreContext from starfish_server.storage.s3 import S3ObjectStore from starfish_server.storage.filesystem import FilesystemObjectStore @@ -370,7 +372,11 @@ async def test_user_actions_executes_appended_action(self): mock_proto.execute_user_action = mock.AsyncMock() await server._user_actions_after_write(event) mock_pm.UserAction.from_json.assert_called_once_with(plain_body) - mock_proto.execute_user_action.assert_awaited_once_with(action, "0xwallet") + mock_proto.execute_user_action.assert_awaited_once_with( + action, + "0xwallet", + source=octobot_node_enums.UserActionSource.SYNC, + ) @pytest.mark.asyncio async def test_user_actions_logs_exception_on_failure(self): diff --git a/packages/sync/tests/test_sync_read_failed_cap_auth_journal.py b/packages/sync/tests/test_sync_read_failed_cap_auth_journal.py new file mode 100644 index 0000000000..5d2fcd1070 --- /dev/null +++ b/packages/sync/tests/test_sync_read_failed_cap_auth_journal.py @@ -0,0 +1,55 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import mock +import pytest +from starfish_server.router.cap_resolver import CapAuthError + +import octobot_sync.app as sync_app + +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module + + +class TestBuildRoleResolverCapAuth: + @pytest.mark.asyncio + async def test_records_sync_read_failed_when_cap_auth_raises(self, journal_persisted_state): + auth_error = CapAuthError(401, "invalid cap") + inner_resolver = mock.AsyncMock(side_effect=auth_error) + with mock.patch("octobot_sync.app.create_cap_cert_role_resolver", return_value=inner_resolver): + gated_resolver = sync_app._build_role_resolver(lambda _identity: True) + with pytest.raises(CapAuthError): + await gated_resolver(mock.Mock()) + events = journal_module.read_events() + assert len(events) == 1 + event_line = events[0] + assert event_line.event == journal_events.NodeJournalEvent.SYNC_READ_FAILED + assert event_line.attributes.collection == "user-data" + assert event_line.attributes.failure_reason == "cap_auth" + assert event_line.attributes.error_category == "CapAuthError" + + @pytest.mark.asyncio + async def test_records_sync_read_failed_when_identity_not_allowed(self, journal_persisted_state): + auth_result = mock.Mock() + auth_result.identity = "blocked-user" + inner_resolver = mock.AsyncMock(return_value=auth_result) + with mock.patch("octobot_sync.app.create_cap_cert_role_resolver", return_value=inner_resolver): + gated_resolver = sync_app._build_role_resolver(lambda identity: identity == "allowed-user") + with pytest.raises(CapAuthError, match="user not allowed"): + await gated_resolver(mock.Mock()) + events = journal_module.read_events() + assert len(events) == 1 + assert events[0].attributes.failure_reason == "cap_auth" diff --git a/packages/sync/tests/test_sync_read_failed_journal.py b/packages/sync/tests/test_sync_read_failed_journal.py new file mode 100644 index 0000000000..cad8af18ed --- /dev/null +++ b/packages/sync/tests/test_sync_read_failed_journal.py @@ -0,0 +1,117 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import mock +import pytest +from starfish_server.storage.base import StoreContext + +import octobot_sync.enums as sync_enums +import octobot_sync.errors as sync_errors +import octobot_sync.server as sync_server +import octobot_sync.sync.collection_backend.errors as collection_errors + +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module + + +def _user_data_context() -> StoreContext: + return StoreContext( + collection=sync_enums.Collections.USER_DATA.value, + params={}, + identity="user-1", + roles=(), + action="read", + ) + + +class TestSyncReadFailureReason: + def test_maps_wallet_not_found(self): + assert sync_server._sync_read_failure_reason( + sync_errors.OctobotSyncWalletNotFoundError("missing"), + ) == "wallet_not_found" + + def test_maps_identity_missing(self): + assert sync_server._sync_read_failure_reason( + sync_errors.OctobotSyncIdentityMissingError("missing"), + ) == "identity_missing" + + def test_maps_storage_error(self): + assert sync_server._sync_read_failure_reason( + collection_errors.CollectionStorageError("broken"), + ) == "storage_error" + + def test_maps_unknown_errors_to_other(self): + assert sync_server._sync_read_failure_reason(RuntimeError("boom")) == "other" + + +class TestSyncReadFailureCollection: + def test_uses_context_collection_when_present(self): + context = StoreContext( + collection="user-accounts", + params={}, + identity="user-1", + roles=(), + action="read", + ) + assert sync_server._sync_read_failure_collection(context) == "user-accounts" + + def test_defaults_to_unknown_without_context(self): + assert sync_server._sync_read_failure_collection(None) == "unknown" + + +class TestGetData: + @pytest.mark.asyncio + async def test_records_sync_read_failed_and_reraises(self, journal_persisted_state): + expected_error = sync_errors.OctobotSyncWalletNotFoundError("wallet missing") + with mock.patch( + "octobot_sync.server.user_data_protocol.get_user_data_state", + new_callable=mock.AsyncMock, + side_effect=expected_error, + ): + with pytest.raises(sync_errors.OctobotSyncWalletNotFoundError): + await sync_server.get_data("user-data-key", _user_data_context()) + + events = journal_module.read_events() + assert len(events) == 1 + event_line = events[0] + assert event_line.event == journal_events.NodeJournalEvent.SYNC_READ_FAILED + assert event_line.attributes.collection == sync_enums.Collections.USER_DATA.value + assert event_line.attributes.failure_reason == "wallet_not_found" + assert event_line.attributes.error_category == "OctobotSyncWalletNotFoundError" + + @pytest.mark.asyncio + async def test_successful_user_data_pull_does_not_record_sync_read_failed(self, journal_persisted_state): + user_data_state = mock.Mock() + user_data_state.to_json.return_value = "{}" + with mock.patch( + "octobot_sync.server.user_data_protocol.get_user_data_state", + new_callable=mock.AsyncMock, + return_value=user_data_state, + ), mock.patch( + "octobot_sync.server._encrypt", + return_value="encrypted", + ), mock.patch( + "octobot_sync.server.node_journal.on_user_data_pull_succeeded", + ): + result = await sync_server.get_data("user-data-key", _user_data_context()) + + assert result is not None + sync_read_failures = [ + event_line + for event_line in journal_module.read_events() + if event_line.event == journal_events.NodeJournalEvent.SYNC_READ_FAILED + ] + assert sync_read_failures == [] diff --git a/packages/tentacles/AGENTS.md b/packages/tentacles/AGENTS.md new file mode 100644 index 0000000000..65ddd26343 --- /dev/null +++ b/packages/tentacles/AGENTS.md @@ -0,0 +1,87 @@ +# Agents: tentacles (sources vs install) + +## Role + +**Default plugin bundle** for OctoBot: concrete evaluators, trading modes, exchange connectors, services, automation pieces, AI agents, and DSL operators on top of framework packages (`octobot_trading`, `octobot_evaluators`, etc.). + +- **Sources vs runtime:** edit **`packages/tentacles/`** (`Evaluator/`, `Trading/`, `Services/`, …). After pack/install, Python loads repo-root **`tentacles/`**—that tree is **generated**; never commit edits there. +- **Layout contract:** each tentacle has `metadata.json`, reference `config/`, and version-gated imports (`check_tentacle_version()` in generated `__init__.py`). Install, update, and packaging live in **`octobot_tentacles_manager`**, not in ad-hoc scripts here. +- **Node product UI:** `Services/Interfaces/node_api_interface` exposes FastAPI routes that submit **`UserAction`** to the node; `node_web_interface` is the Vite/React app (Biome, vitest). Action **business logic** belongs in **node** executors—do not duplicate it only in tentacle routes. +- **Profiles:** shipped profiles and `specific_config/` appear under the **installed** tree; feature work stays under sources paths above. + +## Owns + +- `packages/tentacles/**` for feature work (strategies, exchanges, services, node UI tentacles, etc.) + +## Public surface + +- Tentacle modules discovered after install (paths mirror sources under repo-root `tentacles/`) +- Loaders and activation via `octobot_tentacles_manager`—not a single top-level Python package name in this repo layout + +## May depend on + +- All relevant `octobot_*` packages; `node_web_interface` uses npm build in the tentacles tree + +## Do not + +- Commit changes under repo-root `tentacles/` (generated) +- Skip `bash .cursor/reinstall-tentacles.sh` after source edits in cloud or agent handoff flows +- Hand-edit generated `__init__.py` in sources without understanding tentacles_manager regeneration +- Rewrite core `ExchangeManager` here—subclass connectors and modes in tentacles +- Hand-edit `Services/Interfaces/node_web_interface/openapi.json` (sources or installed `tentacles/` copy) +- Hand-edit `node_web_interface/src/client/*` (generated by `npm run generate-client`) + +## Node REST OpenAPI (node_api_interface → node_web_interface) + +The Node **HTTP** contract for the web UI is **not** `packages/protocol/openapi.json` (wire schemas only). REST OpenAPI is generated from FastAPI in **`node_api_interface`**. + +**Do** + +- Change the HTTP surface in `Services/Interfaces/node_api_interface/` (`api/routes/`, app wiring, Pydantic models used by FastAPI). +- Regenerate the REST spec via the tentacle **build** in `node_api_interface/metadata.json`: + +```json +"build": ["python build_openapi.py > ../node_web_interface/openapi.json"] +``` + +Manual equivalent (from `packages/tentacles/Services/Interfaces/node_api_interface/`, with `source .cursor/env.sh` / PYTHONPATH as for other tentacles Python work): + +```bash +python build_openapi.py > ../node_web_interface/openapi.json +``` + +(`build_openapi.py` imports `node_api` and calls `NodeApiInterface.create_app().openapi()`.) + +Then regenerate the TypeScript client (UI sources under packages path only): + +```bash +cd packages/tentacles/Services/Interfaces/node_web_interface +npm run generate-client +``` + +See `node_web_interface/README.md` (openapi generated by node_api_interface). Wire protocol models: [`../protocol/AGENTS.md`](../protocol/AGENTS.md). + + +## End-user UI + +Node and classic web UI tentacles share end-user copy rules. Before layout or user-visible text changes, read: + +- `packages/tentacles/Services/Interfaces/node_web_interface/AGENTS.md` +- `packages/tentacles/Services/Interfaces/web_interface/AGENTS.md` +- Skill **end-user-ui** (`.cursor/skills/end-user-ui/SKILL.md`) + +Edit sources only under `packages/tentacles/`; never repo-root `tentacles/`. + +## Tests + +- Often via `octobot` job: `pytest --ignore=tentacles/Trading/Exchange tentacles` +- UI: `packages/tentacles/Services/Interfaces/node_web_interface` (npm/vitest); after route changes, regen openapi + `generate-client` before UI work that depends on new types + +## Related human doc + +- `docs/content/developers/packages/tentacles.md` +- `docs/content/guides/octobot-tentacles/` + +## Last reviewed + +- 2026-09-18 \ No newline at end of file diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/symbol_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/symbol_operators.py index b39b93c9bc..98ac737a46 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/symbol_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/symbol_operators.py @@ -71,6 +71,10 @@ class _MarketExpiryOperator(exchange_operator.ExchangeOperator): DESCRIPTION = "Returns the expiry timestamp in milliseconds for the given symbol's market, or None" EXAMPLE = "market_expiry(triggered_symbol())" + @staticmethod + def get_library() -> str: + return commons_constants.CONTEXTUAL_OPERATORS_LIBRARY + @classmethod def get_parameters(cls) -> list: return [ diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_personal_data_operators/test_cancel_order_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_personal_data_operators/test_cancel_order_operators.py index 1d84ca59f3..9d7ab22d6d 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_personal_data_operators/test_cancel_order_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_personal_data_operators/test_cancel_order_operators.py @@ -35,6 +35,7 @@ ) SYMBOL = "BTC/USDT" +TICKER_WISE_SYMBOL = "BTC@BTC/USDT@ETH" EXCHANGE_ORDER_ID = "order-123" @@ -413,3 +414,52 @@ def test_cancel_order_get_dependencies_from_interpreter( assert maybe_exchange_manager_interpreter.get_dependencies() == [ octobot_trading.dsl.SymbolDependency(symbol=symbol), ] + + +class TestCancelOrderOperatorsNetworkQualified: + @pytest.mark.asyncio + async def test_pre_compute_cancels_orders_matching_ticker_wise_symbol( + self, cancel_order_operators_list, backtesting_trader + ): + _config, exchange_manager, _trader = backtesting_trader + cancel_order_op_class, = cancel_order_operators_list + ticker_wise_order = _create_mock_order("ticker-wise-order", symbol=TICKER_WISE_SYMBOL) + plain_order = _create_mock_order("plain-order", symbol=SYMBOL) + with mock.patch.object( + exchange_manager.exchange_personal_data.orders_manager, + "get_open_orders", + mock.Mock(return_value=[ticker_wise_order, plain_order]), + ), mock.patch.object( + exchange_manager.trader, + "cancel_order", + mock.AsyncMock(return_value=True), + ) as cancel_order_mock: + operator = cancel_order_op_class( + TICKER_WISE_SYMBOL, exchange_order_ids=["ticker-wise-order"] + ) + await operator.pre_compute() + assert cancel_order_mock.await_count == 1 + assert cancel_order_mock.await_args[0][0] == ticker_wise_order + + @pytest.mark.asyncio + async def test_cancel_order_call_as_dsl(self, interpreter, backtesting_trader): + _config, exchange_manager, _trader = backtesting_trader + if TICKER_WISE_SYMBOL not in exchange_manager.client_symbols: + exchange_manager.client_symbols.append(TICKER_WISE_SYMBOL) + if TICKER_WISE_SYMBOL not in exchange_manager.exchange_config.traded_symbol_pairs: + exchange_manager.exchange_config.traded_symbol_pairs.append(TICKER_WISE_SYMBOL) + exchange_order_id = "ticker-wise-cancel" + limit_buy = trading_personal_data.BuyLimitOrder(exchange_manager.trader) + limit_buy.update( + order_type=octobot_trading.enums.TraderOrderType.BUY_LIMIT, + symbol=TICKER_WISE_SYMBOL, + exchange_order_id=exchange_order_id, + current_price=decimal.Decimal("50000"), + quantity=decimal.Decimal("0.01"), + price=decimal.Decimal("50000"), + ) + await exchange_manager.exchange_personal_data.orders_manager.upsert_order_instance(limit_buy) + result = await interpreter.interprete( + f"cancel_order('{TICKER_WISE_SYMBOL}', exchange_order_ids=['{exchange_order_id}'])" + ) + assert result == {"cancelled_orders": [exchange_order_id]} diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_personal_data_operators/test_create_order_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_personal_data_operators/test_create_order_operators.py index 035972fba7..9c7f639953 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_personal_data_operators/test_create_order_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_personal_data_operators/test_create_order_operators.py @@ -37,6 +37,7 @@ ) SYMBOL = "BTC/USDT" +TICKER_WISE_SYMBOL = "BTC@BTC/USDT@ETH" AMOUNT = 0.01 PRICE = "50000" MARK_PRICE = decimal.Decimal("50000") @@ -158,6 +159,23 @@ def _ensure_sell_order_trading_context(backtesting_trader): ) +def _ensure_ticker_wise_trading_context(backtesting_trader): + _ensure_portfolio_config(backtesting_trader, {"BTC@BTC": 0, "USDT@ETH": 100000}) + _config, exchange_manager, _trader = backtesting_trader + if TICKER_WISE_SYMBOL not in exchange_manager.client_symbols: + exchange_manager.client_symbols.append(TICKER_WISE_SYMBOL) + if TICKER_WISE_SYMBOL not in exchange_manager.exchange_config.traded_symbol_pairs: + exchange_manager.exchange_config.traded_symbol_pairs.append(TICKER_WISE_SYMBOL) + exchange_manager.exchange_config.traded_symbols.append( + commons_symbols.parse_symbol(TICKER_WISE_SYMBOL) + ) + symbol_data = exchange_manager.exchange_symbols_data.get_exchange_symbol_data( + TICKER_WISE_SYMBOL, allow_creation=True + ) + symbol_data.handle_mark_price_update( + MARK_PRICE, octobot_trading.enums.MarkPriceSources.EXCHANGE_MARK_PRICE.value + ) + class TestCreateOrderOnExchange: @pytest.mark.asyncio @@ -810,3 +828,40 @@ def test_market_order_get_dependencies_from_interpreter_with_exchange_manager(se assert maybe_exchange_manager_interpreter.get_dependencies() == [ octobot_trading.dsl.SymbolDependency(symbol=symbol), ] + + +class TestCreateOrderOperatorsNetworkQualified: + @pytest.mark.asyncio + async def test_market_dsl_creates_order_with_ticker_wise_symbol(self, interpreter, backtesting_trader): + _ensure_ticker_wise_trading_context(backtesting_trader) + result = await interpreter.interprete( + f"market('buy', '{TICKER_WISE_SYMBOL}', {AMOUNT})" + ) + assert result["created_orders"][0]["symbol"] == TICKER_WISE_SYMBOL + + @pytest.mark.asyncio + async def test_limit_dsl_forwards_symbol_to_order_factory( + self, create_order_operators_list, backtesting_trader + ): + _market_op_class, limit_op_class, _stop_loss_op_class = create_order_operators_list + mock_order = _create_mock_order(symbol=TICKER_WISE_SYMBOL) + factory = limit_op_class("buy", TICKER_WISE_SYMBOL, AMOUNT, PRICE).get_order_factory() + with mock.patch.object( + factory, + "create_base_orders_and_associated_elements", + mock.AsyncMock(return_value=[mock_order]), + ), mock.patch.object( + factory, + "create_order_on_exchange", + mock.AsyncMock(return_value=mock_order), + ): + operator = limit_op_class("buy", TICKER_WISE_SYMBOL, AMOUNT, PRICE) + await operator.pre_compute() + call_kwargs = factory.create_base_orders_and_associated_elements.call_args[1] + assert call_kwargs["symbol"] == TICKER_WISE_SYMBOL + + def test_symbol_dependency_uses_ticker_wise_symbol(self, interpreter): + interpreter.prepare(f"market('buy', '{TICKER_WISE_SYMBOL}', {AMOUNT})") + assert interpreter.get_dependencies() == [ + octobot_trading.dsl.SymbolDependency(symbol=TICKER_WISE_SYMBOL), + ] diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_public_data_operators/test_symbol_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_public_data_operators/test_symbol_operators.py index ca2a0d18aa..77c384b5a9 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_public_data_operators/test_symbol_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/tests/exchange_public_data_operators/test_symbol_operators.py @@ -54,6 +54,17 @@ def interpreter(host): ) +def test_market_expiry_operator_excluded_from_global_catalog(host): + symbol_operator_classes = exchange_operators.create_symbol_operators(host) + market_expiry_operators = [ + operator_class + for operator_class in symbol_operator_classes + if operator_class.get_name() == "market_expiry" + ] + assert len(market_expiry_operators) == 1 + assert market_expiry_operators[0] not in dsl_interpreter.get_all_operators() + + @pytest.mark.asyncio async def test_triggered_symbol(interpreter): assert await interpreter.interprete("triggered_symbol()") == SYMBOL diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py index 7f29b0493f..17b089226e 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py @@ -40,7 +40,6 @@ import octobot_commons.configuration import octobot.constants as octobot_constants -import octobot.community.activity_analysis.activity_metrics as activity_metrics import octobot.community.supabase_backend.enums as community_enums import octobot_flow.entities as octobot_flow_entities import octobot_flow.entities.accounts.process_bot_state as process_bot_state_import @@ -92,16 +91,6 @@ def _resolve_state_file_path(recall_state: octobot_process_state_import.OctobotP ) ) -def _report_child_octobot_first_start_if_needed(init_info: dict[str, typing.Any]) -> None: - try: - if init_info.get("already_prepared"): - return - activity_metrics.ActivityMetrics.report_child_octobot_first_start() - except Exception as err: - _get_logger().exception( - err, True, f"Failed to report child OctoBot first start {err}" - ) - # --- Liveness and routing (recall state + child dump) --- @@ -1350,7 +1339,6 @@ async def _pre_compute_first_spawn( environment=child_env, hide_console_window=True, ) - _report_child_octobot_first_start_if_needed(init_info) spawn_pid = self.pid or 0 scheme = str(params.get("http_scheme") or "http").rstrip(":/") http_base_url_host = bind_host or "127.0.0.1" diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py index b099a6a779..fbf6961f8c 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py @@ -1965,159 +1965,6 @@ async def test_returns_recallable_with_init_state_ok_after_first_spawn(self, tmp assert post.updated_exchange_account_elements is not None -class TestReportChildOctobotFirstStartIfNeeded: - def test_skips_when_layout_already_prepared(self): - with mock.patch.object( - octobot_process_ops.activity_metrics.ActivityMetrics, - "report_child_octobot_first_start", - ) as report_mock: - octobot_process_ops._report_child_octobot_first_start_if_needed( - {"already_prepared": True}, - ) - report_mock.assert_not_called() - - def test_reports_first_child_start_when_layout_was_created(self): - with mock.patch.object( - octobot_process_ops.activity_metrics.ActivityMetrics, - "report_child_octobot_first_start", - ) as report_mock: - octobot_process_ops._report_child_octobot_first_start_if_needed( - {"already_prepared": False}, - ) - report_mock.assert_called_once_with() - - def test_logs_exception_when_report_fails(self): - report_error = RuntimeError("sentry unavailable") - logger_mock = mock.Mock() - with mock.patch.object( - octobot_process_ops.activity_metrics.ActivityMetrics, - "report_child_octobot_first_start", - side_effect=report_error, - ), mock.patch.object( - octobot_process_ops, - "_get_logger", - return_value=logger_mock, - ): - octobot_process_ops._report_child_octobot_first_start_if_needed( - {"already_prepared": False}, - ) - logger_mock.exception.assert_called_once_with( - report_error, - True, - f"Failed to report child OctoBot first start {report_error}", - ) - - -class TestEnsureOctobotProcessPrecomputeReportsChildFirstStart: - async def test_reports_child_first_start_on_first_spawn_when_layout_is_new(self, tmp_path): - start_script = tmp_path / "start.py" - start_script.write_text("#", encoding="utf-8") - user_root = str( - tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" - ) - op = EnsureOctobotProcessOperator( - user_folder="ub", - user_id=_PROCESS_TEST_USER_ID, - profile_data=_MINIMAL_PROFILE_DATA, - last_execution_result=None, - ) - with mock.patch.object( - octobot_process_ops.os, - "getcwd", - return_value=str(tmp_path), - ), mock.patch.object( - octobot_process_ops, - "ensure_user_profile_and_layout", - new=mock.AsyncMock( - return_value={ - "user_root": user_root, - "profile_id": "x", - "already_prepared": False, - } - ), - ), mock.patch.object( - octobot_process_ops, - "_listen_port_pair_with_shared_scan_offset", - return_value=(20050, 30050), - ), mock.patch.object( - process_util, - "spawn_managed_subprocess", - ) as spawn_mock, mock.patch.object( - process_util, - "pid_is_running", - side_effect=lambda process_id: process_id == 10001, - ), mock.patch.object( - octobot_process_ops, - "_load_process_bot_state", - new=mock.AsyncMock(side_effect=_async_live_process_bot_state_with_pid_10001), - ), mock.patch.object( - octobot_process_ops, - "_report_child_octobot_first_start_if_needed", - ) as report_child_mock: - spawn_mock.return_value.pid = 10001 - await op.pre_compute() - report_child_mock.assert_called_once_with( - { - "user_root": user_root, - "profile_id": "x", - "already_prepared": False, - }, - ) - - async def test_reports_child_first_start_on_spawn_when_child_not_yet_live(self, tmp_path): - start_script = tmp_path / "start.py" - start_script.write_text("#", encoding="utf-8") - user_root = str( - tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" - ) - op = EnsureOctobotProcessOperator( - user_folder="ub", - user_id=_PROCESS_TEST_USER_ID, - profile_data=_MINIMAL_PROFILE_DATA, - last_execution_result=None, - ) - with mock.patch.object( - octobot_process_ops.os, - "getcwd", - return_value=str(tmp_path), - ), mock.patch.object( - octobot_process_ops, - "ensure_user_profile_and_layout", - new=mock.AsyncMock( - return_value={ - "user_root": user_root, - "profile_id": "x", - "already_prepared": False, - } - ), - ), mock.patch.object( - octobot_process_ops, - "_listen_port_pair_with_shared_scan_offset", - return_value=(20050, 30050), - ), mock.patch.object( - process_util, - "spawn_managed_subprocess", - ) as spawn_mock, mock.patch.object( - octobot_process_ops, - "_load_process_bot_state", - new=mock.AsyncMock(side_effect=_async_return_none_mock), - ), mock.patch.object( - octobot_process_ops, - "_report_child_octobot_first_start_if_needed", - ) as report_child_mock: - spawn_mock.return_value.pid = 10001 - await op.pre_compute() - report_child_mock.assert_called_once_with( - { - "user_root": user_root, - "profile_id": "x", - "already_prepared": False, - }, - ) - first_le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] - assert first_le.get("init_state_ok") is False - - class TestEnsureOctobotProcessPrecomputeRecallPathWhenProcessStateLive: async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_path): start_script = tmp_path / "start.py" @@ -2155,13 +2002,9 @@ async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_pa octobot_process_ops, "_load_process_bot_state", new=mock.AsyncMock(side_effect=_async_return_none_mock), - ), mock.patch.object( - octobot_process_ops.activity_metrics.ActivityMetrics, - "report_child_octobot_first_start", - ) as report_child_mock: + ): spawn_mock.return_value.pid = 10002 await op1.pre_compute() - report_child_mock.assert_not_called() first_value = op1.value assert isinstance(first_value, dict) first_le = first_value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] @@ -2185,12 +2028,8 @@ async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_pa octobot_process_ops, "_load_process_bot_state", new=mock.AsyncMock(side_effect=_async_live_process_bot_state_mock), - ), mock.patch.object( - octobot_process_ops.activity_metrics.ActivityMetrics, - "report_child_octobot_first_start", - ) as report_child_mock: + ): await op2.pre_compute() - report_child_mock.assert_not_called() assert isinstance(op2.value, dict) assert dsl_interpreter.ReCallingOperatorResult.__name__ in op2.value le2 = op2.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_resetting_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_resetting_operators.py index fb5bcea187..ce526d33bf 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_resetting_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_resetting_operators.py @@ -126,11 +126,12 @@ class LoopUntilOperator(dsl_interpreter.PreComputingCallOperator, dsl_interprete NAME = "loop_until" DESCRIPTION = ( "Re-evaluates a condition after retry_interval until it is true. " + "Optional max_retry_interval adds random jitter between checks (same semantics as wait min/max). " "Optional timeout and max_attempts stop the loop with ErrorStatementEncountered; " "if both are omitted, loops until the condition is true. " "Returns the condition value when it becomes true." ) - EXAMPLE = "loop_until(x > 0, 1, timeout=30, max_attempts=10)" + EXAMPLE = "loop_until(x > 0, 1, max_retry_interval=3, timeout=30, max_attempts=10)" CATEGORY = commons_enums.DslKeywordCategory.TRIGGER.value LOOP_START_TIME_KEY = "loop_until_start_time" @@ -153,6 +154,12 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: description="seconds to wait between condition checks", required=True, type=commons_enums.DslValueType.NUMBER.value), + dsl_interpreter.OperatorParameter( + name="max_retry_interval", + description="if set and greater than retry_interval, random wait in [retry_interval, max_retry_interval)", + required=False, + type=commons_enums.DslValueType.NUMBER.value, + default=None), dsl_interpreter.OperatorParameter( name="timeout", description="if set, maximum total seconds; if still false, raises ErrorStatementEncountered", @@ -210,6 +217,23 @@ def _validate_loop_until_params(self, param_by_name: dict[str, typing.Any]) -> N raise octobot_commons.errors.InvalidParametersError( f"loop_until() requires max_attempts >= 1 when set, got {max_attempts_value}" ) + max_retry_interval = param_by_name.get("max_retry_interval") + if max_retry_interval is not None: + if max_retry_interval < 0: + raise octobot_commons.errors.InvalidParametersError( + f"loop_until() requires a non-negative max_retry_interval, got {max_retry_interval}" + ) + if max_retry_interval < retry_interval: + raise octobot_commons.errors.InvalidParametersError( + "loop_until() requires max_retry_interval >= retry_interval when set" + ) + + def _sample_retry_wait(self, param_by_name: dict[str, typing.Any]) -> float: + retry_interval = param_by_name["retry_interval"] + max_retry_interval = param_by_name.get("max_retry_interval") + if max_retry_interval is None or max_retry_interval == retry_interval: + return float(retry_interval) + return random.randrange(int(retry_interval) * 1000, int(max_retry_interval) * 1000) / 1000 def _extra_loop_state(self, last_execution_result: typing.Optional[dict]) -> dict[str, typing.Any]: if not last_execution_result: @@ -225,7 +249,10 @@ def _extra_loop_state(self, last_execution_result: typing.Optional[dict]) -> dic } def _compute_remaining_retry_wait( - self, current_time: float, last_execution_result: typing.Optional[dict] + self, + current_time: float, + last_execution_result: typing.Optional[dict], + param_by_name: dict[str, typing.Any], ) -> typing.Optional[float]: if not last_execution_result: return None @@ -235,8 +262,7 @@ def _compute_remaining_retry_wait( base_waiting_time = last_execution_result[dsl_interpreter.ReCallingOperatorResultKeys.WAITING_TIME.value] waiting_time = base_waiting_time - (current_time - last_execution_time) if waiting_time <= 0: - # reset waiting timee - return base_waiting_time + return self._sample_retry_wait(param_by_name) return waiting_time def _read_loop_start_and_attempts( @@ -267,7 +293,9 @@ async def _compute_return_remaining( timeout = param_by_name.get("timeout") try: - remaining_wait = self._compute_remaining_retry_wait(current_time, last_execution_result) + remaining_wait = self._compute_remaining_retry_wait( + current_time, last_execution_result, param_by_name + ) except KeyError: remaining_wait = None if remaining_wait is None: @@ -282,7 +310,7 @@ async def _compute_return_remaining( raise octobot_commons.errors.MaxAttemptsExceededError( "loop_until: max_attempts exceeded before condition became true" ) - remaining_wait = float(param_by_name["retry_interval"]) + remaining_wait = self._sample_retry_wait(param_by_name) else: # this is not the first execution: check exit conditions if timeout is not None and ( @@ -319,7 +347,6 @@ async def _evaluate_condition_async(self) -> typing.Any: async def _run_blocking_loop( self, param_by_name: dict[str, typing.Any] ) -> typing.Any: - retry_interval = float(param_by_name["retry_interval"]) loop_start_time = time.time() attempt_count = 0 while True: @@ -338,4 +365,4 @@ async def _run_blocking_loop( raise octobot_commons.errors.ErrorStatementEncountered( "loop_until: max_attempts exceeded before condition became true" ) - await asyncio.sleep(retry_interval) + await asyncio.sleep(self._sample_retry_wait(param_by_name)) diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py index df7d650b6b..ed0e1bccc5 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py @@ -26,8 +26,10 @@ import tentacles.Meta.DSL_operators.python_std_operators.base_call_operators as base_call_operators +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support -class RaisesInPreComputeTestOperator(dsl_interpreter.CallOperator): + +class RaisesInPreComputeTestOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator): """Fails in pre_compute only; used to assert if_error runs the on_error DSL.""" NAME = "raises_in_precompute_test_op" @@ -51,7 +53,7 @@ def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return "should_not_reach_compute" -class ContextualNestedTestOperator(dsl_interpreter.CallOperator): +class ContextualNestedTestOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator): """Contextual operator excluded from get_all_operators(); used for nested DSL tests.""" NAME = "contextual_nested_test_op" diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_resetting_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_resetting_operators.py index c34c1233a2..5b65b0c376 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_resetting_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_resetting_operators.py @@ -24,8 +24,10 @@ import octobot_commons.errors import octobot_commons.enums as commons_enums +import octobot_commons.tests.dsl_test_keyword_support as dsl_test_keyword_support -class _LoopUntilSingleEvalTestConditionOperator(dsl_interpreter.CallOperator): + +class _LoopUntilSingleEvalTestConditionOperator(dsl_test_keyword_support.TestKeywordMixin, dsl_interpreter.CallOperator): MIN_PARAMS = 0 MAX_PARAMS = 0 CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @@ -402,10 +404,128 @@ async def test_loop_until_return_remaining_max_attempts_after_wait(self): @pytest.mark.asyncio async def test_loop_until_compute_remaining_retry_wait_no_previous(self): operator = base_resetting_operators.LoopUntilOperator(False, 3, max_attempts=5) - assert operator._compute_remaining_retry_wait( - 1000.0, - None - ) is None + param_by_name = {"retry_interval": 3, "max_retry_interval": None} + assert operator._compute_remaining_retry_wait(1000.0, None, param_by_name) is None + + @pytest.mark.asyncio + async def test_loop_until_invalid_max_retry_interval(self): + operator = base_resetting_operators.LoopUntilOperator( + False, 5, max_retry_interval=3, max_attempts=10, return_remaining_time=True + ) + with pytest.raises(octobot_commons.errors.InvalidParametersError, match="max_retry_interval"): + await operator.pre_compute() + + @pytest.mark.asyncio + async def test_loop_until_jitter_first_schedule_uses_sampled_wait(self): + operator = base_resetting_operators.LoopUntilOperator( + False, + 5, + max_retry_interval=10, + max_attempts=10, + return_remaining_time=True, + ) + with mock.patch.object(base_resetting_operators.random, "randrange", return_value=6500): + with mock.patch.object(base_resetting_operators.time, "time", return_value=1000.0): + await operator.pre_compute() + last_result = operator.value[dsl_interpreter.ReCallingOperatorResult.__name__][ + "last_execution_result" + ] + assert last_result[dsl_interpreter.ReCallingOperatorResultKeys.WAITING_TIME.value] == 6.5 + + @pytest.mark.asyncio + async def test_loop_until_jitter_resamples_after_wait_elapsed(self): + wrapped_previous = { + dsl_interpreter.ReCallableOperatorMixin.LAST_EXECUTION_RESULT_KEY: { + dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + dsl_interpreter.ReCallingOperatorResultKeys.LAST_EXECUTION_TIME.value: 1000.0, + dsl_interpreter.ReCallingOperatorResultKeys.WAITING_TIME.value: 5.0, + base_resetting_operators.LoopUntilOperator.LOOP_START_TIME_KEY: 999.0, + base_resetting_operators.LoopUntilOperator.ATTEMPT_COUNT_KEY: 1, + }, + }, + }, + } + operator = base_resetting_operators.LoopUntilOperator( + False, + 5, + max_retry_interval=10, + max_attempts=10, + return_remaining_time=True, + **wrapped_previous, + ) + with mock.patch.object(base_resetting_operators.random, "randrange", return_value=7500): + with mock.patch.object(base_resetting_operators.time, "time", return_value=1006.0): + await operator.pre_compute() + last_result = operator.value[dsl_interpreter.ReCallingOperatorResult.__name__][ + "last_execution_result" + ] + assert last_result[dsl_interpreter.ReCallingOperatorResultKeys.WAITING_TIME.value] == 7.5 + + @pytest.mark.asyncio + async def test_loop_until_jitter_in_window_decrement_unchanged(self): + wrapped_previous = { + dsl_interpreter.ReCallableOperatorMixin.LAST_EXECUTION_RESULT_KEY: { + dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + dsl_interpreter.ReCallingOperatorResultKeys.LAST_EXECUTION_TIME.value: 1000.0, + dsl_interpreter.ReCallingOperatorResultKeys.WAITING_TIME.value: 5.0, + base_resetting_operators.LoopUntilOperator.LOOP_START_TIME_KEY: 999.0, + base_resetting_operators.LoopUntilOperator.ATTEMPT_COUNT_KEY: 1, + }, + }, + }, + } + operator = base_resetting_operators.LoopUntilOperator( + False, + 5, + max_retry_interval=10, + max_attempts=10, + return_remaining_time=True, + **wrapped_previous, + ) + with mock.patch.object(base_resetting_operators.random, "randrange") as mock_randrange: + with mock.patch.object(base_resetting_operators.time, "time", return_value=1002.0): + await operator.pre_compute() + mock_randrange.assert_not_called() + last_result = operator.value[dsl_interpreter.ReCallingOperatorResult.__name__][ + "last_execution_result" + ] + assert last_result[dsl_interpreter.ReCallingOperatorResultKeys.WAITING_TIME.value] == 3.0 + + def test_loop_until_sample_retry_wait_range_boundaries(self): + operator = base_resetting_operators.LoopUntilOperator(False, 1, max_retry_interval=3) + param_by_name = {"retry_interval": 1, "max_retry_interval": 3} + with mock.patch.object(base_resetting_operators.random, "randrange", return_value=1000): + assert operator._sample_retry_wait(param_by_name) == 1.0 + with mock.patch.object(base_resetting_operators.random, "randrange", return_value=2999): + assert operator._sample_retry_wait(param_by_name) == 2.999 + + def test_loop_until_re_create_script_includes_max_retry_interval(self): + operator = base_resetting_operators.LoopUntilOperator( + False, 5, max_retry_interval=7, max_attempts=10, return_remaining_time=True + ) + param_by_name = operator.get_computed_value_by_parameter() + script = operator.re_create_script(param_by_name) + assert "max_retry_interval=7" in script + + @pytest.mark.asyncio + async def test_loop_until_blocking_uses_sampled_sleep(self): + operator = base_resetting_operators.LoopUntilOperator(False, 1, max_retry_interval=3, max_attempts=3) + with mock.patch.object( + operator, + "_evaluate_condition_async", + new=mock.AsyncMock(side_effect=[False, False, True]), + ): + with mock.patch.object( + operator, + "_sample_retry_wait", + side_effect=[1.5, 2.0], + ) as mock_sample: + with mock.patch.object(asyncio, "sleep", new=mock.AsyncMock()) as mock_sleep: + await operator.pre_compute() + assert mock_sample.call_count == 2 + mock_sleep.assert_has_awaits([mock.call(1.5), mock.call(2.0)]) class TestRecallWhenRelevantForContextOperator: diff --git a/packages/tentacles/Meta/Keywords/scripting_library/backtesting/run_data_analysis.py b/packages/tentacles/Meta/Keywords/scripting_library/backtesting/run_data_analysis.py index 236ef8e85f..b71c7bf391 100644 --- a/packages/tentacles/Meta/Keywords/scripting_library/backtesting/run_data_analysis.py +++ b/packages/tentacles/Meta/Keywords/scripting_library/backtesting/run_data_analysis.py @@ -95,7 +95,8 @@ async def load_historical_values(meta_database, exchange, with_candles=True, contracts = metadata[commons_enums.DBRows.FUTURE_CONTRACTS.value][exchange] if trading_type == "future" else {} # init data for pair in run_global_metadata[commons_enums.DBRows.SYMBOLS.value]: - symbol = symbol_util.parse_symbol(pair).base + # Bare asset ticker: backtest analytics grouping — .base/.quote are network-qualified on ticker-wise pairs; use .base/.quote for portfolio[...] / reference_market. + symbol = symbol_util.parse_symbol(pair).base_asset_ticker() is_inverse_contract = trading_type == "future" and trading_api.is_inverse_future_contract( trading_enums.FutureContractType(contracts[pair]["contract_type"]) ) @@ -358,6 +359,7 @@ async def plot_historical_portfolio_value( def _read_pnl_from_trades(x_data, pnl_data, cumulative_pnl_data, trades_history, x_as_trade_count): + # Bare asset ticker: backtest analytics grouping — .base/.quote are network-qualified on ticker-wise pairs; use .base/.quote for portfolio[...] / reference_market. buy_order_volume_by_price_by_currency = { symbol_util.parse_symbol(symbol).base: {} for symbol in trades_history.keys() @@ -368,7 +370,9 @@ def _read_pnl_from_trades(x_data, pnl_data, cumulative_pnl_data, trades_history, for trades in trades_history.values(): all_trades += trades for trade in sorted(all_trades, key=lambda x: x[commons_enums.PlotAttributes.X.value]): - currency, ref_market = symbol_util.parse_symbol(trade[commons_enums.DBRows.SYMBOL.value]).base_and_quote() + parsed_trade_symbol = symbol_util.parse_symbol(trade[commons_enums.DBRows.SYMBOL.value]) + currency = parsed_trade_symbol.base + ref_market = parsed_trade_symbol.quote trade_volume = trade[commons_enums.PlotAttributes.VOLUME.value] buy_order_volume_by_price = buy_order_volume_by_price_by_currency[currency] if trade[commons_enums.PlotAttributes.SIDE.value] == trading_enums.TradeOrderSide.BUY.value: diff --git a/packages/tentacles/Meta/Keywords/scripting_library/tests/backtesting/test_run_data.py b/packages/tentacles/Meta/Keywords/scripting_library/tests/backtesting/test_run_data.py index f1bdced9fa..d6dc70f83b 100644 --- a/packages/tentacles/Meta/Keywords/scripting_library/tests/backtesting/test_run_data.py +++ b/packages/tentacles/Meta/Keywords/scripting_library/tests/backtesting/test_run_data.py @@ -69,6 +69,21 @@ async def test_get_historical_pnl(default_price_data, default_trades_data, defau "spot", default_spot_metadata) +TICKER_WISE_SYMBOL = "BTC@BTC/USDT@ETH" + + +async def test_total_paid_fees_network_qualified_base_fee_currency(): + trade = { + commons_enums.DBRows.SYMBOL.value: TICKER_WISE_SYMBOL, + commons_enums.DBRows.FEES_CURRENCY.value: "BTC@BTC", + commons_enums.DBRows.FEES_AMOUNT.value: 0.01, + commons_enums.PlotAttributes.Y.value: 50000.0, + } + with mock.patch.object(run_data_analysis, "get_transactions", mock.AsyncMock(return_value=[])): + paid_fees = await run_data_analysis.total_paid_fees(None, [trade]) + assert paid_fees == 0.01 * 50000.0 + + async def test_total_paid_fees(default_trades_data): usdt_fees = sum(trade[commons_enums.DBRows.FEES_AMOUNT.value] for trade in default_trades_data["BTC/USDT"] diff --git a/packages/tentacles/Meta/Keywords/scripting_library/tests/configuration/test_profile_data_configuration.py b/packages/tentacles/Meta/Keywords/scripting_library/tests/configuration/test_profile_data_configuration.py index 9fde09a39e..f880eed273 100644 --- a/packages/tentacles/Meta/Keywords/scripting_library/tests/configuration/test_profile_data_configuration.py +++ b/packages/tentacles/Meta/Keywords/scripting_library/tests/configuration/test_profile_data_configuration.py @@ -22,6 +22,15 @@ +def test_get_traded_coins_includes_qualified_symbol_legs(): + profile = scripting_library.minimal_profile_data() + profile.trading.reference_market = "USDT@ETH" + scripting_library.add_traded_symbols(profile, ["BTC@BTC/USDT@ETH"]) + coins = scripting_library.get_traded_coins(profile, include_stablecoins=False) + assert "USDT@ETH" in coins + assert "BTC@BTC" in coins + + def test_register_historical_configs_adds_traded_pairs(): # Master has no traded pairs, historical has one master = scripting_library.minimal_profile_data() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/auth_errors.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/auth_errors.py new file mode 100644 index 0000000000..5d3601a995 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/auth_errors.py @@ -0,0 +1,40 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import enum +import typing + +import pydantic +from fastapi import HTTPException, status + + +class AuthErrorCode(str, enum.Enum): + AUTH_PASSPHRASE_REQUIRED = "auth_passphrase_required" + AUTH_WALLET_ADDRESS_REQUIRED = "auth_wallet_address_required" + AUTH_WALLET_NOT_FOUND = "auth_wallet_not_found" + AUTH_INVALID_PASSPHRASE = "auth_invalid_passphrase" + AUTH_NODE_NOT_CONFIGURED = "auth_node_not_configured" + + +class NodeAuthErrorDetail(pydantic.BaseModel): + code: AuthErrorCode + message: typing.Optional[str] = None + + +def auth_http_exception( + status_code: int, + code: AuthErrorCode, + message: typing.Optional[str] = None, +) -> HTTPException: + return HTTPException( + status_code=status_code, + detail=NodeAuthErrorDetail(code=code, message=message).model_dump(), + ) + + +def node_not_configured_exception() -> HTTPException: + return auth_http_exception( + status.HTTP_503_SERVICE_UNAVAILABLE, + AuthErrorCode.AUTH_NODE_NOT_CONFIGURED, + message="Node wallet is not configured", + ) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/deps.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/deps.py index 6bed6996df..ecf1b59e3b 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/deps.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/deps.py @@ -17,68 +17,152 @@ import uuid import typing -from fastapi import Depends, HTTPException, status +from fastapi import Depends, HTTPException, Request, status from fastapi.security import HTTPBasic, HTTPBasicCredentials import octobot_node.models import octobot.community.authentication as community_auth import octobot.community.wallet_backend as wallet_backend +from .auth_errors import ( + AuthErrorCode, + auth_http_exception, + node_not_configured_exception, +) +try: + from tentacles.Services.Interfaces.node_api_interface.api.rate_limits.login import ( + get_login_rate_limiter, + login_client_ip, + ) + from tentacles.Services.Interfaces.node_api_interface.core.http_rate_limit import ( + run_with_failure_rate_limit, + ) +except ImportError: + from api.rate_limits.login import get_login_rate_limiter, login_client_ip # type: ignore[no-redef] + from core.http_rate_limit import run_with_failure_rate_limit # type: ignore[no-redef] + security_basic = HTTPBasic(auto_error=False) +_LOGIN_COUNTABLE_FAILURE_CODES = frozenset( + { + AuthErrorCode.AUTH_INVALID_PASSPHRASE, + AuthErrorCode.AUTH_WALLET_NOT_FOUND, + } +) + + +def _should_count_login_failure(err: HTTPException) -> bool: + if err.status_code != status.HTTP_401_UNAUTHORIZED: + return False + detail = err.detail + if not isinstance(detail, dict): + return False + raw_code = detail.get("code") + if raw_code is None: + return False + try: + auth_code = AuthErrorCode(raw_code) + except ValueError: + return False + return auth_code in _LOGIN_COUNTABLE_FAILURE_CODES + + +def _login_failure_counts_toward_limit(err: BaseException) -> bool: + if not isinstance(err, HTTPException): + return False + return _should_count_login_failure(err) + + +def _user_from_wallet_credentials( + auth: community_auth.CommunityAuthentication, + credentials: HTTPBasicCredentials, +) -> octobot_node.models.User: + wallet_address = credentials.username.lower() + passphrase = credentials.password + + try: + wallet_info = auth.authenticate_wallet(wallet_address, passphrase) + except wallet_backend.WalletNotFoundError: + raise auth_http_exception( + status.HTTP_401_UNAUTHORIZED, + AuthErrorCode.AUTH_WALLET_NOT_FOUND, + message="Wallet address is not configured on this node", + ) + except wallet_backend.InvalidPassphraseError: + raise auth_http_exception( + status.HTTP_401_UNAUTHORIZED, + AuthErrorCode.AUTH_INVALID_PASSPHRASE, + message="Passphrase verification failed", + ) + except wallet_backend.WalletError as err: + raise auth_http_exception( + status.HTTP_401_UNAUTHORIZED, + AuthErrorCode.AUTH_INVALID_PASSPHRASE, + message=str(err), + ) + + auth.init_sync_client_for_wallet(wallet_address) + + return octobot_node.models.User( + id=uuid.uuid5(uuid.NAMESPACE_URL, wallet_address), + email=wallet_address, + is_active=True, + is_superuser=wallet_info.is_admin, + full_name=wallet_info.name, + ) + def get_current_user( credentials: typing.Annotated[typing.Optional[HTTPBasicCredentials], Depends(security_basic)], ) -> octobot_node.models.User: auth = community_auth.CommunityAuthentication.instance() if auth is None: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Node not configured", - ) + raise node_not_configured_exception() # Multi-wallet path: username = wallet address, password = passphrase if credentials is None or not credentials.username: # Check whether the node is configured at all (no credentials → can't auth anyway) if not auth.list_wallets(): - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Node not configured", - ) - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Wallet address required as username", + raise node_not_configured_exception() + raise auth_http_exception( + status.HTTP_401_UNAUTHORIZED, + AuthErrorCode.AUTH_WALLET_ADDRESS_REQUIRED, + message="Wallet address required as username", ) - # Normalize to lowercase so wallet_address == task.wallet_address always - wallet_address = credentials.username.lower() passphrase = credentials.password if not passphrase: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Passphrase required", + raise auth_http_exception( + status.HTTP_401_UNAUTHORIZED, + AuthErrorCode.AUTH_PASSPHRASE_REQUIRED, + message="Passphrase required", ) - try: - wallet_info = auth.authenticate_wallet(wallet_address, passphrase) - except wallet_backend.WalletError: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Incorrect address or passphrase", - ) + return _user_from_wallet_credentials(auth, credentials) - auth.init_sync_client_for_wallet(wallet_address) - return octobot_node.models.User( - id=uuid.uuid5(uuid.NAMESPACE_URL, wallet_address), - email=wallet_address, - is_active=True, - is_superuser=wallet_info.is_admin, - full_name=wallet_info.name, +def get_login_rate_limited_user( + request: Request, + credentials: typing.Annotated[typing.Optional[HTTPBasicCredentials], Depends(security_basic)], +) -> octobot_node.models.User: + """Wallet login with in-process rate limit (see api/rate_limits/login.py).""" + client_ip = login_client_ip(request) + return run_with_failure_rate_limit( + get_login_rate_limiter(), + dimensions={"client_ip": client_ip}, + action=lambda: get_current_user(credentials), + should_record_failure=_login_failure_counts_toward_limit, ) +# Route parameter aliases: Annotated[User, Depends(fn)] gives static type User and tells +# FastAPI to run fn before the handler. A plain `user: User` annotation would not inject auth. CurrentUser = typing.Annotated[octobot_node.models.User, Depends(get_current_user)] +# Same as CurrentUser, but only for GET /login/test — counts failed passphrases per client IP. +LoginRateLimitedUser = typing.Annotated[ + octobot_node.models.User, + Depends(get_login_rate_limited_user), +] def get_optional_current_user( diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py index 292bc2239b..6a2e43ddbc 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py @@ -36,10 +36,28 @@ logs, octobots, dsl, + feedback, + journal_client_event, ) except ImportError: from api.route_provider import register_all_provider_routes # type: ignore[no-redef] - from api.routes import login, nodes, users, tasks, setup, exchanges, wallets, accounts, debug, config, logs, octobots, dsl # type: ignore[no-redef] + from api.routes import ( # type: ignore[no-redef] + login, + nodes, + users, + tasks, + setup, + exchanges, + wallets, + accounts, + debug, + config, + logs, + octobots, + dsl, + feedback, + journal_client_event, + ) def build_api_router() -> APIRouter: @@ -58,4 +76,7 @@ def build_api_router() -> APIRouter: api_router.include_router(config.router, prefix="/config") api_router.include_router(logs.router, prefix="/logs") api_router.include_router(dsl.router, prefix="/dsl") + # See feedback / journal_client_event module docstrings for unauthenticated write endpoints. + api_router.include_router(feedback.router, prefix="/feedback") + api_router.include_router(journal_client_event.router, prefix="/journal") return api_router diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/__init__.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/__init__.py new file mode 100644 index 0000000000..ad25c24157 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/__init__.py @@ -0,0 +1,15 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/login.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/login.py new file mode 100644 index 0000000000..4c1962575e --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/login.py @@ -0,0 +1,50 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +from fastapi import Request + +import octobot_commons.in_process_rate_limit as in_process_rate_limit + +try: + from tentacles.Services.Interfaces.node_api_interface.core.http_rate_limit import ( + HTTPRateLimiter, + ) +except ImportError: + from core.http_rate_limit import HTTPRateLimiter # type: ignore[no-redef] + + +_LOGIN_CLIENT_IP_POLICY = in_process_rate_limit.FailureWindowPolicy( + name="client_ip", + max_failures=10, + window_seconds=15 * 60, +) + +LOGIN_RATE_LIMITED_DETAIL = "Too many login attempts. Try again later." + +_login_rate_limiter = HTTPRateLimiter( + (_LOGIN_CLIENT_IP_POLICY,), + rate_limited_detail=LOGIN_RATE_LIMITED_DETAIL, +) + + +def get_login_rate_limiter() -> HTTPRateLimiter: + return _login_rate_limiter + + +def login_client_ip(request: Request) -> str: + if request.client is not None: + return request.client.host + return "unknown" diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/rate_limit_response.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/rate_limit_response.py new file mode 100644 index 0000000000..03b963f321 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/rate_limit_response.py @@ -0,0 +1,25 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import pydantic + + +class RateLimitedDetail(pydantic.BaseModel): + message: str + unblock_at: int = pydantic.Field( + ..., + description="Unix epoch seconds when the client may retry.", + ) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/recover_passphrase.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/recover_passphrase.py new file mode 100644 index 0000000000..ffc1609109 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/rate_limits/recover_passphrase.py @@ -0,0 +1,74 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import typing + +from fastapi import Request + +import octobot.community.wallet_backend as wallet_backend +import octobot_commons.in_process_rate_limit as in_process_rate_limit + +try: + from tentacles.Services.Interfaces.node_api_interface.core.http_rate_limit import ( + HTTPRateLimiter, + ) +except ImportError: + from core.http_rate_limit import HTTPRateLimiter # type: ignore[no-redef] + + +_RECOVER_CLIENT_IP_POLICY = in_process_rate_limit.FailureWindowPolicy( + name="client_ip", + max_failures=5, + window_seconds=15 * 60, +) +_RECOVER_ADDRESS_POLICY = in_process_rate_limit.FailureWindowPolicy( + name="address", + max_failures=10, + window_seconds=60 * 60, + normalize_key=str.lower, +) + +RECOVER_PASSPHRASE_RATE_LIMITED_DETAIL = ( + "Too many recovery attempts. Try again later." +) + +RECOVER_PASSPHRASE_FAILURE_EXCEPTIONS: tuple[type[Exception], ...] = ( + wallet_backend.WalletNotFoundError, + wallet_backend.WalletProofMismatchError, + wallet_backend.InvalidPrivateKeyError, + wallet_backend.PassphraseTooShortError, + wallet_backend.WalletError, +) + +_recover_passphrase_rate_limiter = HTTPRateLimiter( + (_RECOVER_CLIENT_IP_POLICY, _RECOVER_ADDRESS_POLICY), + rate_limited_detail=RECOVER_PASSPHRASE_RATE_LIMITED_DETAIL, +) + + +def get_recover_passphrase_rate_limiter() -> HTTPRateLimiter: + return _recover_passphrase_rate_limiter + + +def recover_passphrase_rate_dimensions( + body: typing.Any, + request: Request, +) -> dict[str, str]: + if request.client is not None: + client_ip = request.client.host + else: + client_ip = "unknown" + return {"client_ip": client_ip, "address": body.address} diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py index 06e65e2f9e..93b2033ea8 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py @@ -20,7 +20,9 @@ from fastapi import APIRouter, Body, HTTPException, Query, Response, status from fastapi.responses import JSONResponse +import octobot_node.agent_seed.demo_wallet as demo_agent_seed_wallet import octobot_node.models +import octobot_node.enums as octobot_node_enums import octobot_node.protocol.debug as debug_protocol import octobot_node.protocol.user_actions as user_actions_protocol import octobot_node.scheduler @@ -190,7 +192,21 @@ async def execute_user_action( user_action = _parse_user_action_payload(payload) resolved_user_id = await _resolve_execution_user_id(current_user, wallet_address, user_action) try: - await user_actions_protocol.execute_user_action(user_action, resolved_user_id) + demo_agent_seed_wallet.validate_demo_agent_seed_user_action( + resolved_user_id, + user_action, + ) + except demo_agent_seed_wallet.DemoAgentSeedUserActionForbiddenError as error: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=str(error), + ) from error + try: + await user_actions_protocol.execute_user_action( + user_action, + resolved_user_id, + source=octobot_node_enums.UserActionSource.DEBUG_API, + ) except RuntimeError as error: if str(error) == "Scheduler is not initialized": raise HTTPException( diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/feedback.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/feedback.py new file mode 100644 index 0000000000..4af7d0bc67 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/feedback.py @@ -0,0 +1,153 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import io +import json +import typing +import zipfile + +import fastapi +import pydantic +from fastapi import APIRouter + +import octobot.constants +import octobot.community.node_journal as node_journal + +try: + from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser +except ImportError: + from api.deps import CurrentUser # type: ignore[no-redef] + +router = APIRouter(tags=["feedback"]) + +FEEDBACK_JOURNAL_JSON_FILENAME = "node_journal.json" +FEEDBACK_JOURNAL_ZIP_FILENAME = "node_journal.zip" + + +class FeedbackUploadEnvelope(pydantic.BaseModel): + install_id: str + app_version: str + onboarding_started_at: float | None + onboarding_complete: bool + journey_summary: dict[str, typing.Any] + events: list[dict[str, typing.Any]] + uploaded: bool + ready: bool + event_count: int + note: str | None = None + ui_error_name: str | None = None + ui_error_route: str | None = None + + +class FeedbackPreviewResponse(pydantic.BaseModel): + journey_summary: dict[str, typing.Any] + upload_envelope: FeedbackUploadEnvelope + + +class FeedbackUploadRequest(pydantic.BaseModel): + issue_url: str | None = None + note: str | None = None + ui_error_name: str | None = None + ui_error_route: str | None = None + + +def _feedback_events() -> list: + if not node_journal.is_journal_enabled(): + return [] + return node_journal.read_events() + + +def _build_feedback_preview() -> FeedbackPreviewResponse: + events = _feedback_events() + journey_summary = node_journal.build_journey_summary(events) + upload_envelope = node_journal.build_upload_envelope( + events, + app_version=octobot.constants.LONG_VERSION, + ) + return FeedbackPreviewResponse( + journey_summary=journey_summary.to_dict(), + upload_envelope=FeedbackUploadEnvelope(**upload_envelope.to_dict()), + ) + + +def _build_feedback_upload_envelope( + *, + note: str | None = None, + ui_error_name: str | None = None, + ui_error_route: str | None = None, +) -> FeedbackUploadEnvelope: + events = _feedback_events() + upload_envelope = node_journal.build_upload_envelope( + events, + app_version=octobot.constants.LONG_VERSION, + note=note, + ui_error_name=ui_error_name, + ui_error_route=ui_error_route, + ) + return FeedbackUploadEnvelope(**upload_envelope.to_dict()) + + +def build_feedback_journal_zip(envelope: FeedbackUploadEnvelope) -> bytes: + json_body = json.dumps(envelope.model_dump(), indent=2, ensure_ascii=False) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr(FEEDBACK_JOURNAL_JSON_FILENAME, json_body) + return buffer.getvalue() + + +def _envelope_from_export_request( + body: FeedbackUploadRequest | None, +) -> FeedbackUploadEnvelope: + note = None + ui_error_name = None + ui_error_route = None + if body is not None: + note_parts = [] + if body.note: + note_parts.append(body.note) + if body.issue_url: + note_parts.append(f"issue_url: {body.issue_url}") + if note_parts: + note = "\n".join(note_parts) + ui_error_name = body.ui_error_name + ui_error_route = body.ui_error_route + return _build_feedback_upload_envelope( + note=note, + ui_error_name=ui_error_name, + ui_error_route=ui_error_route, + ) + + +@router.get("/preview", response_model=FeedbackPreviewResponse) +def get_feedback_preview(current_user: CurrentUser) -> FeedbackPreviewResponse: + return _build_feedback_preview() + + +@router.post("/export") +def export_feedback( + body: FeedbackUploadRequest | None = None, +) -> fastapi.Response: + envelope = _envelope_from_export_request(body) + archive = build_feedback_journal_zip(envelope) + return fastapi.Response( + content=archive, + media_type="application/zip", + headers={ + "Content-Disposition": ( + f'attachment; filename="{FEEDBACK_JOURNAL_ZIP_FILENAME}"' + ), + }, + ) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/journal_client_event.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/journal_client_event.py new file mode 100644 index 0000000000..15e3b06304 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/journal_client_event.py @@ -0,0 +1,114 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +"""Journal client-event route for the Node API. + +Security model: +- ``POST /client-event`` is intentionally unauthenticated; only allowlisted ``UI_JOURNAL_EVENTS`` are accepted. + Assumes a trusted network / same-origin UI (not a public internet-facing write API). +""" + +import json +import typing + +import pydantic +from fastapi import APIRouter, HTTPException, status + +import octobot.community.node_journal as node_journal + +router = APIRouter(tags=["journal"]) + +_MAX_CLIENT_EVENT_ATTRIBUTES_BYTES = 4096 + + +class JournalClientEventRequest(pydantic.BaseModel): + event: str + client_instance_id: str | None = None + attributes: dict[str, typing.Any] | None = None + + +class JournalClientEventResponse(pydantic.BaseModel): + event: str + timestamp: float + session_id: str + install_id: str + app_version: str + distribution: str + onboarding_complete: bool + attributes: dict[str, typing.Any] + recorded: bool = True + + +def _validate_client_event(event_name: str) -> node_journal.NodeJournalEvent: + try: + parsed_event = node_journal.NodeJournalEvent(event_name) + except ValueError as error: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown journal event: {event_name}", + ) from error + if parsed_event not in node_journal.UI_JOURNAL_EVENTS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Event not allowed: {event_name}", + ) + return parsed_event + + +def _build_client_event_attributes(body: JournalClientEventRequest) -> dict[str, typing.Any]: + attributes = dict(body.attributes or {}) + if body.client_instance_id is not None: + attributes["client_instance_id"] = body.client_instance_id + if len(json.dumps(attributes, default=str)) > _MAX_CLIENT_EVENT_ATTRIBUTES_BYTES: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Client event attributes payload is too large", + ) + return attributes + + +@router.post("/client-event", response_model=JournalClientEventResponse) +def record_journal_client_event(body: JournalClientEventRequest) -> JournalClientEventResponse: + parsed_event = _validate_client_event(body.event) + attributes = _build_client_event_attributes(body) + event_line = node_journal.record(parsed_event, attributes=attributes) + response_attributes = dict(attributes) + if event_line.attributes.raw_event_name is not None: + response_attributes["raw_event_name"] = event_line.attributes.raw_event_name + event_value = event_line.event.value + if event_line.recorded is False: + return JournalClientEventResponse( + event=event_value, + timestamp=event_line.timestamp, + session_id=event_line.session_id, + install_id=event_line.install_id, + app_version=event_line.app_version, + distribution=event_line.distribution, + onboarding_complete=event_line.onboarding_complete, + attributes=response_attributes, + recorded=False, + ) + return JournalClientEventResponse( + event=event_value, + timestamp=event_line.timestamp, + session_id=event_line.session_id, + install_id=event_line.install_id, + app_version=event_line.app_version, + distribution=event_line.distribution, + onboarding_complete=event_line.onboarding_complete, + attributes=response_attributes, + recorded=True, + ) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/login.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/login.py index 5c23d3c2d4..f900de56fc 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/login.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/login.py @@ -19,14 +19,28 @@ from fastapi import APIRouter try: - from api.deps import CurrentUser + from api.auth_errors import NodeAuthErrorDetail # type: ignore[no-redef] + from api.deps import LoginRateLimitedUser + from api.rate_limits.rate_limit_response import RateLimitedDetail except ImportError: - from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser # type: ignore[no-redef] + from tentacles.Services.Interfaces.node_api_interface.api.auth_errors import NodeAuthErrorDetail + from tentacles.Services.Interfaces.node_api_interface.api.deps import LoginRateLimitedUser # type: ignore[no-redef] + from tentacles.Services.Interfaces.node_api_interface.api.rate_limits.rate_limit_response import ( # type: ignore[no-redef] + RateLimitedDetail, + ) import octobot_node.models router = APIRouter(tags=["login"]) -@router.get("/login/test", response_model=octobot_node.models.User) -def test_auth(current_user: CurrentUser) -> typing.Any: +@router.get( + "/login/test", + response_model=octobot_node.models.User, + responses={ + 401: {"model": NodeAuthErrorDetail, "description": "Authentication failed"}, + 429: {"model": RateLimitedDetail, "description": "Too many login attempts"}, + 503: {"model": NodeAuthErrorDetail, "description": "Node not configured"}, + }, +) +def test_auth(current_user: LoginRateLimitedUser) -> typing.Any: return current_user diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py index facaa6bfd1..1fe42f5812 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py @@ -14,22 +14,49 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . +import functools import typing import pydantic -from fastapi import APIRouter, Depends, HTTPException, status +from fastapi import APIRouter, Depends, HTTPException, Request, status from fastapi.security import HTTPBasicCredentials import octobot_node.config as node_config import octobot.community.authentication as community_auth import octobot.community.wallet_backend as wallet_backend +import octobot.community.node_journal as node_journal +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.recording_context as journal_recording_context try: + from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser, security_basic + from tentacles.Services.Interfaces.node_api_interface.core import network +except ImportError: + from api.deps import CurrentUser, security_basic # type: ignore[no-redef] from core import network +try: + from tentacles.Services.Interfaces.node_api_interface.api.rate_limits.recover_passphrase import ( + RECOVER_PASSPHRASE_FAILURE_EXCEPTIONS, + get_recover_passphrase_rate_limiter, + recover_passphrase_rate_dimensions, + ) + from tentacles.Services.Interfaces.node_api_interface.core.http_rate_limit import ( + http_failure_rate_limited, + ) except ImportError: - from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser, security_basic - from tentacles.Services.Interfaces.node_api_interface.core import network + from api.rate_limits.recover_passphrase import ( # type: ignore[no-redef] + RECOVER_PASSPHRASE_FAILURE_EXCEPTIONS, + get_recover_passphrase_rate_limiter, + recover_passphrase_rate_dimensions, + ) + from core.http_rate_limit import http_failure_rate_limited # type: ignore[no-redef] +try: + from tentacles.Services.Interfaces.node_api_interface.api.rate_limits.rate_limit_response import ( + RateLimitedDetail, + ) +except ImportError: + from api.rate_limits.rate_limit_response import RateLimitedDetail # type: ignore[no-redef] router = APIRouter(tags=["setup"]) @@ -63,6 +90,23 @@ class VPNNetworkAddress(pydantic.BaseModel): vpn_network_ip: typing.Optional[str] = None +class RecoverWalletFromSeedBody(pydantic.BaseModel): + address: str + new_passphrase: str + seed: typing.Optional[str] = None + private_key: typing.Optional[str] = None + + +class RecoverWalletFromSeedResult(pydantic.BaseModel): + success: bool = True + + +def _client_ip(request: Request) -> str: + if request.client is not None: + return request.client.host + return "unknown" + + @router.get("/setup/status", response_model=SetupStatus) def get_setup_status() -> SetupStatus: auth = community_auth.CommunityAuthentication.instance() @@ -83,16 +127,27 @@ def get_vpn_network_address() -> VPNNetworkAddress: @router.post("/setup/init", response_model=SetupResult) def init_setup(body: SetupInit) -> SetupResult: auth = community_auth.CommunityAuthentication.instance() + setup_method = journal_enums.WalletSetupMethod.IMPORT if body.private_key else journal_enums.WalletSetupMethod.CREATE if auth is None: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + journal_recording_context.raise_wallet_setup_http_error( + http_status=status.HTTP_503_SERVICE_UNAVAILABLE, + failure_reason=journal_enums.WalletSetupFailureReason.SERVICE_UNAVAILABLE, + setup_method=setup_method, detail="Service not initialized", + error_message="Service not initialized", ) if auth.list_wallets(): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, + journal_recording_context.raise_wallet_setup_http_error( + http_status=status.HTTP_409_CONFLICT, + failure_reason=journal_enums.WalletSetupFailureReason.ALREADY_CONFIGURED, + setup_method=setup_method, detail="Node is already configured", + error_message="Node is already configured", ) + node_journal.record_wallet_setup_attempt( + node_type=body.node_type, + setup_method=setup_method, + ) try: if body.private_key: wallet = auth.import_wallet( @@ -108,17 +163,23 @@ def init_setup(body: SetupInit) -> SetupResult: is_admin=True, ) except (wallet_backend.WalletAlreadyExistsError, wallet_backend.AdminWalletAlreadyExistsError) as err: - # A concurrent request already configured the node — surface 409. - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, + journal_recording_context.raise_wallet_setup_http_error( + http_status=status.HTTP_409_CONFLICT, + failure_reason=journal_enums.WalletSetupFailureReason.CONCURRENT_RACE, + setup_method=setup_method, detail=str(err), - ) from err + error=err, + ) except wallet_backend.WalletError as err: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + journal_recording_context.raise_wallet_setup_http_error( + http_status=status.HTTP_422_UNPROCESSABLE_CONTENT, + failure_reason=journal_enums.WalletSetupFailureReason.WALLET_ERROR, + setup_method=setup_method, detail=str(err), - ) from err + error=err, + ) node_config.settings.IS_MASTER_MODE = body.node_type == "master" + node_journal.record_wallet_setup_succeeded() return SetupResult(address=wallet.address) @@ -145,7 +206,7 @@ def export_wallet( target_passphrase = credentials.password if is_own_wallet else passphrase if not target_passphrase: raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Passphrase required", ) try: @@ -161,3 +222,71 @@ def export_wallet( detail="Invalid passphrase", ) return WalletExport(address=entry.address, private_key=entry.private_key, seed=entry.seed or None) + + +def _recover_wallet_from_seed_http_errors( + wrapped: typing.Callable[..., RecoverWalletFromSeedResult], +) -> typing.Callable[..., RecoverWalletFromSeedResult]: + """Map wallet backend errors to HTTP responses after rate-limit accounting.""" + + @functools.wraps(wrapped) + def wrapper( + body: RecoverWalletFromSeedBody, + request: Request, + ) -> RecoverWalletFromSeedResult: + try: + return wrapped(body, request) + except wallet_backend.WalletNotFoundError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Wallet not found", + ) + except wallet_backend.WalletProofMismatchError as err: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=str(err), + ) + except wallet_backend.WalletStorageReadOnlyError as err: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=str(err), + ) + except (wallet_backend.InvalidPrivateKeyError, wallet_backend.PassphraseTooShortError, wallet_backend.WalletError) as err: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail=str(err), + ) + + return wrapper + + +@router.post( + "/setup/wallet/recover-from-seed", + response_model=RecoverWalletFromSeedResult, + responses={ + 429: {"model": RateLimitedDetail, "description": "Too many recovery attempts"}, + }, +) +@_recover_wallet_from_seed_http_errors +@http_failure_rate_limited( + get_recover_passphrase_rate_limiter(), + get_dimensions=recover_passphrase_rate_dimensions, + record_failure_on=RECOVER_PASSPHRASE_FAILURE_EXCEPTIONS, +) +def recover_wallet_from_seed_route( + body: RecoverWalletFromSeedBody, + request: Request, +) -> RecoverWalletFromSeedResult: + auth = community_auth.CommunityAuthentication.instance() + if auth is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Node not configured", + ) + auth.recover_passphrase_from_ownership_proof( + address=body.address, + new_passphrase=body.new_passphrase, + seed=body.seed, + private_key=body.private_key, + ) + return RecoverWalletFromSeedResult() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/tasks.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/tasks.py index f53718f753..ccfd31b96f 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/tasks.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/tasks.py @@ -36,8 +36,6 @@ router = APIRouter(tags=["tasks"]) logger = logging.get_logger(__name__) -_MAX_PAGE_LIMIT = 500 - @router.post("/", response_model=tuple[int, int]) async def create_tasks( @@ -83,13 +81,9 @@ async def get_tasks( page: int = 1, limit: int = 100, ) -> typing.Any: - limit = max(1, min(limit, _MAX_PAGE_LIMIT)) user_id_filter = None if current_user.is_superuser else evm_to_user_id(current_user.email) tasks_data = await octobot_node.scheduler.api.get_all_tasks(user_id=user_id_filter) - - start_idx = (page - 1) * limit - end_idx = start_idx + limit - return tasks_data[start_idx:end_idx] + return octobot_node.scheduler.api.paginate_tasks(tasks_data, page, limit) @router.put("/", response_model=octobot_node.models.Task) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/wallets.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/wallets.py index cd32c8dd4f..86a03e2d48 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/wallets.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/wallets.py @@ -53,12 +53,15 @@ def list_wallets( credentials: typing.Annotated[typing.Optional[HTTPBasicCredentials], Depends(security_basic)], ) -> list[WalletInfo]: """Return configured wallets (no auth required for login page). - Names and is_admin are only revealed to verified callers to avoid PII disclosure.""" + + Addresses and wallet names are always returned for login and recovery UX. + is_admin is hidden unless Basic auth verifies a wallet passphrase. + """ auth = community_auth.CommunityAuthentication.instance() if auth is None: return [] wallets_data = auth.list_wallets() - # Gate is_admin behind credential verification; names are labels visible on login page + # Always return name; gate is_admin only reveal_admin = ( credentials is not None and bool(credentials.username) @@ -68,7 +71,7 @@ def list_wallets( return [ WalletInfo( address=w.address, - name=w.name if reveal_admin else None, + name=w.name, is_admin=w.is_admin if reveal_admin else False, ) for w in wallets_data @@ -117,7 +120,7 @@ def create_wallet(body: CreateWalletBody, current_user: CurrentUser) -> WalletIn ) from err except wallet_backend.WalletError as err: raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(err), ) from err return WalletInfo(address=wallet.address, name=body.name or None, is_admin=False) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/core/http_rate_limit.py b/packages/tentacles/Services/Interfaces/node_api_interface/core/http_rate_limit.py new file mode 100644 index 0000000000..473636415e --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/core/http_rate_limit.py @@ -0,0 +1,120 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import functools +import inspect +import math +import time +import typing + +from fastapi import HTTPException, status + +import octobot_commons.in_process_rate_limit as in_process_rate_limit + +try: + from tentacles.Services.Interfaces.node_api_interface.api.rate_limits.rate_limit_response import ( + RateLimitedDetail, + ) +except ImportError: + from api.rate_limits.rate_limit_response import RateLimitedDetail # type: ignore[no-redef] + +FailureWindowPolicy = in_process_rate_limit.FailureWindowPolicy + +_T = typing.TypeVar("_T") + + +class HTTPRateLimiter(in_process_rate_limit.InProcessFailureRateLimiter): + """HTTP-facing failure rate limiter; raises FastAPI 429 when a budget is exceeded.""" + + def __init__( + self, + policies: tuple[FailureWindowPolicy, ...], + *, + rate_limited_detail: str = "Too many requests. Try again later.", + ) -> None: + super().__init__(policies) + self._rate_limited_detail = rate_limited_detail + + def raise_if_rate_limited( + self, + *, + detail: str | None = None, + **dimensions: str, + ) -> None: + if not self.is_rate_limited(**dimensions): + return + remaining = self.retry_after_seconds(**dimensions) + now_epoch = int(time.time()) + unblock_at = int(math.ceil(time.time() + remaining)) + if remaining > 0.0: + unblock_at = max(unblock_at, now_epoch + 1) + message = detail or self._rate_limited_detail + payload = RateLimitedDetail(message=message, unblock_at=unblock_at).model_dump() + retry_header = max(1, unblock_at - now_epoch) + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail=payload, + headers={"Retry-After": str(retry_header)}, + ) + + +def run_with_failure_rate_limit( + rate_limiter: HTTPRateLimiter, + *, + dimensions: dict[str, str], + action: typing.Callable[[], _T], + should_record_failure: typing.Callable[[BaseException], bool], +) -> _T: + """Check budgets, run action, record failure/success on the limiter's policies.""" + rate_limiter.raise_if_rate_limited(**dimensions) + try: + result = action() + except BaseException as err: + if should_record_failure(err): + rate_limiter.record_failure(**dimensions) + raise + rate_limiter.record_success(**dimensions) + return result + + +def http_failure_rate_limited( + rate_limiter: HTTPRateLimiter, + *, + get_dimensions: typing.Callable[..., dict[str, str]], + record_failure_on: tuple[type[Exception], ...], +) -> typing.Callable[[typing.Callable[..., typing.Any]], typing.Callable[..., typing.Any]]: + """Decorate a route handler; budgets live on the limiter's policies.""" + + def decorator( + wrapped: typing.Callable[..., typing.Any], + ) -> typing.Callable[..., typing.Any]: + signature = inspect.signature(wrapped) + + @functools.wraps(wrapped) + def wrapper(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + bound = signature.bind(*args, **kwargs) + bound.apply_defaults() + dimensions = get_dimensions(**bound.arguments) + return run_with_failure_rate_limit( + rate_limiter, + dimensions=dimensions, + action=lambda: wrapped(*args, **kwargs), + should_record_failure=lambda err: isinstance(err, record_failure_on), + ) + + return wrapper + + return decorator diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py b/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py index 1700dd5ade..8e5fecebdf 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py @@ -86,11 +86,31 @@ def __init__(self, config): async def _inner_start(self) -> bool: return self.threaded_start() + def run(self) -> None: + thread_name = threading.current_thread().name + self.logger.info("Node API thread: run() started (thread=%s)", thread_name) + try: + asyncio.run(self._async_run()) + except Exception as exc: + self.logger.exception( + exc, + True, + f"Node API thread: _async_run failed: {exc}", + ) + finally: + self.logger.info("Node API thread: run() exited (thread=%s)", thread_name) + async def _async_run(self) -> bool: + self.logger.info("Node API thread: _async_run started") if self.node_api_service is None: self.node_api_service = Service_bases.NodeApiService.instance() self.host = self.node_api_service.get_bind_host() self.port = self.node_api_service.get_bind_port() + self.logger.info( + "Node API thread: binding uvicorn on %s:%s", + self.host, + self.port, + ) node_sqlite_file = self.node_api_service.get_node_sqlite_file() node_postgres_url = self.node_api_service.get_node_postgres_url() if node_sqlite_file: @@ -105,6 +125,15 @@ async def _async_run(self) -> bool: host = self.host port = self.port self.app = self.create_app(external_host=self.node_api_service.get_node_external_host()) + dist_dir_for_log = get_dist_directory() + if dist_dir_for_log is not None: + dist_log_value = str(dist_dir_for_log) + else: + dist_log_value = "none" + self.logger.info( + "Node API thread: FastAPI app created (dist=%s)", + dist_log_value, + ) # Set CORS from service config cors_origins_str = self.node_api_service.get_backend_cors_origins() cors_origins = [i.strip() for i in cors_origins_str.split(",") if i.strip()] if cors_origins_str else [] @@ -125,8 +154,10 @@ async def _async_run(self) -> bool: if dist_dir and self._should_open_node_ui_in_browser(): self._open_node_ui_on_browser() try: + self.logger.info("Node API thread: entering uvicorn serve()") await self.server.serve() finally: + self.logger.info("Node API thread: uvicorn serve() exited") if self._serve_finished is not None: self._serve_finished.set() return True @@ -170,6 +201,9 @@ def _open_node_ui_on_browser(self): def create_app(cls, external_host: str | None = None) -> FastAPI: @asynccontextmanager async def lifespan(app: FastAPI): + octobot_commons_logging.get_logger("NodeApiInterface").info( + "Node API thread: FastAPI lifespan startup (uvicorn accepting requests)", + ) yield # Shutdown: trading signal channel first, then DBOS await scheduler.shutdown_scheduler_and_trading_signal_channel() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/core/test_http_rate_limit.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/core/test_http_rate_limit.py new file mode 100644 index 0000000000..b4df43343d --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/core/test_http_rate_limit.py @@ -0,0 +1,238 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import mock +import pytest +from fastapi import HTTPException, status + +import octobot_commons.in_process_rate_limit as in_process_rate_limit + +try: + import tentacles.Services.Interfaces.node_api_interface.core.http_rate_limit as http_rate_limit +except ImportError: + from core import http_rate_limit # type: ignore[no-redef] + +_DEFAULT_DETAIL = "Too many requests. Try again later." +_CLIENT_IP = "1.2.3.4" +_MONOTONIC_PATCH = "octobot_commons.in_process_rate_limit.time.monotonic" +_TIME_PATCH = "tentacles.Services.Interfaces.node_api_interface.core.http_rate_limit.time.time" + + +def _rate_limit_detail(exc: HTTPException) -> dict: + assert isinstance(exc.detail, dict) + return exc.detail + + +class CountedError(Exception): + pass + + +class OtherError(Exception): + pass + + +def _limiter(max_failures=2, *, rate_limited_detail=_DEFAULT_DETAIL): + policy = in_process_rate_limit.FailureWindowPolicy( + name="client_ip", + max_failures=max_failures, + window_seconds=60.0, + ) + return http_rate_limit.HTTPRateLimiter( + (policy,), + rate_limited_detail=rate_limited_detail, + ) + + +def _decorated_handler(limiter, inner_handler): + return http_rate_limit.http_failure_rate_limited( + limiter, + get_dimensions=lambda **dimensions: {"client_ip": dimensions["client_ip"]}, + record_failure_on=(CountedError,), + )(inner_handler) + + +class TestHTTPRateLimiterRaiseIfRateLimited: + def test_raises_429_with_default_detail(self): + limiter = _limiter(max_failures=2) + with mock.patch(_MONOTONIC_PATCH, return_value=0.0): + limiter.record_failure(client_ip=_CLIENT_IP) + limiter.record_failure(client_ip=_CLIENT_IP) + with mock.patch(_MONOTONIC_PATCH, return_value=0.0), mock.patch( + _TIME_PATCH, + return_value=1000.0, + ): + with pytest.raises(HTTPException) as exc_info: + limiter.raise_if_rate_limited(client_ip=_CLIENT_IP) + assert exc_info.value.status_code == status.HTTP_429_TOO_MANY_REQUESTS + detail = _rate_limit_detail(exc_info.value) + assert detail["message"] == _DEFAULT_DETAIL + assert detail["unblock_at"] == 1060 + assert exc_info.value.headers["Retry-After"] == "60" + + def test_raises_429_with_detail_override(self): + limiter = _limiter(max_failures=2) + with mock.patch(_MONOTONIC_PATCH, return_value=0.0): + limiter.record_failure(client_ip=_CLIENT_IP) + limiter.record_failure(client_ip=_CLIENT_IP) + with mock.patch(_MONOTONIC_PATCH, return_value=0.0), mock.patch( + _TIME_PATCH, + return_value=1000.0, + ): + with pytest.raises(HTTPException) as exc_info: + limiter.raise_if_rate_limited(client_ip=_CLIENT_IP, detail="Custom") + assert exc_info.value.status_code == status.HTTP_429_TOO_MANY_REQUESTS + detail = _rate_limit_detail(exc_info.value) + assert detail["message"] == "Custom" + assert detail["unblock_at"] == 1060 + + def test_no_op_when_not_limited(self): + limiter = _limiter(max_failures=2) + limiter.raise_if_rate_limited(client_ip=_CLIENT_IP) + + +class TestHttpFailureRateLimited: + def test_returns_wrapped_result_when_not_limited(self): + limiter = _limiter(max_failures=2) + + def inner(client_ip: str = _CLIENT_IP): + return "ok" + + handler = _decorated_handler(limiter, inner) + assert handler(client_ip=_CLIENT_IP) == "ok" + assert limiter.is_rate_limited(client_ip=_CLIENT_IP) is False + + def test_skips_wrapped_when_already_limited(self): + limiter = _limiter(max_failures=2) + limiter.record_failure(client_ip=_CLIENT_IP) + limiter.record_failure(client_ip=_CLIENT_IP) + inner_mock = mock.Mock() + + def inner(client_ip: str = _CLIENT_IP): + inner_mock() + return "ok" + + handler = _decorated_handler(limiter, inner) + with pytest.raises(HTTPException) as exc_info: + handler(client_ip=_CLIENT_IP) + assert exc_info.value.status_code == status.HTTP_429_TOO_MANY_REQUESTS + inner_mock.assert_not_called() + + def test_records_failure_on_listed_exception(self): + limiter = _limiter(max_failures=2) + + def inner(client_ip: str = _CLIENT_IP): + raise CountedError("fail") + + handler = _decorated_handler(limiter, inner) + with pytest.raises(CountedError): + handler(client_ip=_CLIENT_IP) + with pytest.raises(CountedError): + handler(client_ip=_CLIENT_IP) + with pytest.raises(HTTPException): + handler(client_ip=_CLIENT_IP) + + def test_does_not_record_failure_on_unlisted_exception(self): + limiter = _limiter(max_failures=2) + limiter.record_failure(client_ip=_CLIENT_IP) + + def inner(client_ip: str = _CLIENT_IP): + raise OtherError("fail") + + handler = _decorated_handler(limiter, inner) + with pytest.raises(OtherError): + handler(client_ip=_CLIENT_IP) + assert limiter.is_rate_limited(client_ip=_CLIENT_IP) is False + + def test_records_success_on_normal_return(self): + limiter = _limiter(max_failures=3) + limiter.record_failure(client_ip=_CLIENT_IP) + limiter.record_failure(client_ip=_CLIENT_IP) + assert limiter.is_rate_limited(client_ip=_CLIENT_IP) is False + + def inner(client_ip: str = _CLIENT_IP): + return "ok" + + handler = _decorated_handler(limiter, inner) + assert handler(client_ip=_CLIENT_IP) == "ok" + assert limiter.is_rate_limited(client_ip=_CLIENT_IP) is False + for _ in range(3): + limiter.record_failure(client_ip=_CLIENT_IP) + assert limiter.is_rate_limited(client_ip=_CLIENT_IP) is True + + def test_get_dimensions_receives_bound_defaults(self): + limiter = _limiter(max_failures=2) + captured_dimensions = {} + + def inner(client_ip: str = "9.9.9.9"): + return "ok" + + def get_dimensions(**dimensions): + captured_dimensions["client_ip"] = dimensions["client_ip"] + return {"client_ip": dimensions["client_ip"]} + + handler = http_rate_limit.http_failure_rate_limited( + limiter, + get_dimensions=get_dimensions, + record_failure_on=(CountedError,), + )(inner) + handler() + assert captured_dimensions["client_ip"] == "9.9.9.9" + + +class TestRunWithFailureRateLimit: + def test_records_failure_when_predicate_matches(self): + limiter = _limiter(max_failures=2) + + def action(): + raise CountedError("fail") + + with pytest.raises(CountedError): + http_rate_limit.run_with_failure_rate_limit( + limiter, + dimensions={"client_ip": _CLIENT_IP}, + action=action, + should_record_failure=lambda err: isinstance(err, CountedError), + ) + with pytest.raises(CountedError): + http_rate_limit.run_with_failure_rate_limit( + limiter, + dimensions={"client_ip": _CLIENT_IP}, + action=action, + should_record_failure=lambda err: isinstance(err, CountedError), + ) + with pytest.raises(HTTPException): + http_rate_limit.run_with_failure_rate_limit( + limiter, + dimensions={"client_ip": _CLIENT_IP}, + action=action, + should_record_failure=lambda err: isinstance(err, CountedError), + ) + + def test_skips_failure_recording_when_predicate_false(self): + limiter = _limiter(max_failures=2) + limiter.record_failure(client_ip=_CLIENT_IP) + + def action(): + raise OtherError("fail") + + with pytest.raises(OtherError): + http_rate_limit.run_with_failure_rate_limit( + limiter, + dimensions={"client_ip": _CLIENT_IP}, + action=action, + should_record_failure=lambda err: isinstance(err, CountedError), + ) + assert limiter.is_rate_limited(client_ip=_CLIENT_IP) is False diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/fastapi_route_test_utils.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/fastapi_route_test_utils.py new file mode 100644 index 0000000000..c6ef8041c1 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/fastapi_route_test_utils.py @@ -0,0 +1,34 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +# Keep in sync with simple_market_making_trading_mode/tests/api/fastapi_route_test_utils.py + +import typing + +import fastapi.routing +import starlette.routing + + +def collect_effective_http_paths( + routes: typing.Sequence[starlette.routing.BaseRoute], +) -> set[str]: + paths: set[str] = set() + for route_context in fastapi.routing.iter_route_contexts(routes): + if isinstance(route_context.original_route, fastapi.routing.APIRoute): + path = route_context.path + if path is not None: + paths.add(path) + return paths diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_deps.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_deps.py index 8e324747b4..492f62d1e8 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_deps.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_deps.py @@ -29,6 +29,12 @@ ) +def _auth_error_code(exc_info) -> str: + detail = exc_info.value.detail + assert isinstance(detail, dict) + return detail["code"] + + def test_not_configured_auth_none(): """auth is None → 503.""" with mock.patch( @@ -74,6 +80,7 @@ def test_multiwallet_wrong_passphrase(mock_auth): with pytest.raises(HTTPException) as exc_info: get_current_user(creds) assert exc_info.value.status_code == 401 + assert _auth_error_code(exc_info) == "auth_invalid_passphrase" def test_multiwallet_missing_username(mock_auth): @@ -81,6 +88,7 @@ def test_multiwallet_missing_username(mock_auth): with pytest.raises(HTTPException) as exc_info: get_current_user(creds) assert exc_info.value.status_code == 401 + assert _auth_error_code(exc_info) == "auth_wallet_address_required" def test_multiwallet_missing_passphrase(mock_auth): @@ -88,6 +96,7 @@ def test_multiwallet_missing_passphrase(mock_auth): with pytest.raises(HTTPException) as exc_info: get_current_user(creds) assert exc_info.value.status_code == 401 + assert _auth_error_code(exc_info) == "auth_passphrase_required" def test_multiwallet_unknown_address(mock_auth): @@ -96,6 +105,7 @@ def test_multiwallet_unknown_address(mock_auth): with pytest.raises(HTTPException) as exc_info: get_current_user(creds) assert exc_info.value.status_code == 401 + assert _auth_error_code(exc_info) == "auth_wallet_not_found" def test_multiwallet_address_case_insensitive(mock_auth): diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_login_rate_limit.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_login_rate_limit.py new file mode 100644 index 0000000000..15d1eb3e20 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_login_rate_limit.py @@ -0,0 +1,36 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +from tentacles.Services.Interfaces.node_api_interface.api.rate_limits.login import ( + get_login_rate_limiter, +) + + +def test_ip_bucket_limits_after_ten_failures(): + limiter = get_login_rate_limiter() + limiter.reset_all() + for _ in range(10): + limiter.record_failure(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is True + + +def test_success_clears_ip_bucket(): + limiter = get_login_rate_limiter() + limiter.reset_all() + for _ in range(9): + limiter.record_failure(client_ip="1.2.3.4") + limiter.record_success(client_ip="1.2.3.4") + assert limiter.is_rate_limited(client_ip="1.2.3.4") is False diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_recover_passphrase.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_recover_passphrase.py new file mode 100644 index 0000000000..9586712bfa --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_recover_passphrase.py @@ -0,0 +1,69 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import mock +import pytest +from fastapi import HTTPException + +from tentacles.Services.Interfaces.node_api_interface.api.rate_limits.recover_passphrase import ( + get_recover_passphrase_rate_limiter, +) + +_MONOTONIC_PATCH = "octobot_commons.in_process_rate_limit.time.monotonic" +_TIME_PATCH = "tentacles.Services.Interfaces.node_api_interface.core.http_rate_limit.time.time" + + +def test_ip_bucket_limits_after_five_failures(): + limiter = get_recover_passphrase_rate_limiter() + address = "0xabc" + for _ in range(5): + limiter.record_failure(client_ip="1.2.3.4", address=address) + assert limiter.is_rate_limited(client_ip="1.2.3.4", address=address) is True + + +def test_address_bucket_limits_after_ten_failures(): + limiter = get_recover_passphrase_rate_limiter() + address = "0xdef" + for index in range(10): + limiter.record_failure(client_ip=f"10.0.0.{index}", address=address) + assert limiter.is_rate_limited(client_ip="10.0.0.99", address=address) is True + + +def test_success_resets_buckets(): + limiter = get_recover_passphrase_rate_limiter() + address = "0xabc" + for _ in range(4): + limiter.record_failure(client_ip="1.2.3.4", address=address) + limiter.record_success(client_ip="1.2.3.4", address=address) + assert limiter.is_rate_limited(client_ip="1.2.3.4", address=address) is False + + +def test_raise_if_rate_limited_unblock_at_uses_address_window(): + limiter = get_recover_passphrase_rate_limiter() + limiter.reset_all() + address = "0xdef" + with mock.patch(_MONOTONIC_PATCH, return_value=0.0): + for index in range(10): + limiter.record_failure(client_ip=f"10.0.0.{index}", address=address) + with mock.patch(_MONOTONIC_PATCH, return_value=0.0), mock.patch( + _TIME_PATCH, + return_value=1000.0, + ): + with pytest.raises(HTTPException) as exc_info: + limiter.raise_if_rate_limited(client_ip="10.0.0.99", address=address) + detail = exc_info.value.detail + assert isinstance(detail, dict) + assert detail["unblock_at"] == 4600 diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_register_all_provider_routes.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_register_all_provider_routes.py index 1263fe3e73..f74255e35b 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_register_all_provider_routes.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_register_all_provider_routes.py @@ -17,17 +17,9 @@ import typing import fastapi -import fastapi.routing import tentacles.Services.Interfaces.node_api_interface.api.route_provider as route_provider - - -def _api_route_paths(api_router: fastapi.APIRouter) -> set[str]: - return { - route.path - for route in api_router.routes - if isinstance(route, fastapi.routing.APIRoute) - } +import tentacles.Services.Interfaces.node_api_interface.tests.fastapi_route_test_utils as fastapi_route_test_utils class RouteProviderTestAlpha(route_provider.RouteProvider): @@ -69,18 +61,18 @@ class TestRegisterAllProviderRoutes: def test_includes_routers_from_concrete_subclasses(self) -> None: api_router = fastapi.APIRouter() route_provider.register_all_provider_routes(api_router) - paths = _api_route_paths(api_router) + paths = fastapi_route_test_utils.collect_effective_http_paths(api_router.routes) assert "/tentacles/route-provider-test-alpha" in paths def test_skips_class_unimplemented_get_router(self) -> None: assert UnimplementedRouteProvider in route_provider.RouteProvider.__subclasses__() api_router = fastapi.APIRouter() route_provider.register_all_provider_routes(api_router) - paths = _api_route_paths(api_router) + paths = fastapi_route_test_utils.collect_effective_http_paths(api_router.routes) assert not any("unimplemented" in path for path in paths) def test_skips_concrete_class_with_abstract_in_name(self) -> None: api_router = fastapi.APIRouter() route_provider.register_all_provider_routes(api_router) - paths = _api_route_paths(api_router) + paths = fastapi_route_test_utils.collect_effective_http_paths(api_router.routes) assert "/route-provider-abstract-in-name-skip" not in paths diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_debug.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_debug.py index 2cbfd94fe3..c87823bff1 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_debug.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_debug.py @@ -16,9 +16,12 @@ import base64 import contextlib +import json import mock import pytest +import octobot_node.agent_seed.constants as demo_agent_seed_constants +import octobot_node.agent_seed.demo_wallet as demo_agent_seed_wallet import octobot_node.config import octobot_node.scheduler import octobot_protocol.models as protocol_models @@ -28,6 +31,8 @@ from .conftest import ADMIN_ADDRESS, ADMIN_PASSPHRASE, ADMIN_USER_ID, TENANT_ADDRESS, TENANT_USER_ID +_DEBUG_ROUTE_MODULE = "tentacles.Services.Interfaces.node_api_interface.api.routes.debug" + def _sample_debug_state() -> protocol_models.DebugState: return protocol_models.DebugState( @@ -201,9 +206,10 @@ def _auth_header(address: str, passphrase: str) -> dict: return {"Authorization": f"Basic {token}"} -def _assert_invalid_auth_response(response) -> None: +def _assert_auth_error_response(response, expected_code: str) -> None: assert response.status_code == 401 - assert response.json() == {"detail": "Incorrect address or passphrase"} + detail = response.json()["detail"] + assert detail["code"] == expected_code class TestGetDebug: @@ -291,14 +297,14 @@ def test_wrong_passphrase_returns_401(self, client, mock_auth): "/api/v1/debug/", headers=_auth_header(TENANT_ADDRESS, "wrong"), ) - _assert_invalid_auth_response(response) + _assert_auth_error_response(response, "auth_invalid_passphrase") def test_unknown_wallet_returns_401(self, client, mock_auth): response = client.get( "/api/v1/debug/", headers=_auth_header("0xdeadbeef", ADMIN_PASSPHRASE), ) - _assert_invalid_auth_response(response) + _assert_auth_error_response(response, "auth_wallet_not_found") def test_when_scheduler_not_initialized_returns_503(self, tenant_client, mock_auth): with mock.patch("octobot_node.scheduler.is_initialized", return_value=False): @@ -416,7 +422,7 @@ def test_wrong_passphrase_returns_401(self, client, mock_auth): json=_minimal_user_action_payload(), headers=_auth_header(TENANT_ADDRESS, "wrong"), ) - _assert_invalid_auth_response(response) + _assert_auth_error_response(response, "auth_invalid_passphrase") def test_unknown_wallet_returns_401(self, client, mock_auth): response = client.post( @@ -424,7 +430,7 @@ def test_unknown_wallet_returns_401(self, client, mock_auth): json=_minimal_user_action_payload(), headers=_auth_header("0xdeadbeef", ADMIN_PASSPHRASE), ) - _assert_invalid_auth_response(response) + _assert_auth_error_response(response, "auth_wallet_not_found") def test_when_scheduler_not_initialized_returns_503(self, tenant_client, mock_auth): with mock.patch("octobot_node.scheduler.is_initialized", return_value=False): @@ -573,3 +579,104 @@ def test_admin_without_wallet_address_uses_terminal_owner_user_id_for_restart( ) assert response.status_code == 204 assert mock_execute_user_action.await_args[0][1] == TENANT_USER_ID + + +def _exchange_config_create_user_action_payload() -> dict: + exchange_config = protocol_models.ExchangeConfig( + id="extra-exchange-config", + name="extra", + exchange="binance", + sandboxed=False, + ) + inner = protocol_models.CreateExchangeConfigConfiguration( + action_type=protocol_models.UserActionType.EXCHANGE_CONFIG_CREATE, + configuration=exchange_config, + ) + user_action = protocol_models.UserAction( + id="ua-demo-exchange-config-create", + configuration=protocol_models.UserActionConfiguration(inner), + ) + return json.loads(user_action.to_json()) + + +class TestExecuteUserActionDemoAgentSeedSandbox: + def test_validates_with_resolved_user_id_before_execute(self, tenant_client, mock_auth): + mock_execute_user_action = mock.AsyncMock(return_value=None) + real_validate = demo_agent_seed_wallet.validate_demo_agent_seed_user_action + validate_spy = mock.Mock(wraps=real_validate) + with mock.patch( + f"{_DEBUG_ROUTE_MODULE}.demo_agent_seed_wallet.validate_demo_agent_seed_user_action", + validate_spy, + ), mock.patch( + "octobot_node.protocol.user_actions.execute_user_action", + new=mock_execute_user_action, + ): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): + with _automation_owned_by_caller(): + response = tenant_client.post( + "/api/v1/debug/", + json=_stop_automation_user_action_payload(), + ) + assert response.status_code == 204 + validate_spy.assert_called_once() + validate_user_id, validate_user_action = validate_spy.call_args[0] + assert validate_user_id == TENANT_USER_ID + assert validate_user_action.id == "ua-stop-api-test" + mock_execute_user_action.assert_awaited_once() + + def test_demo_forbidden_returns_403_and_skips_execute(self, tenant_client, mock_auth): + mock_execute_user_action = mock.AsyncMock(return_value=None) + forbidden_detail = demo_agent_seed_constants.DEMO_AGENT_SEED_FORBIDDEN_ACTION_DETAIL + with mock.patch( + f"{_DEBUG_ROUTE_MODULE}.demo_agent_seed_wallet.validate_demo_agent_seed_user_action", + side_effect=demo_agent_seed_wallet.DemoAgentSeedUserActionForbiddenError( + forbidden_detail, + ), + ), mock.patch( + "octobot_node.protocol.user_actions.execute_user_action", + new=mock_execute_user_action, + ): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): + with _automation_owned_by_caller(): + response = tenant_client.post( + "/api/v1/debug/", + json=_stop_automation_user_action_payload(), + ) + assert response.status_code == 403 + assert response.json()["detail"] == forbidden_detail + mock_execute_user_action.assert_not_awaited() + + @mock.patch.object(demo_agent_seed_wallet.community_authentication.CommunityAuthentication, "instance") + def test_demo_exchange_config_create_returns_403_end_to_end( + self, + auth_mock, + tenant_client, + mock_auth, + ): + wallet_mock = mock.Mock(address=demo_agent_seed_constants.DEMO_AGENT_SEED_WALLET_EVM_ADDRESS) + auth_mock.return_value.get_wallet_by_user_id.return_value = wallet_mock + auth_mock.return_value.get_wallet_name.return_value = ( + demo_agent_seed_constants.DEMO_AGENT_SEED_WALLET_DISPLAY_NAME + ) + mock_execute_user_action = mock.AsyncMock(return_value=None) + mock_resolve_execution_user_id = mock.AsyncMock( + return_value=demo_agent_seed_constants.DEMO_AGENT_SEED_USER_ID, + ) + with mock.patch( + f"{_DEBUG_ROUTE_MODULE}._resolve_execution_user_id", + new=mock_resolve_execution_user_id, + ), mock.patch( + "octobot_node.protocol.user_actions.execute_user_action", + new=mock_execute_user_action, + ): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): + response = tenant_client.post( + "/api/v1/debug/", + json=_exchange_config_create_user_action_payload(), + ) + assert response.status_code == 403 + assert ( + response.json()["detail"] + == demo_agent_seed_constants.DEMO_AGENT_SEED_FORBIDDEN_ACTION_DETAIL + ) + mock_execute_user_action.assert_not_awaited() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_feedback.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_feedback.py new file mode 100644 index 0000000000..662297bfc9 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_feedback.py @@ -0,0 +1,41 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import io +import json +import zipfile + +try: + from tentacles.Services.Interfaces.node_api_interface.api.routes.feedback import ( + FEEDBACK_JOURNAL_JSON_FILENAME, + ) +except ImportError: + from api.routes.feedback import FEEDBACK_JOURNAL_JSON_FILENAME # type: ignore[no-redef] + + +class TestPostFeedbackExport: + def test_export_returns_zip_without_auth(self, client): + response = client.post( + "/api/v1/feedback/export", + json={"note": "http export", "ui_error_name": "boot_failed"}, + ) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("application/zip") + with zipfile.ZipFile(io.BytesIO(response.content)) as zip_file: + assert zip_file.namelist() == [FEEDBACK_JOURNAL_JSON_FILENAME] + payload = json.loads(zip_file.read(FEEDBACK_JOURNAL_JSON_FILENAME)) + assert payload["note"] == "http export" + assert payload["ui_error_name"] == "boot_failed" diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_login.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_login.py new file mode 100644 index 0000000000..f94f5dc6a3 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_login.py @@ -0,0 +1,90 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import base64 +import time + +from tentacles.Services.Interfaces.node_api_interface.api.rate_limits.login import ( + LOGIN_RATE_LIMITED_DETAIL, + get_login_rate_limiter, +) + +from .conftest import ADMIN_ADDRESS, ADMIN_PASSPHRASE + +_LOGIN_TEST_URL = "/api/v1/login/test" +_LOGIN_WINDOW_SECONDS = 15 * 60 + + +def _auth_header(address: str, passphrase: str) -> dict: + token = base64.b64encode(f"{address}:{passphrase}".encode()).decode() + return {"Authorization": f"Basic {token}"} + + +def _assert_structured_rate_limit_response( + response, + *, + expected_message: str, + max_window_seconds: int, +) -> None: + assert response.status_code == 429 + detail = response.json()["detail"] + assert isinstance(detail, dict) + assert detail["message"] == expected_message + now_epoch = int(time.time()) + assert now_epoch <= detail["unblock_at"] <= now_epoch + max_window_seconds + 5 + assert int(response.headers["Retry-After"]) >= 1 + + +def test_login_success(client, mock_auth): + get_login_rate_limiter().reset_all() + resp = client.get( + _LOGIN_TEST_URL, + headers=_auth_header(ADMIN_ADDRESS, ADMIN_PASSPHRASE), + ) + assert resp.status_code == 200 + assert resp.json()["email"] == ADMIN_ADDRESS + + +def test_login_rate_limited_after_ten_failures(client, mock_auth): + limiter = get_login_rate_limiter() + limiter.reset_all() + wrong_headers = _auth_header(ADMIN_ADDRESS, "wrong-passphrase") + for _ in range(10): + resp = client.get(_LOGIN_TEST_URL, headers=wrong_headers) + assert resp.status_code == 401 + resp = client.get(_LOGIN_TEST_URL, headers=wrong_headers) + assert resp.status_code == 429 + _assert_structured_rate_limit_response( + resp, + expected_message=LOGIN_RATE_LIMITED_DETAIL, + max_window_seconds=_LOGIN_WINDOW_SECONDS, + ) + + +def test_login_success_resets_rate_limit(client, mock_auth): + limiter = get_login_rate_limiter() + limiter.reset_all() + wrong_headers = _auth_header(ADMIN_ADDRESS, "wrong-passphrase") + for _ in range(9): + resp = client.get(_LOGIN_TEST_URL, headers=wrong_headers) + assert resp.status_code == 401 + ok_resp = client.get( + _LOGIN_TEST_URL, + headers=_auth_header(ADMIN_ADDRESS, ADMIN_PASSPHRASE), + ) + assert ok_resp.status_code == 200 + resp = client.get(_LOGIN_TEST_URL, headers=wrong_headers) + assert resp.status_code == 401 diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_logs.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_logs.py index 4f9a92d370..fa83de77c6 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_logs.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_logs.py @@ -38,9 +38,10 @@ def _auth_header(address: str, passphrase: str) -> dict: return {"Authorization": f"Basic {token}"} -def _assert_invalid_auth_response(response) -> None: +def _assert_auth_error_response(response, expected_code: str) -> None: assert response.status_code == 401 - assert response.json() == {"detail": "Incorrect address or passphrase"} + detail = response.json()["detail"] + assert detail["code"] == expected_code class TestBuildLogsZip: @@ -141,7 +142,7 @@ def test_wrong_passphrase_returns_401(self, client, mock_auth): json={"task_ids": ["task-a"]}, headers=_auth_header(TENANT_ADDRESS, "wrong"), ) - _assert_invalid_auth_response(response) + _assert_auth_error_response(response, "auth_invalid_passphrase") def test_unknown_wallet_returns_401(self, client, mock_auth): response = client.post( @@ -149,7 +150,7 @@ def test_unknown_wallet_returns_401(self, client, mock_auth): json={"task_ids": ["task-a"]}, headers=_auth_header("0xdeadbeef", ADMIN_PASSPHRASE), ) - _assert_invalid_auth_response(response) + _assert_auth_error_response(response, "auth_wallet_not_found") class TestExportLogs: diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_recover_wallet.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_recover_wallet.py new file mode 100644 index 0000000000..93d9fdc3f9 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_recover_wallet.py @@ -0,0 +1,122 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +from unittest import mock + +import time + +import octobot.community.wallet_backend as wallet_backend + +from tentacles.Services.Interfaces.node_api_interface.api.rate_limits.recover_passphrase import ( + RECOVER_PASSPHRASE_RATE_LIMITED_DETAIL, + get_recover_passphrase_rate_limiter, +) + +from .conftest import ADMIN_ADDRESS + +_TEST_MNEMONIC = "test test test test test test test test test test test junk" +_RECOVER_URL = "/api/v1/setup/wallet/recover-from-seed" +_RECOVER_IP_WINDOW_SECONDS = 15 * 60 + + +def _assert_structured_rate_limit_response( + response, + *, + expected_message: str, + max_window_seconds: int, +) -> None: + assert response.status_code == 429 + detail = response.json()["detail"] + assert isinstance(detail, dict) + assert detail["message"] == expected_message + now_epoch = int(time.time()) + assert now_epoch <= detail["unblock_at"] <= now_epoch + max_window_seconds + 5 + assert int(response.headers["Retry-After"]) >= 1 + + +def _recover_body(**overrides): + body = { + "address": ADMIN_ADDRESS, + "new_passphrase": "new-passphrase99", + "seed": _TEST_MNEMONIC, + } + body.update(overrides) + return body + + +def test_recover_wallet_success(client): + auth = mock.MagicMock() + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ): + get_recover_passphrase_rate_limiter().reset_all() + resp = client.post(_RECOVER_URL, json=_recover_body()) + assert resp.status_code == 200 + assert resp.json() == {"success": True} + auth.recover_passphrase_from_ownership_proof.assert_called_once() + + +def test_recover_wallet_mismatch_returns_401(client): + auth = mock.MagicMock() + auth.recover_passphrase_from_ownership_proof.side_effect = ( + wallet_backend.WalletProofMismatchError("mismatch") + ) + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ): + get_recover_passphrase_rate_limiter().reset_all() + resp = client.post(_RECOVER_URL, json=_recover_body()) + assert resp.status_code == 401 + + +def test_recover_wallet_read_only_returns_503(client): + auth = mock.MagicMock() + auth.recover_passphrase_from_ownership_proof.side_effect = ( + wallet_backend.WalletStorageReadOnlyError("read-only") + ) + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ): + get_recover_passphrase_rate_limiter().reset_all() + resp = client.post(_RECOVER_URL, json=_recover_body()) + assert resp.status_code == 503 + assert "read-only" in resp.json()["detail"].lower() + + +def test_recover_wallet_rate_limited_after_ip_failures(client): + auth = mock.MagicMock() + auth.recover_passphrase_from_ownership_proof.side_effect = ( + wallet_backend.WalletProofMismatchError("mismatch") + ) + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ): + limiter = get_recover_passphrase_rate_limiter() + limiter.reset_all() + for _ in range(5): + resp = client.post(_RECOVER_URL, json=_recover_body()) + assert resp.status_code == 401 + resp = client.post(_RECOVER_URL, json=_recover_body()) + assert resp.status_code == 429 + _assert_structured_rate_limit_response( + resp, + expected_message=RECOVER_PASSPHRASE_RATE_LIMITED_DETAIL, + max_window_seconds=_RECOVER_IP_WINDOW_SECONDS, + ) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py index 5952f8f90c..bd97f249cb 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py @@ -20,6 +20,11 @@ from .conftest import ADMIN_ADDRESS, ADMIN_PASSPHRASE +_WALLET_SETUP_FAILED_PATCH = ( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup." + "journal_recording_context.journal_recording.record_wallet_setup_failed" +) + _INIT_BODY = { "passphrase": "strongpass123", "node_type": "standalone", @@ -58,6 +63,8 @@ def test_setup_init_success(client): with mock.patch( "octobot.community.authentication.CommunityAuthentication.instance", return_value=auth, + ), mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup.node_journal.record_wallet_setup_succeeded", ): with mock.patch("octobot_node.config.settings"): resp = client.post("/api/v1/setup/init", json=_INIT_BODY) @@ -68,6 +75,37 @@ def test_setup_init_success(client): ) +def test_setup_init_emits_wallet_setup_succeeded(client): + auth = mock.MagicMock() + auth.list_wallets.return_value = [] + auth.create_wallet.return_value = mock.MagicMock(address=ADMIN_ADDRESS) + metrics_config = mock.MagicMock() + auth.config = metrics_config + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ), mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup.node_journal.record_wallet_setup_succeeded", + ) as record_wallet_setup_succeeded_mock, mock.patch("octobot_node.config.settings"): + resp = client.post("/api/v1/setup/init", json=_INIT_BODY) + assert resp.status_code == 200 + record_wallet_setup_succeeded_mock.assert_called_once_with() + + +def test_setup_init_skips_wallet_milestone_on_409(client): + auth = mock.MagicMock() + auth.list_wallets.return_value = [{"address": ADMIN_ADDRESS, "is_admin": True}] + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ), mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup.node_journal.record_wallet_setup_succeeded", + ) as record_wallet_setup_succeeded_mock: + resp = client.post("/api/v1/setup/init", json=_INIT_BODY) + assert resp.status_code == 409 + record_wallet_setup_succeeded_mock.assert_not_called() + + def test_setup_init_with_private_key(client): pk = "a" * 64 auth = mock.MagicMock() @@ -76,6 +114,8 @@ def test_setup_init_with_private_key(client): with mock.patch( "octobot.community.authentication.CommunityAuthentication.instance", return_value=auth, + ), mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup.node_journal.record_wallet_setup_succeeded", ): with mock.patch("octobot_node.config.settings"): resp = client.post( @@ -99,6 +139,36 @@ def test_setup_init_already_configured_returns_409(client): assert resp.status_code == 409 +def test_setup_init_records_wallet_setup_failed_on_409(client): + auth = mock.MagicMock() + auth.list_wallets.return_value = [{"address": ADMIN_ADDRESS, "is_admin": True}] + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ), mock.patch( + _WALLET_SETUP_FAILED_PATCH, + ) as record_wallet_setup_failed_mock: + resp = client.post("/api/v1/setup/init", json=_INIT_BODY) + assert resp.status_code == 409 + record_wallet_setup_failed_mock.assert_called_once() + assert record_wallet_setup_failed_mock.call_args.kwargs["http_status"] == 409 + assert record_wallet_setup_failed_mock.call_args.kwargs["failure_reason"] == "already_configured" + + +def test_setup_init_records_wallet_setup_failed_on_503(client): + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=None, + ), mock.patch( + _WALLET_SETUP_FAILED_PATCH, + ) as record_wallet_setup_failed_mock: + resp = client.post("/api/v1/setup/init", json=_INIT_BODY) + assert resp.status_code == 503 + record_wallet_setup_failed_mock.assert_called_once() + assert record_wallet_setup_failed_mock.call_args.kwargs["http_status"] == 503 + assert record_wallet_setup_failed_mock.call_args.kwargs["failure_reason"] == "service_unavailable" + + def test_setup_init_invalid_passphrase_returns_422(client): auth = mock.MagicMock() auth.list_wallets.return_value = [] @@ -115,6 +185,26 @@ def test_setup_init_invalid_passphrase_returns_422(client): assert resp.status_code == 422 +def test_setup_init_records_wallet_setup_failed_on_422(client): + auth = mock.MagicMock() + auth.list_wallets.return_value = [] + auth.create_wallet.side_effect = wallet_backend.WalletError("Passphrase must be at least 8 characters") + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ), mock.patch( + _WALLET_SETUP_FAILED_PATCH, + ) as record_wallet_setup_failed_mock, mock.patch("octobot_node.config.settings"): + resp = client.post( + "/api/v1/setup/init", + json={**_INIT_BODY, "passphrase": "short"}, + ) + assert resp.status_code == 422 + record_wallet_setup_failed_mock.assert_called_once() + assert record_wallet_setup_failed_mock.call_args.kwargs["http_status"] == 422 + assert record_wallet_setup_failed_mock.call_args.kwargs["failure_reason"] == "wallet_error" + + def test_wallet_export_success(admin_client, mock_auth): mock_auth.decrypt_wallet_entry_by_address.return_value = mock.MagicMock( address=ADMIN_ADDRESS, diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_tasks.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_tasks.py index b3c8c66412..d05f7740a4 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_tasks.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_tasks.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . +import datetime from unittest.mock import AsyncMock, patch import octobot_node.models @@ -144,3 +145,71 @@ def test_post_tasks_requires_auth(client, mock_auth): def test_delete_tasks_requires_auth(client, mock_auth): resp = client.delete(f"/api/v1/tasks/?taskIds={TENANT_TASK_ID}") assert resp.status_code == 401 + + +def _task_with_completed_at(task_id: str, completed_at: datetime.datetime) -> octobot_node.models.Task: + return octobot_node.models.Task( + id=task_id, + executions=[ + octobot_node.models.Execution( + id=task_id, + status=octobot_node.models.TaskStatus.COMPLETED, + completed_at=completed_at, + ) + ], + ) + + +def test_get_tasks_sorts_newest_first_when_newest_is_last_in_storage_order(admin_client, mock_auth): + oldest_id = "11111111-1111-1111-1111-111111111111" + middle_id = "22222222-2222-2222-2222-222222222222" + newest_id = "33333333-3333-3333-3333-333333333333" + unsorted_tasks = [ + _task_with_completed_at( + oldest_id, + datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), + ), + _task_with_completed_at( + middle_id, + datetime.datetime(2024, 2, 1, tzinfo=datetime.timezone.utc), + ), + _task_with_completed_at( + newest_id, + datetime.datetime(2024, 3, 1, tzinfo=datetime.timezone.utc), + ), + ] + mock_get = AsyncMock(return_value=unsorted_tasks) + with patch("octobot_node.scheduler.api.get_all_tasks", new=mock_get): + resp = admin_client.get("/api/v1/tasks/?page=1&limit=2") + assert resp.status_code == 200 + data = resp.json() + assert len(data) == 2 + assert data[0]["id"] == newest_id + + +def test_get_tasks_page_two_has_no_overlap_with_page_one(admin_client, mock_auth): + task_ids = [ + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + "cccccccc-cccc-cccc-cccc-cccccccccccc", + "dddddddd-dddd-dddd-dddd-dddddddddddd", + ] + unsorted_tasks = [ + _task_with_completed_at( + task_id, + datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc) + + datetime.timedelta(days=index), + ) + for index, task_id in enumerate(task_ids) + ] + mock_get = AsyncMock(return_value=unsorted_tasks) + with patch("octobot_node.scheduler.api.get_all_tasks", new=mock_get): + page_one = admin_client.get("/api/v1/tasks/?page=1&limit=2") + page_two = admin_client.get("/api/v1/tasks/?page=2&limit=2") + assert page_one.status_code == 200 + assert page_two.status_code == 200 + page_one_ids = {row["id"] for row in page_one.json()} + page_two_ids = {row["id"] for row in page_two.json()} + assert page_one_ids.isdisjoint(page_two_ids) + assert len(page_one_ids) == 2 + assert len(page_two_ids) == 2 diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_wallets.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_wallets.py index 38d35b711d..fe6ac8c405 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_wallets.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_wallets.py @@ -36,6 +36,10 @@ def test_list_wallets_unauthenticated(client, mock_auth): # Admin flags must not be revealed without credentials for w in data: assert w["is_admin"] is False + admin_entry = next(w for w in data if w["address"] == ADMIN_ADDRESS) + tenant_entry = next(w for w in data if w["address"] == TENANT_ADDRESS) + assert admin_entry["name"] == "Admin" + assert tenant_entry["name"] == "Alice" def test_list_wallets_authenticated_reveals_admin_flags(client, mock_auth): @@ -63,12 +67,11 @@ def test_list_wallets_bad_credentials_hides_admin_flags(client, mock_auth): def test_list_wallets_unknown_address_hides_admin_flags(client, mock_auth): - """Credentials for an address not in the wallet list must not reveal names/is_admin.""" + """Credentials for an address not in the wallet list must not reveal is_admin.""" resp = client.get("/api/v1/wallets/", auth=("0xunknown000000000000000000000000000001", "anypass")) assert resp.status_code == 200 for w in resp.json(): assert w["is_admin"] is False - assert w["name"] is None def test_list_wallets_no_auth_service(client): diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/AGENTS.md b/packages/tentacles/Services/Interfaces/node_web_interface/AGENTS.md new file mode 100644 index 0000000000..1f72a2e219 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/AGENTS.md @@ -0,0 +1,38 @@ +# Agents: node_web_interface + +## Role + +Node product **React** web UI (`octobot-node`): Vite, TanStack Router/Query, Tailwind. Built bundle in `dist/` ships with the tentacle. + +## End-user UX (mandatory) + +Read skill **end-user-ui** (`.cursor/skills/end-user-ui/SKILL.md`) before any user-visible copy, layout, error, or help change. + +- Entry-level trading terms OK; no pro-trader jargon. +- Minimal UI; one primary goal per screen. +- No AI slop; no em dash (`—`) in user-visible strings. + +## Owns + +- `src/`, tests, `package.json` under **this** `packages/tentacles/.../node_web_interface/` path +- Not `src/client/*` (generated via `npm run generate-client`) +- Not repo-root `tentacles/` (install output) + +## Do not + +- Edit repo-root `tentacles/`; after source changes run `bash .cursor/reinstall-tentacles.sh` +- Hand-edit `openapi.json` or generated client files (see [`../../../AGENTS.md`](../../../AGENTS.md#node-rest-openapi-node_api_interface--node_web_interface)) +- Add Playwright e2e under `e2e/` (`path.deny_node_web_playwright_e2e`) + +## Tests + +- `npm test` (Vitest) from this directory; see [README.md](README.md) + +## Related + +- Tentacles index: [`packages/tentacles/AGENTS.md`](../../../AGENTS.md) +- Node REST OpenAPI: `node_api_interface` tentacle (same parent `Services/Interfaces/`) + +## Last reviewed + +- 2026-09-19 diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/README.md b/packages/tentacles/Services/Interfaces/node_web_interface/README.md index dcad1a6d95..4850afc809 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/README.md +++ b/packages/tentacles/Services/Interfaces/node_web_interface/README.md @@ -3,6 +3,11 @@ React web UI for the OctoBot node API, packaged as a tentacle. Built with Vite, TanStack Router/Query, and Tailwind. The compiled bundle in `dist/` is what ships in the tentacle package. +**Agents:** User-visible copy and layout follow [AGENTS.md](AGENTS.md) and skill **end-user-ui** +([`.cursor/skills/end-user-ui/SKILL.md`](../../../../../.cursor/skills/end-user-ui/SKILL.md)) +under the OctoBot repo root. Edit sources here in `packages/tentacles/.../node_web_interface/` only; +never repo-root `tentacles/`. + ## Requirements Node 24 (see `.nvmrc`). Run `nvm use` before any npm command. @@ -14,9 +19,18 @@ npm ci # install from the committed package-lock.json npm run generate-client # regenerate the typed API client from openapi.json npm run dev # Vite dev server npm run build # tsc + vite build -> dist/ -npm test +npm test # Vitest — this is the CI test gate for this package ``` +Do not add Playwright e2e under `e2e/` in this repo (agent policy: Vitest + Python/API tests; live UI QA via [`tools/agent_seed/README.md`](../../../../../tools/agent_seed/README.md)). + +### Wallet generate onboarding (seed reveal + quiz) + +Creating a wallet via **Generate wallet** on `/setup/` or **Generate wallet** in Settings → Add wallet +runs a client-only onboarding flow: full BIP39 seed reveal (copy + offline checkbox), then a +three-word confirmation quiz. Hex private-key import skips this flow. The UI loads the phrase from +the existing wallet export API after create; there is no server-side “saved my seed” flag. + `openapi.json` is generated by the `node_api_interface` tentacle (`python build_openapi.py`) and is git-ignored. Generate it once before `npm run generate-client` if it is missing. diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/package-lock.json b/packages/tentacles/Services/Interfaces/node_web_interface/package-lock.json index 89a833aa72..059c28a0e8 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/package-lock.json +++ b/packages/tentacles/Services/Interfaces/node_web_interface/package-lock.json @@ -11,59 +11,58 @@ "@drakkar.software/octobot-client": "^0.8.1", "@drakkar.software/octochat-sdk": "^0.8.1", "@drakkar.software/starfish-identities": "3.0.0-alpha.72", - "@hookform/resolvers": "^5.2.2", - "@noble/curves": "2.2.0", - "@radix-ui/react-avatar": "^1.1.11", - "@radix-ui/react-checkbox": "^1.3.3", - "@radix-ui/react-dialog": "^1.1.15", - "@radix-ui/react-dropdown-menu": "^2.1.16", - "@radix-ui/react-label": "^2.1.8", - "@radix-ui/react-radio-group": "^1.3.8", - "@radix-ui/react-scroll-area": "^1.2.10", - "@radix-ui/react-select": "^2.2.6", - "@radix-ui/react-separator": "^1.1.8", - "@radix-ui/react-slot": "^1.2.4", - "@radix-ui/react-tabs": "^1.1.13", - "@radix-ui/react-tooltip": "^1.2.8", - "@tailwindcss/vite": "^4.2.4", - "@tanstack/react-query": "^5.100.6", - "@tanstack/react-query-devtools": "^5.100.6", - "@tanstack/react-router": "^1.168.26", - "@tanstack/react-router-devtools": "^1.166.13", + "@hookform/resolvers": "^5.9.1", + "@noble/curves": "2.4.0", + "@radix-ui/react-avatar": "^1.2.6", + "@radix-ui/react-checkbox": "^1.3.11", + "@radix-ui/react-dialog": "^1.1.23", + "@radix-ui/react-dropdown-menu": "^2.1.24", + "@radix-ui/react-label": "^2.1.15", + "@radix-ui/react-radio-group": "^1.4.7", + "@radix-ui/react-scroll-area": "^1.2.18", + "@radix-ui/react-select": "^2.3.7", + "@radix-ui/react-separator": "^1.1.15", + "@radix-ui/react-slot": "^1.3.3", + "@radix-ui/react-tabs": "^1.1.21", + "@radix-ui/react-tooltip": "^1.2.16", + "@tailwindcss/vite": "^4.3.3", + "@tanstack/react-query": "^5.103.2", + "@tanstack/react-query-devtools": "^5.103.2", + "@tanstack/react-router": "^1.170.38", + "@tanstack/react-router-devtools": "^1.167.2", "@tanstack/react-table": "^8.21.3", - "axios": "1.15.2", + "axios": "1.20.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "form-data": "4.0.5", - "lucide-react": "^1.14.0", + "form-data": "4.0.6", + "lucide-react": "^1.47.0", "next-themes": "^0.4.6", - "react": "^19.2.5", - "react-dom": "^19.2.5", - "react-error-boundary": "^6.1.1", - "react-hook-form": "^7.74.0", - "react-icons": "^5.6.0", - "react-qr-code": "^2.0.21", - "sonner": "^2.0.7", - "tailwind-merge": "^3.5.0", - "tailwindcss": "^4.2.4", - "zod": "^4.4.1" + "react": "^19.3.0", + "react-dom": "^19.3.0", + "react-error-boundary": "^6.1.6", + "react-hook-form": "^7.88.0", + "react-icons": "^5.7.0", + "react-qr-code": "^2.2.0", + "sonner": "^2.0.8", + "tailwind-merge": "^3.7.0", + "tailwindcss": "^4.3.3", + "zod": "^4.6.5" }, "devDependencies": { - "@biomejs/biome": "^2.4.13", - "@hey-api/openapi-ts": "0.73.0", - "@playwright/test": "1.59.1", - "@tanstack/router-devtools": "^1.166.13", - "@tanstack/router-plugin": "^1.167.29", - "@types/node": "^25.6.0", - "@types/react": "^19.2.14", - "@types/react-dom": "^19.2.3", - "@vitejs/plugin-react-swc": "^4.3.0", + "@biomejs/biome": "^2.5.14", + "@hey-api/openapi-ts": "0.85.0", + "@playwright/test": "1.63.0", + "@tanstack/router-plugin": "^1.168.40", + "@types/node": "^25.9.8", + "@types/react": "^19.3.0", + "@types/react-dom": "^19.3.0", + "@vitejs/plugin-react-swc": "^4.3.3", "dotenv": "^17.4.2", "fake-indexeddb": "^6.2.5", "tw-animate-css": "^1.4.0", "typescript": "^5.9.3", - "vite": "^8.0.10", - "vitest": "^4.1.5" + "vite": "^8.3.0", + "vitest": "^4.1.11" } }, "node_modules/@babel/code-frame": { @@ -122,6 +121,16 @@ "url": "https://opencollective.com/babel" } }, + "node_modules/@babel/core/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/@babel/generator": { "version": "7.29.8", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", @@ -156,6 +165,16 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/helper-compilation-targets/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/@babel/helper-globals": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", @@ -243,9 +262,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.29.8", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", - "integrity": "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==", + "version": "7.29.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.9.tgz", + "integrity": "sha512-CjXrNHTnvqBVqHgdBysY3vk2T8tpJHb5/RMeHJBTyVa9xgugCB0CJTx/3oO8RV2QRQP391RWpB7D6hLjm8V9uA==", "dev": true, "license": "MIT", "dependencies": { @@ -307,9 +326,9 @@ } }, "node_modules/@biomejs/biome": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.12.tgz", - "integrity": "sha512-Lw4VHZRebrReBBnlHa12JQjnIBm3JJAA55PDB9LbBVBF0q4RYphm6KfmIjqtPhf61MxZ5Q9KoK8R8x+7per5Aw==", + "version": "2.5.14", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.14.tgz", + "integrity": "sha512-0FabLIjd4M/dm8VFI86RMaLLdgepzbdfiAL2R8cr7V81OYYrP1w7Z73KfAwPK5h9SrEBXTzNG2y+mBwPo9xRnw==", "dev": true, "license": "MIT OR Apache-2.0", "bin": { @@ -323,20 +342,20 @@ "url": "https://opencollective.com/biome" }, "optionalDependencies": { - "@biomejs/cli-darwin-arm64": "2.5.12", - "@biomejs/cli-darwin-x64": "2.5.12", - "@biomejs/cli-linux-arm64": "2.5.12", - "@biomejs/cli-linux-arm64-musl": "2.5.12", - "@biomejs/cli-linux-x64": "2.5.12", - "@biomejs/cli-linux-x64-musl": "2.5.12", - "@biomejs/cli-win32-arm64": "2.5.12", - "@biomejs/cli-win32-x64": "2.5.12" + "@biomejs/cli-darwin-arm64": "2.5.14", + "@biomejs/cli-darwin-x64": "2.5.14", + "@biomejs/cli-linux-arm64": "2.5.14", + "@biomejs/cli-linux-arm64-musl": "2.5.14", + "@biomejs/cli-linux-x64": "2.5.14", + "@biomejs/cli-linux-x64-musl": "2.5.14", + "@biomejs/cli-win32-arm64": "2.5.14", + "@biomejs/cli-win32-x64": "2.5.14" } }, "node_modules/@biomejs/cli-darwin-arm64": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.12.tgz", - "integrity": "sha512-lCRY1rwgNeWNgTr4DI/u6ZwXTRwRLHAvbaio1YLLGS+4r1nhvB2ssyPqIpfUSmRveNfv0fn/N58C7CAdK2XVrg==", + "version": "2.5.14", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.14.tgz", + "integrity": "sha512-UnzaXO65L4tsZimFITFP2M121GyhDcWFrT3pL5ZJ5U4XcS/0L5VHYytVohdCf/gnDUFHgl2I9xnt5bV/J1kxHQ==", "cpu": [ "arm64" ], @@ -351,9 +370,9 @@ } }, "node_modules/@biomejs/cli-darwin-x64": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.12.tgz", - "integrity": "sha512-vhPgwnh+6tN3ArdAXuET99xaNbFt7CG82Bqn+omHVLC5xdVx45JsYjGPmUIGNzjDek5XdNCP1HKksK7fn8+3bQ==", + "version": "2.5.14", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.14.tgz", + "integrity": "sha512-kiy8qA16K93J7uvFfWi4LgjqDpnRKyePAna6A0Y4jxyUga35SYpIZ2cNWlhy0lsfgqRenCpfymnJWEZ6mgpRgA==", "cpu": [ "x64" ], @@ -368,13 +387,16 @@ } }, "node_modules/@biomejs/cli-linux-arm64": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.12.tgz", - "integrity": "sha512-2gp8aVwXYKdAtmBfRFCUuyDMcfN1ahHqUkGfLYrZlNRFmryMATLVvJgWKvyA8wu4Rwn5OSxM1UcUmOuOFNGeBQ==", + "version": "2.5.14", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.14.tgz", + "integrity": "sha512-vO/9BaU1n30CiFNLx49gMTMtbCAAqlo/EqFoG0BU3deQInTbJrmXSmTQ9e3FoDZIKQnvSLDVNL4lx1ci7y1T1Q==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -385,13 +407,16 @@ } }, "node_modules/@biomejs/cli-linux-arm64-musl": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.12.tgz", - "integrity": "sha512-couHYjFLL5uuI8ne6zhT7KwEsXo5YP7ry/2xmEqah7qanu0YmfDi3mwJg47YXSuv/NpZj22CZzcRH/5c4gjPSQ==", + "version": "2.5.14", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.14.tgz", + "integrity": "sha512-SJ9PrZkBnnH9dHJDnxk34vKs0GB2dbsioaft6/hPhWJ7AHpwYH83Isnhj0FSRpFkGgpVfJ1lds23ApB2czUDLQ==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -402,13 +427,16 @@ } }, "node_modules/@biomejs/cli-linux-x64": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.12.tgz", - "integrity": "sha512-SnvOs3TSTiuia4SQOUNe1aWC9RT4+YkjcKnOhL/nsKOV0k5ycgBkDzF0lUxKn1V7Q8CLTRq6iV23ZAivHomRoA==", + "version": "2.5.14", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.14.tgz", + "integrity": "sha512-VHZRa7CCQBUWKxNwUvHJeuW03WFRgN5NWO//SDGOitc9QIeNyfA42V9zlKm+x82xFSG9Bzi5fi+hbhm9anO8yA==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -419,13 +447,16 @@ } }, "node_modules/@biomejs/cli-linux-x64-musl": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.12.tgz", - "integrity": "sha512-8A0oDW58/w9f/PQNYuq0sGUZtGtGrkNF4Z6n0PUoXpLCshi85vtKTv1XSznQawhdE4MXJ8ufpzHXyLFe87M/+w==", + "version": "2.5.14", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.14.tgz", + "integrity": "sha512-2kI5PrMgW5dcEZYrstLPmUmCwkUwZY39rP4BN93Vxbwcs2O57LDQOktUZZYPvvspN5i0mhGr3TJtF5sU26NUWg==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -436,9 +467,9 @@ } }, "node_modules/@biomejs/cli-win32-arm64": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.12.tgz", - "integrity": "sha512-b9vtoZFsuZt1pdjNwJvXl0f+BpayRzV008uS2+JpmwIKdSE2qdu4A/l04FESwLoou5g2E/Qlec0xwJydZplH+A==", + "version": "2.5.14", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.14.tgz", + "integrity": "sha512-pHgAFffmZtaYoxavEsWEYNvcA7NwOIjLyqw2HXyLbt16xYryX0L4fMV2u0G9ag6LNYPIoqWaWqzUcnc6rLGGXg==", "cpu": [ "arm64" ], @@ -453,9 +484,9 @@ } }, "node_modules/@biomejs/cli-win32-x64": { - "version": "2.5.12", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.12.tgz", - "integrity": "sha512-B1R/l+CwEpKFSuqiwePzPNRk1EiJN8kc0UhdafNz6MZN9v5OFP9HYP1irptvWzHrwVI4blVNGMbxc5zt70m3IA==", + "version": "2.5.14", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.14.tgz", + "integrity": "sha512-oJWmBhoHsnhUKIke+0gXDX0mltJrWHA1UyHsrTlXwX0TL64ilVZAo+TYZm97baecV0esBdpzy3k96q+09JaZUQ==", "cpu": [ "x64" ], @@ -469,6 +500,18 @@ "node": ">=14.21.3" } }, + "node_modules/@corvu/utils": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@corvu/utils/-/utils-0.4.2.tgz", + "integrity": "sha512-Ox2kYyxy7NoXdKWdHeDEjZxClwzO4SKM8plAaVwmAJPxHMqA0rLOoAsa+hBDwRLpctf+ZRnAd/ykguuJidnaTA==", + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.6.11" + }, + "peerDependencies": { + "solid-js": "^1.8" + } + }, "node_modules/@drakkar.software/dk-spaces-platform-sdk": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@drakkar.software/dk-spaces-platform-sdk/-/dk-spaces-platform-sdk-0.3.5.tgz", @@ -541,6 +584,21 @@ "node": ">=22" } }, + "node_modules/@drakkar.software/octobot-client/node_modules/@noble/curves": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.2.0.tgz", + "integrity": "sha512-T/BoHgFXirb0ENSPBquzX0rcjXeM6Lo892a2jlYJkqk83LqZx0l1Of7DzlKJ6jkpvMrkHSnAcgb5JegL8SeIkQ==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "2.2.0" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@drakkar.software/octobot-protocol": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/@drakkar.software/octobot-protocol/-/octobot-protocol-0.8.0.tgz", @@ -982,10 +1040,26 @@ "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", "license": "MIT" }, + "node_modules/@hey-api/codegen-core": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@hey-api/codegen-core/-/codegen-core-0.2.0.tgz", + "integrity": "sha512-c7VjBy/8ed0EVLNgaeS9Xxams1Tuv/WK/b4xXH3Qr4wjzYeJUtxOcoP8YdwNLavqKP8pGiuctjX2Z1Pwc4jMgQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=22.10.0" + }, + "funding": { + "url": "https://github.com/sponsors/hey-api" + }, + "peerDependencies": { + "typescript": ">=5.5.3" + } + }, "node_modules/@hey-api/json-schema-ref-parser": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@hey-api/json-schema-ref-parser/-/json-schema-ref-parser-1.0.6.tgz", - "integrity": "sha512-yktiFZoWPtEW8QKS65eqKwA5MTKp88CyiL8q72WynrBs/73SAaxlSWlA2zW/DZlywZ5hX1OYzrCC0wFdvO9c2w==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@hey-api/json-schema-ref-parser/-/json-schema-ref-parser-1.2.0.tgz", + "integrity": "sha512-BMnIuhVgNmSudadw1GcTsP18Yk5l8FrYrg/OSYNxz0D2E0vf4D5e4j5nUbuY8MU6p1vp7ev0xrfP6A/NWazkzQ==", "dev": true, "license": "MIT", "dependencies": { @@ -1002,19 +1076,21 @@ } }, "node_modules/@hey-api/openapi-ts": { - "version": "0.73.0", - "resolved": "https://registry.npmjs.org/@hey-api/openapi-ts/-/openapi-ts-0.73.0.tgz", - "integrity": "sha512-sUscR3OIGW0k9U//28Cu6BTp3XaogWMDORj9H+5Du9E5AvTT7LZbCEDvkLhebFOPkp2cZAQfd66HiZsiwssBcQ==", + "version": "0.85.0", + "resolved": "https://registry.npmjs.org/@hey-api/openapi-ts/-/openapi-ts-0.85.0.tgz", + "integrity": "sha512-LSBHP2/wTF1BnaccHGX1t+0Ss+2VJQxotrLz/0+LK2z8ocuyVZXOYhfBSd7FP8sK78MDJVDBYrPCsBUvNSlH1g==", "dev": true, "license": "MIT", "dependencies": { - "@hey-api/json-schema-ref-parser": "1.0.6", + "@hey-api/codegen-core": "^0.2.0", + "@hey-api/json-schema-ref-parser": "1.2.0", "ansi-colors": "4.1.3", - "c12": "2.0.1", + "c12": "3.3.0", "color-support": "1.1.3", "commander": "13.0.0", "handlebars": "4.7.8", - "open": "10.1.2" + "open": "10.1.2", + "semver": "7.7.2" }, "bin": { "openapi-ts": "bin/index.cjs" @@ -1026,7 +1102,7 @@ "url": "https://github.com/sponsors/hey-api" }, "peerDependencies": { - "typescript": "^5.5.3" + "typescript": ">=5.5.3" } }, "node_modules/@hookform/resolvers": { @@ -1139,6 +1215,15 @@ } } }, + "node_modules/@internationalized/number": { + "version": "3.6.8", + "resolved": "https://registry.npmjs.org/@internationalized/number/-/number-3.6.8.tgz", + "integrity": "sha512-8UmMFia46DUt+k97zKd9fKWXcWHR+k8ae3eYzILETuT2KbIvLyOfac7zesw+sJdRAAZ7Q9pM1Mk22aXp2LD0Ig==", + "license": "Apache-2.0", + "dependencies": { + "@swc/helpers": "^0.5.0" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -1191,13 +1276,49 @@ "dev": true, "license": "MIT" }, + "node_modules/@kobalte/core": { + "version": "0.13.14", + "resolved": "https://registry.npmjs.org/@kobalte/core/-/core-0.13.14.tgz", + "integrity": "sha512-SBVB1lnvYbYt2x3t7fG0lo5fGLEbHQ6PII3NDPva8vD5e2v53On84ZVQiuIk0XI/55R4GQhden7Yem+R/XR8uQ==", + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.5.1", + "@internationalized/number": "^3.2.1", + "@kobalte/utils": "^0.9.2", + "@solid-primitives/props": "^3.1.8", + "@solid-primitives/resize-observer": "^2.0.26", + "solid-presence": "^0.2.0", + "solid-prevent-scroll": "^0.1.11" + }, + "peerDependencies": { + "solid-js": "^1.9.8" + } + }, + "node_modules/@kobalte/utils": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/@kobalte/utils/-/utils-0.9.2.tgz", + "integrity": "sha512-jRVXr+zsVHxzDXRoh+CDeXzvCsFJ6uiHhqqNQ26Cw9ZsZ3D6nqPUBt1gGVtj2ZPmRL3a9Uk1v8D1aJ8/I12Dow==", + "license": "MIT", + "dependencies": { + "@solid-primitives/event-listener": "^2.2.14", + "@solid-primitives/keyed": "^1.2.0", + "@solid-primitives/map": "^0.4.7", + "@solid-primitives/media": "^2.2.4", + "@solid-primitives/props": "^3.1.8", + "@solid-primitives/refs": "^1.0.5", + "@solid-primitives/utils": "^6.2.1" + }, + "peerDependencies": { + "solid-js": "^1.8.8" + } + }, "node_modules/@noble/curves": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.2.0.tgz", - "integrity": "sha512-T/BoHgFXirb0ENSPBquzX0rcjXeM6Lo892a2jlYJkqk83LqZx0l1Of7DzlKJ6jkpvMrkHSnAcgb5JegL8SeIkQ==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.4.0.tgz", + "integrity": "sha512-P4/62zrgfH33CneE3Dn4WhJVA22YUU0eR51wKIan4NVRvwsA0YnPTwWGpNbpuacSujmSFLvyzpyuR30+fbq2Ew==", "license": "MIT", "dependencies": { - "@noble/hashes": "2.2.0" + "@noble/hashes": "2.4.0" }, "engines": { "node": ">= 20.19.0" @@ -1206,6 +1327,18 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@noble/curves/node_modules/@noble/hashes": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.4.0.tgz", + "integrity": "sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@noble/hashes": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.2.0.tgz", @@ -1219,28 +1352,28 @@ } }, "node_modules/@oxc-project/types": { - "version": "0.148.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.148.0.tgz", - "integrity": "sha512-Nm4s/jB+4FpFsPhWGEC4h7rzksesmtnMXomo6rCMcg/b8zLQuOziRgkCS1fxDCXOlJB/6Q8oABOZ/OP6RIPj9A==", + "version": "0.150.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.150.0.tgz", + "integrity": "sha512-rDS5/31E9HfPl/CIzGrn0DOlvBbXFseQ5URJ9sYMfstbKLD/c6Gm9vmRzRGDdAXyOIL4zmO37lc9RIwYqVruZw==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/oxc-project" } }, "node_modules/@playwright/test": { - "version": "1.59.1", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", - "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.59.1" + "playwright": "1.63.0" }, "bin": { "playwright": "cli.js" }, "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/@radix-ui/number": { @@ -2131,9 +2264,9 @@ "license": "MIT" }, "node_modules/@rolldown/binding-android-arm-eabi": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.7.tgz", - "integrity": "sha512-EypzgnYCwyVY4NDHKzGmNJT5b+XaQEBniHxsMdeIQLB/tcCzZnhqrzHpZFbX9iaxx+5RiB8caATBtfvZP7zVxQ==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.9.tgz", + "integrity": "sha512-tNISae1QEf/vkb3xkRcjV5SEdzPE97We5IVaa2Z8jSszQPZ8U60B/YCYpw4QI7VidYsBtKavczXf+DyDs9WGxw==", "cpu": [ "arm" ], @@ -2147,9 +2280,9 @@ } }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.7.tgz", - "integrity": "sha512-l17HE9EweWaqJZhuUuNBN/FzM62xw+DECVnJyvMsxn8vJFAGLy5QfLDoYAcronkAN8VxKZHezDpulHDPx95vFw==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.9.tgz", + "integrity": "sha512-YC8YsI30o606GTZi0VyzYlsDKFP8W61i/QzayHDkLbNEz/IShqAmTa+hsJRj13xTHA0H+6fk4b2UmGn+Q/cMlg==", "cpu": [ "arm64" ], @@ -2163,9 +2296,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.7.tgz", - "integrity": "sha512-8ED8ELFvHXc6OCETIn4gXObPiaR6bckM/ipXtbzlPVDRMBfEGjCKgO90F9YtfdpDatVx/ZQw7aZ1vUMf/+T3Mw==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.9.tgz", + "integrity": "sha512-IwhlH3qK5urrY8hZiEgGkHKEFN901p/p2bjxCxJlr4GyNnF7wYpUvK+Y43uaRYuC4hpfjzbR3SJC3arX1jGvmw==", "cpu": [ "arm64" ], @@ -2179,9 +2312,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.7.tgz", - "integrity": "sha512-/WPripjtiAIZ2tWY7ddijORT0Ujg87wxWW/qcoFVCKAWVDPhtY0xr7Dj0M3GyNGz60jGwTElhro/mkF9dT7dDQ==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.9.tgz", + "integrity": "sha512-XxpJfVzFh+jilRxIXUqcfYAYcunIc/XEzIizsOL1fcJee5Sf7H3mH8WlLmfHfluz5amqR88QQo9izKtmMlavAw==", "cpu": [ "x64" ], @@ -2195,9 +2328,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.7.tgz", - "integrity": "sha512-14DI4NcqpvbICxSnGLx3PmtDaWqRP/KGSGb6C+JLLVPeZRl6dKdHba3pGsqT3vpdTqhEYIPG0MMQ8c0xYqoJxA==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.9.tgz", + "integrity": "sha512-kSfvhmgeWyfkbT3p/1s5vSgboogoah2zkm9fX2zjg2hHxSV7T4KhMWRUUaRk4OXNqoD3QAUeRqLcs1aZOK4U1g==", "cpu": [ "x64" ], @@ -2211,9 +2344,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.7.tgz", - "integrity": "sha512-bxrWIRvHWQvbJwi+VIie/kDJmQxcNE6xxWwZdqF/ExVAigtHkv54WTLQPb+QsZdnFy18fg7JPfWGL0RH6vwIlQ==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.9.tgz", + "integrity": "sha512-1RVzG17pxqbTfYLC352JlLt6kKLG+6Hr30n8DlIJqsnV5luUDd2Qdx9Ayw1Cabfyb1K9k0jXEZ7evxkRoT+uiw==", "cpu": [ "arm" ], @@ -2227,12 +2360,15 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.7.tgz", - "integrity": "sha512-toOY2BChBZyuxU7OYX6Tn389di4IzAqPTycVcci0O7FSfBqzRB3RZn+K5Is6ANf4tmgRd/K1yZTsNTXbkXsnLg==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.9.tgz", + "integrity": "sha512-BXqPvZ2drqVD+/Z8UpKwcs4Mp7grM+eGFku4CAEKrEtcbAsUpzREphK1sogCRZGreVPiMkiiBtw0n3TPteuqvw==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2243,12 +2379,15 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.7.tgz", - "integrity": "sha512-lAIXTH/aiLRLxsTgQvfhjo4K1ydWIp00+V0voOr9beb/9ZmkUFrSIb03dXNFRgMNvkE6oGsF10ioQ6UsI+vS5Q==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.9.tgz", + "integrity": "sha512-11vWvo8YDwLzukt27J3aYDWU+gg2P7J+ZOmiJ0hkF5BXZDW7pVya7r40MXDy6ya0i9KamoENSVKIugvJNgFXIA==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -2259,12 +2398,15 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.7.tgz", - "integrity": "sha512-kdnwS28Pkenp/mZMRwjXXXwxQ7pIsm+bF919LUK93BOyhcLsrVKdP2p9fxpiPNPAbNuch8ypQt0pm2P2LYCAGg==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.9.tgz", + "integrity": "sha512-a1tijMkdwsIARtc0F39ApURROkf3NwqinI6TOiSSWCTR7dT96dffNvMUtDHnq64wKNTIZOIlzKrFvvFUznJiyw==", "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2275,12 +2417,15 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.7.tgz", - "integrity": "sha512-516OdsyLdr5E65paF3yBF55t8mfm9+gmtCsK3xI7XKXIT7EfRlHhxL8K/NR6Hu8BWSgF5+1w74lTL0+nxcc8Qw==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.9.tgz", + "integrity": "sha512-x6SQNdAvv4c3hWqTMaWuawzMX9myaCs/yEmlGsxJzkdClnHW7FbrjQuSiRDhuSYzEYoEMhsaJy9qHG/XNemJPQ==", "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2291,12 +2436,15 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.7.tgz", - "integrity": "sha512-r8/z8n7GFaYRln3xmP1Cxy0HH/HLM0uBUPkEuSVEfKGDA89M0FsZRZJRSwe/tJjRx+fpH/gjorfhB8tmEbSFLA==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.9.tgz", + "integrity": "sha512-9s0AZ8BFK5/n7B/TBoa2yJE3gI3KURrbXcPBlsAsvjU4VeJKgE90y1YtNxyEUIcHPQkg6/yfF3qihUrcM/Kf0Q==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2307,12 +2455,15 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.7.tgz", - "integrity": "sha512-pAsE8iiDxUg1xBqdhrTfg45AVDVpirjz00sblEYClGNNcMnDb+e8beQgqIAw6LvauX/APvgxUnwrgun/YYGBhw==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.9.tgz", + "integrity": "sha512-P7VWAmV+WdJluH7ovnRGoiv2i8To7GAZ+kGzfGup635cyL7SyYl3lSUaA3Gp5THf0n/Co5EyEqb2zbqq+nMOHQ==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -2323,9 +2474,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.7.tgz", - "integrity": "sha512-lTcIYmmnQQA8Or/2DatS6oSqcdLHvendjS+zLu+FwgToynWMRSmQdpM65fTANJgIS4mjbMOo5KT2lnT9SAb96w==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.9.tgz", + "integrity": "sha512-1qixtsE4BK8h+yS3BfmZ09UhA7O/N4IACva6YBr7EBvCJraByTuRcgOTaiA62Tm0vey3UcKXLOaoGHtYmNGEVg==", "cpu": [ "arm64" ], @@ -2339,9 +2490,9 @@ } }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.7.tgz", - "integrity": "sha512-e3Gu3WxbNk/UqQhxqU7YIYO+9ZBvWNz3U+h/qRFosscMFzdRPbXYSaSWgSnklv2fz1TgzBTcti2z35c/7irsHw==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.9.tgz", + "integrity": "sha512-ok8IQjcEPs1AKZfuEUznVBrJw+gK4soq+bx8b1X2XoMqVClarc1q5JDmVtWXY1xfr6ZuHTAsPXHTgTrqKTZeww==", "cpu": [ "arm64" ], @@ -2355,9 +2506,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.7.tgz", - "integrity": "sha512-W/jg5qoRSqjsEv0+dZi4e687mcHqmVuU0P4fK6qS/xjetW2Gmc1W8j//z5nAeNcC8Ttm0hV46IjcYeuVwYhuiw==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.9.tgz", + "integrity": "sha512-Ip2mXoU0hM0boq3Rf+ekuT653OROSo6aSYcPT1VHE4q52KvyxgFkQgrgb/IEsxOuvQ2fZZbs8khJAyCEPM24/g==", "cpu": [ "x64" ], @@ -2409,6 +2560,159 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@solid-primitives/event-listener": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/@solid-primitives/event-listener/-/event-listener-2.4.6.tgz", + "integrity": "sha512-5I0YJcTVYIWoMmgBSROBZGcz+ymhew/pGTg2dHW74BUjFKsV8Li4bOZYl0YAGP4mHw5o4UBd9/BEesqBci3wxw==", + "license": "MIT", + "dependencies": { + "@solid-primitives/utils": "^6.4.1" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/keyed": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@solid-primitives/keyed/-/keyed-1.5.3.tgz", + "integrity": "sha512-zNadtyYBhJSOjXtogkGHmRxjGdz9KHc8sGGVAGlUABkE8BED2tbIZoxkwSqzOwde8OcUEH0bb5DLZUWIMvyBSA==", + "license": "MIT", + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/map": { + "version": "0.4.13", + "resolved": "https://registry.npmjs.org/@solid-primitives/map/-/map-0.4.13.tgz", + "integrity": "sha512-B1zyFbsiTQvqPr+cuPCXO72sRuczG9Swncqk5P74NCGw1VE8qa/Ry9GlfI1e/VdeQYHjan+XkbE3rO2GW/qKew==", + "license": "MIT", + "dependencies": { + "@solid-primitives/trigger": "^1.1.0" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/media": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@solid-primitives/media/-/media-2.3.6.tgz", + "integrity": "sha512-pk49gPOq/UMRUJ+pTSrOfBiR8xJjRYHXIf1iR/jSnyQ/KroU+ZXhkZzavC7hvfp2vJeOTW5k2/HN0r3Q1VJ7Pw==", + "license": "MIT", + "dependencies": { + "@solid-primitives/event-listener": "^2.4.6", + "@solid-primitives/rootless": "^1.5.4", + "@solid-primitives/static-store": "^0.1.4", + "@solid-primitives/utils": "^6.4.1" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/props": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@solid-primitives/props/-/props-3.2.4.tgz", + "integrity": "sha512-MXXdvi2TSB6d+0N6ueA/HP1j/Kh9SEc4WdF1ZDMLwPagxW6pIHHSS9xbRO0RjqSVpNP4j0nxCWT1hx3CkJNhNA==", + "license": "MIT", + "dependencies": { + "@solid-primitives/utils": "^6.4.1" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/refs": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@solid-primitives/refs/-/refs-1.1.4.tgz", + "integrity": "sha512-bLjwIs6ZPu8NQnuw04sU3Zc8qKSpbc0umUU/O4SHf6oWOdO4+dHY8vb1T7C4b/Tg103+9WGr6sEE0+NFlbaB/A==", + "license": "MIT", + "dependencies": { + "@solid-primitives/utils": "^6.4.1" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/resize-observer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@solid-primitives/resize-observer/-/resize-observer-2.2.0.tgz", + "integrity": "sha512-9Fuu/EWBeGj+atGHRJp70HKhdfalmpjwxY8a32NZixdLNmfCJ45AfhLQNr6uOzETbbiMx4iCKlTrJ8KZCHC2Ww==", + "license": "MIT", + "dependencies": { + "@solid-primitives/event-listener": "^2.4.6", + "@solid-primitives/rootless": "^1.5.4", + "@solid-primitives/static-store": "^0.1.4", + "@solid-primitives/utils": "^6.4.1" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/rootless": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@solid-primitives/rootless/-/rootless-1.5.4.tgz", + "integrity": "sha512-TOIZa1VUfVJ+9nkCcRajw3U4t9vBOP1HxX1WHNTbXq32mXwlqTvUnC4CRIilohcryBkT9u2ZkhUDSHRTaGp55g==", + "license": "MIT", + "dependencies": { + "@solid-primitives/utils": "^6.4.1" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/static-store": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@solid-primitives/static-store/-/static-store-0.1.4.tgz", + "integrity": "sha512-LgtVaVBtB7EbmS4+M0b8xY5Iq6pUWXBsIC4VgtrFKDGDdyCaDt88sHk0fUlx1Enxm/XZnZyLXJABRoa39RjJqA==", + "license": "MIT", + "dependencies": { + "@solid-primitives/utils": "^6.4.1" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/storage": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@solid-primitives/storage/-/storage-1.3.11.tgz", + "integrity": "sha512-PpQWR3TaTxHIJFbI9ZssYTM4Aa67g1vJIgps4TPhcXzHqqomrPAIveFC2FG7SDQoi9YQia8FVBjigELziJpfIg==", + "license": "MIT", + "dependencies": { + "@solid-primitives/utils": "^6.2.0" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/transition-group": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@solid-primitives/transition-group/-/transition-group-1.1.2.tgz", + "integrity": "sha512-gnHS0OmcdjeoHN9n7Khu8KNrOlRc8a2weETDt2YT6o1zeW/XtUC6Db3Q9pkMU/9cCKdEmN4b0a/41MKAHRhzWA==", + "license": "MIT", + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/trigger": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@solid-primitives/trigger/-/trigger-1.2.4.tgz", + "integrity": "sha512-Ju0e+ZOD7hpOp7nptJimvDSZHWFvIvF9iBWMvuwt30smX7c5wmB8Kmc0AdDuv0wOTi06PQ1J5IrP1WJbH2yUBQ==", + "license": "MIT", + "dependencies": { + "@solid-primitives/utils": "^6.4.1" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, + "node_modules/@solid-primitives/utils": { + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/@solid-primitives/utils/-/utils-6.4.1.tgz", + "integrity": "sha512-ISSB5QX1qP2ynrheIpYwc4oKR5Ny4siNuUyf1qZniy+Il+p/PtDB0QK1Dnle8noiHpwRD3gpPdubOC3qI/Zamg==", + "license": "MIT", + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -2522,6 +2826,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -2539,6 +2846,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -2556,6 +2866,9 @@ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -2573,6 +2886,9 @@ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -2590,6 +2906,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -2607,6 +2926,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -2674,6 +2996,15 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/@swc/helpers": { + "version": "0.5.23", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.23.tgz", + "integrity": "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.8.0" + } + }, "node_modules/@swc/types": { "version": "0.1.28", "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.28.tgz", @@ -2809,6 +3140,9 @@ "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2825,6 +3159,9 @@ "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -2841,6 +3178,9 @@ "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2857,6 +3197,9 @@ "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -2942,9 +3285,9 @@ } }, "node_modules/@tanstack/history": { - "version": "1.162.2", - "resolved": "https://registry.npmjs.org/@tanstack/history/-/history-1.162.2.tgz", - "integrity": "sha512-Lemp3DJbzNqcin/nZpWxycDaEqySDbnIshDbyHJMMCapD4ZQMe57szRpBXOfzfP6fyWAtHNrLrcBUyANJ6Vlow==", + "version": "1.162.4", + "resolved": "https://registry.npmjs.org/@tanstack/history/-/history-1.162.4.tgz", + "integrity": "sha512-utTS5L2OkeYUzXGohL1Z8sefu1GLNOJcxe8Hd6iIdc/Xo1K1nDB2JEp4iSFhvYh33xKC9V91TxrS8qfrpoKobQ==", "license": "MIT", "engines": { "node": ">=20.19" @@ -2954,10 +3297,26 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, + "node_modules/@tanstack/match-sorter-utils": { + "version": "8.19.4", + "resolved": "https://registry.npmjs.org/@tanstack/match-sorter-utils/-/match-sorter-utils-8.19.4.tgz", + "integrity": "sha512-Wo1iKt2b9OT7d+YGhvEPD3DXvPv2etTusIMhMUoG7fbhmxcXCtIjJDEygy91Y2JFlwGyjqiBPRozme7UD8hoqg==", + "license": "MIT", + "dependencies": { + "remove-accents": "0.5.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, "node_modules/@tanstack/query-core": { - "version": "5.102.8", - "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.102.8.tgz", - "integrity": "sha512-ZNjkJ33CqvPNec/6lZBnHqLc3EVGPZ9ySLhYahU9TcuRFdmwXewuj0c4hwSWcGHqEUwcSrKeZ+oGcvPBqXcQcg==", + "version": "5.103.2", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.103.2.tgz", + "integrity": "sha512-I8DkFXls5jXLqtm8+QpOhEmG07hIblhSZFzafg9wHsMSEvMzULy9hK17wU1T/ahfhMbtITJhbxutwwCoihkR7A==", "license": "MIT", "funding": { "type": "github", @@ -2965,22 +3324,35 @@ } }, "node_modules/@tanstack/query-devtools": { - "version": "5.102.8", - "resolved": "https://registry.npmjs.org/@tanstack/query-devtools/-/query-devtools-5.102.8.tgz", - "integrity": "sha512-ZgeMKuF5d/zOE+tgWm3cSbD6Zcbr6IugVsnbHzUcSismqLZDSUPBKM6ILUxExgwe6rPAOox2x5bA5T+PSOQG0Q==", + "version": "5.103.2", + "resolved": "https://registry.npmjs.org/@tanstack/query-devtools/-/query-devtools-5.103.2.tgz", + "integrity": "sha512-QW+uVtT7dZcc9KMmKQ846Yw25h7WPhM1cfCYYaSkkmPqkK39g+Cj1tcAt+Ege+6rBVFQvNyYln0FgAAHcYGSdA==", "license": "MIT", + "dependencies": { + "@kobalte/core": "^0.13.4", + "@solid-primitives/keyed": "^1.2.2", + "@solid-primitives/resize-observer": "^2.0.26", + "@solid-primitives/storage": "^1.3.11", + "@tanstack/match-sorter-utils": "^8.19.4", + "@tanstack/query-core": "5.103.2", + "clsx": "^2.1.1", + "goober": "^2.1.16", + "solid-js": "^1.9.7", + "solid-transition-group": "^0.2.3", + "superjson": "^2.2.2" + }, "funding": { "type": "github", "url": "https://github.com/sponsors/tannerlinsley" } }, "node_modules/@tanstack/react-query": { - "version": "5.102.8", - "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.102.8.tgz", - "integrity": "sha512-TYBea4OuXWD7MhaSHq069TWbFe7rcwWN6kzT7JF0OKi1K6c1gTv2IzD6A6ExJsCMozdkqBWeuIUZmu4KQg0O5A==", + "version": "5.103.2", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.103.2.tgz", + "integrity": "sha512-B+fWiYZBc+0uUD5zDZAeLw9dKj7XEsdnuu6zRZ+no6LNKpeE3P3bJ+N6HINjcIlWR6fW3vUoTneEaGa2V6ehqw==", "license": "MIT", "dependencies": { - "@tanstack/query-core": "5.102.8" + "@tanstack/query-core": "5.103.2" }, "funding": { "type": "github", @@ -2991,31 +3363,32 @@ } }, "node_modules/@tanstack/react-query-devtools": { - "version": "5.102.8", - "resolved": "https://registry.npmjs.org/@tanstack/react-query-devtools/-/react-query-devtools-5.102.8.tgz", - "integrity": "sha512-QKb7A44BZOU7nxsGA4gFN1fofjYovar5O0T83Ff4Y+2eRq09RGFrAzzutVdF/6/emfSaMDohp6e759BjB3fxEw==", + "version": "5.103.2", + "resolved": "https://registry.npmjs.org/@tanstack/react-query-devtools/-/react-query-devtools-5.103.2.tgz", + "integrity": "sha512-NV2AwKeS4UoyAJFcU0CNsoABKVf3A9KMX2InuII2LPUW7z6U296mbQITURAoMwDAxC2AMFCbXlMEPCgRGveQhw==", "license": "MIT", "dependencies": { - "@tanstack/query-devtools": "5.102.8" + "@tanstack/query-devtools": "5.103.2" }, "funding": { "type": "github", "url": "https://github.com/sponsors/tannerlinsley" }, "peerDependencies": { - "@tanstack/react-query": "^5.102.8", + "@tanstack/react-query": "^5.103.2", + "@types/react": "^18 || ^19", "react": "^18 || ^19" } }, "node_modules/@tanstack/react-router": { - "version": "1.170.33", - "resolved": "https://registry.npmjs.org/@tanstack/react-router/-/react-router-1.170.33.tgz", - "integrity": "sha512-iNnI98vH3kO/V4dy6YM0CInhqwWBddU0G5wZK5jiMvr3HsK2avDQSRx3RY/y6v+6zQAqb2kD6hUPHIenrJBTSw==", + "version": "1.170.38", + "resolved": "https://registry.npmjs.org/@tanstack/react-router/-/react-router-1.170.38.tgz", + "integrity": "sha512-iHM9b0aDJbuvftmkiQXawzoqsdV68p2Re2safbVLCnxafe+iLKV++2i3hI/BMAP6uR92/Mjw94JKJJfD7pbwHQ==", "license": "MIT", "dependencies": { - "@tanstack/history": "1.162.2", - "@tanstack/react-store": "^0.9.3", - "@tanstack/router-core": "1.171.28", + "@tanstack/history": "1.162.4", + "@tanstack/react-store": "^0.11.0", + "@tanstack/router-core": "1.171.32", "isbot": "^5.1.22" }, "engines": { @@ -3031,12 +3404,12 @@ } }, "node_modules/@tanstack/react-router-devtools": { - "version": "1.167.1", - "resolved": "https://registry.npmjs.org/@tanstack/react-router-devtools/-/react-router-devtools-1.167.1.tgz", - "integrity": "sha512-pjfGrmjj4d7naEPM7oshqFfwBoxDPNo/UxltlHH5ePbHsJ+plBhd+JaAewm1ueYOjZ0js9hckjWWDYXpCrSfKw==", + "version": "1.167.2", + "resolved": "https://registry.npmjs.org/@tanstack/react-router-devtools/-/react-router-devtools-1.167.2.tgz", + "integrity": "sha512-VuoqB7OhDUrBa4WOFSKiAQ8xI8sJpeZdo9sYE1hQa89iMuVFUQu1iWE4iCiQhB+6TYOZDZ9e+iHKxYjop4ZVYw==", "license": "MIT", "dependencies": { - "@tanstack/router-devtools-core": "1.168.1" + "@tanstack/router-devtools-core": "1.168.2" }, "engines": { "node": ">=20.19" @@ -3046,8 +3419,8 @@ "url": "https://github.com/sponsors/tannerlinsley" }, "peerDependencies": { - "@tanstack/react-router": "^1.170.19", - "@tanstack/router-core": "^1.171.16", + "@tanstack/react-router": "^1.170.36", + "@tanstack/router-core": "^1.171.30", "react": ">=18.0.0 || >=19.0.0", "react-dom": ">=18.0.0 || >=19.0.0" }, @@ -3058,12 +3431,12 @@ } }, "node_modules/@tanstack/react-store": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/@tanstack/react-store/-/react-store-0.9.3.tgz", - "integrity": "sha512-y2iHd/N9OkoQbFJLUX1T9vbc2O9tjH0pQRgTcx1/Nz4IlwLvkgpuglXUx+mXt0g5ZDFrEeDnONPqkbfxXJKwRg==", + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@tanstack/react-store/-/react-store-0.11.1.tgz", + "integrity": "sha512-HaIGKI3YLmjBYIvy5DFDY23oNaYZIsTZfngey07Uh5iLVJgM3bIGCnZeOFOqzjFld9JHWcaHJnasD/bKoGKwJQ==", "license": "MIT", "dependencies": { - "@tanstack/store": "0.9.3", + "@tanstack/store": "0.11.1", "use-sync-external-store": "^1.6.0" }, "funding": { @@ -3096,12 +3469,12 @@ } }, "node_modules/@tanstack/router-core": { - "version": "1.171.28", - "resolved": "https://registry.npmjs.org/@tanstack/router-core/-/router-core-1.171.28.tgz", - "integrity": "sha512-PvPWSklhw6i9b0rzScVh0btQsK5u/gBYN3mBHyDzhC/U4LrB3WzPXPkUunQUKvQOGXCp16UEb7Htc/ITGm5DkQ==", + "version": "1.171.32", + "resolved": "https://registry.npmjs.org/@tanstack/router-core/-/router-core-1.171.32.tgz", + "integrity": "sha512-X86Jqk3vB2KJcUfS6oLi7B7yxbaa59QEhZRPPP1fRx3k+Jw/Fu1UYVBcRtdwK/OccQnrlQdVoKvNO3QXmyEBOw==", "license": "MIT", "dependencies": { - "@tanstack/history": "1.162.2", + "@tanstack/history": "1.162.4", "cookie-es": "^3.0.0", "seroval": "^1.6.2", "seroval-plugins": "^1.6.2" @@ -3114,38 +3487,10 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, - "node_modules/@tanstack/router-devtools": { - "version": "1.167.1", - "resolved": "https://registry.npmjs.org/@tanstack/router-devtools/-/router-devtools-1.167.1.tgz", - "integrity": "sha512-DkauJt3A8Rt388N44L7wCG8Z5ub4mxTTRNu6WgUBL/w/tVfQZvt/h6KM9+EJZZ1A2eVJtOB7sPoNKedKjzv8Xw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@tanstack/react-router-devtools": "1.167.1" - }, - "engines": { - "node": ">=20.19" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/tannerlinsley" - }, - "peerDependencies": { - "@tanstack/react-router": "^1.170.19", - "csstype": "^3.0.10", - "react": ">=18.0.0 || >=19.0.0", - "react-dom": ">=18.0.0 || >=19.0.0" - }, - "peerDependenciesMeta": { - "csstype": { - "optional": true - } - } - }, "node_modules/@tanstack/router-devtools-core": { - "version": "1.168.1", - "resolved": "https://registry.npmjs.org/@tanstack/router-devtools-core/-/router-devtools-core-1.168.1.tgz", - "integrity": "sha512-qr4voa4cpSMwQvS3867xkU3AB3MtJbTuovKIy+btjJ/Faju6er9w0nDylmD+005Mk/3YKw9/iueZJl2JAB7JOA==", + "version": "1.168.2", + "resolved": "https://registry.npmjs.org/@tanstack/router-devtools-core/-/router-devtools-core-1.168.2.tgz", + "integrity": "sha512-pCCdQ5M/NUeWdiAbPmjyfcNdls6G4vaVlIQbPf1IqZk1FLYFr00aQvi578yUuWFBNM2KNS2QcwM1uPucbtGf5Q==", "license": "MIT", "dependencies": { "clsx": "^2.1.1", @@ -3159,7 +3504,7 @@ "url": "https://github.com/sponsors/tannerlinsley" }, "peerDependencies": { - "@tanstack/router-core": "^1.171.16", + "@tanstack/router-core": "^1.171.30", "csstype": "^3.0.10" }, "peerDependenciesMeta": { @@ -3169,20 +3514,20 @@ } }, "node_modules/@tanstack/router-generator": { - "version": "1.167.34", - "resolved": "https://registry.npmjs.org/@tanstack/router-generator/-/router-generator-1.167.34.tgz", - "integrity": "sha512-sbCQrpYd2MGQXVXltUyPNLQFl924kLXBYp6Gq4keb1GPuDKfxH+yfcI3Qv9RQxI3ZErjgDZE7sOhRDXIf7xzxQ==", + "version": "1.167.38", + "resolved": "https://registry.npmjs.org/@tanstack/router-generator/-/router-generator-1.167.38.tgz", + "integrity": "sha512-lkqgFleDgfkW+QONVMKBs+ZGDUF0v9R9FkplS/GZvKDFpfnu+tZxmGlV2pryU1TDxyuSyjh56AtIla3NMrlScw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.28.5", - "@tanstack/router-core": "1.171.28", - "@tanstack/router-utils": "1.162.2", + "@babel/types": "^7.29.8", + "@tanstack/router-core": "1.171.32", + "@tanstack/router-utils": "1.162.3", "@tanstack/virtual-file-routes": "1.162.0", "jiti": "^2.7.0", "magic-string": "^0.30.21", - "prettier": "^3.5.0", - "zod": "^4.4.3" + "prettier": "^3.9.6", + "zod": "^4.5.4" }, "engines": { "node": ">=20.19" @@ -3193,21 +3538,21 @@ } }, "node_modules/@tanstack/router-plugin": { - "version": "1.168.36", - "resolved": "https://registry.npmjs.org/@tanstack/router-plugin/-/router-plugin-1.168.36.tgz", - "integrity": "sha512-A/Ama19yfgbgC3QZgQGwC+wceK8cYiqqFgRjTuNxhPCrrrdUrP1SKnCKvLvHISkx6xgLnDDMl2wJyP9PMgrjYg==", + "version": "1.168.40", + "resolved": "https://registry.npmjs.org/@tanstack/router-plugin/-/router-plugin-1.168.40.tgz", + "integrity": "sha512-ifiXjjR4uivxwRDhgYAcfyrcu+Mwx+vlG0QjjQ7N5C5sKmzpt5UtsL21TNVvZDSZl2Vae8DcL84Z02fOs8ZZMQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/core": "^7.28.5", - "@babel/template": "^7.27.2", - "@babel/types": "^7.28.5", - "@tanstack/router-core": "1.171.28", - "@tanstack/router-generator": "1.167.34", - "@tanstack/router-utils": "1.162.2", + "@babel/core": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8", + "@tanstack/router-core": "1.171.32", + "@tanstack/router-generator": "1.167.38", + "@tanstack/router-utils": "1.162.3", "chokidar": "^5.0.0", - "unplugin": "^3.0.0", - "zod": "^4.4.3" + "unplugin": "^3.3.0", + "zod": "^4.5.4" }, "engines": { "node": ">=20.19" @@ -3218,7 +3563,7 @@ }, "peerDependencies": { "@rsbuild/core": ">=1.0.2 || ^2.0.0", - "@tanstack/react-router": "^1.170.33", + "@tanstack/react-router": "^1.170.38", "vite": ">=5.0.0 || >=6.0.0 || >=7.0.0 || >=8.0.0", "vite-plugin-solid": "^2.11.10 || ^3.0.0-0", "webpack": ">=5.92.0" @@ -3242,20 +3587,20 @@ } }, "node_modules/@tanstack/router-utils": { - "version": "1.162.2", - "resolved": "https://registry.npmjs.org/@tanstack/router-utils/-/router-utils-1.162.2.tgz", - "integrity": "sha512-hTWqJtqIFFdvuCl8WXNyrodp2L9zo2G37xKRrcVmVRWpAB2h+U1LuRAfS4tsFTiWOIoE/B+WDVFB8JpoEdw6jQ==", + "version": "1.162.3", + "resolved": "https://registry.npmjs.org/@tanstack/router-utils/-/router-utils-1.162.3.tgz", + "integrity": "sha512-Icb0xGuG1+54IV0WMLRcc3ErTx2HeJWyPG661FkQ8UT6guoBq1FJjFRqlG/JS0xi4mFFkBhvwO7tbLa32f3yxA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/generator": "^7.28.5", - "@babel/parser": "^7.28.5", - "@babel/types": "^7.28.5", - "ansis": "^4.1.0", + "@babel/generator": "^7.29.8", + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "ansis": "^4.3.1", "babel-dead-code-elimination": "^1.0.12", - "diff": "^8.0.2", + "diff": "^8.0.4", "pathe": "^2.0.3", - "tinyglobby": "^0.2.15" + "tinyglobby": "^0.2.17" }, "engines": { "node": ">=20.19" @@ -3266,9 +3611,9 @@ } }, "node_modules/@tanstack/store": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/@tanstack/store/-/store-0.9.3.tgz", - "integrity": "sha512-8reSzl/qGWGGVKhBoxXPMWzATSbZLZFWhwBAFO9NAyp0TxzfBP0mIrGb8CP8KrQTmvzXlR/vFPPUrHTLBGyFyw==", + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@tanstack/store/-/store-0.11.1.tgz", + "integrity": "sha512-mzTOBhypOuDJAy/D8n2MfUZ1HFkXnmSETviRyhqEC8LUE7/IZQExOTxMANj3KjTofYTkFNpBY67qaVrT41YccA==", "license": "MIT", "funding": { "type": "github", @@ -3335,9 +3680,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.9.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.5.tgz", - "integrity": "sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==", + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", "devOptional": true, "license": "MIT", "dependencies": { @@ -3345,23 +3690,22 @@ } }, "node_modules/@types/react": { - "version": "19.2.18", - "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", - "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", - "devOptional": true, + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz", + "integrity": "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg==", "license": "MIT", "dependencies": { "csstype": "^3.2.2" } }, "node_modules/@types/react-dom": { - "version": "19.2.7", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.7.tgz", - "integrity": "sha512-I8bPpDLcHBv1qiIiXDCy71Rt8eQDKJP0sMSWJphDdAcdqiJ1sGpZamavoEIRZmYzjia9LuEb2HlYdDpmoENpvQ==", + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==", "devOptional": true, "license": "MIT", "peerDependencies": { - "@types/react": "^19.2.0" + "@types/react": "^19.3.0" } }, "node_modules/@vitejs/plugin-react-swc": { @@ -3494,17 +3838,16 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/acorn": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", - "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", - "dev": true, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", "license": "MIT", - "bin": { - "acorn": "bin/acorn" + "dependencies": { + "debug": "4" }, "engines": { - "node": ">=0.4.0" + "node": ">= 6.0.0" } }, "node_modules/ansi-colors": { @@ -3518,9 +3861,9 @@ } }, "node_modules/ansis": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/ansis/-/ansis-4.3.1.tgz", - "integrity": "sha512-BJ8/l4R5LRE7hW9WdSuGYrLSHi2ynxeFpDFbH0K/CgNeY/tyhk+vO6TYxXC5r5CpUhNVX310xzPsN/H9lCdfOA==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/ansis/-/ansis-4.4.0.tgz", + "integrity": "sha512-9k3v7xcHwgdO/DruxGIg4HtjvlAZlcnsX/mzqUb1t3NkYnl9kK2UJ+Gq0io+vQf7iT//BD/HB/NBkUR1LWxoeA==", "dev": true, "license": "ISC", "engines": { @@ -3563,13 +3906,14 @@ "license": "MIT" }, "node_modules/axios": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.2.tgz", - "integrity": "sha512-wLrXxPtcrPTsNlJmKjkPnNPK2Ihe0hn0wGSaTEiHRPxwjvJwT3hKmXF4dpqxmPO9SoNb2FsYXj/xEo0gHN+D5A==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", + "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==", "license": "MIT", "dependencies": { - "follow-redirects": "^1.15.11", - "form-data": "^4.0.5", + "follow-redirects": "^1.16.0", + "form-data": "^4.0.6", + "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } }, @@ -3587,9 +3931,9 @@ } }, "node_modules/baseline-browser-mapping": { - "version": "2.11.21", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.21.tgz", - "integrity": "sha512-uh8vpY/1/YyFkunIDFH/12p7/7VdPKA1hejMVEbdkEaWnUz0Hesvx5EbiU6XxjyHZIOju+ZMbQJkRh+es3/spQ==", + "version": "2.11.25", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.25.tgz", + "integrity": "sha512-gMmEShwwq7FJqMwvfRwvCl00v4kN+KOfJqXn+f4nrufak5gNHJOksd/60Dvjuz7sI8Y5WiSFBa8FEYr+zoyqCw==", "dev": true, "license": "Apache-2.0", "bin": { @@ -3600,9 +3944,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.9", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.9.tgz", - "integrity": "sha512-EWazOblFYUvlGZcfGhPUPmYh3nikUxBVb+y9MJun5f3hBi812X+8MSQTujLBtgK3cf51fJWbWfOjyeO954d+Eg==", + "version": "4.29.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.0.tgz", + "integrity": "sha512-3GSvyjvDI4Dur1Meg2BekJquu5uF+9R9a1+5M1Mde192eZoXbeXjzgOsgqPS2V8D5wrrip0gR5Hf/GhWQ9ZzaA==", "dev": true, "funding": [ { @@ -3620,11 +3964,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.11.20", + "baseline-browser-mapping": "^2.11.23", "caniuse-lite": "^1.0.30001810", - "electron-to-chromium": "^1.5.420", - "node-releases": "^2.0.54", - "update-browserslist-db": "^1.3.2" + "electron-to-chromium": "^1.5.427", + "node-releases": "^2.0.55", + "update-browserslist-db": "^1.3.3" }, "bin": { "browserslist": "cli.js" @@ -3650,23 +3994,23 @@ } }, "node_modules/c12": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/c12/-/c12-2.0.1.tgz", - "integrity": "sha512-Z4JgsKXHG37C6PYUtIxCfLJZvo6FyhHJoClwwb9ftUkLpPSkuYqn6Tr+vnaN8hymm0kIbcg6Ey3kv/Q71k5w/A==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/c12/-/c12-3.3.0.tgz", + "integrity": "sha512-K9ZkuyeJQeqLEyqldbYLG3wjqwpw4BVaAqvmxq3GYKK0b1A/yYQdIcJxkzAOWcNVWhJpRXAPfZFueekiY/L8Dw==", "dev": true, "license": "MIT", "dependencies": { - "chokidar": "^4.0.1", - "confbox": "^0.1.7", + "chokidar": "^4.0.3", + "confbox": "^0.2.2", "defu": "^6.1.4", - "dotenv": "^16.4.5", - "giget": "^1.2.3", - "jiti": "^2.3.0", - "mlly": "^1.7.1", - "ohash": "^1.1.4", - "pathe": "^1.1.2", - "perfect-debounce": "^1.0.0", - "pkg-types": "^1.2.0", + "dotenv": "^17.2.2", + "exsolve": "^1.0.7", + "giget": "^2.0.0", + "jiti": "^2.5.1", + "ohash": "^2.0.11", + "pathe": "^2.0.3", + "perfect-debounce": "^2.0.0", + "pkg-types": "^2.3.0", "rc9": "^2.1.2" }, "peerDependencies": { @@ -3694,26 +4038,6 @@ "url": "https://paulmillr.com/funding/" } }, - "node_modules/c12/node_modules/dotenv": { - "version": "16.6.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", - "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/c12/node_modules/pathe": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", - "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", - "dev": true, - "license": "MIT" - }, "node_modules/c12/node_modules/readdirp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", @@ -3788,16 +4112,6 @@ "url": "https://paulmillr.com/funding/" } }, - "node_modules/chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, "node_modules/citty": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/citty/-/citty-0.1.6.tgz", @@ -3862,9 +4176,9 @@ } }, "node_modules/confbox": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", - "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.4.tgz", + "integrity": "sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ==", "dev": true, "license": "MIT" }, @@ -3891,6 +4205,18 @@ "integrity": "sha512-UaXxwISYJPTr9hwQxMFYZ7kNhSXboMXP+Z3TRX6f1/NyaGPfuNUZOWP1pUEb75B2HjfklIYLVRfWiFZJyC6Npg==", "license": "MIT" }, + "node_modules/copy-anything": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/copy-anything/-/copy-anything-4.1.1.tgz", + "integrity": "sha512-AoT6Imdr98feSpFfmFwTFN73ccdr7uFPf27cBCgYvyyRyn1BzLRxMvrHNmwXO5LJMddRy4Rdhw2b1h7vSMKsEw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/mesqueeb" + } + }, "node_modules/csstype": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", @@ -3901,7 +4227,6 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -4034,16 +4359,16 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.422", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.422.tgz", - "integrity": "sha512-UvA/32XqrLDdZSn7Jllo1AYNcWji/G0d5M0GTViE7KoGBiMunw3a34Sb2KO4ZZyrSEhqsxFoVhWWJshdyfKqJA==", + "version": "1.5.434", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.434.tgz", + "integrity": "sha512-7EeFW9OLf1NN9NKQP9xdOE/XKqjxv7JDPP6AEAkHli11+Fae1OaLoNuAf13hQv1PPNtuES0pLQSPk2+fS3s8ww==", "dev": true, "license": "ISC" }, "node_modules/enhanced-resolve": { - "version": "5.24.5", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.24.5.tgz", - "integrity": "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==", + "version": "5.25.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.25.1.tgz", + "integrity": "sha512-nGXts5znJzmWPu+mIE9izCOzdg63oJca2mDzGWWTth7sr4aCToKcoyFVBQwN75Ij5Pf6p510EwkTqViTRzDV+w==", "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", @@ -4135,6 +4460,13 @@ "node": ">=12.0.0" } }, + "node_modules/exsolve": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/exsolve/-/exsolve-1.1.1.tgz", + "integrity": "sha512-9U/jZUgjnSGyntRr6y5Muu1MJcwFl6kPu7k8qLF0IMNfLqvw0NZ4nnVDq0RVoZ0RvCyumib4Ez3KYrVfilrw+g==", + "dev": true, + "license": "MIT" + }, "node_modules/fake-indexeddb": { "version": "6.2.5", "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.2.5.tgz", @@ -4183,59 +4515,25 @@ } }, "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "hasown": "^2.0.4", + "mime-types": "^2.1.35" }, "engines": { "node": ">= 6" } }, - "node_modules/fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "dev": true, - "license": "ISC", - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/fs-minipass/node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/fs-minipass/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, - "license": "ISC" - }, "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "hasInstallScript": true, "license": "MIT", "optional": true, @@ -4312,9 +4610,9 @@ } }, "node_modules/giget": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/giget/-/giget-1.2.5.tgz", - "integrity": "sha512-r1ekGw/Bgpi3HLV3h1MRBIlSAdHoIMklpaQ3OQLFcRw9PwAj2rqigvIbg+dBUI51OxVI2jsEtDywDBjSiuf7Ug==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/giget/-/giget-2.0.0.tgz", + "integrity": "sha512-L5bGsVkxJbJgdnwyuheIunkGatUF/zssUoxxjACCseZYAVbaqdh9Tsmmlkl8vYan09H7sbvKt4pS8GqKLBrEzA==", "dev": true, "license": "MIT", "dependencies": { @@ -4322,9 +4620,8 @@ "consola": "^3.4.0", "defu": "^6.1.4", "node-fetch-native": "^1.6.6", - "nypm": "^0.5.4", - "pathe": "^2.0.3", - "tar": "^6.2.1" + "nypm": "^0.6.0", + "pathe": "^2.0.3" }, "bin": { "giget": "dist/cli.mjs" @@ -4425,14 +4722,27 @@ } }, "node_modules/hono": { - "version": "4.13.7", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.7.tgz", - "integrity": "sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ==", + "version": "4.13.8", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.8.tgz", + "integrity": "sha512-/Gng7NfoykZl2pjukW5Z6+8Yxm3BPRf86GTbQnt0SbySkvax4fyL4H3HhY1cCpBGmiW9XDRFzRV+CXK2W8QudQ==", "license": "MIT", "engines": { "node": ">=16.9.0" } }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/is-docker": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", @@ -4693,6 +5003,9 @@ "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4713,6 +5026,9 @@ "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4733,6 +5049,9 @@ "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4753,6 +5072,9 @@ "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4836,9 +5158,9 @@ } }, "node_modules/lucide-react": { - "version": "1.42.0", - "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.42.0.tgz", - "integrity": "sha512-b3jprplnoLS8n5etw1z8xODe3hF/yjKATrTitsrKrnjUhCef5BdDct6Ppv3zVvzFwmtfWgLO6XNM3C9fAD93ug==", + "version": "1.47.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.47.0.tgz", + "integrity": "sha512-o8C23aXpNQypRY73W7fW02EyvWJinEMXgKeGjFoKym0zj3Q73hD97A1IQps1g8C14HTGsOpZL0Am+xjfgFZAbg==", "license": "ISC", "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" @@ -4893,87 +5215,16 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=8" - } - }, - "node_modules/minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", - "dev": true, - "license": "MIT", - "dependencies": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/minizlib/node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/minizlib/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, - "license": "ISC" - }, - "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, - "license": "MIT", - "bin": { - "mkdirp": "bin/cmd.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/mlly": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.2.tgz", - "integrity": "sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.16.0", - "pathe": "^2.0.3", - "pkg-types": "^1.3.1", - "ufo": "^1.6.3" - } - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.18", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", - "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", "funding": [ { "type": "github", @@ -5013,9 +5264,9 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.54", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz", - "integrity": "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ==", + "version": "2.0.56", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.56.tgz", + "integrity": "sha512-x0InOIyzgdk+eyaWaRJFH5snEtiImgBgblZ2CyPrLmqqcuMQkEvcDPHbzqbD8eDsSeJbVOjn+crzyzHaM4D+/A==", "dev": true, "license": "MIT", "engines": { @@ -5023,26 +5274,30 @@ } }, "node_modules/nypm": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/nypm/-/nypm-0.5.4.tgz", - "integrity": "sha512-X0SNNrZiGU8/e/zAB7sCTtdxWTMSIO73q+xuKgglm2Yvzwlo8UoC5FNySQFCvl84uPaeADkqHUZUkWy4aH4xOA==", + "version": "0.6.10", + "resolved": "https://registry.npmjs.org/nypm/-/nypm-0.6.10.tgz", + "integrity": "sha512-W72Hrj1petq+b3Hk2aAC+9zswetlIFTnDW4s5djseZh2nYqBbyQLOtj472HwcbcWykPBUW1WpWpjOd4nK6gMRw==", "dev": true, "license": "MIT", "dependencies": { - "citty": "^0.1.6", - "consola": "^3.4.0", + "citty": "^0.2.2", "pathe": "^2.0.3", - "pkg-types": "^1.3.1", - "tinyexec": "^0.3.2", - "ufo": "^1.5.4" + "tinyexec": "^1.3.1" }, "bin": { "nypm": "dist/cli.mjs" }, "engines": { - "node": "^14.16.0 || >=16.10.0" + "node": ">=18" } }, + "node_modules/nypm/node_modules/citty": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/citty/-/citty-0.2.2.tgz", + "integrity": "sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w==", + "dev": true, + "license": "MIT" + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -5053,9 +5308,9 @@ } }, "node_modules/obug": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", - "integrity": "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", "dev": true, "funding": [ "https://github.com/sponsors/sxzz", @@ -5067,9 +5322,9 @@ } }, "node_modules/ohash": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/ohash/-/ohash-1.1.6.tgz", - "integrity": "sha512-TBu7PtV8YkAZn0tSxobKY2n2aAQva936lhRrj6957aDaCf9IEtqsKbgMzXE/F/sjqYOwmrukeORHNLe5glk7Cg==", + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.12.tgz", + "integrity": "sha512-65S/5gk9YSsaRjcyf7Nfa6h/d3E8/1gslpXfI4W7Dxn/oap8IKRuNT5VXkLQ1YFKIEg4apRY4Pj6aiwFzrDdmw==", "dev": true, "license": "MIT" }, @@ -5100,9 +5355,9 @@ "license": "MIT" }, "node_modules/perfect-debounce": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/perfect-debounce/-/perfect-debounce-1.0.0.tgz", - "integrity": "sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/perfect-debounce/-/perfect-debounce-2.1.0.tgz", + "integrity": "sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g==", "dev": true, "license": "MIT" }, @@ -5125,47 +5380,51 @@ } }, "node_modules/pkg-types": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", - "integrity": "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-2.3.3.tgz", + "integrity": "sha512-j/lCFdcppV0JxWpCEITdbDltBxPP6cHT+yNJ6Go2OgoSA9518X847X9z0p6LtA4Nc16+eQzCZjRrWanTGvHJ5w==", "dev": true, "license": "MIT", "dependencies": { - "confbox": "^0.1.8", - "mlly": "^1.7.4", - "pathe": "^2.0.1" + "confbox": "^0.3.1", + "exsolve": "^1.1.1", + "pathe": "^2.0.3" } }, + "node_modules/pkg-types/node_modules/confbox": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.3.1.tgz", + "integrity": "sha512-cKUSoKa8YxFZZSmraVi7onONx3amu77ngK3kGpsYHDH7drPwCRkQE1RYMPlLRrMtnciRj274XNRxcHxnKmDSnA==", + "dev": true, + "license": "MIT" + }, "node_modules/playwright": { - "version": "1.59.1", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", - "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.59.1" + "playwright-core": "1.63.0" }, "bin": { "playwright": "cli.js" }, "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "fsevents": "2.3.2" + "node": ">=20" } }, "node_modules/playwright-core": { - "version": "1.59.1", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", - "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", "dev": true, "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" }, "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/postcss": { @@ -5197,9 +5456,9 @@ } }, "node_modules/prettier": { - "version": "3.9.6", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.6.tgz", - "integrity": "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==", + "version": "3.9.8", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.8.tgz", + "integrity": "sha512-WRFq3Wn3WId7LLROfMLdH7xaFr2jR62wU8nLO6rQUOLOxNZUviyJQs1M0iIhLexSFy+L+w0ch66wtoO2jRjG0A==", "dev": true, "license": "MIT", "bin": { @@ -5250,30 +5509,30 @@ } }, "node_modules/react": { - "version": "19.2.8", - "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", - "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", + "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/react-dom": { - "version": "19.2.8", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", - "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", "license": "MIT", "dependencies": { - "scheduler": "^0.27.0" + "scheduler": "^0.28.0" }, "peerDependencies": { - "react": "^19.2.8" + "react": "^19.3.0" } }, "node_modules/react-error-boundary": { - "version": "6.1.5", - "resolved": "https://registry.npmjs.org/react-error-boundary/-/react-error-boundary-6.1.5.tgz", - "integrity": "sha512-l2nk+KhgM6e79tAiHEs3tXTdiTW4ARniKYMQN+cLY5vQFpDOkxuXKIC4uIwYuIKoMpFQ6jEbQz9wBArUmq7GWA==", + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/react-error-boundary/-/react-error-boundary-6.1.6.tgz", + "integrity": "sha512-CDXPnXDGyFIbkwaaJ6u+xgsRmJhSi6YdgUDW1vnyKHfXp1a9pfAlM+ZET2CDu80/A+8iRcmXN0NSY9BCCoWP5A==", "license": "MIT", "peerDependencies": { "@types/react": "^18.0.0 || ^19.0.0", @@ -5286,9 +5545,9 @@ } }, "node_modules/react-hook-form": { - "version": "7.87.0", - "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.87.0.tgz", - "integrity": "sha512-zhFzWvLxNHH+8839OnZcUxgMZw88ah2jZWDWvKWgF3Tpbnd0vKL+dlcuU3nZVWESZQjd81EW8K+wU+cYfYAc0w==", + "version": "7.88.0", + "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.88.0.tgz", + "integrity": "sha512-QRaLOWhX93YCnMiRfnOFRSwWXZNt8qhm2JTZwypoDvKpSffTJHmpzMXt8U6PV5UThvL3IiiLDUWe2nHMtz6Mmw==", "license": "MIT", "engines": { "node": ">=18.0.0" @@ -5412,13 +5671,19 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/remove-accents": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/remove-accents/-/remove-accents-0.5.0.tgz", + "integrity": "sha512-8g3/Otx1eJaVD12e31UbJj1YzdtVvzH85HV7t+9MJYk/u3XmkOUJ5Ys9wQrf9PCPK8+xn4ymzqYCiZl6QWKn+A==", + "license": "MIT" + }, "node_modules/rolldown": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.7.tgz", - "integrity": "sha512-g0EtLvBjTUB7jhyV0S/TCup3v/XSVl45vUIGbOGU4QPiyjTenCe4mKuFvW9fEgYmS2Fo42AUssRmNuMziXdrig==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.9.tgz", + "integrity": "sha512-hx/Pv0N1haXRb11qkfnK5MXB/iqr7i0yjWQqmO9uHqZpBgQSqzc8UsSnEpalsh+j1I8qQ2CkXAkJC8Br3dKSlg==", "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.148.0", + "@oxc-project/types": "=0.150.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -5428,21 +5693,21 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm-eabi": "1.2.7", - "@rolldown/binding-android-arm64": "1.2.7", - "@rolldown/binding-darwin-arm64": "1.2.7", - "@rolldown/binding-darwin-x64": "1.2.7", - "@rolldown/binding-freebsd-x64": "1.2.7", - "@rolldown/binding-linux-arm-gnueabihf": "1.2.7", - "@rolldown/binding-linux-arm64-gnu": "1.2.7", - "@rolldown/binding-linux-arm64-musl": "1.2.7", - "@rolldown/binding-linux-ppc64-gnu": "1.2.7", - "@rolldown/binding-linux-s390x-gnu": "1.2.7", - "@rolldown/binding-linux-x64-gnu": "1.2.7", - "@rolldown/binding-linux-x64-musl": "1.2.7", - "@rolldown/binding-openharmony-arm64": "1.2.7", - "@rolldown/binding-win32-arm64-msvc": "1.2.7", - "@rolldown/binding-win32-x64-msvc": "1.2.7" + "@rolldown/binding-android-arm-eabi": "1.2.9", + "@rolldown/binding-android-arm64": "1.2.9", + "@rolldown/binding-darwin-arm64": "1.2.9", + "@rolldown/binding-darwin-x64": "1.2.9", + "@rolldown/binding-freebsd-x64": "1.2.9", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.9", + "@rolldown/binding-linux-arm64-gnu": "1.2.9", + "@rolldown/binding-linux-arm64-musl": "1.2.9", + "@rolldown/binding-linux-ppc64-gnu": "1.2.9", + "@rolldown/binding-linux-s390x-gnu": "1.2.9", + "@rolldown/binding-linux-x64-gnu": "1.2.9", + "@rolldown/binding-linux-x64-musl": "1.2.9", + "@rolldown/binding-openharmony-arm64": "1.2.9", + "@rolldown/binding-win32-arm64-msvc": "1.2.9", + "@rolldown/binding-win32-x64-msvc": "1.2.9" } }, "node_modules/run-applescript": { @@ -5459,34 +5724,37 @@ } }, "node_modules/scheduler": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", - "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", "license": "MIT" }, "node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" } }, "node_modules/seroval": { - "version": "1.6.5", - "resolved": "https://registry.npmjs.org/seroval/-/seroval-1.6.5.tgz", - "integrity": "sha512-sNG8dL93a6zoDfB/+fw8d6Ovg01ydncHmPefPS54oS/3lk96EAFFWBcH79NrrHuP3Zzcb/w1lXZ5jfRabL2lHw==", + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/seroval/-/seroval-1.6.7.tgz", + "integrity": "sha512-AeDcLh0yO2SFm9W71essgnSzLV9DI8ZH0x0knXn2DMnUZj728mpLbxjlbB6IqKCmqh8JA3cEqRyGoNkt584JcQ==", "license": "MIT", "engines": { "node": ">=10" } }, "node_modules/seroval-plugins": { - "version": "1.6.5", - "resolved": "https://registry.npmjs.org/seroval-plugins/-/seroval-plugins-1.6.5.tgz", - "integrity": "sha512-YJIec8RyWI5ZZzZF9Sj2G1gM7p2ua4f8O2Fi/qYDwR7/fQd1dhEM6mBOl9s9DZ9hyJo5SN5jKRhtsV1pkV7Qng==", + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/seroval-plugins/-/seroval-plugins-1.6.7.tgz", + "integrity": "sha512-4Nk35ttD3DTDJW4hgw5StsVAPeU6qnDFnULAouw6tQ7oLTV/ICXrWpsXo2EE52eSP2joUMazbVf52mFEcADqRw==", "license": "MIT", "engines": { "node": ">=10" @@ -5502,6 +5770,79 @@ "dev": true, "license": "ISC" }, + "node_modules/solid-js": { + "version": "1.9.15", + "resolved": "https://registry.npmjs.org/solid-js/-/solid-js-1.9.15.tgz", + "integrity": "sha512-EeiY2xfpZJqPLjXspVEKjAII4yv8NyG//NxZ3IpOFHdUNnnTyL0uJOeS9LWGvA7cFCz5y94cjFwYlmw5Luncsg==", + "license": "MIT", + "dependencies": { + "csstype": "^3.1.0", + "seroval": "~1.5.4", + "seroval-plugins": "~1.5.4" + } + }, + "node_modules/solid-js/node_modules/seroval": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/seroval/-/seroval-1.5.6.tgz", + "integrity": "sha512-rVQVWjjSvlINzaQPZH5JFqsqEsIWdTxY3iJZCnTL/5gQbXIRooVZKI60tVCkOVfzcRPejboxO2t0P89dg5mQaA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/solid-js/node_modules/seroval-plugins": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/seroval-plugins/-/seroval-plugins-1.5.6.tgz", + "integrity": "sha512-HXuLAX2pu/UByPpaeo/TaMfvMIi+1QqIoPJYCcAtU8QkVNwgR6MPlGuCQTErV1JwraaMbYaWVIBX7mppzGLATQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "seroval": "^1.0" + } + }, + "node_modules/solid-presence": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/solid-presence/-/solid-presence-0.2.0.tgz", + "integrity": "sha512-YM92o+jvpzX3XGaD4rLYmq/Kc2ZVh47GSCLEufHBFQQIurvZTs8SoGJxO8BJGNDxBKdcS8F3dYhW1SDXp4BNjA==", + "license": "MIT", + "dependencies": { + "@corvu/utils": "~0.4.2" + }, + "peerDependencies": { + "solid-js": "^1.8" + } + }, + "node_modules/solid-prevent-scroll": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/solid-prevent-scroll/-/solid-prevent-scroll-0.1.11.tgz", + "integrity": "sha512-2PComVCDHaQN/5t7ogEqUYeHasritZKXZ8Sb/VLkl0Web8o9YhEwOo8LSujJEahvMlYCNyKt0zRJeHldOAeWZg==", + "license": "MIT", + "dependencies": { + "@corvu/utils": "~0.4.2" + }, + "peerDependencies": { + "solid-js": "^1.8" + } + }, + "node_modules/solid-transition-group": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/solid-transition-group/-/solid-transition-group-0.2.3.tgz", + "integrity": "sha512-iB72c9N5Kz9ykRqIXl0lQohOau4t0dhel9kjwFvx81UZJbVwaChMuBuyhiZmK24b8aKEK0w3uFM96ZxzcyZGdg==", + "license": "MIT", + "dependencies": { + "@solid-primitives/refs": "^1.0.5", + "@solid-primitives/transition-group": "^1.0.2" + }, + "engines": { + "node": ">=18.0.0", + "pnpm": ">=8.6.0" + }, + "peerDependencies": { + "solid-js": "^1.6.12" + } + }, "node_modules/sonner": { "version": "2.0.8", "resolved": "https://registry.npmjs.org/sonner/-/sonner-2.0.8.tgz", @@ -5551,10 +5892,22 @@ "dev": true, "license": "MIT" }, + "node_modules/superjson": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/superjson/-/superjson-2.2.6.tgz", + "integrity": "sha512-H+ue8Zo4vJmV2nRjpx86P35lzwDT3nItnIsocgumgr0hHMQ+ZGq5vrERg9kJBo5AWGmxZDhzDo+WVIJqkB0cGA==", + "license": "MIT", + "dependencies": { + "copy-anything": "^4" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/tailwind-merge": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.6.0.tgz", - "integrity": "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w==", + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.7.0.tgz", + "integrity": "sha512-XPPUyAc+cvspz3lHTcR/QgPfW2A0lv/xQNIjX3HGhLR+Nq2lHaLq5MtTesHn8GUr3W3DguT2KT5x3NVgRtYwmA==", "license": "MIT", "funding": { "type": "github", @@ -5580,32 +5933,6 @@ "url": "https://opencollective.com/webpack" } }, - "node_modules/tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", - "deprecated": "Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, - "license": "ISC", - "dependencies": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", - "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/tar/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, - "license": "ISC" - }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -5614,11 +5941,14 @@ "license": "MIT" }, "node_modules/tinyexec": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", - "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz", + "integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=18" + } }, "node_modules/tinyglobby": { "version": "0.2.17", @@ -5676,13 +6006,6 @@ "node": ">=14.17" } }, - "node_modules/ufo": { - "version": "1.6.4", - "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.4.tgz", - "integrity": "sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==", - "dev": true, - "license": "MIT" - }, "node_modules/uglify-js": { "version": "3.19.3", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", @@ -5705,14 +6028,14 @@ "license": "MIT" }, "node_modules/unplugin": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-3.3.0.tgz", - "integrity": "sha512-qa66K+crbfyE6JK10GjvbJeRrOsuC/JpbnHctfyp/i4oBTxWOzJfRZyDiOk1PtErMFRu8JhsU/wPvOdBNWe5Rg==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-3.4.0.tgz", + "integrity": "sha512-9skdIFlCsPdFV7wUfZxNsFInlW+7nJmGu2gkTu0OUhF56aXGsHab9x52/QhdJ4lC7ZDPWTxbiC4ANqVlsuaW3w==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/remapping": "^2.3.5", - "picomatch": "^4.0.4", + "picomatch": "^4.0.7", "webpack-virtual-modules": "^0.6.2" }, "engines": { @@ -5720,6 +6043,7 @@ }, "peerDependencies": { "@farmfe/core": "*", + "@rsbuild/core": "*", "@rspack/core": "*", "bun-types-no-globals": "*", "esbuild": "*", @@ -5733,6 +6057,9 @@ "@farmfe/core": { "optional": true }, + "@rsbuild/core": { + "optional": true + }, "@rspack/core": { "optional": true }, @@ -5760,9 +6087,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz", - "integrity": "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz", + "integrity": "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==", "dev": true, "funding": [ { @@ -5834,24 +6161,24 @@ } }, "node_modules/use-sync-external-store": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", - "integrity": "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.7.0.tgz", + "integrity": "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A==", "license": "MIT", "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "node_modules/vite": { - "version": "8.2.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", - "integrity": "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q==", + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.0.tgz", + "integrity": "sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ==", "license": "MIT", "dependencies": { "lightningcss": "^1.33.0", - "picomatch": "^4.0.5", - "postcss": "^8.5.26", - "rolldown": "~1.2.4", + "picomatch": "^4.0.7", + "postcss": "^8.5.28", + "rolldown": "~1.2.6", "tinyglobby": "^0.2.17" }, "bin": { @@ -5868,7 +6195,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.4.0 || ^0.5.0", + "@vitejs/devtools": "^0.7.1", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", @@ -5919,20 +6246,6 @@ } } }, - "node_modules/vite/node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/vite/node_modules/lightningcss": { "version": "1.33.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", @@ -6069,6 +6382,9 @@ "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6089,6 +6405,9 @@ "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6109,6 +6428,9 @@ "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6129,6 +6451,9 @@ "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6272,16 +6597,6 @@ } } }, - "node_modules/vitest/node_modules/tinyexec": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz", - "integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, "node_modules/webpack-virtual-modules": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/webpack-virtual-modules/-/webpack-virtual-modules-0.6.2.tgz", @@ -6321,9 +6636,9 @@ "license": "ISC" }, "node_modules/zod": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.5.4.tgz", - "integrity": "sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==", + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/package.json b/packages/tentacles/Services/Interfaces/node_web_interface/package.json index d841da7cba..bc1cf19591 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/package.json +++ b/packages/tentacles/Services/Interfaces/node_web_interface/package.json @@ -16,58 +16,57 @@ "@drakkar.software/octobot-client": "^0.8.1", "@drakkar.software/octochat-sdk": "^0.8.1", "@drakkar.software/starfish-identities": "3.0.0-alpha.72", - "@hookform/resolvers": "^5.2.2", - "@noble/curves": "2.2.0", - "@radix-ui/react-avatar": "^1.1.11", - "@radix-ui/react-checkbox": "^1.3.3", - "@radix-ui/react-dialog": "^1.1.15", - "@radix-ui/react-dropdown-menu": "^2.1.16", - "@radix-ui/react-label": "^2.1.8", - "@radix-ui/react-radio-group": "^1.3.8", - "@radix-ui/react-scroll-area": "^1.2.10", - "@radix-ui/react-select": "^2.2.6", - "@radix-ui/react-separator": "^1.1.8", - "@radix-ui/react-slot": "^1.2.4", - "@radix-ui/react-tabs": "^1.1.13", - "@radix-ui/react-tooltip": "^1.2.8", - "@tailwindcss/vite": "^4.2.4", - "@tanstack/react-query": "^5.100.6", - "@tanstack/react-query-devtools": "^5.100.6", - "@tanstack/react-router": "^1.168.26", - "@tanstack/react-router-devtools": "^1.166.13", + "@hookform/resolvers": "^5.9.1", + "@noble/curves": "2.4.0", + "@radix-ui/react-avatar": "^1.2.6", + "@radix-ui/react-checkbox": "^1.3.11", + "@radix-ui/react-dialog": "^1.1.23", + "@radix-ui/react-dropdown-menu": "^2.1.24", + "@radix-ui/react-label": "^2.1.15", + "@radix-ui/react-radio-group": "^1.4.7", + "@radix-ui/react-scroll-area": "^1.2.18", + "@radix-ui/react-select": "^2.3.7", + "@radix-ui/react-separator": "^1.1.15", + "@radix-ui/react-slot": "^1.3.3", + "@radix-ui/react-tabs": "^1.1.21", + "@radix-ui/react-tooltip": "^1.2.16", + "@tailwindcss/vite": "^4.3.3", + "@tanstack/react-query": "^5.103.2", + "@tanstack/react-query-devtools": "^5.103.2", + "@tanstack/react-router": "^1.170.38", + "@tanstack/react-router-devtools": "^1.167.2", "@tanstack/react-table": "^8.21.3", - "axios": "1.15.2", + "axios": "1.20.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "form-data": "4.0.5", - "lucide-react": "^1.14.0", + "form-data": "4.0.6", + "lucide-react": "^1.47.0", "next-themes": "^0.4.6", - "react": "^19.2.5", - "react-dom": "^19.2.5", - "react-error-boundary": "^6.1.1", - "react-hook-form": "^7.74.0", - "react-icons": "^5.6.0", - "react-qr-code": "^2.0.21", - "sonner": "^2.0.7", - "tailwind-merge": "^3.5.0", - "tailwindcss": "^4.2.4", - "zod": "^4.4.1" + "react": "^19.3.0", + "react-dom": "^19.3.0", + "react-error-boundary": "^6.1.6", + "react-hook-form": "^7.88.0", + "react-icons": "^5.7.0", + "react-qr-code": "^2.2.0", + "sonner": "^2.0.8", + "tailwind-merge": "^3.7.0", + "tailwindcss": "^4.3.3", + "zod": "^4.6.5" }, "devDependencies": { - "@biomejs/biome": "^2.4.13", - "@hey-api/openapi-ts": "0.73.0", - "@playwright/test": "1.59.1", - "@tanstack/router-devtools": "^1.166.13", - "@tanstack/router-plugin": "^1.167.29", - "@types/node": "^25.6.0", - "@types/react": "^19.2.14", - "@types/react-dom": "^19.2.3", - "@vitejs/plugin-react-swc": "^4.3.0", + "@biomejs/biome": "^2.5.14", + "@hey-api/openapi-ts": "0.85.0", + "@playwright/test": "1.63.0", + "@tanstack/router-plugin": "^1.168.40", + "@types/node": "^25.9.8", + "@types/react": "^19.3.0", + "@types/react-dom": "^19.3.0", + "@vitejs/plugin-react-swc": "^4.3.3", "dotenv": "^17.4.2", "fake-indexeddb": "^6.2.5", "tw-animate-css": "^1.4.0", "typescript": "^5.9.3", - "vite": "^8.0.10", - "vitest": "^4.1.5" + "vite": "^8.3.0", + "vitest": "^4.1.11" } } diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/__tests__/bootstrap-app.test.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/__tests__/bootstrap-app.test.tsx new file mode 100644 index 0000000000..30a28b4eff --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/__tests__/bootstrap-app.test.tsx @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" + +const mocks = vi.hoisted(() => ({ + createRouter: vi.fn(() => ({ id: "router-mock" })), + isWebCryptoAvailable: vi.fn(() => true), + probeAuthState: vi.fn(async () => "ok" as const), + reportInsecureContext: vi.fn(), + render: vi.fn(), +})) + +vi.mock("react-dom/client", () => ({ + createRoot: () => ({ + render: mocks.render, + }), +})) + +vi.mock("@tanstack/react-router", () => ({ + createRouter: mocks.createRouter, + RouterProvider: () => null, +})) + +vi.mock("@/routeTree.gen", () => ({ + routeTree: {}, +})) + +vi.mock("@/lib/secure-context", () => ({ + isWebCryptoAvailable: mocks.isWebCryptoAvailable, +})) + +vi.mock("@/lib/auth-state-probe", () => ({ + probeAuthState: mocks.probeAuthState, +})) + +vi.mock("@/lib/shell-error-reporting", () => ({ + reportInsecureContext: mocks.reportInsecureContext, + reportAuthStateBroken: vi.fn(), + reportShellFatalError: vi.fn(), +})) + +vi.mock("@/lib/device-key", () => ({ + loadPassword: vi.fn(async () => null), +})) + +import { bootstrapApp } from "@/bootstrap-app" + +describe("bootstrapApp insecure context", () => { + beforeEach(() => { + mocks.createRouter.mockClear() + mocks.isWebCryptoAvailable.mockReturnValue(true) + mocks.probeAuthState.mockResolvedValue("ok") + mocks.reportInsecureContext.mockClear() + mocks.render.mockClear() + vi.stubGlobal("document", { + getElementById: vi.fn(() => ({ id: "root" })), + }) + vi.stubGlobal("window", { + isSecureContext: false, + location: { hostname: "192.168.0.1", pathname: "/app", port: "8000" }, + }) + }) + + it("renders recovery and skips router when web crypto is unavailable", async () => { + mocks.isWebCryptoAvailable.mockReturnValue(false) + + await bootstrapApp() + + expect(mocks.reportInsecureContext).toHaveBeenCalledOnce() + expect(mocks.render).toHaveBeenCalledOnce() + expect(mocks.createRouter).not.toHaveBeenCalled() + expect(mocks.probeAuthState).not.toHaveBeenCalled() + }) +}) diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/__tests__/utils.test.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/__tests__/utils.test.ts new file mode 100644 index 0000000000..09933966be --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/__tests__/utils.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest" + +import { ApiError } from "@/client" +import { extractErrorMessage } from "@/utils" + +describe("extractErrorMessage", () => { + it("returns message from structured auth error detail", () => { + const err = new ApiError( + { method: "GET", url: "/api/v1/login/test" }, + { + url: "/api/v1/login/test", + ok: false, + status: 401, + statusText: "Unauthorized", + body: { + detail: { + code: "auth_invalid_passphrase", + message: "Passphrase verification failed", + }, + }, + }, + "Unauthorized", + ) + expect(extractErrorMessage(err)).toBe("Passphrase verification failed") + }) + + it("returns string detail from FastAPI error body", () => { + const err = new ApiError( + { method: "POST", url: "/api/v1/setup/wallet/recover-from-seed" }, + { + url: "/api/v1/setup/wallet/recover-from-seed", + ok: false, + status: 422, + statusText: "Unprocessable Content", + body: { detail: "Invalid seed phrase or private key" }, + }, + "Invalid seed phrase or private key", + ) + expect(extractErrorMessage(err)).toBe("Invalid seed phrase or private key") + }) + + it("returns plain string body when detail is absent", () => { + const err = new ApiError( + { method: "POST", url: "/x" }, + { + url: "/x", + ok: false, + status: 500, + statusText: "Error", + body: "Server failure text", + }, + "Error", + ) + expect(extractErrorMessage(err)).toBe("Server failure text") + }) + + it("falls back to ApiError message when body has no detail", () => { + const err = new ApiError( + { method: "POST", url: "/x" }, + { + url: "/x", + ok: false, + status: 500, + statusText: "Internal Server Error", + body: {}, + }, + "Custom error message", + ) + expect(extractErrorMessage(err)).toBe("Custom error message") + }) +}) diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/bootstrap-app.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/bootstrap-app.tsx new file mode 100644 index 0000000000..554af8569d --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/bootstrap-app.tsx @@ -0,0 +1,152 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query" +import { createRouter, RouterProvider } from "@tanstack/react-router" +import { StrictMode } from "react" +import { createRoot } from "react-dom/client" +import { ErrorBoundary } from "react-error-boundary" +import { OpenAPI } from "@/client" +import { RecoveryScreen } from "@/components/Common/RecoveryScreen" +import { ThemeProvider } from "@/components/theme-provider" +import { Toaster } from "@/components/ui/sonner" +import { clearAuth } from "@/hooks/useAuth" +import { probeAuthState } from "@/lib/auth-state-probe" +import { shouldRedirectToLoginOn401 } from "@/lib/open-api-401-login-redirect" +import { loadPassword } from "@/lib/device-key" +import { markLoginSessionCleared } from "@/lib/login-session-hint" +import type { RecoveryFailureKind } from "@/components/Common/RecoveryScreen" +import { isWebCryptoAvailable } from "@/lib/secure-context" +import { + reportAuthStateBroken, + reportInsecureContext, + reportShellFatalError, +} from "@/lib/shell-error-reporting" +import { routeTree } from "@/routeTree.gen" + +// Wire generated API client credentials and session recovery before any route loads. +export function configureOpenApi(): void { + OpenAPI.BASE = + import.meta.env.NODE_API_URL || + (import.meta.env.DEV ? "http://localhost:8000" : "") + OpenAPI.USERNAME = async () => { + return localStorage.getItem("auth_username") || "" + } + OpenAPI.PASSWORD = async () => { + return (await loadPassword()) ?? "" + } + + let isRedirectingOnAuthFailure = false + // On 401 for a protected page, clear local session and hard-navigate to login so the UI + // never keeps calling APIs with a stale username and no usable password. + OpenAPI.interceptors.response.use((response) => { + if ( + shouldRedirectToLoginOn401(response, { + pathname: window.location.pathname, + isRedirectingOnAuthFailure, + }) + ) { + isRedirectingOnAuthFailure = true + markLoginSessionCleared() + void clearAuth().finally(() => { + window.location.href = "/app/login" + }) + } + return response + }) +} + +function createAppRouter() { + return createRouter({ + routeTree, + basepath: "/app", + }) +} + +type RecoveryView = + | { mode: "recovery"; failureKind: RecoveryFailureKind } + | { mode: "app" } + +async function resolveStartupView(): Promise { + // probeAuthState: skipped during setup; ok when logged out or password present; + // broken when auth_username exists but IndexedDB password is missing/unreadable. + const authState = await probeAuthState() + if (authState === "broken") { + reportAuthStateBroken() + return { mode: "recovery", failureKind: "auth_broken" } + } + return { mode: "app" } +} + +function renderRecovery( + rootElement: HTMLElement, + failureKind: RecoveryFailureKind, +): void { + const queryClient = new QueryClient() + + createRoot(rootElement).render( + + + + + + + + , + ) +} + +// Shown when a render error escapes the main app tree (ErrorBoundary fallback). +export function ShellErrorFallback() { + return +} + +export function handleShellRenderError(error: unknown): void { + reportShellFatalError( + error instanceof Error ? error : new Error(String(error)), + ) +} + +function renderApp(rootElement: HTMLElement): void { + const queryClient = new QueryClient() + const router = createAppRouter() + + createRoot(rootElement).render( + + + + + + + + + + , + ) +} + +// Startup: configure API → mount root → require Web Crypto → auth probe → app or RecoveryScreen. +export async function bootstrapApp(): Promise { + configureOpenApi() + const rootElement = document.getElementById("root") + if (!rootElement) { + throw new Error("Root element not found") + } + + if (!isWebCryptoAvailable()) { + // Non-secure context (e.g. HTTP remote UI): crypto.subtle unavailable → insecure_context recovery. + reportInsecureContext({ + isSecureContext: window.isSecureContext, + hostname: window.location.hostname, + }) + renderRecovery(rootElement, "insecure_context") + return + } + + const startupView = await resolveStartupView() + if (startupView.mode === "recovery") { + renderRecovery(rootElement, startupView.failureKind) + return + } + renderApp(rootElement) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/client/core/CancelablePromise.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/client/core/CancelablePromise.ts index ccc082e8f2..d95c755e0a 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/client/core/CancelablePromise.ts +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/client/core/CancelablePromise.ts @@ -80,7 +80,7 @@ export class CancelablePromise implements Promise { }); } - get [Symbol.toStringTag]() { + get [Symbol.toStringTag](): string { return "Cancellable Promise"; } diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/AppHeader.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/AppHeader.tsx index 84b39c6e18..fb3b6bfbc4 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/AppHeader.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/AppHeader.tsx @@ -2,6 +2,7 @@ import { Link } from "@tanstack/react-router" import { Plus } from "lucide-react" import { Logo } from "@/components/Common/Logo" +import { ShareFeedbackButton } from "@/components/Common/ShareFeedbackButton" import UserMenu from "@/components/Common/UserMenu" import { Button } from "@/components/ui/button" @@ -11,6 +12,11 @@ export function AppHeader() {
+ - +
+ + + + +
) } diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextGuidance.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextGuidance.tsx new file mode 100644 index 0000000000..2a16075a40 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextGuidance.tsx @@ -0,0 +1,83 @@ +import { buildSameComputerLoopbackUrl } from "@/lib/secure-context" +import { OCTOBOT_CONNECT_GUIDE_TWO_INTERFACES_URL } from "@/lib/external-links" +import { + INSECURE_CONTEXT_OPTION_LOCAL_SUBLABEL, + INSECURE_CONTEXT_OPTION_LOCAL_TITLE, + INSECURE_CONTEXT_OPTION_REMOTE_TITLE, + INSECURE_CONTEXT_REMOTE_LEAD, + INSECURE_CONTEXT_SAME_COMPUTER_LEAD, + INSECURE_CONTEXT_TWO_INTERFACES_GUIDE_BEFORE, + INSECURE_CONTEXT_TWO_INTERFACES_GUIDE_LINK, + INSECURE_CONTEXT_WHY_ADDRESS_AFTER, + INSECURE_CONTEXT_WHY_ADDRESS_BEFORE, + INSECURE_CONTEXT_WHY_LEAD, + TAILSCALE_REMOTE_ACCESS_BUTTON_LABEL, +} from "@/lib/ui-recovery-constants" +import { Button } from "@/components/ui/button" + +type InsecureContextGuidanceProps = { + onSetUpRemoteAccessClick: () => void +} + +export function InsecureContextGuidance({ + onSetUpRemoteAccessClick, +}: InsecureContextGuidanceProps) { + const pageHref = + typeof window !== "undefined" ? window.location.href : "" + const loopbackUrl = + typeof window !== "undefined" + ? buildSameComputerLoopbackUrl(window.location.href) + : "http://127.0.0.1:8000/app" + + return ( +
+
+

{INSECURE_CONTEXT_WHY_LEAD}

+ {pageHref !== "" ? ( + <> +

+ {INSECURE_CONTEXT_WHY_ADDRESS_BEFORE} + {pageHref} + {INSECURE_CONTEXT_WHY_ADDRESS_AFTER} +

+

+ {INSECURE_CONTEXT_TWO_INTERFACES_GUIDE_BEFORE} + + {INSECURE_CONTEXT_TWO_INTERFACES_GUIDE_LINK} + + . +

+ + ) : null} +
+
+

{INSECURE_CONTEXT_OPTION_LOCAL_TITLE}

+

{INSECURE_CONTEXT_OPTION_LOCAL_SUBLABEL}

+

{INSECURE_CONTEXT_SAME_COMPUTER_LEAD}

+

+ + {loopbackUrl} + +

+
+
+

{INSECURE_CONTEXT_OPTION_REMOTE_TITLE}

+

{INSECURE_CONTEXT_REMOTE_LEAD}

+ +
+
+ ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextNotice.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextNotice.tsx deleted file mode 100644 index 0e954c740d..0000000000 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextNotice.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import { getLoopbackUrl } from "@/lib/secure-context" - -const InsecureContextNotice = () => { - const loopbackUrl = getLoopbackUrl(window.location.href) - - return ( -
-
- - 🔒 - -

Secure context required

-

- This browser origin does not support the cryptography the node needs. - Open the node UI from a secure address instead. -

- {loopbackUrl ? ( -

- - {loopbackUrl} - -

- ) : ( -

- On this machine, open{" "} - - http://127.0.0.1:{window.location.port || "8000"}/app - - . To reach the node from another device, serve it over HTTPS. -

- )} -
-
- ) -} - -export default InsecureContextNotice diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/LoginAuthErrorDisplay.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/LoginAuthErrorDisplay.tsx new file mode 100644 index 0000000000..d1e9ba6371 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/LoginAuthErrorDisplay.tsx @@ -0,0 +1,68 @@ +import type { AuthErrorPresentation } from "@/lib/auth-error-messages" + +export const LOGIN_AUTH_ERROR_SUMMARY_ID = "login-auth-error-summary" +export const LOGIN_AUTH_ERROR_TIPS_ID = "login-auth-error-tips" + +function tipsHeading(guidanceCount: number): string { + return guidanceCount === 1 ? "Tip" : "What you can try" +} + +type LoginAuthErrorDisplayProps = { + presentation: AuthErrorPresentation | null +} + +export function loginAuthFieldDescribedBy( + presentation: AuthErrorPresentation | null, +): string | undefined { + if (!presentation) { + return undefined + } + const ids = [LOGIN_AUTH_ERROR_SUMMARY_ID] + if (presentation.guidance.length > 0) { + ids.push(LOGIN_AUTH_ERROR_TIPS_ID) + } + return ids.join(" ") +} + +export function LoginAuthErrorDisplay({ + presentation, +}: LoginAuthErrorDisplayProps) { + if (!presentation) { + return null + } + + return ( +
+ + {presentation.guidance.length > 0 ? ( +
+

+ {tipsHeading(presentation.guidance.length)} +

+
    + {presentation.guidance.map((item) => ( +
  • {item}
    • + ))} +
+
+ ) : null} +
+ ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/MagicDnsRemoteUrlField.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/MagicDnsRemoteUrlField.tsx new file mode 100644 index 0000000000..6e11ee9c5f --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/MagicDnsRemoteUrlField.tsx @@ -0,0 +1,77 @@ +import { SelectableTextBlock } from "@/components/Common/SelectableTextBlock" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" +import { + buildHttpsNodeUrlFromHostname, + parseMagicDnsHostnameInput, +} from "@/lib/secure-context" +import { + MAGIC_DNS_INPUT_LABEL, + MAGIC_DNS_INPUT_PLACEHOLDER, + MAGIC_DNS_OPEN_IN_BROWSER_LABEL, + MAGIC_DNS_REMOTE_URL_LABEL, + TAILSCALE_AVOID_IP_HTTPS_WARNING, +} from "@/lib/ui-recovery-constants" + +type MagicDnsRemoteUrlFieldProps = { + pageHref: string + value: string + onValueChange: (value: string) => void +} + +const REMOTE_URL_ARIA_LABEL = "Remote HTTPS URL" + +export function MagicDnsRemoteUrlField({ + pageHref, + value, + onValueChange, +}: MagicDnsRemoteUrlFieldProps) { + const trimmed = value.trim() + const parsed = + trimmed === "" ? null : parseMagicDnsHostnameInput(value) + const remoteUrl = + parsed?.ok === true + ? buildHttpsNodeUrlFromHostname(parsed.hostname, pageHref) + : null + const errorMessage = + trimmed !== "" && parsed?.ok === false ? parsed.message : null + + return ( +
+
+ + onValueChange(event.target.value)} + autoComplete="off" + /> + {errorMessage ? ( +

{errorMessage}

+ ) : null} +
+ {remoteUrl ? ( +
+ {MAGIC_DNS_REMOTE_URL_LABEL} + + + {MAGIC_DNS_OPEN_IN_BROWSER_LABEL} + +

+ {TAILSCALE_AVOID_IP_HTTPS_WARNING} +

+
+ ) : null} +
+ ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/RecoveryScreen.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/RecoveryScreen.tsx new file mode 100644 index 0000000000..cd3b25ea38 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/RecoveryScreen.tsx @@ -0,0 +1,97 @@ +import { useState } from "react" + +import { InsecureContextGuidance } from "@/components/Common/InsecureContextGuidance" +import { ShareFeedbackButton } from "@/components/Common/ShareFeedbackButton" +import { TailscaleRemoteAccessDialog } from "@/components/Common/TailscaleRemoteAccessDialog" +import { resetClientStorage } from "@/lib/client-storage-reset" +import { + INSECURE_CONTEXT_HEADING, + RECOVERY_EXPLANATION, + RESET_CONFIRM_MESSAGE, +} from "@/lib/ui-recovery-constants" +import { Button } from "@/components/ui/button" + +export type RecoveryFailureKind = + | "boot_failed" + | "auth_broken" + | "fatal_render" + | "insecure_context" + +const FAILURE_HEADINGS: Record = { + boot_failed: "OctoBot Node failed to start", + auth_broken: "Sign-in data is inconsistent", + fatal_render: "OctoBot Node encountered a fatal error", + insecure_context: INSECURE_CONTEXT_HEADING, +} + +type RecoveryScreenProps = { + failureKind: RecoveryFailureKind +} + +export function confirmAndResetLocalBrowserData( + confirmFn: () => boolean = () => window.confirm(RESET_CONFIRM_MESSAGE), +): void { + if (!confirmFn()) { + return + } + void resetClientStorage("manual_recovery") +} + +export function RecoveryScreen({ failureKind }: RecoveryScreenProps) { + const [tailscaleDialogOpen, setTailscaleDialogOpen] = useState(false) + + const handleResetClick = () => { + confirmAndResetLocalBrowserData() + } + + const handleReloadClick = () => { + window.location.reload() + } + + const isInsecureContext = failureKind === "insecure_context" + const explanation = isInsecureContext ? null : RECOVERY_EXPLANATION + const feedbackContext = { source: "recovery" as const, failureKind } + + return ( +
+
+

{FAILURE_HEADINGS[failureKind]}

+ {isInsecureContext ? ( + <> + setTailscaleDialogOpen(true)} + /> +
+ +
+ + ) : ( +

{explanation}

+ )} + {!isInsecureContext ? ( +
+ + + +
+ ) : null} +
+ {isInsecureContext ? ( + + ) : null} +
+ ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/SelectableTextBlock.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/SelectableTextBlock.tsx new file mode 100644 index 0000000000..85f625412e --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/SelectableTextBlock.tsx @@ -0,0 +1,60 @@ +import type { FocusEvent, MouseEvent } from "react" + +import { cn } from "@/lib/utils" + +type SelectableTextBlockProps = { + value: string + ariaLabel: string + multiline?: boolean +} + +const selectableControlClassName = cn( + "w-full cursor-text border-0 bg-transparent p-0 font-mono text-xs text-foreground", + "break-all outline-none focus-visible:ring-0", +) + +function selectAllInControl( + event: FocusEvent, +): void { + event.currentTarget.select() +} + +function selectAllInControlOnClick( + event: MouseEvent, +): void { + event.currentTarget.select() +} + +export function SelectableTextBlock({ + value, + ariaLabel, + multiline = false, +}: SelectableTextBlockProps) { + const textareaRows = Math.max(2, value.split("\n").length) + + return ( +
+ {multiline ? ( +