From 0d49fea6f17a6d514f7f5ce6087e5eae41fd41a2 Mon Sep 17 00:00:00 2001 From: Adam Gohain Date: Tue, 21 Jul 2026 10:02:27 -0400 Subject: [PATCH 1/6] Harden workflow runtime and volume access --- client/src/App.css | 13 + client/src/api.js | 41 +- client/src/components/AppErrorBoundary.js | 81 +++ .../src/components/AppErrorBoundary.test.js | 51 ++ client/src/errors/apiError.js | 170 +++++++ client/src/errors/apiError.test.js | 76 +++ client/src/index.js | 5 +- docs/platform-hardening-branch-scope.md | 104 ++++ pyproject.toml | 1 + server_api/chatbot/logging_utils.py | 15 +- server_api/ehtool/data_manager.py | 6 +- server_api/errors.py | 194 ++++++++ server_api/main.py | 333 +++++++++++-- server_api/workflows/agent_actions.py | 356 ++++++++++++++ server_api/workflows/db_models.py | 68 +++ server_api/workflows/operation_router.py | 273 +++++++++++ server_api/workflows/operation_service.py | 398 +++++++++++++++ server_api/workflows/router.py | 119 ++--- server_api/workflows/volume_io.py | 463 +++++++++++------- tests/test_agent_action_registry.py | 99 ++++ tests/test_error_contract.py | 88 ++++ tests/test_neuroglancer_storage_sources.py | 114 +++++ tests/test_neuroglancer_url_contract.py | 31 ++ tests/test_pytc_runtime_routes.py | 110 ++++- tests/test_volume_io.py | 135 ++++- tests/test_workflow_operations.py | 240 +++++++++ uv.lock | 2 + 27 files changed, 3226 insertions(+), 360 deletions(-) create mode 100644 client/src/components/AppErrorBoundary.js create mode 100644 client/src/components/AppErrorBoundary.test.js create mode 100644 client/src/errors/apiError.js create mode 100644 client/src/errors/apiError.test.js create mode 100644 docs/platform-hardening-branch-scope.md create mode 100644 server_api/errors.py create mode 100644 server_api/workflows/agent_actions.py create mode 100644 server_api/workflows/operation_router.py create mode 100644 server_api/workflows/operation_service.py create mode 100644 tests/test_agent_action_registry.py create mode 100644 tests/test_error_contract.py create mode 100644 tests/test_neuroglancer_storage_sources.py create mode 100644 tests/test_workflow_operations.py diff --git a/client/src/App.css b/client/src/App.css index 336d8ef9..4a30272d 100644 --- a/client/src/App.css +++ b/client/src/App.css @@ -2,6 +2,19 @@ text-align: center; } +.app-error-boundary { + align-items: center; + background: var(--seg-bg-canvas, #f7f4ed); + display: flex; + justify-content: center; + min-height: 100vh; + padding: 24px; +} + +.app-error-boundary .ant-result { + max-width: 680px; +} + :root { --seg-bg-canvas: #f7f4ed; --seg-bg-panel: #fffdfa; diff --git a/client/src/api.js b/client/src/api.js index f3010b37..5ff9c967 100644 --- a/client/src/api.js +++ b/client/src/api.js @@ -1,6 +1,7 @@ import axios from "axios"; import yaml from "js-yaml"; import { logClientEvent } from "./logging/appEventLog"; +import { attachApiError, normalizeApiError } from "./errors/apiError"; import { detectConfigDiagnostics, summarizeConfigText, @@ -190,6 +191,7 @@ const attachApiLogging = (instance, source) => { return response; }, (error) => { + attachApiError(error); const config = error.config || {}; const startedAt = config.metadata?.startedAt; const endedAt = @@ -209,6 +211,8 @@ const attachApiLogging = (instance, source) => { ? Number((endedAt - startedAt).toFixed(2)) : null, detail: error.response?.data?.detail || null, + errorCode: error.apiError?.code || null, + requestId: error.apiError?.requestId || null, }, }); return Promise.reject(error); @@ -232,33 +236,6 @@ const buildFilePath = (file) => { const hasBrowserFile = (file) => file && file.originFileObj instanceof File; -const getErrorDetailMessage = (detail) => { - if (!detail) return ""; - if (typeof detail === "string") return detail; - if (Array.isArray(detail)) { - return detail.map(getErrorDetailMessage).filter(Boolean).join("; "); - } - if (typeof detail === "object") { - if (detail.user_message) { - return getErrorDetailMessage(detail.user_message); - } - const nestedUpstream = - detail.upstream_body !== undefined - ? getErrorDetailMessage(detail.upstream_body) - : ""; - return [ - detail.message, - detail.detail, - detail.reason, - nestedUpstream, - detail.error, - ] - .filter(Boolean) - .join(" | "); - } - return String(detail); -}; - export async function getNeuroglancerViewer( image, label, @@ -392,13 +369,11 @@ export async function checkFile(file) { function handleError(error) { if (error.response) { - const detail = error.response.data?.detail; - const detailMessage = getErrorDetailMessage(detail); - throw new Error( - `${error.response.status}: ${detailMessage || error.response.statusText}`, - ); + const apiError = normalizeApiError(error); + error.apiError = apiError; + error.message = `${error.response.status}: ${apiError.message}`; } - throw error; + throw attachApiError(error); } export async function makeApiRequest(url, method, data = null) { diff --git a/client/src/components/AppErrorBoundary.js b/client/src/components/AppErrorBoundary.js new file mode 100644 index 00000000..daa0a742 --- /dev/null +++ b/client/src/components/AppErrorBoundary.js @@ -0,0 +1,81 @@ +import React from "react"; +import { ArrowLeftOutlined, ReloadOutlined } from "@ant-design/icons"; +import { Button, Result, Space, Typography } from "antd"; +import { logClientEvent } from "../logging/appEventLog"; + +const { Text } = Typography; + +class AppErrorBoundary extends React.Component { + constructor(props) { + super(props); + this.state = { error: null, errorId: null }; + } + + static getDerivedStateFromError(error) { + return { + error, + errorId: `ui-${Date.now().toString(36)}`, + }; + } + + componentDidCatch(error, info) { + logClientEvent("ui_render_failed", { + level: "ERROR", + message: error?.message || "The application failed to render", + source: "AppErrorBoundary", + data: { + errorId: this.state.errorId, + errorName: error?.name, + componentStack: info?.componentStack, + }, + }); + } + + retry = () => { + this.setState({ error: null, errorId: null }); + }; + + reload = () => { + window.location.reload(); + }; + + goBack = () => { + window.history.back(); + }; + + render() { + const { error, errorId } = this.state; + if (!error) return this.props.children; + + return ( +
+ + + + + + } + > + Error reference: {errorId} + +
+ ); + } +} + +export default AppErrorBoundary; diff --git a/client/src/components/AppErrorBoundary.test.js b/client/src/components/AppErrorBoundary.test.js new file mode 100644 index 00000000..38f193e1 --- /dev/null +++ b/client/src/components/AppErrorBoundary.test.js @@ -0,0 +1,51 @@ +import React from "react"; +import "@testing-library/jest-dom"; +import { fireEvent, render, screen } from "@testing-library/react"; +import AppErrorBoundary from "./AppErrorBoundary"; +import { logClientEvent } from "../logging/appEventLog"; + +jest.mock("../logging/appEventLog", () => ({ logClientEvent: jest.fn() })); + +const Broken = ({ broken }) => { + if (broken) throw new Error("render failed"); + return
Recovered content
; +}; + +describe("AppErrorBoundary", () => { + let consoleError; + + beforeEach(() => { + consoleError = jest.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + consoleError.mockRestore(); + jest.clearAllMocks(); + }); + + it("shows recovery actions and records render failures", () => { + const { rerender } = render( + + + , + ); + + expect(screen.getByRole("alert")).toHaveTextContent( + "This screen could not be displayed", + ); + expect(screen.getByText(/Error reference: ui-/)).toBeInTheDocument(); + expect(logClientEvent).toHaveBeenCalledWith( + "ui_render_failed", + expect.objectContaining({ source: "AppErrorBoundary" }), + ); + + rerender( + + + , + ); + fireEvent.click(screen.getByRole("button", { name: /try again/i })); + + expect(screen.getByText("Recovered content")).toBeInTheDocument(); + }); +}); diff --git a/client/src/errors/apiError.js b/client/src/errors/apiError.js new file mode 100644 index 00000000..2019a183 --- /dev/null +++ b/client/src/errors/apiError.js @@ -0,0 +1,170 @@ +const DEFAULT_ERROR = { + code: "request_failed", + category: "request", + title: "Request failed", + message: "The request could not be completed.", + retryable: false, + recoveryActions: [], + validationErrors: [], +}; + +const messageFromDetail = (detail) => { + if (!detail) return ""; + if (typeof detail === "string") return detail; + if (Array.isArray(detail)) { + return detail.map(messageFromDetail).filter(Boolean).join("; "); + } + if (typeof detail === "object") { + if (detail.user_message) return messageFromDetail(detail.user_message); + return [ + detail.message, + detail.detail, + detail.reason, + detail.upstream_body, + detail.error, + ] + .map(messageFromDetail) + .filter(Boolean) + .join(" | "); + } + return String(detail); +}; + +const statusDefaults = (status) => { + if (status === 401) { + return { + code: "authentication_required", + category: "authentication", + title: "Authentication required", + recoveryActions: ["sign_in"], + }; + } + if (status === 403) { + return { + code: "permission_denied", + category: "authorization", + title: "Permission denied", + recoveryActions: ["contact_admin"], + }; + } + if (status === 404) { + return { + code: "not_found", + category: "resource", + title: "Resource not found", + recoveryActions: ["go_back"], + }; + } + if (status === 409) { + return { + code: "conflict", + category: "state", + title: "State conflict", + recoveryActions: ["refresh"], + }; + } + if (status === 422) { + return { + code: "validation_failed", + category: "validation", + title: "Some request values are invalid", + recoveryActions: ["review_input"], + }; + } + if (status === 429 || status >= 500) { + return { + code: status === 429 ? "rate_limited" : "service_unavailable", + category: "availability", + title: + status === 429 + ? "Too many requests" + : "Service temporarily unavailable", + retryable: true, + recoveryActions: ["retry"], + }; + } + return {}; +}; + +export const normalizeApiError = (error) => { + if (error?.apiError) return error.apiError; + + const response = error?.response; + const status = response?.status || null; + const payload = response?.data || {}; + const contract = payload?.error || {}; + const isCanceled = + error?.code === "ERR_CANCELED" || error?.name === "CanceledError"; + const isTimeout = error?.code === "ECONNABORTED"; + const isNetworkError = !response && !isCanceled && !isTimeout; + + let transportDefaults = {}; + if (isCanceled) { + transportDefaults = { + code: "request_canceled", + category: "cancellation", + title: "Request canceled", + message: "The operation was canceled.", + }; + } else if (isTimeout) { + transportDefaults = { + code: "request_timeout", + category: "availability", + title: "Request timed out", + message: "The server did not respond in time.", + retryable: true, + recoveryActions: ["retry"], + }; + } else if (isNetworkError) { + transportDefaults = { + code: "network_unavailable", + category: "availability", + title: "Cannot reach the server", + message: "Check the server connection and try again.", + retryable: true, + recoveryActions: ["retry"], + }; + } + + const defaults = { + ...DEFAULT_ERROR, + ...statusDefaults(status), + ...transportDefaults, + }; + const responseRequestId = + response?.headers?.["x-request-id"] || + response?.headers?.get?.("x-request-id") || + null; + + return { + schemaVersion: contract.schema_version || null, + code: contract.code || defaults.code, + category: contract.category || defaults.category, + title: contract.title || defaults.title, + message: + contract.message || + messageFromDetail(payload?.detail) || + transportDefaults.message || + error?.message || + defaults.message, + status, + retryable: + typeof contract.retryable === "boolean" + ? contract.retryable + : Boolean(defaults.retryable), + requestId: contract.request_id || responseRequestId, + recoveryActions: + contract.recovery_actions || defaults.recoveryActions || [], + validationErrors: contract.validation_errors || [], + }; +}; + +export const attachApiError = (error) => { + if (error && typeof error === "object") { + error.apiError = normalizeApiError(error); + } + return error; +}; + +export const getApiErrorMessage = (error, fallback = DEFAULT_ERROR.message) => + normalizeApiError(error)?.message || fallback; diff --git a/client/src/errors/apiError.test.js b/client/src/errors/apiError.test.js new file mode 100644 index 00000000..b4c7362a --- /dev/null +++ b/client/src/errors/apiError.test.js @@ -0,0 +1,76 @@ +import { + attachApiError, + getApiErrorMessage, + normalizeApiError, +} from "./apiError"; + +describe("API error normalization", () => { + it("normalizes the structured server contract", () => { + const result = normalizeApiError({ + response: { + status: 503, + data: { + detail: "Worker is offline", + error: { + schema_version: 1, + code: "service_unavailable", + category: "availability", + title: "Service temporarily unavailable", + message: "Worker is offline", + retryable: true, + request_id: "req-7", + recovery_actions: ["retry", "view_logs"], + }, + }, + }, + }); + + expect(result).toEqual( + expect.objectContaining({ + code: "service_unavailable", + message: "Worker is offline", + requestId: "req-7", + retryable: true, + recoveryActions: ["retry", "view_logs"], + }), + ); + }); + + it("supports legacy detail responses", () => { + const result = normalizeApiError({ + response: { + status: 404, + headers: { "x-request-id": "legacy-4" }, + data: { detail: "Volume not found" }, + }, + }); + + expect(result.code).toBe("not_found"); + expect(result.message).toBe("Volume not found"); + expect(result.requestId).toBe("legacy-4"); + }); + + it("distinguishes network and cancellation failures", () => { + expect(normalizeApiError(new Error("Network Error"))).toEqual( + expect.objectContaining({ + code: "network_unavailable", + retryable: true, + }), + ); + expect(normalizeApiError({ code: "ERR_CANCELED" })).toEqual( + expect.objectContaining({ + code: "request_canceled", + retryable: false, + }), + ); + }); + + it("attaches the normalized value without replacing the original error", () => { + const error = new Error("Network Error"); + + expect(attachApiError(error)).toBe(error); + expect(getApiErrorMessage(error)).toBe( + "Check the server connection and try again.", + ); + }); +}); diff --git a/client/src/index.js b/client/src/index.js index 71d95547..5bca9ff2 100644 --- a/client/src/index.js +++ b/client/src/index.js @@ -3,6 +3,7 @@ import ReactDOM from "react-dom/client"; import { ConfigProvider } from "antd"; import "./index.css"; import App from "./App"; +import AppErrorBoundary from "./components/AppErrorBoundary"; import { antdWorkflowTheme } from "./design/workflowDesignSystem"; import { installClientLogging } from "./logging/appEventLog"; @@ -12,7 +13,9 @@ const root = ReactDOM.createRoot(document.getElementById("root")); root.render( - + + + , ); diff --git a/docs/platform-hardening-branch-scope.md b/docs/platform-hardening-branch-scope.md new file mode 100644 index 00000000..93f6f208 --- /dev/null +++ b/docs/platform-hardening-branch-scope.md @@ -0,0 +1,104 @@ +# Platform Hardening Branch Scope + +Branch: `agent/platform-hardening` + +Base: `origin/main` at `a991b1b` + +## Objective + +Establish the shared technical substrate needed for a recoverable, resource-bounded, +agent-mediated segmentation loop. This branch hardens existing workflow behavior; +it does not redesign the participant workflow or replace PyTorch Connectomics. + +## In Scope + +1. A structured API error contract with stable machine codes, user-facing recovery + guidance, request correlation, and a consistent frontend representation. +2. Application-level frontend failure boundaries and explicit retry/recovery states + for failures that currently disappear into transient notifications. +3. A bounded volume access contract that separates metadata inspection and region + reads from full materialization across supported volume formats. +4. Persisted operation records for long-running work, including terminal failure and + cancellation states, idempotency, and correlation to workflow evidence. +5. A typed agent action registry that defines allowed inputs, risk, approval policy, + and execution ownership independently of language-model intent routing. +6. Focused tests and compatibility shims that preserve the current closed-loop smoke + path while new primitives are adopted incrementally. + +## Out of Scope + +- Database sharding or horizontal multi-tenant scaling. +- A mandatory bulk conversion of existing datasets. +- Replacing Neuroglancer, PyTorch Connectomics, or every existing API route. +- Full migration of all subprocess execution into a distributed worker system. +- A visual redesign unrelated to loading, failure, recovery, or task state. +- Removing existing workflow evidence records or approval history. + +## Architectural Invariants + +- Base image and imported label artifacts remain immutable by default. +- Volume callers must be able to inspect metadata and request a bounded region without + materializing the complete volume. +- Risky agent actions remain approval-gated and must resolve through a registered, + typed action definition. +- The browser may perform navigation and presentation effects; domain execution and + durable state transitions belong to the server. +- Operation state must survive a server restart even before subprocess reattachment is + implemented. +- User-visible failures include a stable code, recovery guidance, and request ID while + technical details remain available for diagnosis. + +## Acceptance Criteria + +- Backend tests cover error serialization, volume metadata/region reads, operation + lifecycle transitions, and registered agent action validation. +- Frontend tests cover normalized API failures, the application error boundary, and a + user-triggered recovery path. +- Existing workflow, EHTool, runtime, and agent proposal tests remain green or receive + narrowly justified compatibility updates. +- The production frontend build completes. +- No new critical path depends on full-volume NumPy materialization when only metadata + or a subvolume is required. +- New agent proposal types cannot silently bypass registry validation or approval + policy. + +## Delivered + +- Shared FastAPI error envelopes preserve legacy `detail` values while adding stable + codes, categories, retry guidance, recovery actions, and request IDs. The React + client normalizes transport and API failures and has a root recovery boundary. +- `VolumeStore` separates metadata and region access from NumPy materialization for + HDF5, TIFF/OME-TIFF, NPY/NPZ, Zarr/N5, NIfTI, and MRC sources. Neuroglancer now + serves 3D image and label chunks from those stores and closes backing resources + when retained viewers expire or are evicted. +- `WorkflowOperation` persists queued, running, succeeded, failed, and cancelled + work with idempotency, correlation, leases, heartbeats, progress, and cancellation + requests. Approved training commands create a per-attempt operation and commit + command/operation state together. +- Agent effects resolve through strict, discriminated runtime and workflow action + schemas. The registry owns risk, approval, execution-owner, and specialist policy; + unknown proposal types, effect keys, and nested action parameters are rejected. + +## Verification + +- Backend: full suite passes (`278 passed`, plus five subtests), including an actual + on-demand Neuroglancer chunk read from HDF5-backed image and label sources. +- Frontend: all 26 suites pass (`138 passed`). +- Production frontend build succeeds. Existing React hook and bundle-size warnings + remain unchanged. + +## Follow-On Work + +This branch intentionally creates migration points. A subsequent branch should: + +1. Synchronize worker completion, failure, and cancellation into long-running training + and inference operations; current training-command success means the worker accepted + submission. +2. Add a frontend task center backed by operation records so reconnecting clients can + resume progress and cancellation state. +3. Materialize multiscale OME-Zarr for repeated large-volume access and preprocess 4D + prediction tensors into 3D label volumes before Neuroglancer launch. +4. Convert EHTool proofreading persistence from retained full arrays to chunk-aligned + correction overlays; its editing model still requires complete mutable volumes. +5. Move remaining browser-orchestrated runtime launches behind server command handlers + while keeping navigation and form-prefill effects browser-owned. diff --git a/pyproject.toml b/pyproject.toml index 3d8a6e7b..4495d961 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "requests>=2.31", "scikit-image>=0.25", "scipy>=1.11", + "sqlalchemy>=2.0,<3", "tensorboard==2.20.0", "tensorboard-data-server==0.7.2", "tifffile>=2025.5.10", diff --git a/server_api/chatbot/logging_utils.py b/server_api/chatbot/logging_utils.py index 41dcdfe9..906831a7 100644 --- a/server_api/chatbot/logging_utils.py +++ b/server_api/chatbot/logging_utils.py @@ -1,13 +1,26 @@ import logging +import re import time import uuid from typing import Any, Optional logger = logging.getLogger(__name__) +_REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") def request_id_from_request(request: Any) -> str: - return request.headers.get("x-request-id") or str(uuid.uuid4()) + existing = getattr(getattr(request, "state", None), "request_id", None) + if existing: + return existing + supplied_request_id = request.headers.get("x-request-id") + request_id = ( + supplied_request_id + if supplied_request_id and _REQUEST_ID_PATTERN.fullmatch(supplied_request_id) + else str(uuid.uuid4()) + ) + if getattr(request, "state", None) is not None: + request.state.request_id = request_id + return request_id def log_request_summary( diff --git a/server_api/ehtool/data_manager.py b/server_api/ehtool/data_manager.py index 45b7b328..3e4d42fe 100644 --- a/server_api/ehtool/data_manager.py +++ b/server_api/ehtool/data_manager.py @@ -1631,11 +1631,7 @@ def _load_volume(self, path: str) -> Dict[str, Any]: # Single file if path_obj.is_file(): - lower_name = path_obj.name.lower() - if lower_name.endswith((".tif", ".tiff")): - volume = tifffile.imread(file_path) - else: - volume = load_volume(path) + volume = load_volume(path) if volume.ndim == 2: return { diff --git a/server_api/errors.py b/server_api/errors.py new file mode 100644 index 00000000..ffb8cfb9 --- /dev/null +++ b/server_api/errors.py @@ -0,0 +1,194 @@ +"""Shared API error responses and exception handlers.""" + +import logging +from typing import Any, Dict, List, Optional + +from fastapi import FastAPI, Request +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + +from server_api.chatbot.logging_utils import request_id_from_request + + +logger = logging.getLogger(__name__) +ERROR_SCHEMA_VERSION = 1 + + +def _message_from_detail(detail: Any, fallback: str) -> str: + if isinstance(detail, str) and detail.strip(): + return detail.strip() + if isinstance(detail, dict): + for key in ("user_message", "message", "detail", "reason"): + value = detail.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return fallback + + +def _error_metadata(status_code: int) -> Dict[str, Any]: + if status_code == 400: + return { + "code": "invalid_request", + "category": "validation", + "title": "Request could not be completed", + "retryable": False, + "recovery_actions": ["review_input"], + } + if status_code == 401: + return { + "code": "authentication_required", + "category": "authentication", + "title": "Authentication required", + "retryable": False, + "recovery_actions": ["sign_in"], + } + if status_code == 403: + return { + "code": "permission_denied", + "category": "authorization", + "title": "Permission denied", + "retryable": False, + "recovery_actions": ["contact_admin"], + } + if status_code == 404: + return { + "code": "not_found", + "category": "resource", + "title": "Resource not found", + "retryable": False, + "recovery_actions": ["go_back"], + } + if status_code == 409: + return { + "code": "conflict", + "category": "state", + "title": "State conflict", + "retryable": False, + "recovery_actions": ["refresh"], + } + if status_code == 413: + return { + "code": "payload_too_large", + "category": "resource", + "title": "Request is too large", + "retryable": False, + "recovery_actions": ["reduce_request"], + } + if status_code == 422: + return { + "code": "validation_failed", + "category": "validation", + "title": "Some request values are invalid", + "retryable": False, + "recovery_actions": ["review_input"], + } + if status_code == 429: + return { + "code": "rate_limited", + "category": "availability", + "title": "Too many requests", + "retryable": True, + "recovery_actions": ["retry_later"], + } + if status_code in {502, 503, 504}: + return { + "code": "service_unavailable", + "category": "availability", + "title": "Service temporarily unavailable", + "retryable": True, + "recovery_actions": ["retry"], + } + if status_code >= 500: + return { + "code": "internal_error", + "category": "internal", + "title": "Unexpected server error", + "retryable": True, + "recovery_actions": ["retry", "view_logs"], + } + return { + "code": "request_failed", + "category": "request", + "title": "Request failed", + "retryable": False, + "recovery_actions": ["go_back"], + } + + +def build_error_response( + *, + status_code: int, + request_id: str, + detail: Any, + message_fallback: str, + headers: Optional[Dict[str, str]] = None, + validation_errors: Optional[List[Dict[str, Any]]] = None, +) -> JSONResponse: + metadata = _error_metadata(status_code) + error = { + "schema_version": ERROR_SCHEMA_VERSION, + **metadata, + "message": _message_from_detail(detail, message_fallback), + "request_id": request_id, + } + if validation_errors is not None: + error["validation_errors"] = validation_errors + + response_headers = dict(headers or {}) + response_headers.setdefault("x-request-id", request_id) + response_headers.setdefault("cache-control", "no-store") + return JSONResponse( + status_code=status_code, + content=jsonable_encoder({"detail": detail, "error": error}), + headers=response_headers, + ) + + +def install_error_handlers(app: FastAPI) -> None: + @app.exception_handler(StarletteHTTPException) + async def http_exception_handler( + request: Request, exc: StarletteHTTPException + ) -> JSONResponse: + request_id = request_id_from_request(request) + return build_error_response( + status_code=exc.status_code, + request_id=request_id, + detail=exc.detail, + message_fallback="The request could not be completed.", + headers=exc.headers, + ) + + @app.exception_handler(RequestValidationError) + async def validation_exception_handler( + request: Request, exc: RequestValidationError + ) -> JSONResponse: + request_id = request_id_from_request(request) + errors = exc.errors() + return build_error_response( + status_code=422, + request_id=request_id, + detail=errors, + message_fallback="Review the highlighted request values and try again.", + validation_errors=errors, + ) + + @app.exception_handler(Exception) + async def unexpected_exception_handler( + request: Request, exc: Exception + ) -> JSONResponse: + request_id = request_id_from_request(request) + logger.exception( + "Unhandled API error request_id=%s method=%s path=%s", + request_id, + request.method, + request.url.path, + exc_info=exc, + ) + return build_error_response( + status_code=500, + request_id=request_id, + detail="An unexpected server error occurred.", + message_fallback="An unexpected server error occurred.", + ) diff --git a/server_api/main.py b/server_api/main.py index 4d5fd0a3..564d86a6 100644 --- a/server_api/main.py +++ b/server_api/main.py @@ -19,6 +19,7 @@ import requests import uvicorn +import numpy as np from fastapi import Depends, FastAPI, HTTPException, Request, UploadFile from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel @@ -39,12 +40,23 @@ from server_api.auth.database import get_db from server_api.auth.router import get_current_user from server_api.ehtool import router as ehtool_router +from server_api.errors import install_error_handlers from server_api.chatbot.logging_utils import ( log_request_summary, request_id_from_request, ) from server_api.workflows import router as workflow_router -from server_api.workflows.db_models import WorkflowCommand, WorkflowEvent +from server_api.workflows import operation_router as workflow_operation_router +from server_api.workflows.db_models import ( + WorkflowCommand, + WorkflowEvent, + WorkflowOperation, +) +from server_api.workflows.operation_service import ( + create_workflow_operation, + operation_to_dict, + transition_workflow_operation, +) from server_api.workflows.service import ( append_event_for_workflow_if_present, command_to_dict, @@ -56,7 +68,7 @@ update_workflow_fields, ) from server_api.project_manager import router as pm_router -from server_api.workflows.volume_io import load_volume +from server_api.workflows.volume_io import VolumeStore, open_volume_store from server_api.workflows.volume_pairs import ( _is_chunked_volume_directory, _is_neuroglancer_volume_file, @@ -251,6 +263,7 @@ def _ensure_sqlite_column(table_name: str, column_name: str, ddl: str) -> None: ) app = FastAPI() +install_error_handlers(app) # Ensure uploads directory exists os.makedirs("uploads", exist_ok=True) @@ -259,6 +272,11 @@ def _ensure_sqlite_column(table_name: str, column_name: str, ddl: str) -> None: app.include_router(auth_router.router) app.include_router(ehtool_router.router, prefix="/eh", tags=["ehtool"]) app.include_router(workflow_router.router, prefix="/api/workflows", tags=["workflows"]) +app.include_router( + workflow_operation_router.router, + prefix="/api/workflows", + tags=["workflow-operations"], +) app.include_router(pm_router.router, prefix="/api/pm", tags=["project-manager"]) app.add_middleware( @@ -267,6 +285,7 @@ def _ensure_sqlite_column(table_name: str, column_name: str, ddl: str) -> None: allow_credentials=True, allow_methods=["*"], allow_headers=["*"], + expose_headers=["x-request-id"], ) logger = logging.getLogger(__name__) @@ -384,6 +403,79 @@ def _build_neuroglancer_layer( return _with_neuroglancer_image_shader(neuroglancer_module, source, shader) +class _NeuroglancerSegmentationStore: + """Validate and convert segmentation labels one requested chunk at a time.""" + + def __init__(self, store: VolumeStore): + if store.ndim != 3: + raise ValueError( + "Storage-backed segmentation viewing requires a 3D label volume. " + "Convert multi-channel predictions to a 3D label volume first." + ) + self._store = store + self.shape = store.shape + source_dtype = np.dtype(store.dtype) + if np.issubdtype(source_dtype, np.bool_): + self.dtype = np.dtype(np.uint8) + elif np.issubdtype(source_dtype, np.unsignedinteger): + self.dtype = source_dtype + elif np.issubdtype(source_dtype, np.integer) or np.issubdtype( + source_dtype, np.floating + ): + self.dtype = np.dtype(np.uint64) + else: + raise ValueError( + f"Segmentation volume dtype {source_dtype} is not supported." + ) + + def __getitem__(self, key): + chunk = np.asarray(self._store[key]) + if chunk.size == 0: + return chunk.astype(self.dtype, copy=False) + if np.issubdtype(chunk.dtype, np.floating): + if not np.all(np.isfinite(chunk)): + raise ValueError( + "Segmentation volumes must not contain NaN or infinite values." + ) + rounded = np.rint(chunk) + if not np.allclose(chunk, rounded): + raise ValueError("Segmentation volumes must use integer-valued labels.") + chunk = rounded + if np.issubdtype(chunk.dtype, np.signedinteger) or np.issubdtype( + chunk.dtype, np.floating + ): + if int(chunk.min()) < 0: + raise ValueError( + "Segmentation volumes must contain non-negative label ids." + ) + return chunk.astype(self.dtype, copy=False) + + +def _open_neuroglancer_volume_sources( + image_path: pathlib.Path, + label_path: Optional[pathlib.Path], +): + resources: List[VolumeStore] = [] + try: + image_store = open_volume_store(str(image_path)) + resources.append(image_store) + if image_store.ndim != 3: + raise ValueError( + f"Image volume must be 3D for Neuroglancer, got {image_store.shape}." + ) + + label_source = None + if label_path is not None: + label_store = open_volume_store(str(label_path)) + resources.append(label_store) + label_source = _NeuroglancerSegmentationStore(label_store) + return image_store, label_source, resources + except Exception: + for resource in resources: + resource.close() + raise + + class ClientAppLogEvent(BaseModel): event: str level: str = "INFO" @@ -726,6 +818,54 @@ def _build_training_body_from_command( } +def _workflow_command_run_response( + workflow, + command: WorkflowCommand, + operation: WorkflowOperation, +) -> dict[str, Any]: + result = decode_json(operation.result_json) + return { + "workflow_id": workflow.id, + "command": command_to_dict(command), + "operation": operation_to_dict(operation), + "worker": result.get("worker", {}), + "run_id": result.get("run_id"), + "started_event_id": result.get("started_event_id"), + } + + +def _fail_command_operation( + db: Session, + *, + command: WorkflowCommand, + operation: WorkflowOperation, + error_payload: dict[str, Any], + retryable: bool, +) -> None: + fail_workflow_command( + db, + command, + error_payload=error_payload, + retryable=retryable, + commit=False, + ) + if operation.status in {"queued", "running"}: + transition_workflow_operation( + db, + operation, + status="failed", + expected_status=operation.status, + error_payload=error_payload, + lease_owner=( + "server_api.training_runner" if operation.status == "running" else None + ), + commit=False, + ) + db.commit() + db.refresh(command) + db.refresh(operation) + + @app.on_event("startup") async def configure_app_event_logging(): log_path = configure_process_logging("server_api") @@ -911,6 +1051,17 @@ def _neuroglancer_token_from_url(viewer_url: str) -> Optional[str]: return None +def _close_neuroglancer_entry_resources(entry: dict[str, Any]) -> None: + for resource in entry.get("resources") or []: + try: + resource.close() + except Exception: + logger.warning( + "Failed to close an evicted Neuroglancer backing resource.", + exc_info=True, + ) + + def _cleanup_retained_neuroglancer_viewers(now: Optional[float] = None): if now is None: now = time.time() @@ -928,11 +1079,14 @@ def _cleanup_retained_neuroglancer_viewers(now: Optional[float] = None): "mode": entry.get("mode"), } ) - _retained_neuroglancer_viewers.pop(token, None) + removed = _retained_neuroglancer_viewers.pop(token, None) + if removed is not None: + _close_neuroglancer_entry_resources(removed) max_viewers = max(PYTC_NEUROGLANCER_MAX_VIEWERS, 0) while max_viewers and len(_retained_neuroglancer_viewers) > max_viewers: token, entry = _retained_neuroglancer_viewers.popitem(last=False) + _close_neuroglancer_entry_resources(entry) evicted.append( { "token": token, @@ -954,6 +1108,7 @@ def _retain_neuroglancer_viewer( workflow_id: Optional[int] = None, image_path: Optional[str] = None, label_path: Optional[str] = None, + resources: Optional[List[VolumeStore]] = None, ) -> Optional[str]: token = getattr(viewer, "token", None) or _neuroglancer_token_from_url( internal_viewer_url @@ -974,6 +1129,8 @@ def _retain_neuroglancer_viewer( now = time.time() with _retained_neuroglancer_viewers_lock: if PYTC_NEUROGLANCER_MAX_VIEWERS <= 0: + for entry in _retained_neuroglancer_viewers.values(): + _close_neuroglancer_entry_resources(entry) _retained_neuroglancer_viewers.clear() append_app_event( component="server_api", @@ -988,6 +1145,9 @@ def _retain_neuroglancer_viewer( return token evicted = _cleanup_retained_neuroglancer_viewers(now) + replaced = _retained_neuroglancer_viewers.pop(token, None) + if replaced is not None: + _close_neuroglancer_entry_resources(replaced) _retained_neuroglancer_viewers[token] = { "viewer": viewer, "public_url": public_url, @@ -996,6 +1156,7 @@ def _retain_neuroglancer_viewer( "workflow_id": workflow_id, "image_path": image_path, "label_path": label_path, + "resources": list(resources or []), "created_at": now, } _retained_neuroglancer_viewers.move_to_end(token) @@ -1779,23 +1940,15 @@ async def neuroglancer( names=["z", "y", "x"], units=["nm", "nm", "nm"], scales=scales ) try: - im = load_volume(str(resolved_image_path), label="image") - except Exception as e: - raise HTTPException( - status_code=400, detail=f"Failed to read image volume: {str(e)}" + im, gt, volume_resources = _open_neuroglancer_volume_sources( + resolved_image_path, + resolved_label_path, ) - try: - gt = ( - load_volume(str(resolved_label_path), label="label") - if resolved_label_path - else None - ) - if gt is not None: - gt = _normalize_segmentation_volume_for_neuroglancer(gt) except Exception as e: raise HTTPException( - status_code=400, detail=f"Failed to prepare label volume: {str(e)}" - ) + status_code=400, + detail=f"Failed to prepare storage-backed volume layers: {str(e)}", + ) from e def ngLayer( data, @@ -1844,6 +1997,7 @@ def ngLayer( workflow_id=workflow_id, image_path=str(resolved_image_path), label_path=str(resolved_label_path) if resolved_label_path else None, + resources=volume_resources, ) append_app_event( component="server_api", @@ -2051,23 +2205,16 @@ async def neuroglancer_proofread( raise HTTPException(status_code=400, detail=str(exc)) from exc try: - im = load_volume(str(resolved_image_path), label="image") + im, gt, volume_resources = _open_neuroglancer_volume_sources( + resolved_image_path, + resolved_label_path, + ) except Exception as exc: raise HTTPException( - status_code=400, detail=f"Failed to read image volume: {str(exc)}" + status_code=400, + detail=f"Failed to prepare storage-backed proofreading layers: {str(exc)}", ) from exc - gt = None - if resolved_label_path: - try: - gt = load_volume(str(resolved_label_path), label="label") - gt = _normalize_segmentation_volume_for_neuroglancer(gt) - except Exception as exc: - raise HTTPException( - status_code=400, - detail=f"Failed to prepare label volume for proofreading: {str(exc)}", - ) from exc - neuroglancer.set_server_bind_address( PYTC_NEUROGLANCER_BIND_HOST, PYTC_NEUROGLANCER_PORT ) @@ -2281,6 +2428,7 @@ def handle_save_review(_action_state): workflow_id=workflow_id, image_path=str(resolved_image_path), label_path=str(resolved_label_path) if resolved_label_path else None, + resources=volume_resources, ) response_payload = { "url": public_url, @@ -2450,6 +2598,70 @@ async def run_workflow_command( detail=f"Unsupported workflow command type: {command.command_type}", ) + if command.status == "submitted": + completed_operation = ( + db.query(WorkflowOperation) + .filter( + WorkflowOperation.workflow_id == workflow.id, + WorkflowOperation.command_id == command.id, + WorkflowOperation.status == "succeeded", + ) + .order_by(WorkflowOperation.id.desc()) + .first() + ) + if completed_operation is not None: + return _workflow_command_run_response( + workflow, + command, + completed_operation, + ) + raise HTTPException( + status_code=409, detail="Workflow command was already submitted." + ) + + operation_query = db.query(WorkflowOperation).filter( + WorkflowOperation.workflow_id == workflow.id, + WorkflowOperation.command_id == command.id, + ) + latest_operation = operation_query.order_by(WorkflowOperation.id.desc()).first() + if latest_operation is not None and latest_operation.status in { + "queued", + "running", + "succeeded", + }: + operation = latest_operation + else: + operation = create_workflow_operation( + db, + workflow_id=workflow.id, + operation_type="start_training", + idempotency_key=( + f"workflow-command:{command.id}:attempt:{operation_query.count() + 1}" + ), + actor=command.actor, + command_id=command.id, + input_payload=decode_json(command.input_json), + metadata={ + "command_type": command.command_type, + "execution_scope": "worker_submission", + }, + commit=True, + ) + if operation.status == "succeeded": + if command.status != "submitted": + command = submit_workflow_command( + db, + command, + result_payload=decode_json(operation.result_json), + commit=True, + ) + return _workflow_command_run_response(workflow, command, operation) + if operation.status != "queued": + raise HTTPException( + status_code=409, + detail=f"Workflow command operation is already {operation.status}.", + ) + try: body = _build_training_body_from_command(command, workflow) body = _runtime_body_with_workflow_fallbacks(body, workflow, mode="training") @@ -2457,8 +2669,20 @@ async def run_workflow_command( db, command, lease_owner="server_api.training_runner", - commit=True, + commit=False, + ) + operation = transition_workflow_operation( + db, + operation, + status="running", + expected_status="queued", + lease_owner="server_api.training_runner", + metadata={"run_id": body.get("run_id")}, + commit=False, ) + db.commit() + db.refresh(command) + db.refresh(operation) update_workflow_fields( db, workflow, @@ -2494,36 +2718,43 @@ async def run_workflow_command( json_body=body, timeout=30, ) - command = submit_workflow_command( - db, - command, - result_payload={ - "worker": worker_data, - "run_id": body.get("run_id"), - "started_event_id": started_event.id if started_event else None, - "submitted": True, - }, - commit=True, - ) - return { - "workflow_id": workflow.id, - "command": command_to_dict(command), + operation_result = { "worker": worker_data, "run_id": body.get("run_id"), "started_event_id": started_event.id if started_event else None, + "submitted": True, } + command = submit_workflow_command( + db, + command, + result_payload=operation_result, + commit=False, + ) + operation = transition_workflow_operation( + db, + operation, + status="succeeded", + expected_status="running", + result_payload=operation_result, + lease_owner="server_api.training_runner", + commit=False, + ) + db.commit() + db.refresh(command) + db.refresh(operation) + return _workflow_command_run_response(workflow, command, operation) except HTTPException as exc: error_payload = { "error": "HTTPException", "status_code": exc.status_code, "detail": exc.detail, } - fail_workflow_command( + _fail_command_operation( db, - command, + command=command, + operation=operation, error_payload=error_payload, retryable=exc.status_code in {503, 504}, - commit=True, ) append_event_for_workflow_if_present( db, @@ -2545,12 +2776,12 @@ async def run_workflow_command( "error": type(exc).__name__, "detail": str(exc), } - fail_workflow_command( + _fail_command_operation( db, - command, + command=command, + operation=operation, error_payload=error_payload, retryable=False, - commit=True, ) append_event_for_workflow_if_present( db, diff --git a/server_api/workflows/agent_actions.py b/server_api/workflows/agent_actions.py new file mode 100644 index 00000000..b0212c45 --- /dev/null +++ b/server_api/workflows/agent_actions.py @@ -0,0 +1,356 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Annotated, Any, Dict, Literal, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field + + +RiskLevel = Literal[ + "read_only", + "prefills_form", + "loads_editor", + "writes_workflow_record", + "modifies_workspace", + "runs_job", + "controls_job", + "exports_evidence", +] + + +class _StrictActionPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class ChooseProjectDataPayload(_StrictActionPayload): + kind: Literal["choose_project_data"] + + +class LoadVisualizationPayload(_StrictActionPayload): + kind: Literal["load_visualization"] + + +class StartInferencePayload(_StrictActionPayload): + kind: Literal["start_inference"] + + +class StopInferencePayload(_StrictActionPayload): + kind: Literal["stop_inference"] + + +class StartProofreadingPayload(_StrictActionPayload): + kind: Literal["start_proofreading"] + + +class StartTrainingPayload(_StrictActionPayload): + kind: Literal["start_training"] + autopick_parameters: Optional[bool] = None + parameter_mode: Optional[str] = None + volume_subset: Optional[Dict[str, Any]] = None + + +class StopTrainingPayload(_StrictActionPayload): + kind: Literal["stop_training"] + + +RuntimeActionPayload = Annotated[ + Union[ + ChooseProjectDataPayload, + LoadVisualizationPayload, + StartInferencePayload, + StopInferencePayload, + StartProofreadingPayload, + StartTrainingPayload, + StopTrainingPayload, + ], + Field(discriminator="kind"), +] + + +class ComputeEvaluationPayload(_StrictActionPayload): + kind: Literal["compute_evaluation"] + name: Optional[str] = None + baseline_prediction_path: Optional[str] = None + candidate_prediction_path: Optional[str] = None + ground_truth_path: Optional[str] = None + baseline_run_id: Optional[int] = None + candidate_run_id: Optional[int] = None + model_version_id: Optional[int] = None + metadata: Optional[Dict[str, Any]] = None + + +class ExportBundlePayload(_StrictActionPayload): + kind: Literal["export_bundle"] + + +class ProposeRetrainingStagePayload(_StrictActionPayload): + kind: Literal["propose_retraining_stage"] + corrected_mask_path: Optional[str] = None + + +WorkflowActionPayload = Annotated[ + Union[ + ComputeEvaluationPayload, + ExportBundlePayload, + ProposeRetrainingStagePayload, + ], + Field(discriminator="kind"), +] + + +class MountProjectPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + directory_path: str + mount_name: Optional[str] = None + destination_path: Optional[str] = None + workflow_patch: Optional[Dict[str, Any]] = None + + +class ClientEffectsPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + navigate_to: Optional[str] = None + runtime_action: Optional[RuntimeActionPayload] = None + workflow_action: Optional[WorkflowActionPayload] = None + mount_project: Optional[MountProjectPayload] = None + reset_workspace: Optional[bool] = None + start_new_workflow: Optional[Any] = None + show_workflow_context: Optional[bool] = None + refresh_insights: Optional[bool] = None + refresh_project_progress: Optional[bool] = None + training_volume_subset: Optional[Any] = None + set_training_config_preset: Optional[str] = None + set_training_image_path: Optional[str] = None + set_training_label_path: Optional[str] = None + set_training_log_path: Optional[str] = None + set_training_output_path: Optional[str] = None + set_inference_checkpoint_path: Optional[str] = None + set_inference_config_preset: Optional[str] = None + set_inference_image_path: Optional[str] = None + set_inference_label_path: Optional[str] = None + set_inference_output_path: Optional[str] = None + set_visualization_image_path: Optional[str] = None + set_visualization_label_path: Optional[str] = None + set_visualization_scales: Optional[Any] = None + set_proofreading_dataset_path: Optional[str] = None + set_proofreading_image_path: Optional[str] = None + set_proofreading_label_path: Optional[str] = None + set_proofreading_mask_path: Optional[str] = None + set_proofreading_project_name: Optional[str] = None + + +class StageRetrainingPayload(BaseModel): + model_config = ConfigDict(extra="allow") + + corrected_mask_path: Optional[str] = None + + +class StartTrainingRunPayload(BaseModel): + model_config = ConfigDict(extra="allow") + + client_effects: Optional[ClientEffectsPayload] = None + + +class RunClientEffectsPayload(BaseModel): + model_config = ConfigDict(extra="allow") + + client_effects: ClientEffectsPayload + + +@dataclass(frozen=True) +class AgentActionDefinition: + action_type: str + risk_level: RiskLevel + requires_approval: bool + execution_owner: Literal["browser_navigation", "server_workflow", "server_runtime"] + specialist_agent_type: str + + +RUNTIME_ACTIONS: Dict[str, AgentActionDefinition] = { + "choose_project_data": AgentActionDefinition( + "choose_project_data", + "prefills_form", + False, + "browser_navigation", + "data_agent", + ), + "load_visualization": AgentActionDefinition( + "load_visualization", + "read_only", + False, + "browser_navigation", + "visualization_agent", + ), + "start_inference": AgentActionDefinition( + "start_inference", "runs_job", True, "server_runtime", "inference_agent" + ), + "stop_inference": AgentActionDefinition( + "stop_inference", "controls_job", True, "server_runtime", "inference_agent" + ), + "start_proofreading": AgentActionDefinition( + "start_proofreading", + "loads_editor", + True, + "server_workflow", + "proofreading_agent", + ), + "start_training": AgentActionDefinition( + "start_training", "runs_job", True, "server_runtime", "training_agent" + ), + "stop_training": AgentActionDefinition( + "stop_training", "controls_job", True, "server_runtime", "training_agent" + ), +} + + +WORKFLOW_ACTIONS: Dict[str, AgentActionDefinition] = { + "compute_evaluation": AgentActionDefinition( + "compute_evaluation", + "writes_workflow_record", + True, + "server_workflow", + "evaluation_agent", + ), + "export_bundle": AgentActionDefinition( + "export_bundle", + "exports_evidence", + True, + "server_workflow", + "evidence_agent", + ), + "propose_retraining_stage": AgentActionDefinition( + "propose_retraining_stage", + "writes_workflow_record", + True, + "server_workflow", + "training_agent", + ), +} + + +NAVIGATION_SPECIALISTS = { + "files": "data_agent", + "visualization": "visualization_agent", + "mask-proofreading": "proofreading_agent", + "training": "training_agent", + "inference": "inference_agent", + "project-progress": "project_manager", +} + + +def _validate_effects(client_effects: Dict[str, Any]) -> None: + validator = getattr(ClientEffectsPayload, "model_validate", None) + if validator is not None: + validator(client_effects) + else: # pragma: no cover - Pydantic v1 compatibility + ClientEffectsPayload.parse_obj(client_effects) + + +def resolve_agent_action( + action_id: str, client_effects: Optional[Dict[str, Any]] +) -> AgentActionDefinition: + effects = client_effects or {} + _validate_effects(effects) + + runtime_action = effects.get("runtime_action") or {} + runtime_kind = runtime_action.get("kind") + if runtime_kind: + return RUNTIME_ACTIONS[str(runtime_kind)] + + workflow_action = effects.get("workflow_action") or {} + workflow_kind = workflow_action.get("kind") + if workflow_kind: + return WORKFLOW_ACTIONS[str(workflow_kind)] + + if effects.get("mount_project"): + return AgentActionDefinition( + "mount_project", "modifies_workspace", True, "server_workflow", "data_agent" + ) + if effects.get("reset_workspace"): + return AgentActionDefinition( + "reset_workspace", + "modifies_workspace", + True, + "server_workflow", + "data_agent", + ) + if effects.get("start_new_workflow"): + return AgentActionDefinition( + "start_new_workflow", + "writes_workflow_record", + True, + "server_workflow", + "project_manager", + ) + + if any(key.startswith("set_") for key in effects) or effects.get( + "training_volume_subset" + ): + navigate_to = str(effects.get("navigate_to") or "") + return AgentActionDefinition( + action_id, + "prefills_form", + False, + "browser_navigation", + NAVIGATION_SPECIALISTS.get(navigate_to, "project_manager"), + ) + + if effects.get("show_workflow_context") or effects.get("refresh_insights"): + return AgentActionDefinition( + ( + "show_workflow_context" + if effects.get("show_workflow_context") + else "refresh_context" + ), + "read_only", + False, + "browser_navigation", + "project_manager", + ) + + if effects.get("navigate_to"): + destination = str(effects["navigate_to"]) + return AgentActionDefinition( + f"open_{destination}", + "read_only", + False, + "browser_navigation", + NAVIGATION_SPECIALISTS.get(destination, "project_manager"), + ) + + return AgentActionDefinition( + action_id, + "read_only", + False, + "browser_navigation", + "project_manager", + ) + + +def validate_agent_proposal( + action: str, payload: Optional[Dict[str, Any]] +) -> AgentActionDefinition: + params = payload or {} + if action == "stage_retraining_from_corrections": + StageRetrainingPayload.model_validate(params) + return AgentActionDefinition( + action, + "writes_workflow_record", + True, + "server_workflow", + "training_agent", + ) + if action == "start_training_run": + validated = StartTrainingRunPayload.model_validate(params) + if validated.client_effects is not None: + return resolve_agent_action( + action, validated.client_effects.model_dump(exclude_none=True) + ) + return RUNTIME_ACTIONS["start_training"] + if action == "run_client_effects": + validated = RunClientEffectsPayload.model_validate(params) + return resolve_agent_action( + action, validated.client_effects.model_dump(exclude_none=True) + ) + raise ValueError(f"Unsupported agent proposal action: {action}") diff --git a/server_api/workflows/db_models.py b/server_api/workflows/db_models.py index 67f773a8..ebd638ee 100644 --- a/server_api/workflows/db_models.py +++ b/server_api/workflows/db_models.py @@ -1,5 +1,6 @@ from sqlalchemy import ( Boolean, + CheckConstraint, Column, DateTime, Float, @@ -99,6 +100,12 @@ class WorkflowSession(Base): cascade="all, delete-orphan", order_by="WorkflowVolumeState.volume_id", ) + operations = relationship( + "WorkflowOperation", + back_populates="workflow", + cascade="all, delete-orphan", + order_by="WorkflowOperation.created_at", + ) class WorkflowEvent(Base): @@ -166,6 +173,67 @@ class WorkflowCommand(Base): approval_event = relationship("WorkflowEvent", foreign_keys=[approval_event_id]) +class WorkflowOperation(Base): + """Durable lifecycle record for asynchronous workflow work. + + Commands capture an approved intent. Operations capture execution state and can + therefore outlive the API process that requested or started the work. + """ + + __tablename__ = "workflow_operations" + __table_args__ = ( + UniqueConstraint( + "workflow_id", + "idempotency_key", + name="uq_workflow_operations_workflow_id_idempotency_key", + ), + CheckConstraint( + "status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled')", + name="ck_workflow_operations_status", + ), + CheckConstraint( + "progress IS NULL OR (progress >= 0 AND progress <= 1)", + name="ck_workflow_operations_progress", + ), + ) + + id = Column(Integer, primary_key=True, index=True) + workflow_id = Column( + Integer, ForeignKey("workflow_sessions.id"), nullable=False, index=True + ) + operation_type = Column(String, nullable=False, index=True) + status = Column(String, default="queued", nullable=False, index=True) + idempotency_key = Column(String, nullable=False, index=True) + correlation_id = Column(String, nullable=False, index=True) + actor = Column(String, default="system", nullable=False, index=True) + command_id = Column( + Integer, ForeignKey("workflow_commands.id"), nullable=True, index=True + ) + model_run_id = Column( + Integer, ForeignKey("workflow_model_runs.id"), nullable=True, index=True + ) + input_json = Column(Text, nullable=True) + result_json = Column(Text, nullable=True) + error_json = Column(Text, nullable=True) + metadata_json = Column(Text, nullable=True) + progress = Column(Float, nullable=True) + attempt_count = Column(Integer, default=0, nullable=False) + lease_owner = Column(String, nullable=True, index=True) + lease_expires_at = Column(DateTime(timezone=True), nullable=True) + heartbeat_at = Column(DateTime(timezone=True), nullable=True) + cancellation_requested_at = Column(DateTime(timezone=True), nullable=True) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + workflow = relationship("WorkflowSession", back_populates="operations") + command = relationship("WorkflowCommand") + model_run = relationship("WorkflowModelRun") + + class WorkflowArtifact(Base): __tablename__ = "workflow_artifacts" __table_args__ = ( diff --git a/server_api/workflows/operation_router.py b/server_api/workflows/operation_router.py new file mode 100644 index 00000000..6bb3242c --- /dev/null +++ b/server_api/workflows/operation_router.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from server_api.auth import models as auth_models +from server_api.auth.database import get_db +from server_api.auth.router import get_current_user + +from .db_models import WorkflowOperation +from .operation_service import ( + OPERATION_STATUSES, + create_workflow_operation, + get_workflow_operation_or_404, + heartbeat_workflow_operation, + operation_to_dict, + request_workflow_operation_cancellation, + transition_workflow_operation, +) +from .service import get_user_workflow_or_404 + +router = APIRouter() + + +class WorkflowOperationCreateRequest(BaseModel): + operation_type: str = Field(min_length=1, max_length=120) + idempotency_key: str = Field(min_length=1, max_length=255) + correlation_id: Optional[str] = Field(default=None, max_length=255) + actor: str = "system" + command_id: Optional[int] = None + model_run_id: Optional[int] = None + input: Dict[str, Any] = Field(default_factory=dict) + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class WorkflowOperationTransitionRequest(BaseModel): + status: str + expected_status: Optional[str] = None + result: Optional[Dict[str, Any]] = None + error: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None + progress: Optional[float] = None + lease_owner: Optional[str] = None + lease_expires_at: Optional[datetime] = None + + +class WorkflowOperationHeartbeatRequest(BaseModel): + progress: Optional[float] = None + metadata: Optional[Dict[str, Any]] = None + lease_owner: Optional[str] = None + lease_expires_at: Optional[datetime] = None + + +class WorkflowOperationCancellationRequest(BaseModel): + reason: Optional[str] = Field(default=None, max_length=1000) + + +class WorkflowOperationResponse(BaseModel): + id: int + workflow_id: int + operation_type: str + status: str + idempotency_key: str + correlation_id: str + actor: str + command_id: Optional[int] = None + model_run_id: Optional[int] = None + input: Dict[str, Any] = Field(default_factory=dict) + result: Dict[str, Any] = Field(default_factory=dict) + error: Dict[str, Any] = Field(default_factory=dict) + metadata: Dict[str, Any] = Field(default_factory=dict) + progress: Optional[float] = None + attempt_count: int = 0 + lease_owner: Optional[str] = None + lease_expires_at: Any = None + heartbeat_at: Any = None + cancellation_requested_at: Any = None + started_at: Any = None + completed_at: Any = None + created_at: Any + updated_at: Any + + +def _response(operation: WorkflowOperation) -> WorkflowOperationResponse: + return WorkflowOperationResponse(**operation_to_dict(operation)) + + +def _owned_operation( + db: Session, + *, + workflow_id: int, + operation_id: int, + user_id: int, +) -> WorkflowOperation: + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user_id) + return get_workflow_operation_or_404( + db, + workflow_id=workflow_id, + operation_id=operation_id, + ) + + +@router.post( + "/{workflow_id}/operations", + response_model=WorkflowOperationResponse, +) +def create_operation( + workflow_id: int, + body: WorkflowOperationCreateRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + operation = create_workflow_operation( + db, + workflow_id=workflow.id, + operation_type=body.operation_type, + idempotency_key=body.idempotency_key, + correlation_id=body.correlation_id, + actor=body.actor, + command_id=body.command_id, + model_run_id=body.model_run_id, + input_payload=body.input, + metadata=body.metadata, + commit=True, + ) + return _response(operation) + + +@router.get( + "/{workflow_id}/operations", + response_model=List[WorkflowOperationResponse], +) +def list_operations( + workflow_id: int, + status: Optional[str] = None, + operation_type: Optional[str] = None, + limit: int = Query(default=100, ge=1, le=500), + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + if status is not None and status not in OPERATION_STATUSES: + raise HTTPException( + status_code=400, + detail=( + "operation status must be one of: " + f"{', '.join(sorted(OPERATION_STATUSES))}" + ), + ) + query = db.query(WorkflowOperation).filter( + WorkflowOperation.workflow_id == workflow_id + ) + if status is not None: + query = query.filter(WorkflowOperation.status == status) + if operation_type is not None: + query = query.filter(WorkflowOperation.operation_type == operation_type) + operations = query.order_by( + WorkflowOperation.created_at.desc(), WorkflowOperation.id.desc() + ).limit(limit) + return [_response(operation) for operation in operations] + + +@router.get( + "/{workflow_id}/operations/{operation_id}", + response_model=WorkflowOperationResponse, +) +def get_operation( + workflow_id: int, + operation_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + return _response( + _owned_operation( + db, + workflow_id=workflow_id, + operation_id=operation_id, + user_id=user.id, + ) + ) + + +@router.post( + "/{workflow_id}/operations/{operation_id}/transitions", + response_model=WorkflowOperationResponse, +) +def transition_operation( + workflow_id: int, + operation_id: int, + body: WorkflowOperationTransitionRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + operation = _owned_operation( + db, + workflow_id=workflow_id, + operation_id=operation_id, + user_id=user.id, + ) + operation = transition_workflow_operation( + db, + operation, + status=body.status, + expected_status=body.expected_status, + result_payload=body.result, + error_payload=body.error, + metadata=body.metadata, + progress=body.progress, + lease_owner=body.lease_owner, + lease_expires_at=body.lease_expires_at, + commit=True, + ) + return _response(operation) + + +@router.post( + "/{workflow_id}/operations/{operation_id}/heartbeat", + response_model=WorkflowOperationResponse, +) +def heartbeat_operation( + workflow_id: int, + operation_id: int, + body: WorkflowOperationHeartbeatRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + operation = _owned_operation( + db, + workflow_id=workflow_id, + operation_id=operation_id, + user_id=user.id, + ) + operation = heartbeat_workflow_operation( + db, + operation, + progress=body.progress, + metadata=body.metadata, + lease_owner=body.lease_owner, + lease_expires_at=body.lease_expires_at, + commit=True, + ) + return _response(operation) + + +@router.post( + "/{workflow_id}/operations/{operation_id}/cancel", + response_model=WorkflowOperationResponse, +) +def cancel_operation( + workflow_id: int, + operation_id: int, + body: Optional[WorkflowOperationCancellationRequest] = None, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + operation = _owned_operation( + db, + workflow_id=workflow_id, + operation_id=operation_id, + user_id=user.id, + ) + operation = request_workflow_operation_cancellation( + db, + operation, + reason=body.reason if body else None, + commit=True, + ) + return _response(operation) diff --git a/server_api/workflows/operation_service.py b/server_api/workflows/operation_service.py new file mode 100644 index 00000000..06d7bd89 --- /dev/null +++ b/server_api/workflows/operation_service.py @@ -0,0 +1,398 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, Optional +from uuid import uuid4 + +from fastapi import HTTPException +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from .db_models import ( + WorkflowCommand, + WorkflowModelRun, + WorkflowOperation, +) +from .service import decode_json, encode_json, validate_actor + +OPERATION_STATUSES = {"queued", "running", "succeeded", "failed", "cancelled"} +TERMINAL_OPERATION_STATUSES = {"succeeded", "failed", "cancelled"} +OPERATION_TRANSITIONS = { + "queued": {"running", "failed", "cancelled"}, + "running": {"succeeded", "failed", "cancelled"}, + "succeeded": set(), + "failed": set(), + "cancelled": set(), +} + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _merge_dicts( + base: Optional[Dict[str, Any]], patch: Optional[Dict[str, Any]] +) -> Dict[str, Any]: + merged = dict(base or {}) + for key, value in (patch or {}).items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _merge_dicts(merged[key], value) + else: + merged[key] = value + return merged + + +def _validate_progress(progress: Optional[float]) -> Optional[float]: + if progress is None: + return None + value = float(progress) + if value < 0 or value > 1: + raise HTTPException(status_code=400, detail="progress must be between 0 and 1") + return value + + +def operation_to_dict(operation: WorkflowOperation) -> Dict[str, Any]: + return { + "id": operation.id, + "workflow_id": operation.workflow_id, + "operation_type": operation.operation_type, + "status": operation.status, + "idempotency_key": operation.idempotency_key, + "correlation_id": operation.correlation_id, + "actor": operation.actor, + "command_id": operation.command_id, + "model_run_id": operation.model_run_id, + "input": decode_json(operation.input_json), + "result": decode_json(operation.result_json), + "error": decode_json(operation.error_json), + "metadata": decode_json(operation.metadata_json), + "progress": operation.progress, + "attempt_count": operation.attempt_count, + "lease_owner": operation.lease_owner, + "lease_expires_at": operation.lease_expires_at, + "heartbeat_at": operation.heartbeat_at, + "cancellation_requested_at": operation.cancellation_requested_at, + "started_at": operation.started_at, + "completed_at": operation.completed_at, + "created_at": operation.created_at, + "updated_at": operation.updated_at, + } + + +def get_workflow_operation_or_404( + db: Session, *, workflow_id: int, operation_id: int +) -> WorkflowOperation: + operation = ( + db.query(WorkflowOperation) + .filter( + WorkflowOperation.id == operation_id, + WorkflowOperation.workflow_id == workflow_id, + ) + .first() + ) + if operation is None: + raise HTTPException(status_code=404, detail="Workflow operation not found") + return operation + + +def _validate_linked_record( + db: Session, + *, + model: Any, + record_id: Optional[int], + workflow_id: int, + label: str, +) -> None: + if record_id is None: + return + record = db.query(model).filter(model.id == record_id).first() + if record is None or record.workflow_id != workflow_id: + raise HTTPException( + status_code=400, + detail=f"{label} must belong to the workflow", + ) + + +def _assert_idempotent_match( + operation: WorkflowOperation, + *, + operation_type: str, + actor: str, + command_id: Optional[int], + model_run_id: Optional[int], + input_payload: Dict[str, Any], + metadata: Dict[str, Any], +) -> WorkflowOperation: + expected = { + "operation_type": operation_type, + "actor": actor, + "command_id": command_id, + "model_run_id": model_run_id, + "input": input_payload, + "metadata": metadata, + } + actual = { + "operation_type": operation.operation_type, + "actor": operation.actor, + "command_id": operation.command_id, + "model_run_id": operation.model_run_id, + "input": decode_json(operation.input_json), + "metadata": decode_json(operation.metadata_json), + } + if actual != expected: + raise HTTPException( + status_code=409, + detail="idempotency_key is already used by a different operation request", + ) + return operation + + +def create_workflow_operation( + db: Session, + *, + workflow_id: int, + operation_type: str, + idempotency_key: str, + correlation_id: Optional[str] = None, + actor: str = "system", + command_id: Optional[int] = None, + model_run_id: Optional[int] = None, + input_payload: Optional[Dict[str, Any]] = None, + metadata: Optional[Dict[str, Any]] = None, + commit: bool = False, +) -> WorkflowOperation: + operation_type = operation_type.strip() + idempotency_key = idempotency_key.strip() + if not operation_type: + raise HTTPException(status_code=400, detail="operation_type is required") + if not idempotency_key: + raise HTTPException(status_code=400, detail="idempotency_key is required") + correlation_id = (correlation_id or str(uuid4())).strip() + if not correlation_id: + raise HTTPException(status_code=400, detail="correlation_id cannot be blank") + actor = validate_actor(actor) + input_payload = input_payload or {} + metadata = metadata or {} + _validate_linked_record( + db, + model=WorkflowCommand, + record_id=command_id, + workflow_id=workflow_id, + label="command_id", + ) + _validate_linked_record( + db, + model=WorkflowModelRun, + record_id=model_run_id, + workflow_id=workflow_id, + label="model_run_id", + ) + + existing = ( + db.query(WorkflowOperation) + .filter( + WorkflowOperation.workflow_id == workflow_id, + WorkflowOperation.idempotency_key == idempotency_key, + ) + .first() + ) + if existing is not None: + return _assert_idempotent_match( + existing, + operation_type=operation_type, + actor=actor, + command_id=command_id, + model_run_id=model_run_id, + input_payload=input_payload, + metadata=metadata, + ) + + operation = WorkflowOperation( + workflow_id=workflow_id, + operation_type=operation_type, + status="queued", + idempotency_key=idempotency_key, + correlation_id=correlation_id, + actor=actor, + command_id=command_id, + model_run_id=model_run_id, + input_json=encode_json(input_payload), + metadata_json=encode_json(metadata), + ) + try: + with db.begin_nested(): + db.add(operation) + db.flush() + except IntegrityError: + existing = ( + db.query(WorkflowOperation) + .filter( + WorkflowOperation.workflow_id == workflow_id, + WorkflowOperation.idempotency_key == idempotency_key, + ) + .first() + ) + if existing is None: + raise + operation = _assert_idempotent_match( + existing, + operation_type=operation_type, + actor=actor, + command_id=command_id, + model_run_id=model_run_id, + input_payload=input_payload, + metadata=metadata, + ) + if commit: + db.commit() + db.refresh(operation) + return operation + + +def transition_workflow_operation( + db: Session, + operation: WorkflowOperation, + *, + status: str, + expected_status: Optional[str] = None, + result_payload: Optional[Dict[str, Any]] = None, + error_payload: Optional[Dict[str, Any]] = None, + metadata: Optional[Dict[str, Any]] = None, + progress: Optional[float] = None, + lease_owner: Optional[str] = None, + lease_expires_at: Optional[datetime] = None, + commit: bool = False, +) -> WorkflowOperation: + if status not in OPERATION_STATUSES: + raise HTTPException( + status_code=400, + detail=( + "operation status must be one of: " + f"{', '.join(sorted(OPERATION_STATUSES))}" + ), + ) + if expected_status is not None and operation.status != expected_status: + raise HTTPException( + status_code=409, + detail=( + f"Operation status changed: expected {expected_status}, " + f"found {operation.status}" + ), + ) + if ( + operation.status == "running" + and operation.lease_owner + and status in TERMINAL_OPERATION_STATUSES + and lease_owner != operation.lease_owner + ): + raise HTTPException( + status_code=409, detail="Operation is leased by another worker" + ) + if operation.status == status and status in TERMINAL_OPERATION_STATUSES: + return operation + if status not in OPERATION_TRANSITIONS[operation.status]: + raise HTTPException( + status_code=409, + detail=f"Operation cannot transition from {operation.status} to {status}", + ) + + now = _now() + operation.status = status + if metadata is not None: + operation.metadata_json = encode_json( + _merge_dicts(decode_json(operation.metadata_json), metadata) + ) + if progress is not None: + operation.progress = _validate_progress(progress) + + if status == "running": + operation.attempt_count = int(operation.attempt_count or 0) + 1 + operation.started_at = operation.started_at or now + operation.completed_at = None + operation.heartbeat_at = now + operation.lease_owner = lease_owner + operation.lease_expires_at = lease_expires_at + operation.error_json = None + else: + operation.completed_at = now + operation.lease_owner = None + operation.lease_expires_at = None + if status == "succeeded": + operation.progress = 1.0 + operation.result_json = encode_json(result_payload or {}) + operation.error_json = None + elif status == "failed": + operation.error_json = encode_json(error_payload or {}) + elif status == "cancelled" and error_payload is not None: + operation.error_json = encode_json(error_payload) + + db.flush() + if commit: + db.commit() + db.refresh(operation) + return operation + + +def heartbeat_workflow_operation( + db: Session, + operation: WorkflowOperation, + *, + progress: Optional[float] = None, + metadata: Optional[Dict[str, Any]] = None, + lease_owner: Optional[str] = None, + lease_expires_at: Optional[datetime] = None, + commit: bool = False, +) -> WorkflowOperation: + if operation.status != "running": + raise HTTPException( + status_code=409, + detail="Only running operations can receive heartbeats", + ) + if lease_owner and operation.lease_owner and lease_owner != operation.lease_owner: + raise HTTPException( + status_code=409, detail="Operation is leased by another worker" + ) + operation.heartbeat_at = _now() + if progress is not None: + operation.progress = _validate_progress(progress) + if metadata is not None: + operation.metadata_json = encode_json( + _merge_dicts(decode_json(operation.metadata_json), metadata) + ) + if lease_owner is not None: + operation.lease_owner = lease_owner + if lease_expires_at is not None: + operation.lease_expires_at = lease_expires_at + db.flush() + if commit: + db.commit() + db.refresh(operation) + return operation + + +def request_workflow_operation_cancellation( + db: Session, + operation: WorkflowOperation, + *, + reason: Optional[str] = None, + commit: bool = False, +) -> WorkflowOperation: + if operation.status in TERMINAL_OPERATION_STATUSES: + return operation + now = _now() + operation.cancellation_requested_at = operation.cancellation_requested_at or now + if reason: + operation.metadata_json = encode_json( + _merge_dicts( + decode_json(operation.metadata_json), + {"cancellation": {"reason": reason}}, + ) + ) + if operation.status == "queued": + operation.status = "cancelled" + operation.completed_at = now + db.flush() + if commit: + db.commit() + db.refresh(operation) + return operation diff --git a/server_api/workflows/router.py b/server_api/workflows/router.py index 15aeb8f1..8c8081b3 100644 --- a/server_api/workflows/router.py +++ b/server_api/workflows/router.py @@ -10,7 +10,7 @@ import requests from fastapi import APIRouter, Depends, HTTPException -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, ValidationError from sqlalchemy.orm import Session from app_event_logger import append_app_event @@ -64,6 +64,7 @@ ) from .bundle_export import build_export_bundle, write_export_bundle_directory from .agent_plan import build_case_study_plan_graph +from .agent_actions import resolve_agent_action, validate_agent_proposal from .evaluation import compute_before_after_evaluation, write_evaluation_report from .metrics import compute_workflow_metrics from .volume_pairs import discover_neuroglancer_volume_pairs @@ -1390,44 +1391,11 @@ def _client_effects_to_command(client_effects: Dict[str, Any]) -> str: def _infer_action_risk(client_effects: Optional[Dict[str, Any]]) -> str: - effects = client_effects or {} - runtime_kind = (effects.get("runtime_action") or {}).get("kind") - workflow_action_kind = (effects.get("workflow_action") or {}).get("kind") - if runtime_kind in {"start_inference", "start_training"}: - return "runs_job" - if runtime_kind in {"stop_inference", "stop_training"}: - return "controls_job" - if runtime_kind == "start_proofreading": - return "loads_editor" - if runtime_kind == "choose_project_data": - return "prefills_form" - if effects.get("mount_project") or effects.get("reset_workspace"): - return "modifies_workspace" - if effects.get("start_new_workflow"): - return "writes_workflow_record" - if workflow_action_kind == "export_bundle": - return "exports_evidence" - if workflow_action_kind in {"compute_evaluation", "propose_retraining_stage"}: - return "writes_workflow_record" - if any(key.startswith("set_") for key in effects): - return "prefills_form" - if effects.get("navigate_to") or effects.get("show_workflow_context"): - return "read_only" - if effects.get("refresh_insights"): - return "read_only" - return "read_only" + return resolve_agent_action("workflow_action", client_effects).risk_level def _requires_action_approval(client_effects: Optional[Dict[str, Any]]) -> bool: - risk = _infer_action_risk(client_effects) - return risk in { - "runs_job", - "controls_job", - "loads_editor", - "exports_evidence", - "writes_workflow_record", - "modifies_workspace", - } + return resolve_agent_action("workflow_action", client_effects).requires_approval def _action_risk_tier(risk_level: str) -> str: @@ -1540,58 +1508,15 @@ def _agent_trace_kwargs(agent: Dict[str, Any]) -> Dict[str, str]: def _specialist_agent_for_action( action_type: str, client_effects: Dict[str, Any] ) -> Dict[str, Any]: - navigate_to = str((client_effects or {}).get("navigate_to") or "") - if action_type in {"start_training", "open_training"} or "training" in navigate_to: - return _agent_descriptor("training_agent") - if ( - action_type in {"start_inference", "open_inference"} - or "inference" in navigate_to - ): - return _agent_descriptor("inference_agent") - if action_type in {"start_proofreading"} or "proofreading" in navigate_to: - return _agent_descriptor("proofreading_agent") - if "visualization" in navigate_to or action_type.startswith("open_visualization"): - return _agent_descriptor("visualization_agent") - if action_type in {"export_bundle"}: - return _agent_descriptor("evidence_agent") - if action_type in {"compute_evaluation"}: - return _agent_descriptor("evaluation_agent") - if ( - action_type in {"mount_project", "choose_project_data"} - or navigate_to == "files" - ): - return _agent_descriptor("data_agent") - if action_type in { - "show_workflow_context", - "refresh_context", - "open_project-progress", - }: - return _agent_descriptor("project_manager") - return _agent_descriptor("project_manager") + definition = resolve_agent_action(action_type, client_effects) + return _agent_descriptor(definition.specialist_agent_type) def _action_type_from_effects( action_id: str, client_effects: Optional[Dict[str, Any]], ) -> str: - effects = client_effects or {} - runtime_kind = (effects.get("runtime_action") or {}).get("kind") - workflow_kind = (effects.get("workflow_action") or {}).get("kind") - if runtime_kind: - return str(runtime_kind) - if workflow_kind: - return str(workflow_kind) - if effects.get("mount_project"): - return "mount_project" - if effects.get("start_new_workflow"): - return "start_new_workflow" - if effects.get("show_workflow_context"): - return "show_workflow_context" - if effects.get("refresh_insights"): - return "refresh_context" - if effects.get("navigate_to"): - return f"open_{effects.get('navigate_to')}" - return action_id + return resolve_agent_action(action_id, client_effects).action_type def _action_target_from_effects( @@ -1829,7 +1754,8 @@ def _build_action_card_payload( requires_approval: bool, disabled_reason: Optional[str], ) -> Dict[str, Any]: - action_type = _action_type_from_effects(action_id, client_effects) + definition = resolve_agent_action(action_id, client_effects) + action_type = definition.action_type specialist_agent = _specialist_agent_for_action(action_type, client_effects) blockers = [disabled_reason] if disabled_reason else [] return { @@ -1855,6 +1781,12 @@ def _build_action_card_payload( "summary_fields": _summary_fields_from_effects(client_effects), "expected_effects": _expected_effects_from_client_effects(client_effects), "executor": "bounded_app_routine", + "execution_owner": definition.execution_owner, + "registry_policy": { + "risk_level": definition.risk_level, + "requires_approval": definition.requires_approval, + "specialist_agent_type": definition.specialist_agent_type, + }, } @@ -8947,6 +8879,10 @@ async def create_agent_action( db: Session = Depends(get_db), ): workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + try: + definition = validate_agent_proposal(body.action, body.payload) + except (ValidationError, ValueError) as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc summary = body.summary or f"Agent proposed: {body.action}" event = append_workflow_event( db, @@ -8955,7 +8891,17 @@ async def create_agent_action( event_type="agent.proposal_created", stage=workflow.stage, summary=summary, - payload={"action": body.action, "params": body.payload}, + payload={ + "action": body.action, + "params": body.payload, + "registry": { + "action_type": definition.action_type, + "risk_level": definition.risk_level, + "requires_approval": definition.requires_approval, + "execution_owner": definition.execution_owner, + "specialist_agent_type": definition.specialist_agent_type, + }, + }, approval_status="pending", commit=True, ) @@ -8997,6 +8943,7 @@ async def approve_agent_action( if action == "start_training_run": client_effects = _training_run_effects_from_proposal(workflow, params) + resolve_agent_action(str(action), client_effects) corrected_mask_path = ( client_effects.get("set_training_label_path") or params.get("label_path") @@ -9080,6 +9027,10 @@ async def approve_agent_action( status_code=400, detail="Approved client-effect action is missing client_effects.", ) + try: + resolve_agent_action(str(action), client_effects) + except ValidationError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc proposal.approval_status = "approved" db.commit() db.refresh(proposal) diff --git a/server_api/workflows/volume_io.py b/server_api/workflows/volume_io.py index c2e4b1ba..79fdd1f6 100644 --- a/server_api/workflows/volume_io.py +++ b/server_api/workflows/volume_io.py @@ -1,7 +1,9 @@ from __future__ import annotations +from abc import ABC, abstractmethod +from dataclasses import dataclass from pathlib import Path -from typing import Any, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, List, Optional, Sequence, Tuple, Union import numpy as np @@ -34,6 +36,128 @@ ) +@dataclass(frozen=True) +class VolumeMetadata: + """Storage-level metadata available without materializing voxel data.""" + + path: str + format: str + shape: Tuple[int, ...] + dtype: np.dtype + dataset_key: Optional[str] = None + chunks: Optional[Tuple[int, ...]] = None + + @property + def ndim(self) -> int: + return len(self.shape) + + +class VolumeStore(ABC): + """A bounded region-reader for a single array inside a volume artifact.""" + + @property + @abstractmethod + def metadata(self) -> VolumeMetadata: + raise NotImplementedError + + @property + def shape(self) -> Tuple[int, ...]: + return self.metadata.shape + + @property + def dtype(self) -> np.dtype: + return self.metadata.dtype + + @property + def ndim(self) -> int: + return self.metadata.ndim + + @abstractmethod + def read( + self, + crop: CropSpec = None, + *, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", + ) -> np.ndarray: + raise NotImplementedError + + def close(self) -> None: + """Release resources held by the backing artifact.""" + + def __enter__(self) -> "VolumeStore": + return self + + def __exit__(self, *_exc_info: Any) -> None: + self.close() + + +class ArrayVolumeStore(VolumeStore): + """VolumeStore adapter for array-like objects supporting basic indexing.""" + + def __init__( + self, + data: Any, + *, + path: Path, + format: str, + dataset_key: Optional[str] = None, + close: Optional[Callable[[], None]] = None, + ) -> None: + self._data = data + self._close = close + self._closed = False + chunks = getattr(data, "chunks", None) + self._metadata = VolumeMetadata( + path=str(path), + format=format, + shape=tuple(int(value) for value in data.shape), + dtype=np.dtype(data.dtype), + dataset_key=dataset_key, + chunks=( + tuple(int(value) for value in chunks) + if chunks is not None and all(value is not None for value in chunks) + else None + ), + ) + + @property + def metadata(self) -> VolumeMetadata: + return self._metadata + + def read( + self, + crop: CropSpec = None, + *, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", + ) -> np.ndarray: + if self._closed: + raise RuntimeError("Volume store is closed") + return _as_array( + self._data, + parse_crop(crop), + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + + def __getitem__(self, key: Any) -> Any: + """Expose storage-backed slicing to consumers such as Neuroglancer.""" + if self._closed: + raise RuntimeError("Volume store is closed") + return self._data[key] + + def close(self) -> None: + if self._closed: + return + self._closed = True + if self._close is not None: + self._close() + + def split_dataset_ref(path: str) -> Tuple[str, Optional[str]]: if "::" not in path: return path, None @@ -216,28 +340,6 @@ def _select_h5_dataset(handle: Any, dataset_key: Optional[str]) -> Any: return handle[datasets[0]] -def _read_hdf5( - path: Path, - dataset_key: Optional[str], - crop: Optional[Tuple[slice, ...]], - *, - channel: Optional[int] = None, - reference_ndim: Optional[int] = None, - label: str = "volume", -) -> np.ndarray: - import h5py - - with h5py.File(path, "r") as handle: - dataset = _select_h5_dataset(handle, dataset_key) - return _as_array( - dataset, - crop, - channel=channel, - reference_ndim=reference_ndim, - label=label, - ) - - def _is_zarr_array(value: Any) -> bool: return ( hasattr(value, "shape") @@ -287,197 +389,194 @@ def _select_zarr_array(store: Any, dataset_key: Optional[str]) -> Any: return store[arrays[0]] -def _read_zarr( - path: Path, - dataset_key: Optional[str], - crop: Optional[Tuple[slice, ...]], - *, - channel: Optional[int] = None, - reference_ndim: Optional[int] = None, - label: str = "volume", -) -> np.ndarray: - import zarr - - store = zarr.open(str(path), mode="r") - array = _select_zarr_array(store, dataset_key) - return _as_array( - array, - crop, - channel=channel, - reference_ndim=reference_ndim, - label=label, - ) - - -def _read_npz( - path: Path, - dataset_key: Optional[str], - crop: Optional[Tuple[slice, ...]], - *, - channel: Optional[int] = None, - reference_ndim: Optional[int] = None, - label: str = "volume", -) -> np.ndarray: - with np.load(path) as loaded: - key = dataset_key - if not key: - for candidate in COMMON_DATASET_NAMES: - if candidate in loaded: - key = candidate - break - if not key: - keys = list(loaded.keys()) - if not keys: - raise ValueError("NPZ file does not contain any arrays") - key = keys[0] - return _as_array( - loaded[key], - crop, - channel=channel, - reference_ndim=reference_ndim, - label=label, - ) - - -def _read_nifti( - path: Path, - crop: Optional[Tuple[slice, ...]], - *, - channel: Optional[int] = None, - reference_ndim: Optional[int] = None, - label: str = "volume", -) -> np.ndarray: - try: - import nibabel as nib - except Exception as exc: # pragma: no cover - optional dependency - raise RuntimeError("nibabel is required to read NIfTI volumes") from exc - - image = nib.load(str(path)) - data = image.dataobj - return _as_array( - data, - crop, - channel=channel, - reference_ndim=reference_ndim, - label=label, - ) - - -def _read_mrc( - path: Path, - crop: Optional[Tuple[slice, ...]], - *, - channel: Optional[int] = None, - reference_ndim: Optional[int] = None, - label: str = "volume", -) -> np.ndarray: - try: - import mrcfile - except Exception as exc: # pragma: no cover - optional dependency - raise RuntimeError("mrcfile is required to read MRC/MAP volumes") from exc - - with mrcfile.open(str(path), permissive=True) as handle: - return _as_array( - handle.data, - crop, - channel=channel, - reference_ndim=reference_ndim, - label=label, - ) - - -def load_volume( +def _close_all(*resources: Any) -> Callable[[], None]: + def close() -> None: + first_error: Optional[Exception] = None + for resource in resources: + close_resource = getattr(resource, "close", None) + if not callable(close_resource): + continue + try: + close_resource() + except Exception as exc: # pragma: no cover - defensive cleanup + first_error = first_error or exc + if first_error is not None: + raise first_error + + return close + + +def _select_npz_array(loaded: Any, dataset_key: Optional[str]) -> Tuple[str, Any]: + key = dataset_key + if not key: + for candidate in COMMON_DATASET_NAMES: + if candidate in loaded: + key = candidate + break + if not key: + keys = list(loaded.keys()) + if not keys: + raise ValueError("NPZ file does not contain any arrays") + key = keys[0] + if key not in loaded: + raise ValueError(f"NPZ array {key!r} not found") + return key, loaded[key] + + +def open_volume_store( path: str, *, dataset_key: Optional[str] = None, - crop: CropSpec = None, - channel: Optional[int] = None, - reference_ndim: Optional[int] = None, - label: str = "volume", -) -> np.ndarray: +) -> VolumeStore: + """Open a volume for metadata inspection and bounded region reads. + + Callers should use this as a context manager. HDF5, NPY, Zarr/N5, NIfTI, + MRC, and TIFF expose their backing array directly; indexing therefore occurs + before NumPy materialization. Formats without random-access support retain an + eager compatibility fallback. + """ + file_path, inline_dataset_key = split_dataset_ref(str(path)) dataset_key = dataset_key or inline_dataset_key target = Path(file_path).expanduser() if not target.exists(): raise FileNotFoundError(f"Volume artifact does not exist: {target}") - crop_slices = parse_crop(crop) lower_name = target.name.lower() lower_path = str(target).lower() if lower_name.endswith((".h5", ".hdf5", ".hdf")): - return _read_hdf5( - target, - dataset_key, - crop_slices, - channel=channel, - reference_ndim=reference_ndim, - label=label, - ) + import h5py + + handle = h5py.File(target, "r") + try: + data = _select_h5_dataset(handle, dataset_key) + selected_key = data.name.lstrip("/") + return ArrayVolumeStore( + data, + path=target, + format="hdf5", + dataset_key=selected_key, + close=handle.close, + ) + except Exception: + handle.close() + raise + if lower_name.endswith((".tif", ".tiff", ".ome.tif", ".ome.tiff")): import tifffile + import zarr + + handle = tifffile.TiffFile(str(target)) + try: + tiff_store = handle.series[0].aszarr() + data = zarr.open(tiff_store, mode="r") + return ArrayVolumeStore( + data, + path=target, + format="ome-tiff" if ".ome.tif" in lower_name else "tiff", + close=_close_all(tiff_store, handle), + ) + except Exception: + handle.close() + return ArrayVolumeStore( + tifffile.imread(str(target)), + path=target, + format="ome-tiff" if ".ome.tif" in lower_name else "tiff", + ) - return _as_array( - tifffile.imread(str(target)), - crop_slices, - channel=channel, - reference_ndim=reference_ndim, - label=label, - ) if lower_name.endswith(".npy"): - return _as_array( - np.load(target, mmap_mode="r"), - crop_slices, - channel=channel, - reference_ndim=reference_ndim, - label=label, + data = np.load(target, mmap_mode="r") + mmap = getattr(data, "_mmap", None) + return ArrayVolumeStore( + data, + path=target, + format="npy", + close=getattr(mmap, "close", None), ) + if lower_name.endswith(".npz"): - return _read_npz( - target, - dataset_key, - crop_slices, - channel=channel, - reference_ndim=reference_ndim, - label=label, - ) + loaded = np.load(target) + try: + selected_key, data = _select_npz_array(loaded, dataset_key) + return ArrayVolumeStore( + data, + path=target, + format="npz", + dataset_key=selected_key, + close=loaded.close, + ) + except Exception: + loaded.close() + raise + if target.is_dir() or lower_name.endswith((".zarr", ".n5")): - return _read_zarr( - target, - dataset_key, - crop_slices, - channel=channel, - reference_ndim=reference_ndim, - label=label, + import zarr + + root = zarr.open(str(target), mode="r") + data = _select_zarr_array(root, dataset_key) + selected_key = dataset_key or getattr(data, "path", None) or None + return ArrayVolumeStore( + data, + path=target, + format="n5" if lower_name.endswith(".n5") else "zarr", + dataset_key=selected_key, ) + if lower_path.endswith((".nii", ".nii.gz")): - return _read_nifti( - target, - crop_slices, - channel=channel, - reference_ndim=reference_ndim, - label=label, + try: + import nibabel as nib + except Exception as exc: # pragma: no cover - optional dependency + raise RuntimeError("nibabel is required to read NIfTI volumes") from exc + image = nib.load(str(target)) + return ArrayVolumeStore( + image.dataobj, + path=target, + format="nifti", + close=getattr(image, "uncache", None), ) + if lower_name.endswith((".mrc", ".map", ".rec")): - return _read_mrc( - target, - crop_slices, - channel=channel, - reference_ndim=reference_ndim, - label=label, + try: + import mrcfile + except Exception as exc: # pragma: no cover - optional dependency + raise RuntimeError("mrcfile is required to read MRC/MAP volumes") from exc + handle = mrcfile.open(str(target), permissive=True) + return ArrayVolumeStore( + handle.data, + path=target, + format="mrc", + close=handle.close, ) + if lower_name.endswith((".png", ".jpg", ".jpeg", ".bmp")): import imageio.v3 as iio - return _as_array( + return ArrayVolumeStore( iio.imread(target), - crop_slices, - channel=channel, - reference_ndim=reference_ndim, - label=label, + path=target, + format=target.suffix.lower().lstrip("."), ) raise ValueError( f"Unsupported volume format for {target}. Supported formats: " + "; ".join(SUPPORTED_VOLUME_FORMATS) ) + + +def load_volume( + path: str, + *, + dataset_key: Optional[str] = None, + crop: CropSpec = None, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", +) -> np.ndarray: + with open_volume_store(path, dataset_key=dataset_key) as store: + return store.read( + crop, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) diff --git a/tests/test_agent_action_registry.py b/tests/test_agent_action_registry.py new file mode 100644 index 00000000..c48077a0 --- /dev/null +++ b/tests/test_agent_action_registry.py @@ -0,0 +1,99 @@ +import pytest +from pydantic import ValidationError + +from server_api.workflows.agent_actions import ( + resolve_agent_action, + validate_agent_proposal, +) + + +def test_runtime_action_registry_owns_risk_and_approval_policy(): + action = resolve_agent_action( + "launch-model", + { + "navigate_to": "training", + "runtime_action": {"kind": "start_training"}, + }, + ) + + assert action.action_type == "start_training" + assert action.risk_level == "runs_job" + assert action.requires_approval is True + assert action.execution_owner == "server_runtime" + assert action.specialist_agent_type == "training_agent" + + +def test_navigation_action_remains_browser_owned_and_read_only(): + action = resolve_agent_action("show-files", {"navigate_to": "files"}) + + assert action.action_type == "open_files" + assert action.risk_level == "read_only" + assert action.requires_approval is False + assert action.execution_owner == "browser_navigation" + assert action.specialist_agent_type == "data_agent" + + +def test_prefill_action_is_typed_without_becoming_domain_execution(): + action = resolve_agent_action( + "prepare-training", + { + "navigate_to": "training", + "set_training_image_path": "/data/image.zarr", + "set_training_label_path": "/data/labels.zarr", + }, + ) + + assert action.risk_level == "prefills_form" + assert action.requires_approval is False + assert action.execution_owner == "browser_navigation" + + +def test_unknown_client_effect_cannot_bypass_registry_validation(): + with pytest.raises(ValidationError): + resolve_agent_action("mystery", {"silently_delete_project": True}) + + +def test_registered_runtime_kind_rejects_unregistered_parameters(): + with pytest.raises(ValidationError): + resolve_agent_action( + "launch-model", + { + "runtime_action": { + "kind": "start_inference", + "shell_command": "rm -rf /", + } + }, + ) + + +def test_registered_workflow_kind_rejects_unregistered_parameters(): + with pytest.raises(ValidationError): + resolve_agent_action( + "export", + { + "workflow_action": { + "kind": "export_bundle", + "destination": "/unapproved/path", + } + }, + ) + + +def test_persisted_client_effect_proposal_is_validated_before_approval(): + action = validate_agent_proposal( + "run_client_effects", + { + "client_effects": { + "navigate_to": "inference", + "runtime_action": {"kind": "start_inference"}, + } + }, + ) + + assert action.action_type == "start_inference" + assert action.execution_owner == "server_runtime" + + +def test_unknown_persisted_proposal_type_is_rejected(): + with pytest.raises(ValueError, match="Unsupported agent proposal action"): + validate_agent_proposal("invented_action", {}) diff --git a/tests/test_error_contract.py b/tests/test_error_contract.py new file mode 100644 index 00000000..eeaa54b3 --- /dev/null +++ b/tests/test_error_contract.py @@ -0,0 +1,88 @@ +import unittest + +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient +from pydantic import BaseModel + +from server_api.errors import install_error_handlers + + +class ExamplePayload(BaseModel): + count: int + + +def build_test_app() -> FastAPI: + app = FastAPI() + install_error_handlers(app) + + @app.get("/missing") + def missing(): + raise HTTPException(status_code=404, detail="Dataset was not found") + + @app.post("/validate") + def validate(payload: ExamplePayload): + return payload + + @app.get("/failure") + def failure(): + raise RuntimeError("private implementation detail") + + return app + + +class ErrorContractTests(unittest.TestCase): + def setUp(self): + self.client = TestClient(build_test_app(), raise_server_exceptions=False) + + def test_http_error_preserves_detail_and_adds_recovery_contract(self): + response = self.client.get("/missing", headers={"x-request-id": "req-123"}) + + self.assertEqual(response.status_code, 404) + self.assertEqual(response.headers["x-request-id"], "req-123") + self.assertEqual(response.json()["detail"], "Dataset was not found") + self.assertEqual( + response.json()["error"], + { + "schema_version": 1, + "code": "not_found", + "category": "resource", + "title": "Resource not found", + "retryable": False, + "recovery_actions": ["go_back"], + "message": "Dataset was not found", + "request_id": "req-123", + }, + ) + + def test_validation_error_includes_field_diagnostics(self): + response = self.client.post("/validate", json={"count": "invalid"}) + + self.assertEqual(response.status_code, 422) + body = response.json() + self.assertEqual(body["error"]["code"], "validation_failed") + self.assertEqual(body["detail"], body["error"]["validation_errors"]) + self.assertTrue(body["error"]["request_id"]) + + def test_invalid_caller_request_id_is_not_reflected(self): + response = self.client.get( + "/missing", headers={"x-request-id": "invalid request id"} + ) + + self.assertEqual(response.status_code, 404) + self.assertNotEqual(response.headers["x-request-id"], "invalid request id") + self.assertEqual( + response.headers["x-request-id"], response.json()["error"]["request_id"] + ) + + def test_unexpected_error_does_not_expose_exception_text(self): + response = self.client.get("/failure") + + self.assertEqual(response.status_code, 500) + body = response.json() + self.assertEqual(body["error"]["code"], "internal_error") + self.assertNotIn("private implementation detail", response.text) + self.assertEqual(response.headers["x-request-id"], body["error"]["request_id"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_neuroglancer_storage_sources.py b/tests/test_neuroglancer_storage_sources.py new file mode 100644 index 00000000..c63f311e --- /dev/null +++ b/tests/test_neuroglancer_storage_sources.py @@ -0,0 +1,114 @@ +import numpy as np +import pytest + +from server_api.main import ( + _NeuroglancerSegmentationStore, + _build_neuroglancer_local_volume_source, + _open_neuroglancer_volume_sources, +) +from server_api.workflows.volume_io import ArrayVolumeStore + + +class RecordingLabelArray: + shape = (8, 16, 24) + dtype = np.dtype(np.int32) + chunks = (2, 4, 6) + + def __init__(self, fill_value=7): + self.fill_value = fill_value + self.requested_keys = [] + + def __array__(self, *_args, **_kwargs): + raise AssertionError("full label volume was materialized") + + def __getitem__(self, key): + self.requested_keys.append(key) + return np.full((2, 4, 6), self.fill_value, dtype=self.dtype) + + +def test_segmentation_source_validates_and_converts_only_requested_chunk(tmp_path): + backing = RecordingLabelArray() + store = ArrayVolumeStore( + backing, + path=tmp_path / "labels.zarr", + format="zarr", + ) + source = _NeuroglancerSegmentationStore(store) + + chunk = source[1:3, 4:8, 6:12] + + assert source.shape == backing.shape + assert source.dtype == np.dtype(np.uint64) + assert chunk.dtype == np.uint64 + assert backing.requested_keys == [(slice(1, 3), slice(4, 8), slice(6, 12))] + + +def test_segmentation_source_rejects_negative_labels_per_chunk(tmp_path): + store = ArrayVolumeStore( + RecordingLabelArray(fill_value=-1), + path=tmp_path / "negative-labels.zarr", + format="zarr", + ) + source = _NeuroglancerSegmentationStore(store) + + with pytest.raises(ValueError, match="non-negative"): + source[0:2, 0:4, 0:6] + + +def test_segmentation_source_requires_preprocessed_3d_labels(tmp_path): + store = ArrayVolumeStore( + np.zeros((2, 8, 16, 24), dtype=np.float32), + path=tmp_path / "prediction.zarr", + format="zarr", + ) + + with pytest.raises(ValueError, match="requires a 3D label volume"): + _NeuroglancerSegmentationStore(store) + + +def test_neuroglancer_local_volume_reads_hdf5_subvolumes_on_demand(tmp_path): + h5py = pytest.importorskip("h5py") + neuroglancer = pytest.importorskip("neuroglancer") + image_path = tmp_path / "image.h5" + label_path = tmp_path / "labels.h5" + with h5py.File(image_path, "w") as handle: + handle.create_dataset( + "data", + data=np.arange(8 * 16 * 24, dtype=np.uint8).reshape(8, 16, 24), + chunks=(2, 4, 6), + ) + with h5py.File(label_path, "w") as handle: + handle.create_dataset( + "data", + data=np.ones((8, 16, 24), dtype=np.int32), + chunks=(2, 4, 6), + ) + + image, labels, resources = _open_neuroglancer_volume_sources(image_path, label_path) + try: + dimensions = neuroglancer.CoordinateSpace( + names=["z", "y", "x"], units=["nm", "nm", "nm"], scales=[1, 1, 1] + ) + image_volume = _build_neuroglancer_local_volume_source( + neuroglancer, image, dimensions, volume_type="image" + ) + label_volume = _build_neuroglancer_local_volume_source( + neuroglancer, labels, dimensions, volume_type="segmentation" + ) + start = np.array([1, 2, 3], dtype=np.int64) + end = np.array([3, 6, 9], dtype=np.int64) + + image_payload, image_content_type = image_volume.get_encoded_subvolume( + "raw", start, end, "1,1,1" + ) + label_payload, label_content_type = label_volume.get_encoded_subvolume( + "raw", start, end, "1,1,1" + ) + + assert image_payload + assert label_payload + assert image_content_type == "application/octet-stream" + assert label_content_type == "application/octet-stream" + finally: + for resource in resources: + resource.close() diff --git a/tests/test_neuroglancer_url_contract.py b/tests/test_neuroglancer_url_contract.py index e051e31b..b95a8385 100644 --- a/tests/test_neuroglancer_url_contract.py +++ b/tests/test_neuroglancer_url_contract.py @@ -12,6 +12,14 @@ def __init__(self, token): self.token = token +class DummyResource: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def make_request(*, host="localhost:4242", scheme="http", extra_headers=None): headers = [(b"host", host.encode("utf-8"))] for key, value in (extra_headers or {}).items(): @@ -125,6 +133,29 @@ def test_retain_neuroglancer_viewer_evicts_oldest_over_capacity(self): finally: server_main.PYTC_NEUROGLANCER_MAX_VIEWERS = previous_limit + def test_evicted_viewer_closes_storage_backing_resources(self): + previous_limit = server_main.PYTC_NEUROGLANCER_MAX_VIEWERS + server_main.PYTC_NEUROGLANCER_MAX_VIEWERS = 1 + old_resource = DummyResource() + try: + server_main._retain_neuroglancer_viewer( + DummyViewer("old"), + public_url="https://viewer.example.com/ng/v/old/", + internal_viewer_url="http://127.0.0.1:4244/v/old/", + mode="visualization", + resources=[old_resource], + ) + server_main._retain_neuroglancer_viewer( + DummyViewer("new"), + public_url="https://viewer.example.com/ng/v/new/", + internal_viewer_url="http://127.0.0.1:4244/v/new/", + mode="visualization", + ) + + self.assertTrue(old_resource.closed) + finally: + server_main.PYTC_NEUROGLANCER_MAX_VIEWERS = previous_limit + if __name__ == "__main__": unittest.main() diff --git a/tests/test_pytc_runtime_routes.py b/tests/test_pytc_runtime_routes.py index aee1fc9f..ce53075a 100644 --- a/tests/test_pytc_runtime_routes.py +++ b/tests/test_pytc_runtime_routes.py @@ -23,6 +23,8 @@ _resolve_raw_image_shader, ) from server_api.main import _coerce_neuroglancer_scales +from server_api.workflows.db_models import WorkflowCommand +from server_api.workflows.service import encode_json from server_pytc.main import app as server_pytc_app from server_pytc.services import model as model_service @@ -521,6 +523,7 @@ def test_durable_training_command_runner_launches_worker_and_records_state(self) captured = {} def fake_worker(method, endpoint, json_body=None, **_kwargs): + captured["calls"] = captured.get("calls", 0) + 1 captured["method"] = method captured["endpoint"] = endpoint captured["json_body"] = json_body @@ -530,11 +533,28 @@ def fake_worker(method, endpoint, json_body=None, **_kwargs): run_response = self.client.post( f"/api/workflows/{workflow_id}/commands/{command['id']}/run" ) + duplicate_run_response = self.client.post( + f"/api/workflows/{workflow_id}/commands/{command['id']}/run" + ) self.assertEqual(run_response.status_code, 200) + self.assertEqual(duplicate_run_response.status_code, 200) payload = run_response.json() self.assertEqual(payload["command"]["status"], "submitted") self.assertEqual(payload["command"]["attempt_count"], 1) + self.assertEqual(payload["operation"]["status"], "succeeded") + self.assertEqual(payload["operation"]["operation_type"], "start_training") + self.assertEqual(payload["operation"]["command_id"], command["id"]) + self.assertEqual( + payload["operation"]["idempotency_key"], + f"workflow-command:{command['id']}:attempt:1", + ) + self.assertEqual(payload["operation"]["result"]["worker"]["pid"], 4242) + self.assertEqual( + duplicate_run_response.json()["operation"]["id"], + payload["operation"]["id"], + ) + self.assertEqual(captured["calls"], 1) self.assertEqual(payload["worker"]["pid"], 4242) self.assertEqual(captured["method"], "post") self.assertEqual(captured["endpoint"], "/start_model_training") @@ -556,6 +576,13 @@ def fake_worker(method, endpoint, json_body=None, **_kwargs): self.assertEqual(commands_response.status_code, 200) self.assertEqual(commands_response.json()[0]["status"], "submitted") + operations_response = self.client.get( + f"/api/workflows/{workflow_id}/operations" + ) + self.assertEqual(operations_response.status_code, 200) + self.assertEqual(len(operations_response.json()), 1) + self.assertEqual(operations_response.json()[0]["status"], "succeeded") + events_response = self.client.get(f"/api/workflows/{workflow_id}/events") self.assertEqual(events_response.status_code, 200) started_events = [ @@ -570,6 +597,83 @@ def fake_worker(method, endpoint, json_body=None, **_kwargs): f"workflow-command-{command['id']}", ) + def test_durable_training_command_failure_records_retryable_operation(self): + workflow_id = self._workflow_id() + project_root = pathlib.Path(self.temp_dir.name) / "failed-command-project" + image_path = project_root / "image.h5" + label_path = project_root / "label.tif" + output_path = project_root / "output" + project_root.mkdir(parents=True) + output_path.mkdir() + image_path.write_text("image", encoding="utf-8") + label_path.write_text("label", encoding="utf-8") + + db = self.SessionLocal() + try: + command = WorkflowCommand( + workflow_id=workflow_id, + command_type="start_training", + status="queued", + idempotency_key="test:failed-training-command", + actor="user", + input_json=encode_json( + { + "trainingConfig": "DATASET: {}\n", + "outputPath": str(output_path), + "logPath": str(output_path), + "inputImagePath": str(image_path), + "inputLabelPath": str(label_path), + } + ), + ) + db.add(command) + db.commit() + db.refresh(command) + command_id = command.id + finally: + db.close() + + with patch( + "server_api.main._proxy_to_worker", + side_effect=HTTPException(status_code=503, detail="worker unavailable"), + ): + failed_response = self.client.post( + f"/api/workflows/{workflow_id}/commands/{command_id}/run" + ) + + self.assertEqual(failed_response.status_code, 503) + commands_response = self.client.get(f"/api/workflows/{workflow_id}/commands") + failed_command = next( + item for item in commands_response.json() if item["id"] == command_id + ) + self.assertEqual(failed_command["status"], "retry_pending") + self.assertEqual(failed_command["attempt_count"], 1) + + operations_response = self.client.get( + f"/api/workflows/{workflow_id}/operations" + ) + self.assertEqual(operations_response.status_code, 200) + operations = operations_response.json() + self.assertEqual(len(operations), 1) + self.assertEqual(operations[0]["status"], "failed") + self.assertEqual(operations[0]["command_id"], command_id) + self.assertEqual(operations[0]["error"]["status_code"], 503) + + with patch( + "server_api.main._proxy_to_worker", + return_value={"status": "started", "pid": 4243}, + ): + retry_response = self.client.post( + f"/api/workflows/{workflow_id}/commands/{command_id}/run" + ) + + self.assertEqual(retry_response.status_code, 200) + self.assertEqual(retry_response.json()["operation"]["status"], "succeeded") + self.assertEqual( + retry_response.json()["operation"]["idempotency_key"], + f"workflow-command:{command_id}:attempt:2", + ) + class ModelServiceTests(unittest.TestCase): def tearDown(self): @@ -652,12 +756,14 @@ def test_runtime_log_lines_are_exported_to_app_event_log(self): self.assertEqual(records[1]["stream"], "stdout") def test_detect_chunk_tile_mismatch_for_direct_h5_volume(self): - diagnostic = model_service._detect_chunk_tile_mismatch(""" + diagnostic = model_service._detect_chunk_tile_mismatch( + """ DATASET: DO_CHUNK_TITLE: 1 IMAGE_NAME: /tmp/train-volume.h5 LABEL_NAME: /tmp/train-label.h5 -""") +""" + ) self.assertIsNotNone(diagnostic) self.assertEqual(diagnostic["code"], "tile_dataset_direct_volume_mismatch") diff --git a/tests/test_volume_io.py b/tests/test_volume_io.py index 9e6e36d4..b8dd6dc4 100644 --- a/tests/test_volume_io.py +++ b/tests/test_volume_io.py @@ -4,7 +4,12 @@ h5py = pytest.importorskip("h5py") pytest.importorskip("tifffile") -from server_api.workflows.volume_io import load_volume, parse_crop +from server_api.workflows.volume_io import ( + ArrayVolumeStore, + load_volume, + open_volume_store, + parse_crop, +) def test_parse_crop_accepts_voxel_slice_strings(): @@ -27,6 +32,102 @@ def test_load_volume_reads_hdf5_dataset_with_crop(tmp_path): np.testing.assert_array_equal(loaded, volume[1:3, 2:5, 1:4]) +def test_open_volume_store_exposes_metadata_without_reading_data(tmp_path): + path = tmp_path / "volume.h5" + with h5py.File(path, "w") as handle: + handle.create_dataset( + "data", + shape=(64, 128, 256), + chunks=(8, 32, 32), + dtype=np.uint16, + ) + + with open_volume_store(str(path), dataset_key="data") as store: + assert store.metadata.shape == (64, 128, 256) + assert store.metadata.chunks == (8, 32, 32) + assert store.metadata.dtype == np.dtype(np.uint16) + assert store.metadata.dataset_key == "data" + assert store.metadata.format == "hdf5" + + +def test_volume_store_indexes_region_before_numpy_materialization(tmp_path): + class RecordingArray: + shape = (100, 200, 300) + dtype = np.dtype(np.uint16) + chunks = (10, 20, 30) + + def __init__(self): + self.requested_keys = [] + + def __array__(self, *_args, **_kwargs): + raise AssertionError("full backing array was materialized") + + def __getitem__(self, key): + self.requested_keys.append(key) + return np.zeros((2, 3, 4), dtype=self.dtype) + + backing = RecordingArray() + store = ArrayVolumeStore( + backing, + path=tmp_path / "recording.zarr", + format="zarr", + ) + + result = store.read("1:3,10:13,20:24") + + assert result.shape == (2, 3, 4) + assert backing.requested_keys == [(slice(1, 3), slice(10, 13), slice(20, 24))] + + +def test_volume_store_supports_storage_backed_array_protocol(tmp_path): + class RecordingArray: + shape = (100, 200, 300) + dtype = np.dtype(np.uint16) + chunks = (10, 20, 30) + + def __init__(self): + self.requested_keys = [] + + def __array__(self, *_args, **_kwargs): + raise AssertionError("full backing array was materialized") + + def __getitem__(self, key): + self.requested_keys.append(key) + return np.ones((2, 3, 4), dtype=self.dtype) + + backing = RecordingArray() + store = ArrayVolumeStore( + backing, + path=tmp_path / "viewer.zarr", + format="zarr", + ) + + region = store[1:3, 10:13, 20:24] + + assert store.shape == (100, 200, 300) + assert store.ndim == 3 + assert store.dtype == np.dtype(np.uint16) + assert region.shape == (2, 3, 4) + assert backing.requested_keys == [(slice(1, 3), slice(10, 13), slice(20, 24))] + + +def test_volume_store_context_closes_backing_resource(tmp_path): + closed = [] + store = ArrayVolumeStore( + np.zeros((2, 3, 4), dtype=np.uint8), + path=tmp_path / "volume.npy", + format="npy", + close=lambda: closed.append(True), + ) + + with store: + assert store.read("0:1").shape == (1, 3, 4) + + assert closed == [True] + with pytest.raises(RuntimeError, match="closed"): + store.read("0:1") + + def test_load_volume_reads_inline_hdf5_dataset_reference(tmp_path): path = tmp_path / "volume.h5" volume = np.arange(2 * 3 * 4, dtype=np.uint8).reshape(2, 3, 4) @@ -70,3 +171,35 @@ def test_load_volume_rejects_channel_for_already_3d_volume(tmp_path): reference_ndim=3, label="mask", ) + + +def test_load_volume_reads_compressed_tiff_crop(tmp_path): + tifffile = pytest.importorskip("tifffile") + path = tmp_path / "volume.tif" + volume = np.arange(8 * 16 * 24, dtype=np.uint16).reshape(8, 16, 24) + tifffile.imwrite( + path, + volume, + compression="zlib", + metadata={"axes": "ZYX"}, + ) + + loaded = load_volume(str(path), crop="2:5,4:9,6:12") + + np.testing.assert_array_equal(loaded, volume[2:5, 4:9, 6:12]) + + +def test_open_volume_store_reads_zarr_region(tmp_path): + zarr = pytest.importorskip("zarr") + path = tmp_path / "volume.zarr" + volume = np.arange(6 * 12 * 18, dtype=np.uint16).reshape(6, 12, 18) + root = zarr.open(str(path), mode="w") + create_array = getattr(root, "create_array", root.create_dataset) + create_array("raw", data=volume, chunks=(2, 4, 6)) + + with open_volume_store(f"{path}::raw") as store: + loaded = store.read("1:4,3:8,5:11") + assert store.metadata.shape == volume.shape + assert store.metadata.chunks == (2, 4, 6) + + np.testing.assert_array_equal(loaded, volume[1:4, 3:8, 5:11]) diff --git a/tests/test_workflow_operations.py b/tests/test_workflow_operations.py new file mode 100644 index 00000000..1adaaf39 --- /dev/null +++ b/tests/test_workflow_operations.py @@ -0,0 +1,240 @@ +import pathlib +import tempfile +import unittest + +import pytest + +pytest.importorskip("sqlalchemy") +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +pytest.importorskip("fastapi") +from fastapi.testclient import TestClient + +from server_api.auth import database as auth_database +from server_api.auth import models +from server_api.main import app as server_api_app + + +class WorkflowOperationRouteTests(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.db_path = pathlib.Path(self.temp_dir.name) / "operation-test.db" + self.engine = create_engine( + f"sqlite:///{self.db_path}", connect_args={"check_same_thread": False} + ) + self.SessionLocal = sessionmaker( + autocommit=False, autoflush=False, bind=self.engine + ) + models.Base.metadata.create_all(bind=self.engine) + + def override_get_db(): + db = self.SessionLocal() + try: + yield db + finally: + db.close() + + server_api_app.dependency_overrides[auth_database.get_db] = override_get_db + self.client = TestClient(server_api_app) + response = self.client.get("/api/workflows/current") + self.assertEqual(response.status_code, 200) + self.workflow_id = response.json()["workflow"]["id"] + + def tearDown(self): + server_api_app.dependency_overrides.clear() + self.engine.dispose() + self.temp_dir.cleanup() + + def _create_operation(self, key="train:dataset-a:v1", **overrides): + body = { + "operation_type": "training", + "idempotency_key": key, + "actor": "user", + "input": {"config_path": "/tmp/config.yaml"}, + "metadata": {"source": "test"}, + **overrides, + } + return self.client.post( + f"/api/workflows/{self.workflow_id}/operations", + json=body, + ) + + def test_operation_lifecycle_is_persisted_and_queryable(self): + created_response = self._create_operation() + self.assertEqual(created_response.status_code, 200) + created = created_response.json() + self.assertEqual(created["status"], "queued") + self.assertEqual(created["attempt_count"], 0) + self.assertTrue(created["correlation_id"]) + + running_response = self.client.post( + ( + f"/api/workflows/{self.workflow_id}/operations/" + f"{created['id']}/transitions" + ), + json={ + "status": "running", + "expected_status": "queued", + "lease_owner": "worker-1", + "progress": 0.1, + }, + ) + self.assertEqual(running_response.status_code, 200) + running = running_response.json() + self.assertEqual(running["status"], "running") + self.assertEqual(running["attempt_count"], 1) + self.assertEqual(running["lease_owner"], "worker-1") + self.assertIsNotNone(running["started_at"]) + self.assertIsNotNone(running["heartbeat_at"]) + + heartbeat_response = self.client.post( + ( + f"/api/workflows/{self.workflow_id}/operations/" + f"{created['id']}/heartbeat" + ), + json={ + "lease_owner": "worker-1", + "progress": 0.55, + "metadata": {"runtime": {"pid": 123}}, + }, + ) + self.assertEqual(heartbeat_response.status_code, 200) + heartbeat = heartbeat_response.json() + self.assertEqual(heartbeat["progress"], 0.55) + self.assertEqual(heartbeat["metadata"]["runtime"]["pid"], 123) + + succeeded_response = self.client.post( + ( + f"/api/workflows/{self.workflow_id}/operations/" + f"{created['id']}/transitions" + ), + json={ + "status": "succeeded", + "expected_status": "running", + "lease_owner": "worker-1", + "result": {"checkpoint_path": "/tmp/model.pth"}, + }, + ) + self.assertEqual(succeeded_response.status_code, 200) + succeeded = succeeded_response.json() + self.assertEqual(succeeded["status"], "succeeded") + self.assertEqual(succeeded["progress"], 1.0) + self.assertEqual(succeeded["result"]["checkpoint_path"], "/tmp/model.pth") + self.assertIsNone(succeeded["lease_owner"]) + self.assertIsNotNone(succeeded["completed_at"]) + + get_response = self.client.get( + f"/api/workflows/{self.workflow_id}/operations/{created['id']}" + ) + self.assertEqual(get_response.status_code, 200) + self.assertEqual(get_response.json()["status"], "succeeded") + + list_response = self.client.get( + f"/api/workflows/{self.workflow_id}/operations", + params={"status": "succeeded", "operation_type": "training"}, + ) + self.assertEqual(list_response.status_code, 200) + self.assertEqual([item["id"] for item in list_response.json()], [created["id"]]) + + def test_creation_is_idempotent_and_rejects_key_reuse(self): + first = self._create_operation().json() + second_response = self._create_operation() + self.assertEqual(second_response.status_code, 200) + self.assertEqual(second_response.json()["id"], first["id"]) + self.assertEqual( + second_response.json()["correlation_id"], first["correlation_id"] + ) + + conflict_response = self._create_operation( + input={"config_path": "/tmp/different.yaml"} + ) + self.assertEqual(conflict_response.status_code, 409) + self.assertIn("idempotency_key", conflict_response.json()["detail"]) + + def test_running_cancellation_is_requested_before_worker_acknowledgement(self): + operation = self._create_operation(key="inference:dataset-b:v1").json() + transition_url = ( + f"/api/workflows/{self.workflow_id}/operations/" + f"{operation['id']}/transitions" + ) + start_response = self.client.post( + transition_url, + json={"status": "running", "expected_status": "queued"}, + ) + self.assertEqual(start_response.status_code, 200) + + cancel_response = self.client.post( + ( + f"/api/workflows/{self.workflow_id}/operations/" + f"{operation['id']}/cancel" + ), + json={"reason": "User selected another checkpoint."}, + ) + self.assertEqual(cancel_response.status_code, 200) + cancellation_requested = cancel_response.json() + self.assertEqual(cancellation_requested["status"], "running") + self.assertIsNotNone(cancellation_requested["cancellation_requested_at"]) + self.assertEqual( + cancellation_requested["metadata"]["cancellation"]["reason"], + "User selected another checkpoint.", + ) + + acknowledged_response = self.client.post( + transition_url, + json={"status": "cancelled", "expected_status": "running"}, + ) + self.assertEqual(acknowledged_response.status_code, 200) + self.assertEqual(acknowledged_response.json()["status"], "cancelled") + + invalid_response = self.client.post( + transition_url, + json={"status": "succeeded"}, + ) + self.assertEqual(invalid_response.status_code, 409) + + def test_queued_cancellation_is_immediately_terminal(self): + operation = self._create_operation(key="evaluation:cancel-before-start").json() + response = self.client.post( + ( + f"/api/workflows/{self.workflow_id}/operations/" + f"{operation['id']}/cancel" + ) + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["status"], "cancelled") + self.assertIsNotNone(response.json()["completed_at"]) + + def test_expected_status_and_lease_owner_prevent_stale_updates(self): + operation = self._create_operation(key="export:evidence:v1").json() + transition_url = ( + f"/api/workflows/{self.workflow_id}/operations/" + f"{operation['id']}/transitions" + ) + stale_response = self.client.post( + transition_url, + json={"status": "running", "expected_status": "running"}, + ) + self.assertEqual(stale_response.status_code, 409) + + self.client.post( + transition_url, + json={"status": "running", "lease_owner": "worker-1"}, + ) + heartbeat_response = self.client.post( + ( + f"/api/workflows/{self.workflow_id}/operations/" + f"{operation['id']}/heartbeat" + ), + json={"lease_owner": "worker-2", "progress": 0.5}, + ) + self.assertEqual(heartbeat_response.status_code, 409) + + bad_progress_response = self.client.post( + ( + f"/api/workflows/{self.workflow_id}/operations/" + f"{operation['id']}/heartbeat" + ), + json={"lease_owner": "worker-1", "progress": 1.5}, + ) + self.assertEqual(bad_progress_response.status_code, 400) diff --git a/uv.lock b/uv.lock index b6546c54..a2f7d15f 100644 --- a/uv.lock +++ b/uv.lock @@ -2861,6 +2861,7 @@ dependencies = [ { name = "scikit-image", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "scipy", version = "1.17.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "sqlalchemy" }, { name = "tensorboard" }, { name = "tensorboard-data-server" }, { name = "tifffile", version = "2025.5.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -2903,6 +2904,7 @@ requires-dist = [ { name = "requests", specifier = ">=2.31" }, { name = "scikit-image", specifier = ">=0.25" }, { name = "scipy", specifier = ">=1.11" }, + { name = "sqlalchemy", specifier = ">=2.0,<3" }, { name = "tensorboard", specifier = "==2.20.0" }, { name = "tensorboard-data-server", specifier = "==0.7.2" }, { name = "tifffile", specifier = ">=2025.5.10" }, From 6e37ae5f4ce8d774351149b7633ee8d672317dc1 Mon Sep 17 00:00:00 2001 From: Adam Gohain Date: Tue, 21 Jul 2026 11:03:37 -0400 Subject: [PATCH 2/6] Add deterministic synthetic core project --- .gitignore | 1 + client/src/views/FilesManager.js | 1 + docs/synthetic-core-project.md | 57 +++++ scripts/create_synthetic_project.py | 40 +++ scripts/start.sh | 48 ++++ server_api/auth/database.py | 15 +- server_api/auth/router.py | 28 ++- server_api/synthetic_project.py | 361 ++++++++++++++++++++++++++++ server_api/workflows/service.py | 47 ++++ tests/test_synthetic_project.py | 204 ++++++++++++++++ 10 files changed, 800 insertions(+), 2 deletions(-) create mode 100644 docs/synthetic-core-project.md create mode 100644 scripts/create_synthetic_project.py create mode 100644 server_api/synthetic_project.py create mode 100644 tests/test_synthetic_project.py diff --git a/.gitignore b/.gitignore index 0d858bfe..b6cb3514 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ uploads pytorch_connectomics server_api/chatbot/faiss_index/ .logs/ +.pytc/ # Local deployment evidence (may contain transient viewer URLs) demo-proofread-3d.png diff --git a/client/src/views/FilesManager.js b/client/src/views/FilesManager.js index 0daa2a64..465a0542 100644 --- a/client/src/views/FilesManager.js +++ b/client/src/views/FilesManager.js @@ -55,6 +55,7 @@ const HIDDEN_SYSTEM_FILES = new Set([ "thumbs.db", ]); const DEFAULT_REMOTE_PROJECT_PATH = + process.env.REACT_APP_INITIAL_PROJECT_ROOT || process.env.REACT_APP_DEFAULT_PROJECT_PATH || "/home/weidf/demo_data/yixiao_tapereader_xri_case_study"; const IMAGE_EXTENSIONS = new Set([ diff --git a/docs/synthetic-core-project.md b/docs/synthetic-core-project.md new file mode 100644 index 00000000..a137f308 --- /dev/null +++ b/docs/synthetic-core-project.md @@ -0,0 +1,57 @@ +# Synthetic Core Project + +The local app now starts with a deterministic synthetic segmentation project by default. It formalizes one fixed development paradigm across the actual file workspace, project scanner, chunked volume reader, workflow state, progress tracker, viewer, proofreading flow, agent context, training handoff, and evaluation surfaces. + +## Canonical State + +The generated project lives at `.pytc/synthetic-core-project` and contains four compressed, chunked HDF5 image volumes: + +| Volume | Initial state | Intended workflow role | +| --- | --- | --- | +| `train-01` | Ground truth | Training source | +| `train-02` | Ground truth | Training source | +| `review-01` | Imperfect draft | Proofreading and correction | +| `target-01` | Image only | Inference target | + +The expected progress state is always **4 total / 2 ground truth / 1 needs proofreading / 1 missing segmentation**. Baseline and corrected candidate predictions are prepopulated for comparison. The data are only for interaction and systems testing; they are not scientific evidence or a model-quality benchmark. + +## Run + +From the repository root: + +```bash +./scripts/start.sh +``` + +Startup creates the fixture if absent, uses `.pytc/synthetic-core.db`, seeds the current workflow, mounts the project into the file browser, builds the client, and launches Electron. + +Normal startup preserves edits made inside the generated project. Restore the exact canonical files and state with: + +```bash +PYTC_SYNTHETIC_PROJECT_RESET=1 ./scripts/start.sh +``` + +Generate or reset the fixture without starting the app: + +```bash +uv run python scripts/create_synthetic_project.py --reset +``` + +Disable the development fixture and use the previous project/database behavior: + +```bash +PYTC_SYNTHETIC_PROJECT=0 ./scripts/start.sh +``` + +Supplying `PYTC_INITIAL_PROJECT_ROOT` also takes precedence over synthetic mode. + +## Core Test Sequence + +1. Confirm the project is already mounted and the progress tracker reports `2 / 1 / 1`. +2. Open a training image and ground-truth label in the storage-backed viewer. +3. Open `review-01`, compare the baseline and candidate, and proofread the draft. +4. Ask the workflow agent for project status and the next recommended action. +5. Stage training from only the two confirmed ground-truth volumes. +6. Select `target-01` for inference and verify that the output is registered. +7. Inspect operation progress, cancellation, failure, and retry states. +8. Reset the fixture before repeating a canonical acceptance run. diff --git a/scripts/create_synthetic_project.py b/scripts/create_synthetic_project.py new file mode 100644 index 00000000..17cad7b6 --- /dev/null +++ b/scripts/create_synthetic_project.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Create the deterministic local project used by the development app.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from server_api.synthetic_project import create_synthetic_project + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + default=str(REPO_ROOT / ".pytc/synthetic-core-project"), + help="Project directory (default: .pytc/synthetic-core-project)", + ) + parser.add_argument( + "--reset", + action="store_true", + help="Delete and restore the generated project to its canonical state.", + ) + args = parser.parse_args() + print( + json.dumps( + create_synthetic_project(args.output_dir, reset=args.reset), + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/start.sh b/scripts/start.sh index 164dffbc..0b80e4a1 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -44,6 +44,36 @@ fi mkdir -p "${LOG_DIR}" +configure_synthetic_project() { + if [[ "${PYTC_SYNTHETIC_PROJECT:-1}" != "1" || -n "${PYTC_INITIAL_PROJECT_ROOT:-}" ]]; then + return 0 + fi + + local project_root="${PYTC_SYNTHETIC_PROJECT_ROOT:-${ROOT_DIR}/.pytc/synthetic-core-project}" + local generator_args=( + python scripts/create_synthetic_project.py + --output-dir "${project_root}" + ) + if [[ "${PYTC_SYNTHETIC_PROJECT_RESET:-0}" == "1" ]]; then + generator_args+=(--reset) + fi + + echo "Preparing deterministic synthetic project..." + "${UV_BIN}" run --directory "${ROOT_DIR}" "${generator_args[@]}" + export PYTC_INITIAL_PROJECT_ROOT="${project_root}" + export PYTC_INITIAL_PROJECT_KIND="synthetic" + export PYTC_INITIAL_PROJECT_TITLE="Synthetic Segmentation Core Loop" + export PYTC_INITIAL_IMAGE_PATH="${project_root}/data/raw" + export PYTC_INITIAL_LABEL_PATH="${project_root}/data/seg" + export PYTC_INITIAL_MASK_PATH="${project_root}/data/seg" + export PYTC_INITIAL_CONFIG_PATH="${project_root}/configs/Synthetic-Core-Loop-BC.yaml" + export PYTC_DATABASE_URL="${PYTC_DATABASE_URL:-sqlite:///${ROOT_DIR}/.pytc/synthetic-core.db}" + export PYTC_AUTOMOUNT_INITIAL_PROJECT="${PYTC_AUTOMOUNT_INITIAL_PROJECT:-1}" + export REACT_APP_INITIAL_PROJECT_ROOT="${project_root}" +} + +configure_synthetic_project + STARTED_PIDS=() relative_log_path() { @@ -137,6 +167,24 @@ start_service \ "${LOG_DIR}/api-server.log" \ env PYTHONDONTWRITEBYTECODE=1 "${UV_BIN}" run --directory "${ROOT_DIR}" python -m server_api.main +initialize_local_project() { + if [[ "${PYTC_AUTOMOUNT_INITIAL_PROJECT:-0}" != "1" || -z "${PYTC_INITIAL_PROJECT_ROOT:-}" ]]; then + return 0 + fi + + local payload + payload="$(PYTC_MOUNT_ROOT="${PYTC_INITIAL_PROJECT_ROOT}" "${UV_BIN}" run --directory "${ROOT_DIR}" python -c 'import json, os; print(json.dumps({"directory_path": os.environ["PYTC_MOUNT_ROOT"], "destination_path": "root"}))')" + curl -sf "http://localhost:4242/api/workflows/current" >/dev/null + curl -sf \ + -X POST \ + -H "Content-Type: application/json" \ + --data-binary "${payload}" \ + "http://localhost:4242/files/mount" >/dev/null + echo "Initial project mounted: ${PYTC_INITIAL_PROJECT_ROOT}" +} + +initialize_local_project + start_service \ "PyTC server" \ 4243 \ diff --git a/server_api/auth/database.py b/server_api/auth/database.py index 7533c849..80137595 100644 --- a/server_api/auth/database.py +++ b/server_api/auth/database.py @@ -1,9 +1,22 @@ +import os +from pathlib import Path + from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker +DATABASE_URL = os.getenv("PYTC_DATABASE_URL", "sqlite:///server_api/auth/sql_app.db") + +if DATABASE_URL.startswith("sqlite:///"): + database_path = DATABASE_URL.removeprefix("sqlite:///") + if database_path and database_path != ":memory:": + Path(database_path).expanduser().parent.mkdir(parents=True, exist_ok=True) + engine = create_engine( - "sqlite:///server_api/auth/sql_app.db", connect_args={"check_same_thread": False} + DATABASE_URL, + connect_args={"check_same_thread": False} + if DATABASE_URL.startswith("sqlite:") + else {}, ) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) Base = declarative_base() diff --git a/server_api/auth/router.py b/server_api/auth/router.py index a3b271c0..f01c9baf 100644 --- a/server_api/auth/router.py +++ b/server_api/auth/router.py @@ -90,7 +90,8 @@ def _is_ignored_system_file(name: Optional[str]) -> bool: def _project_suggestion_candidates() -> List[dict]: repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) workspace_root = os.path.abspath(os.path.join(repo_root, "..")) - return [ + initial_project_root = os.getenv("PYTC_INITIAL_PROJECT_ROOT", "").strip() + candidates = [ { "id": "yixiao-tapereader-xri-case-study", "name": "yixiao_tapereader_xri_case_study", @@ -169,6 +170,31 @@ def _project_suggestion_candidates() -> List[dict]: "recommended": False, }, ] + if initial_project_root: + candidates.insert( + 0, + { + "id": "initial-project", + "name": os.path.basename(initial_project_root.rstrip(os.sep)), + "directory_path": initial_project_root, + "description": "Active local development project.", + "recommended": True, + }, + ) + for candidate in candidates[1:]: + candidate["recommended"] = False + + unique_candidates = [] + seen_paths = set() + for candidate in candidates: + normalized_path = os.path.abspath( + os.path.expanduser(candidate["directory_path"]) + ) + if normalized_path in seen_paths: + continue + seen_paths.add(normalized_path) + unique_candidates.append(candidate) + return unique_candidates def _lower_path_parts(path: str) -> List[str]: diff --git a/server_api/synthetic_project.py b/server_api/synthetic_project.py new file mode 100644 index 00000000..e6ee2cf5 --- /dev/null +++ b/server_api/synthetic_project.py @@ -0,0 +1,361 @@ +"""Deterministic local project fixture for the core segmentation workflow.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +import h5py +import numpy as np + +GENERATOR_VERSION = "pytc-synthetic-core/v1" +VOLUME_SHAPE = (12, 160, 160) +VOLUME_CHUNKS = (4, 64, 64) +VOXEL_SIZE_NM = (40.0, 8.0, 8.0) + + +def _instance_labels(offset: tuple[int, int, int] = (0, 0, 0)) -> np.ndarray: + z, y, x = np.indices(VOLUME_SHAPE) + labels = np.zeros(VOLUME_SHAPE, dtype=np.uint16) + objects = ( + (1, (4, 43, 48), (3, 18, 24)), + (2, (7, 102, 61), (3, 24, 17)), + (3, (5, 82, 118), (2, 16, 28)), + (4, (8, 125, 126), (2, 18, 20)), + ) + for label_id, center, radius in objects: + shifted_center = tuple(center[index] + offset[index] for index in range(3)) + distance = sum( + ((axis - shifted_center[index]) / radius[index]) ** 2 + for index, axis in enumerate((z, y, x)) + ) + labels[distance <= 1.0] = label_id + return labels + + +def _image_from_labels(labels: np.ndarray, seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + z, y, x = np.indices(labels.shape) + image = ( + 112.0 + + 18.0 * np.sin(x / 9.0) + + 13.0 * np.cos(y / 13.0) + + 7.0 * np.sin((x + y + z * 5) / 17.0) + + rng.normal(0.0, 10.0, labels.shape) + ) + image += np.where(labels > 0, 36.0 + (labels % 3) * 9.0, 0.0) + boundary = np.zeros(labels.shape, dtype=bool) + for axis in range(3): + boundary |= labels != np.roll(labels, 1, axis=axis) + image[boundary] -= 45.0 + return np.clip(image, 0, 255).astype(np.uint8) + + +def _draft_labels(labels: np.ndarray) -> np.ndarray: + draft = labels.copy() + draft[draft == 3] = 0 + draft[:, 92:105, 55:72][draft[:, 92:105, 55:72] == 2] = 0 + draft[3:7, 24:37, 112:130] = 9 + return draft + + +def _write_h5(path: Path, data: np.ndarray, *, role: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with h5py.File(path, "w") as handle: + dataset = handle.create_dataset( + "data", + data=data, + chunks=VOLUME_CHUNKS, + compression="gzip", + compression_opts=4, + shuffle=True, + ) + dataset.attrs["axes"] = "zyx" + dataset.attrs["role"] = role + dataset.attrs["voxel_size_nm"] = VOXEL_SIZE_NM + handle.attrs["generator"] = GENERATOR_VERSION + handle.attrs["synthetic"] = True + + +def _config_text(root: Path) -> str: + train_image = root / "data/raw/train-01_image.h5" + train_label = root / "data/seg/ground_truth/train-01_ground_truth.h5" + target_image = root / "data/raw/target-01_image.h5" + output_path = root / "runtime/training" + inference_path = root / "runtime/inference" + return f"""# Deterministic local workflow fixture. Not a benchmark configuration. +SYSTEM: + NUM_GPUS: 1 + NUM_CPUS: 0 + PARALLEL: single + +MODEL: + ARCHITECTURE: unet_plus_3d + BLOCK_TYPE: residual_se + INPUT_SIZE: [9, 65, 65] + OUTPUT_SIZE: [9, 65, 65] + IN_PLANES: 1 + OUT_PLANES: 2 + NORM_MODE: gn + FILTERS: [8, 12, 16, 24, 32] + TARGET_OPT: ["0", "4-1-1"] + LOSS_OPTION: + - [WeightedBCEWithLogitsLoss, DiceLoss] + - [WeightedBCEWithLogitsLoss, DiceLoss] + LOSS_WEIGHT: [[1.0, 0.5], [1.0, 0.5]] + WEIGHT_OPT: [["1", "0"], ["1", "0"]] + OUTPUT_ACT: [["none", "sigmoid"], ["none", "sigmoid"]] + +DATASET: + INPUT_PATH: "" + IMAGE_NAME: {train_image} + LABEL_NAME: {train_label} + OUTPUT_PATH: {output_path} + PAD_SIZE: [4, 16, 16] + PAD_MODE: reflect + IMAGE_SCALE: [1.0, 1.0, 1.0] + LABEL_SCALE: [1.0, 1.0, 1.0] + DATA_SCALE: [1.0, 1.0, 1.0] + NORMALIZE_RANGE: true + MEAN: 0.5 + STD: 0.5 + MATCH_ACT: none + IS_ABSOLUTE_PATH: true + DISTRIBUTED: false + DO_2D: false + LOAD_2D: false + DO_CHUNK_TITLE: 0 + DROP_CHANNEL: false + ENSURE_MIN_SIZE: false + LABEL_BINARY: false + LABEL_MAG: 0 + LABEL_VAST: false + REDUCE_LABEL: true + VALID_RATIO: 0.25 + VALID_MASK_NAME: null + VAL_IMAGE_NAME: null + VAL_LABEL_NAME: null + VAL_VALID_MASK_NAME: null + VAL_PAD_SIZE: [0, 0, 0] + REJECT_SAMPLING: + SIZE_THRES: -1 + DIVERSITY: -1 + NUM_TRIAL: 10 + P: 0.95 + +AUGMENTOR: + ENABLED: false + +SOLVER: + NAME: AdamW + LR_SCHEDULER_NAME: WarmupCosineLR + BASE_LR: 0.0003 + BETAS: [0.9, 0.999] + WEIGHT_DECAY: 0.0001 + ITERATION_STEP: 1 + ITERATION_SAVE: 5 + ITERATION_VAL: 1000000 + ITERATION_TOTAL: 10 + SAMPLES_PER_BATCH: 1 + ITERATION_RESTART: false + WARMUP_FACTOR: 0.001 + WARMUP_ITERS: 2 + WARMUP_METHOD: linear + +MONITOR: + ITERATION_NUM: [2, 5] + LOG_OPT: [1, 1, 0] + VIS_OPT: [0, 8] + +INFERENCE: + INPUT_SIZE: [9, 65, 65] + OUTPUT_SIZE: [9, 65, 65] + IMAGE_NAME: {target_image} + OUTPUT_PATH: {inference_path} + OUTPUT_NAME: target-01_prediction.h5 + OUTPUT_ACT: ["sigmoid", "sigmoid"] + PAD_SIZE: [4, 16, 16] + AUG_MODE: mean + AUG_NUM: 1 + STRIDE: [8, 64, 64] + SAMPLES_PER_BATCH: 1 + DO_EVAL: false + UNPAD: true +""" + + +def _manifest() -> dict[str, Any]: + return { + "schema_version": 1, + "generator": GENERATOR_VERSION, + "synthetic": True, + "title": "Synthetic Segmentation Core Loop", + "description": "Fixed local fixture for end-to-end workflow and UI testing.", + "imaging_modality": "Synthetic volumetric microscopy", + "target_structure": "synthetic organelles", + "task_family": "3D instance segmentation", + "task": "segmentation, proofreading, retraining, and evaluation", + "voxel_size": {"zyx_nm": list(VOXEL_SIZE_NM)}, + "active_paths": { + "image_root": "data/raw", + "label_root": "data/seg", + "config": "configs/Synthetic-Core-Loop-BC.yaml", + }, + "volumes": [ + { + "id": "train-01", + "split": "train", + "image": "data/raw/train-01_image.h5", + "segmentation": "data/seg/ground_truth/train-01_ground_truth.h5", + "mask_state": "ground_truth", + }, + { + "id": "train-02", + "split": "train", + "image": "data/raw/train-02_image.h5", + "segmentation": "data/seg/ground_truth/train-02_ground_truth.h5", + "mask_state": "ground_truth", + }, + { + "id": "review-01", + "split": "proofreading", + "image": "data/raw/review-01_image.h5", + "segmentation": "data/seg/draft/review-01_seg.h5", + "mask_state": "needs_proofreading", + }, + { + "id": "target-01", + "split": "inference", + "image": "data/raw/target-01_image.h5", + "segmentation": None, + "mask_state": "missing_segmentation", + }, + ], + "initial_progress_summary": { + "total": 4, + "ground_truth": 2, + "needs_proofreading": 1, + "missing_segmentation": 1, + }, + "workflow_split": { + "ground_truth_training_volumes": ["train-01", "train-02"], + "proofreading_volumes": ["review-01"], + "image_only_inference_targets": ["target-01"], + }, + } + + +def _is_current(root: Path) -> bool: + try: + manifest = json.loads((root / "project_manifest.json").read_text()) + except (OSError, json.JSONDecodeError): + return False + required = ( + "data/raw/train-01_image.h5", + "data/raw/train-02_image.h5", + "data/raw/review-01_image.h5", + "data/raw/target-01_image.h5", + "data/seg/ground_truth/train-01_ground_truth.h5", + "data/seg/ground_truth/train-02_ground_truth.h5", + "data/seg/draft/review-01_seg.h5", + "outputs/predictions/baseline_review-01.h5", + "outputs/predictions/candidate_review-01.h5", + "configs/Synthetic-Core-Loop-BC.yaml", + "notes/README.md", + ) + return manifest.get("generator") == GENERATOR_VERSION and all( + (root / relative_path).is_file() for relative_path in required + ) + + +def _has_generator_marker(root: Path) -> bool: + try: + manifest = json.loads((root / "project_manifest.json").read_text()) + except (OSError, json.JSONDecodeError): + return False + return str(manifest.get("generator") or "").startswith("pytc-synthetic-core/") + + +def create_synthetic_project(output_dir: str | Path, *, reset: bool = False) -> dict: + """Create or restore the fixed project and return its resolved contract.""" + root = Path(output_dir).expanduser().resolve() + if root.exists() and any(root.iterdir()) and not _has_generator_marker(root): + raise ValueError( + f"Refusing to modify non-synthetic directory without a generator marker: {root}" + ) + if reset and root.exists(): + shutil.rmtree(root) + if _is_current(root): + return {"created": False, "generator": GENERATOR_VERSION, "root": str(root)} + + root.mkdir(parents=True, exist_ok=True) + cases = { + "train-01": (_instance_labels(), 101), + "train-02": (_instance_labels((0, 7, -5)), 202), + "review-01": (_instance_labels((0, -6, 8)), 303), + "target-01": (_instance_labels((0, 4, 5)), 404), + } + for name, (labels, seed) in cases.items(): + _write_h5( + root / f"data/raw/{name}_image.h5", + _image_from_labels(labels, seed), + role="image", + ) + + for name in ("train-01", "train-02"): + _write_h5( + root / f"data/seg/ground_truth/{name}_ground_truth.h5", + cases[name][0], + role="ground_truth", + ) + + review_labels = cases["review-01"][0] + draft = _draft_labels(review_labels) + (root / "data/seg/draft/review-01_draft_seg.h5").unlink(missing_ok=True) + _write_h5( + root / "data/seg/draft/review-01_seg.h5", + draft, + role="draft_segmentation", + ) + _write_h5( + root / "outputs/predictions/baseline_review-01.h5", + draft, + role="baseline_prediction", + ) + _write_h5( + root / "outputs/predictions/candidate_review-01.h5", + review_labels, + role="candidate_prediction", + ) + + config_path = root / "configs/Synthetic-Core-Loop-BC.yaml" + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.write_text(_config_text(root), encoding="utf-8") + (root / "runtime/training").mkdir(parents=True, exist_ok=True) + (root / "runtime/inference").mkdir(parents=True, exist_ok=True) + + notes_path = root / "notes/README.md" + notes_path.parent.mkdir(parents=True, exist_ok=True) + notes_path.write_text( + """# Synthetic Segmentation Core Loop + +This deterministic project tests the application workflow, not biological validity or model quality. + +- `train-01` and `train-02` have confirmed ground-truth instance labels. +- `review-01` has a deliberately incomplete draft with one missing object and one false positive. +- `target-01` is image-only and should be offered as an inference target. +- Baseline and candidate predictions make comparison and evaluation views available immediately. + +Expected initial progress: 4 volumes, 2 ground truth, 1 needs proofreading, and 1 missing segmentation. +The HDF5 dataset key is `data`; volumes are chunked and compressed to exercise storage-backed reads. +Use the repository reset command before a canonical test run. Generated results under `runtime/` are disposable. +""", + encoding="utf-8", + ) + (root / "project_manifest.json").write_text( + json.dumps(_manifest(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return {"created": True, "generator": GENERATOR_VERSION, "root": str(root)} diff --git a/server_api/workflows/service.py b/server_api/workflows/service.py index 6745f0c2..8bffb626 100644 --- a/server_api/workflows/service.py +++ b/server_api/workflows/service.py @@ -51,6 +51,53 @@ def _initial_project_defaults() -> Dict[str, Any]: if not INITIAL_PROJECT_ROOT: return {} normalized_root = INITIAL_PROJECT_ROOT.lower() + if ( + os.getenv("PYTC_INITIAL_PROJECT_KIND", "").lower() == "synthetic" + or "synthetic-core-project" in normalized_root + ): + image_path = os.getenv( + "PYTC_INITIAL_IMAGE_PATH", + os.path.join(INITIAL_PROJECT_ROOT, "data/raw"), + ) + label_path = os.getenv( + "PYTC_INITIAL_LABEL_PATH", + os.path.join(INITIAL_PROJECT_ROOT, "data/seg"), + ) + return { + "title": os.getenv( + "PYTC_INITIAL_PROJECT_TITLE", + "Synthetic Segmentation Core Loop", + ), + "dataset_path": INITIAL_PROJECT_ROOT, + "image_path": image_path, + "label_path": label_path, + "mask_path": os.getenv("PYTC_INITIAL_MASK_PATH", label_path), + "config_path": os.getenv( + "PYTC_INITIAL_CONFIG_PATH", + os.path.join( + INITIAL_PROJECT_ROOT, + "configs/Synthetic-Core-Loop-BC.yaml", + ), + ), + "metadata": { + "created_from": "initial_project_default", + "synthetic": True, + "project_context": { + "imaging_modality": "Synthetic volumetric microscopy", + "target_structure": "synthetic organelles", + "task_family": "3D instance segmentation", + "task_goal": "segmentation, proofreading, retraining, and evaluation", + "optimization_priority": "workflow contract and recovery behavior", + "mask_status": "mixed: 2 ground-truth masks, 1 draft mask, 1 image-only target", + "training_policy": "train only on confirmed ground-truth masks", + "image_only_strategy": "run inference on image-only volumes after training", + "voxel_size_nm": [40, 8, 8], + "voxel_size_source": "project_manifest.json", + }, + "visualization_scales": [40, 8, 8], + "visualization_scales_source": "project_manifest.json", + }, + } if ( "yixiao" in normalized_root or "tapereader" in normalized_root diff --git a/tests/test_synthetic_project.py b/tests/test_synthetic_project.py new file mode 100644 index 00000000..c15eb321 --- /dev/null +++ b/tests/test_synthetic_project.py @@ -0,0 +1,204 @@ +import json +import pathlib +import tempfile +import unittest +from unittest.mock import patch + +import h5py +import numpy as np +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from server_api.auth import database as auth_database +from server_api.auth import models +from server_api.auth.router import ( + _project_suggestion_candidates, + _scan_project_profile, +) +from server_api.main import app as server_api_app +from server_api.synthetic_project import ( + GENERATOR_VERSION, + VOLUME_CHUNKS, + VOLUME_SHAPE, + create_synthetic_project, +) +from server_api.workflows import service as workflow_service + + +class SyntheticProjectTests(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.project_root = pathlib.Path(self.temp_dir.name) / "synthetic-core-project" + + def tearDown(self): + self.temp_dir.cleanup() + + def test_generator_is_deterministic_idempotent_and_resettable(self): + first = create_synthetic_project(self.project_root) + self.assertTrue(first["created"]) + self.assertEqual(first["generator"], GENERATOR_VERSION) + + with h5py.File(self.project_root / "data/raw/train-01_image.h5") as handle: + first_image = handle["data"][:] + self.assertEqual(handle["data"].shape, VOLUME_SHAPE) + self.assertEqual(handle["data"].chunks, VOLUME_CHUNKS) + self.assertEqual(handle["data"].compression, "gzip") + self.assertTrue(handle.attrs["synthetic"]) + + notes_path = self.project_root / "notes/README.md" + notes_path.write_text("local testing note\n", encoding="utf-8") + second = create_synthetic_project(self.project_root) + self.assertFalse(second["created"]) + self.assertEqual(notes_path.read_text(encoding="utf-8"), "local testing note\n") + + restored = create_synthetic_project(self.project_root, reset=True) + self.assertTrue(restored["created"]) + self.assertIn( + "Expected initial progress", notes_path.read_text(encoding="utf-8") + ) + with h5py.File(self.project_root / "data/raw/train-01_image.h5") as handle: + np.testing.assert_array_equal(handle["data"][:], first_image) + + def test_fixture_encodes_expected_project_and_prediction_states(self): + create_synthetic_project(self.project_root) + manifest = json.loads( + (self.project_root / "project_manifest.json").read_text(encoding="utf-8") + ) + self.assertEqual( + manifest["initial_progress_summary"], + { + "total": 4, + "ground_truth": 2, + "needs_proofreading": 1, + "missing_segmentation": 1, + }, + ) + + with ( + h5py.File( + self.project_root / "outputs/predictions/baseline_review-01.h5" + ) as baseline_handle, + h5py.File( + self.project_root / "outputs/predictions/candidate_review-01.h5" + ) as candidate_handle, + ): + baseline = baseline_handle["data"][:] + candidate = candidate_handle["data"][:] + self.assertGreater(np.count_nonzero(baseline != candidate), 0) + self.assertIn(9, np.unique(baseline)) + self.assertNotIn(9, np.unique(candidate)) + + profile = _scan_project_profile(str(self.project_root), audit_detail="summary") + self.assertEqual(profile["volume_sets"][0]["image_count"], 4) + self.assertEqual(profile["volume_sets"][0]["label_count"], 3) + self.assertEqual(profile["volume_sets"][0]["pair_count"], 3) + self.assertEqual( + profile["context_hints"]["imaging_modality"], + "Synthetic volumetric microscopy", + ) + self.assertEqual(profile["context_hints"]["voxel_size_nm"], [40.0, 8.0, 8.0]) + + def test_generator_refuses_to_overwrite_an_unmarked_directory(self): + self.project_root.mkdir() + (self.project_root / "important.txt").write_text("keep", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "non-synthetic directory"): + create_synthetic_project(self.project_root, reset=True) + + self.assertEqual( + (self.project_root / "important.txt").read_text(encoding="utf-8"), + "keep", + ) + + def test_initial_project_candidate_and_workflow_defaults_use_fixture(self): + create_synthetic_project(self.project_root) + with ( + patch.dict( + "os.environ", + { + "PYTC_INITIAL_PROJECT_ROOT": str(self.project_root), + "PYTC_INITIAL_PROJECT_KIND": "synthetic", + }, + clear=False, + ), + patch.object( + workflow_service, + "INITIAL_PROJECT_ROOT", + str(self.project_root), + ), + ): + candidates = _project_suggestion_candidates() + defaults = workflow_service._initial_project_defaults() + + self.assertEqual(candidates[0]["id"], "initial-project") + self.assertTrue(candidates[0]["recommended"]) + self.assertFalse(any(item["recommended"] for item in candidates[1:])) + self.assertEqual(defaults["title"], "Synthetic Segmentation Core Loop") + self.assertEqual(defaults["image_path"], str(self.project_root / "data/raw")) + self.assertTrue(defaults["metadata"]["synthetic"]) + self.assertEqual( + defaults["metadata"]["project_context"]["training_policy"], + "train only on confirmed ground-truth masks", + ) + + +class SyntheticProjectProgressTests(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.project_root = pathlib.Path(self.temp_dir.name) / "synthetic-core-project" + create_synthetic_project(self.project_root) + + self.engine = create_engine( + f"sqlite:///{pathlib.Path(self.temp_dir.name) / 'test.db'}", + connect_args={"check_same_thread": False}, + ) + self.SessionLocal = sessionmaker( + autocommit=False, + autoflush=False, + bind=self.engine, + ) + models.Base.metadata.create_all(bind=self.engine) + + def override_get_db(): + with self.SessionLocal() as db: + yield db + + server_api_app.dependency_overrides[auth_database.get_db] = override_get_db + self.client = TestClient(server_api_app) + + def tearDown(self): + server_api_app.dependency_overrides.clear() + self.engine.dispose() + self.temp_dir.cleanup() + + def test_real_progress_endpoint_reports_fixture_contract(self): + current_response = self.client.get("/api/workflows/current") + self.assertEqual(current_response.status_code, 200) + workflow_id = current_response.json()["workflow"]["id"] + patch_response = self.client.patch( + f"/api/workflows/{workflow_id}", + json={ + "title": "Synthetic Segmentation Core Loop", + "dataset_path": str(self.project_root), + "image_path": str(self.project_root / "data/raw"), + "label_path": str(self.project_root / "data/seg"), + "config_path": str( + self.project_root / "configs/Synthetic-Core-Loop-BC.yaml" + ), + }, + ) + self.assertEqual(patch_response.status_code, 200) + + progress_response = self.client.get( + f"/api/workflows/{workflow_id}/project-progress" + ) + self.assertEqual(progress_response.status_code, 200) + payload = progress_response.json() + self.assertEqual(payload["summary"]["total"], 4) + self.assertEqual(payload["summary"]["ground_truth"], 2) + self.assertEqual(payload["summary"]["needs_proofreading"], 1) + self.assertEqual(payload["summary"]["missing_segmentation"], 1) + states = {row["name"]: row["status"] for row in payload["volumes"]} + self.assertEqual(states["review-01_image.h5"], "needs_proofreading") + self.assertEqual(states["target-01_image.h5"], "missing_segmentation") From fd2991ea1cf62af5c9fedca329524a4d8121288a Mon Sep 17 00:00:00 2001 From: Adam Gohain Date: Tue, 21 Jul 2026 11:40:26 -0400 Subject: [PATCH 3/6] Harden synthetic workflow UX --- client/src/components/FilePickerModal.js | 56 ++++++-- client/src/components/FilePickerModal.test.js | 50 ++++++++ client/src/contexts/WorkflowContext.js | 20 ++- client/src/contexts/WorkflowContext.test.js | 30 +++++ client/src/views/FilesManager.js | 7 +- client/src/views/Views.js | 33 ++++- client/src/views/Views.test.js | 26 ++++ client/src/views/Visualization.js | 84 ++++++++++-- client/src/views/Visualization.test.js | 121 ++++++++++++++++++ scripts/start.sh | 6 +- server_api/auth/router.py | 23 +++- server_api/ehtool/router.py | 19 +-- server_api/workflows/router.py | 24 +++- server_api/workflows/service.py | 7 +- tests/test_file_workspace_routes.py | 26 ++++ tests/test_synthetic_project.py | 15 ++- tests/test_workflow_routes.py | 24 ++++ 17 files changed, 520 insertions(+), 51 deletions(-) create mode 100644 client/src/components/FilePickerModal.test.js create mode 100644 client/src/views/Visualization.test.js diff --git a/client/src/components/FilePickerModal.js b/client/src/components/FilePickerModal.js index 0ffa724d..aff43938 100644 --- a/client/src/components/FilePickerModal.js +++ b/client/src/components/FilePickerModal.js @@ -1,12 +1,23 @@ import React, { useState, useEffect, useMemo } from "react"; -import { Modal, List, Breadcrumb, Button, Spin, message, Progress } from "antd"; +import { + Modal, + List, + Breadcrumb, + Button, + Spin, + message, + Progress, + Result, +} from "antd"; import { FolderFilled, FileOutlined, ArrowLeftOutlined, UploadOutlined, + ReloadOutlined, } from "@ant-design/icons"; import { apiClient } from "../api"; +import { normalizeApiError } from "../errors/apiError"; const HIDDEN_SYSTEM_FILES = new Set([ "workflow_preference.json", @@ -50,6 +61,7 @@ const FilePickerModal = ({ const [onlyImages, setOnlyImages] = useState(false); const [uploading, setUploading] = useState(false); const [uploadProgress, setUploadProgress] = useState(null); + const [loadError, setLoadError] = useState(null); const previewBaseUrl = apiClient.defaults.baseURL || "http://localhost:4242"; // Refactored fetch to get all files once @@ -70,17 +82,21 @@ const FilePickerModal = ({ setOnlyImages(false); setPreviewStatus({}); setUploadProgress(null); + setLoadError(null); loadAllData(); } }, [visible]); const loadAllData = async () => { setLoading(true); + setLoadError(null); try { const res = await apiClient.get("/files"); setAllData(res.data); } catch (error) { - message.error("Failed to load files"); + const normalizedError = normalizeApiError(error); + setLoadError(normalizedError); + message.error(normalizedError.message); } finally { setLoading(false); } @@ -293,7 +309,7 @@ const FilePickerModal = ({ ) : null } width={600} - bodyStyle={{ padding: 0 }} + styles={{ body: { padding: 0 } }} >
- - {getBreadcrumbs().map((b) => ( - + ({ + key: breadcrumb.id, + title: ( - - ))} - + ), + }))} + />
+ ) : loadError ? ( + } + onClick={loadAllData} + > + Try again + + ) : null + } + /> ) : ( ({ + apiClient: { + get: jest.fn(), + post: jest.fn(), + defaults: { baseURL: "http://localhost:4242" }, + }, +})); + +describe("FilePickerModal", () => { + beforeEach(() => { + jest.clearAllMocks(); + window.matchMedia = jest.fn().mockImplementation((query) => ({ + matches: false, + media: query, + onchange: null, + addListener: jest.fn(), + removeListener: jest.fn(), + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + dispatchEvent: jest.fn(), + })); + }); + + it("keeps a retryable error state when files cannot be loaded", async () => { + apiClient.get + .mockRejectedValueOnce(new Error("Network Error")) + .mockResolvedValueOnce({ data: [] }); + + render( + , + ); + + expect(await screen.findByText("Files unavailable")).toBeTruthy(); + expect( + screen.getByText("Check the server connection and try again."), + ).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { name: /try again/i })); + + await waitFor(() => expect(apiClient.get).toHaveBeenCalledTimes(2)); + await waitFor(() => + expect(screen.queryByText("Files unavailable")).toBeNull(), + ); + }); +}); diff --git a/client/src/contexts/WorkflowContext.js b/client/src/contexts/WorkflowContext.js index 0a651299..99194f85 100644 --- a/client/src/contexts/WorkflowContext.js +++ b/client/src/contexts/WorkflowContext.js @@ -493,8 +493,24 @@ export function WorkflowProvider({ children }) { await resetFileWorkspace(); await clearLocalWorkflowInputs(); const data = await startNewWorkflowApi(body); - applyWorkflowDetail(data); - return data; + const projectPath = data?.workflow?.dataset_path; + if (!projectPath) { + applyWorkflowDetail(data); + return data; + } + try { + await mountProjectDirectory({ + directoryPath: projectPath, + mountName: data?.workflow?.title || "", + destinationPath: "root", + }); + applyWorkflowDetail(data); + return data; + } catch (error) { + console.error("Fresh workflow project mount failed:", error); + applyWorkflowDetail(data); + return { ...data, project_mount_failed: true }; + } }, [applyWorkflowDetail, clearLocalWorkflowInputs], ); diff --git a/client/src/contexts/WorkflowContext.test.js b/client/src/contexts/WorkflowContext.test.js index ce6602d1..862be9e5 100644 --- a/client/src/contexts/WorkflowContext.test.js +++ b/client/src/contexts/WorkflowContext.test.js @@ -96,6 +96,9 @@ function Probe() { > Approve proposal + ) : null } /> + ) : items.length === 0 ? ( + ) : ( - ( - { - if (item.is_folder) { - setCurrentPath(String(item.id)); - } else { - if (selectionType === "file") { - const fullPath = constructFullPath(item); - onSelect({ ...item, logical_path: fullPath }); - } - } - }} - actions={[ - item.is_folder && ( - - ), - (selectionType === "file" || - selectionType === "fileOrDirectory") && - !item.is_folder && ( - - ), - (selectionType === "directory" || - selectionType === "fileOrDirectory") && - item.is_folder && ( - - ), - ]} - > - - ) : isImageFile(item) ? ( -
- {previewStatus[item.id] !== "loaded" && ( - - )} - {previewStatus[item.id] !== "error" && ( - {item.name} markPreviewLoaded(item.id)} - onError={() => markPreviewError(item.id)} + } + } + }} + actions={[ + item.is_folder && ( + + ), + (selectionType === "file" || + selectionType === "fileOrDirectory") && + !item.is_folder && ( + + ), + (selectionType === "directory" || + selectionType === "fileOrDirectory") && + item.is_folder && ( + + ), + ]} + > + - )} -
- ) : ( - - ) - } - title={ - - {item.name} - {item.is_folder && - (item.path === "root" || !item.path) && - item.physical_path && ( - - Mounted - - )} - - } - description={item.size ? item.size : null} - /> - + {previewStatus[item.id] !== "loaded" && ( + + )} + {previewStatus[item.id] !== "error" && ( + {item.name} markPreviewLoaded(item.id)} + onError={() => markPreviewError(item.id)} + style={{ + position: "absolute", + inset: 0, + width: "100%", + height: "100%", + objectFit: "cover", + opacity: + previewStatus[item.id] === "loaded" ? 1 : 0, + transition: "opacity 0.2s ease", + }} + /> + )} + + ) : ( + + ) + } + title={ + + {item.name} + {item.is_folder && + (item.path === "root" || !item.path) && + item.physical_path && ( + + Mounted + + )} + + } + description={item.size ? item.size : null} + /> + + + ); + })} + {filesQuery.isFetchingNextPage && ( +
+ +
)} - /> + )} diff --git a/client/src/components/FilePickerModal.test.js b/client/src/components/FilePickerModal.test.js index cdf85544..50fa08fe 100644 --- a/client/src/components/FilePickerModal.test.js +++ b/client/src/components/FilePickerModal.test.js @@ -2,6 +2,8 @@ import React from "react"; import { fireEvent, render, screen, waitFor } from "@testing-library/react"; import FilePickerModal from "./FilePickerModal"; import { apiClient } from "../api"; +import { QueryClientProvider } from "@tanstack/react-query"; +import { createAppQueryClient } from "../queryClient"; jest.mock("../api", () => ({ apiClient: { @@ -12,6 +14,20 @@ jest.mock("../api", () => ({ })); describe("FilePickerModal", () => { + const renderPicker = (props = {}) => { + const queryClient = createAppQueryClient(); + return render( + + + , + ); + }; + beforeEach(() => { jest.clearAllMocks(); window.matchMedia = jest.fn().mockImplementation((query) => ({ @@ -31,9 +47,7 @@ describe("FilePickerModal", () => { .mockRejectedValueOnce(new Error("Network Error")) .mockResolvedValueOnce({ data: [] }); - render( - , - ); + renderPicker(); expect(await screen.findByText("Files unavailable")).toBeTruthy(); expect( @@ -47,4 +61,50 @@ describe("FilePickerModal", () => { expect(screen.queryByText("Files unavailable")).toBeNull(), ); }); + + it("requests bounded pages for the visible folder", async () => { + apiClient.get + .mockResolvedValueOnce({ + data: { + items: Array.from({ length: 100 }, (_, index) => ({ + id: index + 1, + name: `volume-${index + 1}.tif`, + path: "root", + is_folder: false, + })), + total: 101, + offset: 0, + limit: 100, + has_more: true, + }, + }) + .mockResolvedValueOnce({ + data: { + items: [ + { + id: 101, + name: "volume-101.tif", + path: "root", + is_folder: false, + }, + ], + total: 101, + offset: 100, + limit: 100, + has_more: false, + }, + }); + + renderPicker(); + + await waitFor(() => expect(apiClient.get).toHaveBeenCalled()); + expect(apiClient.get.mock.calls[0][1].params).toEqual({ + parent: "root", + offset: 0, + limit: 100, + volume_only: false, + }); + expect(apiClient.get.mock.calls[0][1].signal).toBeDefined(); + expect(apiClient.get.mock.calls.length).toBeLessThanOrEqual(2); + }); }); diff --git a/client/src/index.js b/client/src/index.js index 5bca9ff2..dfd935d7 100644 --- a/client/src/index.js +++ b/client/src/index.js @@ -1,21 +1,25 @@ import React from "react"; import ReactDOM from "react-dom/client"; import { ConfigProvider } from "antd"; +import { QueryClientProvider } from "@tanstack/react-query"; import "./index.css"; import App from "./App"; import AppErrorBoundary from "./components/AppErrorBoundary"; import { antdWorkflowTheme } from "./design/workflowDesignSystem"; import { installClientLogging } from "./logging/appEventLog"; +import { appQueryClient } from "./queryClient"; installClientLogging(); const root = ReactDOM.createRoot(document.getElementById("root")); root.render( - - - - - + + + + + + + , ); diff --git a/client/src/queryClient.js b/client/src/queryClient.js new file mode 100644 index 00000000..898997b8 --- /dev/null +++ b/client/src/queryClient.js @@ -0,0 +1,17 @@ +import { QueryClient } from "@tanstack/react-query"; + +export const createAppQueryClient = () => + new QueryClient({ + defaultOptions: { + queries: { + refetchOnWindowFocus: false, + retry: false, + staleTime: 15_000, + }, + mutations: { + retry: false, + }, + }, + }); + +export const appQueryClient = createAppQueryClient(); diff --git a/client/src/views/FilesManager.js b/client/src/views/FilesManager.js index b77c284a..92b15eb3 100644 --- a/client/src/views/FilesManager.js +++ b/client/src/views/FilesManager.js @@ -111,6 +111,7 @@ const PROJECT_CONFIRMATION_ROLES = [ provenanceOnly: true, }, ]; +const FILE_MANAGER_PAGE_SIZE = 200; const getSelectedFilePath = (item) => { if (!item) return ""; @@ -757,8 +758,10 @@ function FilesManager() { const currentFolderRef = useRef("root"); const [loadedParents, setLoadedParents] = useState([]); const [loadingParents, setLoadingParents] = useState([]); + const [folderPages, setFolderPages] = useState({}); const loadedParentsRef = useRef(new Set()); const loadingParentsRef = useRef(new Set()); + const folderPagesRef = useRef({}); const [expandedFolders, setExpandedFolders] = useState([]); const [viewMode, setViewMode] = useState("grid"); // 'grid' or 'list' const [selectedItems, setSelectedItems] = useState([]); @@ -847,6 +850,10 @@ function FilesManager() { currentFolderRef.current = currentFolder; }, [currentFolder]); + useEffect(() => { + folderPagesRef.current = folderPages; + }, [folderPages]); + const syncLoadedParents = React.useCallback((nextParents) => { loadedParentsRef.current = new Set(nextParents); setLoadedParents(nextParents); @@ -885,14 +892,17 @@ function FilesManager() { filesRef.current = {}; setFolders([]); setFiles({}); + folderPagesRef.current = {}; + setFolderPages({}); syncLoadedParents([]); syncLoadingParents([]); setExpandedFolders([]); }, [syncLoadedParents, syncLoadingParents]); const replaceFolderChildren = React.useCallback( - (parentKey, fileList) => { + (parentKey, fileList, options = {}) => { const normalizedParent = String(parentKey || "root"); + const { append = false } = options; const previousFolders = foldersRef.current; const previousFiles = filesRef.current; const { folders: nextFolders, files: nextFiles } = @@ -903,18 +913,25 @@ function FilesManager() { const nextDirectChildIds = new Set( nextFolders.map((folder) => folder.key), ); - const removedFolderIds = collectDescendantFolderIds( - previousFolders, - existingDirectChildIds.filter((id) => !nextDirectChildIds.has(id)), - ); + const removedFolderIds = append + ? new Set() + : collectDescendantFolderIds( + previousFolders, + existingDirectChildIds.filter((id) => !nextDirectChildIds.has(id)), + ); + const previousFolderIds = new Set( + previousFolders.map((folder) => folder.key), + ); const mergedFolders = [ ...previousFolders.filter( (folder) => - folder.parent !== normalizedParent && + (append || folder.parent !== normalizedParent) && !removedFolderIds.has(folder.key), ), - ...nextFolders, + ...nextFolders.filter( + (folder) => !append || !previousFolderIds.has(folder.key), + ), ].sort((left, right) => { if (left.parent === right.parent) { return String(left.title || "").localeCompare( @@ -927,12 +944,17 @@ function FilesManager() { }); const mergedFiles = { ...previousFiles }; - delete mergedFiles[normalizedParent]; + if (!append) delete mergedFiles[normalizedParent]; removedFolderIds.forEach((folderId) => { delete mergedFiles[folderId]; }); Object.entries(nextFiles).forEach(([key, value]) => { - mergedFiles[key] = value; + const existing = append ? mergedFiles[key] || [] : []; + const existingIds = new Set(existing.map((item) => item.key)); + mergedFiles[key] = [ + ...existing, + ...value.filter((item) => !existingIds.has(item.key)), + ]; }); foldersRef.current = mergedFolders; @@ -1006,10 +1028,15 @@ function FilesManager() { const fetchFolderContents = React.useCallback( async (parentKey, options = {}) => { const normalizedParent = String(parentKey || "root"); - const { force = false, silentNetworkError = false } = options; + const { + force = false, + silentNetworkError = false, + append = false, + } = options; if ( !force && + !append && loadedParentsRef.current.has(normalizedParent) && !loadingParentsRef.current.has(normalizedParent) ) { @@ -1027,12 +1054,38 @@ function FilesManager() { setParentLoadingState(normalizedParent, true); try { + const previousPage = folderPagesRef.current[normalizedParent]; + const pageOffset = append ? previousPage?.nextOffset || 0 : 0; const res = await apiClient.get("/files", { - params: { parent: normalizedParent }, + params: { + parent: normalizedParent, + offset: pageOffset, + limit: FILE_MANAGER_PAGE_SIZE, + }, }); - replaceFolderChildren(normalizedParent, res.data); + const page = Array.isArray(res.data) + ? { + items: res.data, + total: res.data.length, + offset: 0, + limit: res.data.length || FILE_MANAGER_PAGE_SIZE, + has_more: false, + } + : res.data; + replaceFolderChildren(normalizedParent, page.items || [], { append }); + const nextPage = { + total: page.total, + loaded: pageOffset + (page.items || []).length, + nextOffset: pageOffset + Number(page.limit || FILE_MANAGER_PAGE_SIZE), + hasMore: Boolean(page.has_more), + }; + folderPagesRef.current = { + ...folderPagesRef.current, + [normalizedParent]: nextPage, + }; + setFolderPages(folderPagesRef.current); setServerUnavailable(false); - return res.data; + return page.items || []; } catch (err) { const isNetworkError = !err.response; const isMissingParent = @@ -4303,18 +4356,20 @@ function FilesManager() { onDragOver={handleDragOver} onDrop={(e) => handleDrop(e, currentFolder)} > - {loadingParents.includes(currentFolder) && ( -
- -
- )} + {loadingParents.includes(currentFolder) && + currentFolders.length === 0 && + currentFiles.length === 0 && ( +
+ +
+ )} {!loadingParents.includes(currentFolder) && renderProjectInitializationEmptyState()} {!loadingParents.includes(currentFolder) && @@ -4331,6 +4386,25 @@ function FilesManager() { {currentFolders.map((f) => renderItem(f, "folder"))} {renderNewFolderPlaceholder()} {currentFiles.map((f) => renderItem(f, "file"))} + {folderPages[currentFolder]?.hasMore && ( +
+ +
+ )} {selectionBox && (
{ await waitFor(() => { expect(apiClient.get).toHaveBeenCalledWith("/files", { - params: { parent: "root" }, + params: { parent: "root", offset: 0, limit: 200 }, }); }); expect(apiClient.get).not.toHaveBeenCalledWith("/files"); }); + it("loads additional bounded pages only when requested", async () => { + let filePage = 0; + apiClient.get.mockImplementation((url) => { + if (url === "/files/project-suggestions") { + return Promise.resolve({ data: [] }); + } + if (url !== "/files") { + return Promise.resolve({ data: { exists: false, profile: null } }); + } + filePage += 1; + if (filePage === 1) { + return Promise.resolve({ + data: { + items: [ + { + id: 1, + name: "first-volume.tif", + path: "root", + is_folder: false, + size: "1KB", + type: "image/tiff", + }, + ], + total: 201, + offset: 0, + limit: 200, + has_more: true, + }, + }); + } + return Promise.resolve({ + data: { + items: [ + { + id: 2, + name: "last-volume.tif", + path: "root", + is_folder: false, + size: "1KB", + type: "image/tiff", + }, + ], + total: 201, + offset: 200, + limit: 200, + has_more: false, + }, + }); + }); + + renderFilesManager(); + fireEvent.click(await screen.findByText("Load more files")); + + await waitFor(() => + expect(apiClient.get).toHaveBeenCalledWith("/files", { + params: { parent: "root", offset: 200, limit: 200 }, + }), + ); + expect(await screen.findByText("last-volume.tif")).toBeTruthy(); + expect(screen.queryByText("Load more files")).toBeNull(); + }); + it("opens a confirmation modal before registering a suggested smoke project", async () => { mockProjectSuggestionResponses([smokeSuggestion]); apiClient.post.mockResolvedValue({ @@ -506,7 +568,7 @@ describe("FilesManager", () => { ); }); expect(apiClient.get).toHaveBeenCalledWith("/files", { - params: { parent: "7" }, + params: { parent: "7", offset: 0, limit: 200 }, }); await continueWithProjectContext(); await waitFor(() => { @@ -1049,7 +1111,7 @@ describe("FilesManager", () => { await screen.findByText("Confirm project basics"); await waitFor(() => { expect(apiClient.get).toHaveBeenCalledWith("/files", { - params: { parent: "7" }, + params: { parent: "7", offset: 0, limit: 200 }, }); expect(mockWorkflowContext.consumeRuntimeAction).toHaveBeenCalledWith( "choose-data-action", diff --git a/client/src/views/Visualization.test.js b/client/src/views/Visualization.test.js index fc63a823..59b71f49 100644 --- a/client/src/views/Visualization.test.js +++ b/client/src/views/Visualization.test.js @@ -3,6 +3,8 @@ import { render, screen, waitFor } from "@testing-library/react"; import Visualization from "./Visualization"; import { AppContext } from "../contexts/GlobalContext"; import { useWorkflow } from "../contexts/WorkflowContext"; +import { QueryClientProvider } from "@tanstack/react-query"; +import { createAppQueryClient } from "../queryClient"; jest.mock("../contexts/WorkflowContext", () => ({ useWorkflow: jest.fn(), @@ -17,6 +19,17 @@ jest.mock("../api", () => ({ })); describe("Visualization workflow defaults", () => { + const renderWithQueryClient = (ui) => { + const queryClient = createAppQueryClient(); + return render(ui, { + wrapper: ({ children }) => ( + + {children} + + ), + }); + }; + it("hydrates the canonical volume pair and voxel scales", async () => { const setCurrentImage = jest.fn(); const setCurrentLabel = jest.fn(); @@ -42,7 +55,7 @@ describe("Visualization workflow defaults", () => { }, }); - render( + renderWithQueryClient( { setCurrentLabel, setVisualizationScales, }; - const { rerender } = render( + const { rerender } = renderWithQueryClient( , diff --git a/docs/decisions/dbos-durable-operations-spike.md b/docs/decisions/dbos-durable-operations-spike.md new file mode 100644 index 00000000..a7227fab --- /dev/null +++ b/docs/decisions/dbos-durable-operations-spike.md @@ -0,0 +1,116 @@ +# DBOS Durable Operations Spike + +- **Date:** 2026-07-21 +- **Status:** Conditional go for a Postgres-backed integration prototype; no-go for production-route integration +- **Library evaluated:** `dbos[aiosqlite]==2.28.0` +- **Production code changed:** None + +## Question + +Can DBOS replace the process-global job state used by training and inference while +preserving the product-facing `WorkflowOperation` lifecycle: idempotent submission, +durable progress, cancellation, and restart recovery? + +## Prototype + +The isolated implementation lives in `spikes/dbos_operation`. It models a heavy +operation as a queued DBOS workflow containing checkpointed steps. Each step writes +an idempotent external marker representing a compute-chunk side effect. The workflow +publishes an `operation_progress` DBOS event containing the workflow ID, correlation +ID, completed and total step counts, normalized progress, and status. + +The runner supports four operations without registering a production route: + +```bash +uv run python scripts/run_dbos_operation_spike.py execute \ + --database /tmp/pytc-dbos.sqlite \ + --workspace /tmp/pytc-dbos-work \ + --workflow-id workflow-1 \ + --correlation-id request-1 \ + --duplicate-submission + +uv run python scripts/run_dbos_operation_spike.py status \ + --database /tmp/pytc-dbos.sqlite --workflow-id workflow-1 + +uv run python scripts/run_dbos_operation_spike.py cancel \ + --database /tmp/pytc-dbos.sqlite --workflow-id workflow-1 + +uv run python scripts/run_dbos_operation_spike.py recover \ + --database /tmp/pytc-dbos.sqlite --workflow-id workflow-1 +``` + +Automated evidence is produced by: + +```bash +uv run pytest -q tests/test_dbos_operation_spike.py +``` + +Current result: **4 passed in 21.75 seconds** on Python 3.11 with a local SQLite +system database. + +## Gates + +| Gate | Required evidence | Result | +| --- | --- | --- | +| Runtime compatibility | Installs on the repository's Python 3.10-3.11 range | **Pass.** DBOS 2.28.0 declares Python >=3.10; the spike ran on 3.11. | +| Idempotent submission | Submitting the same workflow ID twice executes one workflow and one set of external markers | **Pass.** Both handles have the same ID; every marker is written once. | +| Durable progress | Progress is queryable outside the worker and remains available after completion or process death | **Pass.** `DBOS.set_event` progress is read through `DBOSClient`. | +| Queued cancellation | Cancelling enqueued work removes it before any external effect | **Pass.** Status becomes `CANCELLED`; no marker directory is created. | +| Running cancellation | Cancellation stops work at a documented durable boundary | **Pass with constraint.** Cancellation preempts at the next step boundary; it does not interrupt an ordinary blocking synchronous step. | +| Single-server restart | A killed process resumes from its last completed step without repeating that step's external effect | **Pass.** A replacement with the same executor identity recovers the `PENDING` workflow and completes the remaining markers. | +| Mid-step crash safety | Killing a process during a non-transactional external side effect cannot duplicate or corrupt that effect | **Not proven.** The test kills after the step and progress event are durable. Production steps still require idempotent outputs or transactional integration. | +| Postgres and multiple executors | Recovery, queue concurrency, and cancellation work with the intended production topology | **Not run; production gate fails.** SQLite is explicitly a development/test backend. | +| PyTC subprocess control | Training/inference subprocesses are killed, reaped, and reconciled correctly on cancel/restart | **Not run; production gate fails.** A blocking `Popen` step is not sufficient. | +| Product-state projection | DBOS state and `WorkflowOperation` cannot diverge under crashes | **Not designed; production gate fails.** A single source of truth and projection strategy is required. | + +## Findings + +1. DBOS workflow IDs directly provide the idempotency behavior currently implemented + around `WorkflowOperation.idempotency_key`. +2. DBOS events are a good fit for durable progress and correlation data. They avoid + writing a high-frequency progress stream into the main workflow table. +3. Queue concurrency maps well to GPU/CPU resource limits without a separate broker. +4. Recovery ownership is operationally significant. In the local single-server + setup, the replacement process must reuse the interrupted executor identity. + Multiple live executors require a production recovery strategy rather than PID- + derived identities. +5. DBOS cancellation is cooperative at workflow/step boundaries. Immediate training + cancellation needs a preemptible async step or a short polling supervisor around + the subprocess, including process-group termination and output reconciliation. +6. DBOS does not make arbitrary filesystem or model-training side effects exactly + once. Those effects must remain idempotent. The spike demonstrates this with + exclusive marker creation. + +## Decision + +**Do not mount DBOS into the FastAPI production routes on this branch.** The local +prototype passes the behavioral gates it can honestly exercise, but the Postgres, +multi-executor, subprocess cancellation, and state-projection gates remain open. + +Proceed with one additional, bounded integration phase only if Postgres is accepted +as backend infrastructure: + +1. Run this suite against Postgres with two executor processes and forced worker + loss. +2. Implement a supervised synthetic subprocess step and prove cancel/restart process + cleanup before using real training. +3. Keep `WorkflowOperation` as the user-facing read model while DBOS owns execution, + and update that read model through an idempotent projection keyed by DBOS workflow + ID. Do not dual-write independent lifecycle transitions. +4. Use stable IDs such as + `workflow:{workflow_id}:operation:{operation_id}:attempt:{attempt}` and retain the + request correlation ID in progress events. +5. Integrate one operation type behind a feature flag, starting with evidence export + or evaluation rather than GPU training. + +The go/no-go after that phase is straightforward: all four currently failing gates +must pass before a production endpoint submits DBOS work. + +## Primary References + +- [DBOS Python workflows and workflow-ID idempotency](https://docs.dbos.dev/python/tutorials/workflow-tutorial) +- [DBOS queues and concurrency](https://docs.dbos.dev/python/reference/queues) +- [DBOS cancellation and resume semantics](https://docs.dbos.dev/python/tutorials/workflow-management) +- [DBOS workflow events for progress](https://docs.dbos.dev/python/tutorials/workflow-communication) +- [DBOS SQLite and Postgres guidance](https://docs.dbos.dev/python/tutorials/database-connection) +- [DBOS workflow recovery](https://docs.dbos.dev/production/workflow-recovery) diff --git a/pyproject.toml b/pyproject.toml index 4495d961..17e38681 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,5 +43,7 @@ dependencies = [ [dependency-groups] dev = [ "black==26.5.1", + "dbos[aiosqlite]==2.28.0", + "playwright>=1.54,<2", "pytest==9.0.2", ] diff --git a/scripts/browser_synthetic_core_smoke.py b/scripts/browser_synthetic_core_smoke.py new file mode 100644 index 00000000..4615e568 --- /dev/null +++ b/scripts/browser_synthetic_core_smoke.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Browser smoke for the deterministic synthetic core project.""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import urlparse + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from scripts.browser_yixiao_case_study_smoke import ( + DEFAULT_BASE_URL, + VIEWPORT_DEFAULT, + _extract_progress_snapshot, + _open_tab, + _parse_viewport, + _playwright_import_error, + _raise_playwright_error, + _resolve_browser_playwright, +) + +PROJECT_TITLE = "Synthetic Segmentation Core Loop" +PROJECT_MOUNT_LABEL = PROJECT_TITLE +EXPECTED_PROGRESS = { + "tracked": 4, + "good": 2, + "proofread": 1, + "missing": 1, +} +DEFAULT_REPORT = "/tmp/synthetic-core-browser-smoke.json" + + +def _build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default=DEFAULT_BASE_URL) + parser.add_argument("--timeout-ms", type=int, default=30_000) + parser.add_argument( + "--viewport", + default=f"{VIEWPORT_DEFAULT[0]}x{VIEWPORT_DEFAULT[1]}", + ) + parser.add_argument("--no-headless", action="store_true") + parser.add_argument("--skip-reload", action="store_true") + parser.add_argument("--report", default=DEFAULT_REPORT) + return parser + + +def _assert_progress(page) -> Dict[str, Any]: + snapshot = _extract_progress_snapshot(page) + for key, expected in EXPECTED_PROGRESS.items(): + actual = (snapshot.get("metrics") or {}).get(key) + if actual != expected: + raise AssertionError( + f"Synthetic progress metric {key!r} expected {expected}, got {actual}" + ) + return snapshot + + +def _open_file_picker(page, timeout_ms: int) -> None: + _open_tab(page, "Visualize") + page.get_by_role("button", name="Browse", exact=True).first.click() + dialog = page.get_by_role("dialog", name="Select File") + dialog.wait_for(state="visible", timeout=timeout_ms) + if dialog.get_by_text("Files unavailable").count(): + raise AssertionError("Browse modal reported that files were unavailable") + + project_row = dialog.locator(".file-picker-item").filter( + has_text=PROJECT_MOUNT_LABEL + ) + project_row.first.wait_for(state="visible", timeout=timeout_ms) + project_row.first.click() + data_row = dialog.locator(".file-picker-item").filter(has_text="data") + data_row.first.wait_for(state="visible", timeout=timeout_ms) + data_row.first.click() + raw_row = dialog.locator(".file-picker-item").filter(has_text="raw") + raw_row.first.wait_for(state="visible", timeout=timeout_ms) + raw_row.first.click() + preview = dialog.get_by_role("img", name="train-01_image.h5") + preview.wait_for(state="attached", timeout=timeout_ms) + page.wait_for_function( + "image => image.complete && image.naturalWidth > 0", + arg=preview.element_handle(), + timeout=timeout_ms, + ) + if dialog.get_by_text("Files unavailable").count(): + raise AssertionError("Browse modal failed after opening the project folder") + dialog.get_by_role("button", name="Close").click() + + +def _exercise_core_ui(page, timeout_ms: int) -> Dict[str, Any]: + page.get_by_text(PROJECT_TITLE, exact=True).first.wait_for( + state="visible", timeout=timeout_ms + ) + _open_tab(page, "Files") + page.get_by_text(PROJECT_MOUNT_LABEL, exact=True).first.wait_for( + state="visible", timeout=timeout_ms + ) + _open_file_picker(page, timeout_ms) + _open_tab(page, "Workflow") + page.wait_for_timeout(300) + return _assert_progress(page) + + +def run_smoke( + *, + base_url: str, + timeout_ms: int, + viewport: Tuple[int, int], + headless: bool, + skip_reload: bool, +) -> Dict[str, Any]: + playwright_error = _playwright_import_error() + if playwright_error is not None: + raise _raise_playwright_error(str(playwright_error)) + + from playwright.sync_api import sync_playwright + + result: Dict[str, Any] = {"passed": True, "base_url": base_url, "checks": []} + with sync_playwright() as playwright: + try: + browser = _resolve_browser_playwright(playwright, headless=headless) + except RuntimeError as exc: + raise _raise_playwright_error(str(exc)) from exc + context = browser.new_context( + viewport={"width": viewport[0], "height": viewport[1]} + ) + page = context.new_page() + page.set_default_timeout(timeout_ms) + page_errors: List[str] = [] + api_failures: List[str] = [] + page.on("pageerror", lambda error: page_errors.append(str(error))) + page.on( + "response", + lambda response: ( + api_failures.append( + f"{response.status} {response.request.method} {response.url}" + ) + if response.status >= 500 and "/api" in response.url + else None + ), + ) + try: + page.goto(base_url, wait_until="domcontentloaded", timeout=timeout_ms) + snapshot = _exercise_core_ui(page, timeout_ms) + result["checks"].extend( + [ + "synthetic project loaded", + "bounded file picker navigated", + "bounded volume preview decoded", + "progress counts verified", + ] + ) + result["progress_snapshot"] = snapshot + + if not skip_reload: + page.reload(wait_until="domcontentloaded", timeout=timeout_ms) + _exercise_core_ui(page, timeout_ms) + result["checks"].append("reload continuity verified") + + if page_errors: + raise AssertionError(f"Browser page errors: {page_errors}") + if api_failures: + raise AssertionError(f"API 5xx responses: {api_failures}") + finally: + context.close() + browser.close() + return result + + +def main(argv: Optional[List[str]] = None) -> int: + args = _build_arg_parser().parse_args(argv) + base_url = args.base_url.rstrip("/") + if urlparse(base_url).scheme not in {"http", "https"}: + raise ValueError("base-url must include an http or https scheme") + + report: Dict[str, Any] = { + "generated_at_unix": time.time(), + "script": str(Path(__file__).resolve()), + } + try: + report.update( + run_smoke( + base_url=base_url, + timeout_ms=max(5_000, args.timeout_ms), + viewport=_parse_viewport(args.viewport), + headless=not args.no_headless, + skip_reload=args.skip_reload, + ) + ) + except Exception as exc: + report.update( + passed=False, + error=str(exc), + error_type=exc.__class__.__name__, + ) + + Path(args.report).write_text( + json.dumps(report, indent=2, sort_keys=True), encoding="utf-8" + ) + print( + "Synthetic browser smoke passed." + if report.get("passed") + else f"Synthetic browser smoke failed: {report.get('error')}" + ) + print(f"Report: {args.report}") + return 0 if report.get("passed") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/browser_yixiao_case_study_smoke.py b/scripts/browser_yixiao_case_study_smoke.py index be50f066..17914a1a 100755 --- a/scripts/browser_yixiao_case_study_smoke.py +++ b/scripts/browser_yixiao_case_study_smoke.py @@ -174,18 +174,17 @@ def _playwright_import_error() -> Optional[BaseException]: return exc if isinstance(exc, ImportError) else ImportError(str(exc)) -def _resolve_browser_playwright(playwright) -> None: +def _resolve_browser_playwright(playwright, *, headless: bool = True): browsers = [ - ("chromium", playwright.chromium), - ("firefox", playwright.firefox), - ("webkit", playwright.webkit), + ("chromium", playwright.chromium, {}), + ("chromium-channel", playwright.chromium, {"channel": "chromium"}), + ("firefox", playwright.firefox, {}), + ("webkit", playwright.webkit, {}), ] errors: List[str] = [] - for name, launcher in browsers: + for name, launcher, options in browsers: try: - browser = launcher.launch(headless=True) - browser.close() - return + return launcher.launch(headless=headless, **options) except Exception as exc: # pragma: no cover - engine-specific runtime behavior errors.append(f"{name}: {exc}") raise RuntimeError( @@ -522,12 +521,10 @@ def run_smoke( from playwright.sync_api import sync_playwright with sync_playwright() as p: - # Pick a working engine once to avoid launch failures at runtime. try: - _resolve_browser_playwright(p) + browser = _resolve_browser_playwright(p, headless=headless) except RuntimeError as exc: raise _raise_playwright_error(str(exc)) - browser = p.chromium.launch(headless=headless) context = browser.new_context(viewport=viewport_config) page = context.new_page() page.set_default_timeout(timeout_ms) diff --git a/scripts/run_dbos_operation_spike.py b/scripts/run_dbos_operation_spike.py new file mode 100644 index 00000000..80057c93 --- /dev/null +++ b/scripts/run_dbos_operation_spike.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import pathlib +import sys + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from spikes.dbos_operation.runner import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server_api/auth/models.py b/server_api/auth/models.py index f45ee28c..3d6e2566 100644 --- a/server_api/auth/models.py +++ b/server_api/auth/models.py @@ -203,6 +203,14 @@ class Config: from_attributes = True +class FilePageResponse(BaseModel): + items: List[FileResponse] + total: int + offset: int + limit: int + has_more: bool + + class Token(BaseModel): access_token: str token_type: str diff --git a/server_api/auth/router.py b/server_api/auth/router.py index 4a079eeb..5543751b 100644 --- a/server_api/auth/router.py +++ b/server_api/auth/router.py @@ -1,12 +1,23 @@ -from fastapi import APIRouter, Depends, HTTPException, status, UploadFile, File, Form +from fastapi import ( + APIRouter, + Depends, + File, + Form, + HTTPException, + Query, + UploadFile, + status, +) from fastapi.responses import Response from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm +from sqlalchemy import func, or_ from sqlalchemy.orm import Session from . import models, utils, database from jose import JWTError, jwt -from typing import Any, List, Optional +from typing import Any, List, Optional, Union from datetime import datetime, timezone from urllib.parse import parse_qs, unquote, urlparse +import anyio import json import shutil import os @@ -15,6 +26,7 @@ import re import math from server_api.utils.utils import resolve_existing_path +from server_api.workflows.volume_io import open_volume_store try: # Optional preview dependency import cv2 @@ -37,6 +49,7 @@ IGNORED_SYSTEM_FILENAMES = { ".ds_store", "thumbs.db", + "workflow_preference.json", ".pytc_proofreading.json", ".pytc_instance_labels.tif", ".pytc_project_context.json", @@ -48,6 +61,11 @@ PROJECT_AUDIT_MAX_SAMPLE_VALUES = 250_000 PROJECT_AUDIT_MAX_FINDINGS = 24 PROJECT_CONTEXT_FILENAME = ".pytc_project_context.json" +FILE_LIST_DEFAULT_LIMIT = 100 +FILE_LIST_MAX_LIMIT = 500 +FILE_LIST_MAINTENANCE_LIMIT = 500 +FILE_PREVIEW_MAX_SOURCE_DIM = 1024 +FILE_PREVIEW_LIMITER = anyio.CapacityLimiter(2) VOLUME_EXTENSIONS = { ".h5", @@ -68,6 +86,14 @@ CONFIG_EXTENSIONS = {".yaml", ".yml", ".json", ".toml"} CHECKPOINT_EXTENSIONS = {".pt", ".pth", ".pth.tar", ".ckpt", ".onnx"} TEXT_SIGNAL_EXTENSIONS = CONFIG_EXTENSIONS | {".md", ".txt", ".csv", ".tsv"} +FILE_PICKER_VOLUME_EXTENSIONS = VOLUME_EXTENSIONS | { + ".png", + ".jpg", + ".jpeg", + ".gif", + ".bmp", + ".webp", +} def _format_size(size_bytes: int) -> str: @@ -2024,16 +2050,24 @@ def _is_managed_upload_path(user_id: int, physical_path: Optional[str]) -> bool: return False -def _repair_stale_mounted_entries(db: Session, user_id: int) -> None: - candidates = ( - db.query(models.File) - .filter( - models.File.user_id == user_id, - models.File.physical_path.isnot(None), - models.File.path != "root", - ) - .all() +def _repair_stale_mounted_entries( + db: Session, + user_id: int, + *, + parent: Optional[str] = None, + max_candidates: Optional[int] = None, +) -> None: + query = db.query(models.File).filter( + models.File.user_id == user_id, + models.File.physical_path.isnot(None), + models.File.path != "root", ) + if parent is not None: + query = query.filter(models.File.path == parent) + query = query.order_by(models.File.id.asc()) + if max_candidates is not None: + query = query.limit(max_candidates) + candidates = query.all() changed = False for entry in candidates: @@ -2066,16 +2100,24 @@ def _repair_stale_mounted_entries(db: Session, user_id: int) -> None: db.commit() -def _prune_missing_managed_upload_entries(db: Session, user_id: int) -> None: - candidates = ( - db.query(models.File) - .filter( - models.File.user_id == user_id, - models.File.is_folder.is_(False), - models.File.physical_path.isnot(None), - ) - .all() +def _prune_missing_managed_upload_entries( + db: Session, + user_id: int, + *, + parent: Optional[str] = None, + max_candidates: Optional[int] = None, +) -> None: + query = db.query(models.File).filter( + models.File.user_id == user_id, + models.File.is_folder.is_(False), + models.File.physical_path.isnot(None), ) + if parent is not None: + query = query.filter(models.File.path == parent) + query = query.order_by(models.File.id.asc()) + if max_candidates is not None: + query = query.limit(max_candidates) + candidates = query.all() removed = False for entry in candidates: @@ -2249,14 +2291,53 @@ def list_current_user_projects( # File Management Endpoints -@router.get("/files", response_model=List[models.FileResponse]) +def _visible_files_query(query): + lower_name = func.lower(models.File.name) + return query.filter( + ~lower_name.in_(sorted(IGNORED_SYSTEM_FILENAMES)), + ~lower_name.like(".__pytc_runtime_%"), + ) + + +def _volume_picker_files_query(query): + lower_name = func.lower(models.File.name) + return query.filter( + or_( + models.File.is_folder.is_(True), + *( + lower_name.like(f"%{extension}") + for extension in sorted(FILE_PICKER_VOLUME_EXTENSIONS) + ), + ) + ) + + +@router.get( + "/files", + response_model=Union[List[models.FileResponse], models.FilePageResponse], +) def get_files( parent: Optional[str] = None, + offset: Optional[int] = Query(default=None, ge=0), + limit: Optional[int] = Query(default=None, ge=1, le=FILE_LIST_MAX_LIMIT), + volume_only: bool = False, current_user: models.User = Depends(get_current_user), db: Session = Depends(database.get_db), ): - _repair_stale_mounted_entries(db, current_user.id) - _prune_missing_managed_upload_entries(db, current_user.id) + paged = offset is not None or limit is not None + maintenance_limit = FILE_LIST_MAINTENANCE_LIMIT if paged else None + _repair_stale_mounted_entries( + db, + current_user.id, + parent=parent, + max_candidates=maintenance_limit, + ) + _prune_missing_managed_upload_entries( + db, + current_user.id, + parent=parent, + max_candidates=maintenance_limit, + ) query = db.query(models.File).filter(models.File.user_id == current_user.id) if parent is not None: if parent != "root": @@ -2280,24 +2361,33 @@ def get_files( ) query = query.filter(models.File.path == parent) - return [ - file - for file in query.order_by( + query = _visible_files_query(query) + ordered_query = query.order_by(models.File.is_folder.desc(), models.File.name.asc()) + if not paged: + return ordered_query.all() + + page_offset = offset or 0 + page_limit = limit or FILE_LIST_DEFAULT_LIMIT + if volume_only: + ordered_query = _volume_picker_files_query(query).order_by( models.File.is_folder.desc(), models.File.name.asc() - ).all() - if not _is_ignored_system_file(file.name) - ] + ) + total = ordered_query.order_by(None).count() + items = ordered_query.offset(page_offset).limit(page_limit).all() + return models.FilePageResponse( + items=items, + total=total, + offset=page_offset, + limit=page_limit, + has_more=page_offset + len(items) < total, + ) -@router.get("/files/preview/{file_id}") -def file_preview( +def _get_preview_file( file_id: int, current_user: models.User = Depends(get_current_user), db: Session = Depends(database.get_db), -): - if cv2 is None or np is None: - raise HTTPException(status_code=500, detail="Preview dependencies missing") - +) -> models.File: file = ( db.query(models.File) .filter(models.File.id == file_id, models.File.user_id == current_user.id) @@ -2307,72 +2397,81 @@ def file_preview( raise HTTPException(status_code=404, detail="File not found") if not file.physical_path or not os.path.exists(file.physical_path): raise HTTPException(status_code=404, detail="File not found on disk") + return file - def to_uint8(arr): - if arr is None: - return None - if arr.dtype == np.uint8: - return arr - arr = arr.astype(np.float32) - min_val = np.nanmin(arr) - max_val = np.nanmax(arr) - if max_val <= min_val: - return np.zeros_like(arr, dtype=np.uint8) - scaled = (arr - min_val) / (max_val - min_val) - return np.clip(scaled * 255.0, 0, 255).astype(np.uint8) - - def load_image(path: str) -> Optional["np.ndarray"]: - ext = os.path.splitext(path)[1].lower() - if ext in {".h5", ".hdf5"}: - try: - import h5py # type: ignore - with h5py.File(path, "r") as handle: - datasets = [] +def _to_preview_uint8(arr): + if arr is None: + return None + if arr.dtype == np.uint8: + return arr + arr = arr.astype(np.float32) + min_val = np.nanmin(arr) + max_val = np.nanmax(arr) + if max_val <= min_val: + return np.zeros_like(arr, dtype=np.uint8) + scaled = (arr - min_val) / (max_val - min_val) + return np.clip(scaled * 255.0, 0, 255).astype(np.uint8) + + +def _volume_preview_crop(shape: tuple[int, ...]) -> tuple[tuple[slice, ...], int]: + if len(shape) < 2: + raise ValueError("Preview volume must have at least two dimensions") + + has_color_axis = len(shape) >= 3 and shape[-1] in (3, 4) + retained_dimensions = 3 if has_color_axis else 2 + leading_dimensions = len(shape) - retained_dimensions + height_axis = leading_dimensions + width_axis = leading_dimensions + 1 + source_max = max(shape[height_axis], shape[width_axis]) + stride = max(1, math.ceil(source_max / FILE_PREVIEW_MAX_SOURCE_DIM)) + crop = [ + slice(dimension // 2, dimension // 2 + 1) + for dimension in shape[:leading_dimensions] + ] + crop.extend( + [ + slice(None, None, stride), + slice(None, None, stride), + ] + ) + if has_color_axis: + crop.append(slice(None)) + return tuple(crop), leading_dimensions - def collect_dataset(_name, obj): - if isinstance(obj, h5py.Dataset) and obj.ndim >= 2: - datasets.append(obj) - handle.visititems(collect_dataset) - if not datasets: - return None - dataset = datasets[0] - leading_indices = tuple( - dimension // 2 for dimension in dataset.shape[:-2] - ) - img = dataset[(*leading_indices, slice(None), slice(None))] - except Exception: - img = None - elif ext in {".tif", ".tiff"} and tifffile is not None: - try: - img = tifffile.imread(path) - except Exception: - img = None - else: - img = cv2.imread(path, cv2.IMREAD_UNCHANGED) - if img is None: +def _load_preview_image(path: str) -> Optional["np.ndarray"]: + extension = _project_extension(path) + if extension in VOLUME_EXTENSIONS: + try: + with open_volume_store(path) as store: + crop, leading_dimensions = _volume_preview_crop(store.metadata.shape) + img = store.read(crop) + img = np.asarray(img) + for _ in range(leading_dimensions): + img = np.take(img, 0, axis=0) + except Exception: return None + else: + img = cv2.imread(path, cv2.IMREAD_UNCHANGED) - img = np.asarray(img) - if img.ndim == 2: - return to_uint8(img) - if img.ndim == 3: - if img.shape[2] in (3, 4): - if img.shape[2] == 4: - img = img[:, :, :3] - return to_uint8(img) - mid = img.shape[0] // 2 - return to_uint8(img[mid]) - if img.ndim == 4: - mid = img.shape[0] // 2 - img = img[mid] - if img.ndim == 3 and img.shape[2] == 4: - img = img[:, :, :3] - return to_uint8(img) + if img is None: return None + img = np.asarray(img) + if img.ndim == 2: + return _to_preview_uint8(img) + if img.ndim == 3 and img.shape[2] in (3, 4): + if img.shape[2] == 4: + img = img[:, :, :3] + return _to_preview_uint8(img) + return None - image = load_image(file.physical_path) + +def _render_file_preview(path: str) -> bytes: + if cv2 is None or np is None: + raise HTTPException(status_code=500, detail="Preview dependencies missing") + + image = _load_preview_image(path) if image is None: raise HTTPException(status_code=415, detail="Unsupported image format") @@ -2386,7 +2485,19 @@ def collect_dataset(_name, obj): success, buffer = cv2.imencode(".png", image) if not success: raise HTTPException(status_code=500, detail="Failed to encode preview") - return Response(content=buffer.tobytes(), media_type="image/png") + return buffer.tobytes() + + +@router.get("/files/preview/{file_id}") +async def file_preview( + file: models.File = Depends(_get_preview_file), +): + content = await anyio.to_thread.run_sync( + _render_file_preview, + file.physical_path, + limiter=FILE_PREVIEW_LIMITER, + ) + return Response(content=content, media_type="image/png") @router.post("/files/upload", response_model=models.FileResponse) diff --git a/server_api/errors.py b/server_api/errors.py index ffb8cfb9..f8e74d8d 100644 --- a/server_api/errors.py +++ b/server_api/errors.py @@ -11,9 +11,10 @@ from server_api.chatbot.logging_utils import request_id_from_request - logger = logging.getLogger(__name__) ERROR_SCHEMA_VERSION = 1 +PROBLEM_JSON_MEDIA_TYPE = "application/problem+json" +PROBLEM_TYPE_BASE = "https://seg.bio/problems" def _message_from_detail(detail: Any, fallback: str) -> str: @@ -117,12 +118,49 @@ def _error_metadata(status_code: int) -> Dict[str, Any]: } +def _accepts_problem_json(request: Request) -> bool: + """Require an explicit media type during the compatibility transition.""" + for media_range in request.headers.get("accept", "").split(","): + parts = [part.strip() for part in media_range.split(";")] + if not parts or parts[0].lower() != PROBLEM_JSON_MEDIA_TYPE: + continue + quality = next( + ( + part.split("=", 1)[1].strip() + for part in parts[1:] + if part.lower().startswith("q=") + ), + "1", + ) + try: + if float(quality) > 0: + return True + except ValueError: + continue + return False + + +def _append_vary_header(headers: Dict[str, str], field: str) -> None: + existing_key = next((key for key in headers if key.lower() == "vary"), "Vary") + existing_fields = { + item.strip().lower() + for item in headers.get(existing_key, "").split(",") + if item.strip() + } + if field.lower() not in existing_fields: + headers[existing_key] = ", ".join( + item for item in (headers.get(existing_key, "").strip(), field) if item + ) + + def build_error_response( *, status_code: int, request_id: str, detail: Any, message_fallback: str, + instance: str, + use_problem_json: bool = False, headers: Optional[Dict[str, str]] = None, validation_errors: Optional[List[Dict[str, Any]]] = None, ) -> JSONResponse: @@ -136,13 +174,33 @@ def build_error_response( if validation_errors is not None: error["validation_errors"] = validation_errors + problem = { + "type": f"{PROBLEM_TYPE_BASE}/{metadata['code'].replace('_', '-')}", + "title": metadata["title"], + "status": status_code, + "instance": f"{instance}#request-{request_id}", + } + if use_problem_json: + problem["detail"] = error["message"] + problem["error"] = error + if detail != error["message"]: + problem["legacy_detail"] = detail + if validation_errors is not None: + problem["validation_errors"] = validation_errors + else: + # Preserve FastAPI's historical detail value for existing consumers. + problem["detail"] = detail + problem["error"] = error + response_headers = dict(headers or {}) response_headers.setdefault("x-request-id", request_id) response_headers.setdefault("cache-control", "no-store") + _append_vary_header(response_headers, "Accept") return JSONResponse( status_code=status_code, - content=jsonable_encoder({"detail": detail, "error": error}), + content=jsonable_encoder(problem), headers=response_headers, + media_type=PROBLEM_JSON_MEDIA_TYPE if use_problem_json else None, ) @@ -157,6 +215,8 @@ async def http_exception_handler( request_id=request_id, detail=exc.detail, message_fallback="The request could not be completed.", + instance=request.url.path, + use_problem_json=_accepts_problem_json(request), headers=exc.headers, ) @@ -171,6 +231,8 @@ async def validation_exception_handler( request_id=request_id, detail=errors, message_fallback="Review the highlighted request values and try again.", + instance=request.url.path, + use_problem_json=_accepts_problem_json(request), validation_errors=errors, ) @@ -191,4 +253,6 @@ async def unexpected_exception_handler( request_id=request_id, detail="An unexpected server error occurred.", message_fallback="An unexpected server error occurred.", + instance=request.url.path, + use_problem_json=_accepts_problem_json(request), ) diff --git a/server_api/workflows/agent_actions.py b/server_api/workflows/agent_actions.py index b0212c45..9496ca2d 100644 --- a/server_api/workflows/agent_actions.py +++ b/server_api/workflows/agent_actions.py @@ -1,10 +1,17 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Annotated, Any, Dict, Literal, Optional, Union - -from pydantic import BaseModel, ConfigDict, Field - +from datetime import datetime +from typing import Annotated, Any, Dict, List, Literal, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StringConstraints, + TypeAdapter, + model_validator, +) RiskLevel = Literal[ "read_only", @@ -239,6 +246,456 @@ class AgentActionDefinition: } +NonEmptyString = Annotated[ + str, + StringConstraints(strip_whitespace=True, min_length=1, max_length=500), +] +IdempotencyKey = Annotated[ + str, + StringConstraints(strip_whitespace=True, min_length=1, max_length=255), +] +CorrelationId = Annotated[ + str, + StringConstraints(strip_whitespace=True, min_length=1, max_length=255), +] +ApprovalStatus = Literal["not_required", "pending", "approved", "rejected"] +ExecutionOwner = Literal["browser_navigation", "server_workflow", "server_runtime"] +OperationStatus = Literal["queued", "running", "succeeded", "failed", "cancelled"] +ReceiptStatus = Literal["accepted", "running", "succeeded", "failed", "cancelled"] + + +class _StrictBoundaryModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class ArtifactReference(_StrictBoundaryModel): + """Stable artifact identity used at the action execution boundary.""" + + artifact_id: Optional[int] = Field(default=None, gt=0) + logical_name: Optional[NonEmptyString] = None + artifact_type: Optional[NonEmptyString] = None + role: Optional[NonEmptyString] = None + path: Optional[NonEmptyString] = None + uri: Optional[NonEmptyString] = None + checksum: Optional[NonEmptyString] = None + media_type: Optional[NonEmptyString] = None + immutable: bool = True + + @model_validator(mode="after") + def require_identity(self) -> "ArtifactReference": + if not any( + ( + self.artifact_id, + self.logical_name, + self.path, + self.uri, + self.checksum, + ) + ): + raise ValueError( + "artifact reference requires artifact_id, logical_name, path, uri, " + "or checksum" + ) + return self + + +class WorkflowStagePrecondition(_StrictBoundaryModel): + kind: Literal["workflow_stage"] + allowed_stages: List[NonEmptyString] = Field(min_length=1) + + +class ArtifactAvailablePrecondition(_StrictBoundaryModel): + kind: Literal["artifact_available"] + artifact: ArtifactReference + + +class WorkflowFieldPrecondition(_StrictBoundaryModel): + kind: Literal["workflow_field_present"] + field_name: NonEmptyString + + +class OperationStatusPrecondition(_StrictBoundaryModel): + kind: Literal["operation_status"] + operation_id: int = Field(gt=0) + allowed_statuses: List[OperationStatus] = Field(min_length=1) + + +ActionPrecondition = Annotated[ + Union[ + WorkflowStagePrecondition, + ArtifactAvailablePrecondition, + WorkflowFieldPrecondition, + OperationStatusPrecondition, + ], + Field(discriminator="kind"), +] + + +class ActionPolicy(_StrictBoundaryModel): + risk_level: RiskLevel + requires_approval: bool + approval_reason: Optional[NonEmptyString] = None + + +class ActionApproval(_StrictBoundaryModel): + status: ApprovalStatus + event_id: Optional[int] = Field(default=None, gt=0) + decided_by: Optional[NonEmptyString] = None + decided_at: Optional[datetime] = None + + @model_validator(mode="after") + def require_decision_evidence(self) -> "ActionApproval": + if self.status in {"approved", "rejected"}: + if self.event_id is None or self.decided_by is None: + raise ValueError( + "approved or rejected actions require event_id and decided_by" + ) + elif any((self.event_id, self.decided_by, self.decided_at)): + raise ValueError( + "approval decision evidence is only valid for approved or rejected " + "actions" + ) + return self + + +ALL_ACTION_DEFINITIONS: Dict[str, AgentActionDefinition] = { + **RUNTIME_ACTIONS, + **WORKFLOW_ACTIONS, +} + + +class _ActionEnvelopeBase(_StrictBoundaryModel): + schema_version: Literal["workflow.action/v1"] = "workflow.action/v1" + action_id: NonEmptyString + kind: str + workflow_id: int = Field(gt=0) + requested_by: Literal["user", "agent", "system"] + idempotency_key: IdempotencyKey + correlation_id: CorrelationId + execution_owner: ExecutionOwner + policy: ActionPolicy + approval: ActionApproval + input_artifacts: List[ArtifactReference] = Field(default_factory=list) + expected_output_artifacts: List[ArtifactReference] = Field(default_factory=list) + preconditions: List[ActionPrecondition] = Field(default_factory=list) + created_at: Optional[datetime] = None + + @model_validator(mode="after") + def enforce_registry_policy(self) -> "_ActionEnvelopeBase": + definition = ALL_ACTION_DEFINITIONS.get(self.kind) + if definition is None: + raise ValueError(f"Unsupported action envelope kind: {self.kind}") + if self.policy.risk_level != definition.risk_level: + raise ValueError( + f"risk_level for {self.kind} must be {definition.risk_level}" + ) + if self.policy.requires_approval != definition.requires_approval: + raise ValueError( + f"requires_approval for {self.kind} must be " + f"{definition.requires_approval}" + ) + if self.execution_owner != definition.execution_owner: + raise ValueError( + f"execution_owner for {self.kind} must be " + f"{definition.execution_owner}" + ) + if definition.requires_approval: + if self.approval.status == "not_required": + raise ValueError(f"{self.kind} requires an approval status") + elif self.approval.status != "not_required": + raise ValueError(f"{self.kind} does not accept an approval decision") + return self + + +class ChooseProjectDataAction(_ActionEnvelopeBase): + kind: Literal["choose_project_data"] + + +class LoadVisualizationAction(_ActionEnvelopeBase): + kind: Literal["load_visualization"] + + +class StartInferenceAction(_ActionEnvelopeBase): + kind: Literal["start_inference"] + + +class StopInferenceAction(_ActionEnvelopeBase): + kind: Literal["stop_inference"] + operation_id: Optional[int] = Field(default=None, gt=0) + + +class StartProofreadingAction(_ActionEnvelopeBase): + kind: Literal["start_proofreading"] + + +class TrainingVolumeSubset(_StrictBoundaryModel): + selection_basis: Optional[NonEmptyString] = None + training_statuses: List[NonEmptyString] = Field(default_factory=list) + train_volume_count: Optional[int] = Field(default=None, ge=0) + target_volume_count: Optional[int] = Field(default=None, ge=0) + review_volume_count: Optional[int] = Field(default=None, ge=0) + manifest_path: Optional[NonEmptyString] = None + + +class StartTrainingAction(_ActionEnvelopeBase): + kind: Literal["start_training"] + autopick_parameters: Optional[bool] = None + parameter_mode: Optional[NonEmptyString] = None + volume_subset: Optional[TrainingVolumeSubset] = None + + +class StopTrainingAction(_ActionEnvelopeBase): + kind: Literal["stop_training"] + operation_id: Optional[int] = Field(default=None, gt=0) + + +class ComputeEvaluationAction(_ActionEnvelopeBase): + kind: Literal["compute_evaluation"] + name: Optional[NonEmptyString] = None + baseline_prediction_path: Optional[NonEmptyString] = None + candidate_prediction_path: Optional[NonEmptyString] = None + ground_truth_path: Optional[NonEmptyString] = None + baseline_run_id: Optional[int] = Field(default=None, gt=0) + candidate_run_id: Optional[int] = Field(default=None, gt=0) + model_version_id: Optional[int] = Field(default=None, gt=0) + + +class ExportBundleAction(_ActionEnvelopeBase): + kind: Literal["export_bundle"] + + +class ProposeRetrainingStageAction(_ActionEnvelopeBase): + kind: Literal["propose_retraining_stage"] + corrected_mask_path: Optional[NonEmptyString] = None + + +AgentActionEnvelope = Annotated[ + Union[ + ChooseProjectDataAction, + LoadVisualizationAction, + StartInferenceAction, + StopInferenceAction, + StartProofreadingAction, + StartTrainingAction, + StopTrainingAction, + ComputeEvaluationAction, + ExportBundleAction, + ProposeRetrainingStageAction, + ], + Field(discriminator="kind"), +] + + +class AgentActionError(_StrictBoundaryModel): + code: NonEmptyString + message: NonEmptyString + retryable: bool = False + details: Dict[str, Any] = Field(default_factory=dict) + + +class _ActionReceiptBase(_StrictBoundaryModel): + schema_version: Literal["workflow.action-receipt/v1"] = "workflow.action-receipt/v1" + receipt_id: NonEmptyString + action_id: NonEmptyString + kind: str + workflow_id: int = Field(gt=0) + idempotency_key: IdempotencyKey + correlation_id: CorrelationId + status: ReceiptStatus + operation_id: Optional[int] = Field(default=None, gt=0) + produced_artifacts: List[ArtifactReference] = Field(default_factory=list) + error: Optional[AgentActionError] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + + @model_validator(mode="after") + def validate_receipt_state(self) -> "_ActionReceiptBase": + if self.status == "failed" and self.error is None: + raise ValueError("failed action receipts require an error") + if self.status != "failed" and self.error is not None: + raise ValueError("error is only valid for failed action receipts") + if self.status in {"succeeded", "failed", "cancelled"}: + if self.completed_at is None: + raise ValueError("terminal action receipts require completed_at") + elif self.completed_at is not None: + raise ValueError("non-terminal action receipts cannot have completed_at") + if ( + self.started_at is not None + and self.completed_at is not None + and self.completed_at < self.started_at + ): + raise ValueError("completed_at cannot precede started_at") + + success_fields = { + "choose_project_data": "selected_artifacts", + "load_visualization": "viewer_url", + "start_inference": "run_id", + "stop_inference": "stopped", + "start_proofreading": "proofreading_session_id", + "start_training": "run_id", + "stop_training": "stopped", + "compute_evaluation": "evaluation_result_id", + "export_bundle": "bundle_artifact", + "propose_retraining_stage": "workflow_event_id", + } + success_field = success_fields.get(self.kind) + if self.status == "succeeded" and success_field: + if not getattr(self, success_field, None): + raise ValueError( + f"succeeded {self.kind} receipts require {success_field}" + ) + return self + + +class ChooseProjectDataReceipt(_ActionReceiptBase): + kind: Literal["choose_project_data"] + selected_artifacts: List[ArtifactReference] = Field(default_factory=list) + + +class LoadVisualizationReceipt(_ActionReceiptBase): + kind: Literal["load_visualization"] + viewer_url: Optional[NonEmptyString] = None + + +class StartInferenceReceipt(_ActionReceiptBase): + kind: Literal["start_inference"] + run_id: Optional[NonEmptyString] = None + + +class StopInferenceReceipt(_ActionReceiptBase): + kind: Literal["stop_inference"] + stopped: Optional[bool] = None + + +class StartProofreadingReceipt(_ActionReceiptBase): + kind: Literal["start_proofreading"] + proofreading_session_id: Optional[int] = Field(default=None, gt=0) + viewer_url: Optional[NonEmptyString] = None + + +class StartTrainingReceipt(_ActionReceiptBase): + kind: Literal["start_training"] + run_id: Optional[NonEmptyString] = None + + +class StopTrainingReceipt(_ActionReceiptBase): + kind: Literal["stop_training"] + stopped: Optional[bool] = None + + +class ComputeEvaluationReceipt(_ActionReceiptBase): + kind: Literal["compute_evaluation"] + evaluation_result_id: Optional[int] = Field(default=None, gt=0) + metrics: Dict[str, float] = Field(default_factory=dict) + + +class ExportBundleReceipt(_ActionReceiptBase): + kind: Literal["export_bundle"] + bundle_artifact: Optional[ArtifactReference] = None + + +class ProposeRetrainingStageReceipt(_ActionReceiptBase): + kind: Literal["propose_retraining_stage"] + workflow_event_id: Optional[int] = Field(default=None, gt=0) + staged_workflow_stage: Literal["retraining_staged"] = "retraining_staged" + + +AgentActionReceipt = Annotated[ + Union[ + ChooseProjectDataReceipt, + LoadVisualizationReceipt, + StartInferenceReceipt, + StopInferenceReceipt, + StartProofreadingReceipt, + StartTrainingReceipt, + StopTrainingReceipt, + ComputeEvaluationReceipt, + ExportBundleReceipt, + ProposeRetrainingStageReceipt, + ], + Field(discriminator="kind"), +] + + +_ACTION_ENVELOPE_ADAPTER = TypeAdapter(AgentActionEnvelope) +_ACTION_RECEIPT_ADAPTER = TypeAdapter(AgentActionReceipt) + + +def canonical_action_policy(kind: str) -> ActionPolicy: + definition = ALL_ACTION_DEFINITIONS.get(kind) + if definition is None: + raise ValueError(f"Unsupported action envelope kind: {kind}") + return ActionPolicy( + risk_level=definition.risk_level, + requires_approval=definition.requires_approval, + ) + + +def validate_action_envelope(payload: Any) -> AgentActionEnvelope: + return _ACTION_ENVELOPE_ADAPTER.validate_python(payload) + + +def validate_action_for_execution(payload: Any) -> AgentActionEnvelope: + envelope = validate_action_envelope(payload) + if envelope.policy.requires_approval and envelope.approval.status != "approved": + raise ValueError( + f"{envelope.kind} cannot execute without an approved action envelope" + ) + return envelope + + +def validate_action_receipt(payload: Any) -> AgentActionReceipt: + return _ACTION_RECEIPT_ADAPTER.validate_python(payload) + + +def validate_receipt_for_action( + action_payload: Any, receipt_payload: Any +) -> AgentActionReceipt: + action = validate_action_envelope(action_payload) + receipt = validate_action_receipt(receipt_payload) + matching_fields = ( + "action_id", + "kind", + "workflow_id", + "idempotency_key", + "correlation_id", + ) + mismatches = [ + field + for field in matching_fields + if getattr(action, field) != getattr(receipt, field) + ] + if mismatches: + raise ValueError( + "action receipt does not match its envelope: " + ", ".join(mismatches) + ) + return receipt + + +def dump_action_envelope_json(payload: Any) -> bytes: + return _ACTION_ENVELOPE_ADAPTER.dump_json(validate_action_envelope(payload)) + + +def load_action_envelope_json(payload: Union[str, bytes]) -> AgentActionEnvelope: + return _ACTION_ENVELOPE_ADAPTER.validate_json(payload) + + +def dump_action_receipt_json(payload: Any) -> bytes: + return _ACTION_RECEIPT_ADAPTER.dump_json(validate_action_receipt(payload)) + + +def load_action_receipt_json(payload: Union[str, bytes]) -> AgentActionReceipt: + return _ACTION_RECEIPT_ADAPTER.validate_json(payload) + + +def action_envelope_json_schema() -> Dict[str, Any]: + return _ACTION_ENVELOPE_ADAPTER.json_schema() + + +def action_receipt_json_schema() -> Dict[str, Any]: + return _ACTION_RECEIPT_ADAPTER.json_schema() + + def _validate_effects(client_effects: Dict[str, Any]) -> None: validator = getattr(ClientEffectsPayload, "model_validate", None) if validator is not None: diff --git a/server_api/workflows/operation_router.py b/server_api/workflows/operation_router.py index 6bb3242c..9097ec16 100644 --- a/server_api/workflows/operation_router.py +++ b/server_api/workflows/operation_router.py @@ -12,6 +12,11 @@ from server_api.auth.router import get_current_user from .db_models import WorkflowOperation +from .agent_actions import ( + AgentActionEnvelope, + action_envelope_json_schema, + validate_action_for_execution, +) from .operation_service import ( OPERATION_STATUSES, create_workflow_operation, @@ -104,6 +109,58 @@ def _owned_operation( ) +@router.get("/action-envelopes/schema") +def get_action_envelope_schema(): + return action_envelope_json_schema() + + +@router.post( + "/{workflow_id}/action-operations", + response_model=WorkflowOperationResponse, + status_code=202, +) +def stage_action_operation( + workflow_id: int, + body: AgentActionEnvelope, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + if body.workflow_id != workflow.id: + raise HTTPException( + status_code=409, + detail="Action envelope workflow_id does not match the request path", + ) + try: + envelope = validate_action_for_execution(body) + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + if envelope.execution_owner == "browser_navigation": + raise HTTPException( + status_code=409, + detail="Browser-owned actions cannot be staged as server operations", + ) + + input_payload = envelope.model_dump(mode="json", exclude_none=True) + operation = create_workflow_operation( + db, + workflow_id=workflow.id, + operation_type=f"agent_action:{envelope.kind}", + idempotency_key=envelope.idempotency_key, + correlation_id=envelope.correlation_id, + actor=envelope.requested_by, + input_payload=input_payload, + metadata={ + "action_id": envelope.action_id, + "action_schema_version": envelope.schema_version, + "execution_owner": envelope.execution_owner, + "risk_level": envelope.policy.risk_level, + }, + commit=True, + ) + return _response(operation) + + @router.post( "/{workflow_id}/operations", response_model=WorkflowOperationResponse, diff --git a/spikes/__init__.py b/spikes/__init__.py new file mode 100644 index 00000000..9d527b6f --- /dev/null +++ b/spikes/__init__.py @@ -0,0 +1 @@ +"""Isolated engineering spikes that are not imported by production services.""" diff --git a/spikes/dbos_operation/__init__.py b/spikes/dbos_operation/__init__.py new file mode 100644 index 00000000..a3f0413f --- /dev/null +++ b/spikes/dbos_operation/__init__.py @@ -0,0 +1 @@ +"""DBOS durable-operation evaluation spike.""" diff --git a/spikes/dbos_operation/runner.py b/spikes/dbos_operation/runner.py new file mode 100644 index 00000000..b96f1b22 --- /dev/null +++ b/spikes/dbos_operation/runner.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import argparse +import json +import pathlib +from typing import Any, Dict, Optional, Sequence + +from dbos import DBOS, DBOSClient, SetWorkflowID + +from .runtime import ( + APP_NAME, + APP_VERSION, + PROGRESS_EVENT, + QUEUE_NAME, + synthetic_operation, +) + +RESULT_PREFIX = "DBOS_SPIKE_RESULT=" + + +def sqlite_database_url(database_path: str) -> str: + path = pathlib.Path(database_path).expanduser().resolve() + path.parent.mkdir(parents=True, exist_ok=True) + return f"sqlite:///{path}" + + +def _workflow_status(database_url: str, workflow_id: str) -> Optional[Dict[str, Any]]: + client = DBOSClient(system_database_url=database_url) + try: + rows = client.list_workflows(workflow_ids=[workflow_id], limit=1) + if not rows: + return None + row = rows[0] + return { + "workflow_id": row.workflow_id, + "status": row.status, + "name": row.name, + "queue_name": row.queue_name, + "executor_id": row.executor_id, + "app_version": row.app_version, + "recovery_attempts": row.recovery_attempts, + "output": row.output, + "error": str(row.error) if row.error else None, + } + finally: + client.destroy() + + +def _progress(database_url: str, workflow_id: str) -> Optional[Dict[str, Any]]: + client = DBOSClient(system_database_url=database_url) + try: + value = client.get_event(workflow_id, PROGRESS_EVENT, timeout_seconds=0) + return value if isinstance(value, dict) else None + finally: + client.destroy() + + +def _launch(database_url: str, *, register_queue: bool) -> None: + DBOS( + config={ + "name": APP_NAME, + "application_version": APP_VERSION, + # A single-server replacement process must reuse the executor identity + # so startup recovery can claim that executor's interrupted workflows. + "executor_id": "pytc-dbos-operation-spike", + "system_database_url": database_url, + } + ) + DBOS.launch() + if register_queue: + DBOS.register_queue( + QUEUE_NAME, + concurrency=1, + worker_concurrency=1, + polling_interval_sec=0.05, + ) + + +def _wait_for_result(handle: Any, database_url: str) -> Dict[str, Any]: + try: + result = handle.get_result() + return {"result": result, "cancelled": False} + except Exception: + status = _workflow_status(database_url, handle.workflow_id) + if status and status["status"] == "CANCELLED": + return {"result": None, "cancelled": True} + raise + + +def execute(args: argparse.Namespace) -> Dict[str, Any]: + database_url = sqlite_database_url(args.database) + pathlib.Path(args.workspace).expanduser().resolve().mkdir( + parents=True, exist_ok=True + ) + _launch(database_url, register_queue=args.register_queue) + queue_name = QUEUE_NAME if args.register_queue else f"{QUEUE_NAME}-paused" + try: + with SetWorkflowID(args.workflow_id): + first = DBOS.enqueue_workflow( + queue_name, + synthetic_operation, + str(pathlib.Path(args.workspace).expanduser().resolve()), + args.workflow_id, + args.correlation_id, + args.steps, + args.step_runtime, + args.inter_step_delay, + ) + duplicate_workflow_id = None + if args.duplicate_submission: + with SetWorkflowID(args.workflow_id): + duplicate = DBOS.enqueue_workflow( + queue_name, + synthetic_operation, + str(pathlib.Path(args.workspace).expanduser().resolve()), + args.workflow_id, + args.correlation_id, + args.steps, + args.step_runtime, + args.inter_step_delay, + ) + duplicate_workflow_id = duplicate.workflow_id + + wait_result = ( + _wait_for_result(first, database_url) if args.wait else {"result": None} + ) + return { + "workflow_id": first.workflow_id, + "duplicate_workflow_id": duplicate_workflow_id, + "database_url": database_url, + "progress": _progress(database_url, args.workflow_id), + "workflow": _workflow_status(database_url, args.workflow_id), + **wait_result, + } + finally: + DBOS.destroy(workflow_completion_timeout_sec=1) + + +def recover(args: argparse.Namespace) -> Dict[str, Any]: + database_url = sqlite_database_url(args.database) + _launch(database_url, register_queue=True) + try: + handle = DBOS.retrieve_workflow(args.workflow_id) + wait_result = _wait_for_result(handle, database_url) + return { + "workflow_id": handle.workflow_id, + "database_url": database_url, + "progress": _progress(database_url, args.workflow_id), + "workflow": _workflow_status(database_url, args.workflow_id), + **wait_result, + } + finally: + DBOS.destroy(workflow_completion_timeout_sec=1) + + +def status(args: argparse.Namespace) -> Dict[str, Any]: + database_url = sqlite_database_url(args.database) + return { + "workflow_id": args.workflow_id, + "database_url": database_url, + "progress": _progress(database_url, args.workflow_id), + "workflow": _workflow_status(database_url, args.workflow_id), + } + + +def cancel(args: argparse.Namespace) -> Dict[str, Any]: + database_url = sqlite_database_url(args.database) + client = DBOSClient(system_database_url=database_url) + try: + client.cancel_workflow(args.workflow_id) + finally: + client.destroy() + return status(args) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run the isolated DBOS durable-operation spike." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + execute_parser = subparsers.add_parser( + "execute", help="Idempotently enqueue a synthetic operation." + ) + execute_parser.add_argument("--database", required=True) + execute_parser.add_argument("--workspace", required=True) + execute_parser.add_argument("--workflow-id", required=True) + execute_parser.add_argument("--correlation-id", default="dbos-spike") + execute_parser.add_argument("--steps", type=int, default=3) + execute_parser.add_argument("--step-runtime", type=float, default=0.01) + execute_parser.add_argument("--inter-step-delay", type=float, default=0.05) + execute_parser.add_argument( + "--register-queue", action=argparse.BooleanOptionalAction, default=True + ) + execute_parser.add_argument( + "--duplicate-submission", action="store_true", default=False + ) + execute_parser.add_argument( + "--wait", action=argparse.BooleanOptionalAction, default=True + ) + execute_parser.set_defaults(handler=execute) + + recover_parser = subparsers.add_parser( + "recover", help="Launch a new executor and wait for recovered work." + ) + recover_parser.add_argument("--database", required=True) + recover_parser.add_argument("--workflow-id", required=True) + recover_parser.set_defaults(handler=recover) + + status_parser = subparsers.add_parser( + "status", help="Read durable status and progress without launching a worker." + ) + status_parser.add_argument("--database", required=True) + status_parser.add_argument("--workflow-id", required=True) + status_parser.set_defaults(handler=status) + + cancel_parser = subparsers.add_parser( + "cancel", help="Cancel queued or running work through DBOSClient." + ) + cancel_parser.add_argument("--database", required=True) + cancel_parser.add_argument("--workflow-id", required=True) + cancel_parser.set_defaults(handler=cancel) + return parser + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = build_parser().parse_args(argv) + result = args.handler(args) + print(RESULT_PREFIX + json.dumps(result, sort_keys=True), flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/spikes/dbos_operation/runtime.py b/spikes/dbos_operation/runtime.py new file mode 100644 index 00000000..42130c6d --- /dev/null +++ b/spikes/dbos_operation/runtime.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import hashlib +import json +import os +import pathlib +import time +from typing import Any, Dict + +from dbos import DBOS + +APP_NAME = "pytc-dbos-operation-spike" +APP_VERSION = "pytc-dbos-operation-spike-v1" +QUEUE_NAME = "synthetic-operations" +PROGRESS_EVENT = "operation_progress" + + +def workflow_storage_key(workflow_id: str) -> str: + return hashlib.sha256(workflow_id.encode("utf-8")).hexdigest()[:20] + + +def marker_directory(workspace: str, workflow_id: str) -> pathlib.Path: + return pathlib.Path(workspace) / "markers" / workflow_storage_key(workflow_id) + + +def _append_json_line(path: pathlib.Path, payload: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + encoded = json.dumps(payload, sort_keys=True) + "\n" + with path.open("a", encoding="utf-8") as stream: + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + + +@DBOS.step(name="pytc.synthetic_operation_step.v1") +def execute_synthetic_step( + workspace: str, + workflow_id: str, + step_index: int, + step_runtime_seconds: float, +) -> Dict[str, Any]: + """Perform an idempotent external effect representative of a compute chunk.""" + + markers = marker_directory(workspace, workflow_id) + markers.mkdir(parents=True, exist_ok=True) + marker = markers / f"step-{step_index:04d}.json" + attempt = { + "workflow_id": workflow_id, + "step_index": step_index, + "pid": os.getpid(), + "attempted_at": time.time(), + } + try: + with marker.open("x", encoding="utf-8") as stream: + stream.write(json.dumps(attempt, sort_keys=True)) + stream.flush() + os.fsync(stream.fileno()) + effect = "created" + except FileExistsError: + effect = "already_exists" + _append_json_line(markers / "duplicate-attempts.jsonl", attempt) + + if step_runtime_seconds > 0: + time.sleep(step_runtime_seconds) + return { + "step_index": step_index, + "effect": effect, + "marker": str(marker), + } + + +@DBOS.workflow(name="pytc.synthetic_operation.v1") +def synthetic_operation( + workspace: str, + workflow_id: str, + correlation_id: str, + total_steps: int, + step_runtime_seconds: float, + inter_step_delay_seconds: float, +) -> Dict[str, Any]: + if total_steps < 1: + raise ValueError("total_steps must be positive") + + progress: Dict[str, Any] = { + "status": "running", + "workflow_id": workflow_id, + "correlation_id": correlation_id, + "completed_steps": 0, + "total_steps": total_steps, + "progress": 0.0, + } + DBOS.set_event(PROGRESS_EVENT, progress) + + step_results = [] + for step_index in range(total_steps): + step_results.append( + execute_synthetic_step( + workspace, + workflow_id, + step_index, + step_runtime_seconds, + ) + ) + completed_steps = step_index + 1 + progress = { + **progress, + "completed_steps": completed_steps, + "progress": completed_steps / total_steps, + } + DBOS.set_event(PROGRESS_EVENT, progress) + if completed_steps < total_steps and inter_step_delay_seconds > 0: + DBOS.sleep(inter_step_delay_seconds) + + result = { + "status": "succeeded", + "workflow_id": workflow_id, + "correlation_id": correlation_id, + "completed_steps": total_steps, + "total_steps": total_steps, + "progress": 1.0, + "step_results": step_results, + } + DBOS.set_event(PROGRESS_EVENT, result) + return result diff --git a/tests/test_agent_action_envelopes.py b/tests/test_agent_action_envelopes.py new file mode 100644 index 00000000..81eb008b --- /dev/null +++ b/tests/test_agent_action_envelopes.py @@ -0,0 +1,307 @@ +from datetime import datetime, timezone + +import pytest +from pydantic import ValidationError + +from server_api.workflows.agent_actions import ( + ALL_ACTION_DEFINITIONS, + action_envelope_json_schema, + action_receipt_json_schema, + canonical_action_policy, + dump_action_envelope_json, + dump_action_receipt_json, + load_action_envelope_json, + load_action_receipt_json, + validate_action_for_execution, + validate_action_envelope, + validate_action_receipt, + validate_receipt_for_action, +) + + +def _envelope(kind="start_training", **overrides): + definition = ALL_ACTION_DEFINITIONS[kind] + payload = { + "action_id": f"action-{kind}", + "kind": kind, + "workflow_id": 17, + "requested_by": "agent", + "idempotency_key": f"workflow:17:{kind}:v1", + "correlation_id": "request-4c0d", + "execution_owner": definition.execution_owner, + "policy": { + "risk_level": definition.risk_level, + "requires_approval": definition.requires_approval, + }, + "approval": { + "status": "pending" if definition.requires_approval else "not_required" + }, + "input_artifacts": [ + { + "artifact_id": 9, + "artifact_type": "volume", + "role": "training_image", + "uri": "zarr:///datasets/mito25/image", + "checksum": "sha256:image-v1", + } + ], + "expected_output_artifacts": [ + { + "logical_name": "candidate-checkpoint", + "artifact_type": "model_checkpoint", + "role": "candidate", + } + ], + "preconditions": [ + { + "kind": "workflow_stage", + "allowed_stages": ["proofreading", "retraining_staged"], + }, + { + "kind": "artifact_available", + "artifact": {"artifact_id": 9}, + }, + ], + **overrides, + } + return payload + + +def _receipt(kind="start_training", **overrides): + action_fields = { + "choose_project_data": { + "selected_artifacts": [{"artifact_id": 2}], + }, + "load_visualization": {"viewer_url": "https://viewer.test/v/1"}, + "start_inference": {"run_id": "inference-1"}, + "stop_inference": {"stopped": True}, + "start_proofreading": {"proofreading_session_id": 8}, + "start_training": {"run_id": "training-1"}, + "stop_training": {"stopped": True}, + "compute_evaluation": {"evaluation_result_id": 11}, + "export_bundle": { + "bundle_artifact": { + "artifact_id": 12, + "artifact_type": "evidence_bundle", + } + }, + "propose_retraining_stage": {"workflow_event_id": 13}, + } + return { + "receipt_id": f"receipt-{kind}", + "action_id": f"action-{kind}", + "kind": kind, + "workflow_id": 17, + "idempotency_key": f"workflow:17:{kind}:v1", + "correlation_id": "request-4c0d", + "status": "succeeded", + "operation_id": 31, + "produced_artifacts": [{"logical_name": "output"}], + "started_at": datetime(2026, 7, 21, 13, 0, tzinfo=timezone.utc), + "completed_at": datetime(2026, 7, 21, 13, 5, tzinfo=timezone.utc), + **action_fields[kind], + **overrides, + } + + +@pytest.mark.parametrize("kind", sorted(ALL_ACTION_DEFINITIONS)) +def test_every_registered_core_action_has_a_discriminated_envelope(kind): + envelope = validate_action_envelope(_envelope(kind)) + + assert envelope.kind == kind + assert envelope.policy == canonical_action_policy(kind) + assert envelope.execution_owner == ALL_ACTION_DEFINITIONS[kind].execution_owner + + +def test_action_envelope_schema_exposes_kind_discriminator_and_core_mappings(): + schema = action_envelope_json_schema() + + assert schema["discriminator"]["propertyName"] == "kind" + assert set(schema["discriminator"]["mapping"]) == set(ALL_ACTION_DEFINITIONS) + assert len(schema["oneOf"]) == len(ALL_ACTION_DEFINITIONS) + + +def test_action_receipt_schema_matches_action_discriminator(): + schema = action_receipt_json_schema() + + assert schema["discriminator"]["propertyName"] == "kind" + assert set(schema["discriminator"]["mapping"]) == set(ALL_ACTION_DEFINITIONS) + + +def test_action_envelope_json_round_trip_preserves_typed_boundary_fields(): + payload = _envelope( + "start_training", + approval={ + "status": "approved", + "event_id": 21, + "decided_by": "user:3", + "decided_at": datetime(2026, 7, 21, 12, 59, tzinfo=timezone.utc), + }, + autopick_parameters=True, + parameter_mode="agent_default", + volume_subset={ + "selection_basis": "project_progress", + "training_statuses": ["ground_truth"], + "train_volume_count": 2, + "target_volume_count": 1, + "review_volume_count": 0, + "manifest_path": "/datasets/mito25/volume_subset_manifest.json", + }, + ) + + original = validate_action_envelope(payload) + restored = load_action_envelope_json(dump_action_envelope_json(original)) + + assert type(restored) is type(original) + assert restored.model_dump() == original.model_dump() + assert restored.input_artifacts[0].artifact_id == 9 + assert restored.preconditions[1].kind == "artifact_available" + + +def test_action_receipt_json_round_trip_preserves_typed_result(): + original = validate_action_receipt( + _receipt( + "compute_evaluation", + metrics={"iou": 0.84, "dice": 0.91}, + ) + ) + restored = load_action_receipt_json(dump_action_receipt_json(original)) + + assert type(restored) is type(original) + assert restored.model_dump() == original.model_dump() + assert restored.evaluation_result_id == 11 + assert restored.metrics["dice"] == pytest.approx(0.91) + + +def test_envelope_rejects_policy_or_execution_owner_spoofing(): + with pytest.raises(ValidationError, match="risk_level for start_training"): + validate_action_envelope( + _envelope( + policy={ + "risk_level": "read_only", + "requires_approval": True, + } + ) + ) + + with pytest.raises(ValidationError, match="execution_owner for start_training"): + validate_action_envelope(_envelope(execution_owner="browser_navigation")) + + +def test_envelope_rejects_invalid_approval_evidence_and_status(): + with pytest.raises(ValidationError, match="require event_id and decided_by"): + validate_action_envelope( + _envelope(approval={"status": "approved", "event_id": 9}) + ) + + with pytest.raises(ValidationError, match="does not accept an approval decision"): + validate_action_envelope( + _envelope("choose_project_data", approval={"status": "pending"}) + ) + + +def test_execution_gate_requires_approved_envelope_for_risky_actions(): + with pytest.raises(ValueError, match="cannot execute without an approved"): + validate_action_for_execution(_envelope("start_training")) + + approved = _envelope( + "start_training", + approval={ + "status": "approved", + "event_id": 23, + "decided_by": "user:4", + }, + ) + assert validate_action_for_execution(approved).kind == "start_training" + assert validate_action_for_execution(_envelope("choose_project_data")).kind == ( + "choose_project_data" + ) + + +def test_envelope_rejects_untyped_artifacts_preconditions_and_action_fields(): + with pytest.raises(ValidationError, match="artifact reference requires"): + validate_action_envelope(_envelope(input_artifacts=[{"role": "image"}])) + + with pytest.raises(ValidationError, match="precondition"): + validate_action_envelope( + _envelope(preconditions=[{"kind": "shell_exit_code", "value": 0}]) + ) + + with pytest.raises(ValidationError): + validate_action_envelope(_envelope(shell_command="rm -rf /")) + + with pytest.raises(ValidationError): + validate_action_envelope(_envelope(volume_subset={"shell_command": "rm -rf /"})) + + +def test_failed_receipt_requires_typed_error_and_terminal_timestamp(): + failed_payload = _receipt( + "start_inference", + status="failed", + run_id=None, + error={ + "code": "worker_unavailable", + "message": "The inference worker did not respond.", + "retryable": True, + "details": {"status_code": 503}, + }, + ) + receipt = validate_action_receipt(failed_payload) + + assert receipt.error.code == "worker_unavailable" + assert receipt.error.retryable is True + + with pytest.raises(ValidationError, match="require an error"): + validate_action_receipt({**failed_payload, "error": None}) + + with pytest.raises(ValidationError, match="require completed_at"): + validate_action_receipt({**failed_payload, "completed_at": None}) + + with pytest.raises(ValidationError, match="cannot precede started_at"): + validate_action_receipt( + { + **failed_payload, + "completed_at": datetime(2026, 7, 21, 12, 0, tzinfo=timezone.utc), + } + ) + + +def test_receipt_must_match_action_identity_and_correlation(): + action = _envelope( + "start_training", + approval={ + "status": "approved", + "event_id": 23, + "decided_by": "user:4", + }, + ) + receipt = _receipt("start_training") + + assert validate_receipt_for_action(action, receipt).run_id == "training-1" + with pytest.raises(ValueError, match="correlation_id"): + validate_receipt_for_action( + action, + {**receipt, "correlation_id": "different-request"}, + ) + + +@pytest.mark.parametrize("kind", sorted(ALL_ACTION_DEFINITIONS)) +def test_succeeded_receipts_require_action_specific_result(kind): + receipt = validate_action_receipt(_receipt(kind)) + assert receipt.kind == kind + + result_field = { + "choose_project_data": "selected_artifacts", + "load_visualization": "viewer_url", + "start_inference": "run_id", + "stop_inference": "stopped", + "start_proofreading": "proofreading_session_id", + "start_training": "run_id", + "stop_training": "stopped", + "compute_evaluation": "evaluation_result_id", + "export_bundle": "bundle_artifact", + "propose_retraining_stage": "workflow_event_id", + }[kind] + missing_value = [] if result_field == "selected_artifacts" else None + with pytest.raises(ValidationError, match=f"require {result_field}"): + validate_action_receipt(_receipt(kind, **{result_field: missing_value})) diff --git a/tests/test_browser_synthetic_core_smoke.py b/tests/test_browser_synthetic_core_smoke.py new file mode 100644 index 00000000..051d6482 --- /dev/null +++ b/tests/test_browser_synthetic_core_smoke.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json + +from scripts import browser_synthetic_core_smoke as smoke + + +def test_parser_defaults_target_local_synthetic_app() -> None: + args = smoke._build_arg_parser().parse_args([]) + + assert args.base_url == smoke.DEFAULT_BASE_URL + assert args.report == smoke.DEFAULT_REPORT + assert args.viewport == "1280x900" + assert not args.skip_reload + + +def test_main_records_a_successful_smoke(monkeypatch, tmp_path) -> None: + monkeypatch.setattr( + smoke, + "run_smoke", + lambda **kwargs: { + "passed": True, + "base_url": kwargs["base_url"], + "checks": ["bounded file picker navigated"], + }, + ) + report_path = tmp_path / "report.json" + + result = smoke.main(["--skip-reload", "--report", str(report_path)]) + + assert result == 0 + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["passed"] is True + assert report["checks"] == ["bounded file picker navigated"] + + +def test_main_records_browser_failures(monkeypatch, tmp_path) -> None: + monkeypatch.setattr( + smoke, + "run_smoke", + lambda **_kwargs: (_ for _ in ()).throw(AssertionError("picker failed")), + ) + report_path = tmp_path / "failed.json" + + result = smoke.main(["--report", str(report_path)]) + + assert result == 1 + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["passed"] is False + assert report["error"] == "picker failed" diff --git a/tests/test_dbos_operation_spike.py b/tests/test_dbos_operation_spike.py new file mode 100644 index 00000000..4ef0bf04 --- /dev/null +++ b/tests/test_dbos_operation_spike.py @@ -0,0 +1,268 @@ +import hashlib +import json +import pathlib +import subprocess +import sys +import time + +import pytest + +dbos = pytest.importorskip("dbos") +from dbos import DBOSClient + +RESULT_PREFIX = "DBOS_SPIKE_RESULT=" +RUNNER_MODULE = "spikes.dbos_operation.runner" +PROGRESS_EVENT = "operation_progress" + + +def _command(*args): + return [sys.executable, "-m", RUNNER_MODULE, *map(str, args)] + + +def _parse_result(output): + for line in reversed(output.splitlines()): + if line.startswith(RESULT_PREFIX): + return json.loads(line[len(RESULT_PREFIX) :]) + raise AssertionError(f"Spike runner did not emit a result: {output[-2000:]}") + + +def _run(*args, timeout=30): + completed = subprocess.run( + _command(*args), + check=True, + capture_output=True, + text=True, + timeout=timeout, + ) + return _parse_result(completed.stdout) + + +def _database_url(path): + return f"sqlite:///{path.resolve()}" + + +def _client_state(database, workflow_id): + client = DBOSClient(system_database_url=_database_url(database)) + try: + rows = client.list_workflows(workflow_ids=[workflow_id], limit=1) + status = rows[0].status if rows else None + progress = client.get_event( + workflow_id, + PROGRESS_EVENT, + timeout_seconds=0, + ) + return status, progress + finally: + client.destroy() + + +def _wait_for(database, workflow_id, predicate, timeout=15): + deadline = time.monotonic() + timeout + last_state = (None, None) + while time.monotonic() < deadline: + try: + last_state = _client_state(database, workflow_id) + except Exception: + time.sleep(0.05) + continue + if predicate(*last_state): + return last_state + time.sleep(0.05) + raise AssertionError(f"Timed out waiting for DBOS state; last state={last_state}") + + +def _markers(workspace, workflow_id): + storage_key = hashlib.sha256(workflow_id.encode("utf-8")).hexdigest()[:20] + return workspace / "markers" / storage_key + + +def test_idempotent_submission_executes_external_effects_once(tmp_path): + database = tmp_path / "dbos.sqlite" + workspace = tmp_path / "workspace" + workflow_id = "operation-idempotency-1" + + result = _run( + "execute", + "--database", + database, + "--workspace", + workspace, + "--workflow-id", + workflow_id, + "--correlation-id", + "request-idempotency-1", + "--steps", + 3, + "--duplicate-submission", + ) + + assert result["workflow_id"] == workflow_id + assert result["duplicate_workflow_id"] == workflow_id + assert result["workflow"]["status"] == "SUCCESS" + assert result["progress"]["progress"] == 1.0 + marker_dir = _markers(workspace, workflow_id) + assert len(list(marker_dir.glob("step-*.json"))) == 3 + assert not (marker_dir / "duplicate-attempts.jsonl").exists() + + +def test_queued_cancellation_prevents_execution(tmp_path): + database = tmp_path / "dbos.sqlite" + workspace = tmp_path / "workspace" + workflow_id = "operation-cancel-queued" + + submitted = _run( + "execute", + "--database", + database, + "--workspace", + workspace, + "--workflow-id", + workflow_id, + "--steps", + 3, + "--no-register-queue", + "--no-wait", + ) + assert submitted["workflow"]["status"] == "ENQUEUED" + + cancelled = _run( + "cancel", + "--database", + database, + "--workflow-id", + workflow_id, + ) + assert cancelled["workflow"]["status"] == "CANCELLED" + assert not _markers(workspace, workflow_id).exists() + + +def test_running_cancellation_stops_at_durable_boundary(tmp_path): + database = tmp_path / "dbos.sqlite" + workspace = tmp_path / "workspace" + workflow_id = "operation-cancel-running" + process = subprocess.Popen( + _command( + "execute", + "--database", + database, + "--workspace", + workspace, + "--workflow-id", + workflow_id, + "--steps", + 20, + "--step-runtime", + 0.01, + "--inter-step-delay", + 0.5, + ), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + try: + _wait_for( + database, + workflow_id, + lambda status, progress: ( + status == "PENDING" + and isinstance(progress, dict) + and progress.get("completed_steps", 0) >= 1 + ), + ) + client = DBOSClient(system_database_url=_database_url(database)) + try: + client.cancel_workflow(workflow_id) + finally: + client.destroy() + _wait_for( + database, + workflow_id, + lambda status, _progress: status == "CANCELLED", + ) + output, _ = process.communicate(timeout=15) + result = _parse_result(output) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + assert result["cancelled"] is True + marker_dir = _markers(workspace, workflow_id) + completed_markers = list(marker_dir.glob("step-*.json")) + assert 1 <= len(completed_markers) < 20 + assert not (marker_dir / "duplicate-attempts.jsonl").exists() + + +def test_process_kill_recovers_from_last_completed_step(tmp_path): + database = tmp_path / "dbos.sqlite" + workspace = tmp_path / "workspace" + workflow_id = "operation-restart-recovery" + process = subprocess.Popen( + _command( + "execute", + "--database", + database, + "--workspace", + workspace, + "--workflow-id", + workflow_id, + "--correlation-id", + "request-recovery-1", + "--steps", + 3, + "--step-runtime", + 0.01, + "--inter-step-delay", + 3, + ), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + status_before, progress_before = _wait_for( + database, + workflow_id, + lambda status, progress: ( + status == "PENDING" + and isinstance(progress, dict) + and progress.get("completed_steps") == 1 + ), + ) + assert status_before == "PENDING" + assert progress_before["correlation_id"] == "request-recovery-1" + process.kill() + process.wait(timeout=5) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + status_after_kill, persisted_progress = _wait_for( + database, + workflow_id, + lambda status, progress: ( + status == "PENDING" + and isinstance(progress, dict) + and progress.get("completed_steps") == 1 + ), + ) + assert status_after_kill == "PENDING" + assert persisted_progress["progress"] == pytest.approx(1 / 3) + + recovered = _run( + "recover", + "--database", + database, + "--workflow-id", + workflow_id, + timeout=20, + ) + assert recovered["workflow"]["status"] == "SUCCESS" + assert recovered["workflow"]["recovery_attempts"] >= 2 + assert recovered["progress"]["completed_steps"] == 3 + assert recovered["progress"]["correlation_id"] == "request-recovery-1" + + marker_dir = _markers(workspace, workflow_id) + assert len(list(marker_dir.glob("step-*.json"))) == 3 + assert not (marker_dir / "duplicate-attempts.jsonl").exists() diff --git a/tests/test_error_contract.py b/tests/test_error_contract.py index eeaa54b3..3e080282 100644 --- a/tests/test_error_contract.py +++ b/tests/test_error_contract.py @@ -4,7 +4,7 @@ from fastapi.testclient import TestClient from pydantic import BaseModel -from server_api.errors import install_error_handlers +from server_api.errors import PROBLEM_JSON_MEDIA_TYPE, install_error_handlers class ExamplePayload(BaseModel): @@ -23,6 +23,16 @@ def missing(): def validate(payload: ExamplePayload): return payload + @app.get("/upstream") + def upstream(): + raise HTTPException( + status_code=503, + detail={ + "user_message": "The model worker is offline", + "error": "ConnectionError", + }, + ) + @app.get("/failure") def failure(): raise RuntimeError("private implementation detail") @@ -39,7 +49,13 @@ def test_http_error_preserves_detail_and_adds_recovery_contract(self): self.assertEqual(response.status_code, 404) self.assertEqual(response.headers["x-request-id"], "req-123") + self.assertEqual(response.headers["content-type"], "application/json") + self.assertIn("Accept", response.headers["vary"]) self.assertEqual(response.json()["detail"], "Dataset was not found") + self.assertEqual(response.json()["status"], 404) + self.assertEqual(response.json()["type"], "https://seg.bio/problems/not-found") + self.assertEqual(response.json()["title"], "Resource not found") + self.assertEqual(response.json()["instance"], "/missing#request-req-123") self.assertEqual( response.json()["error"], { @@ -63,6 +79,74 @@ def test_validation_error_includes_field_diagnostics(self): self.assertEqual(body["detail"], body["error"]["validation_errors"]) self.assertTrue(body["error"]["request_id"]) + def test_problem_json_representation_is_rfc_9457_compatible(self): + response = self.client.get( + "/upstream", + headers={ + "accept": PROBLEM_JSON_MEDIA_TYPE, + "x-request-id": "problem-503", + }, + ) + + self.assertEqual(response.status_code, 503) + self.assertEqual(response.headers["content-type"], PROBLEM_JSON_MEDIA_TYPE) + body = response.json() + self.assertEqual( + { + "type": body["type"], + "title": body["title"], + "status": body["status"], + "detail": body["detail"], + "instance": body["instance"], + }, + { + "type": "https://seg.bio/problems/service-unavailable", + "title": "Service temporarily unavailable", + "status": 503, + "detail": "The model worker is offline", + "instance": "/upstream#request-problem-503", + }, + ) + self.assertEqual( + body["legacy_detail"], + { + "user_message": "The model worker is offline", + "error": "ConnectionError", + }, + ) + self.assertEqual(body["error"]["code"], "service_unavailable") + + def test_problem_json_validation_detail_is_string_with_extensions(self): + response = self.client.post( + "/validate", + json={"count": "invalid"}, + headers={"accept": PROBLEM_JSON_MEDIA_TYPE}, + ) + + self.assertEqual(response.status_code, 422) + body = response.json() + self.assertIsInstance(body["detail"], str) + self.assertEqual(body["legacy_detail"], body["validation_errors"]) + self.assertEqual(body["validation_errors"], body["error"]["validation_errors"]) + + def test_default_representation_preserves_non_string_detail(self): + response = self.client.get("/upstream") + + self.assertEqual( + response.json()["detail"], + { + "user_message": "The model worker is offline", + "error": "ConnectionError", + }, + ) + + def test_problem_json_must_be_explicitly_acceptable(self): + response = self.client.get( + "/missing", headers={"accept": "application/problem+json;q=0, */*"} + ) + + self.assertEqual(response.headers["content-type"], "application/json") + def test_invalid_caller_request_id_is_not_reflected(self): response = self.client.get( "/missing", headers={"x-request-id": "invalid request id"} diff --git a/tests/test_file_workspace_routes.py b/tests/test_file_workspace_routes.py index 735d7c4c..97c1b937 100644 --- a/tests/test_file_workspace_routes.py +++ b/tests/test_file_workspace_routes.py @@ -3,6 +3,8 @@ import tempfile import unittest import json +from types import SimpleNamespace +from unittest import mock from fastapi.testclient import TestClient from sqlalchemy import create_engine @@ -133,6 +135,45 @@ def test_missing_parent_file_listing_returns_404(self): self.assertEqual(response.status_code, 404) self.assertIn("no longer mounted", response.json()["detail"]) + def test_file_listing_pagination_is_opt_in_and_filters_before_count(self): + self._create_file(name="nested", is_folder=True) + self._create_file(name="a-image.h5") + self._create_file(name="b-mask.tif") + self._create_file(name="notes.txt") + self._create_file(name=".DS_Store") + self._create_file(name="workflow_preference.json") + + legacy_response = self.client.get("/files") + self.assertEqual(legacy_response.status_code, 200) + self.assertIsInstance(legacy_response.json(), list) + self.assertEqual( + [item["name"] for item in legacy_response.json()], + ["nested", "a-image.h5", "b-mask.tif", "notes.txt"], + ) + + first_page = self.client.get( + "/files", + params={"offset": 0, "limit": 2, "volume_only": True}, + ) + self.assertEqual(first_page.status_code, 200) + payload = first_page.json() + self.assertEqual( + [item["name"] for item in payload["items"]], ["nested", "a-image.h5"] + ) + self.assertEqual(payload["total"], 3) + self.assertEqual(payload["offset"], 0) + self.assertEqual(payload["limit"], 2) + self.assertTrue(payload["has_more"]) + + second_page = self.client.get( + "/files", + params={"offset": 2, "limit": 2, "volume_only": True}, + ).json() + self.assertEqual( + [item["name"] for item in second_page["items"]], ["b-mask.tif"] + ) + self.assertFalse(second_page["has_more"]) + def test_file_preview_reads_one_hdf5_volume_slice(self): try: import h5py @@ -159,6 +200,47 @@ def test_file_preview_reads_one_hdf5_volume_slice(self): self.assertEqual(response.headers["content-type"], "image/png") self.assertTrue(response.content.startswith(b"\x89PNG\r\n\x1a\n")) + def test_file_preview_requests_bounded_middle_volume_region(self): + try: + import numpy as np + except Exception: + self.skipTest("NumPy preview dependency is not installed") + + volume_path = pathlib.Path(self.temp_dir.name) / "bounded.tif" + volume_path.write_bytes(b"placeholder") + file_id = self._create_file( + name="bounded.tif", + physical_path=str(volume_path), + file_type="image/tiff", + ) + requested_crops = [] + + class FakeStore: + metadata = SimpleNamespace(shape=(20, 2000, 3000)) + + def __enter__(self): + return self + + def __exit__(self, *_exc_info): + return None + + def read(self, crop): + requested_crops.append(crop) + return np.zeros((1, 667, 1000), dtype=np.uint16) + + with mock.patch( + "server_api.auth.router.open_volume_store", + return_value=FakeStore(), + ): + response = self.client.get(f"/files/preview/{file_id}") + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(requested_crops), 1) + crop = requested_crops[0] + self.assertEqual(crop[0], slice(10, 11)) + self.assertEqual(crop[1], slice(None, None, 3)) + self.assertEqual(crop[2], slice(None, None, 3)) + def test_project_context_profile_is_saved_in_hidden_project_file(self): project_root = pathlib.Path(self.temp_dir.name) / "profile-project" project_root.mkdir() diff --git a/tests/test_workflow_operations.py b/tests/test_workflow_operations.py index 1adaaf39..4993af52 100644 --- a/tests/test_workflow_operations.py +++ b/tests/test_workflow_operations.py @@ -60,6 +60,104 @@ def _create_operation(self, key="train:dataset-a:v1", **overrides): json=body, ) + def _training_action(self, **overrides): + body = { + "action_id": "action-training-1", + "kind": "start_training", + "workflow_id": self.workflow_id, + "requested_by": "agent", + "idempotency_key": "action:start-training:1", + "correlation_id": "conversation-1", + "execution_owner": "server_runtime", + "policy": { + "risk_level": "runs_job", + "requires_approval": True, + "approval_reason": "Training consumes compute resources.", + }, + "approval": { + "status": "approved", + "event_id": 1, + "decided_by": "test-user", + }, + "input_artifacts": [ + { + "logical_name": "synthetic-training-config", + "role": "config", + "path": "configs/Synthetic-Core-Loop-BC.yaml", + } + ], + "preconditions": [ + { + "kind": "workflow_stage", + "allowed_stages": ["training_ready"], + } + ], + "autopick_parameters": True, + } + body.update(overrides) + return body + + def test_approved_action_envelope_stages_one_idempotent_operation(self): + url = f"/api/workflows/{self.workflow_id}/action-operations" + body = self._training_action() + + first_response = self.client.post(url, json=body) + second_response = self.client.post(url, json=body) + + self.assertEqual(first_response.status_code, 202) + self.assertEqual(second_response.status_code, 202) + first = first_response.json() + self.assertEqual(second_response.json()["id"], first["id"]) + self.assertEqual(first["status"], "queued") + self.assertEqual(first["operation_type"], "agent_action:start_training") + self.assertEqual(first["input"]["kind"], "start_training") + self.assertEqual(first["metadata"]["action_id"], "action-training-1") + self.assertEqual(first["correlation_id"], "conversation-1") + + def test_action_operation_rejects_unapproved_and_wrong_workflow_envelopes(self): + url = f"/api/workflows/{self.workflow_id}/action-operations" + unapproved = self._training_action( + approval={"status": "pending"}, + idempotency_key="action:start-training:pending", + ) + unapproved_response = self.client.post(url, json=unapproved) + self.assertEqual(unapproved_response.status_code, 409) + self.assertIn("approved", unapproved_response.json()["detail"]) + + wrong_workflow = self._training_action( + workflow_id=self.workflow_id + 1, + idempotency_key="action:start-training:wrong-workflow", + ) + wrong_response = self.client.post(url, json=wrong_workflow) + self.assertEqual(wrong_response.status_code, 409) + self.assertIn("workflow_id", wrong_response.json()["detail"]) + + def test_action_operation_rejects_browser_owned_actions(self): + url = f"/api/workflows/{self.workflow_id}/action-operations" + body = { + **self._training_action(), + "action_id": "action-choose-project-1", + "kind": "choose_project_data", + "idempotency_key": "action:choose-project:1", + "execution_owner": "browser_navigation", + "policy": { + "risk_level": "prefills_form", + "requires_approval": False, + }, + "approval": {"status": "not_required"}, + } + body.pop("autopick_parameters") + response = self.client.post(url, json=body) + self.assertEqual(response.status_code, 409) + self.assertIn("Browser-owned", response.json()["detail"]) + + def test_action_envelope_schema_is_exposed_with_a_discriminator(self): + response = self.client.get("/api/workflows/action-envelopes/schema") + self.assertEqual(response.status_code, 200) + schema = response.json() + self.assertEqual(schema["discriminator"]["propertyName"], "kind") + self.assertGreaterEqual(len(schema["oneOf"]), 10) + def test_operation_lifecycle_is_persisted_and_queryable(self): created_response = self._create_operation() self.assertEqual(created_response.status_code, 200) diff --git a/uv.lock b/uv.lock index a2f7d15f..aec60937 100644 --- a/uv.lock +++ b/uv.lock @@ -93,6 +93,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "aiosqlite" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/8a/64761f4005f17809769d23e518d915db74e6310474e733e3593cfc854ef1/aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650", size = 14821, upload-time = "2025-12-23T19:25:43.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405, upload-time = "2025-12-23T19:25:42.139Z" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -685,6 +703,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, ] +[[package]] +name = "dbos" +version = "2.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "sqlalchemy", extra = ["asyncio"] }, + { name = "typer-slim" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/b1/8a6c8ace5764e19850f2fd3f010251c3135daf1d682f43566c18dd0d432f/dbos-2.28.0.tar.gz", hash = "sha256:e738d62baf3953639d3e70988912b533ffd9de047d8b90d12755c10c03cac6fe", size = 592187, upload-time = "2026-07-21T15:24:08.551Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/d4/17f83581209153dff0bb7f6369c83d98d06303f8674746a7d47875add155/dbos-2.28.0-py3-none-any.whl", hash = "sha256:ba1bec5fcb2505b1bdd99251056cecf79669337699a81d5ef2b7598f701f31b6", size = 236036, upload-time = "2026-07-21T15:24:06.354Z" }, +] + +[package.optional-dependencies] +aiosqlite = [ + { name = "aiosqlite" }, +] + [[package]] name = "debugpy" version = "1.8.20" @@ -1826,6 +1866,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/1b/6ef961f543593969d25b2afe57a3564200280528caa9bd1082eecdd7b3bc/markdown-3.10.1-py3-none-any.whl", hash = "sha256:867d788939fe33e4b736426f5b9f651ad0c0ae0ecf89df0ca5d1176c70812fe3", size = 107684, upload-time = "2026-01-21T18:09:27.203Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -1919,6 +1971,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "mistune" version = "3.2.0" @@ -2578,6 +2639,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" }, ] +[[package]] +name = "playwright" +version = "1.61.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/ee/31e4e0db36588b817a10b299a0285082545fde7d36543c2abe498bb3d61a/playwright-1.61.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:ff138c3a604f69911e9d42fd036e55c2a171e5616edf04c1e7f60a2a285540b0", size = 43421877, upload-time = "2026-06-29T10:32:48.428Z" }, + { url = "https://files.pythonhosted.org/packages/42/35/71395dd3ecc798965be4a3ef8c443217d4abca168e7cb34536304f9489e6/playwright-1.61.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:009588c2a7e499bc5a8b425b61fa65490968bbda9cd69e0cf2cff10f8304659a", size = 42205016, upload-time = "2026-06-29T10:32:52.104Z" }, + { url = "https://files.pythonhosted.org/packages/f4/44/323164cf5cd1647bdefce76ffce27651aadb959d089b48f53ea40918276e/playwright-1.61.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9f7de4536088d12037c13a52b7ea34b59270b78926bb56935070597ffac6b1af", size = 43421884, upload-time = "2026-06-29T10:32:55.773Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f8/a35bf179e4ba2522c1893635094a64e407572547bd61528820fc0abc87fe/playwright-1.61.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:54f3b39f6eab832e33458c1dd7da0b5682aedab3b09ae731b5c59fa12fd2024e", size = 47421381, upload-time = "2026-06-29T10:32:59.903Z" }, + { url = "https://files.pythonhosted.org/packages/b7/eb/e3f922348ec17c315f98c463f72faa1181a1c3de0bfe31a8d2edf6561723/playwright-1.61.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93454322ade8c11d5d6c211bfd91bdfb9ffb4810e3e026371bcbc4bec1b7ee4c", size = 47120545, upload-time = "2026-06-29T10:33:03.574Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a6/5be4e52b40a9c0c8a073e7c5b0785c05cf5a9ea8f8a7b5b260e32d970342/playwright-1.61.0-py3-none-win32.whl", hash = "sha256:372d55a6f1248fa1dd47599686980cb8fb5bbe6fcda59eab793eb657c11d8a9b", size = 37844841, upload-time = "2026-06-29T10:33:07.361Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fd/2b78036e5fbe9d5f5645bbe08a1eac7160c51243c0093963edbcf67c35d9/playwright-1.61.0-py3-none-win_amd64.whl", hash = "sha256:35c6cc4589a5d00964a59d7b3e59641e0aac0c02f15479a7af77d20f6bc79597", size = 37844846, upload-time = "2026-06-29T10:33:10.637Z" }, + { url = "https://files.pythonhosted.org/packages/27/0d/1b0f3c4ee4eb0514bc805b5c2f9a223e5b6de4f11a926f5235d51d0fc81b/playwright-1.61.0-py3-none-win_arm64.whl", hash = "sha256:e9fcbffcf557a8620fdedd92491eb59a32d18e23d6f3b4f6214b952be324fe51", size = 33955127, upload-time = "2026-06-29T10:33:14.008Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -2678,6 +2758,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, ] +[[package]] +name = "psycopg" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/bf/70d8a60488f9955cbbcd538beae44d56bb2f1d19e673b72788f2d343ff55/psycopg_binary-3.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a", size = 4609750, upload-time = "2026-05-01T23:24:20.06Z" }, + { url = "https://files.pythonhosted.org/packages/db/b0/29e98ba210c9dbc75a6dc91e3f99b9e06ea901a62ca95804e02a1ae13e6b/psycopg_binary-3.3.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a", size = 4676700, upload-time = "2026-05-01T23:25:21.727Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ab/3df087b3c12bf74e47c08204172b2fabb5a144679110d5c7ad12d9201323/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429", size = 5496319, upload-time = "2026-05-01T23:25:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/87/9a/f088207b4cd6772f9e0d8a91807e79fa2458d4eb9eb1ae406c68415f2bec/psycopg_binary-3.3.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765", size = 5171906, upload-time = "2026-05-01T23:25:34.004Z" }, + { url = "https://files.pythonhosted.org/packages/48/45/4523a857f253871d75c22e1c2e79fd47e599e736bcba1bad58d83e24be02/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13", size = 6762621, upload-time = "2026-05-01T23:25:41.392Z" }, + { url = "https://files.pythonhosted.org/packages/7c/d1/925bf776503345bef428e6c45fb017d0139ddbe0e211814b585c4253dca8/psycopg_binary-3.3.4-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc", size = 5006319, upload-time = "2026-05-01T23:25:51.419Z" }, + { url = "https://files.pythonhosted.org/packages/6f/aa/99727337206fbba357ca084bf4ea8b29dc986f61842a2685859af61416db/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28", size = 4535388, upload-time = "2026-05-01T23:25:57.957Z" }, + { url = "https://files.pythonhosted.org/packages/0b/a4/567ba2c37d19d8c2f63d836385dfd2495aa5897bbee6cfab104d9ee58624/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e", size = 4224544, upload-time = "2026-05-01T23:26:03.832Z" }, + { url = "https://files.pythonhosted.org/packages/b7/23/86457f5a82731685d7701de7bfaa5eb783dd1fecbf875321897d9d9ce33a/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d", size = 3956282, upload-time = "2026-05-01T23:26:09.983Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d8/249456df16d47de082abd9b73bce8ccdeb0293eb12e590f9150c7cbdb788/psycopg_binary-3.3.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744", size = 4261736, upload-time = "2026-05-01T23:26:16.798Z" }, + { url = "https://files.pythonhosted.org/packages/15/6b/c4abe228acafd8a385c1fb615d4f1e3c9b8ad7a4e4f0e84118ba3ffeed9c/psycopg_binary-3.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949", size = 3570620, upload-time = "2026-05-01T23:26:22.655Z" }, + { url = "https://files.pythonhosted.org/packages/b6/82/df3312c0ca083d5b43b352f27d4dd8b1e614bd334473074715d9e0000da4/psycopg_binary-3.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da", size = 4609813, upload-time = "2026-05-01T23:26:30.612Z" }, + { url = "https://files.pythonhosted.org/packages/1f/b5/d74d542458d3e8ac0571d8a88f57ca369999b9a82f4fa528052d0d7d3e4c/psycopg_binary-3.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e", size = 4676799, upload-time = "2026-05-01T23:26:38.475Z" }, + { url = "https://files.pythonhosted.org/packages/09/67/06bab9c60671999f4c6ceff1b334f3ac1f9fc5789eb467c714623ea21de9/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992", size = 5497050, upload-time = "2026-05-01T23:26:47.061Z" }, + { url = "https://files.pythonhosted.org/packages/72/9b/023433e2b20f970de1e22d29132a95281277646da0b2e2879dd4ee94b8c1/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4", size = 5172428, upload-time = "2026-05-01T23:26:56.708Z" }, + { url = "https://files.pythonhosted.org/packages/08/cd/ae16da8fde228a38b2fe9269bbc13cf89e0186173f2265600f02d6a71e64/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e", size = 6762746, upload-time = "2026-05-01T23:27:07.023Z" }, + { url = "https://files.pythonhosted.org/packages/4f/81/0ba09fa5f5f88779093a2541a8e02489825721f258ab88058b11d68b3eb5/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097", size = 5006033, upload-time = "2026-05-01T23:27:12.221Z" }, + { url = "https://files.pythonhosted.org/packages/73/6a/629136040cc3497adb442a305710b5913f2a754d4630fc3d3717c4c0df65/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95", size = 4534175, upload-time = "2026-05-01T23:27:18.248Z" }, + { url = "https://files.pythonhosted.org/packages/7c/32/1027f843c6dc2d5d51960ee62cc0c2cf755a4c39455aff1371173edbef7d/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839", size = 4224203, upload-time = "2026-05-01T23:27:24.3Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e1/380a724d9093c74adb14d4fce920ea8327838abb61f760b1448586b14a8e/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007", size = 3954509, upload-time = "2026-05-01T23:27:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/db/cd/895893ae575a09c97ccfd5def070d88993d955ef34df45a881fd5ff506d6/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c", size = 4259551, upload-time = "2026-05-01T23:27:38.828Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c6/2330a20794e37a3ec609ef2fd8522919ec7a4395a1abf979a8e2d1775cd5/psycopg_binary-3.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d", size = 3572054, upload-time = "2026-05-01T23:27:45.455Z" }, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -2813,6 +2940,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl", hash = "sha256:fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809", size = 51880, upload-time = "2025-11-10T14:25:45.546Z" }, ] +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -2876,6 +3015,8 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "black" }, + { name = "dbos", extra = ["aiosqlite"] }, + { name = "playwright" }, { name = "pytest" }, ] @@ -2917,6 +3058,8 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "black", specifier = "==26.5.1" }, + { name = "dbos", extras = ["aiosqlite"], specifier = "==2.28.0" }, + { name = "playwright", specifier = ">=1.54,<2" }, { name = "pytest", specifier = "==9.0.2" }, ] @@ -3173,6 +3316,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/71/44ce230e1b7fadd372515a97e32a83011f906ddded8d03e3c6aafbdedbb7/rfc3987_syntax-1.1.0-py3-none-any.whl", hash = "sha256:6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f", size = 8046, upload-time = "2025-07-18T01:05:03.843Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "rpds-py" version = "0.30.0" @@ -3430,6 +3586,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/b8/f1f62a5e3c0ad2ff1d189590bfa4c46b4f3b6e49cef6f26c6ee4e575394d/setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173", size = 1064234, upload-time = "2026-01-25T22:38:15.216Z" }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -3475,6 +3640,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/a1/9c4efa03300926601c19c18582531b45aededfb961ab3c3585f1e24f120b/sqlalchemy-2.0.46-py3-none-any.whl", hash = "sha256:f9c11766e7e7c0a2767dda5acb006a118640c9fc0a4104214b96269bfb78399e", size = 1937882, upload-time = "2026-01-21T18:22:10.456Z" }, ] +[package.optional-dependencies] +asyncio = [ + { name = "greenlet" }, +] + [[package]] name = "stack-data" version = "0.6.3" @@ -3754,6 +3924,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3", size = 188214640, upload-time = "2026-01-20T16:00:35.869Z" }, ] +[[package]] +name = "typer" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" }, +] + +[[package]] +name = "typer-slim" +version = "0.24.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/a7/e6aecc4b4eb59598829a3b5076a93aff291b4fdaa2ded25efc4e1f4d219c/typer_slim-0.24.0.tar.gz", hash = "sha256:f0ed36127183f52ae6ced2ecb2521789995992c521a46083bfcdbb652d22ad34", size = 4776, upload-time = "2026-02-16T22:08:51.2Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/24/5480c20380dfd18cf33d14784096dca45a24eae6102e91d49a718d3b6855/typer_slim-0.24.0-py3-none-any.whl", hash = "sha256:d5d7ee1ee2834d5020c7c616ed5e0d0f29b9a4b1dd283bdebae198ec09778d0e", size = 3394, upload-time = "2026-02-16T22:08:49.92Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -3894,6 +4091,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, ] +[[package]] +name = "websockets" +version = "16.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/f7/bc3a25c5ec26ce62ce487690becc2f3710bbc7b33338f005ad390db0b986/websockets-16.1.1.tar.gz", hash = "sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57", size = 182204, upload-time = "2026-07-17T22:51:05.858Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/e7/d1671fb984f9dd844e1da5288070c7c23c9eaba3082d3871aae19c3ab8b9/websockets-16.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:49ae99bdfcae803a885c926bf14f886196e84925395bb3f568fef5c0f0979d7d", size = 179570, upload-time = "2026-07-17T22:48:24.032Z" }, + { url = "https://files.pythonhosted.org/packages/99/f5/70df723bf571f5e0b1b845e0a4ff1c966eeb84f667599fc251caa37d15a3/websockets-16.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5bfd1ac19b1b9986a9c95a82d5e23a391ebb09e12c34d7be6094b86efcc35731", size = 177252, upload-time = "2026-07-17T22:48:25.775Z" }, + { url = "https://files.pythonhosted.org/packages/90/72/2f14b2e167170b8bf1c8bb7f9b0d78000f470d41a2085a91f33e3917b6c9/websockets-16.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9246a0d063cfcbcc85f2359dd6876d681213f4790832272aa16641b4ed5d64d4", size = 177530, upload-time = "2026-07-17T22:48:27.337Z" }, + { url = "https://files.pythonhosted.org/packages/f3/18/a17e2f0cde02dc10154c808deed7e1d8528afff93612f70d3f0a5b19b011/websockets-16.1.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1214e673c404684b9bf7154f5cf43b45025b1a6160fac3a9e438e9c1a97e22cb", size = 186038, upload-time = "2026-07-17T22:48:28.756Z" }, + { url = "https://files.pythonhosted.org/packages/d5/b0/41de283899cf5929d637b72a508cdbc9aa40dc0f317c6b77613fd1000488/websockets-16.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90001d893bc368e302ef168d82130b4e4fdd27b85fa094682df9b667c2d48838", size = 187278, upload-time = "2026-07-17T22:48:30.328Z" }, + { url = "https://files.pythonhosted.org/packages/50/61/874aab5257e027f9f61b5004cec65e592babca7942b1bc09f38e72b7f1fd/websockets-16.1.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:130937b167a52af203c8d58e78d67705874e82759862e3b9671a452fec4abc87", size = 189936, upload-time = "2026-07-17T22:48:31.896Z" }, + { url = "https://files.pythonhosted.org/packages/a6/1a/42173913ac5519607220849ed417c864d77384e4119f06dbba964a50f096/websockets-16.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c9f23004a3d40e89c01a7955d186a6cc83418d93b749701944ce2de3e95a1f3", size = 187796, upload-time = "2026-07-17T22:48:33.344Z" }, + { url = "https://files.pythonhosted.org/packages/1b/f4/37c1840bd89b529479aec41470b97b7c683b107ca90b6399ac5afb99dedf/websockets-16.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f55f0b01956a094c8587146d9558c91937e78789c333860ffaf35931a6e5dbc4", size = 186481, upload-time = "2026-07-17T22:48:34.843Z" }, + { url = "https://files.pythonhosted.org/packages/9e/70/652d9b964adcfbeb056f42e0ca6bece34d108fe75534e74df20643cae199/websockets-16.1.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6aaface73b9c71974c6497366d8b9628357f6c9749e09c4ea3610176c63f2ae3", size = 184351, upload-time = "2026-07-17T22:48:36.307Z" }, + { url = "https://files.pythonhosted.org/packages/13/f1/af3850e5d48d482921985be72ebcb169c6180b3a77b57bd612deebcee23b/websockets-16.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dc0fad4933f427acd5b1cec210f3ea6dce7089e1724e4b9ec6ef47c6c04d1b3b", size = 186791, upload-time = "2026-07-17T22:48:37.762Z" }, + { url = "https://files.pythonhosted.org/packages/1d/40/1a4e3ed4969ec378dcad337e5f1472c5e292cb3e733bc392f0dc2e230abd/websockets-16.1.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f2769a0344a09e9ccf5b3cce538bc75a51b53eff3275d3896310c8552049195d", size = 185413, upload-time = "2026-07-17T22:48:39.127Z" }, + { url = "https://files.pythonhosted.org/packages/aa/3e/4e3fa1afe8f1a6a780434cd9ba8eb422632b044eff3dd73f6af67523c147/websockets-16.1.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f70541f3104339f59f830522d94ebadb1bf47426287381623443d8bb1cdbf33d", size = 187178, upload-time = "2026-07-17T22:48:40.676Z" }, + { url = "https://files.pythonhosted.org/packages/71/ab/dd742766aa5dda7f349be0de49e4d565b84cf6f7f7fa02e07692f0f2bdd9/websockets-16.1.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:dc385593a42e31cd6fb60c19f0ecb015b386603818fc2c6c274fb42bd2bb4165", size = 185051, upload-time = "2026-07-17T22:48:42.098Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f5/76438c6560f416f1c0a7f587679fb97cc6e99ed336011d43ce2002dd27c1/websockets-16.1.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:387e8e4aa5df2f90b198fa3cad3478822a89cf905b6a6d6c97dc3664689640cc", size = 185846, upload-time = "2026-07-17T22:48:43.472Z" }, + { url = "https://files.pythonhosted.org/packages/62/12/5c0320f2127823d27b2d56d611d31b0b284ad4edcb41364d66bf4c92b537/websockets-16.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fd46fff7eb62c24804d234f0051c7a8ea81285ad63e0337d3dcf33ca82aee58a", size = 186066, upload-time = "2026-07-17T22:48:44.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/97/875986b857b955c3f9dd192cb8a1af81254dfb2ea22cc9590f0a1e020b8b/websockets-16.1.1-cp310-cp310-win32.whl", hash = "sha256:7883388947767080f094950b342b30d35a2a06b849cd967c422fa0db72b40ea9", size = 179940, upload-time = "2026-07-17T22:48:46.481Z" }, + { url = "https://files.pythonhosted.org/packages/54/82/1013a5fe7ddae8e102bc3b4b39db81d8d28fd02100a324ce6ede8cd832b1/websockets-16.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:d57685547e0060cc6fd90ee6a28405d6bd395e525545f13c8d7cd99c78afd79f", size = 180239, upload-time = "2026-07-17T22:48:48.043Z" }, + { url = "https://files.pythonhosted.org/packages/2b/03/47debfe28e9d6d354be5d777b67fd44c359b9eb299a5d103500bd7cc3e37/websockets-16.1.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d0fcf657e9f13ff4b177960ab2200237b12994232dfb6df16f1cfe1d4339f93c", size = 179566, upload-time = "2026-07-17T22:48:49.596Z" }, + { url = "https://files.pythonhosted.org/packages/72/93/31efa1ed78c17e5cfc229fd449e3966e1b9cc15753204cd585cc8dd01f4a/websockets-16.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b852788aa51764e2d8e4cf5493d559326bcae5e38d16ba25ffa322b034df272a", size = 177250, upload-time = "2026-07-17T22:48:50.942Z" }, + { url = "https://files.pythonhosted.org/packages/01/4a/542378ab3972b0c1cf1df3df3eff9591cea0d30c58c3aa3c4ddbc244e787/websockets-16.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1427fb4cf0d72f66333e2cacc3ff5f575bf2d7008166ce991a4a470b21d51a22", size = 177528, upload-time = "2026-07-17T22:48:52.59Z" }, + { url = "https://files.pythonhosted.org/packages/33/d9/162321f63c7eed558e9e1798ed7a1e34a4f6dab51f35419e4ed7a4907979/websockets-16.1.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:da4ca1a9d72f9030b3146b8d7022719a9f3d478f61efe6f7dd51d243f61c51b2", size = 186859, upload-time = "2026-07-17T22:48:53.915Z" }, + { url = "https://files.pythonhosted.org/packages/de/09/87df740f7430ce564bd52402e9c9458d4d0459cc7d2ee29e530c8204851b/websockets-16.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86d7f0f8bdb25d2c632b72527325e4776430fd5bc61b9118de4e2b8ddb5f5b01", size = 188095, upload-time = "2026-07-17T22:48:55.384Z" }, + { url = "https://files.pythonhosted.org/packages/d2/12/3d2703af7cc095f3c81904c92208cc1ae79affbc67376944b50ee9301f73/websockets-16.1.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dfcad78ea1492ee3a9ec765cb7f51bbc17d477107aaf6b22abf7b2558d1c5a0", size = 191385, upload-time = "2026-07-17T22:48:56.742Z" }, + { url = "https://files.pythonhosted.org/packages/1d/69/986aa0234a964a00f5149cfc46e136e96c8faad1c783474550f40d31aef4/websockets-16.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fb9a0a6dc3d1b3986cb88091b6899f0396651e0f74e2c9766ab8d6ffc3842e29", size = 188653, upload-time = "2026-07-17T22:48:58.134Z" }, + { url = "https://files.pythonhosted.org/packages/35/6b/10f9d03e3970a69ba67bd3b46b87a929b586d0300fadbfe14f57c1f85490/websockets-16.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29dfa8114c4a620c69591c5973860f768eac29d3fd6904f37f34266cb219c512", size = 187426, upload-time = "2026-07-17T22:48:59.515Z" }, + { url = "https://files.pythonhosted.org/packages/56/db/bb3aad62bf63d8bb3f0634b2eabffcfb3677a34bd19492110ff6869cf703/websockets-16.1.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ff9417c0ada4d0f7d212f928303e5579bdf3ace4c802fa4afabb30995da58c3", size = 184882, upload-time = "2026-07-17T22:49:00.916Z" }, + { url = "https://files.pythonhosted.org/packages/6c/4c/c09a2ea9bfbeccce52fdc383e5f28af4bc8843338aabac28c81489af6120/websockets-16.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fe0b50da2d84535fb4f7b4bfa951280f97ce3d558a0443b541166d609e67b57", size = 187584, upload-time = "2026-07-17T22:49:02.283Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8b/31bb4eb4d9eaacf1fdd39d115772a8aeaedfc19b5dc262e57ffbc8a9d42c/websockets-16.1.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:34420aaa64440ebd51ac72ca8a45ef4626429438c9b02e633ae412ed43f925d3", size = 186174, upload-time = "2026-07-17T22:49:03.973Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e4/dc02d725610a1ad49e193ef91a548194d71bdc6cdf27da83067dd1f73995/websockets-16.1.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a6a61aff018180c9c50b7b0da33bfd29d378af3497429c95006c589a23a11648", size = 187986, upload-time = "2026-07-17T22:49:05.553Z" }, + { url = "https://files.pythonhosted.org/packages/e0/73/30ed84c8bfd14c73d4af29d5ed9323c3073b48e0b7b23b67070f4e7fd59b/websockets-16.1.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:04fd29a0e2fe9414a95b00e92c67ae51bf900c50c0f8a4b2dafdad621f49ea1d", size = 185565, upload-time = "2026-07-17T22:49:06.959Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d3/4be8d4959f51e31b4f8fc0ece12b45bd3b6c0d15ea23b9990d9c11fc805f/websockets-16.1.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5c31aa7e39ee3e8a358573257f1c0bb5c52430d1b637030dd9c8cc2c282926be", size = 186598, upload-time = "2026-07-17T22:49:08.293Z" }, + { url = "https://files.pythonhosted.org/packages/26/fa/abb38597a52d84ed9cfacadc7a0c6f2db282c0ab23cdf72b58a666a21227/websockets-16.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d14bfb217eb4701e850f1525c9d29d79c44794cdf1c299ead25f39f8c78dea81", size = 186834, upload-time = "2026-07-17T22:49:09.766Z" }, + { url = "https://files.pythonhosted.org/packages/59/80/1119ad08a228b90c4eb77fbe48df7836731a605f5f881ba701ca826a4a65/websockets-16.1.1-cp311-cp311-win32.whl", hash = "sha256:2e28e602bb13da44fbe518c1781a88e3b9d4c3d48d02c9bad83e546164336f57", size = 179940, upload-time = "2026-07-17T22:49:11.196Z" }, + { url = "https://files.pythonhosted.org/packages/71/b2/e511c1c6f64a95c2f3fc54bffda0e14eaa7e9442be605c29270f7589b918/websockets-16.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:7421fad442de870a8cbf2287d1cad7e706ece0dbfeba5e911df132cbdc1cb56a", size = 180239, upload-time = "2026-07-17T22:49:12.519Z" }, + { url = "https://files.pythonhosted.org/packages/e1/ed/71fea6e141590cafc40b14dc5943b0845606bee87bdb52a21b6a73eb4311/websockets-16.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:820fb8450edddae3812fd58cbc08e2bf22812cb248ecb5f06dbb82119a56e869", size = 177185, upload-time = "2026-07-17T22:50:56.665Z" }, + { url = "https://files.pythonhosted.org/packages/01/ec/00e7eeca200facf9266a83e4cbbf1bed0e67fba1d4d45031d3e5b3d81b5c/websockets-16.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:125f22dbefaf1554fea66fc83851490edb284ce4f501d37ffed2752f418332d9", size = 177459, upload-time = "2026-07-17T22:50:58.197Z" }, + { url = "https://files.pythonhosted.org/packages/75/fd/5774c4b33f7c0d8f0c51809c8b3a93456c48e3543579262cfa64eb5f522e/websockets-16.1.1-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:30bbe120437b5648a77d3519b7024ea09530e0b5b18d3698c5a0ae536fe0cc2e", size = 178294, upload-time = "2026-07-17T22:50:59.641Z" }, + { url = "https://files.pythonhosted.org/packages/37/c3/48e2c03d2bd79bb45948841c592d24156312dd5f58cdf8f549febe652fb6/websockets-16.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b6b9dadbef0cccd9f4c4ee96b08898afa73e26803bbe0f6aeb5bb12b0074206d", size = 179190, upload-time = "2026-07-17T22:51:01.129Z" }, + { url = "https://files.pythonhosted.org/packages/2d/3f/73e511ecf2496ceac57dd4ed8388efe2bcf0769338a2dbf242c8366ae87e/websockets-16.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56cd5fc4f10a9ea8aa0804bddb7b42506cf9e136046f3b4c27de8fec9e2ecba5", size = 180330, upload-time = "2026-07-17T22:51:02.603Z" }, + { url = "https://files.pythonhosted.org/packages/be/4d/2d0d67834092e354d2b0498f014a41249a89556bc406cf86f3e1557bb463/websockets-16.1.1-py3-none-any.whl", hash = "sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3", size = 173814, upload-time = "2026-07-17T22:51:04.184Z" }, +] + [[package]] name = "werkzeug" version = "3.1.5" From d25bbd3bf5559df3a5a97433a80fcd1ecfe72729 Mon Sep 17 00:00:00 2001 From: Adam Gohain Date: Tue, 21 Jul 2026 14:42:51 -0400 Subject: [PATCH 5/6] Complete durable agent action hardening --- .github/workflows/ci.yml | 56 +++ client/src/App.css | 28 ++ client/src/api.js | 30 ++ client/src/api.test.js | 19 + .../workflow/WorkflowOperationsPanel.css | 68 +++ .../workflow/WorkflowOperationsPanel.js | 379 ++++++++++++++++ .../workflow/WorkflowOperationsPanel.test.js | 215 +++++++++ client/src/contexts/WorkflowContext.js | 28 +- client/src/contexts/WorkflowContext.test.js | 48 ++ client/src/views/ProjectProgress.js | 5 + client/src/views/ProjectProgress.test.js | 4 + client/src/views/Views.js | 15 +- client/src/views/Views.test.js | 2 + docs/platform-hardening-branch-scope.md | 27 +- scripts/browser_synthetic_core_smoke.py | 120 ++++- .../run_browser_synthetic_core_smoke_ci.sh | 128 ++++++ .../workflows/agent_action_execution.py | 421 ++++++++++++++++++ server_api/workflows/agent_actions.py | 17 + server_api/workflows/evaluation_service.py | 108 +++++ server_api/workflows/operation_router.py | 37 +- server_api/workflows/operation_service.py | 2 + server_api/workflows/router.py | 238 +++++++--- tests/test_agent_evaluation_execution.py | 261 +++++++++++ 23 files changed, 2179 insertions(+), 77 deletions(-) create mode 100644 client/src/components/workflow/WorkflowOperationsPanel.css create mode 100644 client/src/components/workflow/WorkflowOperationsPanel.js create mode 100644 client/src/components/workflow/WorkflowOperationsPanel.test.js create mode 100755 scripts/run_browser_synthetic_core_smoke_ci.sh create mode 100644 server_api/workflows/agent_action_execution.py create mode 100644 server_api/workflows/evaluation_service.py create mode 100644 tests/test_agent_evaluation_execution.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32169baf..569e86a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,3 +76,59 @@ jobs: - name: Build frontend run: npm run build + + synthetic-browser-smoke: + name: Synthetic browser smoke + needs: + - backend + - frontend + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.11" + + - name: Set up uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + version: "0.7.3" + enable-cache: true + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + cache: npm + cache-dependency-path: client/package-lock.json + + - name: Fetch pinned PyTorch Connectomics runtime + run: bash scripts/setup_pytorch_connectomics.sh + + - name: Install Python dependencies + run: uv sync --frozen --python 3.11 --group dev + + - name: Install frontend dependencies + working-directory: client + run: npm ci --fetch-retries=5 --fetch-retry-maxtimeout=120000 + + - name: Install Chromium + run: uv run playwright install --with-deps chromium + + - name: Run deterministic synthetic browser smoke + run: bash scripts/run_browser_synthetic_core_smoke_ci.sh + + - name: Upload browser smoke diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: synthetic-browser-smoke-${{ github.run_attempt }} + path: .ci/synthetic-browser-smoke/ + if-no-files-found: warn + retention-days: 14 diff --git a/client/src/App.css b/client/src/App.css index 4a30272d..dfa67459 100644 --- a/client/src/App.css +++ b/client/src/App.css @@ -205,6 +205,34 @@ padding-right: 2px; } +.pytc-top-nav, +.pytc-top-menu { + min-width: 0; +} + +.pytc-top-nav { + max-width: 100vw; + overflow: hidden; +} + +.pytc-top-nav-action { + flex: 0 0 auto; +} + +@media (max-width: 720px) { + .pytc-top-menu .ant-menu-item { + padding-inline: 10px 14px !important; + } + + .pytc-top-nav-action { + padding-inline: 8px; + } + + .pytc-top-nav-action__label { + display: none; + } +} + .app-logo { height: 40vmin; pointer-events: none; diff --git a/client/src/api.js b/client/src/api.js index 44313c25..026841c2 100644 --- a/client/src/api.js +++ b/client/src/api.js @@ -951,6 +951,36 @@ export async function listWorkflowEvents(workflowId) { } } +export async function listWorkflowOperations(workflowId, { limit = 12 } = {}) { + try { + const res = await apiClient.get( + canonicalizeApiPath(`/api/workflows/${workflowId}/operations`), + { params: { limit } }, + ); + return res.data; + } catch (error) { + handleError(error); + } +} + +export async function cancelWorkflowOperation( + workflowId, + operationId, + reason = "", +) { + try { + const res = await apiClient.post( + canonicalizeApiPath( + `/api/workflows/${workflowId}/operations/${operationId}/cancel`, + ), + reason ? { reason } : undefined, + ); + return res.data; + } catch (error) { + handleError(error); + } +} + export async function getWorkflowHotspots(workflowId) { try { const res = await apiClient.get( diff --git a/client/src/api.test.js b/client/src/api.test.js index d9a2d09e..fb4f1c90 100644 --- a/client/src/api.test.js +++ b/client/src/api.test.js @@ -104,4 +104,23 @@ describe("api canonicalization", () => { "/99/commands/321/run", ); }); + + it("uses canonical durable operation list and cancellation paths", async () => { + const { api, apiClientMock } = loadApiModule(BASE_WITH_API_PREFIX); + apiClientMock.get.mockResolvedValue({ data: [] }); + apiClientMock.post.mockResolvedValue({ + data: { id: 8, status: "cancelled" }, + }); + + await api.listWorkflowOperations(42, { limit: 6 }); + expect(apiClientMock.get).toHaveBeenCalledWith("/workflows/42/operations", { + params: { limit: 6 }, + }); + + await api.cancelWorkflowOperation(42, 8, "No longer needed"); + expect(apiClientMock.post).toHaveBeenCalledWith( + "/workflows/42/operations/8/cancel", + { reason: "No longer needed" }, + ); + }); }); diff --git a/client/src/components/workflow/WorkflowOperationsPanel.css b/client/src/components/workflow/WorkflowOperationsPanel.css new file mode 100644 index 00000000..44877c53 --- /dev/null +++ b/client/src/components/workflow/WorkflowOperationsPanel.css @@ -0,0 +1,68 @@ +.workflow-operations-panel__loading { + align-items: center; + display: flex; + gap: 8px; + justify-content: center; + min-height: 96px; +} + +.workflow-operations-panel__list { + border-block: 1px solid var(--seg-border-subtle, #e4ded2); +} + +.workflow-operation-row { + align-items: center; + display: grid; + gap: 12px; + grid-template-columns: minmax(0, 1fr) auto; + min-height: 76px; + padding: 10px 2px; +} + +.workflow-operation-row + .workflow-operation-row { + border-top: 1px solid var(--seg-border-subtle, #e4ded2); +} + +.workflow-operation-row__main { + display: grid; + gap: 6px; + min-width: 0; +} + +.workflow-operation-row__heading, +.workflow-operation-row__metadata { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: 6px 10px; + min-width: 0; +} + +.workflow-operation-row__reference, +.workflow-operation-row__error { + overflow-wrap: anywhere; +} + +.workflow-operation-row__metadata .ant-typography { + font-size: 12px; +} + +.workflow-operation-row__actions { + justify-self: end; +} + +.workflow-operations-panel__syncing { + align-self: flex-end; + font-size: 12px; +} + +@media (max-width: 640px) { + .workflow-operation-row { + align-items: stretch; + grid-template-columns: minmax(0, 1fr); + } + + .workflow-operation-row__actions { + justify-self: start; + } +} diff --git a/client/src/components/workflow/WorkflowOperationsPanel.js b/client/src/components/workflow/WorkflowOperationsPanel.js new file mode 100644 index 00000000..c7377c9d --- /dev/null +++ b/client/src/components/workflow/WorkflowOperationsPanel.js @@ -0,0 +1,379 @@ +import React, { useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { + Alert, + Button, + Card, + Empty, + Popconfirm, + Progress, + Space, + Spin, + Tag, + Tooltip, + Typography, +} from "antd"; +import { RedoOutlined, ReloadOutlined, StopOutlined } from "@ant-design/icons"; +import { + cancelWorkflowOperation, + listWorkflowOperations, + runWorkflowCommand, +} from "../../api"; +import { getApiErrorMessage } from "../../errors/apiError"; +import { useWorkflow } from "../../contexts/WorkflowContext"; +import "./WorkflowOperationsPanel.css"; + +const { Text } = Typography; +const ACTIVE_OPERATION_STATUSES = new Set(["queued", "running"]); +const CANCELLABLE_OPERATION_STATUSES = new Set(["queued", "running"]); +const OPERATION_POLL_INTERVAL_MS = 2500; + +const STATUS_CONFIG = { + queued: { label: "Queued", color: "default" }, + running: { label: "Running", color: "processing" }, + succeeded: { label: "Succeeded", color: "success" }, + failed: { label: "Failed", color: "error" }, + cancelled: { label: "Cancelled", color: "warning" }, +}; + +export const hasActiveOperations = (operations = []) => + operations.some((operation) => + ACTIVE_OPERATION_STATUSES.has(operation?.status), + ); + +export const getOperationsRefetchInterval = (query) => + hasActiveOperations(query?.state?.data) ? OPERATION_POLL_INTERVAL_MS : false; + +export const getWorkflowOperationsQueryOptions = ( + workflowId, + { compact = false } = {}, +) => ({ + queryKey: ["workflow", workflowId, "operations"], + queryFn: () => + listWorkflowOperations(workflowId, { limit: compact ? 6 : 12 }), + enabled: Boolean(workflowId), + refetchInterval: getOperationsRefetchInterval, + refetchOnReconnect: "always", +}); + +const hasReplayableInput = (input) => + Boolean( + input && + typeof input === "object" && + !Array.isArray(input) && + Object.keys(input).length, + ); + +export const canRetryOperation = (operation) => { + if (operation?.status !== "failed" || !operation?.command_id) return false; + + const retry = operation.metadata?.retry; + if ( + retry?.allowed === true && + retry?.kind === "workflow_command" && + hasReplayableInput(operation.input) + ) { + return true; + } + + return Boolean( + operation.metadata?.command_type === "start_training" && + operation.metadata?.execution_scope === "worker_submission" && + [503, 504].includes(Number(operation.error?.status_code)) && + hasReplayableInput(operation.input), + ); +}; + +const operationLabel = (value) => { + const normalized = String(value || "operation") + .replace(/^agent_action:/, "") + .replace(/[_:]+/g, " ") + .trim(); + return normalized.charAt(0).toUpperCase() + normalized.slice(1); +}; + +const operationErrorMessage = (operation) => { + const error = operation?.error; + if (!error) return ""; + if (typeof error === "string") return error; + if (typeof error.detail === "string") return error.detail; + if (typeof error.message === "string") return error.message; + if (typeof error.error === "string") return error.error; + return "The operation did not complete."; +}; + +const formatUpdatedAt = (value) => { + if (!value) return ""; + const date = new Date(value); + if (Number.isNaN(date.getTime())) return ""; + return date.toLocaleString([], { + month: "short", + day: "numeric", + hour: "numeric", + minute: "2-digit", + }); +}; + +function OperationRow({ operation, cancelling, retrying, onCancel, onRetry }) { + const status = STATUS_CONFIG[operation.status] || { + label: operationLabel(operation.status), + color: "default", + }; + const cancellationRequested = Boolean(operation.cancellation_requested_at); + const cancellationPending = + cancellationRequested && operation.status === "running"; + const cancellationAcknowledged = + cancellationRequested && operation.status === "cancelled"; + const canCancel = + CANCELLABLE_OPERATION_STATUSES.has(operation.status) && + !cancellationRequested; + const canRetry = canRetryOperation(operation); + const progress = Number(operation.progress); + const hasProgress = + operation.progress !== null && + operation.progress !== undefined && + Number.isFinite(progress); + const errorMessage = operationErrorMessage(operation); + + return ( +
+
+
+ {operationLabel(operation.operation_type)} + + {status.label} + {cancellationPending && ( + Cancellation requested + )} + {cancellationAcknowledged && ( + Cancellation acknowledged + )} + +
+ +
+ + Ref {operation.correlation_id || `operation-${operation.id}`} + + {formatUpdatedAt(operation.updated_at) && ( + + Updated {formatUpdatedAt(operation.updated_at)} + + )} + {operation.attempt_count > 0 && ( + Attempt {operation.attempt_count} + )} +
+ + {hasProgress && operation.status === "running" && ( + + )} + + {errorMessage && operation.status === "failed" && ( + + {errorMessage} + + )} +
+ + {(canCancel || canRetry) && ( + + {canRetry && ( + + )} + {canCancel && ( + onCancel(operation)} + > + + + )} + + )} +
+ ); +} + +function WorkflowOperationsPanel({ compact = false }) { + const workflow = useWorkflow()?.workflow; + const workflowId = workflow?.id; + const queryClient = useQueryClient(); + const queryKey = ["workflow", workflowId, "operations"]; + const [actionError, setActionError] = useState(""); + + const operationsQuery = useQuery( + getWorkflowOperationsQueryOptions(workflowId, { compact }), + ); + + const refreshOperations = () => { + setActionError(""); + operationsQuery.refetch(); + }; + + const updateOperation = (updatedOperation) => { + queryClient.setQueryData(queryKey, (current = []) => + current.map((operation) => + operation.id === updatedOperation.id ? updatedOperation : operation, + ), + ); + }; + + const cancelMutation = useMutation({ + mutationFn: (operation) => + cancelWorkflowOperation( + workflowId, + operation.id, + "Cancelled from the workflow operations panel.", + ), + onMutate: () => setActionError(""), + onSuccess: (operation) => { + updateOperation(operation); + queryClient.invalidateQueries({ queryKey }); + }, + onError: (error) => setActionError(getApiErrorMessage(error)), + }); + + const retryMutation = useMutation({ + mutationFn: (operation) => + runWorkflowCommand(workflowId, operation.command_id), + onMutate: () => setActionError(""), + onSuccess: () => queryClient.invalidateQueries({ queryKey }), + onError: (error) => setActionError(getApiErrorMessage(error)), + }); + + if (!workflowId) return null; + + const operations = operationsQuery.data || []; + const activeCount = operations.filter((operation) => + ACTIVE_OPERATION_STATUSES.has(operation.status), + ).length; + + return ( + + Operations + {activeCount > 0 && ( + {activeCount} active + )} + + } + extra={ + + -