From 2db5e8137082cfb822392fdedb5e13bc46552e1b Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Wed, 4 Nov 2026 11:04:41 +0100 Subject: [PATCH 1/8] System audit: fix security, data-integrity and correctness defects Iterative audit (32 passes) of the whole system; every confirmed defect is fixed with a regression test. Findings, evidence, open owner decisions and continuation steps are in Audit.md. Highlights: - Secrets: connection keys can no longer name platform secrets or be sent to a host other than the one they were resolved for; the Open WebUI admin token only goes to Studio's own Open WebUI; embedded Open WebUI no longer inherits Studio secrets; auth tokens are revoked on password change. - Authorization: workspace-scoped role checks on experiment visibility, agents/KB/tools API, connections, OTLP credentials, run cancel and monitor run-now; stale sessions lose access to removed workspaces; visualizer API requires login; public run pages hide endpoint URLs. - Run integrity: conditional cancel/fail/finalize, no downgrade of completed results, per-run auditor dispatch for concurrent runs, frozen agent endpoint snapshots, per-rep/per-attempt trace correlation, judge params no longer inherit target defaults, repeated-run exports. - Data integrity: scenario revert/publish pins revisions, stale-form guards for scenarios and judges, cross-set delete, import validation, field-length caps for Postgres, CLI mock mode no longer rewrites or wipes user connections. - UX/a11y, input validation (500s on malformed input), docs/env examples. Tests: 1448 -> 1589 passing; ruff clean; no migrations. --- .env.example | 12 + accounts/serializers.py | 24 +- accounts/services.py | 15 + accounts/views.py | 20 +- accounts/workos_auth.py | 18 +- audits/agentic/metrics.py | 11 +- audits/agentic/rule_engine.py | 18 + audits/agentic/rules.py | 2 +- audits/agentic/tests/test_metrics.py | 5 + .../tests/test_rule_malformed_expect.py | 26 + audits/comparison.py | 6 + audits/events.py | 5 + audits/experiments.py | 29 +- audits/monitors.py | 13 +- audits/services.py | 66 +- audits/tests/test_export.py | 12 + audits/views.py | 19 +- chat/access_sync.py | 11 +- chat/api.py | 2 +- chat/api_passthrough.py | 4 +- chat/config.py | 16 +- chat/proxy.py | 27 +- chat/tests/test_chat.py | 31 + docker-compose.yml | 6 +- docs/chat.md | 5 + docs/deployment.md | 5 +- infra/codegen.py | 72 +- infra/context_processors.py | 13 +- infra/engine.py | 134 ++- infra/export_utils.py | 15 + infra/health.py | 18 +- infra/health_api.py | 2 +- infra/middleware.py | 6 + infra/minimal_config.py | 4 +- infra/runs_table.py | 27 +- infra/seed.py | 6 +- infra/tests/test_agent_resources.py | 20 + infra/tests/test_audit_events.py | 9 + infra/tests/test_audit_submission.py | 61 ++ infra/tests/test_codegen.py | 22 + infra/tests/test_engine_integration.py | 193 ++++- infra/tests/test_experiments.py | 35 + infra/tests/test_malformed_input.py | 360 ++++++++ infra/tests/test_minimal_config.py | 18 +- infra/tests/test_pass12_fixes.py | 135 +++ infra/tests/test_pass24_regressions.py | 195 +++++ infra/tests/test_security_hardening.py | 810 ++++++++++++++++++ infra/tests/test_seed_demo_agent.py | 7 + infra/trace_target.py | 36 +- infra/ui.py | 641 ++++++++++---- infra/visualizer.py | 22 +- infra/worker.py | 38 +- judges/services.py | 19 +- judges/views.py | 31 +- model_registry/agent_serializers.py | 21 + model_registry/agent_views.py | 24 +- model_registry/api_credential_services.py | 7 +- model_registry/models.py | 5 + model_registry/otlp_views.py | 37 +- model_registry/serializers.py | 4 +- model_registry/services.py | 94 +- .../tests/test_agent_target_resolution.py | 29 + model_registry/tests/test_api_passthrough.py | 19 + prompts/services.py | 4 + prompts/views.py | 11 +- scenarios/serializers.py | 25 +- scenarios/services.py | 15 +- scenarios/views.py | 37 +- simpleaudit_studio/cli.py | 74 +- static/visualizer.html | 4 +- templates/admin.html | 10 +- templates/admin/rag_settings.html | 17 +- templates/agents/agent_detail.html | 6 +- templates/auth/register.html | 2 +- templates/base.html | 33 +- templates/compare.html | 13 +- templates/connections.html | 12 +- templates/judge_detail.html | 4 + templates/partials/agentic_rules_editor.html | 19 +- templates/partials/run_detail_body.html | 6 +- templates/partials/run_settings.html | 4 +- templates/scenarios.html | 26 +- 82 files changed, 3435 insertions(+), 484 deletions(-) create mode 100644 audits/agentic/tests/test_rule_malformed_expect.py create mode 100644 infra/export_utils.py create mode 100644 infra/tests/test_malformed_input.py create mode 100644 infra/tests/test_pass12_fixes.py create mode 100644 infra/tests/test_pass24_regressions.py create mode 100644 infra/tests/test_security_hardening.py diff --git a/.env.example b/.env.example index 9e09d7a8..521da651 100644 --- a/.env.example +++ b/.env.example @@ -155,3 +155,15 @@ SENTRY_ENVIRONMENT=development # WORKOS_CLIENT_ID is public; WORKOS_API_KEY is server-side secret (keep out of git). WORKOS_CLIENT_ID= WORKOS_API_KEY= + +# --- Proxy / HTTPS ------------------------------------------------------------ +# Full origins allowed for CSRF when served behind a proxy, comma-separated. +# DJANGO_CSRF_TRUSTED_ORIGINS=https://studio.example.com + +# --- Gunicorn tuning (defaults shown) ------------------------------------------- +# GUNICORN_BIND=0.0.0.0:8000 +# GUNICORN_WORKERS=2 +# GUNICORN_THREADS=16 +# GUNICORN_TIMEOUT=60 +# GUNICORN_GRACEFUL_TIMEOUT=30 +# GUNICORN_KEEPALIVE=5 diff --git a/accounts/serializers.py b/accounts/serializers.py index 503e9f55..ff1d49a2 100644 --- a/accounts/serializers.py +++ b/accounts/serializers.py @@ -1,4 +1,5 @@ from django.contrib.auth.password_validation import validate_password +from django.contrib.auth.validators import UnicodeUsernameValidator from rest_framework import serializers from accounts.models import Project, ProjectMembership, User @@ -12,11 +13,11 @@ class Meta: class RegisterSerializer(serializers.Serializer): - username = serializers.CharField(max_length=150) - email = serializers.EmailField() + username = serializers.CharField(max_length=150, validators=[UnicodeUsernameValidator()]) + email = serializers.EmailField(max_length=254) password = serializers.CharField(write_only=True, style={"input_type": "password"}) - first_name = serializers.CharField(required=False, allow_blank=True, default="") - last_name = serializers.CharField(required=False, allow_blank=True, default="") + first_name = serializers.CharField(required=False, allow_blank=True, default="", max_length=150) + last_name = serializers.CharField(required=False, allow_blank=True, default="", max_length=150) def validate_username(self, value): if User.objects.filter(username__iexact=value).exists(): @@ -91,7 +92,7 @@ class ProfileUpdateSerializer(serializers.Serializer): first_name = serializers.CharField(required=False, allow_blank=True, max_length=150) last_name = serializers.CharField(required=False, allow_blank=True, max_length=150) - email = serializers.EmailField(required=False, allow_blank=True) + email = serializers.EmailField(required=False, allow_blank=True, max_length=254) username = serializers.CharField(required=False, max_length=150) current_password = serializers.CharField(required=False, style={"input_type": "password"}, write_only=True) new_password = serializers.CharField(required=False, style={"input_type": "password"}, write_only=True) @@ -142,6 +143,9 @@ def save(self): if attrs.get("new_password"): user.set_password(attrs["new_password"]) user.save(update_fields=["password"]) + from accounts.services import revoke_auth_tokens + + revoke_auth_tokens(user) return user @@ -149,11 +153,11 @@ def save(self): class UserCreateSerializer(serializers.Serializer): - username = serializers.CharField(max_length=150) - email = serializers.EmailField(required=False, allow_blank=True) + username = serializers.CharField(max_length=150, validators=[UnicodeUsernameValidator()]) + email = serializers.EmailField(required=False, allow_blank=True, max_length=254) password = serializers.CharField(write_only=True, style={"input_type": "password"}) - first_name = serializers.CharField(required=False, allow_blank=True, default="") - last_name = serializers.CharField(required=False, allow_blank=True, default="") + first_name = serializers.CharField(required=False, allow_blank=True, default="", max_length=150) + last_name = serializers.CharField(required=False, allow_blank=True, default="", max_length=150) class UserAdminUpdateSerializer(serializers.Serializer): @@ -161,7 +165,7 @@ class UserAdminUpdateSerializer(serializers.Serializer): first_name = serializers.CharField(required=False, allow_blank=True, max_length=150) last_name = serializers.CharField(required=False, allow_blank=True, max_length=150) - email = serializers.EmailField(required=False, allow_blank=True) + email = serializers.EmailField(required=False, allow_blank=True, max_length=254) is_active = serializers.BooleanField(required=False) is_superuser = serializers.BooleanField(required=False) password = serializers.CharField(required=False, write_only=True, style={"input_type": "password"}) diff --git a/accounts/services.py b/accounts/services.py index cd7d820a..e2ca82e1 100644 --- a/accounts/services.py +++ b/accounts/services.py @@ -44,6 +44,17 @@ def has_local_password(user) -> bool: @transaction.atomic +def revoke_auth_tokens(user) -> None: + """Drop the user's DRF auth tokens (call after a password change or reset). + + Django sessions are tied to the password hash and end on their own; tokens + never expire, so without this an old token keeps full access. + """ + from rest_framework.authtoken.models import Token + + Token.objects.filter(user=user).delete() + + def bootstrap_admin_and_default_project( *, username: str, @@ -63,6 +74,7 @@ def bootstrap_admin_and_default_project( if created or not user.check_password(password): user.set_password(password) user.save(update_fields=["password"]) + revoke_auth_tokens(user) if not user.is_active: user.is_active = True user.save(update_fields=["is_active"]) @@ -236,6 +248,7 @@ def delete_workspace(*, user, project: Project) -> None: """Delete an EMPTY workspace. Refuses while any audit content exists so a mistaken click can never destroy experiment history.""" _require_workspace_admin(user, project) + require_project_writable(user, project) # archived workspaces are read-only from audits.models import AuditRun from model_registry.models import ModelConnection @@ -358,6 +371,8 @@ def admin_update_user( _validate_password_or_raise(password) target.set_password(password) target.save() + if password: + revoke_auth_tokens(target) return target diff --git a/accounts/views.py b/accounts/views.py index 37910a55..74c56b86 100644 --- a/accounts/views.py +++ b/accounts/views.py @@ -81,8 +81,11 @@ def obtain_token(request): The SPA uses token auth (``Authorization: Token ``) so it works across origins and without CSRF handling. Returns the token plus the user payload. """ - username = request.data.get("username", "") - password = request.data.get("password", "") + data = request.data if isinstance(request.data, dict) else {} + username = data.get("username", "") + password = data.get("password", "") + if not isinstance(username, str) or not isinstance(password, str): + raise StableAPIError(detail="Invalid credentials.", code="invalid_credentials", http_status=401) user = authenticate(request, username=username, password=password) if user is None: raise StableAPIError(detail="Invalid credentials.", code="invalid_credentials", http_status=401) @@ -237,8 +240,9 @@ def workspace_detail(request, project_id): @permission_classes([IsAuthenticated]) def switch_workspace(request): """Set the session's active workspace so all UI pages scope to it.""" - raw_id = request.data.get("project_id") or request.data.get("workspace_id") - if not raw_id or not str(raw_id).isdigit(): + data = request.data if isinstance(request.data, dict) else {} + raw_id = data.get("project_id") or data.get("workspace_id") + if not raw_id or not str(raw_id).isascii() or not str(raw_id).isdecimal() or len(str(raw_id)) > 18: raise StableAPIError(detail="project_id is required.", code="invalid_workspace_id") project = _get_project_or_404(int(raw_id)) if not ensure_project_access(request.user, project): @@ -314,8 +318,12 @@ def add_member(request, project_id): defaults={"role": role, "created_by": request.user}, ) if not created and membership.role != role: - membership.role = role - membership.save(update_fields=["role"]) + # Role changes go through update_or_remove_member, which keeps the last + # admin and refuses to touch superusers. + raise StableAPIError( + detail="Already a member; change the role with the member update endpoint.", + code="already_member", http_status=409, + ) return Response(ProjectMembershipSerializer(membership).data, status=status.HTTP_201_CREATED) diff --git a/accounts/workos_auth.py b/accounts/workos_auth.py index 243f48f8..5090b81f 100644 --- a/accounts/workos_auth.py +++ b/accounts/workos_auth.py @@ -34,11 +34,16 @@ def send_magic_auth_code(email: str, *, ip_address: str | None = None, user_agen ) -def authenticate_magic_auth(code: str, email: str, *, ip_address: str | None = None, user_agent: str | None = None): +def authenticate_magic_auth( + code: str, email: str, *, ip_address: str | None = None, user_agent: str | None = None, + allow_create: bool = True, +): """Verify a Magic Auth code and return the local Django user. Returns ``(user, created)`` where ``created`` is True when the account was - provisioned by this sign-in. + provisioned by this sign-in. Raises ``ValueError`` when a new account would + be needed but ``allow_create`` is False (registration disabled), or when + the email already belongs to another local user (emails are chat identity). """ response = _client().user_management.authenticate_with_magic_auth( code=code, @@ -51,6 +56,11 @@ def authenticate_magic_auth(code: str, email: str, *, ip_address: str | None = N raise ValueError("WorkOS returned no user for the magic auth code.") username = _derive_username(wo_user.email or wo_user.id) + if not User.objects.filter(workos_user_id=wo_user.id).exists(): + if not allow_create: + raise ValueError("New sign-ups are disabled on this deployment.") + if wo_user.email and User.objects.filter(email__iexact=wo_user.email).exists(): + raise ValueError("An account with this email already exists; sign in with it instead.") user, created = User.objects.get_or_create( workos_user_id=wo_user.id, defaults={ @@ -64,7 +74,9 @@ def authenticate_magic_auth(code: str, email: str, *, ip_address: str | None = N # Keep profile fields fresh on subsequent sign-ins. if not created: changed = False - if wo_user.email and user.email != wo_user.email: + if (wo_user.email and user.email != wo_user.email + and not User.objects.filter(email__iexact=wo_user.email).exclude(pk=user.pk).exists()): + # Skipped on a collision: two users with one email share a chat account. user.email = wo_user.email changed = True if wo_user.first_name and user.first_name != wo_user.first_name: diff --git a/audits/agentic/metrics.py b/audits/agentic/metrics.py index a1e6e2a1..0aaea055 100644 --- a/audits/agentic/metrics.py +++ b/audits/agentic/metrics.py @@ -10,6 +10,9 @@ _SEVERITY_RANK = {"pass": 0, "low": 1, "medium": 2, "high": 3, "critical": 4, "error": 5, "unknown": 6} +_NO_VERDICT = frozenset({"error", "ungraded", "unknown"}) + + def _severity(value: Any) -> str: return str(value or "unknown").strip().lower() @@ -102,13 +105,17 @@ def compute_run_metrics(results: list[dict[str, Any]]) -> dict[str, Any]: semantic_status = str(semantic.get("status", "INCONCLUSIVE")).upper() semantic_statuses[semantic_status if semantic_status in {"PASS", "FAIL", "INCONCLUSIVE"} else "INCONCLUSIVE"] += 1 - passed = sum(severity in {"pass", "low"} for severity in severities) + # Same definition as the rest of Studio (runs table, monitors, experiments): + # only "pass" passes, and rows without a verdict (ERROR, ungraded, unknown) + # are left out of the denominator rather than counted as failures. + graded = [s for s in severities if s not in _NO_VERDICT] + passed = sum(s == "pass" for s in graded) agentic_total = sum(statuses.values()) deterministic_total = sum(deterministic_statuses.values()) semantic_total = sum(semantic_statuses.values()) return { "total_scenarios": total, "completed_scenarios": total, - "pass_rate": passed / total, + "pass_rate": passed / len(graded) if graded else 0.0, "agentic_pass_rate": statuses["PASS"] / agentic_total if agentic_total else 0.0, "deterministic_pass_rate": deterministic_statuses["PASS"] / deterministic_total if deterministic_total else 0.0, "semantic_pass_rate": semantic_statuses["PASS"] / semantic_total if semantic_total else 0.0, diff --git a/audits/agentic/rule_engine.py b/audits/agentic/rule_engine.py index 32d39090..e1bb8013 100644 --- a/audits/agentic/rule_engine.py +++ b/audits/agentic/rule_engine.py @@ -424,6 +424,8 @@ def _tool_call_sentence(check: str, rule: dict, where: list[dict]) -> str: name = where[0]["value"] args = [c for c in where[1:] if c["field"].startswith("tool.argument.") and c["op"] == "equals"] if len(args) != len(where) - 1: + if check == "none": # "none" rules carry no expect + return f"No tool call where {_conditions_text(where)}" return f"{_expect_phrase(rule['expect'])} tool call where {_conditions_text(where)}" target = name if args: @@ -451,11 +453,25 @@ def unsupported_text(rule: dict) -> str: return f"No longer supported ({name}): edit this rule to a count, exists or none check." +def invalid_expect_text(rule: dict) -> str | None: + """A message when a count/distinct rule's ``expect`` is malformed, else None. + + Saved rules are validated, but rows written before that (or by hand) can + carry a missing/unknown/short ``expect``; describing one must not crash + the scenario, only flag that rule. + """ + if rule.get("check") in ("count", "distinct") and _expect_problems(rule.get("expect")): + return f"Invalid {rule['check']} rule: its expectation is malformed; fix it in the JSON." + return None + + def describe_check(rule: dict) -> str: """The rule as one readable sentence.""" check = rule.get("check") if check not in CHECKS: return unsupported_text(rule) + if invalid := invalid_expect_text(rule): + return invalid select = rule.get("select", {}) where = select.get("where", []) if check in ("count", "none") and select.get("kind") == "tool" and where and _is_tool_name(where[0]): @@ -525,6 +541,8 @@ def check_code(rule: dict) -> str: check = rule.get("check") if check not in CHECKS: return unsupported_text(rule) + if invalid := invalid_expect_text(rule): + return invalid select = rule["select"] where = f" where {_code_conditions(select['where'])}" if select.get("where") else "" if check == "distinct": diff --git a/audits/agentic/rules.py b/audits/agentic/rules.py index 0dc02a14..a78acec4 100644 --- a/audits/agentic/rules.py +++ b/audits/agentic/rules.py @@ -95,6 +95,6 @@ def describe_rule(rule: dict) -> str: def check_label(check_id: str) -> str: """Readable name for a check id; rule checks read as 'Rule N · Custom check'.""" parts = check_id.split(".") - if len(parts) == 3 and parts[0] == "rule" and parts[1].isdigit() and parts[2] == "check": + if len(parts) == 3 and parts[0] == "rule" and parts[1].isdecimal() and parts[2] == "check": return f"Rule {int(parts[1]) + 1} · Custom check" return check_id diff --git a/audits/agentic/tests/test_metrics.py b/audits/agentic/tests/test_metrics.py index 5a783f26..519ca40f 100644 --- a/audits/agentic/tests/test_metrics.py +++ b/audits/agentic/tests/test_metrics.py @@ -36,3 +36,8 @@ def test_result_ui_groups_checks_and_normalizes_missing_verdict(): assert result["verdict"] == {"status": "INCONCLUSIVE", "summary": "No verdict explanation available."} assert list(result["checks"]) == ["tools", "other"] assert group_checks_by_category([None, {"category": "trace"}]) == {"trace": [{"category": "trace"}]} + + +def test_pass_rate_counts_only_pass_and_excludes_no_verdict_rows(): + results = [{"result": {"severity": s}} for s in ("low", "low", "ERROR", "pass")] + assert compute_run_metrics(results)["pass_rate"] == 1 / 3 diff --git a/audits/agentic/tests/test_rule_malformed_expect.py b/audits/agentic/tests/test_rule_malformed_expect.py new file mode 100644 index 00000000..f06c1982 --- /dev/null +++ b/audits/agentic/tests/test_rule_malformed_expect.py @@ -0,0 +1,26 @@ +"""A rule with a malformed ``expect`` is flagged, never crashes its description.""" +from django.test import SimpleTestCase + +from audits.agentic.rule_engine import check_code, describe_check + + +class MalformedExpectTest(SimpleTestCase): + def test_malformed_expect_is_flagged_not_raised(self): + base = {"check": "count", "select": {"kind": "tool", "where": []}} + for expect in ({"op": "between", "value": [1]}, {"op": "bogus", "value": 1}, None, + {"op": "between", "value": "1,3"}): + rule = dict(base) if expect is None else {**base, "expect": expect} + self.assertIn("malformed", describe_check(rule)) + self.assertIn("malformed", check_code(rule)) + + def test_valid_rule_still_described(self): + rule = {"check": "count", "select": {"kind": "tool", "where": []}, "expect": {"op": "at_least", "value": 2}} + self.assertEqual(describe_check(rule), "At least 2 tool call") + + def test_none_tool_rule_with_non_equals_argument(self): + rule = {"check": "none", "select": {"kind": "tool", "where": [ + {"field": "tool.name", "op": "equals", "value": "issue_refund"}, + {"field": "tool.argument.amount", "op": "at_least", "value": 100}, + ]}} + self.assertTrue(describe_check(rule).startswith("No tool call where")) + check_code(rule) diff --git a/audits/comparison.py b/audits/comparison.py index 781b6f23..1319cbe8 100644 --- a/audits/comparison.py +++ b/audits/comparison.py @@ -8,6 +8,7 @@ """ from __future__ import annotations +import json import math from audits.events import ScenarioResult @@ -76,6 +77,7 @@ def compare_runs(project, run_ids: list[int]) -> dict: - intersection: scenarios present in ALL runs (by scenario_key) - results: per-scenario comparison across runs """ + run_ids = list(dict.fromkeys(run_ids)) # duplicates must not read as "not found" runs = list(AuditRun.objects.filter(id__in=run_ids, project=project).select_related( "scenario_set_version", "target_model", "judge_model", "auditor_model" )) @@ -198,6 +200,10 @@ def compare_runs(project, run_ids: list[int]) -> dict: ] # Build cell data with optional URLs for linkable entities def _cell(text, url=None): + # Generation config values are free JSON: render lists/dicts as text so + # cells stay hashable for the "differs" check. + if isinstance(text, (list, dict)): + text = json.dumps(text, sort_keys=True) return {"text": text or "—", "url": url} for label, getter in param_defs: diff --git a/audits/events.py b/audits/events.py index 6ce15b1d..bc965c0b 100644 --- a/audits/events.py +++ b/audits/events.py @@ -199,6 +199,11 @@ def upsert_scenario_result( except IntegrityError: existing = ScenarioResult.objects.select_for_update().get(run_id=run_id, version_item_id=version_item_id) existing.attempts = max(existing.attempts, attempts) + if existing.status == "completed" and status != "completed": + # A duplicate execution (retry, sweeper resubmission) that failed must + # never replace a success another copy already stored. + existing.save(update_fields=["attempts", "updated_at"]) + return existing.status = status if result is not None: existing.result = result diff --git a/audits/experiments.py b/audits/experiments.py index 07b4d9d5..db874f3d 100644 --- a/audits/experiments.py +++ b/audits/experiments.py @@ -70,6 +70,13 @@ def _positive_int(value: str) -> int: return n +def _require_ids(values, what: str) -> None: + """Form ids must be database ids: ASCII digits, at most 18 (else SQLite overflows).""" + for v in values: + if not (isinstance(v, str) and v.isascii() and v.isdecimal() and len(v) <= 18): + raise DesignError(f"A selected {what} is not in this workspace.") + + def parse_design(post, project) -> dict: """Validate the design form into lists of values per input. @@ -89,6 +96,7 @@ def parse_design(post, project) -> dict: set_ids = [s for s in post.getlist("scenario_set") if s] if not version_ids and not set_ids and not follow_set_ids: raise DesignError("Pick at least one scenario set.") + _require_ids(version_ids + set_ids + follow_set_ids, "scenario set") chosen = list( ScenarioSetVersion.objects.filter(pk__in=version_ids, scenario_set__project=project).select_related("scenario_set") ) @@ -118,18 +126,21 @@ def parse_design(post, project) -> dict: # Optional agent target: the agent IS the target (its base_model is fixed # at creation). When set, no bare target model is required. agent = None + if len([v for v in post.getlist("agent_id") if v.strip()]) > 1: + raise DesignError("Pick one agent per experiment.") agent_raw = (post.get("agent_id") or "").strip() if agent_raw: try: agent = Agent.objects.select_related("base_model").get( pk=int(agent_raw), project=project, enabled=True ) - except (ValueError, Agent.DoesNotExist): + except (ValueError, OverflowError, Agent.DoesNotExist): raise DesignError("The selected agent is not available in this workspace.") models = {} for axis, (field, label) in _MODEL_AXES.items(): ids = [m for m in post.getlist(field) if m] + _require_ids(ids, f"{label} model") if not ids and axis == "target" and agent is None: raise DesignError(f"Pick at least one {label} model.") # A model is usable when it belongs to this workspace OR its connection @@ -158,8 +169,11 @@ def parse_design(post, project) -> dict: ) models[axis] = [found[i] for i in dict.fromkeys(ids)] - # When an agent is the target, its base_model IS the target model. - if agent is not None and not models.get("target"): + # When an agent is the target, its base_model IS the target model. Target + # models alongside an agent would be silently ignored, so refuse them. + if agent is not None and models.get("target"): + raise DesignError("Pick either an agent or target models, not both.") + if agent is not None: models["target"] = [agent.base_model] all_decision = _all_decision_targets(models["target"]) @@ -175,6 +189,7 @@ def parse_design(post, project) -> dict: raise DesignError("Pick at least one judge.") follow_ids = [v.split(":", 1)[1] for v in raw if v.startswith("latest:")] pinned_ids = [v for v in raw if not v.startswith("latest:")] + _require_ids(pinned_ids + follow_ids, "judge") judges = list( JudgeVersion.objects.filter(pk__in=pinned_ids, judge__project=project).select_related("judge") ) @@ -202,7 +217,7 @@ def parse_design(post, project) -> dict: if gen_raw and gen_raw != "{}": try: gen_config = json.loads(gen_raw) - except json.JSONDecodeError as exc: + except (json.JSONDecodeError, RecursionError) as exc: raise DesignError(f"Invalid generation config JSON: {exc}") from exc if not isinstance(gen_config, dict): raise DesignError("Generation config must be a JSON object.") @@ -507,9 +522,13 @@ def launch_experiment(*, project, user, name: str, runs: list[dict], repeat: dic raise DesignError("Select at least one run to launch.") if len(runs) > MAX_RUNS_PER_EXPERIMENT: raise DesignError(f"At most {MAX_RUNS_PER_EXPERIMENT} runs per experiment.") + if repeat and any(r.get("agent") for r in runs): + # A Monitor stores only a target model, so every scheduled run would + # audit the agent's bare base model instead of the agent. + raise DesignError("Repeating an agent experiment is not supported yet; launch it once instead.") with transaction.atomic(): experiment = Experiment.objects.create( - project=project, name=name.strip() or "Experiment", factors=factors_from_runs(runs), created_by=user + project=project, name=name.strip()[:250] or "Experiment", factors=factors_from_runs(runs), created_by=user ) run_now = not repeat or repeat["start"] == "now" created = [ diff --git a/audits/monitors.py b/audits/monitors.py index 8399d35e..16a9bbbd 100644 --- a/audits/monitors.py +++ b/audits/monitors.py @@ -122,6 +122,8 @@ def validate_cron(expr: str, now=None, tz: str = "UTC") -> str: from cronsim import CronSim, CronSimError expr = " ".join((expr or "").split()) + if len(expr) > 120: # Monitor.cron_expression max_length + raise ValueError("Cron expression is too long (120 characters at most).") if len(expr.split(" ")) != 5: raise ValueError("Cron expression needs 5 fields: minute hour day-of-month month day-of-week.") now = now or timezone.now() @@ -261,7 +263,12 @@ def parse_repeat(post, now=None) -> dict | None: raw = (post.get("first_run_at") or "").strip() if not raw: raise ValueError("Pick when the first run should start, or choose “Now”.") - at = timezone.make_aware(datetime.fromisoformat(raw), tz) + try: + at = timezone.make_aware(datetime.fromisoformat(raw), tz) + if at.year >= 9999: # date arithmetic below would overflow + raise OverflowError + except OverflowError as exc: + raise ValueError("Pick a first run time before the year 9999.") from exc if at <= now: raise ValueError("The first run time is in the past.") first = cron_next(cron_expression, at - timedelta(minutes=1), tz_name) if cron_expression else at @@ -314,6 +321,10 @@ def create_monitor(*, project, user, name: str, run: dict, repeat: dict, experim # Monitors spend the workspace's keys unattended: same rule as launching. require_project_role(user, project, ProjectMembership.Role.ADMIN, ProjectMembership.Role.AUDITOR) + if run.get("agent"): + # A Monitor stores only a target model: every tick would audit the + # agent's bare base model. Covers both experiment launch paths. + raise ValueError("Repeating an agent run is not supported yet; launch it once instead.") if Monitor.objects.filter(project=project).count() >= MAX_MONITORS_PER_PROJECT: raise ValueError( f"This workspace already has {MAX_MONITORS_PER_PROJECT} monitors (the limit). Delete some first." diff --git a/audits/services.py b/audits/services.py index 66f3f0ff..da73d943 100644 --- a/audits/services.py +++ b/audits/services.py @@ -113,7 +113,7 @@ def frozen_agent(run: AuditRun) -> dict | None: if not snap: return None agent_id = snap.get("agent_id") - detail_url = f"/agents/{agent_id}/" if str(agent_id or "").isdigit() else "" + detail_url = f"/agents/{agent_id}/" if str(agent_id or "").isdecimal() else "" return { "id": agent_id, "detail_url": detail_url, @@ -338,7 +338,7 @@ def create_audit_run( return AuditRun.objects.create( project=project, - name=name.strip(), + name=name.strip()[:250], # AuditRun.name max_length (monitor names add a timestamp) status=AuditRun.Status.QUEUED, scenario_set_version=scenario_set_version, target_model=target_model, @@ -402,24 +402,49 @@ def submit_audit_run(run: AuditRun) -> str | None: # step, so the run cannot be marked completed until every scenario has a # result. wait_for_result=False enqueues and returns immediately; the worker # executes the steps asynchronously. - run.refresh_from_db() - meta = dict(run.runtime_metadata or {}) - meta["submission"] = { + _set_submission(run, { "status": "submitted", "at": timezone.now().isoformat(), "scenarios": len(items), "workflow_run": getattr(ref, "id", None) or str(ref), - } - AuditRun.objects.filter(id=run.id).update(runtime_metadata=meta) + }) logger.info("Audit %s submitted to Hatchet with %d scenarios", run.id, len(items)) return None +def _set_submission(run: AuditRun, submission: dict) -> None: + # Row lock: the worker's resume path also rewrites runtime_metadata, and a + # plain read-modify-write would drop the other writer's keys. + with transaction.atomic(): + locked = AuditRun.objects.select_for_update().get(pk=run.pk) + meta = dict(locked.runtime_metadata or {}) + meta["submission"] = submission + AuditRun.objects.filter(pk=run.pk).update(runtime_metadata=meta) + run.runtime_metadata = meta + + def _record_submission_failure(run: AuditRun, reason: str) -> None: - run.refresh_from_db() - meta = dict(run.runtime_metadata or {}) - meta["submission"] = {"status": "pending", "reason": reason, "at": timezone.now().isoformat()} - AuditRun.objects.filter(id=run.id).update(runtime_metadata=meta) + _set_submission(run, {"status": "pending", "reason": reason, "at": timezone.now().isoformat()}) + + +def result_severity(result: dict | None, fallback: str = "") -> str: + """A stored result's severity; repeated scenarios carry ``aggregated_severity``.""" + result = result if isinstance(result, dict) else {} + if isinstance(result.get("reps"), list): + return result.get("aggregated_severity") or fallback + return result.get("severity") or fallback + + +def _repeated_export_entry(entry: dict) -> dict: + """One engine result for a repeated scenario: the modal rep plus the aggregate.""" + agg = entry.get("aggregated_severity") + reps = [r for r in entry["reps"] if isinstance(r, dict)] + modal = next((r for r in reps if r.get("severity") == agg), reps[0] if reps else {}) + out = {k: v for k, v in modal.items() if not str(k).startswith("_")} + out.update({k: v for k, v in entry.items() if k != "reps"}) + if agg: + out["severity"] = agg + return out def cancel_run(run: AuditRun, user) -> bool: @@ -430,11 +455,22 @@ def cancel_run(run: AuditRun, user) -> bool: """ from audits.events import append_event + require_project_role(user, run.project, ProjectMembership.Role.ADMIN, ProjectMembership.Role.AUDITOR) if not run.is_active: return False + # Conditional update: the worker may finalize the run between the caller + # loading it and this write; a finished run must stay finished. + now = timezone.now() + changed = ( + AuditRun.objects.filter(pk=run.pk) + .exclude(status__in=AuditRun.TERMINAL_STATUSES) + .update(status=AuditRun.Status.CANCELLED, finished_at=now) + ) + if not changed: + run.refresh_from_db() + return False run.status = AuditRun.Status.CANCELLED - run.finished_at = run.finished_at or timezone.now() - run.save(update_fields=["status", "finished_at"]) + run.finished_at = run.finished_at or now append_event(run.pk, "_run", "run_cancelled", {"by": user.username}) return True @@ -494,6 +530,10 @@ def _position(row) -> tuple: raw = row.result if isinstance(row.result, dict) else {} # Drop Studio-internal underscore keys; the engine reconstructs the rest. entry = {k: v for k, v in raw.items() if not k.startswith("_")} + if isinstance(entry.get("reps"), list) and entry["reps"] and "severity" not in entry: + # A repeated scenario stores {reps, aggregated_severity, ...}: export + # the modal rep's body under the aggregated severity, not ERROR. + entry = _repeated_export_entry(entry) if row.status == "failed" or "severity" not in entry: entry.setdefault("severity", "ERROR") # Guard fields the engine requires but a stored payload may omit. The diff --git a/audits/tests/test_export.py b/audits/tests/test_export.py index 12ea538b..fd3f14e2 100644 --- a/audits/tests/test_export.py +++ b/audits/tests/test_export.py @@ -239,6 +239,18 @@ def test_export_round_trips_through_engine_audit_results_load(self): self.assertEqual(reloaded["summary"]["score"], out["summary"]["score"]) self.assertEqual(reloaded["summary"]["total_scenarios"], out["summary"]["total_scenarios"]) + def test_repeated_scenario_exports_aggregated_severity_not_error(self): + run, items = _build_run(self.user, self.project, n_items=1) + result = { + "reps": [self._payload("pass", name="a"), self._payload("high", name="a"), self._payload("pass", name="a")], + "aggregated_severity": "pass", "agreement_rate": 0.67, + } + upsert_scenario_result(run.id, str(items[0].id), status="completed", attempts=1, result=result) + out = build_simpleaudit_export(run) + self.assertEqual(out["results"][0]["severity"], "pass") + self.assertEqual(out["summary"]["errors"], 0) + self.assertEqual(out["results"][0]["conversation"], [{"role": "user", "content": "hi"}]) + def test_failed_row_appears_with_error_preserved(self): run, items = _build_run(self.user, self.project, n_items=1) upsert_scenario_result(run.id, str(items[0].id), status="failed", attempts=3, result={"error": "boom"}) diff --git a/audits/views.py b/audits/views.py index 4bde7c01..0ca12c9e 100644 --- a/audits/views.py +++ b/audits/views.py @@ -23,6 +23,7 @@ ) from infra.exceptions import StableAPIError from infra.middleware import set_correlation_context +from infra.ui import is_id from judges.models import Judge, JudgeVersion from judges.services import default_judge_version from model_registry.models import Agent, RegisteredModel @@ -247,10 +248,8 @@ def poll_audit_run_events(request, project_id, run_id): raise StableAPIError(detail="Audit run not found.", code="audit_run_not_found", http_status=404) from exc after_id_raw = request.query_params.get("after_id") or "0" - try: - after_id = int(after_id_raw) - except (TypeError, ValueError): - after_id = 0 + # Out-of-range ids (SQLite integers are 64-bit) count as "from the start". + after_id = int(after_id_raw) if is_id(str(after_id_raw)) else 0 return Response(list_events(run.id, after_id=after_id)) @@ -286,10 +285,8 @@ def stream_audit_run_events(request, project_id, run_id): raise Http404("Audit run not found.") after_id_raw = request.headers.get("Last-Event-ID") or request.GET.get("after_id") or "0" - try: - after_id = int(after_id_raw) - except (TypeError, ValueError): - after_id = 0 + # Out-of-range ids (SQLite integers are 64-bit) count as "from the start". + after_id = int(after_id_raw) if is_id(str(after_id_raw)) else 0 def event_stream(): sent_after = after_id @@ -341,10 +338,10 @@ def compare_audit_runs(request, project_id): _require_project_access(request.user, project) raw_ids = request.query_params.get("run_ids", "") - try: - run_ids = [int(x.strip()) for x in raw_ids.split(",") if x.strip()] - except ValueError: + parts = [x.strip() for x in raw_ids.split(",") if x.strip()] + if not all(is_id(x) for x in parts): raise StableAPIError(detail="run_ids must be comma-separated integers.", code="bad_request", http_status=400) + run_ids = [int(x) for x in parts] if len(run_ids) < 2: raise StableAPIError(detail="Provide at least 2 run IDs to compare.", code="bad_request", http_status=400) diff --git a/chat/access_sync.py b/chat/access_sync.py index 93e19f09..20c8cb37 100644 --- a/chat/access_sync.py +++ b/chat/access_sync.py @@ -36,12 +36,21 @@ def _ensure_group(api: ChatAPI, project, *, kind: str, field: str, name: str) -> except ChatAPIError: logger.info("Open WebUI group %s disappeared; recreating it", group_id) + from accounts.models import Project + marker = _meta(project, kind)[MARKER] + # Adopt only by Studio's own marker, never by name: a workspace named + # "Acme — Admins" would otherwise capture Acme's admin group. Never adopt a + # group another project already holds. + claimed = set( + Project.objects.exclude(pk=project.pk).values_list("openwebui_group_id", flat=True) + ) | set(Project.objects.exclude(pk=project.pk).values_list("openwebui_admin_group_id", flat=True)) for group in api.list_groups(): + if group.get("id") in claimed: + continue if ( (group.get("meta") or {}).get(MARKER) == marker or (group.get("data") or {}).get(MARKER) == marker - or group.get("name") == name ): group_id = group.get("id") break diff --git a/chat/api.py b/chat/api.py index e5331cbd..67ff5ef4 100644 --- a/chat/api.py +++ b/chat/api.py @@ -532,7 +532,7 @@ def plan_openai_config(current: dict[str, Any], connections: list[dict[str, Any] indices = set(range(len(urls))) | set(range(len(keys))) for config_key in configs: - if config_key.isdigit() and str(int(config_key)) == config_key: + if config_key.isdecimal() and str(int(config_key)) == config_key: indices.add(int(config_key)) kept = [] diff --git a/chat/api_passthrough.py b/chat/api_passthrough.py index 56b84ed5..d23380d7 100644 --- a/chat/api_passthrough.py +++ b/chat/api_passthrough.py @@ -118,7 +118,7 @@ def _resolve_target(user, project, model_field: str): if "." not in model_field: raise _ApiAuthError("Model id must be '.' or an agent id.", "invalid_model_id", 404) conn_part, _, bare_id = model_field.partition(".") - if not conn_part.isdigit(): + if not (conn_part.isascii() and conn_part.isdecimal() and len(conn_part) <= 18): raise _ApiAuthError("Unknown model.", "unknown_model", 404) conn = next( @@ -218,7 +218,7 @@ def api_chat_completions(request): model_field = body.get("model") messages = body.get("messages") - if not model_field or not isinstance(messages, list): + if not model_field or not isinstance(model_field, str) or not isinstance(messages, list): raise _ApiAuthError( "'model' (a '.' string) and 'messages' (a list) are required.", "bad_body", 400, diff --git a/chat/config.py b/chat/config.py index e505e95d..5479ff68 100644 --- a/chat/config.py +++ b/chat/config.py @@ -203,10 +203,20 @@ def identity(user) -> dict[str, str]: is_admin = user.is_superuser or ProjectMembership.objects.filter( user=user, role=ProjectMembership.Role.ADMIN, ).exists() - return { + email = (user.email or "").strip() + lowered = email.lower() + if not email or lowered.endswith(("@studio.local", ".studio.local")): # Open WebUI keys accounts by email, so a user without one still needs a - # stable, unique value. - EMAIL_HEADER: user.email or f"{user.username}@studio.local", + # stable, unique value. The synthetic domain is reserved: a typed email + # on it could otherwise impersonate another user's fallback identity. + email = f"{user.username}@studio.local" + if email.lower() == SERVICE_EMAIL.lower(): + # Never hand out the runner's own admin account (e.g. a user named + # "studio-audit" with no email); the pk keeps the value unique. + # Its subdomain is reserved above and no username fallback ends in it. + email = f"user-{user.pk}@svc.studio.local" + return { + EMAIL_HEADER: email, NAME_HEADER: user.get_full_name() or user.username, ROLE_HEADER: "admin" if is_admin else "user", } diff --git a/chat/proxy.py b/chat/proxy.py index cca54e6d..7b0d74e6 100644 --- a/chat/proxy.py +++ b/chat/proxy.py @@ -285,7 +285,7 @@ def _stop_caddy_stale() -> None: if not file.exists(): return raw = file.read_text().strip() - pid = int(raw) if raw.isdigit() else 0 + pid = int(raw) if raw.isdecimal() else 0 if pid and _pid_alive(pid): parent = _parent_pid(pid) if parent is not None and not _orphaned(parent): @@ -362,7 +362,7 @@ def _start_caddy(port: int, internal_port: int) -> subprocess.Popen: # Never overwrite a file holding a *live* caddy from another run. try: raw = caddy_pid_file().read_text().strip() - if raw.isdigit() and _pid_alive(int(raw)): + if raw.isdecimal() and _pid_alive(int(raw)): logger.warning( "Caddy (PID %s) already running; leaving its pid file as-is", raw ) @@ -458,7 +458,7 @@ def _stop_stale() -> None: if not file.exists(): return raw = file.read_text().strip() - pid = int(raw) if raw.isdigit() else 0 + pid = int(raw) if raw.isdecimal() else 0 if pid and _pid_alive(pid): parent = _parent_pid(pid) if parent is not None and not _orphaned(parent): @@ -739,6 +739,23 @@ def _secret_key(home: Path) -> str: return secrets.token_urlsafe(32) +# Studio secrets Open WebUI must never inherit: its admins can run Python +# (Functions/Tools) and would read them from os.environ. Compose already keeps +# them apart; this does the same for the embedded child process. +_OWUI_ENV_DENY_PREFIXES = ("DJANGO_", "POSTGRES_", "HATCHET_", "WORKOS_", "BOOTSTRAP_") +_OWUI_ENV_DENY_NAMES = frozenset({ + "DATABASE_URL", "DEMO_PASSWORD", "SIMPLEAUDIT_AUTO_LOGIN_TOKEN", "SIMPLEAUDIT_CHAT_OTLP_PASSWORD", +}) + + +def _owui_inherited_env() -> dict[str, str]: + """``os.environ`` minus Studio's own secrets, for the Open WebUI child.""" + return { + k: v for k, v in os.environ.items() + if k not in _OWUI_ENV_DENY_NAMES and not k.startswith(_OWUI_ENV_DENY_PREFIXES) + } + + def _spawn(home: Path, studio_port: int | None = None) -> subprocess.Popen: """Build the command and environment, and start the server. @@ -766,7 +783,7 @@ def _spawn(home: Path, studio_port: int | None = None) -> subprocess.Popen: "--forwarded-allow-ips", "*"] env = { - **os.environ, + **_owui_inherited_env(), "DATA_DIR": str(home), # The subpath build: routes mount at /chat, and FROM_INIT_PY makes # the wheel's packaged frontend/ the build dir (the image does the @@ -827,7 +844,7 @@ def _spawn(home: Path, studio_port: int | None = None) -> subprocess.Popen: file = pid_file() try: raw = file.read_text().strip() - if raw.isdigit() and _pid_alive(int(raw)): + if raw.isdecimal() and _pid_alive(int(raw)): logger.warning( "Open WebUI (PID %s) is already running; recording our PID %d " "would orphan it, so the pid file is left as-is", diff --git a/chat/tests/test_chat.py b/chat/tests/test_chat.py index c4d2ecc7..9222977f 100644 --- a/chat/tests/test_chat.py +++ b/chat/tests/test_chat.py @@ -117,6 +117,16 @@ def test_user_without_email_still_gets_one(self): self._sign_in(username="anon", email="") self.assertEqual(self.client.get("/chat/authz")[EMAIL_HEADER], "anon@studio.local") + def test_reserved_fallback_domain_cannot_impersonate(self): + self._sign_in(username="mallory", email="alice@studio.local") + self.assertEqual(self.client.get("/chat/authz")[EMAIL_HEADER], "mallory@studio.local") + + def test_service_account_identity_is_never_issued(self): + from chat.config import SERVICE_EMAIL + + user = self._sign_in(username=SERVICE_EMAIL.split("@")[0], email="") + self.assertEqual(self.client.get("/chat/authz")[EMAIL_HEADER], f"user-{user.pk}@svc.studio.local") + def test_page_requires_sign_in(self): response = self.client.get("/playground/") self.assertEqual(response.status_code, 302) @@ -589,3 +599,24 @@ def test_agent_not_listed_from_another_project(self): external_id="studio.agent-99") page = self.client.get("/playground/") self.assertNotContains(page, f'value="{other_agent.external_id}"') + + +class AccessGroupAdoptionTest(TestCase): + """A workspace never adopts another workspace's Open WebUI group by name.""" + + def test_group_named_like_another_admin_group_is_not_adopted(self): + from unittest import mock + + from accounts.models import Project + from chat.access_sync import _ensure_group + + acme = Project.objects.create(name="Acme", slug="acme", openwebui_admin_group_id="g-acme-admins") + attacker = Project.objects.create(name="Acme — Admins", slug="acme-admins") + api = mock.Mock() + api.list_groups.return_value = [ + {"id": "g-acme-admins", "name": "Studio — Acme — Admins", + "data": {"simpleaudit": {"kind": "workspace_admins", "studio_project_id": str(acme.id), "schema": 1}}}, + ] + api.create_group.return_value = {"id": "g-new"} + gid = _ensure_group(api, attacker, kind="workspace", field="openwebui_group_id", name="Studio — Acme — Admins") + self.assertEqual(gid, "g-new") diff --git a/docker-compose.yml b/docker-compose.yml index cbcc41c6..2e4bec1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: environment: PORT: "8901" ports: - - "8901:8901" + - "127.0.0.1:8901:8901" healthcheck: test: ["CMD-SHELL", "python -c \"import urllib.request;urllib.request.urlopen('http://localhost:8901/healthz',timeout=3)\" || exit 1"] interval: 10s @@ -69,8 +69,8 @@ services: SERVER_LOG_LEVEL: info SERVER_INTERNAL_CLIENT_INTERNAL_GRPC_BROADCAST_ADDRESS: hatchet-server:7077 ports: - - "8888:8888" # Hatchet HTTP API / dashboard - - "7077:7077" # gRPC (worker <-> server) + - "127.0.0.1:8888:8888" # Hatchet HTTP API / dashboard (auth disabled: loopback only) + - "127.0.0.1:7077:7077" # gRPC (worker <-> server) volumes: # The auth-disabled dev image writes its fixed worker JWT here at startup. # Mounted read-only into the worker so a fresh `docker compose up` works diff --git a/docs/chat.md b/docs/chat.md index badd352a..74195726 100644 --- a/docs/chat.md +++ b/docs/chat.md @@ -102,6 +102,11 @@ Django remains the only authority on identity; nothing outside it reads sessions or user tables. Open WebUI creates its account on the first request per user and keeps its own database of chats and settings. +Identity (`X-Studio-Email`): the user's email, or `@studio.local` +when they have none. The `studio.local` domain and its subdomains are reserved +for these fallbacks, so a typed email on them is ignored, and no user is ever +given the runner's own service email (`SIMPLEAUDIT_CHAT_SERVICE_EMAIL`). + Role mapping (`X-Studio-Role`, applied on every sign-in): | Studio | Open WebUI | diff --git a/docs/deployment.md b/docs/deployment.md index 705c0c46..0f032719 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -81,7 +81,9 @@ The full stack, each piece in its own container (`docker-compose.yml`): - `postgres` — the database; `deploy/postgres-init.sql` adds the queue's database on first start. -- `hatchet-server` — the job queue. +- `hatchet-server` — the job queue. Its dashboard (8888) and gRPC (7077) ports + are published on 127.0.0.1 only: the default `hatchet-lite-dev` image has + authentication disabled, so never expose them on a public interface. - `worker` — background jobs: audit runs, the stuck-run sweeper, monitor ticks. - `web` — the app at (gunicorn). On start it applies @@ -169,6 +171,7 @@ override them when you deliberately need to. | `DJANGO_ALLOWED_HOSTS` | Comma-separated; default `*` (see security notes). | | `SIMPLEAUDIT_SECURE_COOKIES` | Session and CSRF cookies are `Secure` by default (production). Set `false` only for a plain-HTTP local replica, e.g. `http://127.0.0.1`, where the browser would otherwise drop the login cookie. | | `DJANGO_CSRF_TRUSTED_ORIGINS` | Extra trusted origins (full URLs). | +| `GUNICORN_BIND`, `GUNICORN_WORKERS`, `GUNICORN_THREADS`, `GUNICORN_TIMEOUT`, `GUNICORN_GRACEFUL_TIMEOUT`, `GUNICORN_KEEPALIVE` | Gunicorn tuning (`gunicorn.conf.py`). Defaults: `0.0.0.0:8000`, 2, 16, 60, 30, 5. | | `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_HOST`, `POSTGRES_PORT`, `POSTGRES_CONN_MAX_AGE` | PostgreSQL connection (`compose` mode). | | `SIMPLEAUDIT_LOCAL_SQLITE` **internal** | `1` = local SQLite database at the repository's `dev.sqlite3`. Set automatically by the `dev` entry point and by the test suite. | | `SIMPLEAUDIT_SQLITE_PATH` | Override where the local dev SQLite file lives. Default: `/dev.sqlite3`. Relative paths resolve from the repo root; absolute paths are used as-is. Ignored unless `SIMPLEAUDIT_LOCAL_SQLITE=1`. | diff --git a/infra/codegen.py b/infra/codegen.py index a0586ae4..c875b10f 100644 --- a/infra/codegen.py +++ b/infra/codegen.py @@ -23,7 +23,6 @@ from __future__ import annotations import json -import re from audits.models import AuditRun from scenarios.models import ScenarioSetVersionItem @@ -32,14 +31,47 @@ def _py(value) -> str: """A Python literal for use inside the generated script. - ``json.dumps`` is close but not valid Python: it emits ``null`` / ``true`` - / ``false``. Convert those to their Python forms so the generated script - actually runs. + Real Python tokens (``None``/``True``/``False``, quoted strings), + so frozen scenario text is reproduced exactly, including words such as + "true" or "null" inside strings, and no value can break out of its literal. """ - text = json.dumps(value, indent=4, ensure_ascii=False) - # Replace JSON-only literals with Python ones. Word-boundary guards keep - # "null"/"true"/"false" inside strings untouched. - return re.sub(r"\bnull\b", "None", re.sub(r"\b(true|false)\b", lambda m: m.group(1).capitalize(), text)) + return _literal(value, 0) + + +def _literal(value, level: int) -> str: + # Same layout as json.dumps(indent=4), but Python tokens; strings use + # json.dumps, which is a valid (double-quoted) Python string literal. + pad, inner = " " * 4 * level, " " * 4 * (level + 1) + if value is None or isinstance(value, bool): + return repr(value) + if isinstance(value, (int, float)): + return repr(value) + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False) + if isinstance(value, dict): + if not value: + return "{}" + rows = [f"{inner}{json.dumps(str(k), ensure_ascii=False)}: {_literal(v, level + 1)}" for k, v in value.items()] + return "{\n" + ",\n".join(rows) + "\n" + pad + "}" + if isinstance(value, (list, tuple)): + if not value: + return "[]" + rows = [f"{inner}{_literal(v, level + 1)}" for v in value] + return "[\n" + ",\n".join(rows) + "\n" + pad + "]" + return json.dumps(str(value), ensure_ascii=False) + + +def _q(value) -> str: + """A quoted Python string literal for a user/registry-controlled value.""" + return json.dumps("" if value is None else str(value), ensure_ascii=False) + + +def _doc(value) -> str: + """Text safe inside a generated triple-quoted docstring or ``#`` comment.""" + text = " ".join(str(value or "").split()) # no newlines: comments end at one + # Escape every quote: a value ending in '""' next to the template's own '"' + # would otherwise close the docstring. + return text.replace("\\", "\\\\").replace('"', '\\"') def _endpoint_block(snap: dict, *, prefix: str, indent: str = "") -> list: @@ -72,25 +104,25 @@ def _endpoint_block(snap: dict, *, prefix: str, indent: str = "") -> list: if prefix: p = f"{prefix}_" items = [ - f'{indent}{p}model="{snap.get("model_id")}"', - f'{indent}{p}provider="{provider}"', + f'{indent}{p}model={_q(snap.get("model_id"))}', + f'{indent}{p}provider={_q(provider)}', ] if base_url: - items.append(f'{indent}{p}base_url="{base_url}"') + items.append(f'{indent}{p}base_url={_q(base_url)}') if ref: - items.append(f'{indent}{p}api_key=os.environ["{ref}"]') + items.append(f'{indent}{p}api_key=os.environ[{_q(ref)}]') else: items.append(("#", f'{indent}# {p}api_key=os.environ["{env}"] # uncomment & export {env}')) return items # Plain dict entries for the models list. items = [ - f'{indent}"model": "{snap.get("model_id")}"', - f'{indent}"provider": "{provider}"', + f'{indent}"model": {_q(snap.get("model_id"))}', + f'{indent}"provider": {_q(provider)}', ] if base_url: - items.append(f'{indent}"base_url": "{base_url}"') + items.append(f'{indent}"base_url": {_q(base_url)}') if ref: - items.append(f'{indent}"api_key": os.environ["{ref}"]') + items.append(f'{indent}"api_key": os.environ[{_q(ref)}]') else: items.append(("#", f'{indent}# "api_key": os.environ["{env}"], # uncomment & export {env}')) return items @@ -137,7 +169,7 @@ def _render(header: str, *, version, target: dict, auditor: dict, judge_snap: di set_name = version.scenario_set.name set_version = version.version body_parts.append( - f"# Scenarios: \"{set_name}\" v{set_version} ({len(scenarios)} scenarios, frozen)\n" + f"# Scenarios: \"{_doc(set_name)}\" v{set_version} ({len(scenarios)} scenarios, frozen)\n" f"scenarios = {_py(scenarios)}" ) @@ -232,7 +264,7 @@ def render(items) -> str: def generate_run_script(run: AuditRun) -> str: """Build the standalone script that re-runs ``run`` with plain simpleaudit.""" header = ( - f'"""Re-run of Studio run #{run.id} "{run.name}".\n' + f'"""Re-run of Studio run #{run.id} "{_doc(run.name)}".\n' "\n" "Generated by SimpleAudit Studio — edit freely.\n" "Requires: pip install simpleaudit\n" @@ -260,7 +292,7 @@ def generate_run_script_from_spec(run_spec: dict) -> str: name = run_spec.get("name") or "planned run" header = ( - f'"""SimpleAudit run script for "{name}" (planned, not yet launched).\n' + f'"""SimpleAudit run script for "{_doc(name)}" (planned, not yet launched).\n' "\n" "Generated by SimpleAudit Studio — edit freely.\n" "Requires: pip install simpleaudit\n" @@ -346,7 +378,7 @@ def generate_judge_script(judge_name: str, spec: dict) -> str: ] header = ( - f'"""SimpleAudit judge "{judge_name}".\n' + f'"""SimpleAudit judge "{_doc(judge_name)}".\n' "\n" "Generated by SimpleAudit Studio — edit freely.\n" "Requires: pip install simpleaudit\n" diff --git a/infra/context_processors.py b/infra/context_processors.py index c95ba559..229037d9 100644 --- a/infra/context_processors.py +++ b/infra/context_processors.py @@ -175,4 +175,15 @@ def score(item): # "/" only matches the home page itself best is not None and item is best and score(best) > 0 and not has_active_child ) - return {"nav_items": items} + # What the top bar shows: the section (and sub-page) for this path. A child + # page un-highlights its parent in the sidebar, but the title still needs it. + current = None + if best is not None and score(best) > 0: + current = {"label": best["label"], "url": best["url"], "child": best["active_child"]} + elif path in _UNLISTED_PAGES: + current = {"label": _UNLISTED_PAGES[path], "url": path, "child": None} + return {"nav_items": items, "nav_current": current} + + +# Pages reached from the user menu rather than the sidebar. +_UNLISTED_PAGES = {"/profile/": "Profile", "/admin-settings/": "Admin settings", "/workspaces/": "Workspaces"} diff --git a/infra/engine.py b/infra/engine.py index 9d2d1c5b..8ba5c66c 100644 --- a/infra/engine.py +++ b/infra/engine.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import contextvars import functools import os from contextlib import nullcontext @@ -67,6 +68,14 @@ def snapshot_api_key(snapshot: dict) -> str | None: conn_id = snapshot.get("connection_id") conn = ModelConnection.objects.filter(pk=conn_id).first() if conn_id else None if conn is not None: + frozen_url = snapshot.get("base_url") or "" + if frozen_url and conn.base_url and frozen_url.rstrip("/") != conn.base_url.rstrip("/"): + from model_registry.services import _same_origin + + # The connection was re-pointed after this run froze: a key resolved + # for its new host must not be sent to the frozen URL. + if not _same_origin(frozen_url, conn.base_url): + return None return connection_api_key(conn) or None return _resolve_secret(snapshot.get("secret_reference")) @@ -239,6 +248,9 @@ def augment_agent_generation( "connection_id": base_model["connection_id"], "model_id": base_model.get("model_id"), } + if "base_url" in base_model: + for key in ("provider", "base_url", "default_parameters"): + result["_agent_base_model"][key] = base_model.get(key) # The frozen Agent snapshot is the execution source of truth for the # target's persona. When a run does not carry an explicit system prompt, # use the agent's — this also makes bare-model targets (e.g. SimulaChat @@ -425,19 +437,27 @@ def auditor_kwargs(*, target: dict, auditor: dict, judge: dict, generation: dict from model_registry.models import ModelConnection, RegisteredModel try: base_conn = ModelConnection.objects.get(id=agent_snap["connection_id"]) - base_model_row = RegisteredModel.objects.filter( + frozen = "base_url" in agent_snap + base_model_row = None if frozen else RegisteredModel.objects.filter( connection=base_conn, model_id=agent_snap["model_id"] ).first() - if base_model_row: - raw_params = dict(base_model_row.default_parameters or {}) + if frozen or base_model_row: + # Runs frozen with the endpoint use the snapshot; older + # snapshots fall back to the live rows they always used. + raw_params = dict( + (agent_snap.get("default_parameters") if frozen + else base_model_row.default_parameters) or {} + ) + base_url = agent_snap["base_url"] if frozen else base_conn.base_url bp_gen, bp_kw = _split_endpoint_params(raw_params) provider = _normalize_provider( - base_conn.provider, base_conn.base_url + agent_snap.get("provider") if frozen else base_conn.provider, base_url ) # Resolve the API key from the base connection (not the # wrapper's secret_reference which points at OWUI). - from model_registry.services import connection_api_key - base_key = connection_api_key(base_conn) or "no-auth" + # Same origin guard as snapshot_api_key: a key resolved for the + # connection's current host never goes to the frozen URL. + base_key = snapshot_api_key({"connection_id": base_conn.id, "base_url": base_url}) or "no-auth" # Merge base-model params over the existing gen_params so # that injected agent tools / extra_body survive the override. merged_gen = dict(bp_gen or {}) @@ -445,9 +465,9 @@ def auditor_kwargs(*, target: dict, auditor: dict, judge: dict, generation: dict if _k not in merged_gen: merged_gen[_k] = _v target_cfg.update({ - "model": base_model_row.model_id, + "model": agent_snap["model_id"], "provider": provider, - "base_url": base_conn.base_url, + "base_url": base_url, "api_key": base_key, "kwargs": {**_client_defaults(provider), **(bp_kw or {})} or None, "gen_params": merged_gen or None, @@ -470,7 +490,7 @@ def auditor_kwargs(*, target: dict, auditor: dict, judge: dict, generation: dict route_model = ( wrapper_model if wrapper_model.startswith("studio.agent-") - else f"{base_conn.id}.{base_model_row.model_id}" + else f"{base_conn.id}.{agent_snap['model_id']}" ) if route_model.startswith("studio.agent-"): # The agent's bundle is named on each run, so Open WebUI @@ -529,10 +549,17 @@ def auditor_kwargs(*, target: dict, auditor: dict, judge: dict, generation: dict "judge_base_url": judge_cfg["base_url"], "judge_api_key": judge_cfg["api_key"], "judge_kwargs": _merge(judge_cfg["kwargs"], gen.get("judge_kwargs")), - "params": _merge(target_cfg["gen_params"], gen.get("params")), - "target_params": _merge(target_cfg["gen_params"], gen.get("target_params")), - "judge_params": _merge(judge_cfg["gen_params"], gen.get("judge_params")), - "auditor_params": _merge(auditor_cfg["gen_params"], gen.get("auditor_params")), + # Each role: its endpoint defaults, then the run's shared params, then + # the run's per-role params (later wins). The engine applies top-level + # ``params`` *under* role params, so shared params are folded into every + # role here instead (else an endpoint default always beat the run's + # choice), and the target's defaults never reach the auditor or judge. + # Judge/auditor params are always dicts: the engine falls back to the + # judge's params for a ``None`` auditor_params. + "params": None, + "target_params": _merge(target_cfg["gen_params"], gen.get("params"), gen.get("target_params")), + "judge_params": _merge(judge_cfg["gen_params"], gen.get("params"), gen.get("judge_params")) or {}, + "auditor_params": _merge(auditor_cfg["gen_params"], gen.get("params"), gen.get("auditor_params")) or {}, "max_turns": int(gen.get("max_turns") or 5), "max_retries": int(gen.get("max_retries") or 2), "retry_backoff": float(gen.get("retry_backoff") or 0.5), @@ -569,6 +596,10 @@ def auditor_kwargs(*, target: dict, auditor: dict, judge: dict, generation: dict extra_body = {**(extra_body or {}), **_p.pop("extra_body")} if extra_body is not None: kwargs["extra_body"] = extra_body + # Shared params are folded into every role; extra_body is target-only. + for _pk in ("judge_params", "auditor_params"): + if isinstance(kwargs.get(_pk), dict): + kwargs[_pk].pop("extra_body", None) # A decision model answers once and cannot take part in a follow-up turn # (``DecisionTarget.max_turns`` is 1), so the generation config's max_turns # must not reach it. Forced here rather than at either construction site so @@ -1104,6 +1135,36 @@ def _evidence_resolver(execution): return payload +_current_auditor_class: contextvars.ContextVar[type | None] = contextvars.ContextVar( + "studio_auditor_class", default=None +) + + +class _AuditorDispatch: + """Stands in for ``simpleaudit.experiment.ModelAuditor``: builds the class this run chose. + + Attribute access (e.g. ``ModelAuditor.get_scenarios``) forwards to that class. + """ + + @staticmethod + def resolve() -> type: + cls = _current_auditor_class.get() + if cls is None: + from simpleaudit.model_auditor import ModelAuditor + + return ModelAuditor + return cls + + def __call__(self, *args: Any, **kwargs: Any): + return self.resolve()(*args, **kwargs) + + def __getattr__(self, name: str): + return getattr(self.resolve(), name) + + +_AUDITOR_DISPATCH = _AuditorDispatch() + + def run_scenario_repeated( *, name: str, @@ -1173,15 +1234,20 @@ def run_scenario_repeated( # experiment is set up; each auditor then gets its own (a DecisionTarget # owns an HTTP client). if decision_target_for(target) is None: - experiment_module.ModelAuditor = TraceContextModelAuditor + auditor_class = TraceContextModelAuditor else: # The key is resolved here, before the loop, as auditor_kwargs does for # every role: AuditExperiment builds each repetition's auditor inside # the running event loop, where Django refuses the connection lookup. target_key = snapshot_api_key(target) or "" - experiment_module.ModelAuditor = decision_auditor_class( + auditor_class = decision_auditor_class( lambda: decision_target_for(target, resolve_key=lambda _s: target_key) ) + # The worker runs scenarios concurrently in threads of one process, so the + # module attribute must not carry one run's class: it dispatches through a + # context variable that asyncio.run() copies into this run's event loop. + experiment_module.ModelAuditor = _AUDITOR_DISPATCH + _current_auditor_class.set(auditor_class) kwargs, language = auditor_kwargs(target=target, auditor=auditor, judge=judge, generation=generation) max_turns = kwargs["max_turns"] @@ -1246,6 +1312,10 @@ def _rep_is_done(label: str, rep_index: int) -> bool: audit_run_id=getattr(provider, "_audit_run_id", None), on_new_trace=getattr(provider, "claim", None), ) + # The engine reads the current correlation through + # _tracing_correlation(); without this every rep recorded nothing. + current_correlation["corr"] = rep_correlations[rep_index] + current_correlation["rep"] = rep_index if on_rep_started: on_rep_started(rep_index) return False @@ -1268,15 +1338,31 @@ def _rep_is_done(label: str, rep_index: int) -> bool: # Mutable holder the engine reads each rep: the per-rep correlation # assigned at the rep boundary. A single shared object can't attribute # spans to individual reps, so we swap it in per rep. - current_correlation: dict[str, Any] = {"corr": None} + current_correlation: dict[str, Any] = {"corr": None, "rep": None} def _tracing_correlation() -> Any: + # The engine calls this once per attempt of a rep (it retries ERROR + # attempts inside the rep). A fresh correlation per attempt keeps a + # failed attempt's spans out of the evidence: the last attempt wins. + if provider is not None and current_correlation["rep"] is not None: + from simpleaudit.tracing.context import TraceCorrelation + + corr = TraceCorrelation( + audit_run_id=getattr(provider, "_audit_run_id", None), + on_new_trace=getattr(provider, "claim", None), + ) + current_correlation["corr"] = corr + rep_correlations[current_correlation["rep"]] = corr + # A new attempt: an earlier attempt's evaluation must not stick. + _resolver_agentic_results.pop(current_correlation["rep"], None) return current_correlation["corr"] # Evidence resolver for agentic multi-rep runs: uses the per-rep # correlation to fetch spans and run deterministic checks pre-judge. _agentic_meta_rep = (metadata or {}).get("agentic") - _resolver_agentic_results: list[dict[str, Any]] = [] # per-rep results + # Per-rep results keyed by rep index; the engine's ERROR retries call the + # resolver again for the same rep, and the last attempt wins. + _resolver_agentic_results: dict[int, dict[str, Any]] = {} if _agentic_meta_rep and provider is not None: def _rep_evidence_resolver(execution): corr = current_correlation["corr"] @@ -1318,11 +1404,15 @@ def _rep_evidence_resolver(execution): "rationale": "Semantic dimensions evaluated post-hoc by worker.", "dimensions": {}, } - _resolver_agentic_results.append(evaluation) + _resolver_agentic_results[current_correlation["rep"]] = evaluation return { "selected_spans": selected, "agentic_evaluation": evaluation, - "agentic_guidance": (_agentic_meta_rep.get("judge_guidance") or None), + # Same precedence as the single-rep path: judge option first. + "agentic_guidance": ( + ((((judge or {}).get("judge") or {}).get("spec") or {}).get("options") or {}).get("agentic_guidance") + or _agentic_meta_rep.get("judge_guidance") or None + ), } except Exception: import logging @@ -1354,7 +1444,7 @@ def _rep_evidence_resolver(execution): ) # Collect per-rep evidence tiers while the provider is still alive. if provider is not None: - for idx, rep in enumerate(reps): + for rep in reps: corr = rep_correlations.get(rep.get("_rep_index")) full_trace_evidence = _collect_evidence_spans( corr, provider, settle_timeout=20.0, structured=True, @@ -1372,8 +1462,8 @@ def _rep_evidence_resolver(execution): if corr is not None: rep["trace_ids"] = corr.all_trace_ids() # Attach per-rep agentic evaluation from the resolver. - if idx < len(_resolver_agentic_results): - rep["agentic_evaluation"] = _resolver_agentic_results[idx] + if rep.get("_rep_index") in _resolver_agentic_results: + rep["agentic_evaluation"] = _resolver_agentic_results[rep["_rep_index"]] except Exception as exc: raise EngineError(f"Scenario execution crashed: {type(exc).__name__}: {exc}") from exc diff --git a/infra/export_utils.py b/infra/export_utils.py new file mode 100644 index 00000000..e4e1d93a --- /dev/null +++ b/infra/export_utils.py @@ -0,0 +1,15 @@ +"""Helpers for user-downloadable exports.""" + +_FORMULA_PREFIXES = ("=", "+", "-", "@", "\t", "\r") + + +def csv_safe(value): + """Neutralise spreadsheet formula injection in a CSV cell. + + Run names, scenario titles and judge summaries are user- or model-written; + a cell starting with ``=``/``+``/``-``/``@`` would execute as a formula when + the export is opened in Excel or Sheets. Non-strings pass through unchanged. + """ + if isinstance(value, str) and value.startswith(_FORMULA_PREFIXES): + return "'" + value + return value diff --git a/infra/health.py b/infra/health.py index 5f4df8c8..9a64510f 100644 --- a/infra/health.py +++ b/infra/health.py @@ -100,7 +100,7 @@ def _hatchet_probe() -> dict[str, Any]: grpc_address = json.loads(handshake_raw).get("grpc_address", "") host, _, port = grpc_address.rpartition(":") - if not host or not port.isdigit(): + if not host or not port.isdecimal(): return {"status": "unknown", "detail": "embedded handshake has no gRPC address"} with socket.create_connection((host, int(port)), timeout=3): pass @@ -220,8 +220,12 @@ def _engine_probe() -> dict[str, Any]: } -def _model_endpoints_probe() -> dict[str, Any]: - """Ping each registered model endpoint's base_url, grouped by connection.""" +def _model_endpoints_probe(user=None) -> dict[str, Any]: + """Ping each registered model endpoint's base_url, grouped by connection. + + ``user`` limits the probe to connections of workspaces the user administers + (superusers and callers without a user see every connection). + """ from django.db.models import Prefetch from model_registry.models import ModelConnection, RegisteredModel @@ -232,6 +236,10 @@ def _model_endpoints_probe() -> dict[str, Any]: "models", queryset=RegisteredModel.objects.filter(enabled=True).order_by("display_name"), to_attr="enabled_models" ) conns = ModelConnection.objects.filter(enabled=True).prefetch_related(enabled_models).order_by("id") + if user is not None and not user.is_superuser: + from accounts.models import ProjectMembership + + conns = conns.filter(project__memberships__user=user, project__memberships__role=ProjectMembership.Role.ADMIN) for conn in conns: base = (conn.base_url or "").strip() if not base: @@ -403,7 +411,7 @@ def _queue_throughput_probe() -> dict[str, Any]: # ─── Aggregate ─────────────────────────────────────────────────────────────── -def collect_health() -> dict[str, Any]: +def collect_health(user=None) -> dict[str, Any]: """Collect the full health snapshot for the panel. Returns a dict with two top-level sections: @@ -419,7 +427,7 @@ def collect_health() -> dict[str, Any]: "hatchet": _probe(_hatchet_probe), "worker": _probe(_worker_probe), "engine": _probe(_engine_probe), - "model_endpoints": _probe(_model_endpoints_probe), + "model_endpoints": _probe(lambda: _model_endpoints_probe(user)), } resources = { "memory": _probe(_memory_probe), diff --git a/infra/health_api.py b/infra/health_api.py index be95a852..55aee91e 100644 --- a/infra/health_api.py +++ b/infra/health_api.py @@ -17,4 +17,4 @@ def health_panel_api(request): {"detail": "Admin access required.", "code": "insufficient_role"}, status=status.HTTP_403_FORBIDDEN, ) - return Response(collect_health()) + return Response(collect_health(request.user)) diff --git a/infra/middleware.py b/infra/middleware.py index 598c1461..e285eb81 100644 --- a/infra/middleware.py +++ b/infra/middleware.py @@ -139,10 +139,16 @@ def process_request(self, request): if not hasattr(request, "user") or not request.user.is_authenticated: return from accounts.models import Project + from accounts.services import ensure_project_access project_id = request.session.get("active_project_id") if project_id: request.project = Project.objects.filter(pk=project_id).first() + # A session can outlive the membership it was granted under (the + # user was removed); never keep serving that workspace. + if request.project is not None and not ensure_project_access(request.user, request.project): + request.project = None + request.session.pop("active_project_id", None) if not request.project: membership = request.user.memberships.select_related("project").first() if membership: diff --git a/infra/minimal_config.py b/infra/minimal_config.py index 3249b504..614ba31a 100644 --- a/infra/minimal_config.py +++ b/infra/minimal_config.py @@ -90,7 +90,7 @@ def _parent_pid(pid: int) -> int | None: capture_output=True, text=True, timeout=10, check=False).stdout.strip() except (FileNotFoundError, subprocess.SubprocessError): return None - return int(out) if out.isdigit() else None + return int(out) if out.isdecimal() else None def _wait_gone(pids: list[int], seconds: float) -> None: @@ -117,7 +117,7 @@ def _cleanup_stale_embedded_pg(data_dir: str) -> None: if not pid_file.exists(): return raw = pid_file.read_text().splitlines() - if not raw or not raw[0].strip().isdigit(): + if not raw or not raw[0].strip().isdecimal(): return pid = int(raw[0].strip()) if _pid_alive(pid): diff --git a/infra/runs_table.py b/infra/runs_table.py index ab8e6a37..dd0d603b 100644 --- a/infra/runs_table.py +++ b/infra/runs_table.py @@ -20,7 +20,8 @@ from audits.models import AuditRun from audits.services import frozen_judge -from infra.ui import ProjectMixin, write_block_reason +from infra.export_utils import csv_safe +from infra.ui import ProjectMixin, is_id, write_block_reason PAGE_SIZES = (10, 25, 50, 100) @@ -64,7 +65,12 @@ def _date(raw: str, *, end: bool = False): if d is None: return None moment = timezone.make_aware(datetime.combine(d, time.min)) - return moment + timedelta(days=1) if end else moment + if not end: + return moment + try: + return moment + timedelta(days=1) + except OverflowError: # 9999-12-31: no next day; nothing can be later anyway + return None def filtered_runs(project, params): @@ -90,13 +96,13 @@ def filtered_runs(project, params): | Q(target_model__display_name__icontains=q) | Q(target_model__model_id__icontains=q) ) - if q.lstrip("#").isdigit(): + if is_id(q.lstrip("#")): filt |= Q(pk=int(q.lstrip("#"))) qs = qs.filter(filt) for param, field in (("target", "target_model_id"), ("set", "scenario_set_version__scenario_set_id"), ("judge", "judge_version__judge_id"), ("experiment", "experiment_id"), ("monitor", "monitor_id")): - ids = [v for v in params.getlist(param) if v.isdigit()] + ids = [v for v in params.getlist(param) if is_id(v)] if ids: qs = qs.filter(**{f"{field}__in": ids}) start, end = _date(params.get("from")), _date(params.get("to"), end=True) @@ -201,8 +207,11 @@ def post(self, request): return JsonResponse({"error": reason}, status=403) try: body = json.loads(request.body or b"{}") - ids = [int(i) for i in body.get("ids", [])][:500] - except (ValueError, TypeError): + if not isinstance(body, dict): + raise TypeError("body must be an object") + # Out-of-range ids match nothing; drop them before they overflow the DB driver. + ids = [i for i in (int(i) for i in body.get("ids", [])) if 0 < i < 2**63][:500] + except (ValueError, TypeError, OverflowError): return JsonResponse({"error": "Invalid request."}, status=400) action = body.get("action") runs = AuditRun.objects.filter(project=request.project, pk__in=ids) @@ -223,8 +232,8 @@ def post(self, request): body = json.loads(request.body or b"{}") except ValueError: return JsonResponse({"error": "Invalid JSON."}, status=400) - key = body.get("key") - if key not in allowed_preference_keys(): + key = body.get("key") if isinstance(body, dict) else None + if not isinstance(key, str) or key not in allowed_preference_keys(): return JsonResponse({"error": "Unknown preference."}, status=400) value = body.get("value") if len(json.dumps(value)) > MAX_PREFERENCE_BYTES: @@ -254,7 +263,7 @@ def get(self, request): writer.writerow(cols) for run in runs: row = run_row(run, counts) - writer.writerow([row[c] if row[c] is not None else "" for c in cols]) + writer.writerow([csv_safe(row[c]) if row[c] is not None else "" for c in cols]) response = HttpResponse(buf.getvalue(), content_type="text/csv; charset=utf-8") response["Content-Disposition"] = 'attachment; filename="runs.csv"' return response diff --git a/infra/seed.py b/infra/seed.py index 6c269c57..bfa155ad 100644 --- a/infra/seed.py +++ b/infra/seed.py @@ -739,8 +739,10 @@ def seed_demo_agent(project, user, log=logger.info) -> dict[str, str]: "created_by": user, }, ) - agent.knowledge_bases.set([knowledge_base]) - agent.tools.set([tool]) + if created: + # Only on first seed: a restart must not undo links the user changed. + agent.knowledge_bases.set([knowledge_base]) + agent.tools.set([tool]) agent.save() status["agent"] = "created" if created else "reused" return status diff --git a/infra/tests/test_agent_resources.py b/infra/tests/test_agent_resources.py index 8a844046..da47d2f1 100644 --- a/infra/tests/test_agent_resources.py +++ b/infra/tests/test_agent_resources.py @@ -121,6 +121,26 @@ def test_sync_updates_seeded_kb_by_name_not_duplicate(self): self.assertEqual(KnowledgeBase.objects.count(), 1) self.assertEqual(KnowledgeBase.objects.get(pk=seeded.pk).external_id, "owui-kb-1") + def test_sync_never_repoints_a_linked_row_by_name(self): + """A linked row renamed to a victim KB's name must not be re-linked to it + (its remote delete runs as its creator).""" + from infra.ui import _sync_openwebui_resources + + user = _member(self.project) + linked = KnowledgeBase.objects.create(project=self.project, name="Victim KB", external_id="admin-kb") + api = mock.Mock() + api.knowledge_bases.return_value = [{"id": "victim-kb", "name": "Victim KB", "description": ""}] + api.tools.return_value = [{"id": "victim-tool", "name": "T"}] + from model_registry.models import Tool + tool = Tool.objects.create(project=self.project, name="T", external_id="admin-tool") + + with mock.patch("chat.api.ChatAPI.as_user", return_value=api): + _sync_openwebui_resources(self.project, user) + + self.assertEqual(KnowledgeBase.objects.get(pk=linked.pk).external_id, "admin-kb") + tool.refresh_from_db() + self.assertEqual(tool.external_id, "admin-tool") + def test_sync_renames_kb_by_external_id_not_duplicate(self): """A KB renamed in OWUI keeps the same id. Sync must update the existing row in place (matching on external_id), not CREATE a second row whose diff --git a/infra/tests/test_audit_events.py b/infra/tests/test_audit_events.py index 7eea9af4..0b153b56 100644 --- a/infra/tests/test_audit_events.py +++ b/infra/tests/test_audit_events.py @@ -69,3 +69,12 @@ def test_distinct_version_items_are_tracked_separately(self): self.assertEqual(count_results(1), 2) self.assertIsNotNone(get_result(1, "a")) self.assertIsNotNone(get_result(1, "b")) + + +class CompletedResultNotDowngradedTest(TestCase): + def test_late_failed_duplicate_does_not_replace_completed(self): + upsert_scenario_result(1, "a", status="completed", attempts=1, result={"ok": True}) + upsert_scenario_result(1, "a", status="failed", attempts=3, result={"ok": False}) + result = get_result(1, "a") + self.assertEqual(result["status"], "completed") + self.assertEqual(result["attempts"], 3) diff --git a/infra/tests/test_audit_submission.py b/infra/tests/test_audit_submission.py index 72fd60fa..1361fc8a 100644 --- a/infra/tests/test_audit_submission.py +++ b/infra/tests/test_audit_submission.py @@ -97,3 +97,64 @@ def test_submit_without_live_server_keeps_run_queued_and_records_reason(self): submission = run.runtime_metadata.get("submission", {}) self.assertEqual(submission.get("status"), "pending") self.assertIn("reason", submission) + + +class FinishedRunStaysFinishedTest(AuditSubmissionTest): + """Late cancel/fail writes must not overwrite a run the worker already finished.""" + + def test_cancel_with_stale_object_does_not_overwrite_completed_run(self): + from audits.services import cancel_run + + run = self._make_run() + stale = AuditRun.objects.get(pk=run.pk) + AuditRun.objects.filter(pk=run.pk).update(status=AuditRun.Status.COMPLETED) + + self.assertFalse(cancel_run(stale, self.user)) + run.refresh_from_db() + self.assertEqual(run.status, AuditRun.Status.COMPLETED) + + def test_mark_run_failed_does_not_overwrite_cancelled_run(self): + from infra.worker import _mark_run_failed + + run = self._make_run() + AuditRun.objects.filter(pk=run.pk).update(status=AuditRun.Status.CANCELLED) + _mark_run_failed(str(run.pk), "SIMPLEAUDIT_VERSION_MISMATCH") + run.refresh_from_db() + self.assertEqual(run.status, AuditRun.Status.CANCELLED) + self.assertNotEqual(run.error_code, "SIMPLEAUDIT_VERSION_MISMATCH") + + def test_mark_run_failed_fails_active_run(self): + from infra.worker import _mark_run_failed + + run = self._make_run() + _mark_run_failed(str(run.pk), "SIMPLEAUDIT_VERSION_MISMATCH") + run.refresh_from_db() + self.assertEqual(run.status, AuditRun.Status.FAILED) + + def test_viewer_cannot_cancel(self): + from audits.services import cancel_run + from infra.exceptions import StableAPIError + + run = self._make_run() + viewer = User.objects.create_user(username="vic", password="pass12345") + ProjectMembership.objects.create(project=self.project, user=viewer, role=ProjectMembership.Role.VIEWER) + with self.assertRaises(StableAPIError): + cancel_run(run, viewer) + run.refresh_from_db() + self.assertTrue(run.is_active) + + +class RunNameLengthTest(AuditSubmissionTest): + def test_overlong_run_name_is_capped(self): + from unittest import mock + + with mock.patch( + "audits.services.resolve_engine_provenance", + return_value=mock.Mock(version="0.1.0", commit="deadbeef", source="metadata"), + ): + run = create_audit_run( + project=self.project, user=self.user, name="m" * 250 + " · 2026-11-04 10:00", + scenario_set_version=self.version, target_model=self.target, auditor_model=self.auditor, + judge_model=self.judge, judge=judge_for(self.judge), + ) + self.assertEqual(len(run.name), 250) diff --git a/infra/tests/test_codegen.py b/infra/tests/test_codegen.py index 1c95b6de..1c7edf6a 100644 --- a/infra/tests/test_codegen.py +++ b/infra/tests/test_codegen.py @@ -390,3 +390,25 @@ def test_other_project_judge_is_404(self): JudgeVersionFactory(judge=other, version=1, base="safety") resp = self.client.get(f"/judges/{other.id}/script/") assert resp.status_code == 404 + + +class CodegenInjectionTests(TestCase): + """User/registry-controlled text must never become code in a generated script.""" + + def test_hostile_run_name_and_model_id_stay_inert(self): + from infra.codegen import generate_run_script + + run = _make_run(name='x"""\nimport os; os.system("pwned")\n"""') + run.target_config_snapshot = {**run.target_config_snapshot, "model_id": 'm"\nimport os #'} + script = generate_run_script(run) + tree = ast.parse(script) + imported = {a.name for node in ast.walk(tree) if isinstance(node, ast.Import) for a in node.names} + assert imported <= {"os", "json"} + calls = [n for n in ast.walk(tree) if isinstance(n, ast.Call) and getattr(n.func, "attr", "") == "system"] + assert not calls + + def test_scenario_text_reproduced_exactly(self): + from infra.codegen import _py + + value = {"text": "Say true or false; null is fine", "flag": True, "none": None} + assert ast.literal_eval(_py(value)) == value diff --git a/infra/tests/test_engine_integration.py b/infra/tests/test_engine_integration.py index 5022e84f..0fa066ef 100644 --- a/infra/tests/test_engine_integration.py +++ b/infra/tests/test_engine_integration.py @@ -325,7 +325,8 @@ class FakeExperiment: def __init__(self, **_kwargs): import simpleaudit.experiment as experiment_module - captured["auditor_class"] = experiment_module.ModelAuditor + # The module attribute dispatches per run; resolve this run's class. + captured["auditor_class"] = experiment_module.ModelAuditor.resolve() async def run_scenario_reps(self, **_kwargs): return [] @@ -381,6 +382,8 @@ async def acompletion(**kwargs): self.assertEqual(calls[0]["extra_body"], {"session_id": "session-1"}) self.assertNotIn("extra_headers", calls[0]) + @mock.patch("chat.config.UPSTREAM", "http://127.0.0.1:8080/chat") + @mock.patch("chat.config.ENABLED", True) def test_model_target_uses_saved_chat_path_for_openwebui_server_tools(self): """Open WebUI's API path must create a chat for server-side tool execution.""" import asyncio @@ -568,6 +571,31 @@ def test_reasoning_effort_is_forwarded_as_a_request_parameter(self): self.assertEqual(kwargs["judge_params"], {"reasoning_effort": "none"}) self.assertEqual(kwargs["judge_kwargs"], {"timeout": 180, "max_retries": 1}) + def test_target_defaults_do_not_leak_into_judge_or_auditor(self): + from infra.engine import auditor_kwargs + + kwargs, _ = auditor_kwargs( + target=_snap("tgt", default_parameters={"temperature": 1.3, "max_tokens": 64}), + auditor=_snap("aud"), + judge=_snap("jdg"), + ) + # The engine merges `params` into every role; target defaults go only in target_params. + self.assertFalse(kwargs["params"]) + self.assertEqual(kwargs["target_params"], {"temperature": 1.3, "max_tokens": 64}) + + def test_shared_params_override_endpoint_defaults_and_auditor_never_inherits_judge(self): + from infra.engine import auditor_kwargs + + kwargs, _ = auditor_kwargs( + target=_snap("tgt", default_parameters={"temperature": 0.7}), + auditor=_snap("aud"), + judge=_snap("jdg", default_parameters={"max_tokens": 200}), + generation={"params": {"temperature": 0}}, + ) + self.assertEqual(kwargs["target_params"]["temperature"], 0) + self.assertNotIn("max_tokens", kwargs["auditor_params"]) + self.assertIsNotNone(kwargs["auditor_params"]) + def test_roles_map_to_their_own_snapshot(self): from infra.engine import auditor_kwargs @@ -1354,3 +1382,166 @@ def test_scenario_dict_leaves_out_empty_fields(self): self.assertEqual(scenario_dict(name="n", description="d", test_prompt="", metadata={}), {"name": "n", "description": "d"}) self.assertEqual(scenario_dict(name="n", description="d", severity_ceiling="high")["severity"], "high") + + +def test_concurrent_runs_keep_their_own_auditor_class(monkeypatch): + """Two runs in parallel worker threads must each build their own auditor class.""" + import asyncio + import threading + + import simpleaudit.experiment as experiment_module + + from infra import engine + + monkeypatch.setattr(experiment_module, "ModelAuditor", experiment_module.ModelAuditor) + barrier = threading.Barrier(2) + seen: dict[str, type] = {} + + class FakeExperiment: + def __init__(self, models, **_kwargs): + self.label = models[0]["label"] + + async def run_scenario_reps(self, **_kwargs): + # Both runs have installed their class before either builds one. + await asyncio.get_running_loop().run_in_executor(None, barrier.wait) + seen[self.label] = experiment_module.ModelAuditor.resolve() + return [] + + monkeypatch.setattr(experiment_module, "AuditExperiment", FakeExperiment) + classes = {"a": type("AuditorA", (), {}), "b": type("AuditorB", (), {})} + + def run(label): + with mock.patch("infra.engine.decision_target_for", return_value=object()), \ + mock.patch("infra.trace_target.decision_auditor_class", return_value=classes[label]), \ + mock.patch("infra.engine.snapshot_api_key", return_value=""): + engine.run_scenario_repeated( + name=label, description="d", expected_behavior=None, test_prompt=None, + target=_snap(label), auditor=_snap("x"), judge=_snap("y"), n_repetitions=2, trace_config=None, + ) + + threads = [threading.Thread(target=run, args=(k,)) for k in classes] + for t in threads: + t.start() + for t in threads: + t.join(timeout=30) + assert {label.split(" ")[0] for label in seen} == {"a", "b"} + for label, cls in seen.items(): + assert cls is classes[label.split(" ")[0]] + + +class StudioOpenWebUIOriginTest(TestCase): + """The trusted-header admin JWT goes only to Studio's own Open WebUI.""" + + @mock.patch("chat.config.UPSTREAM", "http://127.0.0.1:8080/chat") + @mock.patch("chat.config.ENABLED", True) + def test_only_exact_origin_is_studio_owui(self): + import asyncio + + from infra.trace_target import _is_studio_owui, _owui_get_jwt + + self.assertTrue(_is_studio_owui("http://127.0.0.1:8080/chat/api/v1/")) + for url in ("https://attacker.example/chat/api/v1", "http://127.0.0.1:8080@attacker.example/chat/api/v1"): + self.assertFalse(_is_studio_owui(url), url) + with mock.patch("httpx.AsyncClient") as client: + self.assertIsNone(asyncio.run(_owui_get_jwt(url))) + client.assert_not_called() + + +def test_repeated_traced_run_gives_each_rep_its_correlation(monkeypatch): + """With tracing on, the engine's trace_correlation() returns the current rep's correlation.""" + import simpleaudit.experiment as experiment_module + + from infra import engine + + monkeypatch.setattr(experiment_module, "ModelAuditor", experiment_module.ModelAuditor) + seen = [] + + class FakeExperiment: + def __init__(self, models, rep_is_done=None, **_kwargs): + self.rep_is_done = rep_is_done + + async def run_scenario_reps(self, trace_correlation=None, **_kwargs): + for i in range(2): + self.rep_is_done("m", i) + seen.append(trace_correlation()) + return [] + + provider = mock.MagicMock() + provider.__enter__ = lambda self: self + provider.__exit__ = lambda self, *a: False + monkeypatch.setattr(experiment_module, "AuditExperiment", FakeExperiment) + with mock.patch("infra.tracing.build_trace_provider", return_value=provider): + engine.run_scenario_repeated( + name="s", description="d", expected_behavior=None, test_prompt=None, + target=_snap("t"), auditor=_snap("a"), judge=_snap("j"), n_repetitions=2, + trace_config={"mode": "studio"}, + ) + assert len(seen) == 2 and all(c is not None for c in seen) + assert seen[0] is not seen[1] + + +def test_retry_inside_rep_gets_fresh_correlation(monkeypatch): + """An ERROR retry inside a rep must not reuse the failed attempt's correlation.""" + import simpleaudit.experiment as experiment_module + + from infra import engine + + monkeypatch.setattr(experiment_module, "ModelAuditor", experiment_module.ModelAuditor) + seen = [] + + class FakeExperiment: + def __init__(self, models, rep_is_done=None, **_kwargs): + self.rep_is_done = rep_is_done + + async def run_scenario_reps(self, trace_correlation=None, **_kwargs): + self.rep_is_done("m", 0) + seen.append(trace_correlation()) # first attempt + seen.append(trace_correlation()) # retry of the same rep + return [] + + provider = mock.MagicMock() + provider.__enter__ = lambda self: self + provider.__exit__ = lambda self, *a: False + monkeypatch.setattr(experiment_module, "AuditExperiment", FakeExperiment) + with mock.patch("infra.tracing.build_trace_provider", return_value=provider): + engine.run_scenario_repeated( + name="s", description="d", expected_behavior=None, test_prompt=None, + target=_snap("t"), auditor=_snap("a"), judge=_snap("j"), n_repetitions=2, + trace_config={"mode": "studio"}, + ) + assert seen[0] is not None and seen[1] is not None and seen[0] is not seen[1] + + +def test_repeated_run_uses_judge_agentic_guidance(monkeypatch): + """Judge-level agentic guidance applies on the repeated path as on the single-rep path.""" + import simpleaudit.experiment as experiment_module + + from infra import engine + + monkeypatch.setattr(experiment_module, "ModelAuditor", experiment_module.ModelAuditor) + got = [] + + class FakeExperiment: + def __init__(self, models, rep_is_done=None, **_kwargs): + self.rep_is_done = rep_is_done + + async def run_scenario_reps(self, trace_correlation=None, evidence_resolver=None, **_kwargs): + self.rep_is_done("m", 0) + trace_correlation() + got.append(evidence_resolver(None)) + return [] + + provider = mock.MagicMock() + provider.__enter__ = lambda self: self + provider.__exit__ = lambda self, *a: False + monkeypatch.setattr(experiment_module, "AuditExperiment", FakeExperiment) + judge = {**_snap("j"), "judge": {"spec": {"base": "safety", "output": "", "criteria": "", "probe_prompt": "", + "options": {"agentic_guidance": "Judge says"}}}} + with mock.patch("infra.tracing.build_trace_provider", return_value=provider), \ + mock.patch("infra.engine._collect_evidence_spans", return_value={"all_spans": [], "selected_spans": []}): + engine.run_scenario_repeated( + name="s", description="d", expected_behavior=None, test_prompt=None, + target=_snap("t"), auditor=_snap("a"), judge=judge, n_repetitions=2, + trace_config={"mode": "studio"}, metadata={"agentic": {"schema_version": 2, "enforcement": {}}}, + ) + assert got and got[0]["agentic_guidance"] == "Judge says" diff --git a/infra/tests/test_experiments.py b/infra/tests/test_experiments.py index d33920b5..be1dde0a 100644 --- a/infra/tests/test_experiments.py +++ b/infra/tests/test_experiments.py @@ -220,6 +220,41 @@ def test_launch_creates_experiment_with_selected_runs(self): self.assertEqual(r.generation_parameters_snapshot["judge_params"], {"temperature": 0}) self.assertEqual(runs[-1].generation_parameters_snapshot["max_turns"], 9) + def test_multi_run_agent_experiment_runs_the_agent(self): + """Every run of a multi-run agent design targets the agent, not its bare base model.""" + agent = AgentFactory(project=self.project, base_model=self.t1, external_id="studio.agent-7") + with mock.patch("chat.config.ENABLED", True): + review = self.client.post( + "/experiments/new/", self._design(target_model=[], agent_id=str(agent.id), max_turns="3, 5") + ) + self.assertEqual(len(review.context["rows"]), 2) + resp = self.client.post("/experiments/new/", self._review_to_launch_payload(review)) + exp = Experiment.objects.get() + self.assertRedirects(resp, f"/experiments/{exp.id}/", fetch_redirect_response=False) + runs = list(exp.runs.all()) + self.assertEqual(len(runs), 2) + for r in runs: + self.assertEqual(r.agent_id, agent.id) + + def test_repeating_agent_experiment_is_refused(self): + from audits.experiments import DesignError, launch_experiment + + agent = AgentFactory(project=self.project, base_model=self.t1, external_id="studio.agent-9") + run = {"name": "r", "version": self.v_a, "target": self.t1, "auditor": self.judge, "judge_model": self.judge, + "judge": judge_for(self.judge), "max_turns": None, "language": None, "n_repetitions": None, + "gen_config": None, "agent": agent} + with self.assertRaises(DesignError): + launch_experiment(project=self.project, user=self.user, name="x", runs=[run], + repeat={"start": "now", "every": "day"}) + self.assertFalse(Experiment.objects.exists()) + + def test_agent_with_target_models_is_refused(self): + agent = AgentFactory(project=self.project, base_model=self.t1, external_id="studio.agent-8") + with mock.patch("chat.config.ENABLED", True): + resp = self.client.post("/experiments/new/", self._design(agent_id=str(agent.id))) + self.assertFalse(Experiment.objects.exists()) + self.assertContains(resp, "not both") + def test_multiple_scenario_sets_become_a_factor(self): review = self.client.post( "/experiments/new/", self._design(scenario_set=[self.set_a.id, self.set_b.id], target_model=[self.t1.id], max_turns="") diff --git a/infra/tests/test_malformed_input.py b/infra/tests/test_malformed_input.py new file mode 100644 index 00000000..83f7008e --- /dev/null +++ b/infra/tests/test_malformed_input.py @@ -0,0 +1,360 @@ +"""Malformed query/form/JSON input must yield a client error or a normal page, never a 500.""" +from unittest import mock + +from django.test import TestCase + +from accounts.models import Project, ProjectMembership, User +from scenarios.models import Scenario + + +class MalformedInputTest(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="pa", password="pass12345") + self.project = Project.objects.create(name="P", slug="p") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + self.client.raise_request_exception = False + + def assertNot500(self, resp, label): + self.assertLess(resp.status_code, 500, label) + + def test_unicode_digit_query_params(self): + # "²".isdigit() is True but int("²") raises. + for url in ["/scenarios/?set=%C2%B2", "/workspaces/?manage=%C2%B2", "/experiments/new/?clone_from=%C2%B2", + "/compare/?runs=%C2%B2,1", "/runs/data/?target=%C2%B2", "/runs/data/?q=%23%C2%B2", + "/runs/export.csv?set=%C2%B2"]: + self.assertNot500(self.client.get(url), url) + self.assertNot500(self.client.post("/compare/", {"runs[]": ["²", "1"]}), "compare post") + + def test_bulk_and_preference_bodies(self): + for body in ["[1]", '{"ids":[1e400],"action":"archive"}']: + self.assertNot500(self.client.post("/runs/bulk/", body, content_type="application/json"), body) + for body in ["[]", '{"key":[1]}']: + self.assertNot500(self.client.post("/me/preferences/", body, content_type="application/json"), body) + + def test_scenario_forms_with_bad_set_id(self): + self.assertNot500(self.client.post("/scenarios/create/", {"name": "X", "set_id": "abc"}), "create") + scenario = Scenario.objects.first() + self.assertIsNotNone(scenario) + self.assertNot500(self.client.post( + f"/scenarios/edit/{scenario.id}/", {"title": "Y", "description": "changed", "set_id": "abc"}), "edit") + self.assertNot500(self.client.post(f"/scenarios/delete/{scenario.id}/", {"set_id": "abc"}), "delete") + + def test_connection_forms_with_bad_ids(self): + for data in [ + {"action": "edit_connection", "conn_id": "abc", "conn_name": "n", "conn_base_url": "https://x.example/v1"}, + {"action": "add_models", "conn_id": "abc"}, + {"action": "edit_model", "rm_id": "abc"}, + ]: + self.assertNot500(self.client.post("/connections/", data), data["action"]) + self.assertNot500(self.client.post("/connections/discover/", {"connection_id": "abc"}), "discover") + + def test_agent_form_with_bad_base_model(self): + with mock.patch("infra.ui.chat_enabled", return_value=True): + self.assertNot500(self.client.post("/agents/new/", {"name": "A", "base_model": "abc"}), "abc") + self.assertNot500(self.client.post("/agents/new/", {"name": "A"}), "missing") + + def test_oversized_numeric_ids(self): + huge, long = "9" * 25, "9" * 5000 + for url in [f"/scenarios/?set={long}", f"/workspaces/?manage={long}", f"/runs/data/?q={long}", + f"/runs/data/?target={huge}", f"/runs/export.csv?set={huge}", f"/compare/?runs={huge},1"]: + self.assertNot500(self.client.get(url), url[:40]) + body = '{"ids":[9999999999999999999999999],"action":"archive"}' + self.assertNot500(self.client.post("/runs/bulk/", body, content_type="application/json"), "bulk huge") + + +class MalformedApiInputTest(TestCase): + def setUp(self): + self.client.raise_request_exception = False + + def test_token_endpoint_bad_bodies(self): + for body in ["[1]", '{"username":"x","password":[1]}']: + resp = self.client.post("/api/auth/token/", body, content_type="application/json") + self.assertLess(resp.status_code, 500, body) + + def test_switch_workspace_bad_body(self): + user = User.objects.create_user(username="sw", password="pass12345") + self.client.force_login(user) + resp = self.client.post("/api/projects/switch/", "[1]", content_type="application/json") + self.assertLess(resp.status_code, 500) + + def test_agent_api_not_found_is_404(self): + user = User.objects.create_user(username="ag", password="pass12345") + project = Project.objects.create(name="AG", slug="ag") + ProjectMembership.objects.create(project=project, user=user, role=ProjectMembership.Role.ADMIN) + self.client.force_login(user) + session = self.client.session + session["active_project_id"] = project.id + session.save() + for url in ["/api/agents/999999/", "/api/knowledge-bases/999999/", "/api/tools/999999/"]: + self.assertEqual(self.client.get(url).status_code, 404, url) + + +class Pass14InputTest(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="p14", password="pass12345") + self.project = Project.objects.create(name="P14", slug="p14") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + self.client.raise_request_exception = False + + def test_end_of_calendar_date_filter(self): + for url in ["/runs/data/?to=9999-12-31", "/runs/export.csv?to=9999-12-31"]: + self.assertLess(self.client.get(url).status_code, 500, url) + + def test_oversized_ids_in_audit_api(self): + huge = "9" * 25 + resp = self.client.get(f"/api/projects/{self.project.id}/audit-runs/compare/?run_ids={huge},1") + self.assertLess(resp.status_code, 500) + + def test_publish_api_with_duplicate_ids(self): + from scenarios.models import ScenarioSet + from scenarios.services import create_scenario, publish_scenario_set_version + + sset = ScenarioSet.objects.create(project=self.project, name="S") + sc = create_scenario(project=self.project, user=self.user, key="k", title="T", description="d", + expected_behavior=["x"]) + version = publish_scenario_set_version(scenario_set=sset, user=self.user, scenario_ids=[sc.id, sc.id]) + self.assertEqual(version.items.count(), 1) + + def test_agent_monitor_refused(self): + from audits.monitors import create_monitor + + with self.assertRaises(ValueError): + create_monitor(project=self.project, user=self.user, name="m", run={"agent": object()}, + repeat={"start": "now", "every": "day"}) + + +class ApiBoundaryValidationTest(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="ab", password="pass12345") + self.project = Project.objects.create(name="AB", slug="ab-api") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + self.client.raise_request_exception = False + + def _post(self, url, body): + import json + + return self.client.post(url, json.dumps(body), content_type="application/json") + + def test_import_validates_items(self): + from scenarios.models import Scenario + + resp = self._post(f"/api/projects/{self.project.id}/scenarios/import/", {"scenarios": [ + {"key": "bad", "title": "T", "description": "d", "expected_behavior": "not a list"}, + {"key": "ok", "title": "T", "description": "d", "expected_behavior": ["x"]}, + ]}) + self.assertLess(resp.status_code, 500) + self.assertTrue(Scenario.objects.filter(project=self.project, key="ok").exists()) + self.assertFalse(Scenario.objects.filter(project=self.project, key="bad").exists()) + + def test_bad_agentic_metadata_is_400(self): + resp = self._post(f"/api/projects/{self.project.id}/scenarios/create/", { + "title": "T", "description": "d", "expected_behavior": ["x"], "metadata": {"agentic": {"bogus": 1}}}) + self.assertEqual(resp.status_code, 400) + + def test_long_title_without_key_gets_capped_key(self): + from scenarios.models import Scenario + + resp = self._post(f"/api/projects/{self.project.id}/scenarios/create/", { + "title": "t" * 290, "description": "d", "expected_behavior": ["x"]}) + self.assertLess(resp.status_code, 500) + self.assertLessEqual(len(Scenario.objects.get(project=self.project).key), 200) + + def test_agent_retrieval_settings_must_be_object(self): + from infra.tests.factories import RegisteredModelFactory + + model = RegisteredModelFactory(project=self.project) + resp = self._post("/api/agents/", {"name": "A", "base_model": model.id, "retrieval_settings": [1]}) + self.assertEqual(resp.status_code, 400) + + +class FieldLengthTest(TestCase): + """Values beyond model max_length must be capped or refused (Postgres rejects them).""" + + def setUp(self): + self.user = User.objects.create_user(username="fl", password="pass12345") + self.project = Project.objects.create(name="FL", slug="fl") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def test_scenario_forms_cap_lengths(self): + from scenarios.models import Scenario, ScenarioSet + + self.client.post("/scenarios/set-create/", {"name": "s" * 400}) + sset = ScenarioSet.objects.get(project=self.project) + self.assertLessEqual(len(sset.name), 250) + self.client.post("/scenarios/create/", {"name": "t" * 400, "category": "c" * 200, + "set_id": str(sset.id), "description": "d"}) + self.assertTrue(Scenario.objects.filter(project=self.project).exists()) + for s in Scenario.objects.filter(project=self.project): + self.assertLessEqual(len(s.title), 300) + self.assertLessEqual(len(s.category), 120) + + def test_connection_overlong_key_and_url_refused(self): + from model_registry.models import ModelConnection + + self.client.post("/connections/", {"action": "add_connection", "conn_name": "n" * 300, + "conn_base_url": "https://x.example/" + "a" * 250, "conn_provider": "openai"}) + self.client.post("/connections/", {"action": "add_connection", "conn_name": "ok", + "conn_base_url": "https://x.example/v1", "conn_provider": "openai", + "key_mode": "stored", "conn_api_key": "k" * 900}) + for c in ModelConnection.objects.filter(project=self.project): + self.assertLessEqual(len(c.base_url), 200) + self.assertLessEqual(len(c.api_key_direct), 500) + self.assertLessEqual(len(c.name), 250) + + def test_register_overlong_username_refused(self): + self.client.logout() + self.client.post("/register/", {"username": "u" * 200, "password": "Str0ng-pass-123"}) + self.assertFalse(User.objects.filter(username="u" * 200).exists()) + + +class ScenarioUiFormValidationTest(TestCase): + def setUp(self): + from scenarios.models import ScenarioSet + + self.user = User.objects.create_user(username="uv", password="pass12345") + self.project = Project.objects.create(name="UV", slug="uv") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.sset = ScenarioSet.objects.create(project=self.project, name="S") + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + self.client.raise_request_exception = False + + def test_ui_import_rejects_malformed_items(self): + import json + + from scenarios.models import Scenario + + resp = self.client.post(f"/scenarios/{self.sset.id}/import/", json.dumps({"scenarios": [ + {"key": "bad", "title": "T", "description": "d", "expected_behavior": "abc"}]}), + content_type="application/json") + self.assertEqual(resp.status_code, 400) + self.assertFalse(Scenario.objects.filter(project=self.project, key="bad").exists()) + + def test_documents_must_be_a_list(self): + from scenarios.models import Scenario + + self.client.post("/scenarios/create/", {"name": "D", "description": "d", "documents": '"policy.pdf"', + "set_id": str(self.sset.id)}) + self.assertFalse(Scenario.objects.filter(project=self.project, title="D").exists()) + + def test_duplicate_set_name_does_not_500(self): + from scenarios.models import ScenarioSet + + resp = self.client.post("/scenarios/set-create/", {"name": "S"}) + self.assertLess(resp.status_code, 500) + other = ScenarioSet.objects.create(project=self.project, name="Other") + resp = self.client.post(f"/scenarios/set-rename/{other.id}/", {"name": "S"}) + self.assertLess(resp.status_code, 500) + self.assertEqual(ScenarioSet.objects.filter(project=self.project, name="S").count(), 1) + + +class ScenarioRoundTripAndConnectionFieldsTest(TestCase): + def setUp(self): + from scenarios.models import ScenarioSet + + self.user = User.objects.create_user(username="rt", password="pass12345") + self.project = Project.objects.create(name="RT", slug="rt") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.sset = ScenarioSet.objects.create(project=self.project, name="S") + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def test_export_import_round_trip_with_string_documents(self): + from scenarios.models import ScenarioSet + + self.client.post("/scenarios/create/", {"name": "Docs", "description": "d", "documents": '["policy text"]', + "set_id": str(self.sset.id)}) + exported = self.client.get(f"/scenarios/{self.sset.id}/export/") + self.assertEqual(exported.status_code, 200) + target = ScenarioSet.objects.create(project=self.project, name="T") + resp = self.client.post(f"/scenarios/{target.id}/import/", exported.content, content_type="application/json") + self.assertEqual(resp.status_code, 302, resp.content) + latest = target.versions.order_by("-version").first() + self.assertEqual(latest.items.get().revision.documents, ["policy text"]) + + def test_overlong_model_id_and_secret_ref_refused(self): + from infra.tests.factories import ModelConnectionFactory + from model_registry.models import RegisteredModel + + conn = ModelConnectionFactory(project=self.project) + self.client.post("/connections/", {"action": "add_models", "conn_id": conn.id, "model_id": "m" * 300}) + self.assertFalse(RegisteredModel.objects.filter(connection=conn).exists()) + self.client.post("/connections/", {"action": "edit_connection", "conn_id": conn.id, "conn_name": conn.name, + "conn_base_url": "https://x.example/v1", "conn_provider": "openai", + "key_mode": "env", "conn_secret_ref": "E" * 300}) + conn.refresh_from_db() + self.assertLessEqual(len(conn.secret_reference), 250) + + +class RemainingFormsBoundaryTest(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="rf", password="pass12345") + self.project = Project.objects.create(name="RF", slug="rf") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + self.client.raise_request_exception = False + + def test_profile_refuses_invalid_fields(self): + for data in ({"username": "u" * 200}, {"username": "rf", "email": "not an email"}, + {"username": "a b/c"}, {"username": "rf", "last_name": "x" * 300}): + self.client.post("/profile/", data) + self.user.refresh_from_db() + self.assertEqual(self.user.username, "rf") + self.assertEqual(self.user.email, "") + self.assertLessEqual(len(self.user.last_name), 150) + + def test_connection_provider_capped(self): + from model_registry.models import ModelConnection + + self.client.post("/connections/", {"action": "add_connection", "conn_name": "p", + "conn_base_url": "https://x.example/v1", "conn_provider": "p" * 500}) + for c in ModelConnection.objects.filter(project=self.project): + self.assertLessEqual(len(c.provider), 120) + + def test_cron_and_first_run_bounds(self): + from audits.monitors import validate_cron + + with self.assertRaises(ValueError): + validate_cron("0 0 * * " + ",".join(["1"] * 80)) + + def test_design_form_oversized_ids_do_not_500(self): + huge = "9" * 30 + for field in ("scenario_version", "scenario_set", "target_model", "judge"): + resp = self.client.post("/experiments/new/", {field: huge, "action": "design"}) + self.assertLess(resp.status_code, 500, field) + + +class LegacyUsernameProfileTest(TestCase): + def test_existing_username_with_space_can_still_save_profile(self): + user = User.objects.create_user(username="Jane Doe", password="pass12345") + self.client.force_login(user) + self.client.post("/profile/", {"username": "Jane Doe", "first_name": "Jane"}) + user.refresh_from_db() + self.assertEqual(user.first_name, "Jane") + + def test_new_registration_rejects_invalid_username(self): + self.client.post("/register/", {"username": "Bad Name", "password": "Str0ng-pass-123"}) + self.assertFalse(User.objects.filter(username="Bad Name").exists()) diff --git a/infra/tests/test_minimal_config.py b/infra/tests/test_minimal_config.py index d283aee8..973a848c 100644 --- a/infra/tests/test_minimal_config.py +++ b/infra/tests/test_minimal_config.py @@ -225,15 +225,27 @@ def test_redirects_to_visualizer_when_next_set(self): def test_next_must_be_a_relative_path(self): # An absolute (http://) or protocol-relative (//evil) value must be # ignored so a tampered env var can't turn this into an open redirect. - for bad in ("https://evil.example.com", "//evil.example.com", ""): + for i, bad in enumerate(("https://evil.example.com", "//evil.example.com", "")): + # A fresh token per attempt: each one is single-use. + token = f"{self.TOKEN}-{i}" with patch.dict( os.environ, - {"SIMPLEAUDIT_AUTO_LOGIN_TOKEN": self.TOKEN, "SIMPLEAUDIT_AUTO_LOGIN_NEXT": bad}, + {"SIMPLEAUDIT_AUTO_LOGIN_TOKEN": token, "SIMPLEAUDIT_AUTO_LOGIN_NEXT": bad}, ): - response = self._login(self.TOKEN, follow=False) + response = self._login(token, follow=False) self.assertEqual(response.status_code, 302) self.assertNotIn("evil.example.com", response.headers["Location"]) + @override_settings(MINIMAL_CONFIG=True) + def test_token_cannot_be_replayed_after_reload(self): + # The autoreloader restarts the server with the parent's environment, + # which still carries the token; consumption must survive that. + with patch.dict(os.environ, {"SIMPLEAUDIT_AUTO_LOGIN_TOKEN": self.TOKEN}): + self.assertEqual(self._login(self.TOKEN).status_code, 200) + self.client.logout() + with patch.dict(os.environ, {"SIMPLEAUDIT_AUTO_LOGIN_TOKEN": self.TOKEN}): + self.assertEqual(self._login(self.TOKEN).status_code, 404) + @override_settings(MINIMAL_CONFIG=True) def test_token_cannot_be_replayed(self): with patch.dict(os.environ, {"SIMPLEAUDIT_AUTO_LOGIN_TOKEN": self.TOKEN}): diff --git a/infra/tests/test_pass12_fixes.py b/infra/tests/test_pass12_fixes.py new file mode 100644 index 00000000..7d70662b --- /dev/null +++ b/infra/tests/test_pass12_fixes.py @@ -0,0 +1,135 @@ +"""Regression tests for audit pass 12: visualizer API, connection edits, KB external_id, untrusted spans.""" +import json +import os +import tempfile +from unittest import mock + +from django.test import TestCase, override_settings + +from accounts.models import Project, ProjectMembership, User + + +def _login(client, user, project): + client.force_login(user) + session = client.session + session["active_project_id"] = project.id + session.save() + + +class VisualizerApiAuthTest(TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + with open(os.path.join(self.tmp, "r.json"), "w") as f: + json.dump([{"scenario": "x"}], f) + + def test_api_requires_login(self): + with mock.patch("infra.visualizer.results_dir", return_value=self.tmp): + for url in ["/api/visualizer/api/files/", "/api/visualizer/api/json/r.json", + "/api/visualizer/api/image/?uri=/etc/hosts"]: + resp = self.client.get(url) + self.assertIn(resp.status_code, (302, 401, 403), url) + + def test_image_outside_results_dir_refused(self): + user = User.objects.create_user(username="vz", password="pass12345") + self.client.force_login(user) + with mock.patch("infra.visualizer.results_dir", return_value=self.tmp): + for uri in ["/etc/hosts", "../../etc/hosts"]: + resp = self.client.get("/api/visualizer/api/image/", {"uri": uri}) + self.assertEqual(resp.status_code, 403, uri) + for uri in ["s3://bucket/x.png", "file:///etc/hosts"]: + with mock.patch("simpleaudit.utils.image_data_uri") as loader: + resp = self.client.get("/api/visualizer/api/image/", {"uri": uri}) + self.assertEqual(resp.status_code, 415, uri) + loader.assert_not_called() + + def test_list_shaped_result_file_does_not_500(self): + user = User.objects.create_user(username="vz2", password="pass12345") + self.client.force_login(user) + self.client.raise_request_exception = False + with mock.patch("infra.visualizer.results_dir", return_value=self.tmp), \ + mock.patch("infra.visualizer.is_valid_audit_data", return_value=True): + resp = self.client.get("/api/visualizer/api/json/r.json") + self.assertLess(resp.status_code, 500) + + +class AuditorConnectionEditTest(TestCase): + def setUp(self): + from infra.tests.factories import ModelConnectionFactory + + self.project = Project.objects.create(name="C", slug="c") + self.auditor = User.objects.create_user(username="aud", password="pass12345") + ProjectMembership.objects.create(project=self.project, user=self.auditor, role=ProjectMembership.Role.AUDITOR) + self.conn = ModelConnectionFactory(project=self.project, base_url="https://api.openai.com/v1", api_key_direct="sk-admin") + _login(self.client, self.auditor, self.project) + + def test_auditor_cannot_repoint_connection(self): + self.client.post("/connections/", { + "action": "edit_connection", "conn_id": self.conn.id, "conn_name": "x", + "conn_base_url": "https://attacker.example/v1", "conn_provider": "openai", "key_mode": "stored", + }) + self.conn.refresh_from_db() + self.assertEqual(self.conn.base_url, "https://api.openai.com/v1") + + def test_auditor_cannot_delete_connection(self): + from model_registry.models import ModelConnection + + self.client.post(f"/connections/{self.conn.id}/delete/") + self.assertTrue(ModelConnection.objects.filter(pk=self.conn.pk).exists()) + + +class KnowledgeBaseExternalIdTest(TestCase): + def test_external_id_cannot_be_changed_by_update(self): + from model_registry.models import KnowledgeBase + + project = Project.objects.create(name="K", slug="k") + user = User.objects.create_user(username="kb", password="pass12345") + ProjectMembership.objects.create(project=project, user=user, role=ProjectMembership.Role.AUDITOR) + kb = KnowledgeBase.objects.create(project=project, name="KB", external_id="mine") + _login(self.client, user, project) + resp = self.client.put(f"/api/knowledge-bases/{kb.id}/", {"name": "KB", "external_id": "victim"}, + content_type="application/json") + self.assertLess(resp.status_code, 500) + kb.refresh_from_db() + self.assertEqual(kb.external_id, "mine") + + +class UntrustedSpanTest(TestCase): + def test_malformed_tool_calls_do_not_raise(self): + from infra.ui import _llm_tool_decision + + for msgs in ([{"tool_calls": 5}], [{"tool_calls": [{"function": "x"}]}], [{"tool_calls": ["a"]}]): + self.assertIsNone(_llm_tool_decision({}, {"gen_ai.output.messages": json.dumps(msgs)})) + ok = [{"tool_calls": [{"function": {"name": "lookup", "arguments": "{}"}}]}] + self.assertIsNotNone(_llm_tool_decision({}, {"gen_ai.output.messages": json.dumps(ok)})) + + +class PromptJudgeBadIdsTest(TestCase): + def test_non_numeric_ids_do_not_500(self): + user = User.objects.create_user(username="pj", password="pass12345") + project = Project.objects.create(name="PJ", slug="pj") + ProjectMembership.objects.create(project=project, user=user, role=ProjectMembership.Role.ADMIN) + _login(self.client, user, project) + self.client.raise_request_exception = False + for url in ["/prompts/new/?clone=abc", "/judges/new/?clone=abc"]: + self.assertLess(self.client.get(url).status_code, 500, url) + for url, data in [("/prompts/", {"action": "delete", "prompt_id": "abc"}), + ("/prompts/", {"action": "delete_version", "version_id": "abc"}), + ("/judges/", {"action": "delete", "judge_id": "abc"}), + ("/judges/", {"action": "delete_version", "version_id": "abc"})]: + self.assertLess(self.client.post(url, data).status_code, 500, data) + + +@override_settings() +class AgentPromptDataAttributeTest(TestCase): + def test_prompt_versions_not_escapejs_in_attributes(self): + for path in ("templates/agents/agent_detail.html", "templates/partials/run_settings.html"): + with open(path) as f: + self.assertNotIn("content|escapejs", f.read(), path) + + +class ToolCallNamesTest(TestCase): + def test_all_names_returned_for_multi_tool_step(self): + from infra.ui import _llm_tool_call_names + + msgs = [{"tool_calls": [{"function": {"name": "a"}}, {"function": {"name": "b"}}, {"function": {"name": "c"}}]}] + self.assertEqual(_llm_tool_call_names({"gen_ai.output.messages": json.dumps(msgs)}), ["a", "b", "c"]) diff --git a/infra/tests/test_pass24_regressions.py b/infra/tests/test_pass24_regressions.py new file mode 100644 index 00000000..de675dba --- /dev/null +++ b/infra/tests/test_pass24_regressions.py @@ -0,0 +1,195 @@ +"""Regression tests for audit pass 24: agent rows used as auditor/judge, edits against the pinned revision.""" +from django.test import TestCase + +from accounts.models import Project, ProjectMembership, User +from infra.tests.factories import AgentFactory, AuditRunFactory, ProjectFactory +from model_registry.services import agent_target_model + + +class AgentRowAuditorRoleTest(TestCase): + def test_deleted_agent_row_used_as_auditor_is_not_repointed(self): + project = ProjectFactory() + a = AgentFactory(project=project, external_id="studio.a") + ref_a = agent_target_model(a) + run = AuditRunFactory(project=project, auditor_model=ref_a) + a.delete() + b = AgentFactory(project=project, external_id="studio.b") + ref_b = agent_target_model(b) + ref_a.refresh_from_db() + run.refresh_from_db() + self.assertNotEqual(ref_a.pk, ref_b.pk) + self.assertEqual(run.auditor_model.model_id, "studio.a") + + def test_deleted_agent_row_used_as_judge_is_not_repointed(self): + project = ProjectFactory() + a = AgentFactory(project=project, external_id="studio.a") + ref_a = agent_target_model(a) + AuditRunFactory(project=project, judge_model=ref_a) + a.delete() + b = AgentFactory(project=project, external_id="studio.b") + agent_target_model(b) + ref_a.refresh_from_db() + self.assertEqual(ref_a.model_id, "studio.a") + + +class EditBackToLatestTest(TestCase): + def setUp(self): + from scenarios.models import ScenarioSet + from scenarios.services import create_scenario + + self.user = User.objects.create_user(username="rv", password="pass12345") + self.project = Project.objects.create(name="RV", slug="rv") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.sa = ScenarioSet.objects.create(project=self.project, name="A") + self.sb = ScenarioSet.objects.create(project=self.project, name="B") + self.s = create_scenario(project=self.project, user=self.user, key="s", title="S", description="old", + expected_behavior=["x"]) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def _desc(self, sset): + latest = sset.versions.order_by("-version").first() + return {i.scenario_id: i.revision.description for i in latest.items.select_related("revision")}[self.s.id] + + def test_edit_in_other_set_to_latest_text_is_published(self): + from scenarios.services import publish_scenario_set_version + + publish_scenario_set_version(scenario_set=self.sa, user=self.user, scenario_ids=[self.s.id]) + publish_scenario_set_version(scenario_set=self.sb, user=self.user, scenario_ids=[self.s.id]) + # Edit from set A -> rev2 "new" + self.client.post(f"/scenarios/edit/{self.s.id}/", { + "title": "S", "description": "new", "expected_behavior": "x", "set_id": str(self.sa.id)}) + self.assertEqual(self._desc(self.sa), "new") + self.assertEqual(self._desc(self.sb), "old") + # In set B (showing "old") the user types "new" and saves. + self.client.post(f"/scenarios/edit/{self.s.id}/", { + "title": "S", "description": "new", "expected_behavior": "x", "set_id": str(self.sb.id)}) + self.assertEqual(self._desc(self.sb), "new") + + def test_after_revert_edit_back_is_published(self): + from scenarios.services import publish_scenario_set_version + + publish_scenario_set_version(scenario_set=self.sa, user=self.user, scenario_ids=[self.s.id]) + self.client.post(f"/scenarios/edit/{self.s.id}/", { + "title": "S", "description": "new", "expected_behavior": "x", "set_id": str(self.sa.id)}) + self.client.post(f"/scenarios/revert/{self.sa.id}/", {"target_version": "1"}) + self.assertEqual(self._desc(self.sa), "old") + self.client.post(f"/scenarios/edit/{self.s.id}/", { + "title": "S", "description": "new", "expected_behavior": "x", "set_id": str(self.sa.id)}) + self.assertEqual(self._desc(self.sa), "new") + + +class StaleScenarioFormTest(TestCase): + """A form opened on an older revision must not republish its stale text.""" + + def setUp(self): + from scenarios.models import ScenarioSet + from scenarios.services import create_scenario, publish_scenario_set_version + + self.user = User.objects.create_user(username="ov", password="pass12345") + self.project = Project.objects.create(name="OV", slug="ov") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.sa = ScenarioSet.objects.create(project=self.project, name="A") + self.s = create_scenario(project=self.project, user=self.user, key="s", title="S", description="old", + expected_behavior=["x"]) + publish_scenario_set_version(scenario_set=self.sa, user=self.user, scenario_ids=[self.s.id]) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def test_stale_form_is_refused_and_old_version_view_has_no_edit_form(self): + v1_rev = self.sa.versions.get(version=1).items.get().revision_id + self.client.post(f"/scenarios/edit/{self.s.id}/", { + "title": "S", "description": "new", "expected_behavior": "x", "set_id": str(self.sa.id), + "base_revision_id": str(v1_rev)}) + self.assertEqual(self.sa.versions.count(), 2) + html = self.client.get(f"/scenarios/?set={self.sa.id}&version=1").content.decode() + self.assertNotIn(f"openEditForm({self.s.id}", html) + # A stale form (opened on v1) posting a title-only change must not revert content. + self.client.post(f"/scenarios/edit/{self.s.id}/", { + "title": "S renamed", "description": "old", "expected_behavior": "x", "set_id": str(self.sa.id), + "base_revision_id": str(v1_rev)}) + latest = self.sa.versions.order_by("-version").first() + self.assertEqual(latest.items.get(scenario=self.s).revision.description, "new") + + +class StaleJudgeFormTest(TestCase): + def setUp(self): + from judges.services import create_judge + + self.user = User.objects.create_user(username="sj", password="pass12345") + self.project = Project.objects.create(name="SJ", slug="sj") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.judge = create_judge(project=self.project, name="J1", user=self.user, output="severity", + criteria="old criteria", options={}) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def _form(self, base, **kw): + data = {"name": "J1", "description": "", "start": "format:severity", "criteria": "old criteria", + "probe_prompt": "", "dimensions": "", "question": "", "pass_when": "yes", "note": "", + "base_version_id": str(base)} + data.update(kw) + return data + + def test_stale_form_cannot_revert_newer_criteria(self): + v1 = self.judge.latest.pk + self.client.post(f"/judges/{self.judge.id}/", self._form(v1, criteria="NEW strict criteria")) + self.assertEqual(self.judge.latest.criteria, "NEW strict criteria") + self.client.post(f"/judges/{self.judge.id}/", self._form(v1, name="J1 renamed")) + self.judge.refresh_from_db() + self.assertEqual(self.judge.latest.criteria, "NEW strict criteria") + self.assertEqual(self.judge.name, "J1") # rename waits for the version check + + def test_refused_save_reloads_latest_instead_of_arming_stale_form(self): + v1 = self.judge.latest.pk + self.client.post(f"/judges/{self.judge.id}/", self._form(v1, criteria="NEW strict criteria")) + resp = self.client.post(f"/judges/{self.judge.id}/", self._form(v1, name="J1 renamed")) + self.assertEqual(resp.status_code, 302) + page = self.client.get(resp["Location"]).content.decode() + self.assertIn("NEW strict criteria", page) + self.assertNotIn(">old criteria<", page) + + def test_explicit_restore_of_old_version_still_works(self): + v1 = self.judge.latest.pk + self.client.post(f"/judges/{self.judge.id}/", self._form(v1, criteria="NEW")) + self.client.post(f"/judges/{self.judge.id}/", {**self._form(v1), "restore": "1"}) + self.assertEqual(self.judge.latest.criteria, "old criteria") + + +class PublicRunPageUrlTest(TestCase): + def test_anonymous_public_run_page_hides_endpoint_urls(self): + from audits.models import AuditRun, Experiment + from infra.tests.factories import AuditRunFactory, ProjectFactory + + project = ProjectFactory() + experiment = Experiment.objects.create(project=project, name="Pub", visibility=Experiment.Visibility.PUBLIC) + run = AuditRunFactory(project=project, status="completed") + snap = {**(run.target_config_snapshot or {}), "base_url": "http://frozen-internal.corp/v1"} + AuditRun.objects.filter(pk=run.pk).update(experiment=experiment, target_config_snapshot=snap) + run.target_model.connection.base_url = "http://live-changed.corp/v1" + run.target_model.connection.save() + html = self.client.get(f"/runs/{run.id}/").content.decode() + self.assertNotIn("frozen-internal.corp", html) + self.assertNotIn("live-changed.corp", html) + + +class PublicRunAgentUrlTest(TestCase): + def test_anonymous_public_run_page_hides_agent_base_url(self): + from audits.models import AuditRun, Experiment + from infra.tests.factories import AuditRunFactory, ProjectFactory + + project = ProjectFactory() + experiment = Experiment.objects.create(project=project, name="Pub", visibility=Experiment.Visibility.PUBLIC) + run = AuditRunFactory(project=project, status="completed") + agent_snapshot = {"name": "Ag", "base_model": {"model_id": "m", "display_name": "M", "connection_id": 1, + "base_url": "http://agent-base.corp/v1", + "connection": {"base_url": "http://agent-base.corp/v1"}}} + AuditRun.objects.filter(pk=run.pk).update(experiment=experiment, agent_config_snapshot=agent_snapshot) + html = self.client.get(f"/runs/{run.id}/").content.decode() + self.assertNotIn("agent-base.corp", html) diff --git a/infra/tests/test_security_hardening.py b/infra/tests/test_security_hardening.py new file mode 100644 index 00000000..74e231c2 --- /dev/null +++ b/infra/tests/test_security_hardening.py @@ -0,0 +1,810 @@ +"""Regression tests for authorization and secret-handling defects found in the system audit.""" +from types import SimpleNamespace +from unittest import mock + +from django.test import TestCase + +from accounts.models import Project, ProjectMembership, User +from audits.models import Experiment +from model_registry.api_credential_services import upsert_credential, verify_bearer +from model_registry.services import ( + _same_origin, + connection_api_key, + is_platform_secret_name, +) + + +class ConnectionKeyTest(TestCase): + def test_platform_secret_is_never_used_as_connection_key(self): + conn = SimpleNamespace(api_key_direct="", secret_reference="DJANGO_SECRET_KEY", base_url="https://x.example/v1") + with mock.patch.dict("os.environ", {"DJANGO_SECRET_KEY": "s3cret"}): + self.assertEqual(connection_api_key(conn), "") + + def test_model_key_env_var_still_resolves(self): + conn = SimpleNamespace(api_key_direct="", secret_reference="OPENAI_API_KEY", base_url="https://x.example/v1") + with mock.patch.dict("os.environ", {"OPENAI_API_KEY": "sk-1"}): + self.assertEqual(connection_api_key(conn), "sk-1") + + def test_platform_names(self): + for name in ("DJANGO_SECRET_KEY", "POSTGRES_PASSWORD", "HATCHET_CLIENT_TOKEN", "workos_api_key"): + self.assertTrue(is_platform_secret_name(name), name) + self.assertFalse(is_platform_secret_name("OPENROUTER_API_KEY")) + + def test_upstream_match_is_exact_origin(self): + upstream = "http://open-webui:8080/chat" + self.assertTrue(_same_origin("http://open-webui:8080/chat/api", upstream)) + self.assertFalse(_same_origin("http://open-webui:8080@attacker.example/v1", upstream)) + self.assertFalse(_same_origin("http://open-webui:8080.attacker.example/v1", upstream)) + + +class LoginRedirectTest(TestCase): + def test_backslash_next_is_not_followed(self): + User.objects.create_user(username="bob", password="pass12345") + resp = self.client.post("/login/?next=/%5Cevil.example", {"username": "bob", "password": "pass12345"}) + self.assertEqual(resp.status_code, 302) + self.assertNotIn("evil.example", resp["Location"]) + + def test_local_next_is_followed(self): + User.objects.create_user(username="bob", password="pass12345") + resp = self.client.post("/login/?next=/runs/", {"username": "bob", "password": "pass12345"}) + self.assertEqual(resp["Location"], "/runs/") + + +class ExperimentVisibilityTest(TestCase): + def test_admin_of_other_workspace_cannot_publish(self): + user = User.objects.create_user(username="eve", password="pass12345") + shared = Project.objects.create(name="Shared", slug="shared") + own = Project.objects.create(name="Own", slug="own") + ProjectMembership.objects.create(project=shared, user=user, role=ProjectMembership.Role.VIEWER) + ProjectMembership.objects.create(project=own, user=user, role=ProjectMembership.Role.ADMIN) + exp = Experiment.objects.create(project=shared, name="E") + self.client.force_login(user) + session = self.client.session + session["active_project_id"] = shared.id + session.save() + resp = self.client.post(f"/experiments/{exp.id}/set-visibility/", {"visibility": "public"}) + self.assertEqual(resp.status_code, 403) + exp.refresh_from_db() + self.assertEqual(exp.visibility, Experiment.Visibility.PRIVATE) + + +class BearerRevocationTest(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="carol", password="pass12345") + self.project = Project.objects.create(name="P", slug="p") + self.membership = ProjectMembership.objects.create( + project=self.project, user=self.user, role=ProjectMembership.Role.AUDITOR + ) + self.token = upsert_credential(project=self.project, user=self.user).secret + + def test_active_member_key_works(self): + self.assertIsNotNone(verify_bearer(self.token)) + + def test_removed_member_key_stops_working(self): + self.membership.delete() + self.assertIsNone(verify_bearer(self.token)) + + def test_deactivated_user_key_stops_working(self): + self.user.is_active = False + self.user.save() + self.assertIsNone(verify_bearer(self.token)) + + +class CsvSafeTest(TestCase): + def test_formula_cells_are_neutralised(self): + from infra.export_utils import csv_safe + + self.assertEqual(csv_safe("=HYPERLINK(1)"), "'=HYPERLINK(1)") + self.assertEqual(csv_safe("@SUM(A1)"), "'@SUM(A1)") + self.assertEqual(csv_safe("plain"), "plain") + self.assertEqual(csv_safe(-3), -3) + + +class ScenarioImportViewTest(TestCase): + def setUp(self): + from scenarios.models import ScenarioSet + + self.user = User.objects.create_user(username="imp", password="pass12345") + self.project = Project.objects.create(name="I", slug="i") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.sset = ScenarioSet.objects.create(project=self.project, name="S") + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def _post(self, body): + import json + + return self.client.post(f"/scenarios/{self.sset.id}/import/", json.dumps(body), content_type="application/json") + + def test_non_list_scenarios_rejected(self): + self.assertEqual(self._post({"scenarios": "abc"}).status_code, 400) + + def test_keyless_items_get_distinct_scenarios(self): + from scenarios.models import Scenario + + items = [{"title": f"T{i}", "description": "d", "expected_behavior": ["x"]} for i in range(3)] + self.assertEqual(self._post({"scenarios": items}).status_code, 302) + self.assertEqual(Scenario.objects.filter(project=self.project).count(), 3) + + def test_failure_rolls_back_whole_import(self): + from scenarios.models import Scenario + + items = [{"key": "ok", "title": "OK", "description": "d", "expected_behavior": ["x"]}, {"key": "bad"}] + with mock.patch("infra.ui._publish_new_version", side_effect=RuntimeError("boom")): + self.assertEqual(self._post({"scenarios": items}).status_code, 400) + self.assertFalse(Scenario.objects.filter(project=self.project).exists()) + + +class AgentWriteGuardTest(TestCase): + def setUp(self): + from infra.tests.factories import AgentFactory + + self.viewer = User.objects.create_user(username="view", password="pass12345") + self.project = Project.objects.create(name="A", slug="a") + ProjectMembership.objects.create(project=self.project, user=self.viewer, role=ProjectMembership.Role.VIEWER) + self.agent = AgentFactory(project=self.project) + self.client.force_login(self.viewer) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def test_viewer_cannot_delete_agent(self): + from model_registry.models import Agent + + self.client.post(f"/agents/{self.agent.id}/delete/") + self.assertTrue(Agent.objects.filter(pk=self.agent.pk).exists()) + + def test_viewer_cannot_create_agent(self): + from model_registry.models import Agent + + before = Agent.objects.count() + self.client.post("/agents/new/", {"name": "X", "base_model": self.agent.base_model_id}) + self.assertEqual(Agent.objects.count(), before) + + def test_viewer_cannot_sync(self): + self.assertEqual(self.client.post("/agents/sync/").status_code, 403) + + +class RegistrationValidationTest(TestCase): + def test_duplicate_email_rejected(self): + User.objects.create_user(username="victim", password="pass12345", email="v@example.com") + self.client.post("/register/", {"username": "att", "password": "Str0ng-pass-123", "email": "V@example.com"}) + self.assertFalse(User.objects.filter(username="att").exists()) + + def test_weak_password_rejected(self): + self.client.post("/register/", {"username": "newbie", "password": "1"}) + self.assertFalse(User.objects.filter(username="newbie").exists()) + + def test_case_variant_username_rejected(self): + User.objects.create_user(username="alice", password="pass12345") + self.client.post("/register/", {"username": "Alice", "password": "Str0ng-pass-123"}) + self.assertEqual(User.objects.filter(username__iexact="alice").count(), 1) + + def test_valid_registration_succeeds(self): + resp = self.client.post("/register/", {"username": "fresh", "password": "Str0ng-pass-123", "email": "f@example.com"}) + self.assertEqual(resp.status_code, 302) + self.assertTrue(User.objects.filter(username="fresh").exists()) + + +class ConnectionsPageQueryCountTest(TestCase): + """The connections page must not run queries per connection (N+1).""" + + def _count(self, n): + from django.db import connection + from django.test.utils import CaptureQueriesContext + + from infra.tests.factories import ModelConnectionFactory, RegisteredModelFactory + + for _ in range(n): + conn = ModelConnectionFactory(project=self.project) + RegisteredModelFactory(project=self.project, connection=conn) + with CaptureQueriesContext(connection) as ctx: + self.assertEqual(self.client.get("/connections/").status_code, 200) + return len(ctx) + + def setUp(self): + self.user = User.objects.create_user(username="cq", password="pass12345") + self.project = Project.objects.create(name="Q", slug="q") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def test_query_count_flat_in_connection_count(self): + small = self._count(2) + large = self._count(8) + self.assertEqual(small, large) + + +class EmbeddingSearchLimitTest(TestCase): + def test_bad_limit_does_not_500(self): + admin = User.objects.create_superuser(username="su", password="pass12345") + self.client.force_login(admin) + fake = mock.Mock(status_code=200, json=list) + fake.raise_for_status = lambda: None + with mock.patch("requests.get", return_value=fake) as get: + resp = self.client.get("/api/admin/embedding-models/search/?limit=abc") + self.assertNotEqual(resp.status_code, 500) + self.assertEqual(get.call_args.kwargs["params"]["limit"], "15") + + +class OtlpCredentialApiTest(TestCase): + def setUp(self): + from infra.tests.factories import ModelConnectionFactory + + self.user = User.objects.create_user(username="ot", password="pass12345") + self.project = Project.objects.create(name="O", slug="o") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.conn = ModelConnectionFactory(project=self.project) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def test_revoke_returns_ok_and_disables(self): + from model_registry import otlp_services as otlp + + cred = otlp.create_credential(project=self.project, connection=self.conn, auth_mode="basic", user=self.user).credential + resp = self.client.post(f"/api/otlp/credentials/{cred.id}/revoke/") + self.assertEqual(resp.status_code, 200) + cred.refresh_from_db() + self.assertFalse(cred.enabled) + + def test_create_with_bad_auth_mode_is_400(self): + for mode in ("digest", 1): + resp = self.client.post( + "/api/otlp/credentials/create/", + {"connection_id": self.conn.id, "auth_mode": mode}, + content_type="application/json", + ) + self.assertEqual(resp.status_code, 400, mode) + + +class OpenWebUIEnvTest(TestCase): + def test_studio_secrets_not_inherited(self): + from chat.proxy import _owui_inherited_env + + env = {"DJANGO_SECRET_KEY": "s", "POSTGRES_PASSWORD": "p", "DATABASE_URL": "d", "WORKOS_API_KEY": "w", + "BOOTSTRAP_PASSWORD": "b", "HATCHET_CLIENT_TOKEN": "h", "PATH": "/bin", "ENABLE_RAG_HYBRID_SEARCH": "1"} + with mock.patch.dict("os.environ", env, clear=True): + out = _owui_inherited_env() + self.assertEqual(out, {"PATH": "/bin", "ENABLE_RAG_HYBRID_SEARCH": "1"}) + + +class RemovedMemberSessionTest(TestCase): + def test_removed_member_loses_active_workspace(self): + from audits.models import AuditRun # noqa: F401 (ensures app loaded) + + user = User.objects.create_user(username="bob", password="pass12345") + team = Project.objects.create(name="Team", slug="team") + membership = ProjectMembership.objects.create(project=team, user=user, role=ProjectMembership.Role.AUDITOR) + self.client.force_login(user) + session = self.client.session + session["active_project_id"] = team.id + session.save() + self.assertEqual(self.client.get("/runs/").wsgi_request.project, team) + membership.delete() + resp = self.client.get("/runs/") + self.assertNotEqual(resp.wsgi_request.project, team) + self.assertNotEqual(self.client.session.get("active_project_id"), team.id) + + +class MockEndpointBackupTest(TestCase): + def setUp(self): + from infra.tests.factories import ModelConnectionFactory + + self.project = Project.objects.create(name="M", slug="m") + self.ollama = ModelConnectionFactory(project=self.project, base_url="http://localhost:11434/v1", api_key_direct="") + self.anthropic = ModelConnectionFactory( + project=self.project, base_url="https://api.anthropic.com/v1", api_key_direct="sk-ant" + ) + + def test_mock_run_then_normal_start_restores_everything(self): + from simpleaudit_studio.cli import ( + _restore_real_model_endpoints, + _update_model_endpoints, + ) + + _update_model_endpoints("http://127.0.0.1:50001/v1") + _update_model_endpoints("http://127.0.0.1:50002/v1") # a second mock run keeps the first backup + self.anthropic.refresh_from_db() + self.assertEqual(self.anthropic.api_key_direct, "sk-ant") + _restore_real_model_endpoints() + self.ollama.refresh_from_db() + self.anthropic.refresh_from_db() + self.assertEqual(self.ollama.base_url, "http://localhost:11434/v1") + self.assertEqual(self.anthropic.base_url, "https://api.anthropic.com/v1") + self.assertEqual(self.anthropic.api_key_direct, "sk-ant") + + def test_url_edited_during_mock_run_is_kept(self): + from simpleaudit_studio.cli import ( + _restore_real_model_endpoints, + _update_model_endpoints, + ) + + _update_model_endpoints("http://127.0.0.1:50001/v1") + type(self.ollama).objects.filter(pk=self.ollama.pk).update(base_url="http://localhost:11500/v1") + _restore_real_model_endpoints() + self.ollama.refresh_from_db() + self.anthropic.refresh_from_db() + self.assertEqual(self.ollama.base_url, "http://localhost:11500/v1") + self.assertEqual(self.anthropic.base_url, "https://api.anthropic.com/v1") + + def test_edit_between_two_mock_runs_is_kept(self): + from simpleaudit_studio.cli import ( + _restore_real_model_endpoints, + _update_model_endpoints, + ) + + _update_model_endpoints("http://127.0.0.1:50001/v1") + type(self.ollama).objects.filter(pk=self.ollama.pk).update(base_url="http://localhost:11500/v1") + _update_model_endpoints("http://127.0.0.1:50002/v1") + _restore_real_model_endpoints() + self.ollama.refresh_from_db() + self.anthropic.refresh_from_db() + self.assertEqual(self.ollama.base_url, "http://localhost:11500/v1") + self.assertEqual(self.anthropic.base_url, "https://api.anthropic.com/v1") + + def test_normal_start_leaves_local_endpoints_alone(self): + from simpleaudit_studio.cli import _restore_real_model_endpoints + + _restore_real_model_endpoints() + self.ollama.refresh_from_db() + self.assertEqual(self.ollama.base_url, "http://localhost:11434/v1") + + +class AgentApiWriteGuardTest(TestCase): + """The DRF agents/knowledge-bases/tools API must refuse viewer and archived-workspace writes.""" + + def _login(self, role, archived=False): + from infra.tests.factories import AgentFactory + + user = User.objects.create_user(username=f"u-{role}-{archived}", password="pass12345") + project = Project.objects.create(name=f"P{role}{archived}", slug=f"p-{role}-{archived}", archived=archived) + ProjectMembership.objects.create(project=project, user=user, role=role) + self.agent = AgentFactory(project=project) + self.client.force_login(user) + session = self.client.session + session["active_project_id"] = project.id + session.save() + + def _assert_no_writes(self): + from model_registry.models import Agent + + before = Agent.objects.count() + self.assertEqual(self.client.post("/api/agents/", {"name": "x"}, content_type="application/json").status_code, 403) + self.assertEqual(self.client.put( + f"/api/agents/{self.agent.id}/", {"name": "y"}, content_type="application/json").status_code, 403) + self.assertEqual(self.client.delete(f"/api/agents/{self.agent.id}/").status_code, 403) + self.assertEqual(self.client.post("/api/knowledge-bases/", {"name": "k"}, content_type="application/json").status_code, 403) + self.assertEqual(self.client.post("/api/tools/", {"name": "t"}, content_type="application/json").status_code, 403) + self.assertEqual(Agent.objects.count(), before) + self.agent.refresh_from_db() + self.assertNotEqual(self.agent.name, "y") + + def test_viewer_cannot_write(self): + self._login(ProjectMembership.Role.VIEWER) + self._assert_no_writes() + + def test_archived_workspace_admin_cannot_write(self): + self._login(ProjectMembership.Role.ADMIN, archived=True) + self._assert_no_writes() + + def test_viewer_can_still_read(self): + self._login(ProjectMembership.Role.VIEWER) + self.assertEqual(self.client.get("/api/agents/").status_code, 200) + + +class ConnectionShareScopeTest(TestCase): + def test_cannot_share_into_workspace_not_administered(self): + from infra.tests.factories import ModelConnectionFactory + + user = User.objects.create_user(username="sh", password="pass12345") + own = Project.objects.create(name="Own", slug="own-s") + mine2 = Project.objects.create(name="Mine2", slug="mine2") + victim = Project.objects.create(name="Victim", slug="victim") + ProjectMembership.objects.create(project=own, user=user, role=ProjectMembership.Role.ADMIN) + ProjectMembership.objects.create(project=mine2, user=user, role=ProjectMembership.Role.ADMIN) + conn = ModelConnectionFactory(project=own, base_url="https://x.example/v1") + self.client.force_login(user) + session = self.client.session + session["active_project_id"] = own.id + session.save() + self.client.post("/connections/", { + "action": "edit_connection", "conn_id": conn.id, "conn_name": conn.name, + "conn_base_url": "https://x.example/v1", "conn_provider": "openai", "key_mode": "none", + "conn_visibility": "admins", "conn_shared_with": [str(victim.id), str(mine2.id)], + }) + shared = set(conn.shared_with.values_list("pk", flat=True)) + self.assertIn(mine2.id, shared) # the edit went through + self.assertNotIn(victim.id, shared) + + +class ArchivedWorkspaceWritesTest(TestCase): + def setUp(self): + from infra.tests.factories import ModelConnectionFactory + + self.user = User.objects.create_user(username="arc", password="pass12345") + self.project = Project.objects.create(name="Arc", slug="arc", archived=True) + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.conn = ModelConnectionFactory(project=self.project) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def test_otlp_credential_create_refused(self): + from model_registry.models import OTLPCredential + + resp = self.client.post("/api/otlp/credentials/create/", {"connection_id": self.conn.id}, + content_type="application/json") + self.assertEqual(resp.status_code, 403) + self.assertFalse(OTLPCredential.objects.exists()) + + def test_archived_workspace_delete_refused(self): + from accounts.services import delete_workspace + from infra.exceptions import StableAPIError + + with self.assertRaises(StableAPIError): + delete_workspace(user=self.user, project=self.project) + self.assertTrue(Project.objects.filter(pk=self.project.pk).exists()) + + +class RemoteDeleteActorTest(TestCase): + def test_remote_delete_runs_as_creator_or_requester_not_superuser(self): + from model_registry.services import _remote_actor + + User.objects.create_superuser(username="root", password="pass12345") + requester = User.objects.create_user(username="req", password="pass12345") + creator = User.objects.create_user(username="cre", password="pass12345") + from types import SimpleNamespace + + self.assertEqual(_remote_actor(SimpleNamespace(created_by=creator), requester), creator) + self.assertEqual(_remote_actor(SimpleNamespace(created_by=None), requester), requester) + + +class WorkOSSignInTest(TestCase): + def _wo(self, uid, email): + from types import SimpleNamespace + + client = mock.Mock() + client.user_management.authenticate_with_magic_auth.return_value = SimpleNamespace( + user=SimpleNamespace(id=uid, email=email, first_name="", last_name="") + ) + return mock.patch("accounts.workos_auth._client", return_value=client) + + def test_new_signup_refused_when_registration_disabled(self): + from accounts.workos_auth import authenticate_magic_auth + + with self._wo("wo_1", "new@example.com"), self.assertRaises(ValueError): + authenticate_magic_auth("c", "new@example.com", allow_create=False) + self.assertFalse(User.objects.filter(workos_user_id="wo_1").exists()) + + def test_existing_workos_user_can_sign_in_when_registration_disabled(self): + from accounts.workos_auth import authenticate_magic_auth + + User.objects.create_user(username="w", email="w@example.com", workos_user_id="wo_2") + with self._wo("wo_2", "w@example.com"): + _user, created = authenticate_magic_auth("c", "w@example.com", allow_create=False) + self.assertFalse(created) + + def test_email_collision_refused(self): + from accounts.workos_auth import authenticate_magic_auth + + User.objects.create_user(username="squatter", password="pass12345", email="Victim@example.com") + with self._wo("wo_3", "victim@example.com"), self.assertRaises(ValueError): + authenticate_magic_auth("c", "victim@example.com") + + +class AddMemberRoleBypassTest(TestCase): + def test_add_member_cannot_change_existing_role(self): + admin = User.objects.create_user(username="ad", password="pass12345") + other = User.objects.create_user(username="ot2", password="pass12345") + project = Project.objects.create(name="M2", slug="m2") + ProjectMembership.objects.create(project=project, user=admin, role=ProjectMembership.Role.ADMIN) + ProjectMembership.objects.create(project=project, user=other, role=ProjectMembership.Role.VIEWER) + self.client.force_login(admin) + resp = self.client.post(f"/api/projects/{project.id}/members/add/", {"username": "ad", "role": "viewer"}, + content_type="application/json") + self.assertEqual(resp.status_code, 409) + self.assertEqual(ProjectMembership.objects.get(project=project, user=admin).role, ProjectMembership.Role.ADMIN) + + +class SnapshotKeyOriginTest(TestCase): + """A run sends its key only to the host the key was resolved for.""" + + def test_repointed_connection_yields_no_key(self): + from infra.engine import snapshot_api_key + from infra.tests.factories import ModelConnectionFactory + + project = Project.objects.create(name="SK", slug="sk") + conn = ModelConnectionFactory(project=project, base_url="http://127.0.0.1:8080/chat/api/v1", api_key_direct="sk-x") + frozen = {"connection_id": conn.id, "base_url": "https://attacker.example/v1"} + self.assertIsNone(snapshot_api_key(frozen)) + + def test_same_origin_still_resolves(self): + from infra.engine import snapshot_api_key + from infra.tests.factories import ModelConnectionFactory + + project = Project.objects.create(name="SK2", slug="sk2") + conn = ModelConnectionFactory(project=project, base_url="https://api.example.com/v1", api_key_direct="sk-y") + self.assertEqual(snapshot_api_key({"connection_id": conn.id, "base_url": "https://api.example.com/v1/"}), "sk-y") + + +class AgentBaseKeyOriginTest(TestCase): + def test_repointed_agent_base_connection_gets_no_key(self): + from infra.engine import auditor_kwargs, augment_agent_generation + from infra.tests.factories import ModelConnectionFactory + + project = Project.objects.create(name="AB", slug="ab") + conn = ModelConnectionFactory(project=project, base_url="http://127.0.0.1:8080/chat/api/v1", api_key_direct="sk-live") + agent_snapshot = {"base_model": {"connection_id": conn.id, "model_id": "m", "provider": "openai", + "base_url": "https://attacker.example/v1", "default_parameters": {}}} + gen = augment_agent_generation({}, agent_snapshot) + snap = {"model_id": "studio.agent-1", "provider": "openai", "base_url": "http://w.local/v1"} + kwargs, _ = auditor_kwargs(target=snap, auditor=snap, judge=snap, generation=gen) + self.assertEqual(kwargs["base_url"], "https://attacker.example/v1") + self.assertNotEqual(kwargs["api_key"], "sk-live") + + +class ScenarioKeyCollisionTest(TestCase): + def setUp(self): + from scenarios.models import ScenarioSet + + self.user = User.objects.create_user(username="sk1", password="pass12345") + self.project = Project.objects.create(name="SC", slug="sc") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.sset = ScenarioSet.objects.create(project=self.project, name="S") + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + self.client.raise_request_exception = False + + def test_same_title_twice_does_not_500(self): + from scenarios.models import Scenario + + for _ in range(2): + resp = self.client.post("/scenarios/create/", {"name": "dup", "set_id": str(self.sset.id), "description": "d"}) + self.assertLess(resp.status_code, 500) + self.assertEqual(Scenario.objects.filter(project=self.project, title="dup").count(), 2) + + def test_reimport_of_archived_key_restores_it(self): + import json + + from django.utils import timezone + + from scenarios.models import Scenario + + Scenario.objects.create(project=self.project, key="K", title="old", archived_at=timezone.now()) + body = {"scenarios": [{"key": "K", "title": "Reimport", "description": "d", "expected_behavior": ["x"]}]} + self.client.post(f"/scenarios/{self.sset.id}/import/", json.dumps(body), content_type="application/json") + scenario = Scenario.objects.get(project=self.project, key="K") + self.assertIsNone(scenario.archived_at) + latest = self.sset.versions.order_by("-version").first() + self.assertTrue(latest.items.filter(scenario=scenario).exists()) + + +class ScenarioRevertTest(TestCase): + def setUp(self): + from scenarios.models import ScenarioSet + from scenarios.services import create_scenario + + self.user = User.objects.create_user(username="rv", password="pass12345") + self.project = Project.objects.create(name="RV", slug="rv") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + self.sset = ScenarioSet.objects.create(project=self.project, name="S") + self.a = create_scenario(project=self.project, user=self.user, key="a", title="A", description="old", + expected_behavior=["x"]) + self.c = create_scenario(project=self.project, user=self.user, key="c", title="C", description="c", + expected_behavior=["x"]) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.project.id + session.save() + + def test_revert_survives_an_edit_of_another_scenario(self): + from scenarios.services import ( + publish_scenario_set_version, + update_scenario_content, + ) + + publish_scenario_set_version(scenario_set=self.sset, user=self.user, scenario_ids=[self.a.id, self.c.id]) + update_scenario_content(scenario=self.a, user=self.user, description="BAD", expected_behavior=["x"]) + publish_scenario_set_version(scenario_set=self.sset, user=self.user, scenario_ids=[self.a.id, self.c.id]) + self.client.post(f"/scenarios/revert/{self.sset.id}/", {"target_version": "1"}) + # An ordinary edit of C publishes the next version: A must stay reverted. + self.client.post(f"/scenarios/edit/{self.c.id}/", { + "title": "C", "description": "c2", "expected_behavior": "x", "set_id": str(self.sset.id), + }) + latest = self.sset.versions.order_by("-version").first() + items = {i.scenario_id: i.revision.description for i in latest.items.select_related("revision")} + self.assertEqual(items[self.a.id], "old") + self.assertEqual(items[self.c.id], "c2") + + def test_revert_restores_old_text_and_deleted_scenarios(self): + from django.utils import timezone + + from scenarios.services import ( + publish_scenario_set_version, + update_scenario_content, + ) + + publish_scenario_set_version(scenario_set=self.sset, user=self.user, scenario_ids=[self.a.id, self.c.id]) + update_scenario_content(scenario=self.a, user=self.user, description="NEW", expected_behavior=["x"]) + type(self.c).objects.filter(pk=self.c.pk).update(archived_at=timezone.now()) + publish_scenario_set_version(scenario_set=self.sset, user=self.user, scenario_ids=[self.a.id]) + + self.client.post(f"/scenarios/revert/{self.sset.id}/", {"target_version": "1"}) + latest = self.sset.versions.order_by("-version").first() + self.assertEqual(latest.version, 3) + items = {i.scenario_id: i.revision for i in latest.items.select_related("revision")} + self.assertEqual(items[self.a.id].description, "old") + self.assertIn(self.c.id, items) + self.c.refresh_from_db() + self.assertIsNone(self.c.archived_at) + + +class JudgeGuidanceAndPromptPinTest(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="jg", password="pass12345") + self.project = Project.objects.create(name="JG", slug="jg") + ProjectMembership.objects.create(project=self.project, user=self.user, role=ProjectMembership.Role.ADMIN) + + def test_agentic_guidance_survives_create_and_spec(self): + from judges.services import create_judge, make_spec + + judge = create_judge(project=self.project, name="J", user=self.user, output="severity", criteria="C", + probe_prompt="", options={"agentic_guidance": "Keep me"}) + self.assertEqual(judge.latest.options.get("agentic_guidance"), "Keep me") + spec = make_spec(base="safety", options={"agentic_guidance": "G"}) + self.assertEqual(spec["options"]["agentic_guidance"], "G") + + def test_prompt_version_pinned_by_run_cannot_be_deleted(self): + from audits.models import AuditRun + from infra.tests.factories import AuditRunFactory + from prompts.models import SystemPrompt + from prompts.services import delete_version + + prompt = SystemPrompt.objects.create(project=self.project, name="P") + v1 = prompt.versions.create(version=1, content="A ") + prompt.versions.create(version=2, content="B") + run = AuditRunFactory(project=self.project) + AuditRun.objects.filter(pk=run.pk).update(system_prompt_version=v1) + with self.assertRaises(ValueError): + delete_version(v1) + + +class SharedScenarioDeleteTest(TestCase): + def test_delete_in_one_set_keeps_it_in_the_other(self): + from scenarios.models import ScenarioSet + from scenarios.services import create_scenario, publish_scenario_set_version + + user = User.objects.create_user(username="sd", password="pass12345") + project = Project.objects.create(name="SD", slug="sd") + ProjectMembership.objects.create(project=project, user=user, role=ProjectMembership.Role.ADMIN) + src = ScenarioSet.objects.create(project=project, name="Src") + dst = ScenarioSet.objects.create(project=project, name="Dst") + shared = create_scenario(project=project, user=user, key="s", title="S", description="d", expected_behavior=["x"]) + other = create_scenario(project=project, user=user, key="o", title="O", description="d", expected_behavior=["x"]) + publish_scenario_set_version(scenario_set=src, user=user, scenario_ids=[shared.id, other.id]) + publish_scenario_set_version(scenario_set=dst, user=user, scenario_ids=[shared.id, other.id]) + self.client.force_login(user) + session = self.client.session + session["active_project_id"] = project.id + session.save() + self.client.post(f"/scenarios/delete/{shared.id}/", {"set_id": str(dst.id)}) + dst_latest = dst.versions.order_by("-version").first() + self.assertFalse(dst_latest.items.filter(scenario=shared).exists()) + shared.refresh_from_db() + self.assertIsNone(shared.archived_at) # still used by Src + self.client.post(f"/scenarios/edit/{other.id}/", {"title": "O", "description": "d2", "expected_behavior": "x", + "set_id": str(src.id)}) + src_latest = src.versions.order_by("-version").first() + self.assertTrue(src_latest.items.filter(scenario=shared).exists()) + + +class ConnectionShareLevelTest(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="sl", password="pass12345") + self.own = Project.objects.create(name="SL", slug="sl") + self.other = Project.objects.create(name="SL2", slug="sl2") + for p in (self.own, self.other): + ProjectMembership.objects.create(project=p, user=self.user, role=ProjectMembership.Role.ADMIN) + self.client.force_login(self.user) + session = self.client.session + session["active_project_id"] = self.own.id + session.save() + + def _post(self, **extra): + data = {"conn_name": "C", "conn_base_url": "https://x.example/v1", "conn_provider": "openai", + "key_mode": "none", "conn_shared_with": [str(self.other.id)]} + data.update(extra) + return self.client.post("/connections/", data) + + def test_add_honours_shares_and_workspace_only_clears_them(self): + from model_registry.models import ModelConnection + + self._post(action="add_connection", conn_visibility="admins") + conn = ModelConnection.objects.get(project=self.own, name="C") + self.assertEqual(list(conn.shared_with.values_list("pk", flat=True)), [self.other.id]) + self._post(action="edit_connection", conn_id=conn.id, conn_visibility="workspace") + self.assertFalse(conn.shared_with.exists()) + + +class PasswordChangeRevokesTokensTest(TestCase): + def test_profile_and_admin_password_change_drop_auth_tokens(self): + from rest_framework.authtoken.models import Token + + from accounts.services import admin_update_user + + user = User.objects.create_user(username="pw", password="Old-pass-123x") + Token.objects.create(user=user) + self.client.force_login(user) + self.client.post("/profile/", {"form": "password", "current_password": "Old-pass-123x", + "new_password": "New-pass-456y", "confirm_password": "New-pass-456y"}) + user.refresh_from_db() + self.assertTrue(user.check_password("New-pass-456y")) + self.assertFalse(Token.objects.filter(user=user).exists()) + + Token.objects.create(user=user) + admin = User.objects.create_superuser(username="root2", password="pass12345") + admin_update_user(admin_user=admin, target=user, password="Reset-pass-789z") + self.assertFalse(Token.objects.filter(user=user).exists()) + + +class RunNowOwnerCheckTest(TestCase): + def test_run_now_refused_when_creator_lost_role(self): + from unittest import mock + + from audits.models import AuditRun, Monitor + from infra.tests.factories import AuditRunFactory + + admin = User.objects.create_user(username="adm", password="pass12345") + creator = User.objects.create_user(username="cr", password="pass12345") + project = Project.objects.create(name="RN", slug="rn") + ProjectMembership.objects.create(project=project, user=admin, role=ProjectMembership.Role.ADMIN) + template = AuditRunFactory(project=project) + monitor = Monitor.objects.create( + project=project, name="M", scenario_set=template.scenario_set_version.scenario_set, + scenario_set_version=template.scenario_set_version, target_model=template.target_model, + auditor_model=template.auditor_model, judge_model=template.judge_model, + judge=template.judge_version.judge, judge_version=template.judge_version, + generation_parameters={}, interval_hours=24, created_by=creator, + ) + self.client.force_login(admin) + session = self.client.session + session["active_project_id"] = project.id + session.save() + before = AuditRun.objects.count() + with mock.patch("infra.ui.submit_audit_run"): + self.client.post(f"/monitors/{monitor.id}/run-now/") + self.assertEqual(AuditRun.objects.count(), before) + + +class SecondAdminWorkspaceOnlyTest(TestCase): + def test_workspace_only_clears_shares_made_by_another_admin(self): + from infra.tests.factories import ModelConnectionFactory + + a = User.objects.create_user(username="aa", password="pass12345") + b = User.objects.create_user(username="bb", password="pass12345") + w = Project.objects.create(name="W", slug="w-ws") + x = Project.objects.create(name="X", slug="x-ws") + ProjectMembership.objects.create(project=w, user=a, role=ProjectMembership.Role.ADMIN) + ProjectMembership.objects.create(project=x, user=a, role=ProjectMembership.Role.ADMIN) + ProjectMembership.objects.create(project=w, user=b, role=ProjectMembership.Role.ADMIN) + conn = ModelConnectionFactory(project=w, name="C", base_url="https://x.example/v1", visibility="admins") + conn.shared_with.set([x]) + self.client.force_login(b) + session = self.client.session + session["active_project_id"] = w.id + session.save() + self.client.post("/connections/", {"action": "edit_connection", "conn_id": conn.id, "conn_name": "C", + "conn_base_url": "https://x.example/v1", "conn_provider": "openai", + "key_mode": "none", "conn_visibility": "workspace"}) + conn.refresh_from_db() + self.assertEqual(conn.visibility, "workspace") + self.assertFalse(conn.shared_with.exists()) diff --git a/infra/tests/test_seed_demo_agent.py b/infra/tests/test_seed_demo_agent.py index feca8cbd..d2b1438f 100644 --- a/infra/tests/test_seed_demo_agent.py +++ b/infra/tests/test_seed_demo_agent.py @@ -95,6 +95,13 @@ def test_is_idempotent(self): self.assertEqual(Tool.objects.filter(project=self.project).count(), 1) self.assertEqual(Agent.objects.get(project=self.project).retrieval_settings["top_k"], 5) + def test_reseed_keeps_links_the_user_changed(self): + seed_demo_agent(self.project, self.user) + agent = Agent.objects.get(project=self.project) + agent.tools.clear() + seed_demo_agent(self.project, self.user) + self.assertEqual(agent.tools.count(), 0) + def test_seeds_a_versioned_prompt_the_agent_uses(self): from prompts.models import SystemPrompt diff --git a/infra/trace_target.py b/infra/trace_target.py index 29e3e86b..83dad756 100644 --- a/infra/trace_target.py +++ b/infra/trace_target.py @@ -40,23 +40,38 @@ def _saved_chat_content(message: dict[str, Any]) -> str: return content or "" -# Cached OWUI JWT token (avoid re-authenticating on every turn). -_owui_jwt_cache: dict[str, Any] = {"token": None, "expires": 0.0} +# Cached OWUI JWT tokens per origin (avoid re-authenticating on every turn). +_owui_jwt_cache: dict[str, tuple[str, float]] = {} + + +def _is_studio_owui(base_url: str) -> bool: + """Whether ``base_url`` is Studio's own Open WebUI (exact origin match). + + Only that server may receive the trusted-header admin JWT; a user-chosen + connection URL that merely contains ``/chat/api/v1`` must never get it. + """ + from chat import config as chat_config + from model_registry.services import _same_origin + + return bool(chat_config.ENABLED and base_url and _same_origin(base_url, chat_config.UPSTREAM)) async def _owui_get_jwt(base_url: str) -> str | None: """Obtain a JWT from OWUI's trusted-header signin endpoint.""" import time + if not _is_studio_owui(base_url): + return None + # The base_url is like http://127.0.0.1:8080/api/v1 or + # http://127.0.0.1:8080/chat/api/v1 — we need the origin for /api/v1/auths/signin + origin = base_url.rsplit("/api/v1", 1)[0].rstrip("/") now = time.monotonic() - if _owui_jwt_cache["token"] and _owui_jwt_cache["expires"] > now: - return _owui_jwt_cache["token"] + cached = _owui_jwt_cache.get(origin) + if cached and cached[1] > now: + return cached[0] import httpx - # The base_url is like http://127.0.0.1:8080/api/v1 or - # http://127.0.0.1:8080/chat/api/v1 — we need the origin for /api/v1/auths/signin - origin = base_url.rsplit("/api/v1", 1)[0].rstrip("/") url = f"{origin}/api/v1/auths/signin" headers = { "Content-Type": "application/json", @@ -71,9 +86,8 @@ async def _owui_get_jwt(base_url: str) -> str | None: data = resp.json() token = data.get("token") if token: - _owui_jwt_cache["token"] = token # OWUI default JWT expiry is 4 weeks; cache for 1 hour to be safe. - _owui_jwt_cache["expires"] = now + 3600.0 + _owui_jwt_cache[origin] = (token, now + 3600.0) return token except Exception: logging.getLogger(__name__).warning("Failed to obtain OWUI JWT from %s", url, exc_info=True) @@ -220,7 +234,7 @@ async def _openwebui_saved_chat_call( raw_client = getattr(client, "client", None) base_url = str(getattr(raw_client, "base_url", "")) extra_body = kwargs.get("extra_body") or {} - if raw_client is None or "/chat/api/v1" not in base_url: + if raw_client is None or "/chat/api/v1" not in base_url or not _is_studio_owui(base_url): return None import asyncio @@ -609,7 +623,7 @@ async def send( # through OWUI (the only path that exports OTLP spans). raw_client = getattr(self._client, "client", None) base_url = str(getattr(raw_client, "base_url", "") or "") - if "/chat/api/v1" in base_url: + if "/chat/api/v1" in base_url and _is_studio_owui(base_url): jwt = await _owui_get_jwt(base_url) if jwt: raw_client.api_key = jwt diff --git a/infra/ui.py b/infra/ui.py index 9a5f599d..64c1a336 100644 --- a/infra/ui.py +++ b/infra/ui.py @@ -6,20 +6,24 @@ import itertools import json import logging +import math import os import re import time +import uuid from datetime import UTC, datetime from django.contrib import messages from django.contrib.auth import login, logout from django.contrib.auth.forms import AuthenticationForm from django.contrib.auth.mixins import LoginRequiredMixin +from django.db import transaction from django.db.models import Count, F, Max, ProtectedError, Q, RestrictedError from django.http import Http404, HttpResponse, HttpResponseForbidden, JsonResponse from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.utils import timezone +from django.utils.http import url_has_allowed_host_and_scheme from django.views.generic import DetailView, TemplateView, View from accounts import workos_auth @@ -28,8 +32,14 @@ from audits.comparison import compare_runs from audits.events import ScenarioResult from audits.models import AuditRun -from audits.services import create_audit_run, frozen_name, submit_audit_run +from audits.services import ( + create_audit_run, + frozen_name, + result_severity, + submit_audit_run, +) from infra.chat_feature import chat_enabled +from infra.export_utils import csv_safe from infra.seo import SITE_NAME, _share_context from scenarios.models import ( Scenario, @@ -141,11 +151,23 @@ def write_block_reason(request) -> str | None: return None +def is_id(value) -> bool: + """Whether ``value`` is a string holding a database id. + + ASCII digits only (``"²".isdigit()`` is True but ``int()`` rejects it) and + at most 18 of them: longer values overflow SQLite integers or exceed + Python's int-conversion limit, and would 500 instead of matching nothing. + """ + return isinstance(value, str) and value.isascii() and value.isdecimal() and len(value) <= 18 + + def _int(value, default: int = 0) -> int: """``value`` as an int (query and form values), ``default`` when it isn't one.""" + if isinstance(value, str) and not is_id(value.strip().lstrip("-")): + return default try: return int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): return default @@ -170,7 +192,10 @@ def _merge_structured_agentic_fields(metadata: dict, post) -> dict: if "agentic_rules_json" in post: raw_rules = str(post.get("agentic_rules_json") or "").strip() if raw_rules: - rules = json.loads(raw_rules) + try: + rules = json.loads(raw_rules) + except RecursionError as exc: # absurdly nested JSON: a client error + raise ValueError("Rules JSON is nested too deeply.") from exc problems = validate_rules(rules) if problems: raise ValueError("; ".join( @@ -218,7 +243,7 @@ class HealthView(AdminRequiredMixin, TemplateView): def get_context_data(self, **kw): from infra.health import collect_health - kw.setdefault("health", collect_health()) + kw.setdefault("health", collect_health(self.request.user)) return super().get_context_data(**kw) @@ -266,7 +291,9 @@ def post(self, request, *args, **kwargs): if form.is_valid(): login(request, form.get_user()) next_url = request.GET.get("next") or request.POST.get("next") or "" - if next_url and next_url.startswith("/") and not next_url.startswith("//"): + if next_url and url_has_allowed_host_and_scheme( + next_url, allowed_hosts={request.get_host()}, require_https=request.is_secure() + ): return redirect(next_url) return redirect("dashboard") return self.render_to_response(self.get_context_data(form=form)) @@ -301,8 +328,14 @@ def post(self, request, *args, **kwargs): error = None if not username or not password: error = "Username and password are required." - elif User.objects.filter(username=username).exists(): + elif User.objects.filter(username__iexact=username).exists(): error = "Username already taken." + elif email and User.objects.filter(email__iexact=email).exists(): + # Emails identify chat accounts; a duplicate would sign the new + # user into the existing owner's Open WebUI account. + error = "Email already in use." + elif error := _registration_field_error(username, email, password): + pass else: user = User.objects.create_user(username=username, password=password, email=email) from accounts.services import grant_default_project @@ -313,6 +346,55 @@ def post(self, request, *args, **kwargs): return self.render_to_response(self.get_context_data(error=error)) +def _profile_field_error( + username: str, email: str, first_name: str, last_name: str, *, current_username: str = "", +) -> str | None: + """Same field rules as registration (and the User columns), for profile edits. + + The username format is checked only when it changes: accounts created + before the rule may hold usernames it rejects, and must still save. + """ + from django.contrib.auth.validators import UnicodeUsernameValidator + from django.core.exceptions import ValidationError + from django.core.validators import validate_email + + if len(username) > 150 or len(first_name) > 150 or len(last_name) > 150: + return "Names are limited to 150 characters." + if len(email) > 254: + return "Email is too long." + try: + if username != current_username: + UnicodeUsernameValidator()(username) + if email: + validate_email(email) + except ValidationError as exc: + return " ".join(exc.messages) + return None + + +def _registration_field_error(username: str, email: str, password: str) -> str | None: + """First validation error for a self-registration, else None.""" + from django.contrib.auth.password_validation import validate_password + from django.contrib.auth.validators import UnicodeUsernameValidator + from django.core.exceptions import ValidationError + from django.core.validators import validate_email + + from accounts.models import User + + if len(username) > 150: # User.username max_length + return "Username is too long (150 characters at most)." + if len(email) > 254: + return "Email is too long." + try: + UnicodeUsernameValidator()(username) + if email: + validate_email(email) + validate_password(password, user=User(username=username, email=email)) + except ValidationError as exc: + return " ".join(exc.messages) + return None + + def logout_view(request): logout(request) return redirect("login") @@ -342,7 +424,14 @@ def auto_login_view(request): expected = os.environ.get("SIMPLEAUDIT_AUTO_LOGIN_TOKEN", "") if not expected or not hmac.compare_digest(token, expected): raise Http404 - # Single use: clear it so the URL stops working after this request. + # Single use, durably: the autoreloader restarts this process from a parent + # that still holds the token, so clearing the env var alone is not enough. + from accounts.services import get_site_setting, set_site_setting + + digest = hashlib.sha256(expected.encode()).hexdigest() + if get_site_setting("auto_login_used") == digest: + raise Http404 + set_site_setting("auto_login_used", digest) os.environ.pop("SIMPLEAUDIT_AUTO_LOGIN_TOKEN", None) username = os.environ.get("BOOTSTRAP_USERNAME", "studio") user = User.objects.filter(username=username).first() @@ -420,11 +509,14 @@ def post(self, request): if not email or not code: return self.render_to_response(self.get_context_data(error="Missing email or code.")) try: + from accounts.services import registration_enabled + user, created = workos_auth.authenticate_magic_auth( code, email, ip_address=request.META.get("REMOTE_ADDR"), user_agent=request.META.get("HTTP_USER_AGENT", ""), + allow_create=registration_enabled(), ) except Exception as exc: logger.exception("WorkOS magic auth verification failed") @@ -551,7 +643,7 @@ def get_context_data(self, **kw): # Manage panel: ?manage=, only rendered with controls for admins. manage_id = self.request.GET.get("manage") managed = None - if manage_id and manage_id.isdigit(): + if manage_id and is_id(manage_id): candidate = next((c for c in cards if c["project"].id == int(manage_id)), None) if candidate: memberships = candidate["project"].memberships.select_related("user").order_by("created_at") @@ -629,7 +721,7 @@ def get_context_data(self, **kw): ws["has_content"] = workspace_has_content(project) if project else False manage_id = self.request.GET.get("manage") managed_members = None - if tab == "workspaces" and manage_id and manage_id.isdigit(): + if tab == "workspaces" and manage_id and is_id(manage_id): project = next((p for p in projects if p.id == int(manage_id)), None) if project: managed_members = list( @@ -651,7 +743,7 @@ def get_context_data(self, **kw): superuser_count=superuser_count, registration_enabled=get_site_setting("registration_enabled"), managed_members=managed_members, - manage_id=int(manage_id) if manage_id and manage_id.isdigit() else None, + manage_id=int(manage_id) if manage_id and is_id(manage_id) else None, ) return super().get_context_data(**kw) @@ -955,7 +1047,7 @@ class EmbeddingModelSearchView(SuperuserRequiredMixin, View): def get(self, request): query = (request.GET.get("q") or "").strip()[:200] - limit = min(int(request.GET.get("limit", 15)), self.MAX_RESULTS) + limit = max(1, min(_int(request.GET.get("limit"), 15), self.MAX_RESULTS)) params = { "filter": "sentence-transformers", "limit": str(limit), @@ -1091,6 +1183,9 @@ def post(self, request, *args, **kwargs): return self.render_to_response(self.get_context_data(error=error)) user.set_password(new_password) user.save(update_fields=["password"]) + from accounts.services import revoke_auth_tokens + + revoke_auth_tokens(user) update_session_auth_hash(request, user) messages.success(request, "Password updated.") return redirect("profile") @@ -1102,6 +1197,8 @@ def post(self, request, *args, **kwargs): if not username: error = "Username cannot be empty." + elif error := _profile_field_error(username, email, first_name, last_name, current_username=user.username): + pass elif User.objects.filter(username__iexact=username).exclude(pk=user.pk).exists(): error = "Username already exists." elif email and User.objects.filter(email__iexact=email).exclude(pk=user.pk).exists(): @@ -1124,7 +1221,7 @@ def post(self, request, *args, **kwargs): def _run_from_query(request, param: str): """The active project's run named by ``?=``, or None.""" raw = request.GET.get(param) or "" - if not raw.isdigit(): + if not is_id(raw): return None return ( AuditRun.objects.filter(id=int(raw), project=request.project) @@ -1374,7 +1471,7 @@ def _redesign(self, post, error=None): "name")} rep["repeat"] = rep["repeat"] or "once" source = None - if rep["clone_from"].isdigit(): + if is_id(rep["clone_from"]): source = AuditRun.objects.filter(pk=int(rep["clone_from"]), project=self.request.project).first() return self.render_to_response(self.get_context_data( error=error, sel=_design_selection(post=post), clone=_settings_prefill(post), rep=rep, @@ -1431,7 +1528,7 @@ def _launch_single(self, request, spec, repeat): with transaction.atomic(): experiment = Experiment.objects.create( project=p, - name=custom_name or f"{spec['target'].display_name} · {spec['scenario_set'].label}", + name=(custom_name or f"{spec['target'].display_name} · {spec['scenario_set'].label}")[:250], factors=factors_from_runs([run_spec]), created_by=request.user, ) @@ -1457,7 +1554,7 @@ def _launch_single(self, request, spec, repeat): ) if repeat: first_point = run - if repeat["start"] == "baseline" and repeat["baseline_run"].isdigit(): + if repeat["start"] == "baseline" and is_id(repeat["baseline_run"]): first_point = AuditRun.objects.filter(pk=int(repeat["baseline_run"]), project=p).first() monitor = create_monitor( project=p, @@ -1706,8 +1803,19 @@ def _launch(self, request): rows, runs, errors = self._rows_from_post(post, p) except Exception as e: # noqa: BLE001 - tampered or stale form: start again from the design return self._redesign(design, f"Could not read the review: {e}") + agent = None + if agent_id: + # The review rows carry the agent's base model; without the agent + # every run would audit the bare model instead of the agent. + from model_registry.models import Agent + + agent = Agent.objects.filter(pk=_int(agent_id), project=p, enabled=True).first() + if agent is None: + errors.append("The selected agent is not available in this workspace.") for r in runs: r["target_trace"] = target_trace + if agent is not None: + r["agent"] = agent if not agent_id: values = design.getlist(f"target_trace_{r['target'].id}") r["target_trace"] = "1" in values if values else None @@ -2052,20 +2160,37 @@ def post(self, request, monitor_id, action): monitor.next_run_at = next_after(monitor, timezone.now()) monitor.save(update_fields=["enabled", "next_run_at", "updated_at"]) messages.success(request, f"Monitor '{monitor.name}' {'resumed' if monitor.enabled else 'paused'}.") + elif action == "run-now" and not owner_authorized(monitor): + # Same rule as the scheduler and resume: runs are created on behalf of + # the monitor's creator, who must still hold admin or auditor role. + messages.error( + request, + "Cannot run: the monitor's creator no longer has admin or auditor role. Recreate it under your account.", + ) elif action == "run-now": - last = monitor.last_run - if last and last.status not in (AuditRun.Status.COMPLETED, AuditRun.Status.FAILED, AuditRun.Status.CANCELLED): - messages.error(request, f"Run #{last.id} from this monitor is still {last.status}.") + # Same row lock the scheduler tick takes: a double click, or a click + # racing a tick, must not start overlapping runs of one monitor. + from django.db import transaction + + run, refusal = None, None + try: + with transaction.atomic(): + locked = type(monitor).objects.select_for_update().get(pk=monitor.pk) + last = locked.last_run + if last and last.status not in AuditRun.TERMINAL_STATUSES: + refusal = f"Run #{last.id} from this monitor is still {last.status}." + else: + run = launch_monitor(locked) + locked.last_run = run + locked.last_error = "" + locked.save(update_fields=["last_run", "last_error", "updated_at"]) + except Exception as e: # noqa: BLE001 - surface creation failure to the user + messages.error(request, f"Could not launch: {e}") else: - try: - run = launch_monitor(monitor) - except Exception as e: # noqa: BLE001 - surface creation failure to the user - messages.error(request, f"Could not launch: {e}") + if refusal: + messages.error(request, refusal) else: - monitor.last_run = run - monitor.last_error = "" - monitor.save(update_fields=["last_run", "last_error", "updated_at"]) - submit_audit_run(run) + submit_audit_run(run) # after commit, so the worker sees the run return redirect(f"/runs/{run.id}/") elif action == "delete": name = monitor.name @@ -2091,21 +2216,20 @@ def get_context_data(self, **kw): versions = [] viewing_version = None set_param = self.request.GET.get("set", "").strip() - if set_param.isdigit(): + if is_id(set_param): selected = ScenarioSet.objects.filter(pk=int(set_param), project=p).first() if selected: versions = list(selected.versions.order_by("-version")) # Check if a specific version is being viewed ver_param = self.request.GET.get("version") - if ver_param and ver_param.isdigit(): + if ver_param and is_id(ver_param): viewing_version = selected.versions.filter(version=int(ver_param)).first() if not viewing_version and versions: viewing_version = versions[0] if viewing_version: + # Show everything the version contains (runs execute all of it); + # deleted scenarios carry a badge rather than being hidden. items = list(viewing_version.items.select_related("scenario", "revision")) - # For the latest version, hide archived scenarios (they're still in historical versions) - if viewing_version == versions[0]: - items = [i for i in items if i.scenario.archived_at is None] prev_version = (viewing_version.version - 1) if viewing_version and viewing_version.version > 1 else None # Collect unique categories for filter dropdown categories = sorted({i.scenario.category for i in items if i.scenario.category}) if items else [] @@ -2115,6 +2239,20 @@ def get_context_data(self, **kw): return super().get_context_data(**kw) +def _severity_ceiling(raw: str) -> str: + """The form's severity ceiling when it is a known level, else "" (no ceiling).""" + from scenarios.serializers import SEVERITY_CHOICES + + value = (raw or "").strip().lower() + return value if value in SEVERITY_CHOICES else "" + + +def _post_set_id(request) -> str: + """The form's ``set_id`` when it is a decimal id, else "" (checked before any write).""" + raw = request.POST.get("set_id", "").strip() + return raw if is_id(raw) else "" + + def _scenario_redirect(set_id: str | None): return redirect(f"/scenarios/?set={set_id}" if set_id else "/scenarios/") @@ -2134,6 +2272,8 @@ def _parse_decision_field(raw: str) -> tuple[dict | None, str | None]: decision = json.loads(text) except json.JSONDecodeError as exc: return None, f"Decision question must be valid JSON: {exc.msg}" + except RecursionError: + return None, "Decision question JSON is nested too deeply." from infra.engine import validate_decision_block try: @@ -2143,22 +2283,32 @@ def _parse_decision_field(raw: str) -> tuple[dict | None, str | None]: return decision, None -def _publish_new_version(sset, user, extra_scenario_ids=None): - """Auto-publish a new version capturing all current (non-archived) scenarios in the set.""" +def _publish_new_version(sset, user, extra_scenario_ids=None, changed_ids=None, exclude_ids=None): + """Auto-publish a new version capturing all current (non-archived) scenarios in the set. + + Scenarios this action did not touch keep the revision the latest version + froze (so a revert is not silently undone by an unrelated edit); the + changed and newly added ones take their newest revision. + """ latest = sset.versions.order_by("-version").first() + pins = {} if latest: - scenario_ids = list(latest.items.values_list("scenario_id", flat=True)) + latest_items = list(latest.items.select_related("revision").order_by("position")) + scenario_ids = [i.scenario_id for i in latest_items] + fresh = set(extra_scenario_ids or ()) | set(changed_ids or ()) + pins = {i.scenario_id: i.revision for i in latest_items if i.scenario_id not in fresh} else: scenario_ids = [] if extra_scenario_ids: for sid in extra_scenario_ids: if sid not in scenario_ids: scenario_ids.append(sid) - # Exclude archived scenarios + # Exclude archived scenarios and the ones this action removes from the set archived_ids = set(Scenario.objects.filter(id__in=scenario_ids, archived_at__isnull=False).values_list("id", flat=True)) + archived_ids |= set(exclude_ids or ()) scenario_ids = [sid for sid in scenario_ids if sid not in archived_ids] if scenario_ids: - publish_scenario_set_version(scenario_set=sset, user=user, scenario_ids=scenario_ids) + publish_scenario_set_version(scenario_set=sset, user=user, scenario_ids=scenario_ids, revisions=pins) class ScenarioSetCreateView(ProjectMixin, View): @@ -2166,9 +2316,11 @@ def post(self, request): blocked = _require_write_access(request) if blocked: return blocked - name = request.POST.get("name", "").strip() + name = request.POST.get("name", "").strip()[:250] # ScenarioSet.name max_length desc = request.POST.get("description", "").strip() - if name: + if name and ScenarioSet.objects.filter(project=request.project, name=name).exists(): + messages.error(request, f"A scenario set named '{name}' already exists.") + elif name: ScenarioSet.objects.create( project=request.project, name=name, description=desc, created_by=request.user, ) @@ -2183,8 +2335,11 @@ def post(self, request, set_id): return blocked sset = ScenarioSet.objects.filter(pk=set_id, project=request.project).first() if sset: - name = request.POST.get("name", "").strip() + name = request.POST.get("name", "").strip()[:250] desc = request.POST.get("description", "").strip() + if name and ScenarioSet.objects.filter(project=request.project, name=name).exclude(pk=sset.pk).exists(): + messages.error(request, f"A scenario set named '{name}' already exists.") + return redirect(f"/scenarios/?set={set_id}") if name: sset.name = name sset.description = desc @@ -2253,20 +2408,22 @@ def post(self, request): blocked = _require_write_access(request) if blocked: return blocked - name = request.POST.get("name", "").strip() - category = request.POST.get("category", "").strip() + name = request.POST.get("name", "").strip()[:300] # Scenario.title max_length + category = request.POST.get("category", "").strip()[:120] desc = request.POST.get("description", "") expected_behavior_raw = request.POST.get("expected_behavior", "").strip() expected_behavior = [line.strip() for line in expected_behavior_raw.splitlines() if line.strip()] if expected_behavior_raw else [] test_prompt = request.POST.get("test_prompt", "").strip() - severity_ceiling = request.POST.get("severity_ceiling", "").strip().lower() + severity_ceiling = _severity_ceiling(request.POST.get("severity_ceiling", "")) documents_raw = request.POST.get("documents", "").strip() documents_raw = request.POST.get("documents", "").strip() try: documents = json.loads(documents_raw) if documents_raw else [] - except json.JSONDecodeError: - messages.error(request, "Documents and agentic metadata must be valid JSON.") - return _scenario_redirect(request.POST.get("set_id", "").strip() or None) + if not isinstance(documents, list): + raise TypeError("documents must be a list") + except (json.JSONDecodeError, TypeError, RecursionError): + messages.error(request, "Documents must be a JSON list, and agentic metadata valid JSON.") + return _scenario_redirect(_post_set_id(request) or None) # The raw metadata box is a read-only preview; the structured controls # are the single source of truth. A new scenario has no prior revision # to preserve, so unknown fields start empty. @@ -2277,19 +2434,23 @@ def post(self, request): metadata = _merge_structured_agentic_fields({}, request.POST) except (ValueError, json.JSONDecodeError) as exc: messages.error(request, f"Structured agentic controls are invalid: {exc}") - return _scenario_redirect(request.POST.get("set_id", "").strip() or None) + return _scenario_redirect(_post_set_id(request) or None) file_uri_raw = request.POST.get("file_uri", "").strip() file_uri = file_uri_raw or None decision, decision_error = _parse_decision_field(request.POST.get("decision", "")) if decision_error: messages.error(request, decision_error) - return _scenario_redirect(request.POST.get("set_id", "").strip() or None) + return _scenario_redirect(_post_set_id(request) or None) if decision and request.POST.get("agentic_scenario"): messages.error(request, "A scenario cannot be both agentic and a decision model — pick one.") - return _scenario_redirect(request.POST.get("set_id", "").strip() or None) - set_id = request.POST.get("set_id", "").strip() + return _scenario_redirect(_post_set_id(request) or None) + set_id = _post_set_id(request) if name: key = hashlib.sha256(name.encode()).hexdigest()[:12] + # Keys are unique per workspace and survive archiving: the same title + # in another set, or re-added after a delete, gets a fresh key. + if Scenario.objects.filter(project=request.project, key=key).exists(): + key = f"{key}-{uuid.uuid4().hex[:6]}" scenario = create_scenario( project=request.project, user=request.user, key=key, title=name, category=category, description=desc, expected_behavior=expected_behavior, test_prompt=test_prompt, @@ -2311,20 +2472,22 @@ def post(self, request, scenario_id): if blocked: return blocked scenario = Scenario.objects.filter(pk=scenario_id, project=request.project).first() - set_id = request.POST.get("set_id", "").strip() + set_id = _post_set_id(request) if scenario: - title = request.POST.get("title", "").strip() - category = request.POST.get("category", "").strip() + title = request.POST.get("title", "").strip()[:300] + category = request.POST.get("category", "").strip()[:120] desc = request.POST.get("description", "") expected_behavior_raw = request.POST.get("expected_behavior", "").strip() expected_behavior = [line.strip() for line in expected_behavior_raw.splitlines() if line.strip()] if expected_behavior_raw else [] test_prompt = request.POST.get("test_prompt", "").strip() - severity_ceiling = request.POST.get("severity_ceiling", "").strip().lower() + severity_ceiling = _severity_ceiling(request.POST.get("severity_ceiling", "")) documents_raw = request.POST.get("documents", "").strip() try: documents = json.loads(documents_raw) if documents_raw else [] - except json.JSONDecodeError: - messages.error(request, "Documents and agentic metadata must be valid JSON.") + if not isinstance(documents, list): + raise TypeError("documents must be a list") + except (json.JSONDecodeError, TypeError, RecursionError): + messages.error(request, "Documents must be a JSON list, and agentic metadata valid JSON.") return _scenario_redirect(set_id or None) file_uri_raw = request.POST.get("file_uri", "").strip() file_uri = file_uri_raw or None @@ -2342,11 +2505,25 @@ def post(self, request, scenario_id): # update_scenario_content handles change detection (returns latest if unchanged) latest_rev_before = scenario.revisions.order_by("-revision").first() + # The revision this set currently runs can be older than the newest + # (a revert, or an edit made from another set sharing the scenario); + # compare against it so re-typing the newest text still publishes. + sset = ScenarioSet.objects.filter(pk=set_id, project=request.project).first() if set_id else None + set_latest = sset.versions.order_by("-version").first() if sset else None + pinned_item = set_latest.items.select_related("revision").filter(scenario=scenario).first() if set_latest else None + pinned_rev = pinned_item.revision if pinned_item else latest_rev_before + # The form carries the revision it was opened on: a form from an older + # version, or opened before someone else's edit, would republish + # stale text over the current one. + base_rev = request.POST.get("base_revision_id") + if base_rev and pinned_rev is not None and base_rev != str(pinned_rev.pk): + messages.error(request, "This scenario changed since the form was opened; reload and edit again.") + return _scenario_redirect(set_id or None) # The raw metadata box is a read-only preview; the structured controls # are the single source of truth. Unknown top-level fields survive an # edit because they are carried over from the previous revision rather # than re-read from the (now non-editable) textarea. - prior_metadata = dict(latest_rev_before.metadata or {}) if latest_rev_before else {} + prior_metadata = dict(pinned_rev.metadata or {}) if pinned_rev else {} if not request.POST.get("agentic_scenario"): # The toggle disables agentic expectations, but must not erase # unrelated metadata authored by older clients or imports. @@ -2364,12 +2541,10 @@ def post(self, request, scenario_id): severity_ceiling=severity_ceiling, documents=documents, file_uri=file_uri, decision=decision, metadata=metadata, ) - content_changed = latest_rev_before is None or rev.revision > latest_rev_before.revision + content_changed = pinned_rev is None or rev.pk != pinned_rev.pk if content_changed: - if set_id: - sset = ScenarioSet.objects.filter(pk=set_id, project=request.project).first() - if sset: - _publish_new_version(sset, request.user) + if sset: + _publish_new_version(sset, request.user, changed_ids=[scenario.id]) messages.success(request, f"Scenario '{scenario.title}' updated (new version published).") else: messages.info(request, f"Scenario '{scenario.title}' saved (no content change, version unchanged).") @@ -2381,17 +2556,24 @@ def post(self, request, scenario_id): blocked = _require_write_access(request) if blocked: return blocked - set_id = request.POST.get("set_id", "").strip() + set_id = _post_set_id(request) scenario = Scenario.objects.filter(pk=scenario_id, project=request.project).first() if scenario: - # Archive instead of delete (preserves historical versions) - scenario.archived_at = timezone.now() - scenario.save() - # Publish new version excluding archived scenarios - if set_id: - sset = ScenarioSet.objects.filter(pk=set_id, project=request.project).first() - if sset: - _publish_new_version(sset, request.user) + sset = ScenarioSet.objects.filter(pk=set_id, project=request.project).first() if set_id else None + if sset: + # Remove it from this set only. + _publish_new_version(sset, request.user, exclude_ids=[scenario.id]) + # Archive (never hard-delete: historical versions keep it) only when + # no other set's current version still uses it: a scenario shared + # by import must not vanish from the other sets' next versions. + still_used = any( + (latest := other.versions.order_by("-version").first()) is not None + and latest.items.filter(scenario=scenario).exists() + for other in ScenarioSet.objects.filter(project=request.project).exclude(pk=getattr(sset, "pk", None)) + ) + if not still_used: + scenario.archived_at = timezone.now() + scenario.save() return _scenario_redirect(set_id or None) @@ -2421,9 +2603,17 @@ def post(self, request, set_id): target_ver = _int(request.POST.get("target_version")) old_version = sset.versions.filter(version=target_ver).first() if old_version: - scenario_ids = list(old_version.items.values_list("scenario_id", flat=True)) + old_items = list(old_version.items.select_related("revision").order_by("position")) + scenario_ids = [i.scenario_id for i in old_items] if scenario_ids: - publish_scenario_set_version(scenario_set=sset, user=request.user, scenario_ids=scenario_ids) + # Restore exactly what the preview showed: the old version's frozen + # revisions, and its scenarios even if they were deleted since. + with transaction.atomic(): + Scenario.objects.filter(id__in=scenario_ids, archived_at__isnull=False).update(archived_at=None) + publish_scenario_set_version( + scenario_set=sset, user=request.user, scenario_ids=scenario_ids, + revisions={i.scenario_id: i.revision for i in old_items}, + ) messages.success(request, f"Reverted to v{target_ver} (published as new version).") else: messages.error(request, "Old version has no scenarios.") @@ -2482,33 +2672,66 @@ def post(self, request, set_id): return JsonResponse({"error": "Not found"}, status=404) try: data = json.loads(request.body) + if not isinstance(data, dict): + return JsonResponse({"error": "Expected a JSON object with a 'scenarios' list."}, status=400) if data.get("format_version") not in (None, 2): return JsonResponse({"error": "Unsupported scenario import format_version; expected 2."}, status=400) - new_ids = [] - for item in data.get("scenarios", []): - key = item.get("key", f"imported_{int(timezone.now().timestamp())}") - scenario, _ = Scenario.objects.get_or_create( - project=request.project, key=key, - defaults={"title": item.get("title", "Imported"), "category": item.get("category", "")}, - ) - update_scenario_content( - scenario=scenario, user=request.user, - description=item.get("description", ""), - expected_behavior=item.get("expected_behavior") or [], - test_prompt=item.get("test_prompt", ""), - severity_ceiling=item.get("severity_ceiling", ""), - documents=item.get("documents") or [], - file_uri=item.get("file_uri"), - decision=item.get("decision"), - metadata=item.get("metadata") # Import includes metadata if present - ) - new_ids.append(scenario.id) - # Auto-publish after import (include newly imported scenarios) - _publish_new_version(sset, request.user, extra_scenario_ids=new_ids) + items = data.get("scenarios", []) + if not isinstance(items, list) or not all(isinstance(i, dict) for i in items): + return JsonResponse({"error": "'scenarios' must be a list of objects."}, status=400) + # Same per-item contract as the API importer (types, lengths, severity). + from scenarios.serializers import ScenarioImportItemSerializer + + validated, problems = [], [] + for n, item in enumerate(items, start=1): + key = str(item.get("key") or "").strip() or f"imported_{uuid.uuid4().hex[:12]}" + serializer = ScenarioImportItemSerializer(data={"title": "Imported", **item, "key": key}) + if serializer.is_valid(): + data = dict(serializer.validated_data) + if "metadata" not in item: + data["metadata"] = None # keep an existing scenario's metadata + validated.append(data) + else: + problems.append(f"scenario {n} ({key}): {serializer.errors}") + if problems: + return JsonResponse({"error": "Invalid scenarios: " + "; ".join(problems)}, status=400) + # All-or-nothing: a failure part-way must not leave half-imported content. + with transaction.atomic(): + new_ids = self._import_items(request, validated) + # Auto-publish after import (include newly imported scenarios) + _publish_new_version(sset, request.user, extra_scenario_ids=new_ids) except Exception as e: # noqa: BLE001 - surface any import/publish failure to the user return JsonResponse({"error": str(e)}, status=400) return redirect(f"/scenarios/?set={set_id}") + @staticmethod + def _import_items(request, items: list[dict]) -> list[int]: + new_ids = [] + for item in items: # validated by ScenarioImportItemSerializer + key = item["key"] + scenario, _ = Scenario.objects.get_or_create( + project=request.project, key=key, + defaults={"title": item["title"], "category": item.get("category", "")}, + ) + if scenario.archived_at is not None: + # Re-importing a deleted (archived) scenario restores it; otherwise + # the publish below drops it and the import silently adds nothing. + scenario.archived_at = None + scenario.save(update_fields=["archived_at"]) + update_scenario_content( + scenario=scenario, user=request.user, + description=item["description"], + expected_behavior=item["expected_behavior"], + test_prompt=item["test_prompt"], + severity_ceiling=item["severity_ceiling"], + documents=item["documents"], + file_uri=item.get("file_uri"), + decision=item.get("decision"), + metadata=item.get("metadata"), # None keeps an existing scenario's metadata + ) + new_ids.append(scenario.id) + return new_ids + # ─── Models ────────────────────────────────────────────────────────────────── @@ -2526,7 +2749,6 @@ def get_context_data(self, **kw): from model_registry.services import ( PROVIDER_PRESETS, admin_workspaces, - can_edit_connection, connection_share_label, is_internal_agent_connection, model_usage_counts, @@ -2546,6 +2768,16 @@ def get_context_data(self, **kw): project=p, connection_id__in=[c.id for c in connections] ) } + # One query per relation instead of per connection. + from django.db.models import prefetch_related_objects + + from accounts.models import ProjectMembership + + prefetch_related_objects(connections, "models", "shared_with") + admin_project_ids = set( + ProjectMembership.objects.filter(user=user, role=ProjectMembership.Role.ADMIN) + .values_list("project_id", flat=True) + ) # Attach each connection's models once (owner + shared alike). for conn in connections: conn.otlp_credential = otlp_credentials.get(conn.id) @@ -2555,7 +2787,8 @@ def get_context_data(self, **kw): for m in conn.model_list: m.usage = usage.get(m.id, 0) conn.in_use = any(m.usage for m in conn.model_list) - conn.can_edit = can_edit_connection(user, conn) + # Same rule as can_edit_connection, without a query per connection. + conn.can_edit = user.is_superuser or conn.project_id in admin_project_ids conn.share_label = connection_share_label(conn) conn.is_shared = not conn.is_owner # What the edit dialog needs (never the key itself). Only owner @@ -2566,7 +2799,7 @@ def get_context_data(self, **kw): "key_mode": "stored" if c.api_key_direct else "env" if c.secret_reference else "none", "enabled": c.enabled, "visibility": c.visibility, - "shared_with": [str(x) for x in c.shared_with.values_list("id", flat=True)], + "shared_with": [str(x.id) for x in c.shared_with.all()], "can_edit": c.can_edit, "share_label": c.share_label, "otlp": { @@ -2619,7 +2852,7 @@ def fail(msg): return redirect(reverse("connections") + anchor) if action in ("add_connection", "edit_connection"): - name = post.get("conn_name", "").strip() + name = post.get("conn_name", "").strip()[:250] base_url = post.get("conn_base_url", "").strip() error = _base_url_error(base_url) or (None if name else "Connection name is required.") if action == "add_connection": @@ -2627,9 +2860,13 @@ def fail(msg): else: # Editing is owner-workspace only: a shared connection's id must # belong to this workspace, otherwise it's read-only for us. - conn = ModelConnection.objects.filter(pk=post.get("conn_id"), project=p).first() + conn = ModelConnection.objects.filter(pk=_int(post.get("conn_id")), project=p).first() if conn is None: return fail("Connection not found (or you can't edit a shared connection here).") + # Workspace admins only: an edit can re-point the stored key's URL. + from model_registry.services import can_edit_connection + if not can_edit_connection(request.user, conn): + return fail("Only workspace admins can edit a connection.") conn.enabled = post.get("conn_enabled") == "1" anchor = f"#conn-{conn.pk}" if error: @@ -2638,44 +2875,68 @@ def fail(msg): return fail(f"A connection named “{name}” already exists.") conn.name, conn.base_url = name, base_url conn.description = post.get("conn_description", "").strip()[:DESCRIPTION_MAX] - conn.provider = post.get("conn_provider") or "openai" - # Sharing level + explicit workspaces (owner-only; ignored for adds - # that don't send them, defaulting to "this workspace only"). + conn.provider = (post.get("conn_provider") or "openai").strip()[:120] # max_length + # Sharing level (explicit workspace shares are applied after save). visibility = post.get("conn_visibility", ModelConnection.Visibility.WORKSPACE) if visibility not in dict(ModelConnection.Visibility.choices): visibility = ModelConnection.Visibility.WORKSPACE conn.visibility = visibility - if action == "edit_connection": - shared_ids = [int(x) for x in post.getlist("conn_shared_with") if x.isdigit()] - conn.shared_with.set(shared_ids) # Where the key comes from: stored on the connection, an env var, or none. key_mode = post.get("key_mode", "stored") if key_mode == "env": ref = post.get("conn_secret_ref", "").strip() if not ref: return fail("Enter the environment variable that holds the API key.") + from model_registry.services import is_platform_secret_name + if is_platform_secret_name(ref): + return fail(f"{ref} holds a Studio platform secret and cannot be a model API key.") + if len(ref) > 250: # ModelConnection.secret_reference max_length + return fail("That environment variable name is too long (250 characters at most).") conn.secret_reference, conn.api_key_direct = ref, "" elif key_mode == "none": conn.secret_reference, conn.api_key_direct = "", "" else: conn.secret_reference = "" new_key = post.get("conn_api_key", "").strip() + if len(new_key) > 500: # ModelConnection.api_key_direct max_length + return fail("That API key is longer than 500 characters; store it in an environment variable instead.") if new_key: # blank keeps the stored key conn.api_key_direct = new_key conn.save() + # Explicit workspace shares, applied after every check passed so a + # refused edit changes nothing; adds honour them too. Only the + # "workspaces where I'm admin" level shares (hidden checkboxes still + # submit after switching to "this workspace only"). At that level, + # workspaces the editor doesn't administer keep another admin's share. + from model_registry.services import admin_workspaces + + allowed = {w.id for w in admin_workspaces(request.user)} - {p.id} + requested = ( + {int(x) for x in post.getlist("conn_shared_with") if is_id(x)} & allowed + if conn.visibility == ModelConnection.Visibility.ADMINS else set() + ) + # Any narrower level clears every share, including ones another admin + # made (else "this workspace only" kept leaking to their workspaces). + kept = ( + set(conn.shared_with.exclude(pk__in=allowed).values_list("pk", flat=True)) + if conn.visibility == ModelConnection.Visibility.ADMINS else set() + ) + conn.shared_with.set(requested | kept) anchor = f"#conn-{conn.pk}" messages.success(request, f"Connection “{conn.name}” saved.") elif action == "add_models": # Adding models mutates the connection — owner workspace only. - conn = ModelConnection.objects.filter(pk=post.get("conn_id"), project=p).first() + conn = ModelConnection.objects.filter(pk=_int(post.get("conn_id")), project=p).first() if conn is None: return fail("Connection not found (or you can't edit a shared connection here).") anchor = f"#conn-{conn.pk}" ids = [m.strip() for m in post.getlist("model_id") if m.strip()][:200] if not ids: return fail("Enter a model ID or pick models to add.") + if any(len(m) > 250 for m in ids): # RegisteredModel.model_id max_length + return fail("A model ID is longer than 250 characters.") single = len(ids) == 1 - label = post.get("model_display_name", "").strip() if single else "" + label = post.get("model_display_name", "").strip()[:250] if single else "" desc = post.get("model_description", "").strip()[:DESCRIPTION_MAX] if single else "" existing = set(conn.models.values_list("model_id", flat=True)) new_models = [m for m in dict.fromkeys(ids) if m not in existing] @@ -2687,7 +2948,7 @@ def fail(msg): flagged = post.get(f"decision:{m}") == "1" caps = {"decision": True} if flagged else detect_model_capabilities(conn, m) RegisteredModel.objects.create( - connection=conn, project=p, model_id=m, display_name=label or m, description=desc, + connection=conn, project=p, model_id=m, display_name=(label or m)[:250], description=desc, capabilities=caps, enabled=True, created_by=request.user, ) new = new_models @@ -2698,7 +2959,7 @@ def fail(msg): # Editing/removing a model mutates its connection — owner workspace only. rm = ( RegisteredModel.objects.select_related("connection") - .filter(pk=post.get("rm_id"), project=p, connection__project=p) + .filter(pk=_int(post.get("rm_id")), project=p, connection__project=p) .first() ) if rm is None: @@ -2706,10 +2967,12 @@ def fail(msg): anchor = f"#conn-{rm.connection_id}" if action == "edit_model": new_id = post.get("model_id_new", "").strip() or rm.model_id + if len(new_id) > 250: + return fail("Model ID is too long (250 characters at most).") if new_id != rm.model_id and RegisteredModel.objects.filter(connection=rm.connection, model_id=new_id).exists(): return fail(f"“{new_id}” is already registered on {rm.connection.name}.") rm.model_id = new_id - rm.display_name = post.get("model_display_name", "").strip() or new_id + rm.display_name = (post.get("model_display_name", "").strip() or new_id)[:250] rm.description = post.get("model_description", "").strip()[:DESCRIPTION_MAX] rm.save(update_fields=["model_id", "display_name", "description"]) messages.success(request, f"Saved {rm.display_name}.") @@ -2731,6 +2994,10 @@ def post(self, request, conn_id): if blocked: return blocked conn = ModelConnection.objects.filter(pk=conn_id, project=request.project).first() + from model_registry.services import can_edit_connection + if conn and not can_edit_connection(request.user, conn): + messages.error(request, "Only workspace admins can delete a connection.") + return redirect("connections") if conn: try: conn.delete() @@ -2755,6 +3022,8 @@ def _base_url_error(raw: str) -> str | None: if not raw.strip(): return "Base URL is required." + if len(raw.strip()) > 200: # ModelConnection.base_url max_length + return "Base URL is too long (200 characters at most)." try: EndpointURLField().to_internal_value(raw) except ValidationError: @@ -3052,7 +3321,7 @@ def post(self, request): # Looked up server-side (scoped to the workspace) so the API key never # reaches the browser. - conn = ModelConnection.objects.filter(pk=request.POST.get("connection_id") or 0, project=request.project).first() + conn = ModelConnection.objects.filter(pk=_int(request.POST.get("connection_id")), project=request.project).first() if conn is None: return JsonResponse({"error": "Connection not found in this workspace."}, status=404) try: @@ -3073,7 +3342,7 @@ class CompareView(ProjectMixin, TemplateView): @staticmethod def _parse_run_ids(raw_value: str) -> list[int]: - return [int(x) for x in (raw_value or "").split(",") if x.strip().isdigit()] + return [int(x) for x in (raw_value or "").split(",") if is_id(x.strip())] @staticmethod def _reshape_result(raw: dict) -> dict: @@ -3134,7 +3403,7 @@ def get_context_data(self, **kw): return super().get_context_data(**kw) def post(self, request, *args, **kwargs): - ids = [int(x) for x in request.POST.getlist("runs[]") if x.isdigit()] + ids = [int(x) for x in request.POST.getlist("runs[]") if is_id(x)] result = None error = None if len(ids) < 2: @@ -3266,6 +3535,13 @@ def get_context_data(self, **kw): from audits.services import ROLES, frozen_agent, frozen_judge, frozen_model ctx["frozen_models"] = [frozen_model(run, role) for role in ROLES] + if not self.request.user.is_authenticated: + # Public run pages: endpoint URLs (frozen or live) and connection ids + # are internal infrastructure, not part of what was published. + ctx["frozen_models"] = [ + {k: v for k, v in m.items() if k not in ("base_url", "connection_id", "changes")} + for m in ctx["frozen_models"] + ] from audits.target_metadata import metadata_rows ctx["target_metadata_rows"] = metadata_rows(run) ctx["judge"] = frozen_judge(run) @@ -3273,6 +3549,12 @@ def get_context_data(self, **kw): ctx["system_prompt"] = snap.get("system_prompt", "") ctx["system_prompt_source"] = snap.get("system_prompt_source") ctx["frozen_agent"] = frozen_agent(run) + if not self.request.user.is_authenticated and isinstance(ctx["frozen_agent"], dict): + # Same rule as frozen_models: no endpoint URLs or connection ids publicly. + base = dict(ctx["frozen_agent"].get("base_model") or {}) + for key in ("base_url", "connection_id", "connection"): + base.pop(key, None) + ctx["frozen_agent"] = {**ctx["frozen_agent"], "base_model": base} # Verdict split for the run-level agentic overview bar (percentages are # computed here — the template engine has no float arithmetic). agentic_metrics = (run.summary_metrics or {}).get("agentic_evaluation") @@ -3347,14 +3629,18 @@ class ExperimentSetVisibilityView(LoginRequiredMixin, View): """ def post(self, request, experiment_id): - from accounts.services import is_any_project_admin + from accounts.models import ProjectMembership from audits.models import Experiment + from infra.exceptions import StableAPIError + from scenarios.services import require_project_role - if not is_any_project_admin(request.user): - return HttpResponseForbidden("Admin access required.") - # Scope to the active workspace so an admin of one workspace can't flip - # another workspace's experiment. + # Scope to the active workspace and require ADMIN of THAT workspace: + # being admin of some other workspace is not enough. experiment = get_object_or_404(Experiment, pk=experiment_id, project=request.project) + try: + require_project_role(request.user, experiment.project, ProjectMembership.Role.ADMIN) + except StableAPIError: + return HttpResponseForbidden("Workspace admin access required.") value = request.POST.get("visibility") valid = dict(Experiment.Visibility.choices) if value not in valid: @@ -3566,6 +3852,23 @@ def _trace_card_view(rep: dict, has_timeline: bool = False) -> dict | None: } +def _llm_tool_call_names(attrs: dict) -> list[str]: + """Every tool name an LLM span's output message decided to call.""" + out_msgs = attrs.get("gen_ai.output.messages") + try: + msgs = json.loads(out_msgs) if isinstance(out_msgs, str) else out_msgs + except (ValueError, TypeError): + return [] + names = [] + for m in msgs if isinstance(msgs, list) else []: + calls = m.get("tool_calls") if isinstance(m, dict) else None + for tc in calls if isinstance(calls, list) else []: + fn = tc.get("function") if isinstance(tc, dict) else None + if isinstance(fn, dict) and fn.get("name"): + names.append(str(fn["name"])) + return names + + def _llm_tool_decision(span: dict, attrs: dict) -> tuple[str, str] | None: """Extract the tool call(s) an LLM span decided on, if any. @@ -3587,8 +3890,12 @@ def _llm_tool_decision(span: dict, attrs: dict) -> tuple[str, str] | None: for m in msgs: if not isinstance(m, dict): continue - for tc in m.get("tool_calls") or []: - fn = (tc.get("function") or {}) if isinstance(tc, dict) else {} + # Span attributes come from the audited system's OTLP export: untrusted shape. + calls = m.get("tool_calls") + for tc in calls if isinstance(calls, list) else []: + fn = tc.get("function") if isinstance(tc, dict) else None + if not isinstance(fn, dict): + continue nm = fn.get("name") if nm: names.append(str(nm)) @@ -3908,10 +4215,8 @@ def _merge_synthesized_tool_spans(raw_spans: list[dict], synthetic: list[dict]) # LLM span that decided to call a tool: pull the name(s) out of its # output so we don't re-synthesize those calls below. if kind == "LLM": - decision = _llm_tool_decision(s, attrs) - if decision: - for n in decision[0].split(","): - covered.add(n.strip()) + # All names, not the display label (which joins with " + " and truncates). + covered.update(_llm_tool_call_names(attrs)) extra = [s for s in synthetic if s.get("kind") == "tool" and s.get("attributes", {}).get("tool_name") not in covered] if not extra: return list(raw_spans) @@ -4493,6 +4798,8 @@ class RunExportView(ProjectMixin, View): def get(self, request, run_id): run = get_object_or_404(AuditRun, pk=run_id, project=request.project) fmt = request.GET.get("format", "json").lower() + if fmt not in ("json", "csv"): + fmt = "json" items = {str(vi.pk): vi for vi in ScenarioSetVersionItem.objects.filter(version=run.scenario_set_version).select_related("scenario")} rows = [] @@ -4502,7 +4809,7 @@ def get(self, request, run_id): rows.append({ "id": sr.id, "scenario_name": item.scenario.title if item else str(sr.version_item_id), - "severity": r.get("severity", sr.status), + "severity": result_severity(r, sr.status), "summary": r.get("summary", ""), "result": r, }) @@ -4514,7 +4821,7 @@ def get(self, request, run_id): writer = csv.writer(buf) writer.writerow(["id", "scenario_name", "severity", "summary"]) for row in rows: - writer.writerow([row["id"], row["scenario_name"], row["severity"], row["summary"]]) + writer.writerow([row["id"], csv_safe(row["scenario_name"]), csv_safe(row["severity"]), csv_safe(row["summary"])]) response = HttpResponse(buf.getvalue(), content_type="text/csv; charset=utf-8") else: payload = json.dumps(rows, indent=2, ensure_ascii=False) @@ -4699,18 +5006,26 @@ def _sync_openwebui_resources(project, user): project=project, external_id=external_id ).first() if existing is None: + # Only rows not yet linked: re-pointing a linked row would aim + # its later remote delete (run as its creator) at another KB. existing = KnowledgeBase.objects.filter( - project=project, name=name + project=project, name=name, external_id="" ).first() + if existing is not None: + existing.created_by = user # the user who linked it acts for it if existing is not None: - existing.name = name + # Keep the local name when another row already holds the remote one. + if not KnowledgeBase.objects.filter(project=project, name=name).exclude(pk=existing.pk).exists(): + existing.name = name existing.external_id = external_id existing.description = description - existing.save(update_fields=["name", "external_id", "description", "updated_at"]) + existing.save(update_fields=["name", "external_id", "description", "created_by", "updated_at"]) + elif KnowledgeBase.objects.filter(project=project, name=name).exists(): + continue # a different remote KB already holds this name here else: KnowledgeBase.objects.create( project=project, name=name, - external_id=external_id, description=description, + external_id=external_id, description=description, created_by=user, ) kb_count += 1 except ChatAPIError: @@ -4730,18 +5045,31 @@ def _sync_openwebui_resources(project, user): meta = item.get("meta") if isinstance(item.get("meta"), dict) else {} manifest = meta.get("manifest") if isinstance(meta.get("manifest"), dict) else {} external_id = item.get("id", "") - Tool.objects.update_or_create( - project=project, - name=name, - defaults={ - "external_id": external_id, - # List items carry no top-level description; the toolkit - # description lives in the frontmatter manifest. - "description": manifest.get("description") or meta.get("description") or "", - # Toolkit tools are custom Python; builtins have no row. - "type": "custom", - }, - ) + fields = { + # List items carry no top-level description; the toolkit + # description lives in the frontmatter manifest. + "description": manifest.get("description") or meta.get("description") or "", + # Toolkit tools are custom Python; builtins have no row. + "type": "custom", + } + # Same identity rule as knowledge bases: external_id first, then an + # unlinked row by name (linked by this user), else a new row. + tool = Tool.objects.filter(project=project, external_id=external_id).first() if external_id else None + if tool is None: + tool = Tool.objects.filter(project=project, name=name, external_id="").first() + if tool is not None: + tool.created_by = user + if tool is None: + if Tool.objects.filter(project=project, name=name).exists(): + continue # a different remote tool already holds this name here + Tool.objects.create(project=project, name=name, external_id=external_id, created_by=user, **fields) + else: + if not Tool.objects.filter(project=project, name=name).exclude(pk=tool.pk).exists(): + tool.name = name + tool.external_id = external_id + for key, value in fields.items(): + setattr(tool, key, value) + tool.save() tool_count += 1 except ChatAPIError: pass @@ -4791,6 +5119,9 @@ class AgentSyncView(ProjectMixin, View): """POST /agents/sync/ — pull KBs and tools from OpenWebUI into local models.""" def post(self, request): + reason = write_block_reason(request) + if reason: + return JsonResponse({"error": reason}, status=403) kb_count, tool_count = _sync_openwebui_resources(request.project, request.user) if kb_count is None: return JsonResponse({"error": "Chat is not enabled."}, status=400) @@ -4929,6 +5260,9 @@ def get_context_data(self, **kw): return ctx def post(self, request, *args, **kwargs): + blocked = _require_write_access(request) + if blocked: + return blocked from model_registry.models import ( Agent, KnowledgeBase, @@ -4946,16 +5280,16 @@ def post(self, request, *args, **kwargs): if "agent_id" in self.kwargs: agent = Agent.objects.filter(pk=self.kwargs["agent_id"], project=project).first() - name = request.POST.get("name", "").strip() + name = request.POST.get("name", "").strip()[:250] # Agent.name max_length if not name: messages.error(request, "Name is required.") return self._render(request, agent) + from model_registry.models import RegisteredModel try: base_model_id = int(request.POST.get("base_model")) - from model_registry.models import RegisteredModel base_model = RegisteredModel.objects.get(pk=base_model_id, project=project) - except (ValueError, RegisteredModel.DoesNotExist): + except (TypeError, ValueError, RegisteredModel.DoesNotExist): messages.error(request, "Select a valid base model.") return self._render(request, agent) @@ -4968,9 +5302,11 @@ def _positive_int(name, default): def _optional_float(name): raw = request.POST.get(name, "").strip() try: - return float(raw) if raw else None + value = float(raw) if raw else None except ValueError: return None + # NaN/inf are not JSON (the column's JSON check rejects them). + return value if value is None or math.isfinite(value) else None retrieval = default_retrieval_settings() mode = request.POST.get("search_mode", "semantic") @@ -4984,7 +5320,7 @@ def _optional_float(name): # Resolve the system prompt: a versioned prompt picker or a raw textarea. from prompts.models import SystemPromptVersion prompt_version_id = request.POST.get("system_prompt_version") or "" - if prompt_version_id: + if is_id(prompt_version_id): pv = SystemPromptVersion.objects.filter( pk=prompt_version_id, prompt__project=project).first() system_prompt_text = pv.content if pv else "" @@ -5001,11 +5337,11 @@ def _optional_float(name): agent.save() agent.knowledge_bases.set( KnowledgeBase.objects.filter( - pk__in=request.POST.getlist("knowledge_bases"), project=project + pk__in=[v for v in request.POST.getlist("knowledge_bases") if is_id(v)], project=project ) ) agent.tools.set( - Tool.objects.filter(pk__in=request.POST.getlist("tools"), project=project) + Tool.objects.filter(pk__in=[v for v in request.POST.getlist("tools") if is_id(v)], project=project) ) caps = self._posted_capabilities(request) agent.capabilities = caps @@ -5026,11 +5362,11 @@ def _optional_float(name): ) agent.knowledge_bases.set( KnowledgeBase.objects.filter( - pk__in=request.POST.getlist("knowledge_bases"), project=project + pk__in=[v for v in request.POST.getlist("knowledge_bases") if is_id(v)], project=project ) ) agent.tools.set( - Tool.objects.filter(pk__in=request.POST.getlist("tools"), project=project) + Tool.objects.filter(pk__in=[v for v in request.POST.getlist("tools") if is_id(v)], project=project) ) _sync_agent_form_result(request, agent) messages.success(request, f"Agent '{name}' created.") @@ -5065,6 +5401,9 @@ class AgentDeleteView(ProjectMixin, View): """Delete an agent.""" def post(self, request, agent_id): + blocked = _require_write_access(request) + if blocked: + return blocked from model_registry.models import Agent from model_registry.services import delete_agent_from_openwebui @@ -5073,7 +5412,7 @@ def post(self, request, agent_id): messages.error(request, "Agent not found.") return redirect("agents") name = agent.name - delete_agent_from_openwebui(agent) + delete_agent_from_openwebui(agent, request.user) agent.delete() messages.success(request, f"Agent '{name}' deleted.") return redirect("agents") diff --git a/infra/visualizer.py b/infra/visualizer.py index c7a8f743..a02f4a37 100644 --- a/infra/visualizer.py +++ b/infra/visualizer.py @@ -31,6 +31,7 @@ from django.shortcuts import get_object_or_404 from django.views.generic import View +from audits.services import result_severity from infra.ui import ProjectMixin logger = logging.getLogger(__name__) @@ -597,7 +598,7 @@ def get(self, request): return HttpResponse(html, content_type="text/html; charset=utf-8") -class VisualizerFilesView(View): +class VisualizerFilesView(LoginRequiredMixin, View): """``GET /api/visualizer/api/files/`` → bounded file tree of the results dir. Response shape: @@ -683,7 +684,7 @@ def get(self, request): ) -class VisualizerJsonView(View): +class VisualizerJsonView(LoginRequiredMixin, View): """``GET /api/visualizer/json/`` → the JSON content of a result file.""" def get(self, request, file_path: str): @@ -703,10 +704,10 @@ def get(self, request, file_path: str): return JsonResponse({"error": "Error reading file"}, status=500) if not is_valid_audit_data(data): return JsonResponse({"error": "Not an audit results file"}, status=403) - return JsonResponse(data) + return JsonResponse(data, safe=False) # a result file may be a top-level list -class VisualizerImageView(View): +class VisualizerImageView(LoginRequiredMixin, View): """``GET /api/visualizer/image?uri=...`` → ``{"data_uri": ...}`` for previews. Reuses the audit's own ``image_data_uri`` loader so local paths, http(s) and @@ -721,6 +722,17 @@ def get(self, request): uri = request.GET.get("uri", "") if not uri: return JsonResponse({"error": "Missing uri"}, status=400) + # Only http(s) URLs and files inside the results directory: fsspec would + # otherwise open any server path or protocol (s3://, file://, ...). + scheme = uri.split("://", 1)[0].lower() if "://" in uri else "" + if scheme not in ("http", "https"): + if scheme: + return JsonResponse({"error": "Not a previewable image"}, status=415) + local = _resolve_results_path(uri.lstrip("/")) if not os.path.isabs(uri) else _resolve_results_path( + os.path.relpath(uri, results_dir() or "/")) + if local is None: + return JsonResponse({"error": "Access denied"}, status=403) + uri = local try: data_uri = image_data_uri(uri) except ValueError: @@ -773,7 +785,7 @@ def get(self, request, run_id): results.append( { "scenario_name": item.scenario.title if item else str(sr.version_item_id), - "severity": r.get("severity", sr.status), + "severity": result_severity(r, sr.status), "summary": r.get("summary", ""), "result": r, } diff --git a/infra/worker.py b/infra/worker.py index f2b4e862..2758c91f 100644 --- a/infra/worker.py +++ b/infra/worker.py @@ -447,6 +447,12 @@ def _scenario_execute_impl(workflow_input: ScenarioInput, ctx: Context) -> dict: # target exports no spans); an empty one means "trace off". trace_on = bool(run.trace_config) trace_config = enrich_trace_config(run.trace_config or None, run.target_config_snapshot) + # A scenario may raise its trace content capture (agentic.trace.content_capture); + # it must reach the engine call below, which sanitizes spans at that level. + _agentic_meta = (revision.metadata or {}).get("agentic") + _agentic_trace_cfg = _agentic_meta.get("trace") if isinstance(_agentic_meta, dict) else None + if trace_config and isinstance(_agentic_trace_cfg, dict) and isinstance(_agentic_trace_cfg.get("content_capture"), str): + trace_config = {**trace_config, "content_capture": _agentic_trace_cfg["content_capture"]} gen_params = run.generation_parameters_snapshot or {} if run.agent_config_snapshot: @@ -696,9 +702,6 @@ def _stop_event_flusher() -> None: # recompute the agentic status for UI display. For non-traced agentic # scenarios (no spans available in the resolver), we do the full eval. if trace_on and (revision.metadata or {}).get("agentic"): - _agentic_trace_cfg = (revision.metadata or {}).get("agentic", {}).get("trace") or {} - if isinstance(_agentic_trace_cfg.get("content_capture"), str): - trace_config = {**(trace_config or {}), "content_capture": _agentic_trace_cfg["content_capture"]} try: from audits.agentic.semantic_judge import run_semantic_judge from audits.agentic.trajectory import normalize @@ -991,7 +994,6 @@ def resume_run(run, reason: str) -> int: how many scenarios were resubmitted. """ from audits.events import append_event - from audits.models import AuditRun missing = _missing_version_item_ids(run) if not missing: @@ -1000,12 +1002,16 @@ def resume_run(run, reason: str) -> int: submit_run_workflow( str(run.pk), missing, simpleaudit_version=run.simpleaudit_version, git_commit=run.git_commit, ) - run.refresh_from_db() - meta = dict(run.runtime_metadata or {}) - meta["submission"] = {"status": "submitted", "at": timezone.now().isoformat(), "scenarios": len(missing)} - meta["resumes"] = int(meta.get("resumes") or 0) + 1 - meta["last_resume"] = {"at": timezone.now().isoformat(), "reason": reason, "scenarios": len(missing)} - AuditRun.objects.filter(pk=run.pk).update(runtime_metadata=meta) + from django.db import transaction + + with transaction.atomic(): + # Locked read-modify-write so concurrent submit/resume keep each other's keys. + locked = type(run).objects.select_for_update().get(pk=run.pk) + meta = dict(locked.runtime_metadata or {}) + meta["submission"] = {"status": "submitted", "at": timezone.now().isoformat(), "scenarios": len(missing)} + meta["resumes"] = int(meta.get("resumes") or 0) + 1 + meta["last_resume"] = {"at": timezone.now().isoformat(), "reason": reason, "scenarios": len(missing)} + type(run).objects.filter(pk=run.pk).update(runtime_metadata=meta) append_event(run.pk, "_run", "run_resumed", {"reason": reason, "scenarios": len(missing)}) logger.info("Resumed run %s (%s): resubmitted %d scenario(s)", run.pk, reason, len(missing)) return len(missing) @@ -1067,13 +1073,13 @@ def _mark_run_failed(run_id: str, code: str) -> None: from audits.models import AuditRun try: - run = AuditRun.objects.get(pk=int(run_id)) - except (AuditRun.DoesNotExist, ValueError): + pk = int(run_id) + except ValueError: return - run.status = AuditRun.Status.FAILED - run.error_code = code - run.finished_at = timezone.now() - run.save(update_fields=["status", "error_code", "finished_at"]) + # Never overwrite a run that already completed or was cancelled. + AuditRun.objects.filter(pk=pk, status__in=_ACTIVE_STATUSES).update( + status=AuditRun.Status.FAILED, error_code=code, finished_at=timezone.now() + ) # --- Module-level task registration --------------------------------------- diff --git a/judges/services.py b/judges/services.py index b6e4a778..5b45db07 100644 --- a/judges/services.py +++ b/judges/services.py @@ -159,6 +159,9 @@ def make_spec(*, base: str = "", output: str = "", criteria: str = "", probe_pro default probe) is stored blank, so it follows the base and switching base later doesn't carry stale copies along. Raises ``ValueError``.""" criteria, probe_prompt = (criteria or "").strip(), (probe_prompt or "").strip() + # Agentic guidance is Studio's own option (read by the engine bridge for + # agentic scenarios), valid for any format; keep it through normalisation. + guidance = str((options or {}).get("agentic_guidance") or "").strip() if base: if base not in bases(): raise ValueError(f"Unknown SimpleAudit judge “{base}”.") @@ -178,6 +181,8 @@ def make_spec(*, base: str = "", output: str = "", criteria: str = "", probe_pro default_probe = default_probe_prompt() if probe_prompt == default_probe: probe_prompt = "" + if guidance: + options = {**options, "agentic_guidance": guidance} spec = {"base": base, "output": output, "criteria": criteria, "probe_prompt": probe_prompt, "options": options} library_judge(spec) # SimpleAudit's own checks: dimension names, the question, ... return spec @@ -266,11 +271,19 @@ def parse_start(value: str) -> dict: return {"base": value, "output": ""} -def save_version(judge: Judge, *, note: str = "", user=None, **fields) -> tuple[JudgeVersion, bool]: +class StaleVersionError(ValueError): + """A save based on a version that is no longer the latest.""" + + +def save_version( + judge: Judge, *, note: str = "", user=None, base_version_id: int | None = None, **fields, +) -> tuple[JudgeVersion, bool]: """Save a new version of ``judge`` when the versioned content changed. ``fields`` are ``make_spec`` arguments. Returns ``(version, created)``; - unchanged content returns the latest version. + unchanged content returns the latest version. ``base_version_id`` is the + version the editor was opened on: when another save landed since, a + changed save is refused (``ValueError``) rather than reverting it. """ content = make_spec(**fields) with transaction.atomic(): @@ -279,6 +292,8 @@ def save_version(judge: Judge, *, note: str = "", user=None, **fields) -> tuple[ latest = judge.versions.order_by("-version").first() if latest is not None and latest.content() == content: return latest, False + if base_version_id is not None and latest is not None and base_version_id != latest.pk: + raise StaleVersionError("This judge changed since the form was opened; reload and edit again.") number = (judge.versions.aggregate(n=Max("version"))["n"] or 0) + 1 version = JudgeVersion.objects.create( judge=judge, version=number, **content, note=note.strip()[:250], created_by=user, diff --git a/judges/views.py b/judges/views.py index 1eef983b..70467e60 100644 --- a/judges/views.py +++ b/judges/views.py @@ -7,12 +7,13 @@ from django.shortcuts import redirect from django.views.generic import TemplateView -from infra.ui import ProjectMixin, _require_write_access +from infra.ui import ProjectMixin, _int, _require_write_access from judges.models import Judge, JudgeVersion from judges.services import ( DEFAULT_BASE, DEFAULT_JUDGE_BASE, FORMATS, + StaleVersionError, base, base_choices, create_judge, @@ -118,7 +119,7 @@ def post(self, request): messages.success(request, f"Added {len(made)} SimpleAudit judge{'s' if len(made) != 1 else ''}.") elif action == "delete_version": version = JudgeVersion.objects.select_related("judge").filter( - pk=request.POST.get("version_id") or 0, judge__project=p).first() + pk=_int(request.POST.get("version_id")), judge__project=p).first() if version is None: messages.error(request, "Version not found.") return redirect("judges") @@ -129,7 +130,7 @@ def post(self, request): messages.error(request, str(e)) return redirect("judge_detail", version.judge_id) elif action == "delete": - judge = Judge.objects.filter(pk=request.POST.get("judge_id") or 0, project=p).first() + judge = Judge.objects.filter(pk=_int(request.POST.get("judge_id")), project=p).first() if judge is not None: try: judge.delete() @@ -178,7 +179,7 @@ def get_context_data(self, **kw): cloning = None if judge is None: cloning = JudgeVersion.objects.select_related("judge").filter( - pk=self.request.GET.get("clone") or (form or {}).get("clone") or 0, + pk=_int(self.request.GET.get("clone") or (form or {}).get("clone")), judge__project=self.request.project, ).first() if cloning is not None: @@ -237,8 +238,11 @@ def post(self, request, judge_id=None): p = request.project judge = self._judge() post = request.POST - form = {k: post.get(k, "") for k in ("name", "description", "start", *_SPEC_FIELDS, "note", "clone")} - name = form["name"].strip() + # agentic_guidance is part of the edit form; leaving it out silently + # dropped it from every saved version. + form = {k: post.get(k, "") for k in ("name", "description", "start", *_SPEC_FIELDS, "agentic_guidance", "note", "clone")} + name = form["name"].strip()[:250] # model max_length (Postgres rejects longer) + form["note"] = form["note"][:250] error = None if not name: error = "Give the judge a name." @@ -254,9 +258,20 @@ def post(self, request, judge_id=None): messages.success(request, f"Judge “{judge.name}” created.") return redirect("judge_detail", judge.id) make_spec(**_spec_fields(form)) # validate before renaming + # The version the form was opened on; an old version shown on + # purpose ("restore") may be saved over a newer one. + base = post.get("base_version_id", "") + base_id = int(base) if base.isascii() and base.isdecimal() and not post.get("restore") else None + version, created = save_version( + judge, note=form["note"], user=request.user, base_version_id=base_id, **_spec_fields(form), + ) judge.name, judge.description = name, form["description"].strip() judge.save(update_fields=["name", "description", "updated_at"]) - version, created = save_version(judge, note=form["note"], user=request.user, **_spec_fields(form)) + except StaleVersionError as e: + # Reload the fresh latest: re-rendering the stale fields would + # arm the form (base = latest) to overwrite it on the next click. + messages.error(request, str(e)) + return redirect(f"/judges/{judge.id}/") except ValueError as e: error = str(e) else: @@ -272,7 +287,7 @@ class JudgePreviewView(ProjectMixin, TemplateView): showing what the judge sees when deterministic checks are present.""" def post(self, request): - form = {k: request.POST.get(k, "") for k in ("start", *_SPEC_FIELDS)} + form = {k: request.POST.get(k, "") for k in ("start", *_SPEC_FIELDS, "agentic_guidance")} try: info = resolve(make_spec(**_spec_fields(form))) except ValueError as e: diff --git a/model_registry/agent_serializers.py b/model_registry/agent_serializers.py index 7ae3ae69..4e3b2d35 100644 --- a/model_registry/agent_serializers.py +++ b/model_registry/agent_serializers.py @@ -19,6 +19,12 @@ class Meta: ] read_only_fields = ["id"] + def update(self, instance, validated_data): + # ``external_id`` links a new row to an existing Open WebUI KB on create; + # re-pointing an existing row would aim its remote delete at another KB. + validated_data.pop("external_id", None) + return super().update(instance, validated_data) + class ToolSerializer(serializers.ModelSerializer): # Open WebUI owns the tool's Python source; Studio never stores it. @@ -67,6 +73,21 @@ class Meta: ] read_only_fields = ["id", "external_id"] + def _object(self, value, name): + # Stored JSON is read as a dict (e.g. ``{**defaults, **retrieval_settings}``). + if value is not None and not isinstance(value, dict): + raise serializers.ValidationError(f"{name} must be a JSON object.") + return value + + def validate_retrieval_settings(self, value): + return self._object(value, "retrieval_settings") + + def validate_capabilities(self, value): + return self._object(value, "capabilities") + + def validate_metadata(self, value): + return self._object(value, "metadata") + def validate(self, attrs): project = self.context.get("project") if project: diff --git a/model_registry/agent_views.py b/model_registry/agent_views.py index f5a61a12..944e637c 100644 --- a/model_registry/agent_views.py +++ b/model_registry/agent_views.py @@ -9,10 +9,12 @@ from rest_framework import status from rest_framework.decorators import api_view, permission_classes -from rest_framework.permissions import IsAuthenticated +from rest_framework.permissions import SAFE_METHODS, IsAuthenticated from rest_framework.response import Response +from accounts.models import ProjectMembership from infra.exceptions import StableAPIError +from scenarios.services import require_project_role from .agent_serializers import ( AgentDetailSerializer, @@ -38,9 +40,13 @@ def _project(request): + """The active workspace; writes also need ADMIN/AUDITOR and an unarchived workspace.""" project = getattr(request, "project", None) if not project: - raise StableAPIError("No active workspace.", status=400) + raise StableAPIError(detail="No active workspace.", code="no_workspace", http_status=400) + if request.method not in SAFE_METHODS: + # Same rule as the UI's write_block_reason; deletes also reach Open WebUI. + require_project_role(request.user, project, ProjectMembership.Role.ADMIN, ProjectMembership.Role.AUDITOR) return project @@ -93,7 +99,7 @@ def knowledge_base_detail(request, pk): project = _project(request) kb = KnowledgeBase.objects.filter(pk=pk, project=project).first() if not kb: - raise StableAPIError("Knowledge base not found.", status=404) + raise StableAPIError(detail="Knowledge base not found.", code="knowledge_base_not_found", http_status=404) if request.method == "GET": return Response(KnowledgeBaseSerializer(kb).data) @@ -105,7 +111,7 @@ def knowledge_base_detail(request, pk): sync_knowledge_base_to_openwebui(kb, request.user) return Response(serializer.data) - delete_knowledge_base_from_openwebui(kb) + delete_knowledge_base_from_openwebui(kb, request.user) kb.delete() return Response(status=status.HTTP_204_NO_CONTENT) @@ -137,7 +143,7 @@ def tool_detail(request, pk): project = _project(request) tool = Tool.objects.filter(pk=pk, project=project).first() if not tool: - raise StableAPIError("Tool not found.", status=404) + raise StableAPIError(detail="Tool not found.", code="tool_not_found", http_status=404) if request.method == "GET": return Response(ToolSerializer(tool).data) @@ -155,7 +161,7 @@ def tool_detail(request, pk): sync_tool_to_openwebui(tool, request.user, content=content) return Response(serializer.data) - delete_tool_from_openwebui(tool) + delete_tool_from_openwebui(tool, request.user) tool.delete() return Response(status=status.HTTP_204_NO_CONTENT) @@ -190,7 +196,7 @@ def agent_detail(request, pk): "base_model", "base_model__connection" ).prefetch_related("knowledge_bases", "tools").first() if not agent: - raise StableAPIError("Agent not found.", status=404) + raise StableAPIError(detail="Agent not found.", code="agent_not_found", http_status=404) if request.method == "GET": return Response( @@ -206,7 +212,7 @@ def agent_detail(request, pk): sync_agent_to_openwebui(agent, request.user) return Response(AgentDetailSerializer(agent, context={"request": request}).data) - delete_agent_from_openwebui(agent) + delete_agent_from_openwebui(agent, request.user) agent.delete() return Response(status=status.HTTP_204_NO_CONTENT) @@ -218,5 +224,5 @@ def agent_snapshot(request, pk): project = _project(request) agent = Agent.objects.filter(pk=pk, project=project).first() if not agent: - raise StableAPIError("Agent not found.", status=404) + raise StableAPIError(detail="Agent not found.", code="agent_not_found", http_status=404) return Response(agent.config_snapshot()) diff --git a/model_registry/api_credential_services.py b/model_registry/api_credential_services.py index e562ba6e..dfc65466 100644 --- a/model_registry/api_credential_services.py +++ b/model_registry/api_credential_services.py @@ -27,6 +27,7 @@ verify_secret, ) +from accounts.services import ensure_project_access from model_registry.models import ApiCredential #: Distinct from OTLP's "sa_otlp_" so the two credential spaces never overlap. @@ -104,7 +105,11 @@ def verify_bearer(token: str, *, project=None) -> ApiCredential | None: qs = ApiCredential.objects.filter(token_prefix=token_lookup_prefix(token), enabled=True) if project is not None: qs = qs.filter(project=project) - for cred in qs.select_related("user"): + for cred in qs.select_related("user", "project"): if verify_secret(token, cred.salt, cred.secret_hash): + # A key stops working once its owner is deactivated or loses + # access to the workspace it was issued for. + if not cred.user.is_active or not ensure_project_access(cred.user, cred.project): + return None return cred return None diff --git a/model_registry/models.py b/model_registry/models.py index f84a87c1..bc33b039 100644 --- a/model_registry/models.py +++ b/model_registry/models.py @@ -538,6 +538,11 @@ def config_snapshot( "display_name": self.base_model.display_name, "model_id": self.base_model.model_id, "connection_id": self.base_model.connection_id, + # Frozen endpoint so execution never reads live registry rows; + # only the API key is resolved at execution time. + "provider": self.base_model.connection.provider, + "base_url": self.base_model.connection.base_url, + "default_parameters": dict(self.base_model.default_parameters or {}), }, "system_prompt": self.system_prompt, "knowledge_bases": [ diff --git a/model_registry/otlp_views.py b/model_registry/otlp_views.py index 761c4f19..64516e68 100644 --- a/model_registry/otlp_views.py +++ b/model_registry/otlp_views.py @@ -627,9 +627,18 @@ def _active_project(request): def _require_admin(request, project) -> None: - from accounts.services import _require_workspace_admin + """Workspace ADMIN of an unarchived workspace (every caller changes credentials).""" + from accounts.services import _require_workspace_admin, require_project_writable _require_workspace_admin(request.user, project) + require_project_writable(request.user, project) + + +def _request_origin(request) -> str: + """The browser-supplied ``origin`` field, or "" when absent or not a string.""" + data = request.data if isinstance(request.data, dict) else {} + origin = data.get("origin") + return origin.strip() if isinstance(origin, str) else "" def _endpoint_url(request, origin: str | None = None) -> str: @@ -679,20 +688,26 @@ def create_credential(request): """ project = _active_project(request) _require_admin(request, project) - data = request.data or {} + data = request.data if isinstance(request.data, dict) else {} conn_id = data.get("connection_id") - auth_mode = (data.get("auth_mode") or "basic").strip().lower() - if not conn_id or not str(conn_id).isdigit(): + auth_mode = data.get("auth_mode") or "basic" + if not isinstance(auth_mode, str): + raise StableAPIError(detail="auth_mode must be 'basic' or 'bearer'.", code="invalid_auth_mode") + auth_mode = auth_mode.strip().lower() + if not conn_id or not str(conn_id).isdecimal(): raise StableAPIError(detail="connection_id is required.", code="invalid_connection_id") conn = ModelConnection.objects.filter(pk=int(conn_id), project=project).first() if conn is None: raise StableAPIError(detail="Connection not found in this workspace.", code="conn_not_found", http_status=404) - new_cred = otlp.create_credential( - project=project, connection=conn, auth_mode=auth_mode, user=request.user, - capture_level=data.get("capture_level", "structural"), - ) - endpoint = _endpoint_url(request, origin=(request.data.get("origin") or "").strip()) + try: + new_cred = otlp.create_credential( + project=project, connection=conn, auth_mode=auth_mode, user=request.user, + capture_level=data.get("capture_level", "structural"), + ) + except ValueError as exc: + raise StableAPIError(detail=str(exc), code="invalid_auth_mode") from exc + endpoint = _endpoint_url(request, origin=_request_origin(request)) payload = { "credential": _serialize_cred(new_cred.credential, request), "secret": new_cred.secret, @@ -727,7 +742,7 @@ def rotate_credential(request, cred_id): new_cred = otlp.rotate_credential(cred) except ValueError as e: raise StableAPIError(detail=str(e), code="cannot_rotate", http_status=400) - endpoint = _endpoint_url(request, origin=(request.data.get("origin") or "").strip()) + endpoint = _endpoint_url(request, origin=_request_origin(request)) payload = { "credential": _serialize_cred(new_cred.credential, request), "secret": new_cred.secret, @@ -753,5 +768,5 @@ def revoke_credential(request, cred_id): raise StableAPIError(detail="Credential not found.", code="cred_not_found", http_status=404) cred.enabled = False cred.save(update_fields=["enabled", "updated_at"]) - otlp.clear_target_spans(cred.target_id) + clear_target_spans(cred.target_id) return Response({"ok": True, "credential": _serialize_cred(cred, request)}) diff --git a/model_registry/serializers.py b/model_registry/serializers.py index ae64a3b0..cfedb93a 100644 --- a/model_registry/serializers.py +++ b/model_registry/serializers.py @@ -45,11 +45,11 @@ def to_internal_value(self, data): self.fail("invalid") host = hostport[1:close] rest = hostport[close + 1:] - if rest and not (rest.startswith(":") and rest[1:].isdigit()): + if rest and not (rest.startswith(":") and rest[1:].isdecimal()): self.fail("invalid") elif ":" in hostport: host, _, port = hostport.partition(":") - if not port.isdigit(): + if not port.isdecimal(): self.fail("invalid") else: host = hostport diff --git a/model_registry/services.py b/model_registry/services.py index 76fb6c4b..18cf8365 100644 --- a/model_registry/services.py +++ b/model_registry/services.py @@ -31,6 +31,38 @@ def is_internal_agent_connection(connection) -> bool: ] +# Environment variables that hold Studio's own secrets. A connection's +# ``secret_reference`` may not name them: its key goes to a URL the user chose. +_PLATFORM_SECRET_NAMES = frozenset({ + "BOOTSTRAP_PASSWORD", "DEMO_PASSWORD", "DATABASE_URL", "HATCHET_API_KEY", + "HATCHET_CLIENT_TOKEN", "HATCHET_TOKEN_FILE", "SIMPLEAUDIT_AUTO_LOGIN_TOKEN", + "SIMPLEAUDIT_CHAT_OTLP_PASSWORD", "WEBUI_SECRET_KEY", "WORKOS_API_KEY", +}) +_PLATFORM_SECRET_PREFIXES = ("DJANGO_", "POSTGRES_", "HATCHET_", "WORKOS_", "WEBUI_", "AWS_SECRET") + + +def is_platform_secret_name(name: str) -> bool: + """Whether ``name`` is one of Studio's own secrets rather than a model key.""" + upper = (name or "").strip().upper() + return upper in _PLATFORM_SECRET_NAMES or upper.startswith(_PLATFORM_SECRET_PREFIXES) + + +def _same_origin(url: str, other: str) -> bool: + """Exact scheme/host/port match; URLs carrying credentials never match.""" + from urllib.parse import urlsplit + + a, b = urlsplit(url.strip()), urlsplit(other.strip()) + if a.username or a.password: + return False + defaults = {"http": 80, "https": 443} + try: + port_a = a.port or defaults.get(a.scheme) + port_b = b.port or defaults.get(b.scheme) + except ValueError: + return False + return bool(a.hostname) and (a.scheme, a.hostname, port_a) == (b.scheme, b.hostname, port_b) + + def connection_api_key(conn) -> str: """The connection's API key: the stored key, else the env var it names. @@ -42,11 +74,15 @@ def connection_api_key(conn) -> str: return direct ref = (conn.secret_reference or "").strip() if ref: + if is_platform_secret_name(ref): + # A connection's key is sent to its user-chosen base URL; never let + # it name the platform's own secrets. + logger.warning("Refusing platform secret %s as a connection key", ref) + return "" return os.environ.get(ref, "") from chat import config as chat - base = (conn.base_url or "").strip().rstrip("/") - if chat.ENABLED and base.startswith(chat.UPSTREAM.rstrip("/")): + if chat.ENABLED and _same_origin(conn.base_url or "", chat.UPSTREAM): from chat.api import service_token return service_token() or "" @@ -424,14 +460,29 @@ def _ensure_agent_model_reference(agent, user) -> None: if not agent.external_id: return from chat import config as chat_config - from model_registry.models import ModelConnection, RegisteredModel - - model = ( - RegisteredModel.objects.select_related("connection") - .filter(project=agent.project, connection__name="Open WebUI Agents") - .order_by("id") - .first() - ) + from model_registry.models import Agent, ModelConnection, RegisteredModel + + # One reference row per agent: this agent's own row, else a stale row no + # other agent owns (re-pointed below). Never another agent's row — that + # flipped every agent's runs, filters and picker entry to whichever agent + # synced last. + refs = RegisteredModel.objects.select_related("connection").filter( + project=agent.project, connection__name="Open WebUI Agents" + ).order_by("id") + model = refs.filter(model_id=agent.external_id).first() if agent.external_id else None + if model is None: + owned = set( + Agent.objects.filter(project=agent.project).exclude(pk=agent.pk) + .exclude(external_id="").values_list("external_id", flat=True) + ) + # A row runs or monitors point at belongs to its (possibly deleted) + # agent's history; re-pointing it would relabel those runs. + free = refs.filter( + target_audit_runs__isnull=True, auditor_audit_runs__isnull=True, judge_audit_runs__isnull=True, + target_monitors__isnull=True, auditor_monitors__isnull=True, judge_monitors__isnull=True, + agents__isnull=True, + ).distinct() + model = next((m for m in free if m.model_id not in owned), None) if model is not None: updated = [] if model.display_name != agent.name: @@ -513,12 +564,12 @@ def agent_live_openwebui(agent, user) -> dict | None: return None -def delete_agent_from_openwebui(agent) -> None: +def delete_agent_from_openwebui(agent, user=None) -> None: """Delete an agent's Open WebUI model entry (best-effort, never raises).""" if not _chat_sync_enabled() or not agent.external_id: return try: - _adapter_for(_any_user()).delete_agent(agent) + _adapter_for(_remote_actor(agent, user)).delete_agent(agent) except Exception: logger.exception("Agent '%s' Open WebUI delete failed", agent.name) @@ -545,11 +596,11 @@ def sync_knowledge_base_to_openwebui(kb, user) -> str: return "skipped" -def delete_knowledge_base_from_openwebui(kb) -> None: +def delete_knowledge_base_from_openwebui(kb, user=None) -> None: if not _chat_sync_enabled() or not kb.external_id: return try: - _adapter_for(_any_user())._api.delete_knowledge_base(kb.external_id) + _adapter_for(_remote_actor(kb, user))._api.delete_knowledge_base(kb.external_id) except Exception: logger.exception("Knowledge base '%s' Open WebUI delete failed", kb.name) @@ -578,15 +629,26 @@ def sync_tool_to_openwebui(tool, user, *, content: str) -> str: return "skipped" -def delete_tool_from_openwebui(tool) -> None: +def delete_tool_from_openwebui(tool, user=None) -> None: if not _chat_sync_enabled() or not tool.external_id: return try: - _adapter_for(_any_user())._api.delete_tool(tool.external_id) + _adapter_for(_remote_actor(tool, user))._api.delete_tool(tool.external_id) except Exception: logger.exception("Tool '%s' Open WebUI delete failed", tool.name) +def _remote_actor(obj, user): + """Who performs a remote delete: the object's creator, else the requester. + + Never an arbitrary superuser: an object imported from Open WebUI (sync) is + created_by the importing user, so Open WebUI's own access rules decide + whether that user may delete it. ``_any_user`` remains the last resort for + callers without a requester. + """ + return getattr(obj, "created_by", None) or user or _any_user() + + def _any_user(): """An authenticated user for a deletion-time API call (admin is cleanest).""" from django.contrib.auth import get_user_model diff --git a/model_registry/tests/test_agent_target_resolution.py b/model_registry/tests/test_agent_target_resolution.py index 40c4c98f..234307f7 100644 --- a/model_registry/tests/test_agent_target_resolution.py +++ b/model_registry/tests/test_agent_target_resolution.py @@ -38,3 +38,32 @@ def test_stale_owui_reference_is_repointed_to_base_model(self): # engine can resolve it against the live OpenWebUI endpoint. # (its primary key), which resolves to a live endpoint. self.assertEqual(target.model_id, agent.external_id) + + +class AgentReferencePerAgentTest(TestCase): + def test_each_agent_keeps_its_own_reference_row(self): + from infra.tests.factories import ProjectFactory + + project = ProjectFactory() + a = AgentFactory(project=project, external_id="studio.a") + b = AgentFactory(project=project, external_id="studio.b") + ref_a = agent_target_model(a) + ref_b = agent_target_model(b) + self.assertNotEqual(ref_a.pk, ref_b.pk) + ref_a.refresh_from_db() + self.assertEqual(ref_a.model_id, "studio.a") + self.assertEqual(ref_a.display_name, a.name) + + def test_deleted_agents_row_pinned_by_a_run_is_not_reused(self): + from infra.tests.factories import AuditRunFactory, ProjectFactory + + project = ProjectFactory() + a = AgentFactory(project=project, external_id="studio.a") + ref_a = agent_target_model(a) + AuditRunFactory(project=project, target_model=ref_a) + a.delete() + b = AgentFactory(project=project, external_id="studio.b") + ref_b = agent_target_model(b) + self.assertNotEqual(ref_a.pk, ref_b.pk) + ref_a.refresh_from_db() + self.assertEqual(ref_a.model_id, "studio.a") diff --git a/model_registry/tests/test_api_passthrough.py b/model_registry/tests/test_api_passthrough.py index 83b5819c..f89b68b5 100644 --- a/model_registry/tests/test_api_passthrough.py +++ b/model_registry/tests/test_api_passthrough.py @@ -44,6 +44,8 @@ class ApiCredentialServiceTest(TestCase): def setUp(self): self.user = UserFactory() self.project = ProjectFactory() + # Keys only verify while the owner is a member of the key's workspace. + MembershipFactory(user=self.user, project=self.project) def test_upsert_returns_token_and_stores_hash(self): nc = api_cred.upsert_credential(project=self.project, user=self.user) @@ -330,3 +332,20 @@ def test_unauthenticated_forbidden(self): anon.post("/api/v1/keys/issue/", HTTP_AUTHORIZATION="").status_code, (401, 403), ) + + +class PassthroughBadModelFieldTest(TestCase): + def test_non_string_model_is_400(self): + from model_registry import api_credential_services as cred_services + + user = UserFactory() + project = ProjectFactory() + MembershipFactory(user=user, project=project) + token = cred_services.upsert_credential(project=project, user=user).secret + client = Client() + with mock.patch("chat.config.ENABLED", True): + resp = client.post( + "/api/v1/chat/completions", json.dumps({"model": 123, "messages": []}), + content_type="application/json", HTTP_AUTHORIZATION=f"Bearer {token}", + ) + self.assertEqual(resp.status_code, 400) diff --git a/prompts/services.py b/prompts/services.py index 6482df28..212b505c 100644 --- a/prompts/services.py +++ b/prompts/services.py @@ -60,6 +60,10 @@ def delete_version(version: SystemPromptVersion) -> None: from audits.models import AuditRun + # A run that names this version keeps it: deleting it would null the run's + # provenance link (and then let the whole prompt be deleted). + if version.audit_runs.exists(): + raise ValueError(f"{version.label} is pinned by a run, so it is kept for reproducibility.") # A version is kept when a run froze its exact text (in the agent config # snapshot or the generation parameters), so history stays reproducible. used = AuditRun.objects.filter(project=version.prompt.project).exclude(id__isnull=True) diff --git a/prompts/views.py b/prompts/views.py index 8bfcdd00..567d9dc8 100644 --- a/prompts/views.py +++ b/prompts/views.py @@ -7,7 +7,7 @@ from django.shortcuts import redirect from django.views.generic import TemplateView -from infra.ui import ProjectMixin, _require_write_access +from infra.ui import ProjectMixin, _int, _require_write_access from prompts.models import SystemPrompt, SystemPromptVersion from prompts.services import ( create_prompt, @@ -59,7 +59,7 @@ def post(self, request): action = request.POST.get("action") if action == "delete_version": version = SystemPromptVersion.objects.select_related("prompt").filter( - pk=request.POST.get("version_id") or 0, prompt__project=p).first() + pk=_int(request.POST.get("version_id")), prompt__project=p).first() if version is None: messages.error(request, "Version not found.") return redirect("prompts") @@ -70,7 +70,7 @@ def post(self, request): messages.error(request, str(e)) return redirect("prompt_detail", prompt_id=version.prompt_id) elif action == "delete": - prompt = SystemPrompt.objects.filter(pk=request.POST.get("prompt_id") or 0, project=p).first() + prompt = SystemPrompt.objects.filter(pk=_int(request.POST.get("prompt_id")), project=p).first() if prompt is not None: try: delete_prompt(prompt) @@ -112,7 +112,7 @@ def get_context_data(self, **kw): cloning = None if prompt is None: cloning = SystemPromptVersion.objects.select_related("prompt").filter( - pk=self.request.GET.get("clone") or (form or {}).get("clone") or 0, + pk=_int(self.request.GET.get("clone") or (form or {}).get("clone")), prompt__project=self.request.project, ).first() if form is None: @@ -153,7 +153,8 @@ def post(self, request, prompt_id=None): prompt = self._prompt() post = request.POST form = {k: post.get(k, "") for k in ("name", "description", "content", "note")} - name = form["name"].strip() + name = form["name"].strip()[:250] # model max_length (Postgres rejects longer) + form["note"] = form["note"][:500] error = None if not name: error = "Give the prompt a name." diff --git a/scenarios/serializers.py b/scenarios/serializers.py index e3942abc..2507db36 100644 --- a/scenarios/serializers.py +++ b/scenarios/serializers.py @@ -12,6 +12,13 @@ def _validate_severity_ceiling(value: str) -> str: return value +def _validate_document(value): + # The engine accepts each document as a string (path/URI/text) or an object. + if not isinstance(value, (str, dict)): + raise serializers.ValidationError("Each document must be a string or an object.") + return value + + class ScenarioRevisionSerializer(serializers.ModelSerializer): class Meta: model = ScenarioRevision @@ -37,17 +44,17 @@ def get_latest_revision(self, obj) -> dict | None: class ScenarioCreateSerializer(serializers.Serializer): - key = serializers.CharField(required=False, allow_blank=True, default="") + key = serializers.CharField(required=False, allow_blank=True, default="", max_length=200) title = serializers.CharField(max_length=300) description = serializers.CharField(allow_blank=False) expected_behavior = serializers.ListField(child=serializers.CharField(), allow_empty=False) test_prompt = serializers.CharField(required=False, allow_blank=True, default="") severity_ceiling = serializers.CharField(required=False, allow_blank=True, default="", validators=[_validate_severity_ceiling]) - documents = serializers.ListField(child=serializers.DictField(), required=False, default=list) + documents = serializers.ListField(child=serializers.JSONField(validators=[_validate_document]), required=False, default=list) file_uri = serializers.JSONField(required=False, allow_null=True, default=None) decision = serializers.JSONField(required=False, allow_null=True, default=None) metadata = serializers.DictField(required=False, default=dict) - category = serializers.CharField(required=False, allow_blank=True, default="") + category = serializers.CharField(required=False, allow_blank=True, default="", max_length=120) tags = serializers.ListField(child=serializers.CharField(), required=False, default=list) def validate_decision(self, value): @@ -68,12 +75,20 @@ def validate(self, attrs): return attrs +class ScenarioImportItemSerializer(ScenarioCreateSerializer): + """One imported scenario: the create contract's types and lengths, with the + import endpoint's long-standing leniency (blank description, no behaviours).""" + + description = serializers.CharField(required=False, allow_blank=True, default="") + expected_behavior = serializers.ListField(child=serializers.CharField(), required=False, default=list) + + class ScenarioUpdateSerializer(serializers.Serializer): description = serializers.CharField(allow_blank=False) expected_behavior = serializers.ListField(child=serializers.CharField(), allow_empty=False) test_prompt = serializers.CharField(required=False, allow_blank=True, default="") severity_ceiling = serializers.CharField(required=False, allow_blank=True, default="", validators=[_validate_severity_ceiling]) - documents = serializers.ListField(child=serializers.DictField(), required=False, default=list) + documents = serializers.ListField(child=serializers.JSONField(validators=[_validate_document]), required=False, default=list) file_uri = serializers.JSONField(required=False, allow_null=True, default=None) decision = serializers.JSONField(required=False, allow_null=True, default=None) metadata = serializers.DictField(required=False, default=dict) @@ -109,7 +124,7 @@ class ScenarioSetCreateSerializer(serializers.Serializer): class PublishScenarioSetVersionSerializer(serializers.Serializer): - scenario_ids = serializers.ListField(child=serializers.IntegerField(), allow_empty=False) + scenario_ids = serializers.ListField(child=serializers.IntegerField(min_value=1, max_value=2**63 - 1), allow_empty=False) class ScenarioSetVersionItemSerializer(serializers.Serializer): diff --git a/scenarios/services.py b/scenarios/services.py index 987e65bc..9d3df318 100644 --- a/scenarios/services.py +++ b/scenarios/services.py @@ -52,7 +52,7 @@ def create_scenario(*, project: Project, user, key: str, title: str, description test_prompt: str = "", severity_ceiling: str = "", documents=None, file_uri=None, decision=None, metadata: dict | None = None, category: str = "", tags: list | None = None) -> Scenario: require_project_role(user, project, ProjectMembership.Role.ADMIN, ProjectMembership.Role.AUDITOR) - normalized_key = slugify(key) or slugify(title) + normalized_key = (slugify(key) or slugify(title))[:200] # Scenario.key max_length if not normalized_key: raise StableAPIError(detail="Scenario key or title is required.", code="invalid_scenario_key") @@ -146,14 +146,20 @@ def create_scenario_set(*, project: Project, user, name: str, description: str = @transaction.atomic -def publish_scenario_set_version(*, scenario_set: ScenarioSet, user, scenario_ids: list[int]) -> ScenarioSetVersion: +def publish_scenario_set_version( + *, scenario_set: ScenarioSet, user, scenario_ids: list[int], revisions: dict[int, ScenarioRevision] | None = None, +) -> ScenarioSetVersion: """Publish an immutable ordered version from current latest revisions. + ``revisions`` pins specific revisions per scenario id (a revert restores the + old version's exact text); scenarios not in it use their latest revision. + Concurrency note: two concurrent publishes may attempt the same next version. The unique `(scenario_set, version)` constraint rejects one; callers should retry after re-reading the latest version. """ require_project_role(user, scenario_set.project, ProjectMembership.Role.ADMIN, ProjectMembership.Role.AUDITOR) + scenario_ids = list(dict.fromkeys(scenario_ids)) # a repeated id must not violate the item constraint if not scenario_ids: raise StableAPIError(detail="At least one scenario is required.", code="empty_scenario_set") @@ -165,7 +171,10 @@ def publish_scenario_set_version(*, scenario_set: ScenarioSet, user, scenario_id items = [] for position, scenario_id in enumerate(scenario_ids, start=1): scenario = scenarios[scenario_id] - revision = ScenarioRevision.objects.filter(scenario=scenario).order_by("-revision").first() + pinned = (revisions or {}).get(scenario_id) + if pinned is not None and pinned.scenario_id != scenario_id: + raise StableAPIError(detail="Revision does not belong to its scenario.", code="revision_mismatch") + revision = pinned or ScenarioRevision.objects.filter(scenario=scenario).order_by("-revision").first() if not revision: raise StableAPIError(detail=f"Scenario {scenario.key} has no revision.", code="scenario_missing_revision") items.append( diff --git a/scenarios/views.py b/scenarios/views.py index 4e9dde55..e48bbf6c 100644 --- a/scenarios/views.py +++ b/scenarios/views.py @@ -10,6 +10,7 @@ from scenarios.serializers import ( PublishScenarioSetVersionSerializer, ScenarioCreateSerializer, + ScenarioImportItemSerializer, ScenarioListSerializer, ScenarioRevisionSerializer, ScenarioSetCreateSerializer, @@ -53,7 +54,10 @@ def create_scenario_view(request, project_id): _require_project_access(request.user, project) serializer = ScenarioCreateSerializer(data=request.data) serializer.is_valid(raise_exception=True) - scenario = create_scenario(project=project, user=request.user, **serializer.validated_data) + try: + scenario = create_scenario(project=project, user=request.user, **serializer.validated_data) + except (TypeError, ValueError) as exc: # agentic metadata validation: a client error + raise StableAPIError(detail=str(exc), code="invalid_metadata", http_status=400) from exc return Response(ScenarioListSerializer(scenario).data, status=status.HTTP_201_CREATED) @@ -80,7 +84,10 @@ def update_scenario_view(request, project_id, scenario_id): raise StableAPIError(detail="Scenario not found.", code="scenario_not_found", http_status=404) from exc serializer = ScenarioUpdateSerializer(data=request.data) serializer.is_valid(raise_exception=True) - revision = update_scenario_content(scenario=scenario, user=request.user, **serializer.validated_data) + try: + revision = update_scenario_content(scenario=scenario, user=request.user, **serializer.validated_data) + except (TypeError, ValueError) as exc: # agentic metadata validation: a client error + raise StableAPIError(detail=str(exc), code="invalid_metadata", http_status=400) from exc return Response(ScenarioRevisionSerializer(revision).data, status=status.HTTP_201_CREATED) @@ -212,30 +219,22 @@ def import_scenarios(request, project_id): skipped = 0 errors = [] for item in items: - key = item.get("key", "").strip() + key = item.get("key") if isinstance(item, dict) else None + key = key.strip() if isinstance(key, str) else "" if not key: errors.append({"item": item, "error": "missing key"}) continue if Scenario.objects.filter(project=project, key=key).exists(): skipped += 1 continue + # Same validation as the create endpoint: types, lengths, severity, + # decision and agentic metadata. + item_serializer = ScenarioImportItemSerializer(data={"title": key, **item, "key": key}) + if not item_serializer.is_valid(): + errors.append({"key": key, "error": item_serializer.errors}) + continue try: - create_scenario( - project=project, - user=request.user, - key=key, - title=item.get("title", key), - description=item.get("description", ""), - expected_behavior=item.get("expected_behavior", []), - test_prompt=item.get("test_prompt", ""), - severity_ceiling=item.get("severity_ceiling", ""), - documents=item.get("documents", []), - file_uri=item.get("file_uri"), - decision=item.get("decision"), - metadata=item.get("metadata", {}), - category=item.get("category", ""), - tags=item.get("tags", []), - ) + create_scenario(project=project, user=request.user, **item_serializer.validated_data) created += 1 except Exception as exc: # noqa: BLE001 - collect per-scenario import errors errors.append({"key": key, "error": str(exc)}) diff --git a/simpleaudit_studio/cli.py b/simpleaudit_studio/cli.py index 4a8bb6a8..98ee3163 100644 --- a/simpleaudit_studio/cli.py +++ b/simpleaudit_studio/cli.py @@ -213,10 +213,9 @@ def main() -> None: _update_model_endpoints(mock_url) print("✅ Model endpoints configured (mock).\n") else: - # Repair any connection an earlier version left pointing at the mock, - # then make sure all connections point at OpenAI's real base URL. + # Restore connections a previous --mock run pointed at the mock server. _restore_real_model_endpoints() - print("✅ Model endpoints point at OpenAI (add your API key in the UI).\n") + print("✅ Model endpoints restored (add your API key in the UI if needed).\n") # --- Step 6: Start Django web server in a daemon thread --- port = args.port @@ -566,38 +565,63 @@ def _seed_demo_data() -> None: call_command("seed_agentic_scenarios", project=project.id, verbosity=0) +_MOCK_BACKUP_KEY = "mock_endpoint_backup" + + def _update_model_endpoints(mock_url: str) -> None: - """Point all seeded model connections at the local mock server. + """Point model connections at the local mock server for this run. - The mock server ignores auth entirely, so no API key is set — the - connection stays keyless (shown as "⚠ no key" in the UI). + The original base URLs are saved first (once: a second ``--mock`` run must + not save the mock URL as the "original") and restored on the next normal + start. API keys are left alone; the mock ignores auth. Studio's own agent + connection is never touched. """ + from accounts.services import get_site_setting, set_site_setting from model_registry.models import ModelConnection - - ModelConnection.objects.filter(enabled=True).update( - base_url=mock_url, - api_key_direct="", - secret_reference="", - ) + from model_registry.services import INTERNAL_AGENT_CONNECTION_NAMES + + conns = ModelConnection.objects.filter(enabled=True).exclude(name__in=INTERNAL_AGENT_CONNECTION_NAMES) + backup = dict(get_site_setting(_MOCK_BACKUP_KEY) or {}) + urls = dict(backup.get("urls") or {}) + # Every mock URL used so far (each run binds a new port): the restore skips + # rows the user re-pointed meanwhile, and a URL edited between mock runs + # (anything not a previous mock URL) becomes the one to restore. + mock_urls = list(backup.get("mock_urls") or []) + for conn in conns: + if str(conn.pk) not in urls or conn.base_url not in mock_urls: + urls[str(conn.pk)] = conn.base_url + set_site_setting(_MOCK_BACKUP_KEY, {"urls": urls, "mock_urls": [*mock_urls, mock_url]}) + conns.update(base_url=mock_url) def _restore_real_model_endpoints() -> None: - """Make sure model connections point at OpenAI's real base URL. + """Undo a previous ``--mock`` run: restore each connection's saved base URL. - Seeding already creates the default connection at https://api.openai.com/v1. - This repairs any connection an earlier version left pointing at the local - mock server (http://127.0.0.1:PORT/v1) so future runs target OpenAI. - User-configured custom endpoints (non-local hosts) are left untouched. + Only connections the mock run changed are touched. Legacy repair (versions + that wiped keys and saved no backup): a keyless connection whose URL is + exactly the old mock shape ``http://127.0.0.1:/v1`` goes back to + OpenAI. Local endpoints such as Ollama/vLLM on localhost are left alone. """ - from model_registry.models import ModelConnection + import re - OPENAI_BASE_URL = "https://api.openai.com/v1" - for conn in ModelConnection.objects.filter(enabled=True): - # Extract hostname (strip scheme, path, and port) - netloc = (conn.base_url or "").split("//", 1)[-1].split("/", 1)[0] - host = netloc.rsplit(":", 1)[0] if ":" in netloc else netloc - if host in ("127.0.0.1", "localhost", "0.0.0.0"): - conn.base_url = OPENAI_BASE_URL + from accounts.services import get_site_setting, set_site_setting + from model_registry.models import ModelConnection + from model_registry.services import INTERNAL_AGENT_CONNECTION_NAMES + + backup = get_site_setting(_MOCK_BACKUP_KEY) or {} + mock_urls = backup.get("mock_urls") or [] + for pk, base_url in (backup.get("urls") or {}).items(): + # Only rows still on a mock URL: a URL edited during a mock run stays. + ModelConnection.objects.filter(pk=int(pk), base_url__in=mock_urls).update(base_url=base_url) + if backup: + set_site_setting(_MOCK_BACKUP_KEY, {}) + return + # The old mock bound an OS-assigned (ephemeral, >= 32768) port. + legacy_mock = re.compile(r"^http://127\.0\.0\.1:(\d+)/v1/?$") + for conn in ModelConnection.objects.filter(enabled=True).exclude(name__in=INTERNAL_AGENT_CONNECTION_NAMES): + match = legacy_mock.match(conn.base_url or "") + if match and int(match.group(1)) >= 32768 and not conn.api_key_direct and not conn.secret_reference: + conn.base_url = "https://api.openai.com/v1" conn.save(update_fields=["base_url"]) diff --git a/static/visualizer.html b/static/visualizer.html index 0c1a2d5e..2a382fe3 100644 --- a/static/visualizer.html +++ b/static/visualizer.html @@ -1821,7 +1821,7 @@

${escapeHtml(s.scenario_name || s.name || 'Untitled Scenario')}

- @@ -166,7 +166,7 @@

Members — workspace #{{ manage {{ m.user.username }} {{ m.user.email }} - @@ -259,9 +259,9 @@

All users ({{ users|length
diff --git a/templates/admin/rag_settings.html b/templates/admin/rag_settings.html index 172a3575..dfb0b392 100644 --- a/templates/admin/rag_settings.html +++ b/templates/admin/rag_settings.html @@ -244,12 +244,12 @@

Existing knowledge indexes

['Gated', m.gated ? 'Yes' : 'No'], ]; const tagChips = (m.tags || []).slice(0, 8).map(t => - `${t}` + `${esc(t)}` ).join(''); detailContent.innerHTML = ` -

${m.id}

+

${esc(m.id)}

- ${rows.map(([k, v]) => `
${k}
${v}
`).join('')} + ${rows.map(([k, v]) => `
${esc(k)}
${esc(v)}
`).join('')}
${tagChips ? `
${tagChips}
` : ''}`; } catch (err) { @@ -257,6 +257,9 @@

Existing knowledge indexes

} } + // Hugging Face model metadata is third-party content: escape it. + const esc = (v) => String(v ?? '').replace(/[&<>"']/g, c => ({'&': '&', '<': '<', '>': '>', '"': '"', "'": '''}[c])); + function renderResults(models) { if (!models.length) { resultsDiv.innerHTML = '

No models found.

'; @@ -264,13 +267,13 @@

Existing knowledge indexes

} resultsDiv.innerHTML = models.map(m => `
- - +
`).join(''); resultsDiv.querySelectorAll('[data-select]').forEach(btn => { btn.addEventListener('click', () => { diff --git a/templates/agents/agent_detail.html b/templates/agents/agent_detail.html index 5f4fdee8..d1d6b8e2 100644 --- a/templates/agents/agent_detail.html +++ b/templates/agents/agent_detail.html @@ -22,7 +22,7 @@

{% if agent %}Edit Agent{% else Identity
-
@@ -126,7 +126,7 @@

{% if agent %}Edit Agent{% else
@@ -224,11 +222,22 @@ try { localStorage.setItem('sa-sidebar-collapsed', collapsed ? '1' : '0'); } catch (err) { /* private mode */ } } + const sidebar = document.getElementById('sidebar'); + // On mobile the closed drawer is only translated off-screen; make it inert so + // Tab can't land on invisible links. Desktop keeps it interactive. + function syncDrawerInert() { + if (!sidebar) return; + sidebar.inert = !isDesktop() && !root.classList.contains('sidebar-open'); + } + function setDrawer(open) { + const wasOpen = root.classList.contains('sidebar-open'); root.classList.toggle('sidebar-open', open); if (backdrop) backdrop.hidden = !open; openBtn?.setAttribute('aria-expanded', open); + syncDrawerInert(); if (open) document.querySelector('#sidebar a, #sidebar button')?.focus(); + else if (wasOpen) openBtn?.focus(); } // Header hamburger: collapse on desktop, open drawer on mobile. @@ -254,7 +263,11 @@ } document.addEventListener('keydown', e => { if (e.key === 'Escape' && root.classList.contains('sidebar-open')) setDrawer(false); }); - matchMedia('(min-width: 768px)').addEventListener('change', e => { if (e.matches) setDrawer(false); }); + matchMedia('(min-width: 768px)').addEventListener('change', e => { if (e.matches) setDrawer(false); else syncDrawerInert(); }); + // Pages that reuse this sidebar (the visualizer) toggle html.sidebar-open + // themselves; follow the class so the drawer is never open but inert. + new MutationObserver(syncDrawerInert).observe(root, { attributes: true, attributeFilter: ['class'] }); + syncDrawerInert(); })();