System audit: security, data-integrity and correctness fixes

.env.example

@@ -155,3 +155,15 @@ SENTRY_ENVIRONMENT=development
 # WORKOS_CLIENT_ID is public; WORKOS_API_KEY is server-side secret (keep out of git).
 WORKOS_CLIENT_ID=
 WORKOS_API_KEY=
+
+# --- Proxy / HTTPS ------------------------------------------------------------
+# Full origins allowed for CSRF when served behind a proxy, comma-separated.
+# DJANGO_CSRF_TRUSTED_ORIGINS=https://studio.example.com
+
+# --- Gunicorn tuning (defaults shown) -------------------------------------------
+# GUNICORN_BIND=0.0.0.0:8000
+# GUNICORN_WORKERS=2
+# GUNICORN_THREADS=16
+# GUNICORN_TIMEOUT=60
+# GUNICORN_GRACEFUL_TIMEOUT=30
+# GUNICORN_KEEPALIVE=5

accounts/serializers.py

@@ -1,4 +1,5 @@
 from django.contrib.auth.password_validation import validate_password
+from django.contrib.auth.validators import UnicodeUsernameValidator
 from rest_framework import serializers
 
 from accounts.models import Project, ProjectMembership, User
@@ -12,11 +13,11 @@ class Meta:
 
 
 class RegisterSerializer(serializers.Serializer):
-    username = serializers.CharField(max_length=150)
-    email = serializers.EmailField()
+    username = serializers.CharField(max_length=150, validators=[UnicodeUsernameValidator()])
+    email = serializers.EmailField(max_length=254)
     password = serializers.CharField(write_only=True, style={"input_type": "password"})
-    first_name = serializers.CharField(required=False, allow_blank=True, default="")
-    last_name = serializers.CharField(required=False, allow_blank=True, default="")
+    first_name = serializers.CharField(required=False, allow_blank=True, default="", max_length=150)
+    last_name = serializers.CharField(required=False, allow_blank=True, default="", max_length=150)
 
     def validate_username(self, value):
         if User.objects.filter(username__iexact=value).exists():
@@ -91,7 +92,7 @@ class ProfileUpdateSerializer(serializers.Serializer):
 
     first_name = serializers.CharField(required=False, allow_blank=True, max_length=150)
     last_name = serializers.CharField(required=False, allow_blank=True, max_length=150)
-    email = serializers.EmailField(required=False, allow_blank=True)
+    email = serializers.EmailField(required=False, allow_blank=True, max_length=254)
     username = serializers.CharField(required=False, max_length=150)
     current_password = serializers.CharField(required=False, style={"input_type": "password"}, write_only=True)
     new_password = serializers.CharField(required=False, style={"input_type": "password"}, write_only=True)
@@ -142,26 +143,29 @@ def save(self):
         if attrs.get("new_password"):
             user.set_password(attrs["new_password"])
             user.save(update_fields=["password"])
+            from accounts.services import revoke_auth_tokens
+
+            revoke_auth_tokens(user)
         return user
 
 
 # ─── Super-admin user management ─────────────────────────────────────────────
 
 
 class UserCreateSerializer(serializers.Serializer):
-    username = serializers.CharField(max_length=150)
-    email = serializers.EmailField(required=False, allow_blank=True)
+    username = serializers.CharField(max_length=150, validators=[UnicodeUsernameValidator()])
+    email = serializers.EmailField(required=False, allow_blank=True, max_length=254)
     password = serializers.CharField(write_only=True, style={"input_type": "password"})
-    first_name = serializers.CharField(required=False, allow_blank=True, default="")
-    last_name = serializers.CharField(required=False, allow_blank=True, default="")
+    first_name = serializers.CharField(required=False, allow_blank=True, default="", max_length=150)
+    last_name = serializers.CharField(required=False, allow_blank=True, default="", max_length=150)
 
 
 class UserAdminUpdateSerializer(serializers.Serializer):
     """Super-admin edit of any user. All fields optional."""
 
     first_name = serializers.CharField(required=False, allow_blank=True, max_length=150)
     last_name = serializers.CharField(required=False, allow_blank=True, max_length=150)
-    email = serializers.EmailField(required=False, allow_blank=True)
+    email = serializers.EmailField(required=False, allow_blank=True, max_length=254)
     is_active = serializers.BooleanField(required=False)
     is_superuser = serializers.BooleanField(required=False)
     password = serializers.CharField(required=False, write_only=True, style={"input_type": "password"})

accounts/services.py

@@ -44,6 +44,17 @@ def has_local_password(user) -> bool:
 
 
 @transaction.atomic
+def revoke_auth_tokens(user) -> None:
+    """Drop the user's DRF auth tokens (call after a password change or reset).
+
+    Django sessions are tied to the password hash and end on their own; tokens
+    never expire, so without this an old token keeps full access.
+    """
+    from rest_framework.authtoken.models import Token
+
+    Token.objects.filter(user=user).delete()
+
+
 def bootstrap_admin_and_default_project(
     *,
     username: str,
@@ -63,6 +74,7 @@ def bootstrap_admin_and_default_project(
     if created or not user.check_password(password):
         user.set_password(password)
         user.save(update_fields=["password"])
+        revoke_auth_tokens(user)
     if not user.is_active:
         user.is_active = True
         user.save(update_fields=["is_active"])
@@ -236,6 +248,7 @@ def delete_workspace(*, user, project: Project) -> None:
     """Delete an EMPTY workspace. Refuses while any audit content exists so a
     mistaken click can never destroy experiment history."""
     _require_workspace_admin(user, project)
+    require_project_writable(user, project)   # archived workspaces are read-only
 
     from audits.models import AuditRun
     from model_registry.models import ModelConnection
@@ -358,6 +371,8 @@ def admin_update_user(
         _validate_password_or_raise(password)
         target.set_password(password)
     target.save()
+    if password:
+        revoke_auth_tokens(target)
     return target
 
 

accounts/views.py

@@ -81,8 +81,11 @@ def obtain_token(request):
     The SPA uses token auth (``Authorization: Token <key>``) so it works across
     origins and without CSRF handling. Returns the token plus the user payload.
     """
-    username = request.data.get("username", "")
-    password = request.data.get("password", "")
+    data = request.data if isinstance(request.data, dict) else {}
+    username = data.get("username", "")
+    password = data.get("password", "")
+    if not isinstance(username, str) or not isinstance(password, str):
+        raise StableAPIError(detail="Invalid credentials.", code="invalid_credentials", http_status=401)
     user = authenticate(request, username=username, password=password)
     if user is None:
         raise StableAPIError(detail="Invalid credentials.", code="invalid_credentials", http_status=401)
@@ -237,8 +240,9 @@ def workspace_detail(request, project_id):
 @permission_classes([IsAuthenticated])
 def switch_workspace(request):
     """Set the session's active workspace so all UI pages scope to it."""
-    raw_id = request.data.get("project_id") or request.data.get("workspace_id")
-    if not raw_id or not str(raw_id).isdigit():
+    data = request.data if isinstance(request.data, dict) else {}
+    raw_id = data.get("project_id") or data.get("workspace_id")
+    if not raw_id or not str(raw_id).isascii() or not str(raw_id).isdecimal() or len(str(raw_id)) > 18:
         raise StableAPIError(detail="project_id is required.", code="invalid_workspace_id")
     project = _get_project_or_404(int(raw_id))
     if not ensure_project_access(request.user, project):
@@ -314,8 +318,12 @@ def add_member(request, project_id):
         defaults={"role": role, "created_by": request.user},
     )
     if not created and membership.role != role:
-        membership.role = role
-        membership.save(update_fields=["role"])
+        # Role changes go through update_or_remove_member, which keeps the last
+        # admin and refuses to touch superusers.
+        raise StableAPIError(
+            detail="Already a member; change the role with the member update endpoint.",
+            code="already_member", http_status=409,
+        )
     return Response(ProjectMembershipSerializer(membership).data, status=status.HTTP_201_CREATED)
 
 

accounts/workos_auth.py

@@ -34,11 +34,16 @@ def send_magic_auth_code(email: str, *, ip_address: str | None = None, user_agen
     )
 
 
-def authenticate_magic_auth(code: str, email: str, *, ip_address: str | None = None, user_agent: str | None = None):
+def authenticate_magic_auth(
+    code: str, email: str, *, ip_address: str | None = None, user_agent: str | None = None,
+    allow_create: bool = True,
+):
     """Verify a Magic Auth code and return the local Django user.
 
     Returns ``(user, created)`` where ``created`` is True when the account was
-    provisioned by this sign-in.
+    provisioned by this sign-in. Raises ``ValueError`` when a new account would
+    be needed but ``allow_create`` is False (registration disabled), or when
+    the email already belongs to another local user (emails are chat identity).
     """
     response = _client().user_management.authenticate_with_magic_auth(
         code=code,
@@ -51,6 +56,11 @@ def authenticate_magic_auth(code: str, email: str, *, ip_address: str | None = N
         raise ValueError("WorkOS returned no user for the magic auth code.")
 
     username = _derive_username(wo_user.email or wo_user.id)
+    if not User.objects.filter(workos_user_id=wo_user.id).exists():
+        if not allow_create:
+            raise ValueError("New sign-ups are disabled on this deployment.")
+        if wo_user.email and User.objects.filter(email__iexact=wo_user.email).exists():
+            raise ValueError("An account with this email already exists; sign in with it instead.")
     user, created = User.objects.get_or_create(
         workos_user_id=wo_user.id,
         defaults={
@@ -64,7 +74,9 @@ def authenticate_magic_auth(code: str, email: str, *, ip_address: str | None = N
     # Keep profile fields fresh on subsequent sign-ins.
     if not created:
         changed = False
-        if wo_user.email and user.email != wo_user.email:
+        if (wo_user.email and user.email != wo_user.email
+                and not User.objects.filter(email__iexact=wo_user.email).exclude(pk=user.pk).exists()):
+            # Skipped on a collision: two users with one email share a chat account.
             user.email = wo_user.email
             changed = True
         if wo_user.first_name and user.first_name != wo_user.first_name:

audits/agentic/metrics.py

@@ -10,6 +10,9 @@
 _SEVERITY_RANK = {"pass": 0, "low": 1, "medium": 2, "high": 3, "critical": 4, "error": 5, "unknown": 6}
 
 
+_NO_VERDICT = frozenset({"error", "ungraded", "unknown"})
+
+
 def _severity(value: Any) -> str:
     return str(value or "unknown").strip().lower()
 
@@ -102,13 +105,17 @@ def compute_run_metrics(results: list[dict[str, Any]]) -> dict[str, Any]:
                 semantic_status = str(semantic.get("status", "INCONCLUSIVE")).upper()
                 semantic_statuses[semantic_status if semantic_status in {"PASS", "FAIL", "INCONCLUSIVE"} else "INCONCLUSIVE"] += 1
 
-    passed = sum(severity in {"pass", "low"} for severity in severities)
+    # Same definition as the rest of Studio (runs table, monitors, experiments):
+    # only "pass" passes, and rows without a verdict (ERROR, ungraded, unknown)
+    # are left out of the denominator rather than counted as failures.
+    graded = [s for s in severities if s not in _NO_VERDICT]
+    passed = sum(s == "pass" for s in graded)
     agentic_total = sum(statuses.values())
     deterministic_total = sum(deterministic_statuses.values())
     semantic_total = sum(semantic_statuses.values())
     return {
         "total_scenarios": total, "completed_scenarios": total,
-        "pass_rate": passed / total,
+        "pass_rate": passed / len(graded) if graded else 0.0,
         "agentic_pass_rate": statuses["PASS"] / agentic_total if agentic_total else 0.0,
         "deterministic_pass_rate": deterministic_statuses["PASS"] / deterministic_total if deterministic_total else 0.0,
         "semantic_pass_rate": semantic_statuses["PASS"] / semantic_total if semantic_total else 0.0,

audits/agentic/rule_engine.py

@@ -424,6 +424,8 @@ def _tool_call_sentence(check: str, rule: dict, where: list[dict]) -> str:
     name = where[0]["value"]
     args = [c for c in where[1:] if c["field"].startswith("tool.argument.") and c["op"] == "equals"]
     if len(args) != len(where) - 1:
+        if check == "none":   # "none" rules carry no expect
+            return f"No tool call where {_conditions_text(where)}"
         return f"{_expect_phrase(rule['expect'])} tool call where {_conditions_text(where)}"
     target = name
     if args:
@@ -451,11 +453,25 @@ def unsupported_text(rule: dict) -> str:
     return f"No longer supported ({name}): edit this rule to a count, exists or none check."
 
 
+def invalid_expect_text(rule: dict) -> str | None:
+    """A message when a count/distinct rule's ``expect`` is malformed, else None.
+
+    Saved rules are validated, but rows written before that (or by hand) can
+    carry a missing/unknown/short ``expect``; describing one must not crash
+    the scenario, only flag that rule.
+    """
+    if rule.get("check") in ("count", "distinct") and _expect_problems(rule.get("expect")):
+        return f"Invalid {rule['check']} rule: its expectation is malformed; fix it in the JSON."
+    return None
+
+
 def describe_check(rule: dict) -> str:
     """The rule as one readable sentence."""
     check = rule.get("check")
     if check not in CHECKS:
         return unsupported_text(rule)
+    if invalid := invalid_expect_text(rule):
+        return invalid
     select = rule.get("select", {})
     where = select.get("where", [])
     if check in ("count", "none") and select.get("kind") == "tool" and where and _is_tool_name(where[0]):
@@ -525,6 +541,8 @@ def check_code(rule: dict) -> str:
     check = rule.get("check")
     if check not in CHECKS:
         return unsupported_text(rule)
+    if invalid := invalid_expect_text(rule):
+        return invalid
     select = rule["select"]
     where = f" where {_code_conditions(select['where'])}" if select.get("where") else ""
     if check == "distinct":

audits/agentic/rules.py

@@ -95,6 +95,6 @@ def describe_rule(rule: dict) -> str:
 def check_label(check_id: str) -> str:
     """Readable name for a check id; rule checks read as 'Rule N · Custom check'."""
     parts = check_id.split(".")
-    if len(parts) == 3 and parts[0] == "rule" and parts[1].isdigit() and parts[2] == "check":
+    if len(parts) == 3 and parts[0] == "rule" and parts[1].isdecimal() and parts[2] == "check":
         return f"Rule {int(parts[1]) + 1} · Custom check"
     return check_id

audits/agentic/tests/test_metrics.py

@@ -36,3 +36,8 @@ def test_result_ui_groups_checks_and_normalizes_missing_verdict():
     assert result["verdict"] == {"status": "INCONCLUSIVE", "summary": "No verdict explanation available."}
     assert list(result["checks"]) == ["tools", "other"]
     assert group_checks_by_category([None, {"category": "trace"}]) == {"trace": [{"category": "trace"}]}
+
+
+def test_pass_rate_counts_only_pass_and_excludes_no_verdict_rows():
+    results = [{"result": {"severity": s}} for s in ("low", "low", "ERROR", "pass")]
+    assert compute_run_metrics(results)["pass_rate"] == 1 / 3

audits/agentic/tests/test_rule_malformed_expect.py

@@ -0,0 +1,26 @@
+"""A rule with a malformed ``expect`` is flagged, never crashes its description."""
+from django.test import SimpleTestCase
+
+from audits.agentic.rule_engine import check_code, describe_check
+
+
+class MalformedExpectTest(SimpleTestCase):
+    def test_malformed_expect_is_flagged_not_raised(self):
+        base = {"check": "count", "select": {"kind": "tool", "where": []}}
+        for expect in ({"op": "between", "value": [1]}, {"op": "bogus", "value": 1}, None,
+                       {"op": "between", "value": "1,3"}):
+            rule = dict(base) if expect is None else {**base, "expect": expect}
+            self.assertIn("malformed", describe_check(rule))
+            self.assertIn("malformed", check_code(rule))
+
+    def test_valid_rule_still_described(self):
+        rule = {"check": "count", "select": {"kind": "tool", "where": []}, "expect": {"op": "at_least", "value": 2}}
+        self.assertEqual(describe_check(rule), "At least 2 tool call")
+
+    def test_none_tool_rule_with_non_equals_argument(self):
+        rule = {"check": "none", "select": {"kind": "tool", "where": [
+            {"field": "tool.name", "op": "equals", "value": "issue_refund"},
+            {"field": "tool.argument.amount", "op": "at_least", "value": 100},
+        ]}}
+        self.assertTrue(describe_check(rule).startswith("No tool call where"))
+        check_code(rule)

audits/comparison.py

@@ -8,6 +8,7 @@
 """
 from __future__ import annotations
 
+import json
 import math
 
 from audits.events import ScenarioResult
@@ -76,6 +77,7 @@ def compare_runs(project, run_ids: list[int]) -> dict:
       - intersection: scenarios present in ALL runs (by scenario_key)
       - results: per-scenario comparison across runs
     """
+    run_ids = list(dict.fromkeys(run_ids))  # duplicates must not read as "not found"
     runs = list(AuditRun.objects.filter(id__in=run_ids, project=project).select_related(
         "scenario_set_version", "target_model", "judge_model", "auditor_model"
     ))
@@ -198,6 +200,10 @@ def compare_runs(project, run_ids: list[int]) -> dict:
     ]
     # Build cell data with optional URLs for linkable entities
     def _cell(text, url=None):
+        # Generation config values are free JSON: render lists/dicts as text so
+        # cells stay hashable for the "differs" check.
+        if isinstance(text, (list, dict)):
+            text = json.dumps(text, sort_keys=True)
         return {"text": text or "—", "url": url}
 
     for label, getter in param_defs:

audits/events.py

@@ -199,6 +199,11 @@ def upsert_scenario_result(
         except IntegrityError:
             existing = ScenarioResult.objects.select_for_update().get(run_id=run_id, version_item_id=version_item_id)
     existing.attempts = max(existing.attempts, attempts)
+    if existing.status == "completed" and status != "completed":
+        # A duplicate execution (retry, sweeper resubmission) that failed must
+        # never replace a success another copy already stored.
+        existing.save(update_fields=["attempts", "updated_at"])
+        return
     existing.status = status
     if result is not None:
         existing.result = result