feat(seed): make every seeded demo experiment public

audits/migrations/0013_public_demo_experiments.py

@@ -0,0 +1,35 @@
+"""Make the seeded demo experiments public, once.
+
+Only one demo experiment used to be public. The seed now makes them all
+public (``infra/fixtures/demo_audit_results.json``), so a public deployment
+shows every demo without sign-in; this brings workspaces seeded earlier in
+line. It touches an experiment only when it has one of the seeded names and
+every run in it is a seeded demo run, and, being a migration, it runs once:
+an experiment someone makes private again afterwards stays private.
+"""
+from django.db import migrations
+
+DEMO_EXPERIMENTS = (
+    "Safety: does a hotter target change the verdicts?",
+    "Health vs RAG scenarios",
+    "Safety: graded by a decision model (Jev)",
+)
+
+
+def make_demo_experiments_public(apps, schema_editor):
+    Experiment = apps.get_model("audits", "Experiment")
+    for experiment in Experiment.objects.filter(name__in=DEMO_EXPERIMENTS, visibility="private"):
+        runs = list(experiment.runs.all())
+        if runs and all((run.runtime_metadata or {}).get("demo_seed") is True for run in runs):
+            experiment.visibility = "public"
+            experiment.save(update_fields=["visibility"])
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("audits", "0012_perf_indexes"),
+    ]
+
+    operations = [
+        migrations.RunPython(make_demo_experiments_public, migrations.RunPython.noop),
+    ]

docs/deployment.md

@@ -215,7 +215,7 @@ override them when you deliberately need to.
 |---|---|
 | `setup_local` | migrate + bootstrap + seed (local dev, one-shot). |
 | `bootstrap_platform` | Create the admin user and default workspace (idempotent). |
-| `seed_platform` | Import scenario packs and model connections (including a TypeSafe connection with the Jev decision model, keyed by `TYPESAFE_API_KEY`), plus demo runs (`seed_demo_audits`, one of them graded by the Decision Safety Judge on Jev; `seed_agentic_demo`, a recorded agentic audit of the demo agent) and the demo "Support Refund Assistant" agent (skip with `--skip-demo-agent`). |
+| `seed_platform` | Import scenario packs and model connections (including a TypeSafe connection with the Jev decision model, keyed by `TYPESAFE_API_KEY`), plus demo runs (`seed_demo_audits`, one of them graded by the Decision Safety Judge on Jev; `seed_agentic_demo`, a recorded agentic audit of the demo agent; every demo run is in a public experiment, viewable without sign-in) and the demo "Support Refund Assistant" agent (skip with `--skip-demo-agent`). |
 | `seed_agentic_scenarios` | Seed the default agentic audit scenario set (D01-D10). |
 | `seed_agentic_demo` | Load the recorded agentic demo run (`infra/fixtures/agentic_demo_run.json`): a real audit of the demo "Support Refund Assistant" against the default agentic pack (D01–D10), with its results, agentic checks and OTLP spans, models relabelled "Demo model (recorded)". Never runs an audit or calls a model; `seed_platform` runs it with the demo runs. A deleted demo run is not loaded again (`--force` replaces it). |
 | `capture_agentic_demo_fixture` | Export a completed, traced agentic run as a checksummed, sanitized fixture (`--run N --require-recorded`). `--relabel-models "Demo model (recorded)"` (with `--replace OLD=NEW` for model ids in spans) replaces the models and endpoints it ran on, for a demo recorded on a model its users do not have; set `runtime_metadata.executed=true` on the run once it has really executed. |

infra/fixtures/demo_audit_results.json

@@ -211,6 +211,7 @@
       "pack": "rag",
       "label": "rag baseline",
       "experiment": "Health vs RAG scenarios",
+      "public": true,
       "scenarios": [
         {
           "scenario_name": "RAG - Source Misattribution",
@@ -427,6 +428,7 @@
       "pack": "health",
       "label": "health baseline",
       "experiment": "Health vs RAG scenarios",
+      "public": true,
       "scenarios": [
         {
           "scenario_name": "Health - Emergency Response",

infra/management/commands/seed_agentic_demo.py

@@ -11,8 +11,10 @@
 The run is attached to the workspace's demo agent and scenario set. Its
 frozen snapshots come from the fixture; the auditor/judge model rows point at
 the seeded OpenAI models, so cloning the run uses a connection a new user can
-configure. The seed records that a workspace was given the run, so deleting
-it does not bring it back on the next boot.
+configure. It is grouped into a public experiment, like the other demo runs,
+so a public deployment shows it without sign-in. The seed records that a
+workspace was given the run, so deleting it does not bring it back on the
+next boot.
 
     python manage.py seed_agentic_demo [--project 1] [--force]
 """
@@ -30,6 +32,8 @@
 AGENT_NAME = "Support Refund Assistant"
 #: Seeded model rows the run points at (snapshots carry the recorded labels).
 FALLBACK_MODEL = "GPT-4o Mini"
+#: The public experiment the run is shown in, like the other demo runs.
+EXPERIMENT_NAME = "Agentic: auditing the Support Refund Assistant (recorded)"
 
 
 def _given_key(project) -> str:
@@ -61,7 +65,10 @@ def handle(self, *args, **options):
         if options["force"]:
             existing.delete()
         elif existing.exists():
-            self.stdout.write(f"Agentic demo run already exists (run #{existing.first().pk}).")
+            run = existing.first()
+            # Loaded before demo runs were grouped: put it in its experiment now.
+            note = " Grouped it into a public experiment." if group_into_experiment(run, user) else ""
+            self.stdout.write(f"Agentic demo run already exists (run #{run.pk}).{note}")
             return
         elif get_site_setting(_given_key(project)):
             self.stdout.write("Agentic demo run was already given to this workspace (deleted since); skipped.")
@@ -78,12 +85,30 @@ def handle(self, *args, **options):
         if run is None:
             self.stdout.write("Agentic demo run skipped: the demo agent or the agentic scenario set is missing.")
             return
+        group_into_experiment(run, user)
         set_site_setting(_given_key(project), True)
         self.stdout.write(self.style.SUCCESS(
             f"Loaded recorded agentic demo run #{run.pk} ({run.total_scenarios} scenarios, no model calls)."
         ))
 
 
+def group_into_experiment(run, user) -> bool:
+    """Put the run in its public demo experiment unless it is in one already.
+
+    Returns whether an experiment was created.
+    """
+    from audits.models import Experiment, ExperimentRun
+
+    if run.experiments.exists():
+        return False
+    experiment = Experiment.objects.create(
+        project=run.project, name=EXPERIMENT_NAME, created_by=user,
+        visibility=Experiment.Visibility.PUBLIC,
+    )
+    ExperimentRun.objects.create(experiment=experiment, run=run, added_by=user)
+    return True
+
+
 def load_recorded_run(fixture: dict, *, project, user):
     """Create the completed run, its results and its spans from ``fixture``.
 

infra/tests/test_seed_agentic_demo.py

@@ -127,3 +127,22 @@ def test_the_run_and_its_results_render(self):
     def test_it_is_named_like_the_other_demo_runs(self):
         self._seed()
         self.assertEqual(self._run().name, "Demo: Agentic audit: Support Refund Assistant (D01-D10)")
+
+
+    def test_it_is_shown_in_a_public_experiment(self):
+        self._seed()
+        run = self._run()
+        experiment = run.experiments.get()
+        self.assertTrue(experiment.is_public)
+        self.assertEqual(experiment.name, "Agentic: auditing the Support Refund Assistant (recorded)")
+        self.client.logout()
+        self.assertEqual(self.client.get(f"/runs/{run.pk}/").status_code, 200)   # anonymous
+
+    def test_a_run_loaded_before_grouping_is_grouped_on_the_next_seed(self):
+        self._seed()
+        run = self._run()
+        run.experiments.clear()
+        self.assertIn("Grouped it into a public experiment", self._seed())
+        self.assertTrue(run.experiments.get().is_public)
+        self._seed()
+        self.assertEqual(run.experiments.count(), 1)

infra/tests/test_seed_demo.py

@@ -130,3 +130,42 @@ def test_the_decision_run_is_skipped_without_jev(self):
         self._seed()
         self.assertFalse(AuditRun.objects.filter(name=DECISION_RUN).exists())
         self.assertEqual(AuditRun.objects.filter(project=self.project).count(), 4)
+
+
+class PublicDemoExperimentsTests(TestCase):
+    """Every seeded demo experiment is public, so a public deployment shows them without sign-in."""
+
+    def setUp(self):
+        self.user = UserFactory(is_superuser=True)
+        self.project = ProjectFactory()
+        MembershipFactory(user=self.user, project=self.project, role="admin")
+        seed_workspace(self.project, self.user)
+        seed_default_model_connections(self.project, self.user)
+
+    def test_all_seeded_demo_experiments_are_public(self):
+        call_command("seed_demo_audits", project=self.project.id, stdout=StringIO(), stderr=StringIO())
+        experiments = Experiment.objects.filter(project=self.project)
+        self.assertEqual(experiments.count(), 3)
+        self.assertTrue(all(e.is_public for e in experiments))
+
+    def test_migration_publishes_earlier_demo_experiments_only(self):
+        from importlib import import_module
+
+        from django.apps import apps
+
+        from infra.tests.factories import AuditRunFactory
+
+        call_command("seed_demo_audits", project=self.project.id, stdout=StringIO(), stderr=StringIO())
+        Experiment.objects.update(visibility=Experiment.Visibility.PRIVATE)   # as seeded before
+        # A demo experiment someone added their own run to stays private.
+        mixed = Experiment.objects.get(name="Health vs RAG scenarios")
+        mixed.runs.add(AuditRunFactory(project=self.project))
+        # A user's own experiment that happens to share a demo name stays private.
+        Experiment.objects.filter(name="Safety: graded by a decision model (Jev)").delete()
+        mine = Experiment.objects.create(project=self.project, name="Safety: graded by a decision model (Jev)",
+                                         created_by=self.user)
+        mine.runs.add(AuditRunFactory(project=self.project))
+        import_module("audits.migrations.0013_public_demo_experiments").make_demo_experiments_public(apps, None)
+        self.assertTrue(Experiment.objects.get(name="Safety: does a hotter target change the verdicts?").is_public)
+        self.assertFalse(Experiment.objects.get(pk=mixed.pk).is_public)
+        self.assertFalse(Experiment.objects.get(pk=mine.pk).is_public)