From 827b29bfb161812c3030073742cc37ae06cc9e8e Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Thu, 12 Nov 2026 13:39:27 +0100 Subject: [PATCH] feat(seed): make every seeded demo experiment public Only the hotter-target safety experiment was public, so a public deployment (the HF Space) showed one demo without sign-in and hid the rest. - demo_audit_results.json: "Health vs RAG scenarios" is public too (the decision-model experiment already was). - seed_agentic_demo: the recorded agentic run goes into its own public experiment, "Agentic: auditing the Support Refund Assistant (recorded)"; a run loaded before this is grouped on the next seed. - audits 0013: a one-time data migration makes the seeded demo experiments public in workspaces seeded earlier. It only touches an experiment with a seeded name whose runs are all seeded demo runs; an experiment someone makes private afterwards stays private. --- .../0013_public_demo_experiments.py | 35 +++++++++++++++++ docs/deployment.md | 2 +- infra/fixtures/demo_audit_results.json | 2 + .../management/commands/seed_agentic_demo.py | 31 +++++++++++++-- infra/tests/test_seed_agentic_demo.py | 19 +++++++++ infra/tests/test_seed_demo.py | 39 +++++++++++++++++++ 6 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 audits/migrations/0013_public_demo_experiments.py diff --git a/audits/migrations/0013_public_demo_experiments.py b/audits/migrations/0013_public_demo_experiments.py new file mode 100644 index 00000000..2aa98f56 --- /dev/null +++ b/audits/migrations/0013_public_demo_experiments.py @@ -0,0 +1,35 @@ +"""Make the seeded demo experiments public, once. + +Only one demo experiment used to be public. The seed now makes them all +public (``infra/fixtures/demo_audit_results.json``), so a public deployment +shows every demo without sign-in; this brings workspaces seeded earlier in +line. It touches an experiment only when it has one of the seeded names and +every run in it is a seeded demo run, and, being a migration, it runs once: +an experiment someone makes private again afterwards stays private. +""" +from django.db import migrations + +DEMO_EXPERIMENTS = ( + "Safety: does a hotter target change the verdicts?", + "Health vs RAG scenarios", + "Safety: graded by a decision model (Jev)", +) + + +def make_demo_experiments_public(apps, schema_editor): + Experiment = apps.get_model("audits", "Experiment") + for experiment in Experiment.objects.filter(name__in=DEMO_EXPERIMENTS, visibility="private"): + runs = list(experiment.runs.all()) + if runs and all((run.runtime_metadata or {}).get("demo_seed") is True for run in runs): + experiment.visibility = "public" + experiment.save(update_fields=["visibility"]) + + +class Migration(migrations.Migration): + dependencies = [ + ("audits", "0012_perf_indexes"), + ] + + operations = [ + migrations.RunPython(make_demo_experiments_public, migrations.RunPython.noop), + ] diff --git a/docs/deployment.md b/docs/deployment.md index 89357677..d00ec4e7 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -215,7 +215,7 @@ override them when you deliberately need to. |---|---| | `setup_local` | migrate + bootstrap + seed (local dev, one-shot). | | `bootstrap_platform` | Create the admin user and default workspace (idempotent). | -| `seed_platform` | Import scenario packs and model connections (including a TypeSafe connection with the Jev decision model, keyed by `TYPESAFE_API_KEY`), plus demo runs (`seed_demo_audits`, one of them graded by the Decision Safety Judge on Jev; `seed_agentic_demo`, a recorded agentic audit of the demo agent) and the demo "Support Refund Assistant" agent (skip with `--skip-demo-agent`). | +| `seed_platform` | Import scenario packs and model connections (including a TypeSafe connection with the Jev decision model, keyed by `TYPESAFE_API_KEY`), plus demo runs (`seed_demo_audits`, one of them graded by the Decision Safety Judge on Jev; `seed_agentic_demo`, a recorded agentic audit of the demo agent; every demo run is in a public experiment, viewable without sign-in) and the demo "Support Refund Assistant" agent (skip with `--skip-demo-agent`). | | `seed_agentic_scenarios` | Seed the default agentic audit scenario set (D01-D10). | | `seed_agentic_demo` | Load the recorded agentic demo run (`infra/fixtures/agentic_demo_run.json`): a real audit of the demo "Support Refund Assistant" against the default agentic pack (D01–D10), with its results, agentic checks and OTLP spans, models relabelled "Demo model (recorded)". Never runs an audit or calls a model; `seed_platform` runs it with the demo runs. A deleted demo run is not loaded again (`--force` replaces it). | | `capture_agentic_demo_fixture` | Export a completed, traced agentic run as a checksummed, sanitized fixture (`--run N --require-recorded`). `--relabel-models "Demo model (recorded)"` (with `--replace OLD=NEW` for model ids in spans) replaces the models and endpoints it ran on, for a demo recorded on a model its users do not have; set `runtime_metadata.executed=true` on the run once it has really executed. | diff --git a/infra/fixtures/demo_audit_results.json b/infra/fixtures/demo_audit_results.json index 8e93c970..a62c00c5 100644 --- a/infra/fixtures/demo_audit_results.json +++ b/infra/fixtures/demo_audit_results.json @@ -211,6 +211,7 @@ "pack": "rag", "label": "rag baseline", "experiment": "Health vs RAG scenarios", + "public": true, "scenarios": [ { "scenario_name": "RAG - Source Misattribution", @@ -427,6 +428,7 @@ "pack": "health", "label": "health baseline", "experiment": "Health vs RAG scenarios", + "public": true, "scenarios": [ { "scenario_name": "Health - Emergency Response", diff --git a/infra/management/commands/seed_agentic_demo.py b/infra/management/commands/seed_agentic_demo.py index b4696bdc..21d2ab90 100644 --- a/infra/management/commands/seed_agentic_demo.py +++ b/infra/management/commands/seed_agentic_demo.py @@ -11,8 +11,10 @@ The run is attached to the workspace's demo agent and scenario set. Its frozen snapshots come from the fixture; the auditor/judge model rows point at the seeded OpenAI models, so cloning the run uses a connection a new user can -configure. The seed records that a workspace was given the run, so deleting -it does not bring it back on the next boot. +configure. It is grouped into a public experiment, like the other demo runs, +so a public deployment shows it without sign-in. The seed records that a +workspace was given the run, so deleting it does not bring it back on the +next boot. python manage.py seed_agentic_demo [--project 1] [--force] """ @@ -30,6 +32,8 @@ AGENT_NAME = "Support Refund Assistant" #: Seeded model rows the run points at (snapshots carry the recorded labels). FALLBACK_MODEL = "GPT-4o Mini" +#: The public experiment the run is shown in, like the other demo runs. +EXPERIMENT_NAME = "Agentic: auditing the Support Refund Assistant (recorded)" def _given_key(project) -> str: @@ -61,7 +65,10 @@ def handle(self, *args, **options): if options["force"]: existing.delete() elif existing.exists(): - self.stdout.write(f"Agentic demo run already exists (run #{existing.first().pk}).") + run = existing.first() + # Loaded before demo runs were grouped: put it in its experiment now. + note = " Grouped it into a public experiment." if group_into_experiment(run, user) else "" + self.stdout.write(f"Agentic demo run already exists (run #{run.pk}).{note}") return elif get_site_setting(_given_key(project)): self.stdout.write("Agentic demo run was already given to this workspace (deleted since); skipped.") @@ -78,12 +85,30 @@ def handle(self, *args, **options): if run is None: self.stdout.write("Agentic demo run skipped: the demo agent or the agentic scenario set is missing.") return + group_into_experiment(run, user) set_site_setting(_given_key(project), True) self.stdout.write(self.style.SUCCESS( f"Loaded recorded agentic demo run #{run.pk} ({run.total_scenarios} scenarios, no model calls)." )) +def group_into_experiment(run, user) -> bool: + """Put the run in its public demo experiment unless it is in one already. + + Returns whether an experiment was created. + """ + from audits.models import Experiment, ExperimentRun + + if run.experiments.exists(): + return False + experiment = Experiment.objects.create( + project=run.project, name=EXPERIMENT_NAME, created_by=user, + visibility=Experiment.Visibility.PUBLIC, + ) + ExperimentRun.objects.create(experiment=experiment, run=run, added_by=user) + return True + + def load_recorded_run(fixture: dict, *, project, user): """Create the completed run, its results and its spans from ``fixture``. diff --git a/infra/tests/test_seed_agentic_demo.py b/infra/tests/test_seed_agentic_demo.py index 3f1e3f6f..0de1df99 100644 --- a/infra/tests/test_seed_agentic_demo.py +++ b/infra/tests/test_seed_agentic_demo.py @@ -127,3 +127,22 @@ def test_the_run_and_its_results_render(self): def test_it_is_named_like_the_other_demo_runs(self): self._seed() self.assertEqual(self._run().name, "Demo: Agentic audit: Support Refund Assistant (D01-D10)") + + + def test_it_is_shown_in_a_public_experiment(self): + self._seed() + run = self._run() + experiment = run.experiments.get() + self.assertTrue(experiment.is_public) + self.assertEqual(experiment.name, "Agentic: auditing the Support Refund Assistant (recorded)") + self.client.logout() + self.assertEqual(self.client.get(f"/runs/{run.pk}/").status_code, 200) # anonymous + + def test_a_run_loaded_before_grouping_is_grouped_on_the_next_seed(self): + self._seed() + run = self._run() + run.experiments.clear() + self.assertIn("Grouped it into a public experiment", self._seed()) + self.assertTrue(run.experiments.get().is_public) + self._seed() + self.assertEqual(run.experiments.count(), 1) diff --git a/infra/tests/test_seed_demo.py b/infra/tests/test_seed_demo.py index 7b6c7b1e..be80dc51 100644 --- a/infra/tests/test_seed_demo.py +++ b/infra/tests/test_seed_demo.py @@ -130,3 +130,42 @@ def test_the_decision_run_is_skipped_without_jev(self): self._seed() self.assertFalse(AuditRun.objects.filter(name=DECISION_RUN).exists()) self.assertEqual(AuditRun.objects.filter(project=self.project).count(), 4) + + +class PublicDemoExperimentsTests(TestCase): + """Every seeded demo experiment is public, so a public deployment shows them without sign-in.""" + + def setUp(self): + self.user = UserFactory(is_superuser=True) + self.project = ProjectFactory() + MembershipFactory(user=self.user, project=self.project, role="admin") + seed_workspace(self.project, self.user) + seed_default_model_connections(self.project, self.user) + + def test_all_seeded_demo_experiments_are_public(self): + call_command("seed_demo_audits", project=self.project.id, stdout=StringIO(), stderr=StringIO()) + experiments = Experiment.objects.filter(project=self.project) + self.assertEqual(experiments.count(), 3) + self.assertTrue(all(e.is_public for e in experiments)) + + def test_migration_publishes_earlier_demo_experiments_only(self): + from importlib import import_module + + from django.apps import apps + + from infra.tests.factories import AuditRunFactory + + call_command("seed_demo_audits", project=self.project.id, stdout=StringIO(), stderr=StringIO()) + Experiment.objects.update(visibility=Experiment.Visibility.PRIVATE) # as seeded before + # A demo experiment someone added their own run to stays private. + mixed = Experiment.objects.get(name="Health vs RAG scenarios") + mixed.runs.add(AuditRunFactory(project=self.project)) + # A user's own experiment that happens to share a demo name stays private. + Experiment.objects.filter(name="Safety: graded by a decision model (Jev)").delete() + mine = Experiment.objects.create(project=self.project, name="Safety: graded by a decision model (Jev)", + created_by=self.user) + mine.runs.add(AuditRunFactory(project=self.project)) + import_module("audits.migrations.0013_public_demo_experiments").make_demo_experiments_public(apps, None) + self.assertTrue(Experiment.objects.get(name="Safety: does a hotter target change the verdicts?").is_public) + self.assertFalse(Experiment.objects.get(pk=mixed.pk).is_public) + self.assertFalse(Experiment.objects.get(pk=mine.pk).is_public)