diff --git a/.github/workflows/science.yml b/.github/workflows/science.yml index bc1a8f8e..4a356ea2 100644 --- a/.github/workflows/science.yml +++ b/.github/workflows/science.yml @@ -54,7 +54,7 @@ permissions: env: AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} - AZURE_API_VERSION: "2024-12-01-preview" + AZURE_API_VERSION: "2025-03-01-preview" # Long-context judge required — gpt-5.4-nano/mini fail mechanically on # realistic agent transcripts (>10 turns) by not returning the required # verdict schema. See user note 2026-05-09. diff --git a/CHANGELOG.md b/CHANGELOG.md index a3480000..0fab0f70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Clean-install checks for every documented ASSERT/example environment, including `pip check` and credential-free target imports. +### Changed + +### Fixed + +## [0.3.0] - 2026-09-04 + +Adds evidence-backed eval authoring, stronger host-authoritative sandbox mediation, a broader atomic behavior library, and cleaner optional dependency ownership. This release also changes the default multi-turn budget and includes security, compatibility, and viewer fixes. + +### Added + +- Evidence-backed eval config generation for the `run-assert-eval` skill, including primary-source research, citation review, human approval before write, and one-risk-per-suite output (#337). +- Atomic behavior presets with broader built-in risk coverage and CI checks that enforce atomicity and parity with the example behavior specs (#293). +- Clean-install checks for every documented ASSERT/example environment, including `pip check` and credential-free target imports (#336). +- Real Docker containment tests that run on every pull request and push to `main` (#328). ### Changed -- **Breaking, planned for 0.3.0:** optional dependencies now describe ASSERT capabilities rather than repository examples. `otel` is replaced by `phoenix`, `azure-aad` by `azure-auth`, and `embeddings` remains the offline/local embedding backend. The `langgraph`, `dspy`, `examples`, and `regression` extras are removed; install each agent framework from the target project's dependency manifest or the repository example's adjacent `requirements.txt`. `all` now means all optional ASSERT product capabilities: `phoenix`, `analysis`, `embeddings`, `azure-auth`, and `acs`. +- **Breaking:** optional dependencies now describe ASSERT capabilities rather than repository examples. `otel` is replaced by `phoenix`, `azure-aad` by `azure-auth`, and `embeddings` remains the offline/local embedding backend. The `langgraph`, `dspy`, `examples`, and `regression` extras are removed; install each agent framework from the target project's dependency manifest or the repository example's adjacent `requirements.txt`. `all` now means all optional ASSERT product capabilities: `phoenix`, `analysis`, `embeddings`, `azure-auth`, and `acs` (#336). - OpenInference instrumentors are owned by the runtime they instrument. ASSERT installs its LiteLLM instrumentor directly for Prompt Agent tracing; LangChain, OpenAI, and other framework instrumentors live with their target/example dependencies. - Analysis keeps NumPy and the OpenAI embedding backend, while Phoenix owns the Pandas dependency used to convert its DataFrame spans. @@ -27,14 +40,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 | `assert-ai[examples]` | Each example's adjacent `requirements.txt` | | `assert-ai[regression]` | `uv sync --group dev` from a repository checkout; SciPy is no longer required | -- **Breaking, planned for 0.3.0:** `pipeline.inference.max_turns` now resolves to `6` instead of `10` when omitted. Scenario cases that relied on the implicit budget run four fewer turns, which can lower observed violation rates for multi-turn erosion harms, so results from before and after this change are not directly comparable. Set `max_turns: 10` explicitly to keep the previous behavior. Single-turn `prompt` cases do not use this setting and are unaffected. Because the resolved value feeds the inference resume fingerprint, a run interrupted before the upgrade re-executes rather than resumes. +- **Breaking:** `pipeline.inference.max_turns` now resolves to `6` instead of `10` when omitted. Scenario cases that relied on the implicit budget run four fewer turns, which can lower observed violation rates for multi-turn erosion harms, so results from before and after this change are not directly comparable. Set `max_turns: 10` explicitly to keep the previous behavior. Single-turn `prompt` cases do not use this setting and are unaffected. Because the resolved value feeds the inference resume fingerprint, a run interrupted before the upgrade re-executes rather than resumes (#337). ### Fixed -- `assert-ai --version` now reads the installed distribution metadata instead of reporting a hard-coded stale version. -- Declare `aiohttp`, which is directly used by the HTTP endpoint target, instead of relying on LiteLLM to install it transitively. -- Keep Bank Manager's GPT and non-GPT Azure routes compatible with ASSERT's OpenAI dependency range, and verify its documented installation with `pip check`. -- Traced callable targets that capture no spans now record an actionable invalid-trace warning instead of silently reporting valid trace metadata. +- Sandbox policy evaluation, mock resolution, and action evidence can now run in an authenticated host service instead of trusting the evaluated target to report its own decisions (#329). +- Sandbox mediation now propagates and validates one case ID across policy selection, mocks, execution, and evidence (#327). +- Proxy-generated network evidence is stored in a host-only directory outside the evaluated target's writable mount (#326). +- `assert-ai --version` now reads the installed distribution metadata instead of reporting a hard-coded stale version (#334). +- Declare `aiohttp`, which is directly used by the HTTP endpoint target, instead of relying on LiteLLM to install it transitively (#336). +- Update LiteLLM to patched version 1.84.1 and cap it below 2.0 (#330). +- Update vulnerable website, viewer, and Python dependencies to patched releases (#335, #336). +- Keep Bank Manager's GPT and non-GPT Azure routes compatible with ASSERT's OpenAI dependency range, and verify its documented installation with `pip check` (#336). +- Traced callable targets that capture no spans now record an actionable invalid-trace warning instead of silently reporting valid trace metadata (#337). +- Keep viewer result tables within their containers, wrap or truncate long labels, and position metric dropdowns so overflow cannot clip their options (#342). ## [0.2.0] - 2026-08-14 @@ -82,6 +101,7 @@ front-doors. - Docs site: rewrite repo-relative `.md` links so cross-references resolve on `responsibleai.github.io` (#225); repair broken links and remove missing logo asset (#227); remove maintainer-agent docs from the public site (#251). - Docs / packaging polish: absolute README links for PyPI (#214), static MIT badge (#213), redeploy site when `docs/` or `assets/` change (#212), local viewer guide typos (#208, #210). -[Unreleased]: https://github.com/responsibleai/ASSERT/compare/v0.2.0...HEAD +[Unreleased]: https://github.com/responsibleai/ASSERT/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/responsibleai/ASSERT/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/responsibleai/ASSERT/compare/v0.1.0...v0.2.0 [0.1.0]: https://github.com/responsibleai/ASSERT/releases/tag/v0.1.0 diff --git a/pyproject.toml b/pyproject.toml index 4b112170..11f011d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "assert-ai" -version = "0.2.0" +version = "0.3.0" description = "YAML-driven safety evaluation pipeline with LiteLLM-backed stages" readme = "README.md" requires-python = ">=3.11" diff --git a/uv.lock b/uv.lock index 4be9c736..12ea4c1d 100644 --- a/uv.lock +++ b/uv.lock @@ -393,7 +393,7 @@ wheels = [ [[package]] name = "assert-ai" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "aiohttp" },