From 9e6951ed1ece6bcf128b0811cde7dcd155cf0e38 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Mon, 12 Oct 2026 17:23:23 -0700 Subject: [PATCH 1/9] sign protein binder design workflow Signed-off-by: Ohad Mosafi --- .github/scripts/prune-orphans.sh | 4 +- .github/workflows/plugin-sync.yml | 1 + .github/workflows/skill-security.yml | 1 + CONTRIBUTING.md | 12 +- README.md | 2 +- catalog-exceptions.yml | 10 +- plugins.d/bionemo-agent-toolkit.yml | 1 + scripts/plugin_sync.py | 2 +- .../skills/protein-binder-design/BENCHMARK.md | 117 ++++ .../skills/protein-binder-design/README.md | 22 +- .../skills/protein-binder-design/SKILL.md | 247 ++++++- .../config/skillspector-baseline.yml | 86 +++ .../protein-binder-design/evals/README.md | 128 ++++ .../protein-binder-design/evals/config.yml | 12 + .../evals/deferred/live-campaigns.json | 42 ++ .../evals/environment/Dockerfile | 5 + .../protein-binder-design/evals/evals.json | 146 +++- .../evals/files/boltz2_only.json | 64 ++ .../evals/files/bookkeeping.json | 125 ++++ .../evals/files/cache_contract.md | 42 ++ .../evals/files/mixed_thresholds.json | 81 +++ .../evals/files/no_survivors.json | 62 ++ .../evals/files/provenance_cache/backbone.pdb | 4 + .../files/provenance_cache/manifest.json | 212 ++++++ .../responses/candidate_swap.json | 11 + .../responses/response_tampered.json | 1 + .../responses/sample_swap.json | 11 + .../responses/sequence_swap.json | 11 + .../responses/target_swap.json | 11 + .../provenance_cache/responses/valid.json | 11 + .../evals/files/provenance_cache/target.pdb | 7 + .../files/raw_refolds/boltz2/backbone.pdb | 4 + .../files/raw_refolds/boltz2/manifest.json | 150 +++++ .../raw_refolds/boltz2/responses/control.json | 11 + .../boltz2/responses/incomplete.json | 11 + .../boltz2/responses/poor_binder.json | 11 + .../boltz2/responses/selected.json | 16 + .../evals/files/raw_refolds/boltz2/target.pdb | 7 + .../files/raw_refolds/openfold3/backbone.pdb | 4 + .../files/raw_refolds/openfold3/manifest.json | 121 ++++ .../openfold3/responses/composite_only.json | 15 + .../openfold3/responses/poor_binder.json | 16 + .../openfold3/responses/selected.json | 23 + .../files/raw_refolds/openfold3/target.pdb | 7 + .../evals/files/resume_cache/backbone.pdb | 4 + .../evals/files/resume_cache/below.pdb | 10 + .../evals/files/resume_cache/candidates.csv | 3 + .../evals/files/resume_cache/done.pdb | 10 + .../evals/files/resume_cache/manifest.json | 206 ++++++ .../files/resume_cache/responses/below.json | 16 + .../files/resume_cache/responses/control.json | 16 + .../files/resume_cache/responses/done.json | 16 + .../resume_cache/responses/recoverable.json | 16 + .../evals/files/resume_cache/target.pdb | 7 + .../evals/files/target_prep/CONTRACT.md | 11 + .../evals/files/target_prep/request.json | 22 + .../evals/files/target_prep/target.pdb | 12 + .../protein-binder-design/evals/grader.py | 633 ++++++++++++++++++ .../references/local-nim-setup.md | 144 ++-- .../references/manifest.md | 78 ++- .../references/offline.md | 106 +++ .../references/pipeline.md | 112 +++- .../references/validation.md | 58 +- .../requirements-dev.txt | 3 + .../protein-binder-design/requirements.txt | 4 + .../protein-binder-design/scripts/controls.py | 40 +- .../scripts/hosted_env.sh | 26 + .../protein-binder-design/scripts/manifest.py | 77 ++- .../protein-binder-design/scripts/metrics.py | 57 +- .../scripts/offline_bookkeeping.py | 135 ++++ .../scripts/pdb_utils.py | 114 +++- .../scripts/refold_cache.py | 246 +++++++ .../protein-binder-design/scripts/registry.py | 10 +- .../protein-binder-design/skill-card.md | 99 +++ .../protein-binder-design/skill.oms.sig | 1 + .../tests/test_controls.py | 52 ++ .../tests/test_hosted_env.py | 73 ++ .../tests/test_manifest.py | 226 +++++++ .../tests/test_metrics.py | 61 ++ .../tests/test_offline.py | 474 +++++++++++++ .../protein-binder-design/LICENSE | 0 .../protein-binder-design/README.md | 22 +- .../protein-binder-design/SKILL.md | 305 +++++++++ .../protein-binder-design/assets/targets.json | 0 .../config/skillspector-baseline.yml | 86 +++ .../protein-binder-design/evals/README.md | 128 ++++ .../protein-binder-design/evals/config.yml | 12 + .../evals/deferred/live-campaigns.json | 42 ++ .../evals/environment/Dockerfile | 5 + .../protein-binder-design/evals/evals.json | 136 ++++ .../evals/files/boltz2_only.json | 64 ++ .../evals/files/bookkeeping.json | 125 ++++ .../evals/files/cache_contract.md | 42 ++ .../evals/files/mixed_thresholds.json | 81 +++ .../evals/files/no_survivors.json | 62 ++ .../evals/files/provenance_cache/backbone.pdb | 4 + .../files/provenance_cache/manifest.json | 212 ++++++ .../responses/candidate_swap.json | 11 + .../responses/response_tampered.json | 1 + .../responses/sample_swap.json | 11 + .../responses/sequence_swap.json | 11 + .../responses/target_swap.json | 11 + .../provenance_cache/responses/valid.json | 11 + .../evals/files/provenance_cache/target.pdb | 7 + .../files/raw_refolds/boltz2/backbone.pdb | 4 + .../files/raw_refolds/boltz2/manifest.json | 150 +++++ .../raw_refolds/boltz2/responses/control.json | 11 + .../boltz2/responses/incomplete.json | 11 + .../boltz2/responses/poor_binder.json | 11 + .../boltz2/responses/selected.json | 16 + .../evals/files/raw_refolds/boltz2/target.pdb | 7 + .../files/raw_refolds/openfold3/backbone.pdb | 4 + .../files/raw_refolds/openfold3/manifest.json | 121 ++++ .../openfold3/responses/composite_only.json | 15 + .../openfold3/responses/poor_binder.json | 16 + .../openfold3/responses/selected.json | 23 + .../files/raw_refolds/openfold3/target.pdb | 7 + .../evals/files/resume_cache/backbone.pdb | 4 + .../evals/files/resume_cache/below.pdb | 10 + .../evals/files/resume_cache/candidates.csv | 3 + .../evals/files/resume_cache/done.pdb | 10 + .../evals/files/resume_cache/manifest.json | 206 ++++++ .../files/resume_cache/responses/below.json | 16 + .../files/resume_cache/responses/control.json | 16 + .../files/resume_cache/responses/done.json | 16 + .../resume_cache/responses/recoverable.json | 16 + .../evals/files/resume_cache/target.pdb | 7 + .../evals/files/target_prep/CONTRACT.md | 11 + .../evals/files/target_prep/request.json | 22 + .../evals/files/target_prep/target.pdb | 12 + .../protein-binder-design/evals/grader.py | 633 ++++++++++++++++++ .../evals/trigger_evals.json | 0 .../references/local-nim-setup.md | 118 ++++ .../references/manifest.md | 137 ++++ .../references/offline.md | 106 +++ .../references/pipeline.md | 235 +++++++ .../references/validation.md | 113 ++++ .../requirements-dev.txt | 3 + .../protein-binder-design/requirements.txt | 4 + .../protein-binder-design/scripts/controls.py | 56 ++ .../scripts/hosted_env.sh | 26 + .../protein-binder-design/scripts/manifest.py | 77 ++- .../protein-binder-design/scripts/metrics.py | 84 +++ .../scripts/offline_bookkeeping.py | 135 ++++ .../scripts/pdb_utils.py | 167 +++++ .../scripts/refold_cache.py | 246 +++++++ .../protein-binder-design/scripts/registry.py | 10 +- .../tests/test_controls.py | 52 ++ .../tests/test_hosted_env.py | 73 ++ .../tests/test_manifest.py | 226 +++++++ .../tests/test_metrics.py | 61 ++ .../tests/test_offline.py | 474 +++++++++++++ .../README.md | 2 +- .../protein-binder-design/SKILL.md | 116 ---- .../protein-binder-design/evals/evals.json | 42 -- .../references/local-nim-setup.md | 80 --- .../references/manifest.md | 71 -- .../references/pipeline.md | 151 ----- .../references/validation.md | 69 -- .../protein-binder-design/scripts/controls.py | 26 - .../protein-binder-design/scripts/metrics.py | 39 -- .../scripts/pdb_utils.py | 91 --- .../workflow-rules.mdc | 2 +- 163 files changed, 10030 insertions(+), 894 deletions(-) create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/BENCHMARK.md create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/config/skillspector-baseline.yml create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/deferred/live-campaigns.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/environment/Dockerfile create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/boltz2_only.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/bookkeeping.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/cache_contract.md create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/mixed_thresholds.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/no_survivors.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/backbone.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/manifest.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/candidate_swap.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/response_tampered.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/sample_swap.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/sequence_swap.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/target_swap.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/valid.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/target.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/backbone.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/manifest.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/control.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/incomplete.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/poor_binder.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/selected.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/target.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/backbone.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/manifest.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/composite_only.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/poor_binder.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/selected.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/target.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/backbone.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/below.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/candidates.csv create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/done.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/manifest.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/below.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/control.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/done.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/recoverable.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/target.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/CONTRACT.md create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/request.json create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/target.pdb create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/grader.py create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/references/offline.md create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements-dev.txt create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/hosted_env.sh create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/offline_bookkeeping.py create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/refold_cache.py create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/skill-card.md create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/skill.oms.sig create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_controls.py create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_hosted_env.py create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_manifest.py create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_metrics.py create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_offline.py rename {workflows => workflow-skills}/generative-protein-binder-design/protein-binder-design/LICENSE (100%) rename {workflows => workflow-skills}/generative-protein-binder-design/protein-binder-design/README.md (64%) create mode 100755 workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md rename {workflows => workflow-skills}/generative-protein-binder-design/protein-binder-design/assets/targets.json (100%) create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/config/skillspector-baseline.yml create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/deferred/live-campaigns.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/environment/Dockerfile create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/evals.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/boltz2_only.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/bookkeeping.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/cache_contract.md create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/mixed_thresholds.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/no_survivors.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/backbone.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/manifest.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/candidate_swap.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/response_tampered.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/sample_swap.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/sequence_swap.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/target_swap.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/valid.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/target.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/backbone.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/manifest.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/control.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/incomplete.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/poor_binder.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/selected.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/target.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/backbone.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/manifest.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/composite_only.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/poor_binder.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/selected.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/target.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/backbone.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/below.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/candidates.csv create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/done.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/manifest.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/below.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/control.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/done.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/recoverable.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/target.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/CONTRACT.md create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/request.json create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/target.pdb create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/grader.py rename {workflows => workflow-skills}/generative-protein-binder-design/protein-binder-design/evals/trigger_evals.json (100%) create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/references/local-nim-setup.md create mode 100755 workflow-skills/generative-protein-binder-design/protein-binder-design/references/manifest.md create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/references/offline.md create mode 100755 workflow-skills/generative-protein-binder-design/protein-binder-design/references/pipeline.md create mode 100755 workflow-skills/generative-protein-binder-design/protein-binder-design/references/validation.md create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/requirements-dev.txt create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/controls.py create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/hosted_env.sh rename {workflows => workflow-skills}/generative-protein-binder-design/protein-binder-design/scripts/manifest.py (64%) create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/metrics.py create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/offline_bookkeeping.py create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/pdb_utils.py create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/refold_cache.py rename {workflows => workflow-skills}/generative-protein-binder-design/protein-binder-design/scripts/registry.py (61%) create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_controls.py create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_hosted_env.py create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_manifest.py create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_metrics.py create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_offline.py delete mode 100755 workflows/generative-protein-binder-design/protein-binder-design/SKILL.md delete mode 100644 workflows/generative-protein-binder-design/protein-binder-design/evals/evals.json delete mode 100644 workflows/generative-protein-binder-design/protein-binder-design/references/local-nim-setup.md delete mode 100755 workflows/generative-protein-binder-design/protein-binder-design/references/manifest.md delete mode 100755 workflows/generative-protein-binder-design/protein-binder-design/references/pipeline.md delete mode 100755 workflows/generative-protein-binder-design/protein-binder-design/references/validation.md delete mode 100644 workflows/generative-protein-binder-design/protein-binder-design/scripts/controls.py delete mode 100644 workflows/generative-protein-binder-design/protein-binder-design/scripts/metrics.py delete mode 100644 workflows/generative-protein-binder-design/protein-binder-design/scripts/pdb_utils.py diff --git a/.github/scripts/prune-orphans.sh b/.github/scripts/prune-orphans.sh index 0ed6f5a4..4c492de9 100755 --- a/.github/scripts/prune-orphans.sh +++ b/.github/scripts/prune-orphans.sh @@ -5,8 +5,8 @@ # FORK of NVIDIA/skills' prune-orphans.sh, adapted for this repo's GROUPED, # multi-root layout. Differences from upstream: # - Upstream scans a single flat `skills/*/` root. We scan the SOURCED group -# roots in PRUNE_ROOTS (the wholly-native roots nim-skills/ and workflows/ -# are never scanned, so they can't be pruned). +# roots in PRUNE_ROOTS (the wholly-native roots nim-skills/, workflows/, +# and workflow-skills/ are never scanned, so they can't be pruned). # - catalog_dir values are full grouped paths (e.g. library-skills/nvmolkit-usage), # kept at DEPTH 1 under their group root, so the scan and the expected-set # comparison both use full relative paths. diff --git a/.github/workflows/plugin-sync.yml b/.github/workflows/plugin-sync.yml index a606d03b..0af96dc3 100644 --- a/.github/workflows/plugin-sync.yml +++ b/.github/workflows/plugin-sync.yml @@ -14,6 +14,7 @@ on: - "library-skills/**" - "open-models-skills/**" - "workflows/**" + - "workflow-skills/**" - "skills/**" - "plugins" - "skills.sh.json" diff --git a/.github/workflows/skill-security.yml b/.github/workflows/skill-security.yml index bb6a933d..ba3cd5e5 100644 --- a/.github/workflows/skill-security.yml +++ b/.github/workflows/skill-security.yml @@ -11,6 +11,7 @@ on: - "library-skills/**" - "open-models-skills/**" - "workflows/**" + - "workflow-skills/**" - ".github/workflows/skill-security.yml" - "scripts/scan_skills.py" - "scripts/plugin_sync.py" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da33559e..5033dcfe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,8 +34,8 @@ This repo is an **aggregator**. Most skills are authored in *other* GitHub repos and **vendored** in by the nightly sync (`.github/workflows/sync-skills.yml`), which reads `components.d/*.yml`, clones each source repo at its declared `ref`, and `rsync`s the declared skill folders into this repo's grouped catalog dirs -(`nim-skills/`, `library-skills/`, `open-models-skills/`, `workflows/`), then -opens a PR a maintainer reviews. +(`nim-skills/`, `library-skills/`, `open-models-skills/`, `workflows/`, +`workflow-skills/`), then opens a PR a maintainer reviews. A skill lives here in exactly one of two ways: @@ -44,7 +44,7 @@ A skill lives here in exactly one of two ways: every run (`rsync --delete`), so **do not edit a sourced skill dir in this repo** — the change reverts on the next sync. Fix it upstream. - **Native** — authored directly here and listed in `catalog-exceptions.yml` - (e.g. the NIM skills and the `workflows/` meta-skills). + (e.g. the NIM skills and the `workflows/` and `workflow-skills/` meta-skills). Every skill directory must be claimed by **exactly one** registry (`components.d` or `catalog-exceptions.yml`), or the orphan pruner @@ -56,6 +56,12 @@ copy directly; run `python scripts/plugin_sync.py --write` after changing a source skill. The root `plugins -> skills` symlink is a temporary compatibility path for existing plugin consumers. +The `protein-binder-design` source skill uses `workflow-skills/` because NVCarps +treats the `workflows/` prefix as a whole-workflow validation target. This lets +NVCarps validate the generated skill individually while plugin-sync and security +checks cover its source. Both roots remain registered so existing workflow +sources can stay in `workflows/` until they are migrated. + ### Onboarding a source repo Add `components.d/.yml`, meet the requirements below, and open a PR: diff --git a/README.md b/README.md index 9b27e76e..93c18a24 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ flow as well. Skills are also discoverable by partner harnesses directly from th | Product | Description | Skills | |---------|-------------|--------| -| **Protein Binder Design** | End-to-end de novo binder design workflows — a NIM route and a Proteina-Complexa route. | [`protein-binder-design`](workflows/generative-protein-binder-design/protein-binder-design), [`complexa-binder-design`](workflows/generative-protein-binder-design/complexa-binder-design) | +| **Protein Binder Design** | End-to-end de novo binder design workflows — a NIM route and a Proteina-Complexa route. | [`protein-binder-design`](workflow-skills/generative-protein-binder-design/protein-binder-design), [`complexa-binder-design`](workflows/generative-protein-binder-design/complexa-binder-design) | | **Generative Virtual Screening workflow** | Generate candidate molecules, dock them to a target, and score binding affinity (GenMol → DiffDock → Boltz-2). | [`drug-discovery-pipeline`](nim-skills/meta-skills/drug-discovery-pipeline) | | **MSA-enabled protein structure prediction workflow** | Build a multiple sequence alignment, then predict structure (MSA-Search → OpenFold3). | [`msa-structure-prediction-pipeline`](nim-skills/meta-skills/msa-structure-prediction-pipeline) | | **Boltz-2** | Biomolecular structure prediction + binding affinity (NIM). | [`boltz2-nim`](nim-skills/boltz2-nim) | diff --git a/catalog-exceptions.yml b/catalog-exceptions.yml index d3755949..255e2ae0 100644 --- a/catalog-exceptions.yml +++ b/catalog-exceptions.yml @@ -2,9 +2,9 @@ # registration. The forked prune-orphans.sh scans the SOURCED roots # (library-skills/, open-models-skills/) and deletes any dir there that is # neither declared in components.d/*.yml nor listed below. The wholly-native -# roots (nim-skills/, workflows/) are not scanned at all (see PRUNE_ROOTS in -# .github/scripts/prune-orphans.sh), so they need no entry — but nim-skills and -# workflows are listed here for documentation. +# roots (nim-skills/, workflows/, workflow-skills/) are not scanned at all +# (see PRUNE_ROOTS in .github/scripts/prune-orphans.sh), so they need no entry +# — but they are listed here for documentation. # # Add an entry only with a documented reason and an owner. exceptions: @@ -17,6 +17,10 @@ exceptions: reason: Native meta-skill workflows (generative-protein-binder-design, virtual_screening, ai_q) authored in-repo. owner: trvachov component: Workflows + - dir: workflow-skills + reason: Native workflow skills migrated out of workflows/ for individual skill validation in NVCarps. + owner: trvachov + component: Workflows # --- native skills inside a SCANNED (sourced) root — MUST be listed or the pruner deletes them --- - dir: library-skills/parabricks diff --git a/plugins.d/bionemo-agent-toolkit.yml b/plugins.d/bionemo-agent-toolkit.yml index ba1d7da4..5a98fce7 100644 --- a/plugins.d/bionemo-agent-toolkit.yml +++ b/plugins.d/bionemo-agent-toolkit.yml @@ -40,6 +40,7 @@ include_skills: - open-models-skills/proteina-complexa/ # Workflows - workflows/generative-protein-binder-design/ + - workflow-skills/generative-protein-binder-design/protein-binder-design/ # Inherits version/author/homepage/repository/license/brand_color/capabilities # from plugins.d/_defaults.yml. diff --git a/scripts/plugin_sync.py b/scripts/plugin_sync.py index c9ba2a23..13f98341 100644 --- a/scripts/plugin_sync.py +++ b/scripts/plugin_sync.py @@ -42,7 +42,7 @@ CONFIG = REPO / "skills.sh.json" # Source roots that hold distributable skills. -SOURCE_ROOTS = ["nim-skills", "library-skills", "open-models-skills", "workflows"] +SOURCE_ROOTS = ["nim-skills", "library-skills", "open-models-skills", "workflows", "workflow-skills"] # Path segments that never contain a distributable skill. EXCLUDE_SEGMENTS = {"plugins", "vendor", "evals", "node_modules", ".git"} # Files/dirs ignored when copying/comparing (never part of the payload). diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/BENCHMARK.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/BENCHMARK.md new file mode 100644 index 00000000..6a62bcbf --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/BENCHMARK.md @@ -0,0 +1,117 @@ +# Skill Benchmark: protein-binder-design + +> **Overall verdict: NEUTRAL — One or more dimensions remain below PASS** + +Live evaluation did not show a material gain or regression. Collect more evidence or improve the skill before making a publication decision. + +## Evaluation Metadata + +- Skill: `protein-binder-design` +- Evaluation date: 2026-10-02 +- Evaluator version: `1.5.6` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-5`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 1 evaluation tasks (1 positive) +- Dataset digest: `sha256:109a2793e5be03aca71b3dc7844f3adb21799632e13edcf2fc73cc66cdfd1303` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 83.8% — baseline ran, but no comparable score was available; uplift unavailable | 85.5% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 35.0% — baseline ran, but no comparable score was available; uplift unavailable | 85.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 100.0% → 100.0% (±0.0 points) | 100.0% → 58.3% (-41.7 points) | +| Efficiency | 83.9% — baseline ran, but no comparable score was available; uplift unavailable | 84.2% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 387,458 | 733,717 | -346,259 | -47.19% | skill 1/1; base 1/1 | +| claude-code | offline-manifest | 387,458 | 733,717 | -346,259 | -47.19% | skill 1/1; base 1/1 | +| codex | All cases | 204,321 | 182,661 | +21,660 | +11.86% | skill 1/1; base 1/1 | +| codex | offline-manifest | 204,321 | 182,661 | +21,660 | +11.86% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 591,779 | 916,378 | -324,599 | -35.42% | skill 2/2; base 2/2 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 41 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 1 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: No documented scripts in table format (`skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: Instructions don't mention 'run_script' (`skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md`) +- **MEDIUM** SCHEMA/folder_hierarchy: Unexpected nesting depth for general skill (`skills/bionemo-agent-toolkit/skills/protein-binder-design`) +- 36 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md index 282aaa0c..a984980c 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md @@ -20,7 +20,7 @@ protein-binder-design/ │ ├── metrics.py # Kabsch CA-RMSD (self-consistency) │ ├── controls.py # scrambled negative controls │ └── registry.py # target-registry loader -├── assets/targets.json # EXAMPLE target registry (verify before real use) +├── assets/targets.json # empty target-registry template (populate before use) └── evals/ # trigger + assertion evals ``` @@ -30,7 +30,25 @@ protein-binder-design/ - Access to the BioNeMo NIMs you intend to use (RFdiffusion, ProteinMPNN, Boltz2 and/or OpenFold3, optional MSA-Search) — hosted at [build.nvidia.com](https://build.nvidia.com) or self-hosted via NGC. -- Python ≥ 3.10 with `numpy` (`pip install numpy`). Scripts are otherwise stdlib. +- Python ≥ 3.12 for the pinned helper environment. Install dependencies with + `python3 -m pip install -r requirements.txt`; mmCIF parsing and cached refold + recovery require Biotite. Manifest and PDB-only helpers use the standard library. + +## Offline tests + +Install `requirements-dev.txt` in a virtual environment, then run +`python3 -m pytest tests`. To inspect coverage, run +`python3 -m pytest --cov=scripts --cov-report=term-missing tests`. +These tests use synthetic fixtures and credentials; they make no model calls. + +## Tier 1 validation + +NVCARPS selects `config/skillspector-baseline.yml` before the repository-wide +baseline. Include this file, the requirements files, and the tests when committing +the skill; synchronize the aggregate copy with the repository's plugin sync tool. +The baseline records reviewed false positives as exact source fingerprints for +SkillSpector 2.11.2. Source edits or a scanner upgrade require another review; +the environment-copy finding is fixed in code and is not suppressed. ## Configure diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md index 5cf456d7..4a212df6 100755 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md @@ -1,18 +1,65 @@ --- name: protein-binder-design description: > - Orchestrate an end-to-end de novo protein binder design campaign against a protein target by composing BioNeMo NIM skills. Use for binder design, minibinder design, de novo binders, RFdiffusion + ProteinMPNN + Boltz2/OpenFold3 pipelines, epitope/hotspot-targeted design, in-silico binder validation, and ranking designs by interface confidence. + Use when designing RFdiffusion/ProteinMPNN binders or validating and resuming binder campaigns. Not for code or report review. license: Apache-2.0 -compatibility: "numpy>=1.24; requests>=2.28" +compatibility: "Python>=3.12 with requirements.txt; Bash for hosted_env.sh" +metadata: + author: "NVIDIA BioNeMo " + tags: [protein-design, binder-design, bionemo] allowed-tools: Bash, Read, Write, AskUserQuestion +permissions: + - env # reads NVIDIA_API_KEY/NGC_API_KEY and configured NIM endpoints + - network # calls hosted NVIDIA APIs or user-configured local NIM endpoints --- # Protein Binder Design (workflow) +## Purpose + Run a de novo binder design campaign by composing atomic NIM skills. This skill owns orchestration, handoff contracts, filtering, validation, and the run -manifest. It does NOT duplicate per-NIM API details — defer those to each -atomic skill's `SKILL.md`. +manifest. For per-NIM API details, consult the corresponding NIM skill. + +Use for requests such as “design protein binders”, “run an RFdiffusion campaign”, +“validate and rank these binders”, or “resume this binder manifest”. A request to +review code, explain methods, or inspect an evaluation report does not start a +design campaign. Offline bookkeeping stays offline. + +## Prerequisites + +- Python 3.12+; install [requirements.txt](requirements.txt) for RMSD, mmCIF, + cache recovery, and Python HTTP examples. Manifest and PDB-only helpers use + the standard library. Install [requirements-dev.txt](requirements-dev.txt) + to run `python3 -m pytest tests` with coverage. +- Live inference needs hosted NIM credentials or reachable local NIM services; + local deployment also needs Docker, the NVIDIA Container Toolkit, and a + supported GPU. See Configuration below and [local setup](references/local-nim-setup.md). + +## Inputs + +Required inputs from the user prompt or saved manifest: target structure/sequence, +target chain, epitope/hotspots, binder length range, backbone/sequence counts, +output directory, and hosted or local endpoints. Resolve the confidence +selection policy before live inference. Resume requests require the saved +manifest and artifacts; offline imports require a synthetic score fixture or +cache bindings as defined in [offline contracts](references/offline.md). +Optional inputs include a target MSA, sourced positive controls, and explicit +overrides to the default selection thresholds. + +Apply the user's explicit corrections first. Otherwise preserve saved campaign +settings, then use supplied task context; ask only for unresolved inputs. Record +intentional changes to saved settings so resumed results remain interpretable. + +## Data handling + +Hosted mode sends target structures/sequences, candidate sequences, and any +requested alignments to the selected NVIDIA NIM endpoints under that service's +terms. Use the user's selected deployment; resolve the destination before +uploading data if it is unspecified. Local mode sends inference inputs to the +configured local service; initial image/weight downloads still contact NGC. +All modes write JSON/CSV and, when produced, PDB/mmCIF/FASTA artifacts under the +chosen run directory. Keep credentials out of those artifacts and logs. ## Composed skills @@ -28,7 +75,32 @@ NIM skill set). They are **not required**: `references/pipeline.md` carries the concrete request shape for every NIM call, so an agent with NIM access can follow this skill standalone. For endpoints/auth see **Configuration** below. -## Pipeline +## Instructions + +Resolve the campaign inputs before inference. The bundled `assets/targets.json` +is a template, not a populated PD-L1 registry. A named target needs a verified +structure, chain, and epitope; controls need existing candidate sequences and +sourced positive controls; resuming needs the original manifest and artifacts. +Check the supplied paths first. If an input or NIM credential is missing, record +the blocker and the unexecuted stages. Do not invent registry entries, measured +scores, or an interrupted run. An empty CSV alone does not establish a completed +campaign; report which stages ran and their saved artifacts. + +### Offline inputs + +For a synthetic fixture with `target` and `profiles`, run +`python3 scripts/offline_bookkeeping.py /path/to/bookkeeping.json --output-dir /path/to/output`. +This saves and reloads each manifest, writes both CSVs and `summary.json`, and +reports missing scores and provisional counts in one call. For an interrupted +synthetic run with saved responses, use +`python3 scripts/refold_cache.py /path/to/manifest.json --output-dir /path/to/new-run`. +Read [offline contracts](references/offline.md) for cache bindings, sample +selection, chain confidence and blocked work. These helpers need no credentials +and make no model calls. Inspect exit status and saved artifacts before claiming +completion; a failed command followed by a successful retry is recovered +execution, with both attempts retained as evidence. + +### Pipeline 1. **Target prep** — get the target PDB + epitope; map epitope/hotspot author residue numbers to RFdiffusion `hotspot_res` strings; optionally build a @@ -37,13 +109,40 @@ this skill standalone. For endpoints/auth see **Configuration** below. 3. **Sequences** (`proteinmpnn-nim`) — k sequences per backbone; drop the native/WT row from `mfasta`. 4. **Co-fold + score** (`boltz2-nim` / `openfold3-nim`) — co-fold binder+target; - collect interface confidence (ipTM) and binder pLDDT. + collect explicit ipTM or Boltz2 composite confidence, plus binder pLDDT. 5. **Self-consistency** — CA-RMSD between the RFdiffusion backbone and the predicted binder (`scripts/metrics.py`). -6. **Filter + rank** — apply thresholds; rank survivors; write manifest + CSV. +6. **Filter + rank** — apply the saved campaign thresholds, exclude controls, + sort surviving designs, and export that exact list (see below). Full handoff contracts, branching, and the cost funnel: `references/pipeline.md`. +### Filter, rank, and export + +For `m` returned by `Manifest.create()` or `Manifest.load()`, use this sequence +after scoring. Keep the supplied filters, ranking metric, scores, and control +flags unchanged: + +```python +from pathlib import Path + +m.apply_filters() +rank_by = m.data["params"].get("rank_by", "iptm") +ranked = m.rank(by=rank_by, descending=True, passed_only=True, include_controls=False) +run_dir = Path(m.data["run_dir"]) +m.to_csv(run_dir / "all_candidates.csv") # complete audit, including controls +m.to_csv(run_dir / "candidates.csv", candidates=ranked) +print(m.summary()) # controls excluded from candidate/pass counts +print([candidate["id"] for candidate in ranked]) +``` + +`rank()` returns a new list; it does not reorder or shrink the manifest. +`to_csv()` without `candidates=ranked` writes **all** entries in insertion order. +Pass an empty ranked list through unchanged: zero survivors means a header-only +`candidates.csv`, while the audit CSV and manifest retain every entry. A high +ranking score cannot override a failed or missing enabled filter metric. Report +missing scores and label the passing fraction provisional when scoring is incomplete. + ## Handoff contracts (the fragile glue) - RFdiffusion `output_pdb` → ProteinMPNN `input_pdb` (inline PDB text). @@ -52,32 +151,69 @@ Full handoff contracts, branching, and the cost funnel: `references/pipeline.md` - Epitope author residue numbers → 1-based sequence indices: remap with `scripts/pdb_utils.py:remap_to_seq_index`. RFdiffusion `hotspot_res` uses chain+author strings like `"A50"`; Boltz2 pocket/contacts use 1-based indices. + Keep insertion IDs distinct (`50` versus `50A`), select the target chain/model, + and block absent hotspots instead of substituting nearby residues. Verify + downstream support before requesting an insertion-coded hotspot. - Boltz2 complex `.cif` → binder chain → self-consistency RMSD vs the backbone. + Use the same saved sample for confidence and coordinates; mmCIF request polymer + IDs are label chain IDs. Whole-complex pLDDT does not supply binder-only pLDDT. -## Run manifest (reproducibility backbone) +## Output Format Every campaign writes `manifest.json` (+ `candidates.csv`) under a run dir via `scripts/manifest.py`. It records lineage, params, scores, artifacts, filter status, and controls — enabling ranking, resumability, validation, and the -final report. Schema and usage: `references/manifest.md`. +final report. Schema and usage: [manifest contract](references/manifest.md). +Keep all entries in `all_candidates.csv`; `candidates.csv` contains only ranked +survivors and retains its header when none pass. Offline helpers also write +`summary.json`. The final response identifies the mode, completed/blocked stages, +artifact paths, selection thresholds, ranked IDs, and missing scores. Report +control counts separately and mark an incomplete passing fraction provisional. ## Filters (defaults) - ipTM ≥ 0.8, binder pLDDT ≥ 80, self-consistency RMSD ≤ 2.0 Å. - Override per campaign and record overrides in the manifest `filters`. +- Every applicable enabled metric needs a finite score to pass. Sequence-only + controls have no designed backbone: their RMSD is not applicable and the manifest + records that exemption; design candidates still require RMSD. Keep a composite confidence + value in its own field; do not substitute it for an unavailable ipTM. +- For Boltz2-only campaigns without explicit ipTM, resolve the composite + confidence cutoff before inference. Set `iptm_min: null`, enable + `boltz2_confidence_min` with that cutoff, retain the pLDDT/RMSD thresholds, + and save `params.rank_by: boltz2_confidence`. Filter with `apply_filters()`, + rank with `rank(by="boltz2_confidence", passed_only=True)`, and export that + list with `to_csv(candidates=ranked)`. Report this as composite-confidence + selection; it does not establish the default ipTM criterion. Complete example: + `references/manifest.md`. **Default decision when no cutoff was supplied:** use + a refolder that returns explicit ipTM (OpenFold3) and the default ipTM profile. + If only composite-confidence Boltz2 is available, obtain a campaign cutoff and + its rationale from the user or a separate same-target calibration before design + inference. Save it in `params.confidence_selection`. Without either, stop at + input preparation and report `selection_policy_unresolved`; do not guess 0.8 + or present unfiltered rankings as validated binders. Calibration procedure: + `references/validation.md`. ## Validation -Always run controls and report a **success rate**, not just top scores. +Run scrambled controls and report a **success rate**, not just top scores. Negative controls via `scripts/controls.py` (scrambled sequences); positive -controls = published binders re-scored through the same pipeline. Benchmark -targets live in `assets/targets.json` (`scripts/registry.py`). Methodology and +controls = sourced published binders re-scored through the same pipeline. The +bundled `assets/targets.json` is an empty control template, not a positive-control +dataset. Use user-provided sequences with citations, or retrieve and verify an +exact sequence from a primary publication or structure before registering it. +If none is available, record `positive_controls_unavailable`, complete the +computational screen with negatives, and label it **uncalibrated screening**; +do not claim positive-control benchmarking or calibrated binder success. A +request requiring a benchmark remains incomplete until a positive is supplied. +Methodology and metric definitions: `references/validation.md`. ## Human-in-the-loop + cost -- Confirm target, epitope/hotspots, binder length range, and hosted-vs-local - with the user before generating backbones (AskUserQuestion). +- Reuse target, epitope/hotspots, binder length range, and hosted-vs-local + choices already supplied by the user. Ask only for unresolved inputs before + generating backbones. - Co-folding is the expensive stage: co-fold a capped shortlist, review, then expand. State hosted vs local once and reuse it across all NIM calls. @@ -93,24 +229,77 @@ Each composed NIM is reached over HTTP; choose **hosted** or **local** once and reuse it for every call: - **Hosted** (managed): base URL `https://health.api.nvidia.com/v1/...` per NIM at - [build.nvidia.com](https://build.nvidia.com); set `NVIDIA_API_KEY` (sent as - `Authorization: Bearer`). Read keys from the env — never hardcode them. + [build.nvidia.com](https://build.nvidia.com); use `NGC_API_KEY` or the + `NVIDIA_API_KEY` fallback (sent as `Authorization: Bearer`). Read keys from the + environment. Check only presence with + `bool(os.getenv("NGC_API_KEY") or os.getenv("NVIDIA_API_KEY"))`; never print + environment dumps, key values, or authorization headers. An `OPENAI_API_KEY` + belongs to the agent runtime and must not be used as a NIM credential. If both + NIM keys are absent, report missing access before sending authenticated calls. + Run **each delegated hosted command** through the credential wrapper: + `bash scripts/hosted_env.sh python3 your_nim_request.py`. + Use the helper's absolute path when running outside the skill directory. + It exports the selected key as `NGC_API_KEY`, which the atomic NIM skills read, + preserves an existing nonempty `NGC_API_KEY`, and stops before launching the + child if neither key exists. The wrapper loads the key and executes the command + in one invocation, so no exported state from a previous tool shell is needed. + It disables shell tracing and preserves arguments and the child's exit code. - **Local** (self-hosted NGC containers): point each NIM at its local URL - (e.g. `http://localhost:8000/...`); local NIMs need no auth header. To **launch** the - NIMs yourself (docker run per NIM, persistent caches, health checks, and the GPU - **profile‑selection gotcha** — some NIMs (e.g. Boltz2) need `NIM_MODEL_PROFILE` pinned - on GPUs that have no bundled profile, while others (RFdiffusion/ProteinMPNN) auto‑select - by compute capability): see **`references/local-nim-setup.md`**. + (e.g. `http://localhost:8000/...`); local NIMs need no auth header. Follow + [local setup](references/local-nim-setup.md) for digest-pinned images, private + cache mounts, loopback ports, readiness checks, and GPU-specific profile selection. Per-NIM paths, request/response schemas, and worked `curl`/Python examples live in `references/pipeline.md`. -## Scripts +## Examples + +- **New campaign:** with a verified target entry and NIM access, create the + manifest, remap residues with `pdb_utils.py`, generate backbones, design binder + sequences, co-fold a shortlist, and save response-derived scores before + filtering and exporting. Record any stages that could not run. +- **Controls:** load the existing campaign, use `make_scrambled_controls` on + its binder sequences, and add sourced published binders. Mark all controls + `is_control=True`, co-fold with the same settings, then report + `n_passed / n_candidates` from `summary()` with controls excluded. With no + candidates the rate is unavailable; disclose incomplete scoring. +- **Resume:** call `Manifest.load("runs/")` and + `missing_scores(candidate_id)`; use the artifact checklist in + `references/manifest.md` to execute only missing stages. A finite below-threshold + score is complete, not a reason to repeat inference. Reuse saved predictions + for missing metrics; preserve completed candidates and explicit failure records. + +## Available Scripts + +Paths below are relative to this skill. Run CLIs with `python3 scripts/.py` +or `bash scripts/hosted_env.sh`; import library helpers after adding the skill's +`scripts` directory to the Python import path. No harness-specific runner is required. -- `scripts/manifest.py` — campaign manifest (create / load / score / filter / rank / CSV). -- `scripts/pdb_utils.py` — PDB parse, chain extract, sequence, residue remap, CA coords. -- `scripts/metrics.py` — Kabsch CA-RMSD for self-consistency. -- `scripts/controls.py` — scrambled negative controls. -- `scripts/registry.py` + `assets/targets.json` — **example** benchmark target - registry (illustrative epitopes — verify against the cited structure before a - real campaign). Replace with your own targets. +| Script | Purpose | Arguments | +|---|---|---| +| [offline_bookkeeping.py](scripts/offline_bookkeeping.py) | CLI: import synthetic score profiles | `input.json --output-dir DIR` | +| [refold_cache.py](scripts/refold_cache.py) | CLI: recover metrics from verified cache bindings | `manifest.json --output-dir DIR` | +| [metrics.py](scripts/metrics.py) | CLI/library: binder CA-RMSD | `backbone.pdb prediction.cif --backbone-chain A --predicted-chain B` | +| [hosted_env.sh](scripts/hosted_env.sh) | Credential wrapper | `COMMAND [ARG ...]` | +| [manifest.py](scripts/manifest.py) | Library: save, filter, rank, export | `Manifest.create(run_dir, target, mode, params, filters)` / `Manifest.load(path)` | +| [pdb_utils.py](scripts/pdb_utils.py) | Library: select chain and remap residues | `remap_to_seq_index(pdb_text, chain, author_resnums)` | +| [controls.py](scripts/controls.py) | Library: composition-preserving negative controls | `make_scrambled_controls(seqs, n=5, seed=0)` | +| [registry.py](scripts/registry.py) | Library: load a user-populated target registry | `get_target(name, path=None)`; default is the empty `assets/targets.json` template | + +## Limitations + +Confidence thresholds and scrambled controls provide computational screening, +not experimental proof of binding. The target registry contains no verified +positive controls. Boltz2 composite confidence is distinct from ipTM. Cached +synthetic fixtures test bookkeeping, not model performance or biological validity. + +## Troubleshooting + +| Error or symptom | Cause | Resolution | +|---|---|---| +| Missing NIM key | Neither supported key is set | Configure the selected hosted credential or use an available local service | +| `selection_policy_unresolved` | No explicit ipTM or calibrated composite cutoff | Resolve the refolder/cutoff before design inference | +| Absent hotspot or chain | Author numbering/chain differs from input | Inspect the selected model and insertion codes; supply the correct residue IDs | +| Cache binding/digest failure | Response identity or bytes do not match | Keep the failed candidate visible; obtain matching artifacts before retrying | +| No ranked survivors | Enabled metric missing or below threshold | Inspect the audit CSV; retain the empty ranking and disclose incomplete scoring | +| Missing Python module / tests cannot collect | Helper or test dependencies absent | Install the appropriate requirements in the same Python environment used to run the command | diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/config/skillspector-baseline.yml b/skills/bionemo-agent-toolkit/skills/protein-binder-design/config/skillspector-baseline.yml new file mode 100644 index 00000000..161a053a --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/config/skillspector-baseline.yml @@ -0,0 +1,86 @@ +# Reviewed 2026-10-07 against SkillSpector 2.11.2, the scanner used by NVCARPS. +# Exact fingerprints bind the complete file contents and scanner version. +# File edits or a scanner upgrade invalidate these exceptions and require review. +# No rule-wide suppressions; credential harvesting is fixed in code, not waived. +# Environment-isolation regression rechecked 2026-10-08; its AST4 hash refreshed. +version: 2 +scanner_version: 2.11.2 +rules: [] +fingerprints: + - hash: sha256:9996d6fb97fdfb596f2f50c4cf3e79570283746d312372f175e15db370fc0831 + rule_id: AS3 + file: BENCHMARK.md + reason: >- + Generated historical evaluation quotes this skill's own manifest path in + four diagnostic rows. No instruction or code reads other installed skills. + The benchmark evidence and scores remain unmodified. + - hash: sha256:daaafc2bfa26f14b989e67b1e96716fcf3f4d8f9989dde5b033821ad887d9f10 + rule_id: PE2 + file: references/local-nim-setup.md + reason: >- + chmod 700 restricts three explicitly named, user-owned cache directories. + It neither invokes sudo nor grants elevated privileges. Container UID/GID + matches the invoking user; no recursive chmod or wildcard is used. + - hash: sha256:4eb3e8ba0df55f36be8791053aef6e0998b6e4a4b45536296e05def0f16496d8 + rule_id: RP1 + file: references/local-nim-setup.md + reason: >- + The launch block validates all three image references against the expected + NGC repositories and 64-hex SHA-256 digests before any service starts. + The static Docker parser cannot resolve these checked shell variables. + Mocked execution verified that tags, wrong registries, and malformed + digests fail before any command with side effects. + - hash: sha256:3bf6b75e39342d08503840522a956796814679807c077c7a7fa32191d4ee9661 + rule_id: RP1 + file: references/local-nim-setup.md + reason: >- + The optional profile-listing example explicitly reuses the Boltz2 digest + validated by the preceding launch block. It specifies no moving/default + tag. The static Docker parser cannot resolve the checked variable. + - hash: sha256:49a8eccca8415a52025878019d22ad62d4407bb3e0bac773944c5e33cfa357ff + rule_id: AST4 + file: tests/test_hosted_env.py + reason: >- + Fixed test commands exercise credential selection with synthetic values, + an explicit argument list, shell=False, and an isolated environment. + The legacy bash -c source test uses positional parameters with fixed code; + it interpolates no user input and makes no network calls. + - hash: sha256:9a86caae52eb514ddfe60d96005e799e15e99d09abc0c657cde9ca81e54c5734 + rule_id: AST4 + file: tests/test_hosted_env.py + reason: >- + Fixed bash tracing test checks that a synthetic credential stays out of + output and that the wrapper preserves an argument containing spaces and + the child's exit status. Uses shell=False and no inherited credentials. + - hash: sha256:410d6f5fceb976f9e8783fe51370814d9f4c6112cde35bdf4f7474073a20be80 + rule_id: AST4 + file: tests/test_offline.py + reason: >- + Executes this repository's standalone offline grader with the current + Python executable, isolated mode, a fixed argument list, and shell=False. + The child receives only three temporary fixture paths. It receives no + developer/CI credentials and submits no inference or network requests. + A parent-only reward-path override verifies environment isolation: restoring + bulk inheritance makes the output-location assertion fail. + # Additional occurrences of the same reviewed benchmark/path and launch text. + # Each line is fingerprinted separately; the scanner expands compacted groups. + - hash: sha256:d52d9f5c67c895424f94182a0e365ebf673b1de593e196f607ef46c557aa5990 + rule_id: AS3 + file: BENCHMARK.md + reason: Generated diagnostic quotes this skill's own manifest path; no peer-skill access. + - hash: sha256:f5ea4acc4cd746d227dc072baf0af37bc71905295cae7d8d542d87a18ce2038b + rule_id: AS3 + file: BENCHMARK.md + reason: Generated diagnostic quotes this skill's own manifest path; no peer-skill access. + - hash: sha256:757ebd760479cde3d81fd3a0d32fdf58a7d00c4c1607cb529906d7d648c632ed + rule_id: AS3 + file: BENCHMARK.md + reason: Generated diagnostic quotes this skill's own manifest path; no peer-skill access. + - hash: sha256:23cec21cbfc72046b9fec2058f664eccab0821054e7c9ac9497fae60424ffea3 + rule_id: RP1 + file: references/local-nim-setup.md + reason: ProteinMPNN launch uses the expected NGC repository and SHA-256 digest checked before any service starts. + - hash: sha256:658d33910213c2b483f90ad16b6aa0cc2f1a3067813cf9b2e85101e488b9b343 + rule_id: RP1 + file: references/local-nim-setup.md + reason: Boltz2 launch uses the expected NGC repository and SHA-256 digest checked before any service starts. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md new file mode 100644 index 00000000..5377e065 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md @@ -0,0 +1,128 @@ +# Evaluation coverage and deferred live campaigns + +## Active coverage: offline software workflows + +`evals.json` runs eight offline exercises. The four original prompts, assertions, +expected outputs and score fixtures remain unchanged for historical comparison. +Four additional cases require target mapping, saved-response interpretation, +recovery decisions and provenance checks. Their output contracts and all required +inputs are staged in both conditions; equivalent implementations are accepted. + +| Case | Input under `/workspace/input/` | Required result | +| --- | --- | --- | +| `offline-manifest` | `bookkeeping.json` | Persistence and ranked exports for ipTM and Boltz2 profiles. | +| `offline-boltz2-ranking` | `boltz2_only.json` | Composite confidence stays separate from ipTM; passing controls do not enter the ranked CSV. | +| `offline-mixed-threshold-ranking` | `mixed_thresholds.json` | All filters apply before sorting; a high ipTM cannot rescue another failing or invalid metric. | +| `offline-empty-ranking` | `no_survivors.json` | A header-only ranked CSV and a complete audit when no designs pass. | +| `offline-target-preparation` | `target_prep/` | Select chain/model/alternate CA; keep insertion codes distinct; block absent hotspots. | +| `offline-refold-extraction` | `raw_refolds/`, `cache_contract.md` | Bind scores to the recorded sample, separate confidence metrics, and use binder-only pLDDT. | +| `offline-cache-resume` | `resume_cache/`, `cache_contract.md` | Preserve complete scores and failure history; recover missing metrics from cache; rebuild stale exports. | +| `offline-provenance-rejection` | `provenance_cache/`, `cache_contract.md` | Reject candidate, sequence, target, sample and response-file mismatches before scoring. | + +Every case stages identical inputs for the with/without-skill conditions. +`environment/Dockerfile` supplies the same pinned NumPy and Biotite versions to +both conditions during environment setup, before the offline task starts. +The original prompts and fixtures are preserved; environment and verifier +changes should still be recorded when comparing with historical reports. +The added prompts request campaign outputs without giving the expected order; +the assertions check saved artifacts, not use of a particular helper. +All sequences, toy coordinates, numeric inputs, saved responses and control labels are synthetic. +The active `config.yml` forwards no NIM credentials and the cases make no NIM +calls. Passing them establishes **offline software workflow coverage only**, not live model +execution, biological validity, or end-to-end campaign coverage. + +Compare agents separately when rerunning: a passing best-agent summary can hide +another agent's regression. Compare the original case against its own previous +result; the expanded suite's aggregate is not directly comparable to the old +one-case score. Positive lift is an observation, not a target for the grader: +keep the baseline inputs and success criteria identical. + +Run the bundled regression tests without model access: + +```bash +python3 -m unittest discover -s tests -v +``` + +## Deterministic artifact checks + +`grading.mode: aces_plus_custom` runs the existing ACES judges and the standalone +`grader.py`. The verifier reads actual manifests, CSVs, selected complex files +and summaries. It pins input digests, checks score types and absence, ranking +order, control exclusion, preserved state, source identity and artifact bytes. +It imports no skill helpers. Tests exercise all eight workflows and deliberately +corrupt their outputs to check that the verifier rejects them. The old ambiguous +phrase "ranked candidates.csv" accepts `candidates.csv`, `ranked candidates.csv` +or `ranked_candidates.csv`; new contracts specify `candidates.csv` exactly. + +**SkillEvaluator 1.5.6 keeps the ACES headline score and pass@k authoritative.** +The custom binary metric `artifact_correctness` and individual failures appear +in `custom_reward.json` and the rich `aces_reward.json` custom fields. They do +not change the headline score or turn its "All Passed" label into an artifact +gate. Inspect this metric per trial alongside the judges. Do not interpret an +ACES pass with `artifact_correctness=0` as a completed artifact-producing task. + +Artifact checks cannot prove an agent actually performed a save/reload step or +made no model calls; the trajectory judges retain those responsibilities. They +also do not prove biological validity. Keep recovery attempts and their tool +results in the judge evidence. To audit a suspicious grade, compare the full +trajectory, the captured judge `prompt_evidence` when available, and verifier +artifacts. Missing transcript evidence is not proof that files were never made. + +The eight-case suite uses 32 runs at one trial per case, two agents and both +conditions. Compare the unchanged four-case subset separately from new cases, +and compare agents separately. A fresh live agent evaluation is needed to +measure lift; passing local tests does not predict an improved agent score. + +Fixture changes require updating the corresponding `INPUT_HASHES` entries in +`grader.py` and reviewing the hand-calculated `CACHE_RUNS` outcomes. Never derive +expected truth from unverified agent-mutated inputs. The local regression suite +fails on digest drift. Keep contracts identical in both conditions and express +real workflow decisions, rather than making prompts vague or targeting one model. + +## Deferred: live design, controls, and resume + +The original three live cases are retained in `deferred/live-campaigns.json`, +outside the active `evals.json` dataset. **They are not runnable or counted as +covered.** Their required verified inputs are not bundled; do not activate them +with empty `files` lists or substitute synthetic scores. Each case and each +with/without-skill attempt starts independently, so cases 2 and 3 cannot reuse +outputs from case 1. + +| Case | Required inputs | +| --- | --- | +| 1: PD-L1 design | Verified PD-L1 registry entry, matching target structure and chain, author-numbered epitope, binder length range, and shortlist size. | +| 2: Controls | Existing PD-L1 campaign manifest and candidate sequences, matching target structure, and published positive-control sequences with source citations. | +| 3: Resume | The interrupted campaign's manifest and referenced artifacts, including both completed candidates and candidates with missing scores. | + +When restoring live cases, configure `harbor.runtime_env` to forward the +available NIM credential (`NGC_API_KEY` or `NVIDIA_API_KEY`) to both conditions. +Normalize it with `scripts/hosted_env.sh` in the same shell as each hosted +command. Forwarding a key does not supply campaign inputs or prove endpoint +access. The agent runtime's `OPENAI_API_KEY` is not a substitute for a NIM key. + +## Staging inputs + +Put verified fixtures under `evals/files/` and list the relevant files or +directory in each case's `files` array. SkillEvaluator 1.5.6 stages the selected +paths beneath `/workspace/input/`, preserving their paths relative to +`evals/files/`. For example, selecting `evals/files/pdl1_design/` makes its +contents available under `/workspace/input/pdl1_design/`. The same inputs must +be available to both conditions. + +Point prompts at those explicit staged paths. For the resume case, either ask +the agent to resume the staged run there or copy it to `runs/rbd_binders` during +pre-agent setup, retaining its artifact paths. A directory name in a prompt +does not create that directory. Record fixture provenance and preserve the +existing completed outputs so the grader can check that they were not recomputed. + +Before moving a deferred case back into `evals.json`, update its prompt to name +those staged paths and replace plan-only assertions with execution requirements: +invoke the model APIs, save actual responses, +extract metrics, and update the manifest. Empty CSVs, prepared scripts, and +synthetic confidence values do not establish successful inference. A separate +offline bookkeeping test may use explicitly synthetic data; it cannot stand in +for these live integration cases. + +The workflow can be followed through `references/pipeline.md`. +Companion NIM skills are optional; include them in both evaluation conditions +if testing the workflow as a group. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml new file mode 100644 index 00000000..e54e6139 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml @@ -0,0 +1,12 @@ +schema_version: 1 + +harbor: + task_source: evals_json + agents: + claude-code: + model: aws/anthropic/bedrock-claude-opus-5 + # The active dataset uses staged synthetic bookkeeping and workflow inputs only. + # Deferred live campaigns must supply fixtures before they are re-enabled. + +grading: + mode: aces_plus_custom diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/deferred/live-campaigns.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/deferred/live-campaigns.json new file mode 100644 index 00000000..9845a15f --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/deferred/live-campaigns.json @@ -0,0 +1,42 @@ +{ + "skill_name": "protein-binder-design", + "evals": [ + { + "id": "1", + "prompt": "Design 50 de novo mini-binders against PD-L1 (use the target registry entry) at its defined interface epitope. Generate backbones, design 8 sequences each, co-fold the shortlist against the target, and rank by interface confidence. Save a run manifest and a ranked CSV.", + "expected_output": "An orchestration that composes the rfdiffusion-nim, proteinmpnn-nim, and boltz2-nim/openfold3-nim skills in order, remaps epitope residues, writes a run manifest via scripts/manifest.py, applies the ipTM/pLDDT/RMSD filter, and exports a ranked candidates.csv.", + "files": [], + "assertions": [ + "[composes-rfdiffusion] Uses RFdiffusion for binder backbones: Plan invokes the rfdiffusion-nim skill with a binder contig and hotspot_res", + "[composes-proteinmpnn] Uses ProteinMPNN for sequence design: Plan invokes the proteinmpnn-nim skill and drops the native/WT row from mfasta", + "[composes-cofold] Co-folds binder with target: Plan invokes boltz2-nim or openfold3-nim to co-fold the binder with the target and collects ipTM or confidence", + "[writes-manifest] Writes a run manifest: Plan uses scripts/manifest.py (Manifest.create, set_scores, apply_filters, to_csv)", + "[applies-filter] Applies default filter thresholds: Plan filters on ipTM >= 0.8, binder pLDDT >= 80, self-consistency RMSD <= 2.0", + "[remaps-epitope] Remaps epitope residues: Plan remaps epitope author numbering using scripts/pdb_utils.py before use" + ] + }, + { + "id": "2", + "prompt": "For my binder campaign against PD-L1, include proper controls and tell me the success rate.", + "expected_output": "Adds scrambled-sequence negative controls and published-binder positive controls through the same co-folding pipeline, marks them is_control in the manifest, and reports success rate as n_passed / n_candidates.", + "files": [], + "assertions": [ + "[negative-controls] Generates scrambled negative controls: Plan uses scripts/controls.py to scramble binder sequences and marks them is_control with control_type scrambled", + "[positive-controls] Uses published binders as positive controls: Plan re-scores published binders from the registry through the same pipeline as positive controls", + "[success-rate] Reports a success rate: Plan reports success rate as n_passed divided by n_candidates from manifest.summary()", + "[same-pipeline] Controls run through identical pipeline: Controls are co-folded and scored with the same settings as the designs" + ] + }, + { + "id": "3", + "prompt": "Continue my interrupted binder run in runs/rbd_binders and only process candidates that are missing scores.", + "expected_output": "Loads the existing manifest with Manifest.load, identifies candidates lacking the needed scores/artifacts, and resumes without re-running completed stages.", + "files": [], + "assertions": [ + "[loads-manifest] Loads the existing manifest: Plan calls Manifest.load on the run directory", + "[resumes-incomplete] Skips completed candidates: Plan only processes candidates missing required scores or artifacts", + "[no-recompute] Does not recompute finished stages: Plan avoids re-running stages already recorded in the manifest" + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/environment/Dockerfile b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/environment/Dockerfile new file mode 100644 index 00000000..323dfaad --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim + +# Shared by both conditions, before the offline agent task starts. Pin the +# versions used by the local regression suite; no NIM clients or credentials. +RUN python3 -m pip install --no-cache-dir numpy==2.5.1 biotite==1.7.1 diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/evals.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/evals.json index 1dadc67b..b8711b6f 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/evals.json +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/evals.json @@ -2,40 +2,134 @@ "skill_name": "protein-binder-design", "evals": [ { - "id": "1", - "prompt": "Design 50 de novo mini-binders against PD-L1 (use the target registry entry) at its defined interface epitope. Generate backbones, design 8 sequences each, co-fold the shortlist against the target, and rank by interface confidence. Save a run manifest and a ranked CSV.", - "expected_output": "An orchestration that composes the rfdiffusion-nim, proteinmpnn-nim, and boltz2-nim/openfold3-nim skills in order, remaps epitope residues, writes a run manifest via scripts/manifest.py, applies the ipTM/pLDDT/RMSD filter, and exports a ranked candidates.csv.", - "files": [], - "assertions": [ - {"id": "composes-rfdiffusion", "description": "Uses RFdiffusion for binder backbones", "check": "Plan invokes the rfdiffusion-nim skill with a binder contig and hotspot_res"}, - {"id": "composes-proteinmpnn", "description": "Uses ProteinMPNN for sequence design", "check": "Plan invokes the proteinmpnn-nim skill and drops the native/WT row from mfasta"}, - {"id": "composes-cofold", "description": "Co-folds binder with target", "check": "Plan invokes boltz2-nim or openfold3-nim to co-fold the binder with the target and collects ipTM or confidence"}, - {"id": "writes-manifest", "description": "Writes a run manifest", "check": "Plan uses scripts/manifest.py (Manifest.create, set_scores, apply_filters, to_csv)"}, - {"id": "applies-filter", "description": "Applies default filter thresholds", "check": "Plan filters on ipTM >= 0.8, binder pLDDT >= 80, self-consistency RMSD <= 2.0"}, - {"id": "remaps-epitope", "description": "Remaps epitope residues", "check": "Plan remaps epitope author numbering using scripts/pdb_utils.py before use"} + "id": "offline-manifest", + "prompt": "Run an offline software bookkeeping exercise using /workspace/input/bookkeeping.json. All scores, controls, and thresholds in this file are synthetic; do not call model APIs or describe these numbers as scientific predictions. For each profile, create a separate run directory and manifest using the supplied target, filters, ranking metric, candidates, scores, and control flags. Use the bundled manifest helper if available, or implement equivalent bookkeeping. Save and reload each manifest, apply the supplied filters, exclude controls from rankings and success counts, and export both all_candidates.csv and ranked candidates.csv. Show the resulting counts and ranked IDs for each profile. Leave absent scores absent, including ipTM for the Boltz2 profile. Label all results as simulated bookkeeping, disclose the pending candidate, and explain that these results provide no live inference coverage.", + "expected_output": "Two saved and reloaded manifests and two CSVs per profile. Each profile has four non-control candidates, two passing candidates, and one control. Both rankings contain best then boundary. Below and pending fail; the control is excluded from rankings and counts. The Boltz2 profile retains boltz2_confidence without inventing ipTM. The simulated 2/4 passing fraction is provisional because pending lacks required scores. No model calls or scientific validation claims.", + "files": [ + "evals/files/bookkeeping.json" + ], + "assertions": [ + "[executes-bookkeeping] Executes the bookkeeping and produces a manifest and both CSV artifacts per profile; a proposed plan alone does not pass.", + "[resumes-state] Saves and reloads each profile without changing the supplied scores, filters, rank_by metric, or control flags.", + "[filters-missing-scores] Each profile reports n_candidates=4, n_passed=2, n_controls=1; below and pending fail.", + "[ranks-and-exports] Each ranked candidates.csv contains best then boundary, excludes the control, and all_candidates.csv retains all five entries.", + "[separates-metrics] Boltz2 is filtered and ranked by boltz2_confidence, with iptm_min disabled and no invented ipTM score.", + "[states-coverage] Labels every result as simulated offline bookkeeping, discloses the incomplete candidate and provisional 2/4 ratio, and makes no live inference or scientific validation claim." + ] + }, + { + "id": "offline-boltz2-ranking", + "prompt": "Review my protein binder campaign in /workspace/input/boltz2_only.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/boltz2-only. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "expected_output": "The supplied composite-confidence cutoff is 0.73; ipTM remains disabled and absent. Ranked IDs are binder_a then binder_b. Counts are n_candidates=4, n_passed=2, n_controls=1; the audit retains all five entries. The sequence-only control is exempt only from RMSD. Pending lacks RMSD and low_confidence fails. The 2/4 fraction is provisional and synthetic.", + "files": [ + "evals/files/boltz2_only.json" + ], + "assertions": [ + "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/boltz2-only; a plan alone does not pass.", + "[separate-metrics] Keeps the supplied 0.73 confidence cutoff, disabled ipTM and enabled pLDDT/RMSD filters; does not invent ipTM or rename confidence to ipTM.", + "[ranked-artifact] candidates.csv contains exactly binder_a then binder_b, excluding the higher-scoring control and pending.", + "[complete-audit] The manifest and all_candidates.csv retain all five entries; reports n_candidates=4, n_passed=2, n_controls=1.", + "[control-applicability] The sequence-only control has no invented RMSD or backbone and is exempt only from RMSD; its confidence and pLDDT still pass the supplied filters.", + "[honest-coverage] Discloses pending's missing RMSD and a provisional 2/4 fraction, labels scores synthetic, and makes no live inference or scientific validation claim." + ] + }, + { + "id": "offline-mixed-threshold-ranking", + "prompt": "Review my protein binder campaign in /workspace/input/mixed_thresholds.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/mixed-thresholds. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "expected_output": "All supplied thresholds are enforced. Ranked IDs are binder_a then binder_b, despite higher ipTM on the control and designs failing other criteria. Counts are n_candidates=6, n_passed=2, n_controls=1; all seven entries remain in the audit. The string-valued interface score stays invalid rather than being coerced; pending lacks RMSD. Results are synthetic and 2/6 is provisional.", + "files": [ + "evals/files/mixed_thresholds.json" + ], + "assertions": [ + "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/mixed-thresholds; a plan alone does not pass.", + "[all-thresholds] Keeps every supplied filter; high_confidence_low_plddt and high_rmsd fail despite higher ipTM than selected designs.", + "[ranked-artifact] candidates.csv contains exactly binder_a then binder_b in descending numeric ipTM order and excludes controls, failures and incomplete designs.", + "[complete-audit] The manifest and all_candidates.csv retain all seven entries; reports n_candidates=6, n_passed=2, n_controls=1.", + "[invalid-and-missing] Preserves the string score as supplied and rejects it as invalid; pending is incomplete. Does not invent, coerce, or repair scores.", + "[honest-coverage] Reports a provisional 2/6 fraction because scoring is incomplete and labels the outputs synthetic offline bookkeeping, with no live inference claim." + ] + }, + { + "id": "offline-empty-ranking", + "prompt": "Review my protein binder campaign in /workspace/input/no_survivors.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/no-survivors. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "expected_output": "No design passes every applicable filter. candidates.csv has a header and zero data rows even though the synthetic control passes. Counts are n_candidates=4, n_passed=0, n_controls=1; all five entries remain in the audit and manifest. Pending lacks pLDDT and RMSD. The 0/4 fraction is provisional; thresholds are not relaxed to manufacture a shortlist.", + "files": [ + "evals/files/no_survivors.json" + ], + "assertions": [ + "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/no-survivors; a plan alone does not pass.", + "[empty-ranked-artifact] candidates.csv is header-only with zero data rows; the file is not omitted and there is no fallback to all candidates.", + "[control-exclusion] Excludes the passing control from the ranked design shortlist and design/pass counts.", + "[complete-audit] The manifest and all_candidates.csv retain all five entries; reports n_candidates=4, n_passed=0, n_controls=1.", + "[preserves-policy] Keeps all supplied scores, flags and thresholds; does not relax criteria to obtain a nonempty shortlist.", + "[honest-coverage] Discloses pending's missing pLDDT and RMSD, a provisional 0/4 fraction, and synthetic offline coverage; no live inference or scientific validation claim." + ] + }, + { + "id": "offline-target-preparation", + "prompt": "Prepare the supplied target for a binder workflow from /workspace/input/target_prep/request.json. Follow /workspace/input/target_prep/CONTRACT.md and write the preparation artifacts under /workspace/output/target-prep. Resolve each requested hotspot handoff independently and report which can proceed. Work offline using these synthetic inputs; do not run inference.", + "files": [ + "evals/files/target_prep/" + ], + "expected_output": "Target chain E from model 1 gives AGST, author map 10:1,42:2,42A:3,77:4. Present hotspots produce E42,E77 and sequence indices 2,4. Missing author 43 blocks that request without a nearest-residue guess. Target PDB retains chain E and author numbering.", + "assertions": [ + "[structure-selection] Uses chain E, first model and one alternate CA per residue.", + "[author-map] Preserves 42 versus 42A and sparse numbering in the map and saved PDB.", + "[handoff] Saves present hotspots as E42/E77 and sequence indices 2/4.", + "[blocked-input] Missing 43 blocks only its request; no inferred replacement or partial payload.", + "[artifacts] Writes target.pdb and preparation.json with the specified fields.", + "[coverage] Discloses synthetic offline coverage and makes no inference claim." + ] + }, + { + "id": "offline-refold-extraction", + "prompt": "Reconcile the saved refold results under /workspace/input/raw_refolds/ into two output runs /workspace/output/raw-refolds/boltz2 and /workspace/output/raw-refolds/openfold3. Follow /workspace/input/cache_contract.md, preserve the campaign policies, and report the resulting rankings and any blocked candidates. The responses are synthetic offline fixtures; make no model calls.", + "files": [ + "evals/files/raw_refolds/", + "evals/files/cache_contract.md" + ], + "expected_output": "Both rankings contain selected only. Boltz2 selected uses sample 1 confidence 0.81 and binder pLDDT 82; ipTM absent. OpenFold3 selected uses sample 1 ipTM 0.83 and binder pLDDT 86. poor_binder fails pLDDT 60 in both. Boltz2 incomplete is blocked for length mismatch; OpenFold3 composite_only is blocked for missing explicit ipTM. Counts 3/1/1 and 3/1/0, respectively; both fractions provisional.", + "assertions": [ + "[sample-binding] Scores and saved complex use the recorded sample, not the highest confidence sample.", + "[chain-confidence] Uses binder CA confidence with explicit scale and mmCIF label IDs, not target or whole-complex means.", + "[metric-separation] Does not turn a Boltz2 composite or OpenFold3 confidence_score into ipTM.", + "[partial-response] Blocks incomplete chains and absent required metrics without partial score imports.", + "[saved-results] Writes both reloaded manifests, audit/ranked CSVs and summaries with correct counts and provenance.", + "[coverage] Reports simulated offline results, unresolved candidates and provisional fractions without model calls." ] }, { - "id": "2", - "prompt": "For my binder campaign against PD-L1, include proper controls and tell me the success rate.", - "expected_output": "Adds scrambled-sequence negative controls and published-binder positive controls through the same co-folding pipeline, marks them is_control in the manifest, and reports success rate as n_passed / n_candidates.", - "files": [], + "id": "offline-cache-resume", + "prompt": "Finish the recoverable offline work in /workspace/input/resume_cache/manifest.json and write the resumed run under /workspace/output/resumed. Use /workspace/input/cache_contract.md. Preserve the original run and its completed results and failure history, reconcile available cached responses, and replace stale exports. Report what remains blocked. All inputs are synthetic; do not submit inference or retries.", + "files": [ + "evals/files/resume_cache/", + "evals/files/cache_contract.md" + ], + "expected_output": "done and below are complete and unchanged; a finite low score is not pending. recoverable retains ipTM 0.86, gains binder pLDDT 84 and aligned RMSD approximately zero from cache. missing and failed stay blocked without invented metrics. Failure history and existing artifacts remain unchanged. Counts 5/2/1; ranked IDs done then recoverable; stale control/missing rows removed; 2/5 provisional.", "assertions": [ - {"id": "negative-controls", "description": "Generates scrambled negative controls", "check": "Plan uses scripts/controls.py to scramble binder sequences and marks them is_control with control_type scrambled"}, - {"id": "positive-controls", "description": "Uses published binders as positive controls", "check": "Plan re-scores published binders from the registry through the same pipeline as positive controls"}, - {"id": "success-rate", "description": "Reports a success rate", "check": "Plan reports success rate as n_passed divided by n_candidates from manifest.summary()"}, - {"id": "same-pipeline", "description": "Controls run through identical pipeline", "check": "Controls are co-folded and scored with the same settings as the designs"} + "[resume-preservation] Keeps complete passing and failing scores, artifact bytes and historical failed stages.", + "[cache-recovery] Recovers only missing metrics; existing ipTM 0.86 is not replaced by raw 0.98.", + "[geometry] Aligns explicit full binder chains to obtain RMSD near zero.", + "[blocked-work] Records missing_artifact and missing_prediction without retries or fabricated scores.", + "[exports] Replaces stale ranking with done/recoverable; counts 5/2/1 and full audit retained.", + "[coverage] Reports pending work and provisional simulated 2/5 with no live inference coverage." ] }, { - "id": "3", - "prompt": "Continue my interrupted binder run in runs/rbd_binders and only process candidates that are missing scores.", - "expected_output": "Loads the existing manifest with Manifest.load, identifies candidates lacking the needed scores/artifacts, and resumes without re-running completed stages.", - "files": [], + "id": "offline-provenance-rejection", + "prompt": "Audit and reconcile /workspace/input/provenance_cache/manifest.json under /workspace/output/provenance. Follow /workspace/input/cache_contract.md. Decide which saved responses can supply scores to their candidates and produce the manifest, exports, summary and rejection reasons. Preserve the supplied evidence. These are synthetic offline fixtures; do not call models or repair provenance by guessing.", + "files": [ + "evals/files/provenance_cache/", + "evals/files/cache_contract.md" + ], + "expected_output": "Only valid is recovered and ranked, with boltz2_confidence 0.9 and binder_plddt 84. Reject candidate_swap, target_swap, sequence_swap, sample_swap and response_tampered for their respective candidate/target/sequence/selected-sample/file hash mismatch; leave all their scores empty and retain original evidence. Counts 6/1/0; 1/6 provisional; no ipTM invented.", "assertions": [ - {"id": "loads-manifest", "description": "Loads the existing manifest", "check": "Plan calls Manifest.load on the run directory"}, - {"id": "resumes-incomplete", "description": "Skips completed candidates", "check": "Plan only processes candidates missing required scores or artifacts"}, - {"id": "no-recompute", "description": "Does not recompute finished stages", "check": "Plan avoids re-running stages already recorded in the manifest"} + "[identity] Checks candidate and sequence identity before accepting scores.", + "[target] Rejects a response binding to another target digest.", + "[artifact-integrity] Checks response bytes and selected-complex digest; does not rewrite bindings to fit cached data.", + "[fail-closed] Leaves rejected candidates unscored and present in the audit with specific reasons.", + "[exports] Ranks only valid and records counts 6/1/0 with source hashes/sample provenance.", + "[coverage] Reports provisional synthetic results and no live inference coverage." ] } ] diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/boltz2_only.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/boltz2_only.json new file mode 100644 index 00000000..87c42e22 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/boltz2_only.json @@ -0,0 +1,64 @@ +{ + "provenance": "Synthetic software bookkeeping fixture. Scores and thresholds are illustrative, not predictions or scientific validation. No biological sequences or live inference are represented.", + "target": { + "name": "synthetic ranking fixture" + }, + "profiles": [ + { + "name": "boltz2-only", + "rank_by": "boltz2_confidence", + "filters": { + "iptm_min": null, + "boltz2_confidence_min": 0.73, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2 + }, + "candidates": [ + { + "id": "binder_b", + "scores": { + "boltz2_confidence": 0.77, + "binder_plddt": 89, + "self_consistency_rmsd": 1.2 + }, + "is_control": false + }, + { + "id": "control", + "scores": { + "boltz2_confidence": 0.99, + "binder_plddt": 99 + }, + "is_control": true, + "control_type": "scrambled" + }, + { + "id": "binder_a", + "scores": { + "boltz2_confidence": 0.89, + "binder_plddt": 86, + "self_consistency_rmsd": 1.4 + }, + "is_control": false + }, + { + "id": "low_confidence", + "scores": { + "boltz2_confidence": 0.729, + "binder_plddt": 90, + "self_consistency_rmsd": 1 + }, + "is_control": false + }, + { + "id": "pending", + "scores": { + "boltz2_confidence": 0.95, + "binder_plddt": 90 + }, + "is_control": false + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/bookkeeping.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/bookkeeping.json new file mode 100644 index 00000000..43177c5a --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/bookkeeping.json @@ -0,0 +1,125 @@ +{ + "provenance": "Synthetic software bookkeeping fixture. Scores and thresholds are illustrative, not predictions or scientific validation. No target structure, biological sequences, model output, or endpoint access is represented.", + "target": { + "name": "synthetic bookkeeping fixture" + }, + "profiles": [ + { + "name": "iptm", + "rank_by": "iptm", + "filters": { + "iptm_min": 0.8, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2.0 + }, + "candidates": [ + { + "id": "boundary", + "scores": { + "iptm": 0.8, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "below", + "scores": { + "iptm": 0.79, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "best", + "scores": { + "iptm": 0.9, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "control", + "scores": { + "iptm": 0.99, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": true, + "control_type": "scrambled" + }, + { + "id": "pending", + "scores": { + "iptm": 0.95 + }, + "is_control": false + } + ] + }, + { + "name": "boltz2", + "rank_by": "boltz2_confidence", + "filters": { + "iptm_min": null, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2.0, + "boltz2_confidence_min": 0.8 + }, + "candidates": [ + { + "id": "boundary", + "scores": { + "boltz2_confidence": 0.8, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "below", + "scores": { + "boltz2_confidence": 0.79, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "best", + "scores": { + "boltz2_confidence": 0.9, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "control", + "scores": { + "boltz2_confidence": 0.99, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": true, + "control_type": "scrambled" + }, + { + "id": "pending", + "scores": { + "boltz2_confidence": 0.95 + }, + "is_control": false + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/cache_contract.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/cache_contract.md new file mode 100644 index 00000000..7dabe34d --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/cache_contract.md @@ -0,0 +1,42 @@ +# Saved-response fixture contract +All sequences, coordinates, responses, scores, and policies here are synthetic. Work offline; +do not read credentials, call endpoints, or claim live inference or scientific validation. +This local cache metadata is a fixture/campaign format, not an extra field promised by a NIM API. + +Copy each input run to its required output directory before modifying it. Preserve input files, +original candidates, target, params, filters, existing finite scores, artifacts, and failed stages. +Relative paths refer to that run directory. Save and reload the resulting manifest, apply every +enabled filter, and export all_candidates.csv (all entries) and candidates.csv (passing designs +in descending rank_by order, excluding controls). Empty rankings retain a header. + +For each candidate, record cache_status as complete, recovered, or blocked. Complete means all +applicable enabled scores are finite (even if below a threshold); do not recompute or replace them. +A sequence-only control without a backbone is exempt only from RMSD. For incomplete candidates, +validate the prediction binding before importing any scores: candidate ID, SHA-256 of the exact +sequence string, target.structure bytes, response file bytes, and the selected structure text. +All digests use UTF-8 without newline normalization. Copy the complete prediction binding into +score_provenance when recovering. Keep all prediction fields as supplied. A mismatch blocks the +whole import; preserve any existing scores. Never select a different sample to get a better score. + +Boltz2: structures[sample_index] pairs with confidence_scores[sample_index], in +boltz2_confidence only; the arrays must have equal length. No ipTM substitution. +OpenFold3: choose outputs by matching input_id, then structures_with_scores[sample_index]; +only iptm_score supplies ipTM. confidence_score and complex_plddt_score do not supply it. +The selected structure's binder-chain CA B factors encode per-residue pLDDT here, with the +explicit prediction.plddt_scale (0-1 or 0-100). Average the binder only and report 0-100. +Use mmCIF label_asym_id, not auth_asym_id, and require both chain roles and the full binder +length. These fixture encodings do not establish B-factor semantics for arbitrary experimental PDBs. +Recover missing RMSD by aligning the full backbone binder chain and full selected predicted +binder chain with a proper rigid rotation (Kabsch); never truncate to match lengths. + +Import only missing/invalid enabled metrics and only when all of them are recoverable. Save +the exact selected complex under the output run and add artifacts.complex with its relative path. +If blocked, leave scores unchanged and set cache_reason to candidate_mismatch, target_mismatch, +sequence_mismatch, artifact_digest_mismatch, sample_digest_mismatch, binder_length_mismatch, +missing_required_scores, missing_artifact, or missing_prediction as applicable. No API retry. +Existing failure_reason and stage entries remain as historical evidence, even after recovery. + +Write summary.json with n_candidates (non-controls), n_passed, n_controls, ranked_ids, +missing_scores (IDs to missing/invalid required metric names), passing_fraction_provisional, +label containing 'simulated', and live_inference: false. Describe unresolved candidates and +provisional fractions in the final response. A count or score alone is not evidence of execution. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/mixed_thresholds.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/mixed_thresholds.json new file mode 100644 index 00000000..990dc63f --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/mixed_thresholds.json @@ -0,0 +1,81 @@ +{ + "provenance": "Synthetic software bookkeeping fixture. Scores and thresholds are illustrative, not predictions or scientific validation. No biological sequences or live inference are represented.", + "target": { + "name": "synthetic ranking fixture" + }, + "profiles": [ + { + "name": "mixed-thresholds", + "rank_by": "iptm", + "filters": { + "iptm_min": 0.8, + "binder_plddt_min": 85, + "self_consistency_rmsd_max": 1.5 + }, + "candidates": [ + { + "id": "binder_b", + "scores": { + "iptm": 0.84, + "binder_plddt": 90, + "self_consistency_rmsd": 1 + }, + "is_control": false + }, + { + "id": "high_confidence_low_plddt", + "scores": { + "iptm": 0.99, + "binder_plddt": 84, + "self_consistency_rmsd": 1 + }, + "is_control": false + }, + { + "id": "binder_a", + "scores": { + "iptm": 0.87, + "binder_plddt": 90, + "self_consistency_rmsd": 1.2 + }, + "is_control": false + }, + { + "id": "high_rmsd", + "scores": { + "iptm": 0.98, + "binder_plddt": 95, + "self_consistency_rmsd": 1.51 + }, + "is_control": false + }, + { + "id": "pending", + "scores": { + "iptm": 0.96, + "binder_plddt": 90 + }, + "is_control": false + }, + { + "id": "control", + "scores": { + "iptm": 0.999, + "binder_plddt": 99 + }, + "is_control": true, + "control_type": "scrambled" + }, + { + "id": "invalid_score", + "scores": { + "iptm": "0.95", + "binder_plddt": 90, + "self_consistency_rmsd": 1 + }, + "is_control": false + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/no_survivors.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/no_survivors.json new file mode 100644 index 00000000..93edda5d --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/no_survivors.json @@ -0,0 +1,62 @@ +{ + "provenance": "Synthetic software bookkeeping fixture. Scores and thresholds are illustrative, not predictions or scientific validation. No biological sequences or live inference are represented.", + "target": { + "name": "synthetic ranking fixture" + }, + "profiles": [ + { + "name": "no-survivors", + "rank_by": "iptm", + "filters": { + "iptm_min": 0.8, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2 + }, + "candidates": [ + { + "id": "control", + "scores": { + "iptm": 0.99, + "binder_plddt": 99 + }, + "is_control": true, + "control_type": "scrambled" + }, + { + "id": "below", + "scores": { + "iptm": 0.79, + "binder_plddt": 90, + "self_consistency_rmsd": 1 + }, + "is_control": false + }, + { + "id": "pending", + "scores": { + "iptm": 0.95 + }, + "is_control": false + }, + { + "id": "low_plddt", + "scores": { + "iptm": 0.91, + "binder_plddt": 79, + "self_consistency_rmsd": 1 + }, + "is_control": false + }, + { + "id": "high_rmsd", + "scores": { + "iptm": 0.92, + "binder_plddt": 96, + "self_consistency_rmsd": 2.01 + }, + "is_control": false + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/backbone.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/backbone.pdb new file mode 100644 index 00000000..3f1e5d16 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/backbone.pdb @@ -0,0 +1,4 @@ +ATOM 1 CA ALA D 1 0.000 0.000 0.000 1.00 80.00 C +ATOM 2 CA ALA D 2 4.000 0.000 0.000 1.00 80.00 C +ATOM 3 CA ALA D 3 0.000 3.000 0.000 1.00 80.00 C +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/manifest.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/manifest.json new file mode 100644 index 00000000..7e34d60a --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/manifest.json @@ -0,0 +1,212 @@ +{ + "schema_version": "1.0", + "campaign": "protein-binder-design", + "mode": "offline-bookkeeping", + "run_dir": ".", + "provenance": "Synthetic offline software fixture. Toy coordinates and scores are invented; no live inference or biological validity.", + "target": { + "name": "synthetic target", + "structure": { + "path": "target.pdb", + "sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10" + } + }, + "params": { + "rank_by": "boltz2_confidence" + }, + "filters": { + "iptm_min": null, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": null, + "boltz2_confidence_min": 0.8 + }, + "stages": [ + { + "stage": "cofold", + "status": "interrupted", + "attempts": 1 + } + ], + "candidates": [ + { + "id": "valid", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "valid", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/valid.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "0c8bd86d48741f1b39a19688a25602a7a7c7dfdcea6b7154100f70895028adeb", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "candidate_swap", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "valid", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/candidate_swap.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "0c8bd86d48741f1b39a19688a25602a7a7c7dfdcea6b7154100f70895028adeb", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "target_swap", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "target_swap", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "response": { + "path": "responses/target_swap.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "0c8bd86d48741f1b39a19688a25602a7a7c7dfdcea6b7154100f70895028adeb", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "sequence_swap", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "sequence_swap", + "sequence_sha256": "4c573d7486ff3e432b9fcd07d5558134dfd4963eb541ea4ad688d42870548dfb", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/sequence_swap.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "0c8bd86d48741f1b39a19688a25602a7a7c7dfdcea6b7154100f70895028adeb", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "sample_swap", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "sample_swap", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/sample_swap.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "response_tampered", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "response_tampered", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/response_tampered.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "0c8bd86d48741f1b39a19688a25602a7a7c7dfdcea6b7154100f70895028adeb", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/candidate_swap.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/candidate_swap.json new file mode 100644 index 00000000..d00fb2cd --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/candidate_swap.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 82.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 84.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 86.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.9 + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/response_tampered.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/response_tampered.json new file mode 100644 index 00000000..8d7314a1 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/response_tampered.json @@ -0,0 +1 @@ +{"tampered": true} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/sample_swap.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/sample_swap.json new file mode 100644 index 00000000..d00fb2cd --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/sample_swap.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 82.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 84.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 86.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.9 + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/sequence_swap.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/sequence_swap.json new file mode 100644 index 00000000..d00fb2cd --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/sequence_swap.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 82.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 84.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 86.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.9 + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/target_swap.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/target_swap.json new file mode 100644 index 00000000..d00fb2cd --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/target_swap.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 82.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 84.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 86.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.9 + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/valid.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/valid.json new file mode 100644 index 00000000..d00fb2cd --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/responses/valid.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 82.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 84.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 86.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.9 + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/target.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/target.pdb new file mode 100644 index 00000000..b28e2f70 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/provenance_cache/target.pdb @@ -0,0 +1,7 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 80.00 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 80.00 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 80.00 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 80.00 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 80.00 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 80.00 C +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/backbone.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/backbone.pdb new file mode 100644 index 00000000..3f1e5d16 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/backbone.pdb @@ -0,0 +1,4 @@ +ATOM 1 CA ALA D 1 0.000 0.000 0.000 1.00 80.00 C +ATOM 2 CA ALA D 2 4.000 0.000 0.000 1.00 80.00 C +ATOM 3 CA ALA D 3 0.000 3.000 0.000 1.00 80.00 C +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/manifest.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/manifest.json new file mode 100644 index 00000000..9195aa1d --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/manifest.json @@ -0,0 +1,150 @@ +{ + "schema_version": "1.0", + "campaign": "protein-binder-design", + "mode": "offline-bookkeeping", + "run_dir": ".", + "provenance": "Synthetic offline software fixture. Toy coordinates and scores are invented; no live inference or biological validity.", + "target": { + "name": "synthetic target", + "structure": { + "path": "target.pdb", + "sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10" + } + }, + "params": { + "rank_by": "boltz2_confidence" + }, + "filters": { + "iptm_min": null, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": null, + "boltz2_confidence_min": 0.8 + }, + "stages": [ + { + "stage": "cofold", + "status": "interrupted", + "attempts": 1 + } + ], + "candidates": [ + { + "id": "selected", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "selected", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/selected.json", + "sha256": "fb0fc1994076830ceeea1d8dd2d40485a6e8658b22ff289a9e676bb480c53a2c" + }, + "sample_index": 1, + "complex_sha256": "f33530439320b437faa35521d3f526d04383f945244dfa6845ca1ee9450e2f3c", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "poor_binder", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "poor_binder", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/poor_binder.json", + "sha256": "45049bc8801df9bf9b763ec0e4539f6ce0f4828f287c326e1b657e006c084339" + }, + "sample_index": 0, + "complex_sha256": "2f65e462aa25e26b29c9ef1941287e8e7336440316dbf18e7c88e3ea6a6e885a", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "incomplete", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "incomplete", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/incomplete.json", + "sha256": "c23989c01579afb22e2321e24c0b10daea5ad1357cca6af74146678d5d9d9f11" + }, + "sample_index": 0, + "complex_sha256": "5657f98f6a7305a50b0c26579ea25457608cf47a987b704ad4d093899d5a6785", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "control", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": true, + "artifacts": {}, + "prediction": { + "provider": "boltz2", + "candidate_id": "control", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/control.json", + "sha256": "d3b2430f50a5e20c5af3d4b11cdc5a2d59c7717d342fedef98220e8fbd5d5ae5" + }, + "sample_index": 0, + "complex_sha256": "464735ea62f21ebc8ae58331773d6dcb85af7bfb6977999451daf066a83328e1", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/control.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/control.json new file mode 100644 index 00000000..c4d26264 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/control.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 99.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 99.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 99.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.999 + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/incomplete.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/incomplete.json new file mode 100644 index 00000000..2509f504 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/incomplete.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 85.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 85.0 2 8.0 11.0 0.0 1 8\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.95 + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/poor_binder.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/poor_binder.json new file mode 100644 index 00000000..48b3dfd4 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/poor_binder.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 58.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 60.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 62.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.99 + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/selected.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/selected.json new file mode 100644 index 00000000..5a5236f7 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/responses/selected.json @@ -0,0 +1,16 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 99.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 99.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 99.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + }, + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 80.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 82.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 84.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.99, + 0.81 + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/target.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/target.pdb new file mode 100644 index 00000000..b28e2f70 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/boltz2/target.pdb @@ -0,0 +1,7 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 80.00 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 80.00 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 80.00 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 80.00 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 80.00 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 80.00 C +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/backbone.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/backbone.pdb new file mode 100644 index 00000000..3f1e5d16 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/backbone.pdb @@ -0,0 +1,4 @@ +ATOM 1 CA ALA D 1 0.000 0.000 0.000 1.00 80.00 C +ATOM 2 CA ALA D 2 4.000 0.000 0.000 1.00 80.00 C +ATOM 3 CA ALA D 3 0.000 3.000 0.000 1.00 80.00 C +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/manifest.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/manifest.json new file mode 100644 index 00000000..d461f70a --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/manifest.json @@ -0,0 +1,121 @@ +{ + "schema_version": "1.0", + "campaign": "protein-binder-design", + "mode": "offline-bookkeeping", + "run_dir": ".", + "provenance": "Synthetic offline software fixture. Toy coordinates and scores are invented; no live inference or biological validity.", + "target": { + "name": "synthetic target", + "structure": { + "path": "target.pdb", + "sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10" + } + }, + "params": { + "rank_by": "iptm" + }, + "filters": { + "iptm_min": 0.8, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": null + }, + "stages": [ + { + "stage": "cofold", + "status": "interrupted", + "attempts": 1 + } + ], + "candidates": [ + { + "id": "selected", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "selected", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/selected.json", + "sha256": "605dd0759e2642108810f6a88baef0f12427a86f2e5acfe3349f19d9a4b7399b" + }, + "sample_index": 1, + "complex_sha256": "a0b0b48b1a155ac4a12fc720db0e7d18afb08052035a84d60abc1e98999cf355", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "composite_only", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "composite_only", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/composite_only.json", + "sha256": "5123c9b20050bc54cdb559537b2ffc32d3d630d8683af02746bad5a234a133c4" + }, + "sample_index": 0, + "complex_sha256": "2c33a2375e2be4a7ffe87376f45952734d46ce9e03209a6cfe363366cb8cf045", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "poor_binder", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "poor_binder", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/poor_binder.json", + "sha256": "119c9a8719156a0d169d9258bc52dabe4ad497fffdfa0ab0c3497bc1dec32fb1" + }, + "sample_index": 0, + "complex_sha256": "3bc1ef048158c2b026eb491a4e916b2d9fa5fb0d8571f0fe7679b4aff5437f95", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/composite_only.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/composite_only.json new file mode 100644 index 00000000..85a99fa2 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/composite_only.json @@ -0,0 +1,15 @@ +{ + "outputs": [ + { + "input_id": "composite_only", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.90 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.90 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.90 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97 + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/poor_binder.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/poor_binder.json new file mode 100644 index 00000000..b8cd92e2 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/poor_binder.json @@ -0,0 +1,16 @@ +{ + "outputs": [ + { + "input_id": "poor_binder", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.58 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.60 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.62 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.99 + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/selected.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/selected.json new file mode 100644 index 00000000..1a354376 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/responses/selected.json @@ -0,0 +1,23 @@ +{ + "outputs": [ + { + "input_id": "selected", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.99 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.99 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.99 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.99 + }, + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.84 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.86 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.88 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.83 + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/target.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/target.pdb new file mode 100644 index 00000000..b28e2f70 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/raw_refolds/openfold3/target.pdb @@ -0,0 +1,7 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 80.00 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 80.00 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 80.00 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 80.00 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 80.00 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 80.00 C +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/backbone.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/backbone.pdb new file mode 100644 index 00000000..3f1e5d16 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/backbone.pdb @@ -0,0 +1,4 @@ +ATOM 1 CA ALA D 1 0.000 0.000 0.000 1.00 80.00 C +ATOM 2 CA ALA D 2 4.000 0.000 0.000 1.00 80.00 C +ATOM 3 CA ALA D 3 0.000 3.000 0.000 1.00 80.00 C +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/below.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/below.pdb new file mode 100644 index 00000000..c1059fc9 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/below.pdb @@ -0,0 +1,10 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C +ATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C +ATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C +ATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/candidates.csv b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/candidates.csv new file mode 100644 index 00000000..f9c62572 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/candidates.csv @@ -0,0 +1,3 @@ +id,iptm +control,0.99 +missing,1.0 diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/done.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/done.pdb new file mode 100644 index 00000000..c1059fc9 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/done.pdb @@ -0,0 +1,10 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C +ATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C +ATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C +ATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/manifest.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/manifest.json new file mode 100644 index 00000000..7e09e729 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/manifest.json @@ -0,0 +1,206 @@ +{ + "schema_version": "1.0", + "campaign": "protein-binder-design", + "mode": "offline-bookkeeping", + "run_dir": ".", + "provenance": "Synthetic offline software fixture. Toy coordinates and scores are invented; no live inference or biological validity.", + "target": { + "name": "synthetic target", + "structure": { + "path": "target.pdb", + "sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10" + } + }, + "params": { + "rank_by": "iptm" + }, + "filters": { + "iptm_min": 0.8, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2 + }, + "stages": [ + { + "stage": "cofold", + "status": "interrupted", + "attempts": 1 + } + ], + "candidates": [ + { + "id": "done", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": { + "iptm": 0.9, + "binder_plddt": 90, + "self_consistency_rmsd": 0 + }, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb", + "complex": "done.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "done", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/done.json", + "sha256": "94b16fe78c2cea20d2f2179d1a763feb98ec9de71e0222825cb2cc4bd1f1669f" + }, + "sample_index": 0, + "complex_sha256": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "below", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": { + "iptm": 0.7, + "binder_plddt": 85, + "self_consistency_rmsd": 0 + }, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb", + "complex": "below.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "below", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/below.json", + "sha256": "9486ff3d9be5100cacbf77d25a604c27db8caa89e3ee82d161056138ae45db0c" + }, + "sample_index": 0, + "complex_sha256": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "recoverable", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": { + "iptm": 0.86 + }, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "recoverable", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/recoverable.json", + "sha256": "214ee2ae319ad54656a6a3406ecacc571bee992c178881d61199d35c9f8d81af" + }, + "sample_index": 0, + "complex_sha256": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "missing", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "missing", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/missing.json", + "sha256": "5ed61eea6199299dba8ef59ffea5fa7fc23e0e54c81d1792a75551e09e7c8c8e" + }, + "sample_index": 0, + "complex_sha256": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "failed", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "failure_reason": "HTTP 503: retries exhausted", + "attempts": 3 + }, + { + "id": "control", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": { + "iptm": 0.99, + "binder_plddt": 88 + }, + "is_control": true, + "artifacts": {}, + "prediction": { + "provider": "openfold3", + "candidate_id": "control", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/control.json", + "sha256": "7c074850d88c42357c6983784e8c7d28dcc14fa728dc3b9ea278ee0167b14b94" + }, + "sample_index": 0, + "complex_sha256": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/below.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/below.json new file mode 100644 index 00000000..bc8a7909 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/below.json @@ -0,0 +1,16 @@ +{ + "outputs": [ + { + "input_id": "below", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.9 + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/control.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/control.json new file mode 100644 index 00000000..cbbe342e --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/control.json @@ -0,0 +1,16 @@ +{ + "outputs": [ + { + "input_id": "control", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.9 + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/done.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/done.json new file mode 100644 index 00000000..aaf3d163 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/done.json @@ -0,0 +1,16 @@ +{ + "outputs": [ + { + "input_id": "done", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.9 + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/recoverable.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/recoverable.json new file mode 100644 index 00000000..85b66a7b --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/responses/recoverable.json @@ -0,0 +1,16 @@ +{ + "outputs": [ + { + "input_id": "recoverable", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.98 + } + ] + } + ] +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/target.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/target.pdb new file mode 100644 index 00000000..b28e2f70 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/resume_cache/target.pdb @@ -0,0 +1,7 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 80.00 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 80.00 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 80.00 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 80.00 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 80.00 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 80.00 C +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/CONTRACT.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/CONTRACT.md new file mode 100644 index 00000000..6163462f --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/CONTRACT.md @@ -0,0 +1,11 @@ +# Offline target preparation contract +This is a synthetic software fixture. No API calls, inference, or biological validation. +Use model 1 and the chain in request.json. Keep author numbers and insertion codes distinct; +choose one CA per residue using the supplied alternate-location policy and preserve file order. +Write target.pdb (only the chosen chain/model, original author numbering) and preparation.json: +`chain`, `sequence`, `author_to_sequence` (string author ID to 1-based integer), and `requests` +(keyed by request ID). Each request has `status`: `ready` or `blocked`. A ready request has +`hotspot_res` (chain+author strings) and `sequence_indices`. A missing hotspot blocks that +request and records `missing_author_residues`; do not choose a nearby residue or emit its +partial handoff. Other requests can succeed independently. Add `simulated: true` and +`live_inference: false`. Do not renumber the PDB or combine alternate locations/models. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/request.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/request.json new file mode 100644 index 00000000..5de39eaa --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/request.json @@ -0,0 +1,22 @@ +{ + "provenance": "Synthetic toy PDB, no real target or inference.", + "structure": "target.pdb", + "chain": "E", + "requests": [ + { + "id": "present", + "hotspots": [ + "42", + "77" + ] + }, + { + "id": "missing", + "hotspots": [ + "43" + ] + } + ], + "model": 1, + "alternate_location_policy": "Prefer blank; otherwise highest occupancy; break ties by lexicographic altloc." +} diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/target.pdb b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/target.pdb new file mode 100644 index 00000000..f72ac714 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/target_prep/target.pdb @@ -0,0 +1,12 @@ +MODEL 1 +ATOM 1 CA ALA A 42 99.000 99.000 99.000 1.00 80.00 C +ATOM 2 CA ALA E 10 0.000 0.000 0.000 1.00 80.00 C +ATOM 3 CA AGLY E 42 9.000 9.000 9.000 0.20 80.00 C +ATOM 4 CA BGLY E 42 4.000 0.000 0.000 0.80 80.00 C +ATOM 5 CA SER E 42A 4.000 3.000 0.000 1.00 80.00 C +ATOM 6 CA THR E 77 8.000 3.000 0.000 1.00 80.00 C +ENDMDL +MODEL 2 +ATOM 7 CA VAL E 10 50.000 50.000 50.000 1.00 80.00 C +ENDMDL +END diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/grader.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/grader.py new file mode 100644 index 00000000..e2575799 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/grader.py @@ -0,0 +1,633 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Standalone artifact verifier; no agent helper imports, network or LLM calls. + +Harbor copies only this file to /tests. Input digests below pin the fixture truth; +an agent cannot change that truth by editing its input copies. ACES plus custom +keeps the ACES headline score: artifact_correctness is a separate binary metric. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import math +import os +from pathlib import Path + + +BOOKKEEPING = { + "offline-manifest": ("bookkeeping.json", None, ["best", "boundary"]), + "offline-boltz2-ranking": ( + "boltz2_only.json", + "boltz2-only", + ["binder_a", "binder_b"], + ), + "offline-mixed-threshold-ranking": ( + "mixed_thresholds.json", + "mixed-thresholds", + ["binder_a", "binder_b"], + ), + "offline-empty-ranking": ("no_survivors.json", "no-survivors", []), +} +# Hand-calculated outcomes, independent of the campaign helper and its parser. +CACHE_RUNS = { + "raw_refolds/boltz2": ( + "raw-refolds/boltz2", + ["selected"], + { + "selected": ( + {"boltz2_confidence": 0.81, "binder_plddt": 82}, + "recovered", + None, + ), + "poor_binder": ( + {"boltz2_confidence": 0.99, "binder_plddt": 60}, + "recovered", + None, + ), + "incomplete": ({}, "blocked", "binder_length_mismatch"), + "control": ( + {"boltz2_confidence": 0.999, "binder_plddt": 99}, + "recovered", + None, + ), + }, + ), + "raw_refolds/openfold3": ( + "raw-refolds/openfold3", + ["selected"], + { + "selected": ({"iptm": 0.83, "binder_plddt": 86}, "recovered", None), + "composite_only": ({}, "blocked", "missing_required_scores"), + "poor_binder": ({"iptm": 0.99, "binder_plddt": 60}, "recovered", None), + }, + ), + "resume_cache": ( + "resumed", + ["done", "recoverable"], + { + "done": ( + {"iptm": 0.9, "binder_plddt": 90, "self_consistency_rmsd": 0}, + "complete", + None, + ), + "below": ( + {"iptm": 0.7, "binder_plddt": 85, "self_consistency_rmsd": 0}, + "complete", + None, + ), + "recoverable": ( + {"iptm": 0.86, "binder_plddt": 84, "self_consistency_rmsd": 0}, + "recovered", + None, + ), + "missing": ({}, "blocked", "missing_artifact"), + "failed": ({}, "blocked", "missing_prediction"), + "control": ({"iptm": 0.99, "binder_plddt": 88}, "complete", None), + }, + ), + "provenance_cache": ( + "provenance", + ["valid"], + { + "valid": ( + {"boltz2_confidence": 0.9, "binder_plddt": 84}, + "recovered", + None, + ), + "candidate_swap": ({}, "blocked", "candidate_mismatch"), + "target_swap": ({}, "blocked", "target_mismatch"), + "sequence_swap": ({}, "blocked", "sequence_mismatch"), + "sample_swap": ({}, "blocked", "sample_digest_mismatch"), + "response_tampered": ({}, "blocked", "artifact_digest_mismatch"), + }, + ), +} +CACHE_CASES = { + "offline-refold-extraction": ["raw_refolds/boltz2", "raw_refolds/openfold3"], + "offline-cache-resume": ["resume_cache"], + "offline-provenance-rejection": ["provenance_cache"], +} + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def numeric(value): + return type(value) in (int, float) and math.isfinite(value) + + +def same(actual, expected, *, approximate=False): + if isinstance(expected, dict): + return ( + isinstance(actual, dict) + and actual.keys() == expected.keys() + and all( + same(actual[k], v, approximate=approximate) for k, v in expected.items() + ) + ) + if isinstance(expected, list): + return ( + isinstance(actual, list) + and len(actual) == len(expected) + and all( + same(a, e, approximate=approximate) for a, e in zip(actual, expected) + ) + ) + if numeric(expected): + return numeric(actual) and ( + math.isclose(actual, expected, rel_tol=1e-7, abs_tol=1e-6) + if approximate + else actual == expected + ) + return type(actual) is type(expected) and actual == expected + + +def read_json(path): + return json.loads(Path(path).read_text()) + + +def metrics(candidate, filters): + required = {} + for key, value in filters.items(): + if value is None: + continue + metric, direction = key.rsplit("_", 1) + if ( + metric == "self_consistency_rmsd" + and candidate.get("is_control") + and not candidate.get("artifacts", {}).get("backbone_pdb") + ): + continue + required[metric] = (direction, value) + return required + + +def passes(candidate, filters): + required = metrics(candidate, filters) + for key, (direction, cutoff) in required.items(): + value = candidate["scores"].get(key) + if not numeric(value) or ( + value < cutoff if direction == "min" else value > cutoff + ): + return False + return bool(required) + + +def check_csv(path, expected, ids, check, *, ordered, approximate=False): + with path.open(newline="") as stream: + reader = csv.DictReader(stream) + rows = list(reader) + headers = reader.fieldnames or [] + check(bool(headers) and "id" in headers, f"{path}: CSV header") + actual_ids = [row.get("id") for row in rows] + check( + actual_ids == ids if ordered else sorted(actual_ids) == sorted(ids), + f"{path}: candidate IDs", + ) + lookup = {c["id"]: c for c in expected} + score_keys = {key for c in expected for key in c["scores"]} + check(score_keys.issubset(headers), f"{path}: supplied score columns") + if "iptm" not in score_keys: + check( + "iptm" not in [key.lower() for key in headers], + f"{path}: absent ipTM stays absent", + ) + for row in rows: + if row.get("id") not in lookup: + continue + candidate = lookup[row["id"]] + for key in score_keys: + expected_value, actual = candidate["scores"].get(key), row.get(key) + if expected_value is None: + ok = actual == "" + elif numeric(expected_value): + try: + ok = same(float(actual), expected_value, approximate=approximate) + except (ValueError, TypeError): + ok = False + else: + ok = actual == str(expected_value) + check(ok, f"{path}: {row['id']}.{key}") + if "is_control" in headers: + check( + str(row["is_control"]).lower() + == str(candidate.get("is_control", False)).lower(), + f"{path}: {row['id']} control flag", + ) + + +def verify_run(path, source, expected, ranked_ids, check, *, cache=False): + data = read_json(path / "manifest.json") + check(same(data.get("target"), source["target"]), f"{path}: target") + check(same(data.get("filters"), source["filters"]), f"{path}: filters") + rank_by = data.get("params", {}).get("rank_by", data.get("rank_by")) + check(rank_by == source["params"]["rank_by"], f"{path}: rank metric") + actual = data.get("candidates", []) + check( + sorted(c["id"] for c in actual) == sorted(c["id"] for c in expected), + f"{path}: complete manifest", + ) + lookup = {c["id"]: c for c in actual} + for candidate in expected: + cid = candidate["id"] + record = lookup.get(cid, {}) + check( + same(record.get("scores"), candidate["scores"], approximate=cache), + f"{path}: {cid} score values/types/absence", + ) + check( + record.get("is_control", False) is candidate.get("is_control", False), + f"{path}: {cid} control flag", + ) + check( + record.get("control_type") == candidate.get("control_type"), + f"{path}: {cid} control type", + ) + for flag in ("passed_filter", "passed"): + if flag in record: + check( + record[flag] is passes(candidate, source["filters"]), + f"{path}: {cid} filter status", + ) + check_csv( + path / "all_candidates.csv", + expected, + [c["id"] for c in expected], + check, + ordered=False, + approximate=cache, + ) + # Older prompts used the ambiguous phrase "ranked candidates.csv". Accept + # the three literal interpretations without changing historical prompts. + alternatives = [ + path / name + for name in ("candidates.csv", "ranked candidates.csv", "ranked_candidates.csv") + ] + ranked_paths = [p for p in alternatives if p.exists()] + check(bool(ranked_paths), f"{path}: ranked CSV exists") + if cache: + check((path / "candidates.csv").exists(), f"{path}: required candidates.csv") + for ranked_path in ranked_paths: + check_csv( + ranked_path, expected, ranked_ids, check, ordered=True, approximate=cache + ) + return data, lookup + + +def find_manifests(workspace): + for root, dirs, names in os.walk(workspace): + dirs[:] = [ + d + for d in dirs + if d not in {"input", "skills", "node_modules"} and not d.startswith(".") + ] + if "manifest.json" in names: + yield Path(root) + + +def verify_bookkeeping(entry_id, workspace, check): + filename, destination, ranked = BOOKKEEPING[entry_id] + fixture = read_json(workspace / "input" / filename) + for profile in fixture["profiles"]: + source = { + "target": fixture["target"], + "filters": profile["filters"], + "params": {"rank_by": profile["rank_by"]}, + } + paths = ( + [workspace / "output" / destination] + if destination + else list(find_manifests(workspace)) + ) + attempts = [] + for path in paths: + checks = [] + try: + data = read_json(path / "manifest.json") + if ( + data.get("params", {}).get("rank_by", data.get("rank_by")) + != profile["rank_by"] + ): + continue + verify_run( + path, + source, + profile["candidates"], + ranked, + lambda ok, msg: checks.append((ok, msg)), + ) + except (OSError, ValueError, KeyError, TypeError) as exc: + checks.append((False, f"{path}: {type(exc).__name__}: {exc}")) + attempts.append(checks) + valid = next((a for a in attempts if a and all(ok for ok, _ in a)), None) + chosen = ( + valid + if valid is not None + else ( + max(attempts, key=lambda a: sum(ok for ok, _ in a)) if attempts else [] + ) + ) + check(bool(chosen), f"{profile['name']}: output manifest found") + for ok, msg in chosen: + check(ok, msg) + + +def verify_cache(name, workspace, check): + destination, ranked, outcomes = CACHE_RUNS[name] + source_root, output = workspace / "input" / name, workspace / "output" / destination + source = read_json(source_root / "manifest.json") + expected = [{**c, "scores": outcomes[c["id"]][0]} for c in source["candidates"]] + data, lookup = verify_run(output, source, expected, ranked, check, cache=True) + check( + same(data.get("params"), source["params"]), f"{output}: complete saved params" + ) + check( + same(data.get("stages", [])[: len(source["stages"])], source["stages"]), + f"{output}: stage history", + ) + for original in source["candidates"]: + cid = original["id"] + actual = lookup.get(cid, {}) + _, status, reason = outcomes[cid] + check(actual.get("cache_status") == status, f"{output}: {cid} decision") + if reason: + check( + actual.get("cache_reason") == reason, + f"{output}: {cid} rejection reason", + ) + for key, value in original.items(): + if key not in {"scores", "artifacts"}: + check(same(actual.get(key), value), f"{output}: {cid} preserves {key}") + for key, value in original["scores"].items(): + check( + same(actual.get("scores", {}).get(key), value), + f"{output}: {cid} preserves score {key}", + ) + for key, value in original.get("artifacts", {}).items(): + check( + actual.get("artifacts", {}).get(key) == value, + f"{output}: {cid} preserves artifact {key}", + ) + if status == "recovered": + binding = original["prediction"] + check( + same(actual.get("score_provenance"), binding), + f"{output}: {cid} score provenance", + ) + complex_path = ( + output / actual.get("artifacts", {}).get("complex", "") + ).resolve() + check( + complex_path.is_relative_to(output.resolve()) + and complex_path.is_file() + and sha(complex_path) == binding["complex_sha256"], + f"{output}: {cid} saved selected complex", + ) + # Saved evidence is copied without changing bytes, even when it is rejected. + for rel, digest in INPUT_HASHES.items(): + prefix = name + "/" + if rel.startswith(prefix) and rel[len(prefix) :] not in { + "manifest.json", + "candidates.csv", + }: + path = output / rel[len(prefix) :] + check( + path.is_file() and sha(path) == digest, + f"{path}: preserved evidence bytes", + ) + summary = read_json(output / "summary.json") + missing = { + c["id"]: [ + key + for key in metrics(c, source["filters"]) + if not numeric(c["scores"].get(key)) + ] + for c in expected + } + missing = {key: value for key, value in missing.items() if value} + required_summary = { + "n_candidates": sum(not c.get("is_control", False) for c in expected), + "n_passed": len(ranked), + "n_controls": sum(c.get("is_control", False) for c in expected), + "ranked_ids": ranked, + "passing_fraction_provisional": True, + "live_inference": False, + } + for key, value in required_summary.items(): + check(same(summary.get(key), value), f"{output}: summary {key}") + actual_missing = summary.get("missing_scores", {}) + check( + isinstance(actual_missing, dict) + and {k: sorted(v) for k, v in actual_missing.items()} + == {k: sorted(v) for k, v in missing.items()}, + f"{output}: missing scores disclosure", + ) + check( + "simulated" in str(summary.get("label", "")).lower(), + f"{output}: simulation label", + ) + + +def target_atoms(text): + """Read structural fields without depending on PDB writer formatting.""" + atoms = {} + explicit_model = False + in_model = False + ended = False + for line in text.splitlines(): + record = line[:6].strip() + if record == "MODEL": + if explicit_model or atoms or ended or int(line[10:14]) != 1: + raise ValueError("target must contain only the first model") + explicit_model = in_model = True + elif record == "ENDMDL": + if not in_model: + raise ValueError("unexpected ENDMDL in target") + in_model = False + elif record == "END": + ended = True + elif record in {"ATOM", "HETATM"}: + if ended or (explicit_model and not in_model): + raise ValueError("target atom outside its model") + key = (line[21], int(line[22:26]), line[26].strip(), line[12:16].strip()) + if key in atoms: + raise ValueError(f"duplicate target atom/conformer: {key}") + coordinates = tuple( + float(line[start : start + 8]) for start in (30, 38, 46) + ) + if not all(math.isfinite(value) for value in coordinates): + raise ValueError("non-finite target coordinates") + atoms[key] = (line[17:20].strip(), line[16].strip(), coordinates) + if in_model: + raise ValueError("unclosed target model") + return atoms + + +def verify_target(workspace, check): + root = workspace / "output/target-prep" + data = read_json(root / "preparation.json") + expected = { + "chain": "E", + "sequence": "AGST", + "author_to_sequence": {"10": 1, "42": 2, "42A": 3, "77": 4}, + "simulated": True, + "live_inference": False, + } + for key, value in expected.items(): + check(same(data.get(key), value), f"target preparation: {key}") + requests = data.get("requests", {}) + present, absent = requests.get("present", {}), requests.get("missing", {}) + check( + present.get("status") == "ready" + and present.get("hotspot_res") == ["E42", "E77"] + and same(present.get("sequence_indices"), [2, 4]), + "present hotspot handoff", + ) + check( + absent.get("status") == "blocked" + and absent.get("missing_author_residues") == ["43"] + and not absent.get("hotspot_res") + and not absent.get("sequence_indices"), + "missing hotspot blocks its request", + ) + atoms = target_atoms((root / "target.pdb").read_text()) + # Hand-checked first-model atoms from the digest-pinned synthetic input. + expected_atoms = { + ("E", 10, "", "CA"): ("ALA", "", (0.0, 0.0, 0.0)), + ("E", 42, "", "CA"): ("GLY", "B", (4.0, 0.0, 0.0)), + ("E", 42, "A", "CA"): ("SER", "", (4.0, 3.0, 0.0)), + ("E", 77, "", "CA"): ("THR", "", (8.0, 3.0, 0.0)), + } + check( + list(atoms) == list(expected_atoms) + and all( + atoms[key][0] == residue + # Writers may clear altLoc after selecting the required conformer. + and atoms[key][1] in {"", altloc} + and all( + math.isclose(actual, expected, rel_tol=0, abs_tol=0.0005) + for actual, expected in zip(atoms[key][2], coordinates) + ) + for key, (residue, altloc, coordinates) in expected_atoms.items() + ), + "extracted PDB chain/model/author numbering/selected coordinates", + ) + + +def grade(entry_id, workspace): + workspace = Path(workspace) + checks = [] + + def check(ok, message): + checks.append({"passed": bool(ok), "message": message}) + + if entry_id in BOOKKEEPING: + prefixes = [BOOKKEEPING[entry_id][0]] + elif entry_id in CACHE_CASES: + prefixes = [name + "/" for name in CACHE_CASES[entry_id]] + [ + "cache_contract.md" + ] + elif entry_id == "offline-target-preparation": + prefixes = ["target_prep/"] + else: + prefixes = [] + check(False, f"unknown case: {entry_id}") + for rel, digest in INPUT_HASHES.items(): + if any( + rel == prefix or (prefix.endswith("/") and rel.startswith(prefix)) + for prefix in prefixes + ): + path = workspace / "input" / rel + check(path.is_file() and sha(path) == digest, f"immutable fixture: {rel}") + if checks and all(item["passed"] for item in checks): + try: + if entry_id in BOOKKEEPING: + verify_bookkeeping(entry_id, workspace, check) + elif entry_id in CACHE_CASES: + for name in CACHE_CASES[entry_id]: + verify_cache(name, workspace, check) + else: + verify_target(workspace, check) + except (OSError, ValueError, KeyError, TypeError) as exc: + check( + False, f"artifact read/validation failed: {type(exc).__name__}: {exc}" + ) + passed = bool(checks) and all(item["passed"] for item in checks) + return { + "overall": float(passed), + "custom_metrics": {"artifact_correctness": float(passed)}, + "details": { + "artifact_correctness": { + "passed": passed, + "checks": checks, + "scope": "saved artifacts only; no proof of live inference or save/reload execution", + } + }, + } + + +def main(): + tests = Path(os.environ.get("HARBOR_TESTS_DIR", "/tests")) + verifier = Path(os.environ.get("HARBOR_VERIFIER_DIR", "/logs/verifier")) + entry = read_json(tests / "entry.json") + result = grade( + entry["id"], Path(os.environ.get("BINDER_EVAL_WORKSPACE", "/workspace")) + ) + verifier.mkdir(parents=True, exist_ok=True) + Path( + os.environ.get("HARBOR_REWARD_JSON", str(verifier / "reward.json")) + ).write_text(json.dumps(result, indent=2) + "\n") + + +# INPUT_HASHES is pinned below, outside the agent's mutable /workspace/input. +INPUT_HASHES = { + "boltz2_only.json": "c7b760c45bb2950f59c277eb08587789d8b945f03ca12f0d5407aa147e74ee59", + "bookkeeping.json": "23a769f5f3f02c96c6758eb3b5787b99befbe69b898a64f170fe416c24a80fd3", + "cache_contract.md": "24744e774ff688963bf99b540c7cfd9c525b8c870b2ce863cf298a210262edb6", + "mixed_thresholds.json": "fcc5b8aa40870a1fb0e9deb4362a60996fc155121f21365bb1640aae97ae5420", + "no_survivors.json": "2bfde692731de2850bee4f126dcc2c75cbd4e0b7eb97b1bbaa9566ea077f3022", + "provenance_cache/backbone.pdb": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764", + "provenance_cache/manifest.json": "5f08e89bd297f0c42e8e5302353a03d31f8e885c87178c807c938f4568c622cc", + "provenance_cache/responses/candidate_swap.json": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89", + "provenance_cache/responses/response_tampered.json": "7943c5202ba623e735fafa2d2661810171ada09095bed9038d0a06f9232eda76", + "provenance_cache/responses/sample_swap.json": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89", + "provenance_cache/responses/sequence_swap.json": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89", + "provenance_cache/responses/target_swap.json": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89", + "provenance_cache/responses/valid.json": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89", + "provenance_cache/target.pdb": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "raw_refolds/boltz2/backbone.pdb": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764", + "raw_refolds/boltz2/manifest.json": "b2666564b055d96269d619d6c0b6fe33229878fa5625342c567444d99d243055", + "raw_refolds/boltz2/responses/control.json": "d3b2430f50a5e20c5af3d4b11cdc5a2d59c7717d342fedef98220e8fbd5d5ae5", + "raw_refolds/boltz2/responses/incomplete.json": "c23989c01579afb22e2321e24c0b10daea5ad1357cca6af74146678d5d9d9f11", + "raw_refolds/boltz2/responses/poor_binder.json": "45049bc8801df9bf9b763ec0e4539f6ce0f4828f287c326e1b657e006c084339", + "raw_refolds/boltz2/responses/selected.json": "fb0fc1994076830ceeea1d8dd2d40485a6e8658b22ff289a9e676bb480c53a2c", + "raw_refolds/boltz2/target.pdb": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "raw_refolds/openfold3/backbone.pdb": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764", + "raw_refolds/openfold3/manifest.json": "c1080b79c08db16a76ec89cf754f5d859fbb9ee0e18fee74249e49151990f06a", + "raw_refolds/openfold3/responses/composite_only.json": "5123c9b20050bc54cdb559537b2ffc32d3d630d8683af02746bad5a234a133c4", + "raw_refolds/openfold3/responses/poor_binder.json": "119c9a8719156a0d169d9258bc52dabe4ad497fffdfa0ab0c3497bc1dec32fb1", + "raw_refolds/openfold3/responses/selected.json": "605dd0759e2642108810f6a88baef0f12427a86f2e5acfe3349f19d9a4b7399b", + "raw_refolds/openfold3/target.pdb": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "resume_cache/backbone.pdb": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764", + "resume_cache/below.pdb": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "resume_cache/candidates.csv": "0d4f8840021deaf5107cb8eb02577acd1db45f26b8cb4ae043a6f0957db32f0e", + "resume_cache/done.pdb": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "resume_cache/manifest.json": "35a9958917c527b27fdb679df481aaa34b7a0f4bd59ef6ec5219752be08bf119", + "resume_cache/responses/below.json": "9486ff3d9be5100cacbf77d25a604c27db8caa89e3ee82d161056138ae45db0c", + "resume_cache/responses/control.json": "7c074850d88c42357c6983784e8c7d28dcc14fa728dc3b9ea278ee0167b14b94", + "resume_cache/responses/done.json": "94b16fe78c2cea20d2f2179d1a763feb98ec9de71e0222825cb2cc4bd1f1669f", + "resume_cache/responses/recoverable.json": "214ee2ae319ad54656a6a3406ecacc571bee992c178881d61199d35c9f8d81af", + "resume_cache/target.pdb": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "target_prep/CONTRACT.md": "886e330fa4562df5029c874418a92bf108ec12331b69e16506f57d9ed9e7382b", + "target_prep/request.json": "d8d0c7e0e92d0c18a1a493a4853f0e7e1718be9bb30c2cbd44c9410747652298", + "target_prep/target.pdb": "6ef9dfdb6bf827b336d34aa768c36ed238581d365fab228ac2452ed0e019ca45", +} + + +if __name__ == "__main__": + main() diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/local-nim-setup.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/local-nim-setup.md index 434fec95..b8495e91 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/local-nim-setup.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/local-nim-setup.md @@ -1,80 +1,118 @@ -# Running the BioNeMo NIMs locally (self-hosted) +# Running the BioNeMo NIMs locally -Use this when you want to self-host RFdiffusion, ProteinMPNN, and a co-folder (Boltz2) -instead of the managed `build.nvidia.com` endpoints — e.g. to avoid rate limits or to -keep a campaign self-contained on one node. The pipeline logic is unchanged; only the -base URL changes (no `Authorization` header for local NIMs). +Use this guide when the user chooses self-hosted RFdiffusion, ProteinMPNN, and +Boltz2. Inference inputs go to the configured local services without an +Authorization header. Pulling containers and downloading model weights still +requires access to NVIDIA NGC. ## Prerequisites -- Docker with the NVIDIA container runtime (`docker run --gpus ...` works). -- An NGC API key in `NGC_API_KEY` (used to pull images and download model weights). -- `docker login nvcr.io -u '$oauthtoken' -p "$NGC_API_KEY"` once. +- Docker with the NVIDIA Container Toolkit and a supported GPU/driver stack. +- An NGC key in `NGC_API_KEY`, with access to the selected model versions and + their governing terms accepted. +- Choose the image version and digest from each model's NGC catalog: + [RFdiffusion](https://catalog.ngc.nvidia.com/orgs/nim/teams/ipd/containers/rfdiffusion), + [ProteinMPNN](https://catalog.ngc.nvidia.com/orgs/nim/teams/ipd/containers/proteinmpnn), + [Boltz2](https://catalog.ngc.nvidia.com/orgs/nim/teams/mit/containers/boltz2). -## Images +Authenticate with the existing environment key through stdin, with shell tracing +disabled. This avoids putting the secret in the command's argument list: -| NIM | Image | Idle GPU | Serves | -|---|---|---|---| -| RFdiffusion | `nvcr.io/nim/ipd/rfdiffusion:latest` | ~24 GB | `:8000/v1/biology/ipd/rfdiffusion/generate` | -| ProteinMPNN | `nvcr.io/nim/ipd/proteinmpnn:latest` | ~1.5 GB | `:8000/v1/biology/ipd/proteinmpnn/predict` | -| Boltz2 | `nvcr.io/nim/mit/boltz2:latest` | ~8 GB | `:8000/biology/mit/boltz2/predict` | +```bash +set +x +printf '%s' "${NGC_API_KEY:?Set NGC_API_KEY}" | docker login nvcr.io --username '$oauthtoken' --password-stdin +``` -All three co-fit on one ≥48 GB GPU (~33 GB idle together). +## Image and API contracts + +Set `RFD_IMAGE`, `PMPNN_IMAGE`, and `BOLTZ2_IMAGE` to the respective full NGC +repository reference ending in `@sha256:` plus its 64-character digest. Obtain +the digest from the selected catalog version or the `RepoDigests` output of +`docker image inspect` after pulling that explicit version. Do not invent a +digest or use a moving tag. Save the three references and selected GPU profiles +in the campaign parameters so a resumed run uses the same containers. + +| NIM | Image variable | Cache mount | Local prediction path | +|---|---|---|---| +| RFdiffusion | `RFD_IMAGE` | `/opt/nim/.cache` | `/biology/ipd/rfdiffusion/generate` | +| ProteinMPNN | `PMPNN_IMAGE` | `/home/nvs/.cache/nim` | `/biology/ipd/proteinmpnn/predict` | +| Boltz2 | `BOLTZ2_IMAGE` | `/opt/nim/.cache` | `/biology/mit/boltz2/predict` | + +These paths follow the published +[RFdiffusion 2.3.0](https://docs.nvidia.com/nim/bionemo/rfdiffusion/2.3.0/quickstart-guide.html), +[ProteinMPNN 1.2.0](https://docs.nvidia.com/nim/bionemo/proteinmpnn/1.2.0/quickstart-guide.html), +and [Boltz2 1.10.0](https://docs.nvidia.com/nim/bionemo/boltz2/1.10.0/getting-started.html) +guides; check the selected release's API/cache contract when changing versions. +Hosted prediction URLs additionally include `/v1`. GPU memory depends on the +model, profile, and input length; check each support matrix and run the stages +sequentially if the models do not fit together. ## Launch pattern -Give each NIM its own persistent cache (so weights download once), a name, and a port. -Mount the cache at `/opt/nim/.cache` and make it writable: +Run the following Bash block after setting the three image variables. It checks +every digest before launching any service. Each container receives only its own +cache directory and exposes an unauthenticated API on the host's loopback address. ```bash -mkdir -p ~/nimcache_rfd ~/nimcache_pmpnn ~/nimcache_boltz2 && chmod 700 ~/nimcache_* +( +set +x +set -euo pipefail +: "${NGC_API_KEY:?Set NGC_API_KEY}" +: "${RFD_IMAGE:?Set the RFdiffusion repository digest}" +: "${PMPNN_IMAGE:?Set the ProteinMPNN repository digest}" +: "${BOLTZ2_IMAGE:?Set the Boltz2 repository digest}" + +require_digest() { + local image_ref="$1" repository="$2" digest + digest="${image_ref#"$repository@sha256:"}" + if [[ "$image_ref" != "$repository@sha256:$digest" || ! "$digest" =~ ^[a-f0-9]{64}$ ]]; then + printf 'Expected an immutable image digest for %s\n' "$repository" >&2 + exit 1 + fi +} +require_digest "$RFD_IMAGE" nvcr.io/nim/ipd/rfdiffusion +require_digest "$PMPNN_IMAGE" nvcr.io/nim/ipd/proteinmpnn +require_digest "$BOLTZ2_IMAGE" nvcr.io/nim/mit/boltz2 + +mkdir -p "$HOME/nimcache_rfd" "$HOME/nimcache_pmpnn" "$HOME/nimcache_boltz2" +chmod 700 "$HOME/nimcache_rfd" "$HOME/nimcache_pmpnn" "$HOME/nimcache_boltz2" docker run -d --name rfdiffusion --gpus device=0 --shm-size=4g \ - -e NGC_API_KEY -v ~/nimcache_rfd:/opt/nim/.cache -p 8081:8000 \ - nvcr.io/nim/ipd/rfdiffusion:latest + --user "$(id -u):$(id -g)" -e NGC_API_KEY \ + -v "$HOME/nimcache_rfd:/opt/nim/.cache" -p 127.0.0.1:8081:8000 "$RFD_IMAGE" docker run -d --name proteinmpnn --gpus device=0 --shm-size=4g \ - -e NGC_API_KEY -v ~/nimcache_pmpnn:/opt/nim/.cache -p 8082:8000 \ - nvcr.io/nim/ipd/proteinmpnn:latest + --user "$(id -u):$(id -g)" -e NGC_API_KEY \ + -v "$HOME/nimcache_pmpnn:/home/nvs/.cache/nim" -p 127.0.0.1:8082:8000 "$PMPNN_IMAGE" docker run -d --name boltz2 --gpus device=0 --shm-size=8g \ - -e NGC_API_KEY -v ~/nimcache_boltz2:/opt/nim/.cache -p 8083:8000 \ - nvcr.io/nim/mit/boltz2:latest + --user "$(id -u):$(id -g)" -e NGC_API_KEY \ + -v "$HOME/nimcache_boltz2:/opt/nim/.cache" -p 127.0.0.1:8083:8000 "$BOLTZ2_IMAGE" +) ``` -Wait for readiness (first start downloads weights — minutes): +The three caches are owned by the invoking user and mode 700; the containers run +with that user's UID/GID. For releases requiring a fixed container user, arrange +access to these exact directories for that UID before launch. Do not widen +permissions on unrelated caches. First startup downloads weights and can take +several minutes. Check readiness with local GET requests that send no credentials +or protein data and bypass configured HTTP proxies: ```bash -curl -fsS http://localhost:8081/v1/health/ready && echo RFD_OK -curl -fsS http://localhost:8082/v1/health/ready && echo PMPNN_OK -curl -fsS http://localhost:8083/v1/health/ready && echo BOLTZ2_OK +curl --noproxy '*' -fsS http://127.0.0.1:8081/v1/health/ready +curl --noproxy '*' -fsS http://127.0.0.1:8082/v1/health/ready +curl --noproxy '*' -fsS http://127.0.0.1:8083/v1/health/ready ``` -If the NIMs and your client share a user-defined docker network, reach them by container -name instead of published ports (e.g. `http://rfdiffusion:8000/...`). - ## GPU profile selection -Most of these NIMs **auto-select** a profile by compute capability (SM) and just work on -a supported GPU. Some NIMs match profiles by **exact GPU model**, so on a GPU that has no -bundled profile the container exits early with `NIMProfileIDNotFound` / "0 profiles found". -If that happens, list the bundled profiles and pin the one that matches **your** GPU's -compute capability: +If startup reports `NIMProfileIDNotFound` or no compatible profiles, inspect the +selected release's supported GPUs and available profiles. For releases exposing +`list-model-profiles`, reuse the already validated image digest: ```bash -# 1) list the profiles this NIM ships and their tags (gpu / compute capability / precision / backend) -docker run --rm --gpus device=0 -e NGC_API_KEY \ - list-model-profiles -# 2) choose the profile whose tags match YOUR GPU (compute capability first), then pin it: -docker run -d --name --gpus device=0 --shm-size=8g \ - -e NGC_API_KEY -e NIM_MODEL_PROFILE= \ - -v ~/nimcache_:/opt/nim/.cache -p 8083:8000 +docker run --rm --gpus device=0 -e NGC_API_KEY "${BOLTZ2_IMAGE:?Set the validated Boltz2 digest}" list-model-profiles ``` -A TRT-optimized engine built for one GPU generally loads on another of the **same compute -capability** (you may see a benign cross-device warning). Always select the profile for the -hardware you are running on — **do not copy a `NIM_MODEL_PROFILE` id from another machine**, -and check the NIM's support matrix for your exact GPU. - -## Then point the pipeline at local - -Set the local base URLs and drop the `Authorization` header. Request/response shapes for -every NIM are in `references/pipeline.md` — only the URL/auth changes between hosted and -local. +Select a profile documented for that GPU and add `-e NIM_MODEL_PROFILE` to the +corresponding launch command after setting the chosen profile in the environment. +Matching compute capability alone does not establish engine compatibility. +Record the profile with the image digest. Then use the local prediction paths +above in the pipeline and omit hosted authentication headers. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/manifest.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/manifest.md index 0f520e00..f4e6a7cd 100755 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/manifest.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/manifest.md @@ -56,16 +56,82 @@ m.upsert_candidate("bb003_seq02", backbone_id="bb003", sequence="MKT...") m.set_scores("bb003_seq02", iptm=0.84, binder_plddt=86.2, self_consistency_rmsd=1.4) m.add_artifact("bb003_seq02", "complex_cif", "complexes/bb003_seq02.cif") m.apply_filters() -top = m.rank(by="iptm", passed_only=True) -m.to_csv() +ranked = m.rank(by="iptm", passed_only=True) +m.to_csv(candidates=ranked) # resume a campaign later m2 = Manifest.load("runs/demo") ``` +`apply_filters()` requires every applicable enabled metric to be present and finite. Missing +or invalid scores cannot pass. Set a threshold to `null` only when deliberately +disabling that criterion for the campaign, and disclose the override. A composite +confidence field does not satisfy an ipTM requirement. Sequence-only controls +without `artifacts.backbone_pdb` skip only RMSD, with the reason saved under +`filter_exemptions`. `filter_metrics` records the criteria actually evaluated. + +### Boltz2-only filtering and ranking + +Use a separate `boltz2_confidence_min` filter when the co-fold response has no +explicit ipTM. Resolve its cutoff with the campaign inputs before inference; +there is no default composite-confidence threshold. The numbers below are +**synthetic bookkeeping examples**, including the illustrative 0.8 cutoff. +Replace scores with values extracted from the saved prediction, and choose the +cutoff for the real campaign using `references/validation.md` and save its source +in `params.confidence_selection`. If no cutoff is supplied, the default route is +explicit ipTM from OpenFold3; if that is unavailable, record +`selection_policy_unresolved` and stop before design inference. Keep binder +pLDDT/RMSD criteria enabled for designs. + +```python +m = Manifest.create( + run_dir="runs/boltz2_demo", + target={"name": "synthetic example"}, + params={"rank_by": "boltz2_confidence"}, + filters={ + "iptm_min": None, + "boltz2_confidence_min": 0.8, + "binder_plddt_min": 80.0, + "self_consistency_rmsd_max": 2.0, + }, +) +m.set_scores("example", boltz2_confidence=0.9, binder_plddt=86.2, self_consistency_rmsd=1.4) +m.apply_filters() +ranked = m.rank(by=m.data["params"]["rank_by"], passed_only=True) +m.to_csv(candidates=ranked) +``` + +This candidate can pass without an `iptm` value; its composite confidence remains +in `boltz2_confidence`. A missing or non-finite enabled metric still fails. +Label the metric and cutoff in the report, and apply the same profile to controls. + +`to_csv()` with no candidate list exports the full audit table in insertion +order. Pass the list returned by `rank()` to export ranked survivors, including +an empty list when none pass. `rank()` does not mutate the manifest; calling +`to_csv()` without that list afterward still writes all entries. Keep the audit +in `all_candidates.csv` and the selected list in `candidates.csv`. An empty +selected list produces only the CSV header; never replace it with all candidates. +The manifest always retains all candidates and +controls. `rank()` excludes missing, nonnumeric, boolean, and non-finite scores. + ## Resumability -Because state lives in `manifest.json`, an interrupted campaign resumes by -loading the manifest and skipping candidates that already have the needed -scores/artifacts. Long campaigns should checkpoint after each expensive stage -(`m.save()` is called automatically by the mutation helpers). +Load `Manifest.load(run_dir)` and retain its filters, mode, parameters, and ranking +metric. `m.required_metrics(cid)` lists the applicable enabled metrics; +`m.missing_scores(cid)` lists missing/non-finite values. A finite failing score is +complete and should not trigger another prediction. + +| Stage | Completed evidence | Resume action when missing | +|---|---|---| +| Backbone (designs only) | nonempty `artifacts.backbone_pdb` file | resolve saved backbone by `backbone_id`; regenerate only if genuinely absent | +| Sequence | nonempty candidate `sequence` plus saved MPNN response for designs | parse saved response before making another MPNN call | +| Co-fold | nonempty `artifacts.complex_cif` or `complex_pdb`, plus saved response | reuse a saved complex/response; otherwise co-fold this candidate | +| Metrics | `missing_scores(cid) == []` | extract from saved response/structure; run `metrics.py` for design RMSD | +| Filtering/report | current `filter_metrics`, `passed_filter`, CSV | rerun deterministic filtering and export; no inference | + +Resolve artifact paths as recorded (the examples include the run directory), and +verify files exist and are nonempty before skipping a stage. Sequence-only controls +do not require a designed backbone or its RMSD. An explicit `failure_reason` from +a failed request remains visible; retry only after the cause or retry decision is +recorded. Checkpoint every successful or failed expensive stage with the mutation +helpers. Preserve all candidates, including incomplete and failed ones. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/offline.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/offline.md new file mode 100644 index 00000000..03f45844 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/offline.md @@ -0,0 +1,106 @@ +# Offline bookkeeping and saved responses + +Use these helpers for explicitly synthetic software exercises. They never read +credentials or submit inference. Outputs establish software behavior only, with +no biological validity, scientific calibration or endpoint coverage. Real +campaign response parsing must first establish the provider's actual schema and +confidence encoding. + +## Score fixtures + +From the skill directory: + +```bash +python3 scripts/offline_bookkeeping.py /workspace/input/bookkeeping.json --output-dir /workspace/runs/bookkeeping +``` + +The input has `target`, optional `provenance`, and `profiles`. Each profile has +`name`, `rank_by`, `filters`, and `candidates` with `id`, `scores`, and control +flags. Names must be unique; a profile name is a single directory component. +One profile writes directly to `--output-dir`; multiple profiles create named +subdirectories. The destination must be empty. Existing scores, explicit nulls, +invalid strings and absent keys are preserved. Unknown filters are rejected. + +Each run contains `manifest.json`, `all_candidates.csv`, `candidates.csv`, and +`summary.json`. The manifest is saved and reloaded before filtering. The audit +retains controls and incomplete candidates. The ranked file contains only +passing non-controls and retains a header when no design passes. Summary counts +exclude controls from the design denominator and disclose missing-score IDs and +a provisional-fraction flag. A finite below-threshold score is complete. + +Check the exit status and inspect saved files before reporting their counts and +ranked IDs. A script on disk or an intended path is not an executed result. + +## Recovering cached metrics + +```bash +python3 scripts/refold_cache.py /workspace/input/resume_cache/manifest.json --output-dir /workspace/output/resumed +``` + +This copies the source run to a new destination, preserves failed stages and +existing finite scores, imports missing required metrics from supported cached +responses, and regenerates exports and the summary. The destination must not +exist or be inside the source run. Relative artifact paths resolve within the +copied run. Missing predictions remain blocked; this command never retries an +endpoint. Completed low-scoring candidates are preserved, not resubmitted. + +The fixture format extends the normal manifest with +`target.structure: {path, sha256}` and a per-candidate `prediction` binding: + +```json +{ + "provider": "boltz2", + "candidate_id": "candidate-1", + "sequence_sha256": "", + "target_sha256": "", + "response": {"path": "responses/candidate-1.json", "sha256": ""}, + "sample_index": 1, + "complex_sha256": "", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": {"path": "backbone.pdb", "sha256": ""}, + "backbone_chain": "D" +} +``` + +These are local provenance fields, not fields promised by a model endpoint. +Digests establish consistency with recorded inputs; they do not authenticate a +model or prove inference occurred. Record bindings when the original request +and response are saved; do not invent them from a convenient cached file. +A changed candidate, sequence, target, response or selected sample blocks the +import. Preserve conflicting evidence and record the reason. + +- **Boltz2:** pair `structures[sample_index]` with the same index in + `confidence_scores`, retained only as `boltz2_confidence`. Require equal array + lengths. Do not select a higher-confidence sample independently of coordinates. +- **OpenFold3:** match `outputs[].input_id` to the candidate, then select + `structures_with_scores[sample_index]`. Only `iptm_score` supplies ipTM; + `confidence_score` and `complex_plddt_score` cannot replace it. +- **Binder pLDDT:** fixtures explicitly encode it in CA B factors with + `plddt_scale` of `0-1` or `0-100`. Average the binder only and normalize to + 0–100. Experimental PDB B factors need not encode pLDDT. mmCIF uses + `label_asym_id` to match the request, which can differ from author chain IDs. + Require the complete binder length and both chain roles. +- **Missing RMSD:** verify the backbone digest and align complete, explicitly + identified binder CA sets with `metrics.ca_rmsd_from_structures()`. Unequal + lengths fail; do not truncate or compare the target instead. + +An incomplete candidate receives scores only after all missing applicable +metrics and bindings validate. Finite existing values are never overwritten. +Recovered candidates record `score_provenance` (the full binding) and +`artifacts.complex` (the selected structure). `cache_status` is `complete`, +`recovered`, or `blocked`; `cache_reason` explains blocked work. Historical +`failure_reason` and stage records remain intact. A provisional passing fraction +still includes incomplete design candidates in its denominator. + +## Target preparation + +`pdb_utils.py` extracts the first model, chooses one CA per author residue, and +keeps insertion codes as distinct map keys. A blank alternate location takes +precedence; otherwise highest occupancy wins, with a lexicographic tie break. +Sequence indices follow CA order, not arithmetic offsets from the first author +number. Specify a chain when author numbering overlaps across chains. +`remap_to_seq_index()` raises for absent hotspots: record the missing IDs and +block that handoff. Verify insertion-code support for a downstream endpoint +before emitting a request containing such a hotspot. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/pipeline.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/pipeline.md index d2af20a6..0863505b 100755 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/pipeline.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/pipeline.md @@ -7,7 +7,17 @@ skill. Deterministic glue (parsing, remapping, RMSD, manifest) uses the bundled ## 0. Setup -- Decide hosted vs local **once** (ask the user) and reuse for every NIM call. +- Reuse an existing hosted/local choice, or resolve it once if unspecified. +- For hosted calls, read `NGC_API_KEY` or `NVIDIA_API_KEY` from the environment + and stop if neither is set. Do not substitute the agent's `OPENAI_API_KEY` or + print environment variables or authorization headers. +- Wrap every hosted command: + `bash scripts/hosted_env.sh python3 your_nim_request.py` (use the helper's + absolute path outside the skill directory). This exports the chosen key as + `NGC_API_KEY` for RFdiffusion, ProteinMPNN, Boltz2, OpenFold3, and optional MSA + subprocesses. The wrapper selects credentials and launches the command together; + it works in a fresh tool shell and disables tracing. Local + inference uses no authentication header and does not need this helper. - Create a run directory and manifest: ```python @@ -17,10 +27,45 @@ m = Manifest.create( run_dir="runs/_", target={"name": "", "pdb_id": "", "chain": ""}, mode="hosted", - params={"n_backbones": 100, "seqs_per_backbone": 8, "binder_len": "60-90"}, + params={"n_backbones": 100, "seqs_per_backbone": 8, "binder_len": "60-90", "rank_by": "iptm"}, ) ``` +Before inference, select the scoring route. The default is a refolder returning +explicit ipTM (OpenFold3) with the manifest's default filters. For Boltz2-only +output without ipTM, require a user-selected or separately calibrated confidence +cutoff and its rationale first (`references/validation.md`). Save that decision: + +```python +m.data["filters"].update(iptm_min=None, boltz2_confidence_min=campaign_cutoff) +m.data["params"]["rank_by"] = "boltz2_confidence" +m.data["params"]["confidence_selection"] = {"source": cutoff_source, "rationale": cutoff_rationale} +m.save() +``` + +This retains the pLDDT/RMSD criteria. The complete Boltz2 manifest example is in +`references/manifest.md`; reuse its saved filters and ranking metric on resume. +If no justified cutoff or explicit-ipTM refolder is available, stop before design +inference with `selection_policy_unresolved`. Never use the synthetic example's +cutoff as an implicit scientific default. + +### Request failures and partial batches + +Use the atomic skill's request client with an explicit per-request timeout (up +to 1200 seconds for inference), at most three retries for HTTP 429/5xx, and bounded +backoff (10, 20, 40 seconds; honor Retry-After up to 120 seconds). Count retries +against the campaign's call/time budget and stop when it is exhausted. A timed-out +generation POST may have run remotely: query its returned job ID if available; +otherwise record `outcome_unknown` and require an explicit retry decision to avoid +duplicate generation charges. Do not retry HTTP 400/401/403 unchanged. + +Save each successful response and update its candidate before the next request. +On exhausted errors, set `failure_reason`, log the failed stage and attempt count +with `m.log_stage()`, and continue independent candidates within the budget. Never +fill missing scores with zeros or drop failed candidates from the denominator. +Abort the batch on missing/invalid credentials. Reuse successful artifacts on +resume; retry failed calls only after the cause or retry decision is recorded. + ## 1. Target prep - Obtain the target structure (experimental PDB, or predict with `openfold2-nim` @@ -37,12 +82,18 @@ seq_idx = remap_to_seq_index(target_pdb, chain="", author_resnums=[ --predicted-chain `. +For sequence-only controls without a designed backbone, RMSD is not applicable; +`Manifest.apply_filters()` records the exemption and still tests confidence/pLDDT. ## 6. Filter + rank + report ```python -m.apply_filters() # uses manifest filters (ipTM/pLDDT/RMSD) -top = m.rank(by="iptm", descending=True, passed_only=True)[:20] -m.to_csv() # candidates.csv next to manifest.json +from pathlib import Path + +m.apply_filters() # uses the saved campaign criteria +rank_by = m.data["params"].get("rank_by", "iptm") +ranked = m.rank(by=rank_by, descending=True, passed_only=True, include_controls=False) +top = ranked[:20] +run_dir = Path(m.data["run_dir"]) +m.to_csv(run_dir / "all_candidates.csv") # full audit, including controls and failures +m.to_csv(candidates=ranked) # ranked survivors; controls excluded print(m.summary()) # {n_candidates, n_passed, n_controls} ``` +`rank()` does not modify the manifest. Always pass its returned list to the +ranked export, including when empty; `to_csv()` alone exports every entry in +insertion order. The complete audit is a separate file. All applicable enabled +filters must pass before ranking, even for a design with the highest interface +score. Disclose missing scores and a provisional passing fraction. + Produce a short report: target + epitope, params, success rate, the top designs with their scores and artifact paths, and how they compare to controls (`references/validation.md`). @@ -146,6 +229,7 @@ designs with their scores and artifact paths, and how they compare to controls - **No target structure** → predict it first (`openfold2/3-nim` or `boltz2-nim`). - **Target needs evolutionary context** → `msa-search-nim` before co-folding. - **Interface metric** → prefer OpenFold3 `iptm_score`; Boltz2 `confidence_scores` - is the fallback complex-confidence signal. + uses its own explicit filter and ranking route. Label the selected metric and + cutoff in the report, and compare controls under that same route. - **Binder MSA** → keep single-sequence for de novo binders (standard); do not fabricate a binder MSA. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/validation.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/validation.md index a1e9441b..2eae3eed 100755 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/validation.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/validation.md @@ -8,7 +8,8 @@ controls. | Metric | Source | Pass guide | Meaning | |---|---|---|---| -| Interface confidence (ipTM) | OpenFold3 `iptm_score` · Boltz2 `confidence_scores` | ≥ 0.8 | predicted interface quality | +| Interface confidence (ipTM) | Explicit ipTM output, e.g. OpenFold3 `iptm_score` | ≥ 0.8 | predicted interface quality | +| Composite complex confidence | Boltz2 `confidence_scores` | campaign-selected `boltz2_confidence_min` | separate filter/ranking route when ipTM is unavailable | | Binder pLDDT | OpenFold3 / Boltz2 | ≥ 80 | binder fold confidence | | Self-consistency RMSD | `scripts/metrics.py` (Kabsch CA-RMSD) | ≤ 2.0 Å | designed backbone vs predicted | | Sequence quality (NLL) | ProteinMPNN `scores` | lower better | sequence–backbone compatibility | @@ -16,10 +17,16 @@ controls. There is no protein–protein affinity NIM, so ipTM + self-consistency RMSD are the binder proxy (the Bennett et al. 2023 filter pattern). +For a Boltz2-only campaign without explicit ipTM, save `iptm_min: null`, a chosen +`boltz2_confidence_min`, and `params.rank_by: boltz2_confidence` before inference. +Keep pLDDT/RMSD requirements; rank with that saved metric. This profile has its +own success rate and must be labeled as composite-confidence selection, not as +passing the default ipTM filter. `references/manifest.md` has executable usage. + ## Controls -Run controls through the **identical** pipeline so score distributions are -comparable. +Run controls through the **same co-folding model and settings** so confidence +and pLDDT distributions are comparable. They do not need RFdiffusion/ProteinMPNN. - **Negative controls** — scrambled binder sequences (preserve composition): @@ -35,8 +42,39 @@ m.upsert_candidate("ctrl_neg_01", is_control=True, control_type="scrambled", seq (`assets/targets.json` → `published_binders`), co-folded the same way and marked `control_type="published"`. -A working pipeline separates designed/published positives from scrambled -negatives in the ipTM and RMSD distributions. +The bundled registry is a template and contains no published binder sequences. +Use cited user inputs or verify exact sequences against a primary paper/structure +and register the target, binder sequence, and source with `scripts/registry.py`. +If none can be sourced, record `positive_controls_unavailable`; return an +**uncalibrated computational screen** with negative controls and disclose that the +positive-control comparison was not performed. Do not invent a positive or label +the missing comparison as successful. If calibrated benchmarking was requested, +that portion remains blocked. + +Sequence-only controls have no RFdiffusion backbone. Leave their self-consistency +RMSD absent, and do not attach a parent's backbone merely to satisfy the filter. +`apply_filters()` records `filter_exemptions.self_consistency_rmsd` for these +controls while retaining confidence and pLDDT requirements. A control with a real +reference `artifacts.backbone_pdb` is subject to the RMSD filter. Compare control +confidence/pLDDT distributions separately from the design-only RMSD distribution. + +### Choosing a Boltz2 composite-confidence cutoff + +The default route uses explicit ipTM and the existing ipTM threshold. Without a +composite cutoff, prefer OpenFold3; if unavailable, stop before design inference +with `selection_policy_unresolved`. No universally validated composite-confidence +cutoff is bundled here, and the synthetic 0.8 example is not a recommendation. + +For a Boltz2-only campaign, reuse the user's supplied cutoff and record its source +and limitations, or perform a separate calibration with sourced positives and +scrambled negatives against the same target, model/version, and folding settings. +Plot/report both score distributions, choose a cutoff under a stated false-positive +tolerance, and assess it on held-out controls; insufficient controls or inseparable +distributions cannot establish a calibrated cutoff. Obtain the campaign decision +before screening the design pool. Save the cutoff, control IDs, settings, rationale, +and calibration limitations in `params.confidence_selection`; never tune the cutoff +on the design pool just to obtain passers. A user-selected cutoff without calibration +supports explicitly labeled exploratory screening only. ## Success rate @@ -44,16 +82,22 @@ The metric the field actually quotes — fraction of designs passing the filter: ```python s = m.summary() -success_rate = s["n_passed"] / max(s["n_candidates"], 1) +success_rate = s["n_passed"] / s["n_candidates"] if s["n_candidates"] else None ``` +Controls are excluded from both counts. A candidate needs finite values for +every enabled filter to pass. Report the number still missing required scores; +until scoring finishes, label the ratio as provisional. With no candidates, +report the rate as unavailable, not zero percent. + Use it to compare pipeline configs (diffusion steps, sampling temperature, sequences/backbone) rather than over-interpreting any single design. ## Published comparison Re-score literature winners through your exact pipeline and check your top -designs land in the same ipTM/RMSD regime. Absolute scores are not comparable +designs land in the same confidence regime. Compare RMSD only when the control has +a real reference backbone. Absolute scores are not comparable across pipelines — only same-pipeline comparisons are meaningful. Benchmark targets come from the target registry (`assets/targets.json`) — add your own diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements-dev.txt b/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements-dev.txt new file mode 100644 index 00000000..a2637ad3 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements-dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt +pytest==9.1.1 +pytest-cov==7.1.0 diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt b/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt new file mode 100644 index 00000000..2faa4494 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt @@ -0,0 +1,4 @@ +# Tested helper environment (Python 3.12+); NumPy/Biotite match the Tier 3 image. +numpy==2.5.1 +biotite==1.7.1 +requests==2.34.2 diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/controls.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/controls.py index e3cc6bc1..ca29f023 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/controls.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/controls.py @@ -1,26 +1,56 @@ +#!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 -"""Negative controls for binder validation: composition-preserving scrambles.""" +"""Negative controls for binder validation: composition-preserving scrambles. + +Usage: Import scramble_sequence or make_scrambled_controls; no command-line API. +Arguments: Uppercase canonical amino-acid sequences, nonnegative count n, integer seed. +Output: A shuffled sequence or list of sequences with unchanged composition. +Exit codes: Not applicable to this library; invalid inputs raise ValueError. +""" from __future__ import annotations import random +AMINO_ACIDS = frozenset("ACDEFGHIKLMNPQRSTVWY") +DEFAULT_CONTROL_COUNT = 5 +MAX_SCRAMBLE_SEED = 2 ** 31 - 1 + + +def _validate_sequence(seq): + if not isinstance(seq, str) or not seq or set(seq) - AMINO_ACIDS: + raise ValueError("sequence must be a nonempty uppercase canonical amino-acid string") + + +def _validate_seed(seed): + if not isinstance(seed, int) or isinstance(seed, bool): + raise ValueError("seed must be an integer") + def scramble_sequence(seq, seed=0): - """Return a shuffled sequence (same amino-acid composition, new order).""" + """Return a shuffled sequence; a shuffle can coincide with the input.""" + _validate_sequence(seq) + _validate_seed(seed) rng = random.Random(seed) chars = list(seq) rng.shuffle(chars) return "".join(chars) -def make_scrambled_controls(seqs, n=5, seed=0): - """Generate ``n`` scrambled negative controls drawn from ``seqs``.""" +def make_scrambled_controls(seqs, n=DEFAULT_CONTROL_COUNT, seed=0): + """Draw controls cyclically from a sequence list; an empty list yields none.""" + if not isinstance(n, int) or isinstance(n, bool) or n < 0: + raise ValueError("n must be a nonnegative integer") + if not isinstance(seqs, (list, tuple)): + raise ValueError("seqs must be a list or tuple of amino-acid sequences") + _validate_seed(seed) + for seq in seqs: + _validate_sequence(seq) rng = random.Random(seed) if not seqs: return [] out = [] for i in range(n): base = seqs[i % len(seqs)] - out.append(scramble_sequence(base, seed=rng.randint(0, 2 ** 31 - 1))) + out.append(scramble_sequence(base, seed=rng.randint(0, MAX_SCRAMBLE_SEED))) return out diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/hosted_env.sh b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/hosted_env.sh new file mode 100644 index 00000000..8eef03ad --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/hosted_env.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +# Run a hosted command with its credential in the child's environment: +# bash scripts/hosted_env.sh python your_nim_request.py +# Sourcing remains supported for existing callers. +set +x # Never trace credential assignments, even when invoked with bash -x. +if [ -n "${NGC_API_KEY:-}" ]; then + export NGC_API_KEY +elif [ -n "${NVIDIA_API_KEY:-}" ]; then + export NGC_API_KEY="$NVIDIA_API_KEY" +else + printf '%s\n' 'Set NGC_API_KEY or NVIDIA_API_KEY for hosted NIM access.' >&2 + if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then + return 1 + fi + exit 1 +fi + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + if [[ $# -eq 0 ]]; then + printf '%s\n' 'Usage: bash hosted_env.sh COMMAND [ARG ...]' >&2 + exit 2 + fi + exec "$@" +fi diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/manifest.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/manifest.py index c29c519f..2807deaa 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/manifest.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/manifest.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 """Run manifest for protein-binder-design campaigns. @@ -5,11 +6,18 @@ A campaign manifest is a single JSON file that records every candidate's lineage, scores, artifacts, and filter status. It is the backbone for ranking, resumability, validation, and the final report. No third-party dependencies. + +Usage: Import Manifest and call create(run_dir, target) or load(path). +Arguments: Campaign paths, target/parameter/filter dictionaries, candidate IDs and scores. +Output: Mutation methods save manifest.json; to_csv writes the selected candidate table. +Exit codes: Not applicable to this library; invalid data and I/O errors raise exceptions. """ from __future__ import annotations import csv import json +import math +import operator from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -22,11 +30,22 @@ "self_consistency_rmsd_max": 2.0, } +FILTER_METRICS = ( + ("iptm_min", "iptm", operator.ge), + ("boltz2_confidence_min", "boltz2_confidence", operator.ge), + ("binder_plddt_min", "binder_plddt", operator.ge), + ("self_consistency_rmsd_max", "self_consistency_rmsd", operator.le), +) + def _now() -> str: return datetime.now(timezone.utc).isoformat() +def _finite_score(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + class Manifest: """Read/write wrapper around a campaign ``manifest.json``.""" @@ -54,7 +73,7 @@ def create( "target": target, "mode": mode, "params": params or {}, - "filters": dict(filters) if filters else dict(DEFAULT_FILTERS), + "filters": dict(filters) if filters is not None else dict(DEFAULT_FILTERS), "stages": [], "candidates": [], } @@ -117,20 +136,45 @@ def add_artifact(self, cid: str, key: str, path: str | Path) -> dict[str, Any]: return c # ---- analysis -------------------------------------------------------- + def required_metrics(self, cid: str) -> list[str]: + """Enabled metrics applicable to a candidate, including on resume. + + Sequence-only controls have no designed backbone to compare against. + Their interface confidence and pLDDT still use the campaign thresholds. + """ + candidate = self._find(cid) + if candidate is None: + raise KeyError(cid) + sequence_control = candidate.get("is_control") and not candidate.get("artifacts", {}).get("backbone_pdb") + return [metric for threshold, metric, _ in FILTER_METRICS + if self.data["filters"].get(threshold) is not None + and not (sequence_control and metric == "self_consistency_rmsd")] + + def missing_scores(self, cid: str) -> list[str]: + """Return absent or invalid required scores; low finite scores are complete.""" + required = self.required_metrics(cid) + scores = self._find(cid).get("scores", {}) + return [metric for metric in required if not _finite_score(scores.get(metric))] + def apply_filters(self) -> None: + """Require all applicable metrics; record control-only RMSD exemptions.""" f = self.data["filters"] for c in self.data["candidates"]: s = c.get("scores", {}) checks = [] - if f.get("iptm_min") is not None and s.get("iptm") is not None: - checks.append(s["iptm"] >= f["iptm_min"]) - if f.get("binder_plddt_min") is not None and s.get("binder_plddt") is not None: - checks.append(s["binder_plddt"] >= f["binder_plddt_min"]) - if ( - f.get("self_consistency_rmsd_max") is not None - and s.get("self_consistency_rmsd") is not None - ): - checks.append(s["self_consistency_rmsd"] <= f["self_consistency_rmsd_max"]) + required = self.required_metrics(c["id"]) + c["filter_metrics"] = required + c["filter_exemptions"] = {} + for threshold, metric, compare in FILTER_METRICS: + if f.get(threshold) is None: + continue + if metric not in required: + c["filter_exemptions"][metric] = "sequence-only control: no designed backbone" + continue + score = s.get(metric) + checks.append( + _finite_score(score) and compare(score, f[threshold]) + ) c["passed_filter"] = bool(checks) and all(checks) self.save() @@ -146,17 +190,24 @@ def rank( cands = [c for c in cands if not c.get("is_control")] if passed_only: cands = [c for c in cands if c.get("passed_filter")] - cands = [c for c in cands if c.get("scores", {}).get(by) is not None] + cands = [c for c in cands if _finite_score(c.get("scores", {}).get(by))] return sorted(cands, key=lambda c: c["scores"][by], reverse=descending) - def to_csv(self, path: str | Path | None = None) -> Path: + def to_csv( + self, + path: str | Path | None = None, + *, + candidates: list[dict[str, Any]] | None = None, + ) -> Path: + """Export all candidates, or a supplied ranked list in its given order.""" path = Path(path) if path else Path(self.data["run_dir"]) / "candidates.csv" + candidates = self.data["candidates"] if candidates is None else candidates score_keys = sorted({k for c in self.data["candidates"] for k in c.get("scores", {})}) cols = ["id", "backbone_id", "is_control", "control_type", "passed_filter"] + score_keys with open(path, "w", newline="") as fh: w = csv.writer(fh) w.writerow(cols) - for c in self.data["candidates"]: + for c in candidates: row = [ c.get("id"), c.get("backbone_id"), diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/metrics.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/metrics.py index 0f6acde6..6bc45ad4 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/metrics.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/metrics.py @@ -1,19 +1,31 @@ +#!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 -"""Deterministic structural metrics for binder design (numpy only).""" +"""Deterministic RMSD metrics (NumPy; Biotite for mmCIF input). + +Usage: python3 metrics.py backbone.pdb prediction.cif --backbone-chain A --predicted-chain B +Arguments: Two structure paths and explicit binder chain IDs (mmCIF label IDs). +Output: CA-RMSD in angstroms on stdout; no files are written. +Exit codes: 0 success; 1 unavailable RMSD/input failure; 2 invalid CLI arguments. + Library functions raise ValueError for invalid coordinates. +""" from __future__ import annotations import numpy as np +COORDINATE_DIMENSIONS = 3 + def kabsch_rmsd(p, q): """Minimal RMSD after optimal superposition of two (N, 3) coord sets.""" p = np.asarray(p, dtype=float) q = np.asarray(q, dtype=float) - if p.shape != q.shape or p.ndim != 2 or p.shape[1] != 3: + if p.shape != q.shape or p.ndim != 2 or p.shape[1] != COORDINATE_DIMENSIONS: raise ValueError(f"coordinate shape mismatch: {p.shape} vs {q.shape}") if p.shape[0] == 0: raise ValueError("no coordinates provided") + if not np.isfinite(p).all() or not np.isfinite(q).all(): + raise ValueError("non-finite coordinates cannot be aligned") pc = p - p.mean(axis=0) qc = q - q.mean(axis=0) h = pc.T @ qc @@ -25,7 +37,7 @@ def kabsch_rmsd(p, q): def ca_rmsd_from_pdb(pdb_a, pdb_b, chain_a=None, chain_b=None): - """CA-RMSD between two structures (by chain). Truncates to common length.""" + """CA-RMSD between equal-length PDB chains; missing residues are an error.""" import os import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -33,7 +45,40 @@ def ca_rmsd_from_pdb(pdb_a, pdb_b, chain_a=None, chain_b=None): a = ca_coords(pdb_a, chain_a) b = ca_coords(pdb_b, chain_b) - n = min(len(a), len(b)) - if n == 0: + if not a or not b: raise ValueError("no CA atoms found for RMSD") - return kabsch_rmsd(a[:n], b[:n]) + return kabsch_rmsd(a, b) + + +def ca_rmsd_from_structures(backbone_pdb, predicted_text, *, backbone_chain, + predicted_chain, predicted_format="cif"): + """Compare explicit binder chains, using mmCIF label IDs for predictions. + + Both chains must contain the full binder in residue order. Do not silently + truncate missing residues or compare against the target chain. + """ + from pdb_utils import structure_ca_coords + + designed = structure_ca_coords(backbone_pdb, backbone_chain, format="pdb") + predicted = structure_ca_coords(predicted_text, predicted_chain, format=predicted_format) + return kabsch_rmsd(designed, predicted) + + +if __name__ == "__main__": + import argparse + from pathlib import Path + + parser = argparse.ArgumentParser(description="Binder CA-RMSD from a PDB backbone and PDB/mmCIF prediction.") + parser.add_argument("backbone", type=Path) + parser.add_argument("prediction", type=Path) + parser.add_argument("--backbone-chain", required=True) + parser.add_argument("--predicted-chain", required=True) + args = parser.parse_args() + try: + print(ca_rmsd_from_structures( + args.backbone.read_text(), args.prediction.read_text(), + backbone_chain=args.backbone_chain, predicted_chain=args.predicted_chain, + predicted_format=args.prediction.suffix.lstrip("."), + )) + except (ValueError, OSError, ImportError) as exc: + parser.exit(1, f"RMSD unavailable: {exc}\n") diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/offline_bookkeeping.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/offline_bookkeeping.py new file mode 100644 index 00000000..e57d4c04 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/offline_bookkeeping.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Import synthetic score fixtures and export audited, reloaded manifests offline. + +Usage: python3 offline_bookkeeping.py fixture.json --output-dir DIR +Arguments: JSON with target and profiles; a new or empty output directory. +Output: Per-profile manifest.json, all_candidates.csv, candidates.csv, summary.json; + JSON summaries on stdout. Multiple profiles use named subdirectories. +Exit codes: 0 success; 1 invalid fixture or I/O failure; 2 invalid CLI arguments. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import math +from pathlib import Path + +from manifest import FILTER_METRICS, Manifest + + +def export_run(manifest: Manifest) -> dict: + """Filter saved state and write both exports, including an empty ranked list.""" + manifest.save() + manifest = Manifest.load(manifest.path) + manifest.apply_filters() + ranked = manifest.rank(by=manifest.data["params"]["rank_by"], passed_only=True) + root = manifest.path.parent + manifest.to_csv(root / "all_candidates.csv") + manifest.to_csv(root / "candidates.csv", candidates=ranked) + pending = { + c["id"]: manifest.missing_scores(c["id"]) + for c in manifest.data["candidates"] + if manifest.missing_scores(c["id"]) + } + summary = { + "label": "simulated offline bookkeeping", + "live_inference": False, + **manifest.summary(), + "ranked_ids": [c["id"] for c in ranked], + "missing_scores": pending, + "passing_fraction_provisional": any( + c["id"] in pending + for c in manifest.data["candidates"] + if not c.get("is_control") + ), + } + (root / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + return summary + + +def run_fixture(input_path: Path, output_dir: Path) -> dict: + """One profile writes directly to output_dir; multiple profiles get subdirs. + + Refuse existing manifests: import is distinct from resuming a real campaign. + All profiles are validated before creating outputs. Scores stay uncoerced. + """ + fixture = json.loads(Path(input_path).read_text()) + profiles = fixture["profiles"] + if not profiles: + raise ValueError("fixture must contain at least one profile") + plans, names = [], set() + for profile in profiles: + name = profile["name"] + if ( + not isinstance(name, str) + or name in {"", ".", ".."} + or "/" in name + or "\\" in name + ): + raise ValueError("profile names must be single directory names") + if name in names: + raise ValueError(f"duplicate profile: {name}") + names.add(name) + ids = [c["id"] for c in profile["candidates"]] + if any(not isinstance(cid, str) or not cid for cid in ids) or len(ids) != len( + set(ids) + ): + raise ValueError(f"candidate IDs must be unique nonempty strings in {name}") + unknown = set(profile["filters"]) - {item[0] for item in FILTER_METRICS} + if unknown: + raise ValueError(f"unsupported filters: {sorted(unknown)}") + if any( + value is not None + and (type(value) not in (int, float) or not math.isfinite(value)) + for value in profile["filters"].values() + ): + raise ValueError("filter thresholds must be finite numbers or null") + if any( + type(c.get("is_control", False)) is not bool for c in profile["candidates"] + ): + raise ValueError("is_control flags must be booleans") + if not isinstance(profile["rank_by"], str) or not profile["rank_by"]: + raise ValueError("rank_by must name a score") + root = Path(output_dir) / name if len(profiles) > 1 else Path(output_dir) + if root.exists() and any(root.iterdir()): + raise FileExistsError(f"output directory is not empty: {root}") + plans.append((profile, root)) + results = {} + for profile, root in plans: + manifest = Manifest.create( + root, + copy.deepcopy(fixture["target"]), + mode="offline-bookkeeping", + params={"rank_by": profile["rank_by"]}, + filters=copy.deepcopy(profile["filters"]), + ) + manifest.data["provenance"] = fixture.get("provenance") + for candidate in profile["candidates"]: + record = manifest.upsert_candidate(candidate["id"]) + record.update( + copy.deepcopy(candidate) + ) # Preserve explicit nulls and absent score keys. + results[profile["name"]] = export_run(manifest) + return results + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input", type=Path, help="synthetic JSON fixture containing target and profiles" + ) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + try: + result = run_fixture(args.input, args.output_dir) + except (ValueError, KeyError, TypeError, OSError) as exc: + parser.exit(1, f"Offline import failed: {exc}\n") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/pdb_utils.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/pdb_utils.py index 74445711..73c4c903 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/pdb_utils.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/pdb_utils.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 """Dependency-free PDB parsing helpers for binder-design handoffs. @@ -7,6 +8,12 @@ - one-letter sequence from CA atoms - map PDB author residue numbers -> 1-based sequence index (hotspot/pocket remap) - CA coordinates for RMSD + +Usage: Import extract_chain, sequence, remap_to_seq_index, or structure_ca_coords. +Arguments: Structure text, explicit chain ID, and author residue IDs where needed. +Output: Selected PDB text, sequence, residue indices, or CA coordinate tuples. +Exit codes: Not applicable to this library; missing/ambiguous residues or chains + raise KeyError/ValueError. mmCIF input requires Biotite. """ from __future__ import annotations @@ -17,37 +24,79 @@ "TYR": "Y", "VAL": "V", "MSE": "M", "SEC": "U", "PYL": "O", } +# Fixed-width PDB ATOM columns, expressed as zero-based offsets/slices. +ATOM_NAME = slice(12, 16) +ALTLOC = 16 +RESIDUE_NAME = slice(17, 20) +CHAIN_ID = 21 +AUTHOR_RESIDUE_ID = slice(22, 27) +AUTHOR_RESIDUE_NUMBER = slice(22, 26) +INSERTION_CODE = 26 +X_COORD = slice(30, 38) +Y_COORD = slice(38, 46) +Z_COORD = slice(46, 54) +OCCUPANCY = slice(54, 60) + + +def _first_model_lines(pdb_text): + for line in pdb_text.splitlines(): + if line.startswith("ENDMDL"): + break + yield line + def _iter_atom_lines(pdb_text, chain=None): - for line in pdb_text.splitlines(): + for line in _first_model_lines(pdb_text): if not line.startswith("ATOM"): continue - if len(line) < 54: + if len(line) < Z_COORD.stop: continue - if chain is not None and line[21] != chain: + if chain is not None and line[CHAIN_ID] != chain: continue yield line def extract_chain(pdb_text, chain): - """Return PDB text containing only records for ``chain``.""" + """First model and chosen CA conformer; retain chain and author numbering.""" keep = [] - for line in pdb_text.splitlines(): - if line.startswith(("ATOM", "HETATM", "TER")) and len(line) > 21 and line[21] == chain: + selected = {line[AUTHOR_RESIDUE_ID]: line for line in _selected_ca_lines(pdb_text, chain)} + for line in _first_model_lines(pdb_text): + if line.startswith(("ATOM", "HETATM", "TER")) and len(line) > CHAIN_ID and line[CHAIN_ID] == chain: + choice = selected.get(line[AUTHOR_RESIDUE_ID]) + if line.startswith("ATOM") and choice: + if line[ATOM_NAME].strip() == "CA" and line != choice: + continue + if choice[ALTLOC].strip() and line[ALTLOC] not in (" ", choice[ALTLOC]): + continue keep.append(line) return "\n".join(keep) -def ca_residues(pdb_text, chain=None): - """Ordered list of (resName, resSeq, iCode, (x, y, z)) for CA atoms.""" - out = [] +def _selected_ca_lines(pdb_text, chain=None): + chosen = {} for line in _iter_atom_lines(pdb_text, chain): - if line[12:16].strip() != "CA": + if line[ATOM_NAME].strip() != "CA": continue - res_name = line[17:20].strip() - res_seq = int(line[22:26]) - icode = line[26].strip() - x = float(line[30:38]); y = float(line[38:46]); z = float(line[46:54]) + key = (line[CHAIN_ID], line[AUTHOR_RESIDUE_ID]) + occupancy = float(line[OCCUPANCY].strip() or 0) if len(line) >= OCCUPANCY.stop else 0 + priority = (bool(line[ALTLOC].strip()), -occupancy, line[ALTLOC]) + if key not in chosen or priority < chosen[key][0]: + chosen[key] = (priority, line) + return [line for _, line in chosen.values()] + + +def ca_residues(pdb_text, chain=None): + """First model, one CA per author residue, preserving insertion codes. + + Prefer a blank alternate location, otherwise highest occupancy, then the + lexicographically first altloc on a tie. Residue order is file order. + """ + out = [] + for line in _selected_ca_lines(pdb_text, chain): + res_name = line[RESIDUE_NAME].strip() + res_seq = int(line[AUTHOR_RESIDUE_NUMBER]) + icode = line[INSERTION_CODE].strip() + x, y, z = (float(line[field]) for field in (X_COORD, Y_COORD, Z_COORD)) out.append((res_name, res_seq, icode, (x, y, z))) return out @@ -60,14 +109,16 @@ def sequence(pdb_text, chain=None): def residue_index_map(pdb_text, chain=None): """Map PDB author residue id -> 1-based sequence index (CA order). - Keys are stored both as the bare author number ('501') and, when an - insertion code is present, as number+icode ('501A'). + '501' and '501A' are distinct keys. Never alias an insertion residue to a + bare number: that silently moves a hotspot. Specify a chain when numbering + overlaps across chains. """ mapping = {} for i, (_, res_seq, icode, _) in enumerate(ca_residues(pdb_text, chain), start=1): - mapping[str(res_seq)] = i - if icode: - mapping[f"{res_seq}{icode}"] = i + key = f"{res_seq}{icode}" + if key in mapping: + raise ValueError(f"ambiguous author residue {key!r}; specify one chain") + mapping[key] = i return mapping @@ -89,3 +140,28 @@ def remap_to_seq_index(pdb_text, chain, author_resnums): def ca_coords(pdb_text, chain=None): """List of (x, y, z) for CA atoms in chain order.""" return [r[3] for r in ca_residues(pdb_text, chain)] + + +def structure_ca_coords(structure_text, chain, *, format="pdb"): + """Extract one chain's C-alpha coordinates from model 1 of PDB or mmCIF. + + mmCIF uses label chain IDs (the polymer IDs in the prediction request) and + requires biotite. PDB-only callers retain the dependency-free parser. + """ + if format.lower() == "pdb": + # Only the first model: concatenating models corrupts RMSD matching. + coords = ca_coords(structure_text.split("ENDMDL", 1)[0], chain) + elif format.lower() in {"cif", "mmcif"}: + from io import StringIO + from biotite.structure.io import pdbx + + atoms = pdbx.get_structure( + pdbx.CIFFile.read(StringIO(structure_text)), model=1, + use_author_fields=False, + ) + coords = atoms.coord[(atoms.chain_id == chain) & (atoms.atom_name == "CA")].tolist() + else: + raise ValueError(f"unsupported structure format: {format}") + if not coords: + raise ValueError(f"no CA atoms found for chain {chain!r}") + return coords diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/refold_cache.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/refold_cache.py new file mode 100644 index 00000000..3a0da8bb --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/refold_cache.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Reconcile a synthetic campaign with saved responses; never submit inference. + +The cache binding contract is documented in references/offline.md. This is an +offline fixture/recovery adapter, not an endpoint client or a provenance attestor. + +Usage: python3 refold_cache.py manifest.json --output-dir DIR +Arguments: A synthetic offline-bookkeeping manifest and a new directory outside its run. +Output: Copied run with verified metrics or per-candidate blocked reasons, both CSVs, + summary.json, recovered complexes, and a JSON summary on stdout. +Exit codes: 0 recovery/export completed (inspect blocked candidates); 1 invalid run + or I/O failure; 2 invalid CLI arguments. No inference requests are submitted. +""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import math +import shutil +from pathlib import Path + +from biotite import InvalidFileError + +from manifest import Manifest +from offline_bookkeeping import export_run + +PLDDT_PERCENT_MAX = 100 +ARTIFACT_ID_HEX_LENGTH = 16 + + +class CacheError(ValueError): + """A saved response cannot safely supply scores for this candidate.""" + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def local_file(root: Path, name: str) -> Path: + path = (root / name).resolve() + if not path.is_relative_to(root.resolve()): + raise CacheError("artifact_outside_run") + return path + + +def checked_bytes(root: Path, record: dict) -> bytes: + try: + value = local_file(root, record["path"]).read_bytes() + except FileNotFoundError as exc: + raise CacheError("missing_artifact") from exc + if digest(value) != record["sha256"]: + raise CacheError("artifact_digest_mismatch") + return value + + +def finite(value): + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + ) + + +def binder_plddt(text, format, chain, sequence_length, scale): + """Mean CA confidence for an explicitly selected binder chain and scale.""" + from biotite.structure.io import pdb, pdbx + + if format == "pdb": + atoms = pdb.PDBFile.read(io.StringIO(text)).get_structure( + model=1, extra_fields=["b_factor"] + ) + elif format in {"cif", "mmcif"}: + atoms = pdbx.get_structure( + pdbx.CIFFile.read(io.StringIO(text)), + model=1, + use_author_fields=False, + extra_fields=["b_factor"], + ) + else: + raise CacheError("unsupported_structure_format") + selected = atoms[(atoms.chain_id == chain) & (atoms.atom_name == "CA")] + if len(selected) != sequence_length or not len(selected): + raise CacheError("binder_length_mismatch") + if scale not in {"0-1", "0-100"}: + raise CacheError("plddt_scale_unresolved") + values = [float(v) for v in selected.b_factor] + upper = 1 if scale == "0-1" else PLDDT_PERCENT_MAX + if any(not math.isfinite(v) or not 0 <= v <= upper for v in values): + raise CacheError("invalid_plddt") + return sum(values) / len(values) * (PLDDT_PERCENT_MAX / upper) + + +def extract_scores(manifest: Manifest, candidate: dict, root: Path): + binding = candidate.get("prediction") + if not binding: + raise CacheError("missing_prediction") + target = checked_bytes(root, manifest.data["target"]["structure"]) + if binding["candidate_id"] != candidate["id"]: + raise CacheError("candidate_mismatch") + if binding["target_sha256"] != digest(target): + raise CacheError("target_mismatch") + if binding["sequence_sha256"] != digest(candidate["sequence"].encode()): + raise CacheError("sequence_mismatch") + if binding["binder_chain"] == binding["target_chain"]: + raise CacheError("chain_roles_overlap") + raw = checked_bytes(root, binding["response"]) + response = json.loads(raw) + index = binding["sample_index"] + if not isinstance(index, int) or isinstance(index, bool) or index < 0: + raise CacheError("invalid_sample_index") + scores = {} + if binding["provider"] == "boltz2": + structures = response["structures"] + confidences = response.get("confidence_scores", []) + if len(confidences) != len(structures): + raise CacheError("sample_count_mismatch") + if index >= len(structures): + raise CacheError("invalid_sample_index") + sample = structures[index] + scores["boltz2_confidence"] = confidences[index] + elif binding["provider"] == "openfold3": + outputs = [ + o for o in response["outputs"] if o.get("input_id") == candidate["id"] + ] + if len(outputs) != 1: + raise CacheError("candidate_mismatch") + samples = outputs[0]["structures_with_scores"] + if index >= len(samples): + raise CacheError("invalid_sample_index") + sample = samples[index] + if "iptm_score" in sample: + scores["iptm"] = sample["iptm_score"] + else: + raise CacheError("unsupported_provider") + if any(not finite(v) or not 0 <= v <= 1 for v in scores.values()): + raise CacheError("invalid_confidence") + text, format = sample["structure"], sample["format"] + if digest(text.encode()) != binding["complex_sha256"]: + raise CacheError("sample_digest_mismatch") + # Both chains must be present; confidence cannot establish a missing complex. + from pdb_utils import structure_ca_coords + + structure_ca_coords(text, binding["target_chain"], format=format) + scores["binder_plddt"] = binder_plddt( + text, + format, + binding["binder_chain"], + len(candidate["sequence"]), + binding["plddt_scale"], + ) + if "self_consistency_rmsd" in manifest.missing_scores(candidate["id"]): + from metrics import ca_rmsd_from_structures + + backbone = checked_bytes(root, binding["backbone"]) + scores["self_consistency_rmsd"] = ca_rmsd_from_structures( + backbone.decode(), + text, + backbone_chain=binding["backbone_chain"], + predicted_chain=binding["binder_chain"], + predicted_format=format, + ) + return scores, text, format, binding + + +def recover(input_manifest: Path, output_dir: Path) -> dict: + """Copy an interrupted fixture; preserve complete results and failed stages.""" + input_manifest, output_dir = ( + Path(input_manifest).resolve(), + Path(output_dir).resolve(), + ) + if input_manifest.name != "manifest.json": + raise ValueError("input must be named manifest.json") + if output_dir.is_relative_to(input_manifest.parent): + raise ValueError("output must be outside the source run") + if Manifest.load(input_manifest).data.get("mode") != "offline-bookkeeping": + raise ValueError( + "this helper accepts explicitly synthetic offline-bookkeeping runs only" + ) + # Copy once; refusing an existing destination prevents accidental overwrites. + shutil.copytree(input_manifest.parent, output_dir) + manifest = Manifest.load(output_dir) + manifest.data["run_dir"] = str(output_dir) + manifest.data["mode"] = "offline-bookkeeping" + manifest.data["source_manifest_sha256"] = digest(input_manifest.read_bytes()) + for candidate in manifest.data["candidates"]: + missing = manifest.missing_scores(candidate["id"]) + if not missing: + candidate["cache_status"] = "complete" + candidate.pop("cache_reason", None) + continue + try: + scores, structure, format, binding = extract_scores( + manifest, candidate, output_dir + ) + updates = {key: value for key, value in scores.items() if key in missing} + if set(missing) - updates.keys(): + raise CacheError("missing_required_scores") + # Commit metrics only after all bindings and required scores validate. + complexes = output_dir / "recovered_complexes" + complexes.mkdir(exist_ok=True) + name = digest(candidate["id"].encode())[:ARTIFACT_ID_HEX_LENGTH] + ( + ".pdb" if format == "pdb" else ".cif" + ) + path = complexes / name + path.write_text(structure) + candidate["scores"].update(updates) + candidate.setdefault("artifacts", {})["complex"] = str( + path.relative_to(output_dir) + ) + candidate["score_provenance"] = dict(binding) + candidate["cache_status"] = "recovered" + candidate.pop("cache_reason", None) + except ( + CacheError, + KeyError, + IndexError, + TypeError, + ValueError, + OSError, + AttributeError, + InvalidFileError, + ) as exc: + candidate["cache_status"] = "blocked" + candidate["cache_reason"] = str(exc) + return export_run(manifest) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + try: + result = recover(args.manifest, args.output_dir) + except (ValueError, KeyError, TypeError, OSError) as exc: + parser.exit(1, f"Cache recovery failed: {exc}\n") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/registry.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/registry.py index cfa644bd..f4e150c7 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/registry.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/scripts/registry.py @@ -1,6 +1,14 @@ +#!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 -"""Benchmark target registry loader (reads assets/targets.json).""" +"""Benchmark target registry loader (default: the empty assets/targets.json template). + +Usage: Import load_registry or get_target; no command-line API. +Arguments: Optional registry JSON path; get_target also takes a target name or alias. +Output: Parsed registry or matching target dictionary; no files are written. +Exit codes: Not applicable to this library; missing targets raise KeyError, + malformed JSON and unreadable paths raise their standard Python exceptions. +""" from __future__ import annotations import json diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/skill-card.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/skill-card.md new file mode 100644 index 00000000..955160c0 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/skill-card.md @@ -0,0 +1,99 @@ +## Description:
+Orchestrate an end-to-end de novo protein binder design campaign against a protein target by composing BioNeMo NIM skills.
+ +Hosted campaigns send target structures/sequences, candidate sequences, and requested alignments to the selected NVIDIA NIM endpoints under the service's terms. Local campaigns use the configured local endpoints; container and model downloads contact NGC. Campaigns write JSON manifests, CSV rankings, and available PDB/mmCIF/FASTA artifacts to the chosen run directory. Offline bookkeeping uses local synthetic fixtures without inference calls.
+ +The published evaluation of revision `1213167` has a **NEUTRAL** verdict; readiness for deployment has not been established. The [benchmark](BENCHMARK.md) covers one offline bookkeeping task, not a live binder-design campaign. Gather further evidence or improve and re-evaluate the skill before a publication or deployment decision. License terms below describe permitted use, not validation readiness.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and computational biologists designing de novo protein binders against protein targets using agent-orchestrated BioNeMo NIM pipelines (RFdiffusion, ProteinMPNN, Boltz2/OpenFold3).
+ +Activate for binder design, validation/ranking, or campaign-resume requests (for example, “design protein binders” or “resume this binder manifest”). Code review, method explanation, and evaluation-report review do not authorize a live design campaign. Resolve missing target, budget, and deployment choices before inference; reuse choices already supplied by the user.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Yes for hosted inference and NGC downloads; no for offline bookkeeping]
+**Credential Type(s):** [API key]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +Risk: Hosted inference transmits potentially confidential protein data to the selected external service; campaign artifacts also persist on local disk.
+Mitigation: Honor the user's selected deployment and output directory, resolve an unspecified destination before uploading data, and keep credentials out of saved outputs. Use local services when required by the user's data restrictions.
+ +Risk: Generated shell commands act on the configured environment; target descriptions and structure annotations may contain untrusted instructions.
+Mitigation: Treat input annotations as data, check generated commands against the user's requested operation and output paths, and pass data as quoted arguments. Use the documented credential wrapper and keep secrets out of command text and logs.
+ +Risk: Predicted confidence does not prove experimental binding.
+Mitigation: Preserve metric provenance, disclose missing scores and uncalibrated controls, and distinguish synthetic bookkeeping from biological validation.
+ +## Reference(s):
+- [pipeline.md](references/pipeline.md)
+- [validation.md](references/validation.md)
+- [manifest.md](references/manifest.md)
+- [local-nim-setup.md](references/local-nim-setup.md)
+- [NVIDIA BioNeMo NIMs](https://build.nvidia.com)
+ + +## Skill Output:
+**Output Type(s):** [Shell commands, Files, Analysis]
+**Output Format:** [Markdown with inline bash code blocks, JSON manifest, CSV ranking]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Produces manifest.json, candidates.csv, and PDB artifact files per campaign run]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-5`)
+- Codex (`openai/openai/gpt-5.5`)
+ + + +## Evaluation Tasks:
+1 evaluation task (1 positive) executed in isolated k8s-sandbox pods; evaluator version 1.5.6.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Final-answer correctness against the reference answer.
+- Discoverability: Whether the expected skill was selected and activated when needed.
+- Effectiveness: Whether the skill helped complete the user's goal (50% goal_accuracy + 50% behavior_check).
+- Efficiency: Tool-call productivity and token usage efficiency (50% skill_efficiency + 50% token_efficiency).
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `skill_execution`: Whether the expected skill was selected, decoys avoided, and workflow executed.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `skill_efficiency`: Tool-call productivity (routing scored under Discoverability).
+- `token_efficiency`: Actual uncached prompt plus completion token usage.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 83.8% — uplift unavailable | 85.5% — uplift unavailable | +| Security | 100.0% → 100.0% (±0.0 pts) | 100.0% → 100.0% (±0.0 pts) | +| Correctness | 100.0% → 100.0% (±0.0 pts) | 100.0% → 100.0% (±0.0 pts) | +| Discoverability | 35.0% — uplift unavailable | 85.0% — uplift unavailable | +| Effectiveness | 100.0% → 100.0% (±0.0 pts) | 100.0% → 58.3% (-41.7 pts) | +| Efficiency | 83.9% — uplift unavailable | 84.2% — uplift unavailable | + +## Skill Version(s):
+1213167 (source: git SHA, committed 2026-10-01)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/skill.oms.sig b/skills/bionemo-agent-toolkit/skills/protein-binder-design/skill.oms.sig new file mode 100644 index 00000000..5121beb0 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAicHJvdGVpbi1iaW5kZXItZGVzaWduIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjA3MzdiMGNkMzcwN2ExOGViYmEzM2EzNDRkNmMyYzZlMDZiZmMzMDJhYzBiMmI4ZDU3YjZlZjM1NDIyZWJlZjkiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjk4YWJjNWUyNWI5NTM5NDA2OWNlMjViZDA5ZGRmMjdjYmE2MDA4Yzk5NjFiOTUzMWM1ZDcyMGFhZGU3NzllZTMiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJMSUNFTlNFIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI3NWVmYTA3YzRkMmFmYjE0ZDgyMjZjZjAwYjc3NjNkZDJlYzFjNTU4NWE5ZGZlOTE5ODgxYTNhM2ZlZjJkZGFjIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiUkVBRE1FLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkM2EzMDdmYzljMDM4NDdkY2Y2ODRmYTM0YTEzYzYwZTJkOTM2NTM0ZjljNjI3ODNiODE3NjA1NjAzZmRhNTdhIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImRjNzFlMDZkOTczMzdmODQ1YjkyN2ZjMmU5ZWU2MTdlOTg2N2RmMDA2MDVkNDBjMjE1YmQ1MmI2YzczZGMzYTEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJhc3NldHMvdGFyZ2V0cy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIyZGNhOGM4MWRjMDlkNzYzNTU1M2MyOWJkMTJhZTA5ODljYmU1ZjQ1OGYwNWM5YWU0NTllNjQ5ZDc2MTNhNjQxIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvUkVBRE1FLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI0OTQ5MjJlYmRkZDUyN2E1NWNhNzMwZjJkZGMwNTkyMzY5NjNjNjAwMTNjMjJmN2E2YmE5MzhmNTVmODE5ZjkwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNjZlNDYwMTYxZWJhYmM4Mzk1ZDEwZmZjNzQyOGMyMmMyMDExODYxOWZlYjVlZWMxY2YyNGNiOTJlOTE2N2JhNSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2RlZmVycmVkL2xpdmUtY2FtcGFpZ25zLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQwZTNlMzhiOGNkNWFlZmMyNDdiNTAzMGVkNjZmNjFkNmRhYzRiYjE2OTdmOWQ2NjlkYjliYjBmYmViOGE0Y2UiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI0YzIyNmNmYTlmYTQ4MWEwNmM2ODdiNjdhNTA4MjZhOTZiY2ZlYzI4N2Y4MWRiMDc3ZDIyOWI2YTEwZGI1MzI2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvYm9va2tlZXBpbmcuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMjNhNzY5ZjVmM2YwMmM5NmM2NzU4ZWIzYjU3ODdiOTliZWZiZTY5Yjg5OGE2NGYxNzBmZTQxNmMyNGE4MGZkMyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL3RyaWdnZXJfZXZhbHMuanNvbiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYWFjMjM5OTYzZGIyMDNlYmU2NGU2ZGExMWNiNjQ1NTE1MDRkODM5OGQ3OWZlNjU1NDdkYTg4ZWMzMzVkNzJiYiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvbG9jYWwtbmltLXNldHVwLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJhYzc2YTA2OTYyZDBkMDA5NmFmMDRlMjJlNWY1OTc5MzAxNjBlYzAzNzc2MDg2MTNiY2VlMDUxMzAwNDdkZWYzIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9tYW5pZmVzdC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMTg2NGMyNjc3MDNhZGJmZDM2ZDQ4NTM2Yzk3NjM4YTcxMTc1OTIwZTEyMDMxM2VlOGU2OTMzMzEwMDY4M2FmMCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcGlwZWxpbmUubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjhkMDYzZTZhY2I0NzRjNTQyNGQ3OWNkOTJmYTI1MDdiNDQwOTY2MWQ4NGMxYzljNWEzZWMxZGIyOGE5ZjNiMTciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3ZhbGlkYXRpb24ubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjg3MTU1ZDc5ODUwNWY1YmQ4NTQzODBmODBkMmRjYjQ2OWU1MjA1ZWI4OTNlZmZiNDU4ODZhNzVkODJkNzc1NDgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL2NvbnRyb2xzLnB5IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJjYWUzYTYxMDE5NzY2ZjQyMzFhYzc2YWI4ZDEwMmE5ZDE1YTkxMjE3NTc5ODg1NjU1MTZiNjZmNDFjMDgwMjYwIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9ob3N0ZWRfZW52LnNoIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlNzA5M2Y2ODFhNThhNDJlODNjM2VjYWEzYmE5MGRmMDZhYmJiMjY1ODQ1MzIwNjM5MTU0N2U2YmQ5M2JlNTNiIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9tYW5pZmVzdC5weSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNmFiM2M2M2NmYzNmNGYyMGFjNWE2YjJlZjg2M2Y4MmYwYTk2MjBjMTBhM2EyMjcxYmI2MTEzMTliNDEzN2YzYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNjcmlwdHMvbWV0cmljcy5weSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMjJmMzJmMjg2ZTU5NWNlNzIzMGMxNDE0NGQ0ZDRlZjUxNGEyODM2YjU1NzZjYjE1ZTVjM2Y4ODM5YjI2Zjc1MSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNjcmlwdHMvcGRiX3V0aWxzLnB5IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxNzcxY2U4ODRhMmU3ZWU1MjhmODc2NDQ5NDQxNmIzYmUxYWM2NTM3ODcyYTNiNWJiNGNlYTRlMGZiNTY0NzdjIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9yZWdpc3RyeS5weSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNmNiMTY3MjI2MWMzMThhNGMzNzRkNjgyN2VhOWYyODE4MDQ4NGVjNTljNjc5NDc1YTAzNjEzOTJlYTNhM2NiNiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImZmODE2NWU4ZTM2NjY2YjZhZDU5MDMzZDFlYzYzMzZhMjExNmRkZTFhZTRkNzVmYWViYzA3YTg5ZDUzMTVkN2UiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJ0ZXN0cy90ZXN0X2hvc3RlZF9lbnYucHkiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjliMTI2YTkxZTcxMGE4YjI1ZDdkNmIxODcyNzljYzI1ZWEzNzc3NTgyMzNlZTE3ZmE3ODk3OTg5ZjllYTZhMjYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJ0ZXN0cy90ZXN0X21hbmlmZXN0LnB5IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI0YWJhODYzNWNhZGQ2M2RlN2NiZjE1MzNjMjY1MWUyNmUxZmI2YzVhYWE1MTY5MmY5OWFiYjFkYzcyODMyMzNmIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDAzU7mXHu/D5Q1kp7Obv9aAjDgFNBUno2+G8RyTHiw5lN+8jaaXmbtb/GMpssliCMCMHj0Z9VNLxChbDNuLovA2CxFdbIEVpXEwN5qsR6KChWDEn3eBi25emsUvK4zOQicwA==","keyid":""}]}} \ No newline at end of file diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_controls.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_controls.py new file mode 100644 index 00000000..9c0ea2c7 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_controls.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Check reproducibility, composition, and input boundaries for negative controls.""" + +import sys +import unittest +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from controls import make_scrambled_controls, scramble_sequence # noqa: E402 + + +class ScrambledControlTests(unittest.TestCase): + def test_controls_preserve_each_source_composition_and_repeat_by_seed(self): + sources = ["ACDEFGHIKLMNPQRSTVWY", "AACCGGTT"] + controls = make_scrambled_controls(sources, n=7, seed=42) + self.assertEqual(len(controls), 7) + self.assertEqual(controls, make_scrambled_controls(sources, n=7, seed=42)) + for index, control in enumerate(controls): + self.assertEqual(Counter(control), Counter(sources[index % 2])) + self.assertEqual(sources, ["ACDEFGHIKLMNPQRSTVWY", "AACCGGTT"]) + + def test_zero_count_and_empty_source_set_produce_no_controls(self): + self.assertEqual(make_scrambled_controls(["ACDE"], n=0), []) + self.assertEqual(make_scrambled_controls([]), []) + self.assertEqual(scramble_sequence("AAAA"), "AAAA") + + def test_invalid_sequences_cannot_be_sent_as_negative_controls(self): + for seq in ("", "acde", "AC X", "AX", "AC*", None, ["A", "C"]): + with self.subTest(seq=seq), self.assertRaises(ValueError): + scramble_sequence(seq) + with self.subTest(seq=seq), self.assertRaises(ValueError): + make_scrambled_controls(["ACDE", seq]) + + def test_invalid_count_seed_or_collection_is_rejected(self): + for count in (-1, 1.5, True, "5"): + with self.subTest(count=count), self.assertRaises(ValueError): + make_scrambled_controls(["ACDE"], n=count) + for seed in (None, True, 0.5, "42"): + with self.subTest(seed=seed), self.assertRaises(ValueError): + scramble_sequence("ACDE", seed=seed) + with self.subTest(seed=seed), self.assertRaises(ValueError): + make_scrambled_controls(["ACDE"], seed=seed) + for sources in ("ACDE", None, {"ACDE"}): + with self.subTest(sources=sources), self.assertRaises(ValueError): + make_scrambled_controls(sources) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_hosted_env.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_hosted_env.py new file mode 100644 index 00000000..b1f563f8 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_hosted_env.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Check that delegated processes inherit the selected hosted credential.""" +import os +import subprocess +import sys +import unittest +from pathlib import Path + + +HELPER = Path(__file__).resolve().parents[1] / "scripts" / "hosted_env.sh" + + +class HostedEnvironmentTests(unittest.TestCase): + def run_child(self, keys, expected="", legacy=False): + # Isolate from real credentials; these values are synthetic test inputs. + env = {"PATH": os.defpath, **keys} + child = [sys.executable, "-c", + "import os, sys; assert os.environ['NGC_API_KEY'] == sys.argv[1]; print('child-ran')", + expected] + command = ["bash", str(HELPER), *child] + if legacy: + command = [ + "bash", "--noprofile", "--norc", "-c", + 'source "$1" && "$2" -c "$3" "$4"', + "test-hosted-env", str(HELPER), sys.executable, + "import os, sys; assert os.environ['NGC_API_KEY'] == sys.argv[1]; print('child-ran')", + expected, + ] + return subprocess.run( + command, + env=env, shell=False, text=True, capture_output=True, + ) + + def test_selected_key_reaches_delegated_process_without_logging(self): + for keys, expected in ( + ({"NGC_API_KEY": "synthetic-ngc"}, "synthetic-ngc"), + ({"NVIDIA_API_KEY": "synthetic-fallback"}, "synthetic-fallback"), + ({"NGC_API_KEY": "", "NVIDIA_API_KEY": "synthetic-fallback"}, "synthetic-fallback"), + ({"NGC_API_KEY": "synthetic-ngc", "NVIDIA_API_KEY": "synthetic-fallback"}, "synthetic-ngc"), + ): + with self.subTest(keys=sorted(keys)): + result = self.run_child(keys, expected) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "child-ran\n") + self.assertEqual(result.stderr, "") + + def test_missing_nim_key_stops_before_delegated_call(self): + for keys in ({}, {"OPENAI_API_KEY": "synthetic-agent-key"}, {"NGC_API_KEY": "", "NVIDIA_API_KEY": ""}): + with self.subTest(keys=sorted(keys)): + result = self.run_child(keys) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + self.assertIn("Set NGC_API_KEY or NVIDIA_API_KEY", result.stderr) + self.assertNotIn("synthetic-agent-key", result.stderr) + + def test_legacy_source_still_exports_to_child(self): + result = self.run_child({"NVIDIA_API_KEY": "synthetic-fallback"}, "synthetic-fallback", legacy=True) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_wrapper_preserves_arguments_and_failure_code_without_tracing_secret(self): + result = subprocess.run( + ["bash", "-x", str(HELPER), sys.executable, "-c", + "import sys; assert sys.argv[1] == 'two words'; sys.exit(7)", "two words"], + env={"PATH": os.defpath, "NVIDIA_API_KEY": "synthetic-secret"}, + shell=False, text=True, capture_output=True, + ) + self.assertEqual(result.returncode, 7) + self.assertNotIn("synthetic-secret", result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_manifest.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_manifest.py new file mode 100644 index 00000000..7de7c759 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_manifest.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Offline bookkeeping tests; numeric fixtures are not scientific predictions.""" +import csv +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +_SPEC = importlib.util.spec_from_file_location( + "binder_manifest", Path(__file__).resolve().parents[1] / "scripts" / "manifest.py" +) +_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_MODULE) +Manifest = _MODULE.Manifest + + +class ManifestFilterTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.manifest = Manifest.create(self.temp.name, {"name": "bookkeeping fixture"}) + + def add(self, cid, **scores): + return self.manifest.set_scores(cid, **scores) + + def test_each_required_score_must_be_present(self): + full = {"iptm": 0.85, "binder_plddt": 85, "self_consistency_rmsd": 1.5} + for missing in full: + with self.subTest(missing=missing): + candidate = self.add(missing, **{k: v for k, v in full.items() if k != missing}) + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + + def test_boundaries_and_failures(self): + passing = self.add("boundary", iptm=0.8, binder_plddt=80, self_consistency_rmsd=2.0) + for cid, scores in ( + ("low_interface", dict(iptm=0.79, binder_plddt=90, self_consistency_rmsd=1.0)), + ("low_plddt", dict(iptm=0.9, binder_plddt=79, self_consistency_rmsd=1.0)), + ("high_rmsd", dict(iptm=0.9, binder_plddt=90, self_consistency_rmsd=2.01)), + ): + self.add(cid, **scores) + self.manifest.apply_filters() + self.assertTrue(passing["passed_filter"]) + self.assertEqual([c["id"] for c in self.manifest.rank(passed_only=True)], ["boundary"]) + + def test_invalid_values_cannot_pass(self): + for value in (float("inf"), float("-inf"), float("nan"), True, "0.9", None): + for metric in ("iptm", "binder_plddt", "self_consistency_rmsd"): + with self.subTest(value=value, metric=metric): + candidate = self.add("invalid", iptm=0.9, binder_plddt=90, self_consistency_rmsd=1.0) + candidate["scores"][metric] = value + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + + def test_explicitly_disabled_metric_is_optional(self): + self.manifest.data["filters"]["self_consistency_rmsd_max"] = None + candidate = self.add("disabled", iptm=0.8, binder_plddt=80) + self.manifest.apply_filters() + self.assertTrue(candidate["passed_filter"]) + + def test_no_scores_or_no_enabled_filters_cannot_pass(self): + candidate = self.add("empty") + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + self.manifest.data["filters"] = {} + self.add("empty", iptm=0.9, binder_plddt=90, self_consistency_rmsd=1.0) + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + + def test_composite_confidence_does_not_replace_iptm(self): + candidate = self.add("composite", boltz2_confidence=0.99, binder_plddt=90, self_consistency_rmsd=1.0) + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + + def test_resume_and_summary_do_not_count_incomplete_as_passed(self): + self.add("done", iptm=0.85, binder_plddt=85, self_consistency_rmsd=1.5) + self.add("pending", iptm=0.85) + self.add("control", iptm=0.85, binder_plddt=85, self_consistency_rmsd=1.5) + self.manifest.upsert_candidate("control", is_control=True, control_type="published") + self.manifest.apply_filters() + loaded = Manifest.load(self.temp.name) + self.assertEqual(loaded.summary(), {"n_candidates": 2, "n_passed": 1, "n_controls": 1}) + loaded.set_scores("pending", binder_plddt=85, self_consistency_rmsd=1.5) + loaded.apply_filters() + self.assertEqual(loaded.summary()["n_passed"], 2) + self.assertEqual(loaded.data["candidates"][0], self.manifest.data["candidates"][0]) + + def test_boltz2_profile_survives_resume_and_exports_ranked_candidates(self): + self.manifest.data["filters"] = { + "iptm_min": None, + "boltz2_confidence_min": 0.8, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2.0, + } + self.manifest.data["params"]["rank_by"] = "boltz2_confidence" + for cid, confidence in (("boundary", 0.8), ("low", 0.79), ("best", 0.9), ("control", 0.99)): + self.add(cid, boltz2_confidence=confidence, binder_plddt=85, self_consistency_rmsd=1.0) + self.add("incomplete", boltz2_confidence=0.95) + self.manifest.upsert_candidate("control", is_control=True, control_type="scrambled") + self.manifest.apply_filters() + + loaded = Manifest.load(self.temp.name) + ranked = loaded.rank(by=loaded.data["params"]["rank_by"], passed_only=True) + self.assertEqual([c["id"] for c in ranked], ["best", "boundary"]) + self.assertEqual(loaded.summary(), {"n_candidates": 4, "n_passed": 2, "n_controls": 1}) + self.assertTrue(all("iptm" not in c["scores"] for c in loaded.data["candidates"])) + with loaded.to_csv(candidates=ranked).open() as stream: + rows = list(csv.DictReader(stream)) + self.assertEqual([row["id"] for row in rows], ["best", "boundary"]) + with loaded.to_csv(candidates=[]).open() as stream: + self.assertEqual(list(csv.DictReader(stream)), []) + + def test_boltz2_missing_or_invalid_confidence_cannot_pass_or_rank(self): + self.manifest.data["filters"] = {"boltz2_confidence_min": 0.8} + for value in (None, float("inf"), float("-inf"), float("nan"), True, "0.9"): + with self.subTest(value=value): + candidate = self.add("invalid") + candidate["scores"]["boltz2_confidence"] = value + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + self.assertEqual(self.manifest.rank(by="boltz2_confidence"), []) + + def test_create_respects_explicit_empty_filters(self): + manifest = Manifest.create(self.temp.name, {"name": "bookkeeping fixture"}, filters={}) + candidate = manifest.set_scores("no_filters", iptm=0.9, binder_plddt=90, self_consistency_rmsd=1.0) + manifest.apply_filters() + self.assertEqual(manifest.data["filters"], {}) + self.assertFalse(candidate["passed_filter"]) + + def test_sequence_controls_skip_only_inapplicable_rmsd(self): + control = self.manifest.upsert_candidate("control", is_control=True, control_type="published") + self.add("control", iptm=0.85, binder_plddt=85) + design = self.add("design", iptm=0.85, binder_plddt=85) + self.manifest.apply_filters() + self.assertTrue(control["passed_filter"]) + self.assertFalse(design["passed_filter"]) + self.assertIn("self_consistency_rmsd", control["filter_exemptions"]) + self.assertEqual(self.manifest.missing_scores("design"), ["self_consistency_rmsd"]) + self.assertEqual(self.manifest.missing_scores("control"), []) + self.add("control", iptm=0.2) + self.manifest.apply_filters() + self.assertFalse(control["passed_filter"]) + + def test_control_with_reference_backbone_requires_rmsd(self): + control = self.manifest.upsert_candidate("control", is_control=True) + self.manifest.add_artifact("control", "backbone_pdb", "reference.pdb") + self.add("control", iptm=0.85, binder_plddt=85) + self.manifest.apply_filters() + self.assertFalse(control["passed_filter"]) + self.assertEqual(control["filter_exemptions"], {}) + self.add("control", self_consistency_rmsd=1.0) + self.manifest.apply_filters() + self.assertTrue(control["passed_filter"]) + + def test_resume_distinguishes_missing_metrics_from_completed_low_scores(self): + self.add("low", iptm=0.1, binder_plddt=85, self_consistency_rmsd=1.0) + self.add("incomplete", iptm=0.9, binder_plddt=float("nan")) + loaded = Manifest.load(self.temp.name) + self.assertEqual(loaded.missing_scores("low"), []) + self.assertEqual(loaded.missing_scores("incomplete"), ["binder_plddt", "self_consistency_rmsd"]) + with self.assertRaises(KeyError): + loaded.missing_scores("unknown") + + def test_evaluation_fixtures_export_only_ranked_survivors_after_resume(self): + fixtures = Path(__file__).resolve().parents[1] / "evals" / "files" + expectations = { + "bookkeeping.json": (["best", "boundary"], 4, 2), + "boltz2_only.json": (["binder_a", "binder_b"], 4, 2), + "mixed_thresholds.json": (["binder_a", "binder_b"], 6, 2), + "no_survivors.json": ([], 4, 0), + } + for filename, (expected_ids, n_candidates, n_passed) in expectations.items(): + fixture = json.loads((fixtures / filename).read_text()) + for profile in fixture["profiles"]: + with self.subTest(fixture=filename, profile=profile["name"]): + run_dir = Path(self.temp.name) / Path(filename).stem / profile["name"] + manifest = Manifest.create( + run_dir, fixture["target"], + filters=profile["filters"], params={"rank_by": profile["rank_by"]}, + ) + for entry in profile["candidates"]: + manifest.upsert_candidate( + entry["id"], is_control=entry.get("is_control", False), + control_type=entry.get("control_type"), + ) + manifest.set_scores(entry["id"], **entry["scores"]) + + loaded = Manifest.load(run_dir) + loaded.apply_filters() + ranked = loaded.rank( + by=loaded.data["params"]["rank_by"], descending=True, + passed_only=True, include_controls=False, + ) + loaded.to_csv(run_dir / "all_candidates.csv") + loaded.to_csv(run_dir / "candidates.csv", candidates=ranked) + + with (run_dir / "candidates.csv").open() as stream: + reader = csv.DictReader(stream) + self.assertIn("id", reader.fieldnames) + rows = list(reader) + self.assertEqual([row["id"] for row in rows], expected_ids) + self.assertTrue(all(row["is_control"] == "False" for row in rows)) + self.assertTrue(all(row["passed_filter"] == "True" for row in rows)) + with (run_dir / "all_candidates.csv").open() as stream: + audit_rows = list(csv.DictReader(stream)) + self.assertEqual( + [row["id"] for row in audit_rows], + [entry["id"] for entry in profile["candidates"]], + ) + saved = Manifest.load(run_dir) + self.assertEqual(saved.summary(), { + "n_candidates": n_candidates, "n_passed": n_passed, "n_controls": 1, + }) + self.assertEqual(saved.data["filters"], profile["filters"]) + self.assertEqual(saved.data["params"]["rank_by"], profile["rank_by"]) + for entry, candidate in zip(profile["candidates"], saved.data["candidates"]): + self.assertEqual(candidate["id"], entry["id"]) + self.assertEqual(candidate["scores"], entry["scores"]) + self.assertEqual(candidate["is_control"], entry.get("is_control", False)) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_metrics.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_metrics.py new file mode 100644 index 00000000..c2f73081 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_metrics.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Offline geometric fixtures for explicit PDB/mmCIF binder-chain handoffs.""" +import io +import sys +import unittest +from pathlib import Path + +import biotite.structure.io.pdb as pdb +import biotite.structure.io.pdbx as pdbx + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from metrics import ca_rmsd_from_pdb, ca_rmsd_from_structures, kabsch_rmsd + + +def pdb_text(chains): + lines = [] + for chain, coords in chains: + for resid, (x, y, z) in enumerate(coords, 1): + lines.append(f"ATOM {len(lines)+1:5d} CA ALA {chain}{resid:4d} " + f"{x:8.3f}{y:8.3f}{z:8.3f}{1.0:6.2f}{80.0:6.2f} C ") + return "\n".join([*lines, "END", ""]) + + +class StructureMetricTests(unittest.TestCase): + def setUp(self): + self.coords = [(0, 0, 0), (4, 0, 0), (0, 3, 0)] + self.backbone = pdb_text([("D", self.coords)]) + rotated = [(8, 7, 0), (8, 11, 0), (5, 7, 0)] + atoms = pdb.PDBFile.read(io.StringIO(pdb_text([("T", [(0, 0, 0)]), ("Y", rotated)]))).get_structure(model=1) + cif = pdbx.CIFFile() + pdbx.set_structure(cif, atoms) + # Prediction label IDs differ from author IDs: the request named binder B. + cif.block["atom_site"]["label_asym_id"] = pdbx.CIFColumn(["A", "B", "B", "B"]) + stream = io.StringIO() + cif.write(stream) + self.prediction = stream.getvalue() + + def test_cif_uses_requested_binder_chain_and_aligns_rigid_rotation(self): + value = ca_rmsd_from_structures(self.backbone, self.prediction, + backbone_chain="D", predicted_chain="B") + self.assertAlmostEqual(value, 0.0, places=6) + + def test_missing_or_wrong_chain_does_not_produce_a_partial_rmsd(self): + for chain in ("Y", "absent", "A"): + with self.subTest(chain=chain), self.assertRaises(ValueError): + ca_rmsd_from_structures(self.backbone, self.prediction, + backbone_chain="D", predicted_chain=chain) + + def test_pdb_length_mismatch_is_not_silently_truncated(self): + with self.assertRaisesRegex(ValueError, "shape mismatch"): + ca_rmsd_from_pdb(self.backbone, pdb_text([("B", self.coords[:2])]), "D", "B") + + def test_nonfinite_coordinates_fail_before_alignment(self): + for value in (float("nan"), float("inf")): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, "non-finite"): + kabsch_rmsd(self.coords, [(value, 0, 0), *self.coords[1:]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_offline.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_offline.py new file mode 100644 index 00000000..74e804e1 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_offline.py @@ -0,0 +1,474 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Execute offline workflows, then challenge the independent artifact verifier.""" + +import copy +import importlib.util +import json +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +SKILL = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SKILL / "scripts")) +from offline_bookkeeping import run_fixture # noqa: E402 - skill scripts are not an installed package +from pdb_utils import ( # noqa: E402 - skill scripts are not an installed package + ca_coords, + extract_chain, + remap_to_seq_index, + residue_index_map, + sequence, +) +from refold_cache import digest, recover # noqa: E402 - skill scripts are not an installed package + +spec = importlib.util.spec_from_file_location( + "artifact_grader", SKILL / "evals/grader.py" +) +grader = importlib.util.module_from_spec(spec) +spec.loader.exec_module(grader) +CASES = { + case["id"]: case + for case in json.loads((SKILL / "evals/evals.json").read_text())["evals"] +} + + +def write_json(path, data): + path.write_text(json.dumps(data, indent=2) + "\n") + + +def prepare_target(workspace): + """Exercise the public PDB helpers; grading uses independently pinned results.""" + source = workspace / "input/target_prep" + request = json.loads((source / "request.json").read_text()) + text = (source / request["structure"]).read_text() + chain = request["chain"] + result = { + "chain": chain, + "sequence": sequence(text, chain), + "author_to_sequence": residue_index_map(text, chain), + "simulated": True, + "live_inference": False, + "requests": {}, + } + for item in request["requests"]: + missing = [r for r in item["hotspots"] if r not in result["author_to_sequence"]] + if missing: + result["requests"][item["id"]] = { + "status": "blocked", + "missing_author_residues": missing, + } + else: + result["requests"][item["id"]] = { + "status": "ready", + "hotspot_res": [chain + r for r in item["hotspots"]], + "sequence_indices": remap_to_seq_index(text, chain, item["hotspots"]), + } + output = workspace / "output/target-prep" + output.mkdir(parents=True) + (output / "target.pdb").write_text(extract_chain(text, chain)) + write_json(output / "preparation.json", result) + + +class OfflineWorkflowTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.workspace = Path(self.temp.name) + + def stage(self, case): + for item in CASES[case]["files"]: + source = SKILL / item + destination = ( + self.workspace / "input" / source.relative_to(SKILL / "evals/files") + ) + destination.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, destination) + else: + shutil.copy2(source, destination) + + def execute(self, case): + if case in grader.BOOKKEEPING: + filename, destination, _ = grader.BOOKKEEPING[case] + output = self.workspace / "output" / (destination or "bookkeeping") + run_fixture(self.workspace / "input" / filename, output) + elif case in grader.CACHE_CASES: + for name in grader.CACHE_CASES[case]: + destination = grader.CACHE_RUNS[name][0] + recover( + self.workspace / "input" / name / "manifest.json", + self.workspace / "output" / destination, + ) + else: + prepare_target(self.workspace) + + def assert_grade(self, case, expected): + result = grader.grade(case, self.workspace) + failures = [ + c["message"] + for c in result["details"]["artifact_correctness"]["checks"] + if not c["passed"] + ] + self.assertEqual( + result["custom_metrics"]["artifact_correctness"], + expected, + "\n".join(failures), + ) + + def test_all_cases_require_artifacts_and_pass_after_real_execution(self): + for case in CASES: + with self.subTest(case=case), tempfile.TemporaryDirectory() as root: + self.workspace = Path(root) + self.stage(case) + before = { + p: digest(p.read_bytes()) + for p in (self.workspace / "input").rglob("*") + if p.is_file() + } + self.assert_grade(case, 0) + self.execute(case) + self.assert_grade(case, 1) + self.assertEqual(before, {p: digest(p.read_bytes()) for p in before}) + + def test_original_scores_are_not_coerced_and_absent_keys_stay_absent(self): + case = "offline-mixed-threshold-ranking" + self.stage(case) + self.execute(case) + data = json.loads( + (self.workspace / "output/mixed-thresholds/manifest.json").read_text() + ) + rows = {c["id"]: c for c in data["candidates"]} + self.assertEqual(rows["invalid_score"]["scores"]["iptm"], "0.95") + self.assertNotIn("self_consistency_rmsd", rows["pending"]["scores"]) + self.assertFalse(rows["invalid_score"]["passed_filter"]) + + def test_bookkeeping_verifier_accepts_equivalent_minimal_manifest(self): + case = "offline-boltz2-ranking" + self.stage(case) + self.execute(case) + path = self.workspace / "output/boltz2-only/manifest.json" + data = json.loads(path.read_text()) + # A baseline need not reproduce helper-specific timestamps or schema extras. + minimal = {key: data[key] for key in ("target", "filters", "candidates")} + minimal["rank_by"] = data["params"]["rank_by"] + for candidate in minimal["candidates"]: + for key in ( + "created", + "backbone_id", + "filter_metrics", + "filter_exemptions", + "passed_filter", + ): + candidate.pop(key, None) + write_json(path, minimal) + self.assert_grade(case, 1) + + def test_target_verifier_accepts_equivalent_pdb_writers(self): + case = "offline-target-preparation" + self.stage(case) + self.execute(case) + path = self.workspace / "output/target-prep/target.pdb" + atoms = [ + line for line in path.read_text().splitlines() if line.startswith("ATOM") + ] + rewritten = [] + for serial, line in enumerate(atoms, 101): + # Change serials, atom-name alignment and numeric formatting; clear + # altLoc after choosing B, and omit unused trailing PDB fields. + xyz = "".join( + f"{float(line[start : start + 8]):8.2f}" for start in (30, 38, 46) + ) + rewritten.append(f"ATOM {serial:5d} CA {line[17:30]}{xyz}") + for model_records in (False, True): + with self.subTest(model_records=model_records): + text = "\n".join(rewritten) + "\n" + if model_records: + text = "MODEL 1\n" + text + "ENDMDL\n" + path.write_text(text + "END\n") + self.assert_grade(case, 1) + + def test_target_verifier_rejects_duplicate_conformers_and_wrong_structure(self): + case = "offline-target-preparation" + self.stage(case) + self.execute(case) + path = self.workspace / "output/target-prep/target.pdb" + atoms = [ + line for line in path.read_text().splitlines() if line.startswith("ATOM") + ] + source = ( + (self.workspace / "input/target_prep/target.pdb").read_text().splitlines() + ) + alternative = next( + line for line in source if line.startswith("ATOM") and line[16] == "A" + ) + changed_serial = atoms[1][:6] + " 999" + atoms[1][11:] + mutations = { + "both conformers": atoms[:2] + [alternative] + atoms[2:], + "same atom, new serial": atoms + [changed_serial], + "wrong conformer": atoms[:1] + [alternative] + atoms[2:], + "missing atom": atoms[:-1], + "wrong chain": [atoms[0][:21] + "A" + atoms[0][22:]] + atoms[1:], + "lost insertion code": atoms[:2] + + [atoms[2][:26] + " " + atoms[2][27:]] + + atoms[3:], + "wrong residue": [atoms[0][:17] + "VAL" + atoms[0][20:]] + atoms[1:], + "wrong coordinates": [atoms[0][:30] + " 1.000" + atoms[0][38:]] + + atoms[1:], + "non-finite coordinates": [atoms[0][:30] + " nan" + atoms[0][38:]] + + atoms[1:], + "wrong order": list(reversed(atoms)), + "second model": ["MODEL 2"] + atoms + ["ENDMDL"], + "two models": ["MODEL 1"] + + atoms + + ["ENDMDL", "MODEL 2"] + + atoms + + ["ENDMDL"], + "outside model": ["MODEL 1"] + atoms + ["ENDMDL", atoms[0]], + } + for name, lines in mutations.items(): + with self.subTest(name=name): + path.write_text("\n".join(lines) + "\nEND\n") + self.assert_grade(case, 0) + + def test_csv_mutations_are_rejected_even_with_correct_manifest_and_summary(self): + case = "offline-boltz2-ranking" + self.stage(case) + self.execute(case) + path = self.workspace / "output/boltz2-only/candidates.csv" + original = path.read_text() + lines = original.splitlines(keepends=True) + for text in ( + lines[0], + lines[0] + "".join(reversed(lines[1:])), + original.replace("0.89", "0.99"), + original + lines[1], + ): + with self.subTest(text=text): + path.write_text(text) + self.assert_grade(case, 0) + path.write_text(original) + self.assert_grade(case, 1) + + def test_empty_rank_must_be_a_header_only_file(self): + case = "offline-empty-ranking" + self.stage(case) + self.execute(case) + root = self.workspace / "output/no-survivors" + shutil.copy2(root / "all_candidates.csv", root / "candidates.csv") + self.assert_grade(case, 0) + + def test_agent_cannot_redefine_fixture_truth(self): + case = "offline-manifest" + self.stage(case) + self.execute(case) + path = self.workspace / "input/bookkeeping.json" + path.write_text(path.read_text().replace('"iptm_min": 0.8', '"iptm_min": 0.1')) + self.assert_grade(case, 0) + + def test_supplied_score_preservation_is_exact(self): + case = "offline-boltz2-ranking" + self.stage(case) + self.execute(case) + path = self.workspace / "output/boltz2-only/manifest.json" + data = json.loads(path.read_text()) + data["candidates"][0]["scores"]["boltz2_confidence"] += 1e-9 + write_json(path, data) + self.assert_grade(case, 0) + + def test_resume_preserves_finite_low_scores_and_only_fills_missing_metrics(self): + case = "offline-cache-resume" + self.stage(case) + self.execute(case) + data = json.loads((self.workspace / "output/resumed/manifest.json").read_text()) + rows = {c["id"]: c for c in data["candidates"]} + self.assertEqual(rows["below"]["cache_status"], "complete") + self.assertEqual(rows["recoverable"]["scores"]["iptm"], 0.86) + self.assertAlmostEqual( + rows["recoverable"]["scores"]["binder_plddt"], 84, places=5 + ) + self.assertAlmostEqual( + rows["recoverable"]["scores"]["self_consistency_rmsd"], 0, places=6 + ) + self.assertEqual( + rows["failed"]["failure_reason"], "HTTP 503: retries exhausted" + ) + self.assertEqual(rows["missing"]["scores"], {}) + + def test_recovered_scores_and_provenance_are_checked_independently(self): + case = "offline-cache-resume" + self.stage(case) + self.execute(case) + path = self.workspace / "output/resumed/manifest.json" + original = json.loads(path.read_text()) + for kind in ("old_score", "sample", "failure", "reason", "decision"): + data = copy.deepcopy(original) + rows = {c["id"]: c for c in data["candidates"]} + if kind == "old_score": + rows["recoverable"]["scores"]["iptm"] = 0.98 + elif kind == "sample": + rows["recoverable"]["score_provenance"]["sample_index"] = 1 + elif kind == "failure": + rows["failed"].pop("failure_reason") + elif kind == "reason": + rows["missing"]["cache_reason"] = "unknown" + else: + rows["below"]["cache_status"] = "blocked" + write_json(path, data) + with self.subTest(kind=kind): + self.assert_grade(case, 0) + + def test_saved_selected_complex_bytes_are_checked(self): + case = "offline-refold-extraction" + self.stage(case) + self.execute(case) + root = self.workspace / "output/raw-refolds/boltz2" + data = json.loads((root / "manifest.json").read_text()) + row = next(c for c in data["candidates"] if c["id"] == "selected") + path = root / row["artifacts"]["complex"] + path.write_text(path.read_text() + "# changed\n") + self.assert_grade(case, 0) + + def test_all_provenance_mismatches_block_import(self): + case = "offline-provenance-rejection" + self.stage(case) + self.execute(case) + data = json.loads( + (self.workspace / "output/provenance/manifest.json").read_text() + ) + for candidate in data["candidates"]: + if candidate["id"] != "valid": + self.assertEqual(candidate["cache_status"], "blocked") + self.assertEqual(candidate["scores"], {}) + self.assertNotIn("score_provenance", candidate) + + def test_invalid_sample_index_is_blocked_without_importing_partial_scores(self): + case = "offline-refold-extraction" + self.stage(case) + path = self.workspace / "input/raw_refolds/boltz2/manifest.json" + original = json.loads(path.read_text()) + for index in (-1, True, 10): + data = copy.deepcopy(original) + data["candidates"][0]["prediction"]["sample_index"] = index + write_json(path, data) + output = self.workspace / str(index) + recover(path, output) + result = json.loads((output / "manifest.json").read_text())["candidates"][0] + self.assertEqual(result["cache_reason"], "invalid_sample_index") + self.assertEqual(result["scores"], {}) + + def test_existing_output_is_not_overwritten(self): + case = "offline-cache-resume" + self.stage(case) + self.execute(case) + with self.assertRaises(FileExistsError): + self.execute(case) + output = self.workspace / "output/resumed/manifest.json" + self.assertTrue(output.is_file()) + + def test_bookkeeping_validates_all_profiles_before_writing(self): + source = self.workspace / "fixture.json" + fixture = json.loads((SKILL / "evals/files/bookkeeping.json").read_text()) + fixture["profiles"][1]["filters"]["binder_plddt_min"] = "80" + write_json(source, fixture) + output = self.workspace / "output" + with self.assertRaisesRegex(ValueError, "finite numbers"): + run_fixture(source, output) + self.assertFalse(output.exists()) + + def test_real_campaign_is_not_relabelled_as_synthetic(self): + case = "offline-cache-resume" + self.stage(case) + path = self.workspace / "input/resume_cache/manifest.json" + data = json.loads(path.read_text()) + data["mode"] = "hosted" + write_json(path, data) + with self.assertRaisesRegex(ValueError, "synthetic"): + self.execute(case) + self.assertFalse((self.workspace / "output/resumed").exists()) + + def test_malformed_structure_blocks_one_candidate_and_continues(self): + case = "offline-refold-extraction" + self.stage(case) + root = self.workspace / "input/raw_refolds/openfold3" + manifest = json.loads((root / "manifest.json").read_text()) + candidate = manifest["candidates"][0] + path = root / candidate["prediction"]["response"]["path"] + data = json.loads(path.read_text()) + data["outputs"][0]["structures_with_scores"][1]["structure"] = "not a PDB" + write_json(path, data) + candidate["prediction"]["response"]["sha256"] = digest(path.read_bytes()) + candidate["prediction"]["complex_sha256"] = digest(b"not a PDB") + write_json(root / "manifest.json", manifest) + recover(root / "manifest.json", self.workspace / "output/broken") + result = json.loads( + (self.workspace / "output/broken/manifest.json").read_text() + ) + self.assertEqual(result["candidates"][0]["cache_status"], "blocked") + self.assertEqual(result["candidates"][0]["scores"], {}) + self.assertEqual(result["candidates"][2]["cache_status"], "recovered") + + def test_standalone_grader_runs_with_only_grader_and_entry_files(self): + case = "offline-boltz2-ranking" + self.stage(case) + self.execute(case) + tests, verifier = self.workspace / "tests", self.workspace / "verifier" + tests.mkdir() + shutil.copy2(SKILL / "evals/grader.py", tests / "grader.py") + write_json(tests / "entry.json", {"id": case}) + parent_reward = self.workspace / "parent-reward.json" + # A parent-only override must not redirect the standalone verifier's + # output. The child needs only these three explicit fixture paths. + with mock.patch.dict("os.environ", {"HARBOR_REWARD_JSON": str(parent_reward)}): + env = { + "HARBOR_TESTS_DIR": str(tests), + "HARBOR_VERIFIER_DIR": str(verifier), + "BINDER_EVAL_WORKSPACE": str(self.workspace), + } + subprocess.run( + [sys.executable, "-I", str(tests / "grader.py")], + env=env, shell=False, check=True, + ) + self.assertFalse(parent_reward.exists()) + self.assertEqual( + json.loads((verifier / "reward.json").read_text())["custom_metrics"], + {"artifact_correctness": 1.0}, + ) + + +class TargetMappingTests(unittest.TestCase): + def setUp(self): + self.text = (SKILL / "evals/files/target_prep/target.pdb").read_text() + + def test_insertion_residue_does_not_overwrite_bare_author_number(self): + self.assertEqual( + residue_index_map(self.text, "E"), {"10": 1, "42": 2, "42A": 3, "77": 4} + ) + self.assertEqual(remap_to_seq_index(self.text, "E", [42, "42A", 77]), [2, 3, 4]) + with self.assertRaises(KeyError): + remap_to_seq_index(self.text, "E", [43]) + + def test_first_model_and_highest_occupancy_altloc_only(self): + self.assertEqual(sequence(self.text, "E"), "AGST") + self.assertEqual( + ca_coords(self.text, "E"), [(0, 0, 0), (4, 0, 0), (4, 3, 0), (8, 3, 0)] + ) + self.assertNotIn("VAL", extract_chain(self.text, "E")) + saved_ca = [ + line + for line in extract_chain(self.text, "E").splitlines() + if line[12:16].strip() == "CA" + ] + self.assertEqual(len(saved_ca), 4) + self.assertEqual(saved_ca[1][16], "B") + + def test_overlapping_author_numbers_require_explicit_chain(self): + with self.assertRaisesRegex(ValueError, "ambiguous"): + residue_index_map(self.text) + + +if __name__ == "__main__": + unittest.main() diff --git a/workflows/generative-protein-binder-design/protein-binder-design/LICENSE b/workflow-skills/generative-protein-binder-design/protein-binder-design/LICENSE similarity index 100% rename from workflows/generative-protein-binder-design/protein-binder-design/LICENSE rename to workflow-skills/generative-protein-binder-design/protein-binder-design/LICENSE diff --git a/workflows/generative-protein-binder-design/protein-binder-design/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/README.md similarity index 64% rename from workflows/generative-protein-binder-design/protein-binder-design/README.md rename to workflow-skills/generative-protein-binder-design/protein-binder-design/README.md index 282aaa0c..a984980c 100644 --- a/workflows/generative-protein-binder-design/protein-binder-design/README.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/README.md @@ -20,7 +20,7 @@ protein-binder-design/ │ ├── metrics.py # Kabsch CA-RMSD (self-consistency) │ ├── controls.py # scrambled negative controls │ └── registry.py # target-registry loader -├── assets/targets.json # EXAMPLE target registry (verify before real use) +├── assets/targets.json # empty target-registry template (populate before use) └── evals/ # trigger + assertion evals ``` @@ -30,7 +30,25 @@ protein-binder-design/ - Access to the BioNeMo NIMs you intend to use (RFdiffusion, ProteinMPNN, Boltz2 and/or OpenFold3, optional MSA-Search) — hosted at [build.nvidia.com](https://build.nvidia.com) or self-hosted via NGC. -- Python ≥ 3.10 with `numpy` (`pip install numpy`). Scripts are otherwise stdlib. +- Python ≥ 3.12 for the pinned helper environment. Install dependencies with + `python3 -m pip install -r requirements.txt`; mmCIF parsing and cached refold + recovery require Biotite. Manifest and PDB-only helpers use the standard library. + +## Offline tests + +Install `requirements-dev.txt` in a virtual environment, then run +`python3 -m pytest tests`. To inspect coverage, run +`python3 -m pytest --cov=scripts --cov-report=term-missing tests`. +These tests use synthetic fixtures and credentials; they make no model calls. + +## Tier 1 validation + +NVCARPS selects `config/skillspector-baseline.yml` before the repository-wide +baseline. Include this file, the requirements files, and the tests when committing +the skill; synchronize the aggregate copy with the repository's plugin sync tool. +The baseline records reviewed false positives as exact source fingerprints for +SkillSpector 2.11.2. Source edits or a scanner upgrade require another review; +the environment-copy finding is fixed in code and is not suppressed. ## Configure diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md new file mode 100755 index 00000000..4a212df6 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md @@ -0,0 +1,305 @@ +--- +name: protein-binder-design +description: > + Use when designing RFdiffusion/ProteinMPNN binders or validating and resuming binder campaigns. Not for code or report review. +license: Apache-2.0 +compatibility: "Python>=3.12 with requirements.txt; Bash for hosted_env.sh" +metadata: + author: "NVIDIA BioNeMo " + tags: [protein-design, binder-design, bionemo] +allowed-tools: Bash, Read, Write, AskUserQuestion +permissions: + - env # reads NVIDIA_API_KEY/NGC_API_KEY and configured NIM endpoints + - network # calls hosted NVIDIA APIs or user-configured local NIM endpoints +--- + +# Protein Binder Design (workflow) + +## Purpose + +Run a de novo binder design campaign by composing atomic NIM skills. This skill +owns orchestration, handoff contracts, filtering, validation, and the run +manifest. For per-NIM API details, consult the corresponding NIM skill. + +Use for requests such as “design protein binders”, “run an RFdiffusion campaign”, +“validate and rank these binders”, or “resume this binder manifest”. A request to +review code, explain methods, or inspect an evaluation report does not start a +design campaign. Offline bookkeeping stays offline. + +## Prerequisites + +- Python 3.12+; install [requirements.txt](requirements.txt) for RMSD, mmCIF, + cache recovery, and Python HTTP examples. Manifest and PDB-only helpers use + the standard library. Install [requirements-dev.txt](requirements-dev.txt) + to run `python3 -m pytest tests` with coverage. +- Live inference needs hosted NIM credentials or reachable local NIM services; + local deployment also needs Docker, the NVIDIA Container Toolkit, and a + supported GPU. See Configuration below and [local setup](references/local-nim-setup.md). + +## Inputs + +Required inputs from the user prompt or saved manifest: target structure/sequence, +target chain, epitope/hotspots, binder length range, backbone/sequence counts, +output directory, and hosted or local endpoints. Resolve the confidence +selection policy before live inference. Resume requests require the saved +manifest and artifacts; offline imports require a synthetic score fixture or +cache bindings as defined in [offline contracts](references/offline.md). +Optional inputs include a target MSA, sourced positive controls, and explicit +overrides to the default selection thresholds. + +Apply the user's explicit corrections first. Otherwise preserve saved campaign +settings, then use supplied task context; ask only for unresolved inputs. Record +intentional changes to saved settings so resumed results remain interpretable. + +## Data handling + +Hosted mode sends target structures/sequences, candidate sequences, and any +requested alignments to the selected NVIDIA NIM endpoints under that service's +terms. Use the user's selected deployment; resolve the destination before +uploading data if it is unspecified. Local mode sends inference inputs to the +configured local service; initial image/weight downloads still contact NGC. +All modes write JSON/CSV and, when produced, PDB/mmCIF/FASTA artifacts under the +chosen run directory. Keep credentials out of those artifacts and logs. + +## Composed skills + +| Step | Skill | Owns | +|---|---|---| +| Backbones | `rfdiffusion-nim` | binder backbone PDBs (contigs + hotspots) | +| Sequences | `proteinmpnn-nim` | sequences for each backbone | +| Co-fold / score | `boltz2-nim` or `openfold3-nim` | binder–target complex + confidence / ipTM | +| MSA (optional) | `msa-search-nim` | target A3M for higher-quality folding | + +The atomic NIM skills are recommended companions (one per NIM, from the BioNeMo +NIM skill set). They are **not required**: `references/pipeline.md` carries the +concrete request shape for every NIM call, so an agent with NIM access can follow +this skill standalone. For endpoints/auth see **Configuration** below. + +## Instructions + +Resolve the campaign inputs before inference. The bundled `assets/targets.json` +is a template, not a populated PD-L1 registry. A named target needs a verified +structure, chain, and epitope; controls need existing candidate sequences and +sourced positive controls; resuming needs the original manifest and artifacts. +Check the supplied paths first. If an input or NIM credential is missing, record +the blocker and the unexecuted stages. Do not invent registry entries, measured +scores, or an interrupted run. An empty CSV alone does not establish a completed +campaign; report which stages ran and their saved artifacts. + +### Offline inputs + +For a synthetic fixture with `target` and `profiles`, run +`python3 scripts/offline_bookkeeping.py /path/to/bookkeeping.json --output-dir /path/to/output`. +This saves and reloads each manifest, writes both CSVs and `summary.json`, and +reports missing scores and provisional counts in one call. For an interrupted +synthetic run with saved responses, use +`python3 scripts/refold_cache.py /path/to/manifest.json --output-dir /path/to/new-run`. +Read [offline contracts](references/offline.md) for cache bindings, sample +selection, chain confidence and blocked work. These helpers need no credentials +and make no model calls. Inspect exit status and saved artifacts before claiming +completion; a failed command followed by a successful retry is recovered +execution, with both attempts retained as evidence. + +### Pipeline + +1. **Target prep** — get the target PDB + epitope; map epitope/hotspot author + residue numbers to RFdiffusion `hotspot_res` strings; optionally build a + target MSA with `msa-search-nim`. +2. **Backbones** (`rfdiffusion-nim`) — binder contig + `hotspot_res`; N backbones. +3. **Sequences** (`proteinmpnn-nim`) — k sequences per backbone; drop the + native/WT row from `mfasta`. +4. **Co-fold + score** (`boltz2-nim` / `openfold3-nim`) — co-fold binder+target; + collect explicit ipTM or Boltz2 composite confidence, plus binder pLDDT. +5. **Self-consistency** — CA-RMSD between the RFdiffusion backbone and the + predicted binder (`scripts/metrics.py`). +6. **Filter + rank** — apply the saved campaign thresholds, exclude controls, + sort surviving designs, and export that exact list (see below). + +Full handoff contracts, branching, and the cost funnel: `references/pipeline.md`. + +### Filter, rank, and export + +For `m` returned by `Manifest.create()` or `Manifest.load()`, use this sequence +after scoring. Keep the supplied filters, ranking metric, scores, and control +flags unchanged: + +```python +from pathlib import Path + +m.apply_filters() +rank_by = m.data["params"].get("rank_by", "iptm") +ranked = m.rank(by=rank_by, descending=True, passed_only=True, include_controls=False) +run_dir = Path(m.data["run_dir"]) +m.to_csv(run_dir / "all_candidates.csv") # complete audit, including controls +m.to_csv(run_dir / "candidates.csv", candidates=ranked) +print(m.summary()) # controls excluded from candidate/pass counts +print([candidate["id"] for candidate in ranked]) +``` + +`rank()` returns a new list; it does not reorder or shrink the manifest. +`to_csv()` without `candidates=ranked` writes **all** entries in insertion order. +Pass an empty ranked list through unchanged: zero survivors means a header-only +`candidates.csv`, while the audit CSV and manifest retain every entry. A high +ranking score cannot override a failed or missing enabled filter metric. Report +missing scores and label the passing fraction provisional when scoring is incomplete. + +## Handoff contracts (the fragile glue) + +- RFdiffusion `output_pdb` → ProteinMPNN `input_pdb` (inline PDB text). +- ProteinMPNN `mfasta` → Boltz2 binder polymer `sequence` (exclude the + native/WT row; pair scores only with designed rows). +- Epitope author residue numbers → 1-based sequence indices: remap with + `scripts/pdb_utils.py:remap_to_seq_index`. RFdiffusion `hotspot_res` uses + chain+author strings like `"A50"`; Boltz2 pocket/contacts use 1-based indices. + Keep insertion IDs distinct (`50` versus `50A`), select the target chain/model, + and block absent hotspots instead of substituting nearby residues. Verify + downstream support before requesting an insertion-coded hotspot. +- Boltz2 complex `.cif` → binder chain → self-consistency RMSD vs the backbone. + Use the same saved sample for confidence and coordinates; mmCIF request polymer + IDs are label chain IDs. Whole-complex pLDDT does not supply binder-only pLDDT. + +## Output Format + +Every campaign writes `manifest.json` (+ `candidates.csv`) under a run dir via +`scripts/manifest.py`. It records lineage, params, scores, artifacts, filter +status, and controls — enabling ranking, resumability, validation, and the +final report. Schema and usage: [manifest contract](references/manifest.md). +Keep all entries in `all_candidates.csv`; `candidates.csv` contains only ranked +survivors and retains its header when none pass. Offline helpers also write +`summary.json`. The final response identifies the mode, completed/blocked stages, +artifact paths, selection thresholds, ranked IDs, and missing scores. Report +control counts separately and mark an incomplete passing fraction provisional. + +## Filters (defaults) + +- ipTM ≥ 0.8, binder pLDDT ≥ 80, self-consistency RMSD ≤ 2.0 Å. +- Override per campaign and record overrides in the manifest `filters`. +- Every applicable enabled metric needs a finite score to pass. Sequence-only + controls have no designed backbone: their RMSD is not applicable and the manifest + records that exemption; design candidates still require RMSD. Keep a composite confidence + value in its own field; do not substitute it for an unavailable ipTM. +- For Boltz2-only campaigns without explicit ipTM, resolve the composite + confidence cutoff before inference. Set `iptm_min: null`, enable + `boltz2_confidence_min` with that cutoff, retain the pLDDT/RMSD thresholds, + and save `params.rank_by: boltz2_confidence`. Filter with `apply_filters()`, + rank with `rank(by="boltz2_confidence", passed_only=True)`, and export that + list with `to_csv(candidates=ranked)`. Report this as composite-confidence + selection; it does not establish the default ipTM criterion. Complete example: + `references/manifest.md`. **Default decision when no cutoff was supplied:** use + a refolder that returns explicit ipTM (OpenFold3) and the default ipTM profile. + If only composite-confidence Boltz2 is available, obtain a campaign cutoff and + its rationale from the user or a separate same-target calibration before design + inference. Save it in `params.confidence_selection`. Without either, stop at + input preparation and report `selection_policy_unresolved`; do not guess 0.8 + or present unfiltered rankings as validated binders. Calibration procedure: + `references/validation.md`. + +## Validation + +Run scrambled controls and report a **success rate**, not just top scores. +Negative controls via `scripts/controls.py` (scrambled sequences); positive +controls = sourced published binders re-scored through the same pipeline. The +bundled `assets/targets.json` is an empty control template, not a positive-control +dataset. Use user-provided sequences with citations, or retrieve and verify an +exact sequence from a primary publication or structure before registering it. +If none is available, record `positive_controls_unavailable`, complete the +computational screen with negatives, and label it **uncalibrated screening**; +do not claim positive-control benchmarking or calibrated binder success. A +request requiring a benchmark remains incomplete until a positive is supplied. +Methodology and +metric definitions: `references/validation.md`. + +## Human-in-the-loop + cost + +- Reuse target, epitope/hotspots, binder length range, and hosted-vs-local + choices already supplied by the user. Ask only for unresolved inputs before + generating backbones. +- Co-folding is the expensive stage: co-fold a capped shortlist, review, then + expand. State hosted vs local once and reuse it across all NIM calls. + +## Responsible use + +De novo binder design is dual-use. Decline requests aimed at enhancing pathogen +fitness, toxin potency, or bioweapon function; keep designs to legitimate +research and therapeutic intent. + +## Configuration (NIM access) + +Each composed NIM is reached over HTTP; choose **hosted** or **local** once and +reuse it for every call: + +- **Hosted** (managed): base URL `https://health.api.nvidia.com/v1/...` per NIM at + [build.nvidia.com](https://build.nvidia.com); use `NGC_API_KEY` or the + `NVIDIA_API_KEY` fallback (sent as `Authorization: Bearer`). Read keys from the + environment. Check only presence with + `bool(os.getenv("NGC_API_KEY") or os.getenv("NVIDIA_API_KEY"))`; never print + environment dumps, key values, or authorization headers. An `OPENAI_API_KEY` + belongs to the agent runtime and must not be used as a NIM credential. If both + NIM keys are absent, report missing access before sending authenticated calls. + Run **each delegated hosted command** through the credential wrapper: + `bash scripts/hosted_env.sh python3 your_nim_request.py`. + Use the helper's absolute path when running outside the skill directory. + It exports the selected key as `NGC_API_KEY`, which the atomic NIM skills read, + preserves an existing nonempty `NGC_API_KEY`, and stops before launching the + child if neither key exists. The wrapper loads the key and executes the command + in one invocation, so no exported state from a previous tool shell is needed. + It disables shell tracing and preserves arguments and the child's exit code. +- **Local** (self-hosted NGC containers): point each NIM at its local URL + (e.g. `http://localhost:8000/...`); local NIMs need no auth header. Follow + [local setup](references/local-nim-setup.md) for digest-pinned images, private + cache mounts, loopback ports, readiness checks, and GPU-specific profile selection. + +Per-NIM paths, request/response schemas, and worked `curl`/Python examples live in +`references/pipeline.md`. + +## Examples + +- **New campaign:** with a verified target entry and NIM access, create the + manifest, remap residues with `pdb_utils.py`, generate backbones, design binder + sequences, co-fold a shortlist, and save response-derived scores before + filtering and exporting. Record any stages that could not run. +- **Controls:** load the existing campaign, use `make_scrambled_controls` on + its binder sequences, and add sourced published binders. Mark all controls + `is_control=True`, co-fold with the same settings, then report + `n_passed / n_candidates` from `summary()` with controls excluded. With no + candidates the rate is unavailable; disclose incomplete scoring. +- **Resume:** call `Manifest.load("runs/")` and + `missing_scores(candidate_id)`; use the artifact checklist in + `references/manifest.md` to execute only missing stages. A finite below-threshold + score is complete, not a reason to repeat inference. Reuse saved predictions + for missing metrics; preserve completed candidates and explicit failure records. + +## Available Scripts + +Paths below are relative to this skill. Run CLIs with `python3 scripts/.py` +or `bash scripts/hosted_env.sh`; import library helpers after adding the skill's +`scripts` directory to the Python import path. No harness-specific runner is required. + +| Script | Purpose | Arguments | +|---|---|---| +| [offline_bookkeeping.py](scripts/offline_bookkeeping.py) | CLI: import synthetic score profiles | `input.json --output-dir DIR` | +| [refold_cache.py](scripts/refold_cache.py) | CLI: recover metrics from verified cache bindings | `manifest.json --output-dir DIR` | +| [metrics.py](scripts/metrics.py) | CLI/library: binder CA-RMSD | `backbone.pdb prediction.cif --backbone-chain A --predicted-chain B` | +| [hosted_env.sh](scripts/hosted_env.sh) | Credential wrapper | `COMMAND [ARG ...]` | +| [manifest.py](scripts/manifest.py) | Library: save, filter, rank, export | `Manifest.create(run_dir, target, mode, params, filters)` / `Manifest.load(path)` | +| [pdb_utils.py](scripts/pdb_utils.py) | Library: select chain and remap residues | `remap_to_seq_index(pdb_text, chain, author_resnums)` | +| [controls.py](scripts/controls.py) | Library: composition-preserving negative controls | `make_scrambled_controls(seqs, n=5, seed=0)` | +| [registry.py](scripts/registry.py) | Library: load a user-populated target registry | `get_target(name, path=None)`; default is the empty `assets/targets.json` template | + +## Limitations + +Confidence thresholds and scrambled controls provide computational screening, +not experimental proof of binding. The target registry contains no verified +positive controls. Boltz2 composite confidence is distinct from ipTM. Cached +synthetic fixtures test bookkeeping, not model performance or biological validity. + +## Troubleshooting + +| Error or symptom | Cause | Resolution | +|---|---|---| +| Missing NIM key | Neither supported key is set | Configure the selected hosted credential or use an available local service | +| `selection_policy_unresolved` | No explicit ipTM or calibrated composite cutoff | Resolve the refolder/cutoff before design inference | +| Absent hotspot or chain | Author numbering/chain differs from input | Inspect the selected model and insertion codes; supply the correct residue IDs | +| Cache binding/digest failure | Response identity or bytes do not match | Keep the failed candidate visible; obtain matching artifacts before retrying | +| No ranked survivors | Enabled metric missing or below threshold | Inspect the audit CSV; retain the empty ranking and disclose incomplete scoring | +| Missing Python module / tests cannot collect | Helper or test dependencies absent | Install the appropriate requirements in the same Python environment used to run the command | diff --git a/workflows/generative-protein-binder-design/protein-binder-design/assets/targets.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/assets/targets.json similarity index 100% rename from workflows/generative-protein-binder-design/protein-binder-design/assets/targets.json rename to workflow-skills/generative-protein-binder-design/protein-binder-design/assets/targets.json diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/config/skillspector-baseline.yml b/workflow-skills/generative-protein-binder-design/protein-binder-design/config/skillspector-baseline.yml new file mode 100644 index 00000000..161a053a --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/config/skillspector-baseline.yml @@ -0,0 +1,86 @@ +# Reviewed 2026-10-07 against SkillSpector 2.11.2, the scanner used by NVCARPS. +# Exact fingerprints bind the complete file contents and scanner version. +# File edits or a scanner upgrade invalidate these exceptions and require review. +# No rule-wide suppressions; credential harvesting is fixed in code, not waived. +# Environment-isolation regression rechecked 2026-10-08; its AST4 hash refreshed. +version: 2 +scanner_version: 2.11.2 +rules: [] +fingerprints: + - hash: sha256:9996d6fb97fdfb596f2f50c4cf3e79570283746d312372f175e15db370fc0831 + rule_id: AS3 + file: BENCHMARK.md + reason: >- + Generated historical evaluation quotes this skill's own manifest path in + four diagnostic rows. No instruction or code reads other installed skills. + The benchmark evidence and scores remain unmodified. + - hash: sha256:daaafc2bfa26f14b989e67b1e96716fcf3f4d8f9989dde5b033821ad887d9f10 + rule_id: PE2 + file: references/local-nim-setup.md + reason: >- + chmod 700 restricts three explicitly named, user-owned cache directories. + It neither invokes sudo nor grants elevated privileges. Container UID/GID + matches the invoking user; no recursive chmod or wildcard is used. + - hash: sha256:4eb3e8ba0df55f36be8791053aef6e0998b6e4a4b45536296e05def0f16496d8 + rule_id: RP1 + file: references/local-nim-setup.md + reason: >- + The launch block validates all three image references against the expected + NGC repositories and 64-hex SHA-256 digests before any service starts. + The static Docker parser cannot resolve these checked shell variables. + Mocked execution verified that tags, wrong registries, and malformed + digests fail before any command with side effects. + - hash: sha256:3bf6b75e39342d08503840522a956796814679807c077c7a7fa32191d4ee9661 + rule_id: RP1 + file: references/local-nim-setup.md + reason: >- + The optional profile-listing example explicitly reuses the Boltz2 digest + validated by the preceding launch block. It specifies no moving/default + tag. The static Docker parser cannot resolve the checked variable. + - hash: sha256:49a8eccca8415a52025878019d22ad62d4407bb3e0bac773944c5e33cfa357ff + rule_id: AST4 + file: tests/test_hosted_env.py + reason: >- + Fixed test commands exercise credential selection with synthetic values, + an explicit argument list, shell=False, and an isolated environment. + The legacy bash -c source test uses positional parameters with fixed code; + it interpolates no user input and makes no network calls. + - hash: sha256:9a86caae52eb514ddfe60d96005e799e15e99d09abc0c657cde9ca81e54c5734 + rule_id: AST4 + file: tests/test_hosted_env.py + reason: >- + Fixed bash tracing test checks that a synthetic credential stays out of + output and that the wrapper preserves an argument containing spaces and + the child's exit status. Uses shell=False and no inherited credentials. + - hash: sha256:410d6f5fceb976f9e8783fe51370814d9f4c6112cde35bdf4f7474073a20be80 + rule_id: AST4 + file: tests/test_offline.py + reason: >- + Executes this repository's standalone offline grader with the current + Python executable, isolated mode, a fixed argument list, and shell=False. + The child receives only three temporary fixture paths. It receives no + developer/CI credentials and submits no inference or network requests. + A parent-only reward-path override verifies environment isolation: restoring + bulk inheritance makes the output-location assertion fail. + # Additional occurrences of the same reviewed benchmark/path and launch text. + # Each line is fingerprinted separately; the scanner expands compacted groups. + - hash: sha256:d52d9f5c67c895424f94182a0e365ebf673b1de593e196f607ef46c557aa5990 + rule_id: AS3 + file: BENCHMARK.md + reason: Generated diagnostic quotes this skill's own manifest path; no peer-skill access. + - hash: sha256:f5ea4acc4cd746d227dc072baf0af37bc71905295cae7d8d542d87a18ce2038b + rule_id: AS3 + file: BENCHMARK.md + reason: Generated diagnostic quotes this skill's own manifest path; no peer-skill access. + - hash: sha256:757ebd760479cde3d81fd3a0d32fdf58a7d00c4c1607cb529906d7d648c632ed + rule_id: AS3 + file: BENCHMARK.md + reason: Generated diagnostic quotes this skill's own manifest path; no peer-skill access. + - hash: sha256:23cec21cbfc72046b9fec2058f664eccab0821054e7c9ac9497fae60424ffea3 + rule_id: RP1 + file: references/local-nim-setup.md + reason: ProteinMPNN launch uses the expected NGC repository and SHA-256 digest checked before any service starts. + - hash: sha256:658d33910213c2b483f90ad16b6aa0cc2f1a3067813cf9b2e85101e488b9b343 + rule_id: RP1 + file: references/local-nim-setup.md + reason: Boltz2 launch uses the expected NGC repository and SHA-256 digest checked before any service starts. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md new file mode 100644 index 00000000..5377e065 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md @@ -0,0 +1,128 @@ +# Evaluation coverage and deferred live campaigns + +## Active coverage: offline software workflows + +`evals.json` runs eight offline exercises. The four original prompts, assertions, +expected outputs and score fixtures remain unchanged for historical comparison. +Four additional cases require target mapping, saved-response interpretation, +recovery decisions and provenance checks. Their output contracts and all required +inputs are staged in both conditions; equivalent implementations are accepted. + +| Case | Input under `/workspace/input/` | Required result | +| --- | --- | --- | +| `offline-manifest` | `bookkeeping.json` | Persistence and ranked exports for ipTM and Boltz2 profiles. | +| `offline-boltz2-ranking` | `boltz2_only.json` | Composite confidence stays separate from ipTM; passing controls do not enter the ranked CSV. | +| `offline-mixed-threshold-ranking` | `mixed_thresholds.json` | All filters apply before sorting; a high ipTM cannot rescue another failing or invalid metric. | +| `offline-empty-ranking` | `no_survivors.json` | A header-only ranked CSV and a complete audit when no designs pass. | +| `offline-target-preparation` | `target_prep/` | Select chain/model/alternate CA; keep insertion codes distinct; block absent hotspots. | +| `offline-refold-extraction` | `raw_refolds/`, `cache_contract.md` | Bind scores to the recorded sample, separate confidence metrics, and use binder-only pLDDT. | +| `offline-cache-resume` | `resume_cache/`, `cache_contract.md` | Preserve complete scores and failure history; recover missing metrics from cache; rebuild stale exports. | +| `offline-provenance-rejection` | `provenance_cache/`, `cache_contract.md` | Reject candidate, sequence, target, sample and response-file mismatches before scoring. | + +Every case stages identical inputs for the with/without-skill conditions. +`environment/Dockerfile` supplies the same pinned NumPy and Biotite versions to +both conditions during environment setup, before the offline task starts. +The original prompts and fixtures are preserved; environment and verifier +changes should still be recorded when comparing with historical reports. +The added prompts request campaign outputs without giving the expected order; +the assertions check saved artifacts, not use of a particular helper. +All sequences, toy coordinates, numeric inputs, saved responses and control labels are synthetic. +The active `config.yml` forwards no NIM credentials and the cases make no NIM +calls. Passing them establishes **offline software workflow coverage only**, not live model +execution, biological validity, or end-to-end campaign coverage. + +Compare agents separately when rerunning: a passing best-agent summary can hide +another agent's regression. Compare the original case against its own previous +result; the expanded suite's aggregate is not directly comparable to the old +one-case score. Positive lift is an observation, not a target for the grader: +keep the baseline inputs and success criteria identical. + +Run the bundled regression tests without model access: + +```bash +python3 -m unittest discover -s tests -v +``` + +## Deterministic artifact checks + +`grading.mode: aces_plus_custom` runs the existing ACES judges and the standalone +`grader.py`. The verifier reads actual manifests, CSVs, selected complex files +and summaries. It pins input digests, checks score types and absence, ranking +order, control exclusion, preserved state, source identity and artifact bytes. +It imports no skill helpers. Tests exercise all eight workflows and deliberately +corrupt their outputs to check that the verifier rejects them. The old ambiguous +phrase "ranked candidates.csv" accepts `candidates.csv`, `ranked candidates.csv` +or `ranked_candidates.csv`; new contracts specify `candidates.csv` exactly. + +**SkillEvaluator 1.5.6 keeps the ACES headline score and pass@k authoritative.** +The custom binary metric `artifact_correctness` and individual failures appear +in `custom_reward.json` and the rich `aces_reward.json` custom fields. They do +not change the headline score or turn its "All Passed" label into an artifact +gate. Inspect this metric per trial alongside the judges. Do not interpret an +ACES pass with `artifact_correctness=0` as a completed artifact-producing task. + +Artifact checks cannot prove an agent actually performed a save/reload step or +made no model calls; the trajectory judges retain those responsibilities. They +also do not prove biological validity. Keep recovery attempts and their tool +results in the judge evidence. To audit a suspicious grade, compare the full +trajectory, the captured judge `prompt_evidence` when available, and verifier +artifacts. Missing transcript evidence is not proof that files were never made. + +The eight-case suite uses 32 runs at one trial per case, two agents and both +conditions. Compare the unchanged four-case subset separately from new cases, +and compare agents separately. A fresh live agent evaluation is needed to +measure lift; passing local tests does not predict an improved agent score. + +Fixture changes require updating the corresponding `INPUT_HASHES` entries in +`grader.py` and reviewing the hand-calculated `CACHE_RUNS` outcomes. Never derive +expected truth from unverified agent-mutated inputs. The local regression suite +fails on digest drift. Keep contracts identical in both conditions and express +real workflow decisions, rather than making prompts vague or targeting one model. + +## Deferred: live design, controls, and resume + +The original three live cases are retained in `deferred/live-campaigns.json`, +outside the active `evals.json` dataset. **They are not runnable or counted as +covered.** Their required verified inputs are not bundled; do not activate them +with empty `files` lists or substitute synthetic scores. Each case and each +with/without-skill attempt starts independently, so cases 2 and 3 cannot reuse +outputs from case 1. + +| Case | Required inputs | +| --- | --- | +| 1: PD-L1 design | Verified PD-L1 registry entry, matching target structure and chain, author-numbered epitope, binder length range, and shortlist size. | +| 2: Controls | Existing PD-L1 campaign manifest and candidate sequences, matching target structure, and published positive-control sequences with source citations. | +| 3: Resume | The interrupted campaign's manifest and referenced artifacts, including both completed candidates and candidates with missing scores. | + +When restoring live cases, configure `harbor.runtime_env` to forward the +available NIM credential (`NGC_API_KEY` or `NVIDIA_API_KEY`) to both conditions. +Normalize it with `scripts/hosted_env.sh` in the same shell as each hosted +command. Forwarding a key does not supply campaign inputs or prove endpoint +access. The agent runtime's `OPENAI_API_KEY` is not a substitute for a NIM key. + +## Staging inputs + +Put verified fixtures under `evals/files/` and list the relevant files or +directory in each case's `files` array. SkillEvaluator 1.5.6 stages the selected +paths beneath `/workspace/input/`, preserving their paths relative to +`evals/files/`. For example, selecting `evals/files/pdl1_design/` makes its +contents available under `/workspace/input/pdl1_design/`. The same inputs must +be available to both conditions. + +Point prompts at those explicit staged paths. For the resume case, either ask +the agent to resume the staged run there or copy it to `runs/rbd_binders` during +pre-agent setup, retaining its artifact paths. A directory name in a prompt +does not create that directory. Record fixture provenance and preserve the +existing completed outputs so the grader can check that they were not recomputed. + +Before moving a deferred case back into `evals.json`, update its prompt to name +those staged paths and replace plan-only assertions with execution requirements: +invoke the model APIs, save actual responses, +extract metrics, and update the manifest. Empty CSVs, prepared scripts, and +synthetic confidence values do not establish successful inference. A separate +offline bookkeeping test may use explicitly synthetic data; it cannot stand in +for these live integration cases. + +The workflow can be followed through `references/pipeline.md`. +Companion NIM skills are optional; include them in both evaluation conditions +if testing the workflow as a group. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml new file mode 100644 index 00000000..e54e6139 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml @@ -0,0 +1,12 @@ +schema_version: 1 + +harbor: + task_source: evals_json + agents: + claude-code: + model: aws/anthropic/bedrock-claude-opus-5 + # The active dataset uses staged synthetic bookkeeping and workflow inputs only. + # Deferred live campaigns must supply fixtures before they are re-enabled. + +grading: + mode: aces_plus_custom diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/deferred/live-campaigns.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/deferred/live-campaigns.json new file mode 100644 index 00000000..9845a15f --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/deferred/live-campaigns.json @@ -0,0 +1,42 @@ +{ + "skill_name": "protein-binder-design", + "evals": [ + { + "id": "1", + "prompt": "Design 50 de novo mini-binders against PD-L1 (use the target registry entry) at its defined interface epitope. Generate backbones, design 8 sequences each, co-fold the shortlist against the target, and rank by interface confidence. Save a run manifest and a ranked CSV.", + "expected_output": "An orchestration that composes the rfdiffusion-nim, proteinmpnn-nim, and boltz2-nim/openfold3-nim skills in order, remaps epitope residues, writes a run manifest via scripts/manifest.py, applies the ipTM/pLDDT/RMSD filter, and exports a ranked candidates.csv.", + "files": [], + "assertions": [ + "[composes-rfdiffusion] Uses RFdiffusion for binder backbones: Plan invokes the rfdiffusion-nim skill with a binder contig and hotspot_res", + "[composes-proteinmpnn] Uses ProteinMPNN for sequence design: Plan invokes the proteinmpnn-nim skill and drops the native/WT row from mfasta", + "[composes-cofold] Co-folds binder with target: Plan invokes boltz2-nim or openfold3-nim to co-fold the binder with the target and collects ipTM or confidence", + "[writes-manifest] Writes a run manifest: Plan uses scripts/manifest.py (Manifest.create, set_scores, apply_filters, to_csv)", + "[applies-filter] Applies default filter thresholds: Plan filters on ipTM >= 0.8, binder pLDDT >= 80, self-consistency RMSD <= 2.0", + "[remaps-epitope] Remaps epitope residues: Plan remaps epitope author numbering using scripts/pdb_utils.py before use" + ] + }, + { + "id": "2", + "prompt": "For my binder campaign against PD-L1, include proper controls and tell me the success rate.", + "expected_output": "Adds scrambled-sequence negative controls and published-binder positive controls through the same co-folding pipeline, marks them is_control in the manifest, and reports success rate as n_passed / n_candidates.", + "files": [], + "assertions": [ + "[negative-controls] Generates scrambled negative controls: Plan uses scripts/controls.py to scramble binder sequences and marks them is_control with control_type scrambled", + "[positive-controls] Uses published binders as positive controls: Plan re-scores published binders from the registry through the same pipeline as positive controls", + "[success-rate] Reports a success rate: Plan reports success rate as n_passed divided by n_candidates from manifest.summary()", + "[same-pipeline] Controls run through identical pipeline: Controls are co-folded and scored with the same settings as the designs" + ] + }, + { + "id": "3", + "prompt": "Continue my interrupted binder run in runs/rbd_binders and only process candidates that are missing scores.", + "expected_output": "Loads the existing manifest with Manifest.load, identifies candidates lacking the needed scores/artifacts, and resumes without re-running completed stages.", + "files": [], + "assertions": [ + "[loads-manifest] Loads the existing manifest: Plan calls Manifest.load on the run directory", + "[resumes-incomplete] Skips completed candidates: Plan only processes candidates missing required scores or artifacts", + "[no-recompute] Does not recompute finished stages: Plan avoids re-running stages already recorded in the manifest" + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/environment/Dockerfile b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/environment/Dockerfile new file mode 100644 index 00000000..323dfaad --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim + +# Shared by both conditions, before the offline agent task starts. Pin the +# versions used by the local regression suite; no NIM clients or credentials. +RUN python3 -m pip install --no-cache-dir numpy==2.5.1 biotite==1.7.1 diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/evals.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/evals.json new file mode 100644 index 00000000..b8711b6f --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/evals.json @@ -0,0 +1,136 @@ +{ + "skill_name": "protein-binder-design", + "evals": [ + { + "id": "offline-manifest", + "prompt": "Run an offline software bookkeeping exercise using /workspace/input/bookkeeping.json. All scores, controls, and thresholds in this file are synthetic; do not call model APIs or describe these numbers as scientific predictions. For each profile, create a separate run directory and manifest using the supplied target, filters, ranking metric, candidates, scores, and control flags. Use the bundled manifest helper if available, or implement equivalent bookkeeping. Save and reload each manifest, apply the supplied filters, exclude controls from rankings and success counts, and export both all_candidates.csv and ranked candidates.csv. Show the resulting counts and ranked IDs for each profile. Leave absent scores absent, including ipTM for the Boltz2 profile. Label all results as simulated bookkeeping, disclose the pending candidate, and explain that these results provide no live inference coverage.", + "expected_output": "Two saved and reloaded manifests and two CSVs per profile. Each profile has four non-control candidates, two passing candidates, and one control. Both rankings contain best then boundary. Below and pending fail; the control is excluded from rankings and counts. The Boltz2 profile retains boltz2_confidence without inventing ipTM. The simulated 2/4 passing fraction is provisional because pending lacks required scores. No model calls or scientific validation claims.", + "files": [ + "evals/files/bookkeeping.json" + ], + "assertions": [ + "[executes-bookkeeping] Executes the bookkeeping and produces a manifest and both CSV artifacts per profile; a proposed plan alone does not pass.", + "[resumes-state] Saves and reloads each profile without changing the supplied scores, filters, rank_by metric, or control flags.", + "[filters-missing-scores] Each profile reports n_candidates=4, n_passed=2, n_controls=1; below and pending fail.", + "[ranks-and-exports] Each ranked candidates.csv contains best then boundary, excludes the control, and all_candidates.csv retains all five entries.", + "[separates-metrics] Boltz2 is filtered and ranked by boltz2_confidence, with iptm_min disabled and no invented ipTM score.", + "[states-coverage] Labels every result as simulated offline bookkeeping, discloses the incomplete candidate and provisional 2/4 ratio, and makes no live inference or scientific validation claim." + ] + }, + { + "id": "offline-boltz2-ranking", + "prompt": "Review my protein binder campaign in /workspace/input/boltz2_only.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/boltz2-only. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "expected_output": "The supplied composite-confidence cutoff is 0.73; ipTM remains disabled and absent. Ranked IDs are binder_a then binder_b. Counts are n_candidates=4, n_passed=2, n_controls=1; the audit retains all five entries. The sequence-only control is exempt only from RMSD. Pending lacks RMSD and low_confidence fails. The 2/4 fraction is provisional and synthetic.", + "files": [ + "evals/files/boltz2_only.json" + ], + "assertions": [ + "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/boltz2-only; a plan alone does not pass.", + "[separate-metrics] Keeps the supplied 0.73 confidence cutoff, disabled ipTM and enabled pLDDT/RMSD filters; does not invent ipTM or rename confidence to ipTM.", + "[ranked-artifact] candidates.csv contains exactly binder_a then binder_b, excluding the higher-scoring control and pending.", + "[complete-audit] The manifest and all_candidates.csv retain all five entries; reports n_candidates=4, n_passed=2, n_controls=1.", + "[control-applicability] The sequence-only control has no invented RMSD or backbone and is exempt only from RMSD; its confidence and pLDDT still pass the supplied filters.", + "[honest-coverage] Discloses pending's missing RMSD and a provisional 2/4 fraction, labels scores synthetic, and makes no live inference or scientific validation claim." + ] + }, + { + "id": "offline-mixed-threshold-ranking", + "prompt": "Review my protein binder campaign in /workspace/input/mixed_thresholds.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/mixed-thresholds. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "expected_output": "All supplied thresholds are enforced. Ranked IDs are binder_a then binder_b, despite higher ipTM on the control and designs failing other criteria. Counts are n_candidates=6, n_passed=2, n_controls=1; all seven entries remain in the audit. The string-valued interface score stays invalid rather than being coerced; pending lacks RMSD. Results are synthetic and 2/6 is provisional.", + "files": [ + "evals/files/mixed_thresholds.json" + ], + "assertions": [ + "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/mixed-thresholds; a plan alone does not pass.", + "[all-thresholds] Keeps every supplied filter; high_confidence_low_plddt and high_rmsd fail despite higher ipTM than selected designs.", + "[ranked-artifact] candidates.csv contains exactly binder_a then binder_b in descending numeric ipTM order and excludes controls, failures and incomplete designs.", + "[complete-audit] The manifest and all_candidates.csv retain all seven entries; reports n_candidates=6, n_passed=2, n_controls=1.", + "[invalid-and-missing] Preserves the string score as supplied and rejects it as invalid; pending is incomplete. Does not invent, coerce, or repair scores.", + "[honest-coverage] Reports a provisional 2/6 fraction because scoring is incomplete and labels the outputs synthetic offline bookkeeping, with no live inference claim." + ] + }, + { + "id": "offline-empty-ranking", + "prompt": "Review my protein binder campaign in /workspace/input/no_survivors.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/no-survivors. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "expected_output": "No design passes every applicable filter. candidates.csv has a header and zero data rows even though the synthetic control passes. Counts are n_candidates=4, n_passed=0, n_controls=1; all five entries remain in the audit and manifest. Pending lacks pLDDT and RMSD. The 0/4 fraction is provisional; thresholds are not relaxed to manufacture a shortlist.", + "files": [ + "evals/files/no_survivors.json" + ], + "assertions": [ + "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/no-survivors; a plan alone does not pass.", + "[empty-ranked-artifact] candidates.csv is header-only with zero data rows; the file is not omitted and there is no fallback to all candidates.", + "[control-exclusion] Excludes the passing control from the ranked design shortlist and design/pass counts.", + "[complete-audit] The manifest and all_candidates.csv retain all five entries; reports n_candidates=4, n_passed=0, n_controls=1.", + "[preserves-policy] Keeps all supplied scores, flags and thresholds; does not relax criteria to obtain a nonempty shortlist.", + "[honest-coverage] Discloses pending's missing pLDDT and RMSD, a provisional 0/4 fraction, and synthetic offline coverage; no live inference or scientific validation claim." + ] + }, + { + "id": "offline-target-preparation", + "prompt": "Prepare the supplied target for a binder workflow from /workspace/input/target_prep/request.json. Follow /workspace/input/target_prep/CONTRACT.md and write the preparation artifacts under /workspace/output/target-prep. Resolve each requested hotspot handoff independently and report which can proceed. Work offline using these synthetic inputs; do not run inference.", + "files": [ + "evals/files/target_prep/" + ], + "expected_output": "Target chain E from model 1 gives AGST, author map 10:1,42:2,42A:3,77:4. Present hotspots produce E42,E77 and sequence indices 2,4. Missing author 43 blocks that request without a nearest-residue guess. Target PDB retains chain E and author numbering.", + "assertions": [ + "[structure-selection] Uses chain E, first model and one alternate CA per residue.", + "[author-map] Preserves 42 versus 42A and sparse numbering in the map and saved PDB.", + "[handoff] Saves present hotspots as E42/E77 and sequence indices 2/4.", + "[blocked-input] Missing 43 blocks only its request; no inferred replacement or partial payload.", + "[artifacts] Writes target.pdb and preparation.json with the specified fields.", + "[coverage] Discloses synthetic offline coverage and makes no inference claim." + ] + }, + { + "id": "offline-refold-extraction", + "prompt": "Reconcile the saved refold results under /workspace/input/raw_refolds/ into two output runs /workspace/output/raw-refolds/boltz2 and /workspace/output/raw-refolds/openfold3. Follow /workspace/input/cache_contract.md, preserve the campaign policies, and report the resulting rankings and any blocked candidates. The responses are synthetic offline fixtures; make no model calls.", + "files": [ + "evals/files/raw_refolds/", + "evals/files/cache_contract.md" + ], + "expected_output": "Both rankings contain selected only. Boltz2 selected uses sample 1 confidence 0.81 and binder pLDDT 82; ipTM absent. OpenFold3 selected uses sample 1 ipTM 0.83 and binder pLDDT 86. poor_binder fails pLDDT 60 in both. Boltz2 incomplete is blocked for length mismatch; OpenFold3 composite_only is blocked for missing explicit ipTM. Counts 3/1/1 and 3/1/0, respectively; both fractions provisional.", + "assertions": [ + "[sample-binding] Scores and saved complex use the recorded sample, not the highest confidence sample.", + "[chain-confidence] Uses binder CA confidence with explicit scale and mmCIF label IDs, not target or whole-complex means.", + "[metric-separation] Does not turn a Boltz2 composite or OpenFold3 confidence_score into ipTM.", + "[partial-response] Blocks incomplete chains and absent required metrics without partial score imports.", + "[saved-results] Writes both reloaded manifests, audit/ranked CSVs and summaries with correct counts and provenance.", + "[coverage] Reports simulated offline results, unresolved candidates and provisional fractions without model calls." + ] + }, + { + "id": "offline-cache-resume", + "prompt": "Finish the recoverable offline work in /workspace/input/resume_cache/manifest.json and write the resumed run under /workspace/output/resumed. Use /workspace/input/cache_contract.md. Preserve the original run and its completed results and failure history, reconcile available cached responses, and replace stale exports. Report what remains blocked. All inputs are synthetic; do not submit inference or retries.", + "files": [ + "evals/files/resume_cache/", + "evals/files/cache_contract.md" + ], + "expected_output": "done and below are complete and unchanged; a finite low score is not pending. recoverable retains ipTM 0.86, gains binder pLDDT 84 and aligned RMSD approximately zero from cache. missing and failed stay blocked without invented metrics. Failure history and existing artifacts remain unchanged. Counts 5/2/1; ranked IDs done then recoverable; stale control/missing rows removed; 2/5 provisional.", + "assertions": [ + "[resume-preservation] Keeps complete passing and failing scores, artifact bytes and historical failed stages.", + "[cache-recovery] Recovers only missing metrics; existing ipTM 0.86 is not replaced by raw 0.98.", + "[geometry] Aligns explicit full binder chains to obtain RMSD near zero.", + "[blocked-work] Records missing_artifact and missing_prediction without retries or fabricated scores.", + "[exports] Replaces stale ranking with done/recoverable; counts 5/2/1 and full audit retained.", + "[coverage] Reports pending work and provisional simulated 2/5 with no live inference coverage." + ] + }, + { + "id": "offline-provenance-rejection", + "prompt": "Audit and reconcile /workspace/input/provenance_cache/manifest.json under /workspace/output/provenance. Follow /workspace/input/cache_contract.md. Decide which saved responses can supply scores to their candidates and produce the manifest, exports, summary and rejection reasons. Preserve the supplied evidence. These are synthetic offline fixtures; do not call models or repair provenance by guessing.", + "files": [ + "evals/files/provenance_cache/", + "evals/files/cache_contract.md" + ], + "expected_output": "Only valid is recovered and ranked, with boltz2_confidence 0.9 and binder_plddt 84. Reject candidate_swap, target_swap, sequence_swap, sample_swap and response_tampered for their respective candidate/target/sequence/selected-sample/file hash mismatch; leave all their scores empty and retain original evidence. Counts 6/1/0; 1/6 provisional; no ipTM invented.", + "assertions": [ + "[identity] Checks candidate and sequence identity before accepting scores.", + "[target] Rejects a response binding to another target digest.", + "[artifact-integrity] Checks response bytes and selected-complex digest; does not rewrite bindings to fit cached data.", + "[fail-closed] Leaves rejected candidates unscored and present in the audit with specific reasons.", + "[exports] Ranks only valid and records counts 6/1/0 with source hashes/sample provenance.", + "[coverage] Reports provisional synthetic results and no live inference coverage." + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/boltz2_only.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/boltz2_only.json new file mode 100644 index 00000000..87c42e22 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/boltz2_only.json @@ -0,0 +1,64 @@ +{ + "provenance": "Synthetic software bookkeeping fixture. Scores and thresholds are illustrative, not predictions or scientific validation. No biological sequences or live inference are represented.", + "target": { + "name": "synthetic ranking fixture" + }, + "profiles": [ + { + "name": "boltz2-only", + "rank_by": "boltz2_confidence", + "filters": { + "iptm_min": null, + "boltz2_confidence_min": 0.73, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2 + }, + "candidates": [ + { + "id": "binder_b", + "scores": { + "boltz2_confidence": 0.77, + "binder_plddt": 89, + "self_consistency_rmsd": 1.2 + }, + "is_control": false + }, + { + "id": "control", + "scores": { + "boltz2_confidence": 0.99, + "binder_plddt": 99 + }, + "is_control": true, + "control_type": "scrambled" + }, + { + "id": "binder_a", + "scores": { + "boltz2_confidence": 0.89, + "binder_plddt": 86, + "self_consistency_rmsd": 1.4 + }, + "is_control": false + }, + { + "id": "low_confidence", + "scores": { + "boltz2_confidence": 0.729, + "binder_plddt": 90, + "self_consistency_rmsd": 1 + }, + "is_control": false + }, + { + "id": "pending", + "scores": { + "boltz2_confidence": 0.95, + "binder_plddt": 90 + }, + "is_control": false + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/bookkeeping.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/bookkeeping.json new file mode 100644 index 00000000..43177c5a --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/bookkeeping.json @@ -0,0 +1,125 @@ +{ + "provenance": "Synthetic software bookkeeping fixture. Scores and thresholds are illustrative, not predictions or scientific validation. No target structure, biological sequences, model output, or endpoint access is represented.", + "target": { + "name": "synthetic bookkeeping fixture" + }, + "profiles": [ + { + "name": "iptm", + "rank_by": "iptm", + "filters": { + "iptm_min": 0.8, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2.0 + }, + "candidates": [ + { + "id": "boundary", + "scores": { + "iptm": 0.8, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "below", + "scores": { + "iptm": 0.79, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "best", + "scores": { + "iptm": 0.9, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "control", + "scores": { + "iptm": 0.99, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": true, + "control_type": "scrambled" + }, + { + "id": "pending", + "scores": { + "iptm": 0.95 + }, + "is_control": false + } + ] + }, + { + "name": "boltz2", + "rank_by": "boltz2_confidence", + "filters": { + "iptm_min": null, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2.0, + "boltz2_confidence_min": 0.8 + }, + "candidates": [ + { + "id": "boundary", + "scores": { + "boltz2_confidence": 0.8, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "below", + "scores": { + "boltz2_confidence": 0.79, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "best", + "scores": { + "boltz2_confidence": 0.9, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": false, + "control_type": null + }, + { + "id": "control", + "scores": { + "boltz2_confidence": 0.99, + "binder_plddt": 85, + "self_consistency_rmsd": 1.0 + }, + "is_control": true, + "control_type": "scrambled" + }, + { + "id": "pending", + "scores": { + "boltz2_confidence": 0.95 + }, + "is_control": false + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/cache_contract.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/cache_contract.md new file mode 100644 index 00000000..7dabe34d --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/cache_contract.md @@ -0,0 +1,42 @@ +# Saved-response fixture contract +All sequences, coordinates, responses, scores, and policies here are synthetic. Work offline; +do not read credentials, call endpoints, or claim live inference or scientific validation. +This local cache metadata is a fixture/campaign format, not an extra field promised by a NIM API. + +Copy each input run to its required output directory before modifying it. Preserve input files, +original candidates, target, params, filters, existing finite scores, artifacts, and failed stages. +Relative paths refer to that run directory. Save and reload the resulting manifest, apply every +enabled filter, and export all_candidates.csv (all entries) and candidates.csv (passing designs +in descending rank_by order, excluding controls). Empty rankings retain a header. + +For each candidate, record cache_status as complete, recovered, or blocked. Complete means all +applicable enabled scores are finite (even if below a threshold); do not recompute or replace them. +A sequence-only control without a backbone is exempt only from RMSD. For incomplete candidates, +validate the prediction binding before importing any scores: candidate ID, SHA-256 of the exact +sequence string, target.structure bytes, response file bytes, and the selected structure text. +All digests use UTF-8 without newline normalization. Copy the complete prediction binding into +score_provenance when recovering. Keep all prediction fields as supplied. A mismatch blocks the +whole import; preserve any existing scores. Never select a different sample to get a better score. + +Boltz2: structures[sample_index] pairs with confidence_scores[sample_index], in +boltz2_confidence only; the arrays must have equal length. No ipTM substitution. +OpenFold3: choose outputs by matching input_id, then structures_with_scores[sample_index]; +only iptm_score supplies ipTM. confidence_score and complex_plddt_score do not supply it. +The selected structure's binder-chain CA B factors encode per-residue pLDDT here, with the +explicit prediction.plddt_scale (0-1 or 0-100). Average the binder only and report 0-100. +Use mmCIF label_asym_id, not auth_asym_id, and require both chain roles and the full binder +length. These fixture encodings do not establish B-factor semantics for arbitrary experimental PDBs. +Recover missing RMSD by aligning the full backbone binder chain and full selected predicted +binder chain with a proper rigid rotation (Kabsch); never truncate to match lengths. + +Import only missing/invalid enabled metrics and only when all of them are recoverable. Save +the exact selected complex under the output run and add artifacts.complex with its relative path. +If blocked, leave scores unchanged and set cache_reason to candidate_mismatch, target_mismatch, +sequence_mismatch, artifact_digest_mismatch, sample_digest_mismatch, binder_length_mismatch, +missing_required_scores, missing_artifact, or missing_prediction as applicable. No API retry. +Existing failure_reason and stage entries remain as historical evidence, even after recovery. + +Write summary.json with n_candidates (non-controls), n_passed, n_controls, ranked_ids, +missing_scores (IDs to missing/invalid required metric names), passing_fraction_provisional, +label containing 'simulated', and live_inference: false. Describe unresolved candidates and +provisional fractions in the final response. A count or score alone is not evidence of execution. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/mixed_thresholds.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/mixed_thresholds.json new file mode 100644 index 00000000..990dc63f --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/mixed_thresholds.json @@ -0,0 +1,81 @@ +{ + "provenance": "Synthetic software bookkeeping fixture. Scores and thresholds are illustrative, not predictions or scientific validation. No biological sequences or live inference are represented.", + "target": { + "name": "synthetic ranking fixture" + }, + "profiles": [ + { + "name": "mixed-thresholds", + "rank_by": "iptm", + "filters": { + "iptm_min": 0.8, + "binder_plddt_min": 85, + "self_consistency_rmsd_max": 1.5 + }, + "candidates": [ + { + "id": "binder_b", + "scores": { + "iptm": 0.84, + "binder_plddt": 90, + "self_consistency_rmsd": 1 + }, + "is_control": false + }, + { + "id": "high_confidence_low_plddt", + "scores": { + "iptm": 0.99, + "binder_plddt": 84, + "self_consistency_rmsd": 1 + }, + "is_control": false + }, + { + "id": "binder_a", + "scores": { + "iptm": 0.87, + "binder_plddt": 90, + "self_consistency_rmsd": 1.2 + }, + "is_control": false + }, + { + "id": "high_rmsd", + "scores": { + "iptm": 0.98, + "binder_plddt": 95, + "self_consistency_rmsd": 1.51 + }, + "is_control": false + }, + { + "id": "pending", + "scores": { + "iptm": 0.96, + "binder_plddt": 90 + }, + "is_control": false + }, + { + "id": "control", + "scores": { + "iptm": 0.999, + "binder_plddt": 99 + }, + "is_control": true, + "control_type": "scrambled" + }, + { + "id": "invalid_score", + "scores": { + "iptm": "0.95", + "binder_plddt": 90, + "self_consistency_rmsd": 1 + }, + "is_control": false + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/no_survivors.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/no_survivors.json new file mode 100644 index 00000000..93edda5d --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/no_survivors.json @@ -0,0 +1,62 @@ +{ + "provenance": "Synthetic software bookkeeping fixture. Scores and thresholds are illustrative, not predictions or scientific validation. No biological sequences or live inference are represented.", + "target": { + "name": "synthetic ranking fixture" + }, + "profiles": [ + { + "name": "no-survivors", + "rank_by": "iptm", + "filters": { + "iptm_min": 0.8, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2 + }, + "candidates": [ + { + "id": "control", + "scores": { + "iptm": 0.99, + "binder_plddt": 99 + }, + "is_control": true, + "control_type": "scrambled" + }, + { + "id": "below", + "scores": { + "iptm": 0.79, + "binder_plddt": 90, + "self_consistency_rmsd": 1 + }, + "is_control": false + }, + { + "id": "pending", + "scores": { + "iptm": 0.95 + }, + "is_control": false + }, + { + "id": "low_plddt", + "scores": { + "iptm": 0.91, + "binder_plddt": 79, + "self_consistency_rmsd": 1 + }, + "is_control": false + }, + { + "id": "high_rmsd", + "scores": { + "iptm": 0.92, + "binder_plddt": 96, + "self_consistency_rmsd": 2.01 + }, + "is_control": false + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/backbone.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/backbone.pdb new file mode 100644 index 00000000..3f1e5d16 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/backbone.pdb @@ -0,0 +1,4 @@ +ATOM 1 CA ALA D 1 0.000 0.000 0.000 1.00 80.00 C +ATOM 2 CA ALA D 2 4.000 0.000 0.000 1.00 80.00 C +ATOM 3 CA ALA D 3 0.000 3.000 0.000 1.00 80.00 C +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/manifest.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/manifest.json new file mode 100644 index 00000000..7e34d60a --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/manifest.json @@ -0,0 +1,212 @@ +{ + "schema_version": "1.0", + "campaign": "protein-binder-design", + "mode": "offline-bookkeeping", + "run_dir": ".", + "provenance": "Synthetic offline software fixture. Toy coordinates and scores are invented; no live inference or biological validity.", + "target": { + "name": "synthetic target", + "structure": { + "path": "target.pdb", + "sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10" + } + }, + "params": { + "rank_by": "boltz2_confidence" + }, + "filters": { + "iptm_min": null, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": null, + "boltz2_confidence_min": 0.8 + }, + "stages": [ + { + "stage": "cofold", + "status": "interrupted", + "attempts": 1 + } + ], + "candidates": [ + { + "id": "valid", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "valid", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/valid.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "0c8bd86d48741f1b39a19688a25602a7a7c7dfdcea6b7154100f70895028adeb", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "candidate_swap", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "valid", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/candidate_swap.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "0c8bd86d48741f1b39a19688a25602a7a7c7dfdcea6b7154100f70895028adeb", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "target_swap", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "target_swap", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "response": { + "path": "responses/target_swap.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "0c8bd86d48741f1b39a19688a25602a7a7c7dfdcea6b7154100f70895028adeb", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "sequence_swap", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "sequence_swap", + "sequence_sha256": "4c573d7486ff3e432b9fcd07d5558134dfd4963eb541ea4ad688d42870548dfb", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/sequence_swap.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "0c8bd86d48741f1b39a19688a25602a7a7c7dfdcea6b7154100f70895028adeb", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "sample_swap", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "sample_swap", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/sample_swap.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "response_tampered", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "response_tampered", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/response_tampered.json", + "sha256": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89" + }, + "sample_index": 0, + "complex_sha256": "0c8bd86d48741f1b39a19688a25602a7a7c7dfdcea6b7154100f70895028adeb", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/candidate_swap.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/candidate_swap.json new file mode 100644 index 00000000..d00fb2cd --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/candidate_swap.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 82.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 84.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 86.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.9 + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/response_tampered.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/response_tampered.json new file mode 100644 index 00000000..8d7314a1 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/response_tampered.json @@ -0,0 +1 @@ +{"tampered": true} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/sample_swap.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/sample_swap.json new file mode 100644 index 00000000..d00fb2cd --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/sample_swap.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 82.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 84.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 86.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.9 + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/sequence_swap.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/sequence_swap.json new file mode 100644 index 00000000..d00fb2cd --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/sequence_swap.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 82.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 84.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 86.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.9 + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/target_swap.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/target_swap.json new file mode 100644 index 00000000..d00fb2cd --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/target_swap.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 82.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 84.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 86.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.9 + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/valid.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/valid.json new file mode 100644 index 00000000..d00fb2cd --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/responses/valid.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 82.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 84.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 86.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.9 + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/target.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/target.pdb new file mode 100644 index 00000000..b28e2f70 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/provenance_cache/target.pdb @@ -0,0 +1,7 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 80.00 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 80.00 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 80.00 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 80.00 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 80.00 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 80.00 C +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/backbone.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/backbone.pdb new file mode 100644 index 00000000..3f1e5d16 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/backbone.pdb @@ -0,0 +1,4 @@ +ATOM 1 CA ALA D 1 0.000 0.000 0.000 1.00 80.00 C +ATOM 2 CA ALA D 2 4.000 0.000 0.000 1.00 80.00 C +ATOM 3 CA ALA D 3 0.000 3.000 0.000 1.00 80.00 C +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/manifest.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/manifest.json new file mode 100644 index 00000000..9195aa1d --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/manifest.json @@ -0,0 +1,150 @@ +{ + "schema_version": "1.0", + "campaign": "protein-binder-design", + "mode": "offline-bookkeeping", + "run_dir": ".", + "provenance": "Synthetic offline software fixture. Toy coordinates and scores are invented; no live inference or biological validity.", + "target": { + "name": "synthetic target", + "structure": { + "path": "target.pdb", + "sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10" + } + }, + "params": { + "rank_by": "boltz2_confidence" + }, + "filters": { + "iptm_min": null, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": null, + "boltz2_confidence_min": 0.8 + }, + "stages": [ + { + "stage": "cofold", + "status": "interrupted", + "attempts": 1 + } + ], + "candidates": [ + { + "id": "selected", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "selected", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/selected.json", + "sha256": "fb0fc1994076830ceeea1d8dd2d40485a6e8658b22ff289a9e676bb480c53a2c" + }, + "sample_index": 1, + "complex_sha256": "f33530439320b437faa35521d3f526d04383f945244dfa6845ca1ee9450e2f3c", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "poor_binder", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "poor_binder", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/poor_binder.json", + "sha256": "45049bc8801df9bf9b763ec0e4539f6ce0f4828f287c326e1b657e006c084339" + }, + "sample_index": 0, + "complex_sha256": "2f65e462aa25e26b29c9ef1941287e8e7336440316dbf18e7c88e3ea6a6e885a", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "incomplete", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "boltz2", + "candidate_id": "incomplete", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/incomplete.json", + "sha256": "c23989c01579afb22e2321e24c0b10daea5ad1357cca6af74146678d5d9d9f11" + }, + "sample_index": 0, + "complex_sha256": "5657f98f6a7305a50b0c26579ea25457608cf47a987b704ad4d093899d5a6785", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "control", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": true, + "artifacts": {}, + "prediction": { + "provider": "boltz2", + "candidate_id": "control", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/control.json", + "sha256": "d3b2430f50a5e20c5af3d4b11cdc5a2d59c7717d342fedef98220e8fbd5d5ae5" + }, + "sample_index": 0, + "complex_sha256": "464735ea62f21ebc8ae58331773d6dcb85af7bfb6977999451daf066a83328e1", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/control.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/control.json new file mode 100644 index 00000000..c4d26264 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/control.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 99.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 99.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 99.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.999 + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/incomplete.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/incomplete.json new file mode 100644 index 00000000..2509f504 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/incomplete.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 85.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 85.0 2 8.0 11.0 0.0 1 8\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.95 + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/poor_binder.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/poor_binder.json new file mode 100644 index 00000000..48b3dfd4 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/poor_binder.json @@ -0,0 +1,11 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 58.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 60.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 62.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.99 + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/selected.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/selected.json new file mode 100644 index 00000000..5a5236f7 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/responses/selected.json @@ -0,0 +1,16 @@ +{ + "structures": [ + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 99.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 99.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 99.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + }, + { + "structure": "data_structure\n#\nloop_\n_atom_site.group_PDB \n_atom_site.type_symbol \n_atom_site.label_atom_id \n_atom_site.label_alt_id \n_atom_site.label_comp_id \n_atom_site.label_asym_id \n_atom_site.label_seq_id \n_atom_site.pdbx_PDB_ins_code \n_atom_site.auth_seq_id \n_atom_site.auth_comp_id \n_atom_site.auth_asym_id \n_atom_site.auth_atom_id \n_atom_site.B_iso_or_equiv \n_atom_site.label_entity_id \n_atom_site.Cartn_x \n_atom_site.Cartn_y \n_atom_site.Cartn_z \n_atom_site.pdbx_PDB_model_num \n_atom_site.id \nATOM C CA . ALA A 1 . 1 ALA T CA 99.0 1 0.0 20.0 0.0 1 1\nATOM C CA . ALA A 2 . 2 ALA T CA 99.0 1 2.0 20.0 0.0 1 2\nATOM C CA . ALA A 3 . 3 ALA T CA 99.0 1 4.0 20.0 0.0 1 3\nATOM C CA . ALA A 4 . 4 ALA T CA 99.0 1 6.0 20.0 0.0 1 4\nATOM C CA . ALA A 5 . 5 ALA T CA 99.0 1 8.0 20.0 0.0 1 5\nATOM C CA . ALA A 6 . 6 ALA T CA 99.0 1 10.0 20.0 0.0 1 6\nATOM C CA . ALA B 1 . 1 ALA Y CA 80.0 2 8.0 7.0 0.0 1 7\nATOM C CA . ALA B 2 . 2 ALA Y CA 82.0 2 8.0 11.0 0.0 1 8\nATOM C CA . ALA B 3 . 3 ALA Y CA 84.0 2 5.0 7.0 0.0 1 9\n#\n", + "format": "mmcif" + } + ], + "confidence_scores": [ + 0.99, + 0.81 + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/target.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/target.pdb new file mode 100644 index 00000000..b28e2f70 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/boltz2/target.pdb @@ -0,0 +1,7 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 80.00 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 80.00 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 80.00 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 80.00 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 80.00 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 80.00 C +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/backbone.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/backbone.pdb new file mode 100644 index 00000000..3f1e5d16 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/backbone.pdb @@ -0,0 +1,4 @@ +ATOM 1 CA ALA D 1 0.000 0.000 0.000 1.00 80.00 C +ATOM 2 CA ALA D 2 4.000 0.000 0.000 1.00 80.00 C +ATOM 3 CA ALA D 3 0.000 3.000 0.000 1.00 80.00 C +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/manifest.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/manifest.json new file mode 100644 index 00000000..d461f70a --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/manifest.json @@ -0,0 +1,121 @@ +{ + "schema_version": "1.0", + "campaign": "protein-binder-design", + "mode": "offline-bookkeeping", + "run_dir": ".", + "provenance": "Synthetic offline software fixture. Toy coordinates and scores are invented; no live inference or biological validity.", + "target": { + "name": "synthetic target", + "structure": { + "path": "target.pdb", + "sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10" + } + }, + "params": { + "rank_by": "iptm" + }, + "filters": { + "iptm_min": 0.8, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": null + }, + "stages": [ + { + "stage": "cofold", + "status": "interrupted", + "attempts": 1 + } + ], + "candidates": [ + { + "id": "selected", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "selected", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/selected.json", + "sha256": "605dd0759e2642108810f6a88baef0f12427a86f2e5acfe3349f19d9a4b7399b" + }, + "sample_index": 1, + "complex_sha256": "a0b0b48b1a155ac4a12fc720db0e7d18afb08052035a84d60abc1e98999cf355", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "composite_only", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "composite_only", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/composite_only.json", + "sha256": "5123c9b20050bc54cdb559537b2ffc32d3d630d8683af02746bad5a234a133c4" + }, + "sample_index": 0, + "complex_sha256": "2c33a2375e2be4a7ffe87376f45952734d46ce9e03209a6cfe363366cb8cf045", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "poor_binder", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "poor_binder", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/poor_binder.json", + "sha256": "119c9a8719156a0d169d9258bc52dabe4ad497fffdfa0ab0c3497bc1dec32fb1" + }, + "sample_index": 0, + "complex_sha256": "3bc1ef048158c2b026eb491a4e916b2d9fa5fb0d8571f0fe7679b4aff5437f95", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/composite_only.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/composite_only.json new file mode 100644 index 00000000..85a99fa2 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/composite_only.json @@ -0,0 +1,15 @@ +{ + "outputs": [ + { + "input_id": "composite_only", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.90 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.90 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.90 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97 + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/poor_binder.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/poor_binder.json new file mode 100644 index 00000000..b8cd92e2 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/poor_binder.json @@ -0,0 +1,16 @@ +{ + "outputs": [ + { + "input_id": "poor_binder", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.58 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.60 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.62 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.99 + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/selected.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/selected.json new file mode 100644 index 00000000..1a354376 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/responses/selected.json @@ -0,0 +1,23 @@ +{ + "outputs": [ + { + "input_id": "selected", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.99 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.99 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.99 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.99 + }, + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.84 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.86 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.88 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.83 + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/target.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/target.pdb new file mode 100644 index 00000000..b28e2f70 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/raw_refolds/openfold3/target.pdb @@ -0,0 +1,7 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 80.00 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 80.00 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 80.00 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 80.00 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 80.00 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 80.00 C +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/backbone.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/backbone.pdb new file mode 100644 index 00000000..3f1e5d16 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/backbone.pdb @@ -0,0 +1,4 @@ +ATOM 1 CA ALA D 1 0.000 0.000 0.000 1.00 80.00 C +ATOM 2 CA ALA D 2 4.000 0.000 0.000 1.00 80.00 C +ATOM 3 CA ALA D 3 0.000 3.000 0.000 1.00 80.00 C +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/below.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/below.pdb new file mode 100644 index 00000000..c1059fc9 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/below.pdb @@ -0,0 +1,10 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C +ATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C +ATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C +ATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/candidates.csv b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/candidates.csv new file mode 100644 index 00000000..f9c62572 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/candidates.csv @@ -0,0 +1,3 @@ +id,iptm +control,0.99 +missing,1.0 diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/done.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/done.pdb new file mode 100644 index 00000000..c1059fc9 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/done.pdb @@ -0,0 +1,10 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C +ATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C +ATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C +ATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/manifest.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/manifest.json new file mode 100644 index 00000000..7e09e729 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/manifest.json @@ -0,0 +1,206 @@ +{ + "schema_version": "1.0", + "campaign": "protein-binder-design", + "mode": "offline-bookkeeping", + "run_dir": ".", + "provenance": "Synthetic offline software fixture. Toy coordinates and scores are invented; no live inference or biological validity.", + "target": { + "name": "synthetic target", + "structure": { + "path": "target.pdb", + "sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10" + } + }, + "params": { + "rank_by": "iptm" + }, + "filters": { + "iptm_min": 0.8, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2 + }, + "stages": [ + { + "stage": "cofold", + "status": "interrupted", + "attempts": 1 + } + ], + "candidates": [ + { + "id": "done", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": { + "iptm": 0.9, + "binder_plddt": 90, + "self_consistency_rmsd": 0 + }, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb", + "complex": "done.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "done", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/done.json", + "sha256": "94b16fe78c2cea20d2f2179d1a763feb98ec9de71e0222825cb2cc4bd1f1669f" + }, + "sample_index": 0, + "complex_sha256": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "below", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": { + "iptm": 0.7, + "binder_plddt": 85, + "self_consistency_rmsd": 0 + }, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb", + "complex": "below.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "below", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/below.json", + "sha256": "9486ff3d9be5100cacbf77d25a604c27db8caa89e3ee82d161056138ae45db0c" + }, + "sample_index": 0, + "complex_sha256": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "recoverable", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": { + "iptm": 0.86 + }, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "recoverable", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/recoverable.json", + "sha256": "214ee2ae319ad54656a6a3406ecacc571bee992c178881d61199d35c9f8d81af" + }, + "sample_index": 0, + "complex_sha256": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "missing", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "prediction": { + "provider": "openfold3", + "candidate_id": "missing", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/missing.json", + "sha256": "5ed61eea6199299dba8ef59ffea5fa7fc23e0e54c81d1792a75551e09e7c8c8e" + }, + "sample_index": 0, + "complex_sha256": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + }, + { + "id": "failed", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": {}, + "is_control": false, + "artifacts": { + "backbone_pdb": "backbone.pdb" + }, + "failure_reason": "HTTP 503: retries exhausted", + "attempts": 3 + }, + { + "id": "control", + "sequence": "AAA", + "backbone_id": "toy-backbone", + "scores": { + "iptm": 0.99, + "binder_plddt": 88 + }, + "is_control": true, + "artifacts": {}, + "prediction": { + "provider": "openfold3", + "candidate_id": "control", + "sequence_sha256": "cb1ad2119d8fafb69566510ee712661f9f14b83385006ef92aec47f523a38358", + "target_sha256": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "response": { + "path": "responses/control.json", + "sha256": "7c074850d88c42357c6983784e8c7d28dcc14fa728dc3b9ea278ee0167b14b94" + }, + "sample_index": 0, + "complex_sha256": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "binder_chain": "B", + "target_chain": "T", + "plddt_scale": "0-1", + "backbone": { + "path": "backbone.pdb", + "sha256": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764" + }, + "backbone_chain": "D" + } + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/below.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/below.json new file mode 100644 index 00000000..bc8a7909 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/below.json @@ -0,0 +1,16 @@ +{ + "outputs": [ + { + "input_id": "below", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.9 + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/control.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/control.json new file mode 100644 index 00000000..cbbe342e --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/control.json @@ -0,0 +1,16 @@ +{ + "outputs": [ + { + "input_id": "control", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.9 + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/done.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/done.json new file mode 100644 index 00000000..aaf3d163 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/done.json @@ -0,0 +1,16 @@ +{ + "outputs": [ + { + "input_id": "done", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.9 + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/recoverable.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/recoverable.json new file mode 100644 index 00000000..85b66a7b --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/responses/recoverable.json @@ -0,0 +1,16 @@ +{ + "outputs": [ + { + "input_id": "recoverable", + "structures_with_scores": [ + { + "structure": "ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 0.99 C \nATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 0.99 C \nATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 0.99 C \nATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 0.99 C \nATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 0.99 C \nATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 0.99 C \nATOM 7 CA ALA B 1 8.000 7.000 0.000 1.00 0.82 C \nATOM 8 CA ALA B 2 8.000 11.000 0.000 1.00 0.84 C \nATOM 9 CA ALA B 3 5.000 7.000 0.000 1.00 0.86 C \nEND\n", + "format": "pdb", + "confidence_score": 0.99, + "complex_plddt_score": 0.97, + "iptm_score": 0.98 + } + ] + } + ] +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/target.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/target.pdb new file mode 100644 index 00000000..b28e2f70 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/resume_cache/target.pdb @@ -0,0 +1,7 @@ +ATOM 1 CA ALA T 1 0.000 20.000 0.000 1.00 80.00 C +ATOM 2 CA ALA T 2 2.000 20.000 0.000 1.00 80.00 C +ATOM 3 CA ALA T 3 4.000 20.000 0.000 1.00 80.00 C +ATOM 4 CA ALA T 4 6.000 20.000 0.000 1.00 80.00 C +ATOM 5 CA ALA T 5 8.000 20.000 0.000 1.00 80.00 C +ATOM 6 CA ALA T 6 10.000 20.000 0.000 1.00 80.00 C +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/CONTRACT.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/CONTRACT.md new file mode 100644 index 00000000..6163462f --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/CONTRACT.md @@ -0,0 +1,11 @@ +# Offline target preparation contract +This is a synthetic software fixture. No API calls, inference, or biological validation. +Use model 1 and the chain in request.json. Keep author numbers and insertion codes distinct; +choose one CA per residue using the supplied alternate-location policy and preserve file order. +Write target.pdb (only the chosen chain/model, original author numbering) and preparation.json: +`chain`, `sequence`, `author_to_sequence` (string author ID to 1-based integer), and `requests` +(keyed by request ID). Each request has `status`: `ready` or `blocked`. A ready request has +`hotspot_res` (chain+author strings) and `sequence_indices`. A missing hotspot blocks that +request and records `missing_author_residues`; do not choose a nearby residue or emit its +partial handoff. Other requests can succeed independently. Add `simulated: true` and +`live_inference: false`. Do not renumber the PDB or combine alternate locations/models. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/request.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/request.json new file mode 100644 index 00000000..5de39eaa --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/request.json @@ -0,0 +1,22 @@ +{ + "provenance": "Synthetic toy PDB, no real target or inference.", + "structure": "target.pdb", + "chain": "E", + "requests": [ + { + "id": "present", + "hotspots": [ + "42", + "77" + ] + }, + { + "id": "missing", + "hotspots": [ + "43" + ] + } + ], + "model": 1, + "alternate_location_policy": "Prefer blank; otherwise highest occupancy; break ties by lexicographic altloc." +} diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/target.pdb b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/target.pdb new file mode 100644 index 00000000..f72ac714 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/target_prep/target.pdb @@ -0,0 +1,12 @@ +MODEL 1 +ATOM 1 CA ALA A 42 99.000 99.000 99.000 1.00 80.00 C +ATOM 2 CA ALA E 10 0.000 0.000 0.000 1.00 80.00 C +ATOM 3 CA AGLY E 42 9.000 9.000 9.000 0.20 80.00 C +ATOM 4 CA BGLY E 42 4.000 0.000 0.000 0.80 80.00 C +ATOM 5 CA SER E 42A 4.000 3.000 0.000 1.00 80.00 C +ATOM 6 CA THR E 77 8.000 3.000 0.000 1.00 80.00 C +ENDMDL +MODEL 2 +ATOM 7 CA VAL E 10 50.000 50.000 50.000 1.00 80.00 C +ENDMDL +END diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/grader.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/grader.py new file mode 100644 index 00000000..e2575799 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/grader.py @@ -0,0 +1,633 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Standalone artifact verifier; no agent helper imports, network or LLM calls. + +Harbor copies only this file to /tests. Input digests below pin the fixture truth; +an agent cannot change that truth by editing its input copies. ACES plus custom +keeps the ACES headline score: artifact_correctness is a separate binary metric. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import math +import os +from pathlib import Path + + +BOOKKEEPING = { + "offline-manifest": ("bookkeeping.json", None, ["best", "boundary"]), + "offline-boltz2-ranking": ( + "boltz2_only.json", + "boltz2-only", + ["binder_a", "binder_b"], + ), + "offline-mixed-threshold-ranking": ( + "mixed_thresholds.json", + "mixed-thresholds", + ["binder_a", "binder_b"], + ), + "offline-empty-ranking": ("no_survivors.json", "no-survivors", []), +} +# Hand-calculated outcomes, independent of the campaign helper and its parser. +CACHE_RUNS = { + "raw_refolds/boltz2": ( + "raw-refolds/boltz2", + ["selected"], + { + "selected": ( + {"boltz2_confidence": 0.81, "binder_plddt": 82}, + "recovered", + None, + ), + "poor_binder": ( + {"boltz2_confidence": 0.99, "binder_plddt": 60}, + "recovered", + None, + ), + "incomplete": ({}, "blocked", "binder_length_mismatch"), + "control": ( + {"boltz2_confidence": 0.999, "binder_plddt": 99}, + "recovered", + None, + ), + }, + ), + "raw_refolds/openfold3": ( + "raw-refolds/openfold3", + ["selected"], + { + "selected": ({"iptm": 0.83, "binder_plddt": 86}, "recovered", None), + "composite_only": ({}, "blocked", "missing_required_scores"), + "poor_binder": ({"iptm": 0.99, "binder_plddt": 60}, "recovered", None), + }, + ), + "resume_cache": ( + "resumed", + ["done", "recoverable"], + { + "done": ( + {"iptm": 0.9, "binder_plddt": 90, "self_consistency_rmsd": 0}, + "complete", + None, + ), + "below": ( + {"iptm": 0.7, "binder_plddt": 85, "self_consistency_rmsd": 0}, + "complete", + None, + ), + "recoverable": ( + {"iptm": 0.86, "binder_plddt": 84, "self_consistency_rmsd": 0}, + "recovered", + None, + ), + "missing": ({}, "blocked", "missing_artifact"), + "failed": ({}, "blocked", "missing_prediction"), + "control": ({"iptm": 0.99, "binder_plddt": 88}, "complete", None), + }, + ), + "provenance_cache": ( + "provenance", + ["valid"], + { + "valid": ( + {"boltz2_confidence": 0.9, "binder_plddt": 84}, + "recovered", + None, + ), + "candidate_swap": ({}, "blocked", "candidate_mismatch"), + "target_swap": ({}, "blocked", "target_mismatch"), + "sequence_swap": ({}, "blocked", "sequence_mismatch"), + "sample_swap": ({}, "blocked", "sample_digest_mismatch"), + "response_tampered": ({}, "blocked", "artifact_digest_mismatch"), + }, + ), +} +CACHE_CASES = { + "offline-refold-extraction": ["raw_refolds/boltz2", "raw_refolds/openfold3"], + "offline-cache-resume": ["resume_cache"], + "offline-provenance-rejection": ["provenance_cache"], +} + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def numeric(value): + return type(value) in (int, float) and math.isfinite(value) + + +def same(actual, expected, *, approximate=False): + if isinstance(expected, dict): + return ( + isinstance(actual, dict) + and actual.keys() == expected.keys() + and all( + same(actual[k], v, approximate=approximate) for k, v in expected.items() + ) + ) + if isinstance(expected, list): + return ( + isinstance(actual, list) + and len(actual) == len(expected) + and all( + same(a, e, approximate=approximate) for a, e in zip(actual, expected) + ) + ) + if numeric(expected): + return numeric(actual) and ( + math.isclose(actual, expected, rel_tol=1e-7, abs_tol=1e-6) + if approximate + else actual == expected + ) + return type(actual) is type(expected) and actual == expected + + +def read_json(path): + return json.loads(Path(path).read_text()) + + +def metrics(candidate, filters): + required = {} + for key, value in filters.items(): + if value is None: + continue + metric, direction = key.rsplit("_", 1) + if ( + metric == "self_consistency_rmsd" + and candidate.get("is_control") + and not candidate.get("artifacts", {}).get("backbone_pdb") + ): + continue + required[metric] = (direction, value) + return required + + +def passes(candidate, filters): + required = metrics(candidate, filters) + for key, (direction, cutoff) in required.items(): + value = candidate["scores"].get(key) + if not numeric(value) or ( + value < cutoff if direction == "min" else value > cutoff + ): + return False + return bool(required) + + +def check_csv(path, expected, ids, check, *, ordered, approximate=False): + with path.open(newline="") as stream: + reader = csv.DictReader(stream) + rows = list(reader) + headers = reader.fieldnames or [] + check(bool(headers) and "id" in headers, f"{path}: CSV header") + actual_ids = [row.get("id") for row in rows] + check( + actual_ids == ids if ordered else sorted(actual_ids) == sorted(ids), + f"{path}: candidate IDs", + ) + lookup = {c["id"]: c for c in expected} + score_keys = {key for c in expected for key in c["scores"]} + check(score_keys.issubset(headers), f"{path}: supplied score columns") + if "iptm" not in score_keys: + check( + "iptm" not in [key.lower() for key in headers], + f"{path}: absent ipTM stays absent", + ) + for row in rows: + if row.get("id") not in lookup: + continue + candidate = lookup[row["id"]] + for key in score_keys: + expected_value, actual = candidate["scores"].get(key), row.get(key) + if expected_value is None: + ok = actual == "" + elif numeric(expected_value): + try: + ok = same(float(actual), expected_value, approximate=approximate) + except (ValueError, TypeError): + ok = False + else: + ok = actual == str(expected_value) + check(ok, f"{path}: {row['id']}.{key}") + if "is_control" in headers: + check( + str(row["is_control"]).lower() + == str(candidate.get("is_control", False)).lower(), + f"{path}: {row['id']} control flag", + ) + + +def verify_run(path, source, expected, ranked_ids, check, *, cache=False): + data = read_json(path / "manifest.json") + check(same(data.get("target"), source["target"]), f"{path}: target") + check(same(data.get("filters"), source["filters"]), f"{path}: filters") + rank_by = data.get("params", {}).get("rank_by", data.get("rank_by")) + check(rank_by == source["params"]["rank_by"], f"{path}: rank metric") + actual = data.get("candidates", []) + check( + sorted(c["id"] for c in actual) == sorted(c["id"] for c in expected), + f"{path}: complete manifest", + ) + lookup = {c["id"]: c for c in actual} + for candidate in expected: + cid = candidate["id"] + record = lookup.get(cid, {}) + check( + same(record.get("scores"), candidate["scores"], approximate=cache), + f"{path}: {cid} score values/types/absence", + ) + check( + record.get("is_control", False) is candidate.get("is_control", False), + f"{path}: {cid} control flag", + ) + check( + record.get("control_type") == candidate.get("control_type"), + f"{path}: {cid} control type", + ) + for flag in ("passed_filter", "passed"): + if flag in record: + check( + record[flag] is passes(candidate, source["filters"]), + f"{path}: {cid} filter status", + ) + check_csv( + path / "all_candidates.csv", + expected, + [c["id"] for c in expected], + check, + ordered=False, + approximate=cache, + ) + # Older prompts used the ambiguous phrase "ranked candidates.csv". Accept + # the three literal interpretations without changing historical prompts. + alternatives = [ + path / name + for name in ("candidates.csv", "ranked candidates.csv", "ranked_candidates.csv") + ] + ranked_paths = [p for p in alternatives if p.exists()] + check(bool(ranked_paths), f"{path}: ranked CSV exists") + if cache: + check((path / "candidates.csv").exists(), f"{path}: required candidates.csv") + for ranked_path in ranked_paths: + check_csv( + ranked_path, expected, ranked_ids, check, ordered=True, approximate=cache + ) + return data, lookup + + +def find_manifests(workspace): + for root, dirs, names in os.walk(workspace): + dirs[:] = [ + d + for d in dirs + if d not in {"input", "skills", "node_modules"} and not d.startswith(".") + ] + if "manifest.json" in names: + yield Path(root) + + +def verify_bookkeeping(entry_id, workspace, check): + filename, destination, ranked = BOOKKEEPING[entry_id] + fixture = read_json(workspace / "input" / filename) + for profile in fixture["profiles"]: + source = { + "target": fixture["target"], + "filters": profile["filters"], + "params": {"rank_by": profile["rank_by"]}, + } + paths = ( + [workspace / "output" / destination] + if destination + else list(find_manifests(workspace)) + ) + attempts = [] + for path in paths: + checks = [] + try: + data = read_json(path / "manifest.json") + if ( + data.get("params", {}).get("rank_by", data.get("rank_by")) + != profile["rank_by"] + ): + continue + verify_run( + path, + source, + profile["candidates"], + ranked, + lambda ok, msg: checks.append((ok, msg)), + ) + except (OSError, ValueError, KeyError, TypeError) as exc: + checks.append((False, f"{path}: {type(exc).__name__}: {exc}")) + attempts.append(checks) + valid = next((a for a in attempts if a and all(ok for ok, _ in a)), None) + chosen = ( + valid + if valid is not None + else ( + max(attempts, key=lambda a: sum(ok for ok, _ in a)) if attempts else [] + ) + ) + check(bool(chosen), f"{profile['name']}: output manifest found") + for ok, msg in chosen: + check(ok, msg) + + +def verify_cache(name, workspace, check): + destination, ranked, outcomes = CACHE_RUNS[name] + source_root, output = workspace / "input" / name, workspace / "output" / destination + source = read_json(source_root / "manifest.json") + expected = [{**c, "scores": outcomes[c["id"]][0]} for c in source["candidates"]] + data, lookup = verify_run(output, source, expected, ranked, check, cache=True) + check( + same(data.get("params"), source["params"]), f"{output}: complete saved params" + ) + check( + same(data.get("stages", [])[: len(source["stages"])], source["stages"]), + f"{output}: stage history", + ) + for original in source["candidates"]: + cid = original["id"] + actual = lookup.get(cid, {}) + _, status, reason = outcomes[cid] + check(actual.get("cache_status") == status, f"{output}: {cid} decision") + if reason: + check( + actual.get("cache_reason") == reason, + f"{output}: {cid} rejection reason", + ) + for key, value in original.items(): + if key not in {"scores", "artifacts"}: + check(same(actual.get(key), value), f"{output}: {cid} preserves {key}") + for key, value in original["scores"].items(): + check( + same(actual.get("scores", {}).get(key), value), + f"{output}: {cid} preserves score {key}", + ) + for key, value in original.get("artifacts", {}).items(): + check( + actual.get("artifacts", {}).get(key) == value, + f"{output}: {cid} preserves artifact {key}", + ) + if status == "recovered": + binding = original["prediction"] + check( + same(actual.get("score_provenance"), binding), + f"{output}: {cid} score provenance", + ) + complex_path = ( + output / actual.get("artifacts", {}).get("complex", "") + ).resolve() + check( + complex_path.is_relative_to(output.resolve()) + and complex_path.is_file() + and sha(complex_path) == binding["complex_sha256"], + f"{output}: {cid} saved selected complex", + ) + # Saved evidence is copied without changing bytes, even when it is rejected. + for rel, digest in INPUT_HASHES.items(): + prefix = name + "/" + if rel.startswith(prefix) and rel[len(prefix) :] not in { + "manifest.json", + "candidates.csv", + }: + path = output / rel[len(prefix) :] + check( + path.is_file() and sha(path) == digest, + f"{path}: preserved evidence bytes", + ) + summary = read_json(output / "summary.json") + missing = { + c["id"]: [ + key + for key in metrics(c, source["filters"]) + if not numeric(c["scores"].get(key)) + ] + for c in expected + } + missing = {key: value for key, value in missing.items() if value} + required_summary = { + "n_candidates": sum(not c.get("is_control", False) for c in expected), + "n_passed": len(ranked), + "n_controls": sum(c.get("is_control", False) for c in expected), + "ranked_ids": ranked, + "passing_fraction_provisional": True, + "live_inference": False, + } + for key, value in required_summary.items(): + check(same(summary.get(key), value), f"{output}: summary {key}") + actual_missing = summary.get("missing_scores", {}) + check( + isinstance(actual_missing, dict) + and {k: sorted(v) for k, v in actual_missing.items()} + == {k: sorted(v) for k, v in missing.items()}, + f"{output}: missing scores disclosure", + ) + check( + "simulated" in str(summary.get("label", "")).lower(), + f"{output}: simulation label", + ) + + +def target_atoms(text): + """Read structural fields without depending on PDB writer formatting.""" + atoms = {} + explicit_model = False + in_model = False + ended = False + for line in text.splitlines(): + record = line[:6].strip() + if record == "MODEL": + if explicit_model or atoms or ended or int(line[10:14]) != 1: + raise ValueError("target must contain only the first model") + explicit_model = in_model = True + elif record == "ENDMDL": + if not in_model: + raise ValueError("unexpected ENDMDL in target") + in_model = False + elif record == "END": + ended = True + elif record in {"ATOM", "HETATM"}: + if ended or (explicit_model and not in_model): + raise ValueError("target atom outside its model") + key = (line[21], int(line[22:26]), line[26].strip(), line[12:16].strip()) + if key in atoms: + raise ValueError(f"duplicate target atom/conformer: {key}") + coordinates = tuple( + float(line[start : start + 8]) for start in (30, 38, 46) + ) + if not all(math.isfinite(value) for value in coordinates): + raise ValueError("non-finite target coordinates") + atoms[key] = (line[17:20].strip(), line[16].strip(), coordinates) + if in_model: + raise ValueError("unclosed target model") + return atoms + + +def verify_target(workspace, check): + root = workspace / "output/target-prep" + data = read_json(root / "preparation.json") + expected = { + "chain": "E", + "sequence": "AGST", + "author_to_sequence": {"10": 1, "42": 2, "42A": 3, "77": 4}, + "simulated": True, + "live_inference": False, + } + for key, value in expected.items(): + check(same(data.get(key), value), f"target preparation: {key}") + requests = data.get("requests", {}) + present, absent = requests.get("present", {}), requests.get("missing", {}) + check( + present.get("status") == "ready" + and present.get("hotspot_res") == ["E42", "E77"] + and same(present.get("sequence_indices"), [2, 4]), + "present hotspot handoff", + ) + check( + absent.get("status") == "blocked" + and absent.get("missing_author_residues") == ["43"] + and not absent.get("hotspot_res") + and not absent.get("sequence_indices"), + "missing hotspot blocks its request", + ) + atoms = target_atoms((root / "target.pdb").read_text()) + # Hand-checked first-model atoms from the digest-pinned synthetic input. + expected_atoms = { + ("E", 10, "", "CA"): ("ALA", "", (0.0, 0.0, 0.0)), + ("E", 42, "", "CA"): ("GLY", "B", (4.0, 0.0, 0.0)), + ("E", 42, "A", "CA"): ("SER", "", (4.0, 3.0, 0.0)), + ("E", 77, "", "CA"): ("THR", "", (8.0, 3.0, 0.0)), + } + check( + list(atoms) == list(expected_atoms) + and all( + atoms[key][0] == residue + # Writers may clear altLoc after selecting the required conformer. + and atoms[key][1] in {"", altloc} + and all( + math.isclose(actual, expected, rel_tol=0, abs_tol=0.0005) + for actual, expected in zip(atoms[key][2], coordinates) + ) + for key, (residue, altloc, coordinates) in expected_atoms.items() + ), + "extracted PDB chain/model/author numbering/selected coordinates", + ) + + +def grade(entry_id, workspace): + workspace = Path(workspace) + checks = [] + + def check(ok, message): + checks.append({"passed": bool(ok), "message": message}) + + if entry_id in BOOKKEEPING: + prefixes = [BOOKKEEPING[entry_id][0]] + elif entry_id in CACHE_CASES: + prefixes = [name + "/" for name in CACHE_CASES[entry_id]] + [ + "cache_contract.md" + ] + elif entry_id == "offline-target-preparation": + prefixes = ["target_prep/"] + else: + prefixes = [] + check(False, f"unknown case: {entry_id}") + for rel, digest in INPUT_HASHES.items(): + if any( + rel == prefix or (prefix.endswith("/") and rel.startswith(prefix)) + for prefix in prefixes + ): + path = workspace / "input" / rel + check(path.is_file() and sha(path) == digest, f"immutable fixture: {rel}") + if checks and all(item["passed"] for item in checks): + try: + if entry_id in BOOKKEEPING: + verify_bookkeeping(entry_id, workspace, check) + elif entry_id in CACHE_CASES: + for name in CACHE_CASES[entry_id]: + verify_cache(name, workspace, check) + else: + verify_target(workspace, check) + except (OSError, ValueError, KeyError, TypeError) as exc: + check( + False, f"artifact read/validation failed: {type(exc).__name__}: {exc}" + ) + passed = bool(checks) and all(item["passed"] for item in checks) + return { + "overall": float(passed), + "custom_metrics": {"artifact_correctness": float(passed)}, + "details": { + "artifact_correctness": { + "passed": passed, + "checks": checks, + "scope": "saved artifacts only; no proof of live inference or save/reload execution", + } + }, + } + + +def main(): + tests = Path(os.environ.get("HARBOR_TESTS_DIR", "/tests")) + verifier = Path(os.environ.get("HARBOR_VERIFIER_DIR", "/logs/verifier")) + entry = read_json(tests / "entry.json") + result = grade( + entry["id"], Path(os.environ.get("BINDER_EVAL_WORKSPACE", "/workspace")) + ) + verifier.mkdir(parents=True, exist_ok=True) + Path( + os.environ.get("HARBOR_REWARD_JSON", str(verifier / "reward.json")) + ).write_text(json.dumps(result, indent=2) + "\n") + + +# INPUT_HASHES is pinned below, outside the agent's mutable /workspace/input. +INPUT_HASHES = { + "boltz2_only.json": "c7b760c45bb2950f59c277eb08587789d8b945f03ca12f0d5407aa147e74ee59", + "bookkeeping.json": "23a769f5f3f02c96c6758eb3b5787b99befbe69b898a64f170fe416c24a80fd3", + "cache_contract.md": "24744e774ff688963bf99b540c7cfd9c525b8c870b2ce863cf298a210262edb6", + "mixed_thresholds.json": "fcc5b8aa40870a1fb0e9deb4362a60996fc155121f21365bb1640aae97ae5420", + "no_survivors.json": "2bfde692731de2850bee4f126dcc2c75cbd4e0b7eb97b1bbaa9566ea077f3022", + "provenance_cache/backbone.pdb": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764", + "provenance_cache/manifest.json": "5f08e89bd297f0c42e8e5302353a03d31f8e885c87178c807c938f4568c622cc", + "provenance_cache/responses/candidate_swap.json": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89", + "provenance_cache/responses/response_tampered.json": "7943c5202ba623e735fafa2d2661810171ada09095bed9038d0a06f9232eda76", + "provenance_cache/responses/sample_swap.json": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89", + "provenance_cache/responses/sequence_swap.json": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89", + "provenance_cache/responses/target_swap.json": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89", + "provenance_cache/responses/valid.json": "b7529dacb3c5123b550074006b88536e17ead761ddb79b71d12c4a8a7bbf3d89", + "provenance_cache/target.pdb": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "raw_refolds/boltz2/backbone.pdb": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764", + "raw_refolds/boltz2/manifest.json": "b2666564b055d96269d619d6c0b6fe33229878fa5625342c567444d99d243055", + "raw_refolds/boltz2/responses/control.json": "d3b2430f50a5e20c5af3d4b11cdc5a2d59c7717d342fedef98220e8fbd5d5ae5", + "raw_refolds/boltz2/responses/incomplete.json": "c23989c01579afb22e2321e24c0b10daea5ad1357cca6af74146678d5d9d9f11", + "raw_refolds/boltz2/responses/poor_binder.json": "45049bc8801df9bf9b763ec0e4539f6ce0f4828f287c326e1b657e006c084339", + "raw_refolds/boltz2/responses/selected.json": "fb0fc1994076830ceeea1d8dd2d40485a6e8658b22ff289a9e676bb480c53a2c", + "raw_refolds/boltz2/target.pdb": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "raw_refolds/openfold3/backbone.pdb": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764", + "raw_refolds/openfold3/manifest.json": "c1080b79c08db16a76ec89cf754f5d859fbb9ee0e18fee74249e49151990f06a", + "raw_refolds/openfold3/responses/composite_only.json": "5123c9b20050bc54cdb559537b2ffc32d3d630d8683af02746bad5a234a133c4", + "raw_refolds/openfold3/responses/poor_binder.json": "119c9a8719156a0d169d9258bc52dabe4ad497fffdfa0ab0c3497bc1dec32fb1", + "raw_refolds/openfold3/responses/selected.json": "605dd0759e2642108810f6a88baef0f12427a86f2e5acfe3349f19d9a4b7399b", + "raw_refolds/openfold3/target.pdb": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "resume_cache/backbone.pdb": "217f6524bd2ce0e9ddac2cf44fc2c3ef091768bf8539f269be520f4fad5de764", + "resume_cache/below.pdb": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "resume_cache/candidates.csv": "0d4f8840021deaf5107cb8eb02577acd1db45f26b8cb4ae043a6f0957db32f0e", + "resume_cache/done.pdb": "038c31607d7610be18bf20022f25adb00866733c70d26d372458d7d73e516b86", + "resume_cache/manifest.json": "35a9958917c527b27fdb679df481aaa34b7a0f4bd59ef6ec5219752be08bf119", + "resume_cache/responses/below.json": "9486ff3d9be5100cacbf77d25a604c27db8caa89e3ee82d161056138ae45db0c", + "resume_cache/responses/control.json": "7c074850d88c42357c6983784e8c7d28dcc14fa728dc3b9ea278ee0167b14b94", + "resume_cache/responses/done.json": "94b16fe78c2cea20d2f2179d1a763feb98ec9de71e0222825cb2cc4bd1f1669f", + "resume_cache/responses/recoverable.json": "214ee2ae319ad54656a6a3406ecacc571bee992c178881d61199d35c9f8d81af", + "resume_cache/target.pdb": "c8d7a6f455e0a524ec91d0689377243becb1f9395b5d83d51fc7089c091f9d10", + "target_prep/CONTRACT.md": "886e330fa4562df5029c874418a92bf108ec12331b69e16506f57d9ed9e7382b", + "target_prep/request.json": "d8d0c7e0e92d0c18a1a493a4853f0e7e1718be9bb30c2cbd44c9410747652298", + "target_prep/target.pdb": "6ef9dfdb6bf827b336d34aa768c36ed238581d365fab228ac2452ed0e019ca45", +} + + +if __name__ == "__main__": + main() diff --git a/workflows/generative-protein-binder-design/protein-binder-design/evals/trigger_evals.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/trigger_evals.json similarity index 100% rename from workflows/generative-protein-binder-design/protein-binder-design/evals/trigger_evals.json rename to workflow-skills/generative-protein-binder-design/protein-binder-design/evals/trigger_evals.json diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/local-nim-setup.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/local-nim-setup.md new file mode 100644 index 00000000..b8495e91 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/local-nim-setup.md @@ -0,0 +1,118 @@ +# Running the BioNeMo NIMs locally + +Use this guide when the user chooses self-hosted RFdiffusion, ProteinMPNN, and +Boltz2. Inference inputs go to the configured local services without an +Authorization header. Pulling containers and downloading model weights still +requires access to NVIDIA NGC. + +## Prerequisites + +- Docker with the NVIDIA Container Toolkit and a supported GPU/driver stack. +- An NGC key in `NGC_API_KEY`, with access to the selected model versions and + their governing terms accepted. +- Choose the image version and digest from each model's NGC catalog: + [RFdiffusion](https://catalog.ngc.nvidia.com/orgs/nim/teams/ipd/containers/rfdiffusion), + [ProteinMPNN](https://catalog.ngc.nvidia.com/orgs/nim/teams/ipd/containers/proteinmpnn), + [Boltz2](https://catalog.ngc.nvidia.com/orgs/nim/teams/mit/containers/boltz2). + +Authenticate with the existing environment key through stdin, with shell tracing +disabled. This avoids putting the secret in the command's argument list: + +```bash +set +x +printf '%s' "${NGC_API_KEY:?Set NGC_API_KEY}" | docker login nvcr.io --username '$oauthtoken' --password-stdin +``` + +## Image and API contracts + +Set `RFD_IMAGE`, `PMPNN_IMAGE`, and `BOLTZ2_IMAGE` to the respective full NGC +repository reference ending in `@sha256:` plus its 64-character digest. Obtain +the digest from the selected catalog version or the `RepoDigests` output of +`docker image inspect` after pulling that explicit version. Do not invent a +digest or use a moving tag. Save the three references and selected GPU profiles +in the campaign parameters so a resumed run uses the same containers. + +| NIM | Image variable | Cache mount | Local prediction path | +|---|---|---|---| +| RFdiffusion | `RFD_IMAGE` | `/opt/nim/.cache` | `/biology/ipd/rfdiffusion/generate` | +| ProteinMPNN | `PMPNN_IMAGE` | `/home/nvs/.cache/nim` | `/biology/ipd/proteinmpnn/predict` | +| Boltz2 | `BOLTZ2_IMAGE` | `/opt/nim/.cache` | `/biology/mit/boltz2/predict` | + +These paths follow the published +[RFdiffusion 2.3.0](https://docs.nvidia.com/nim/bionemo/rfdiffusion/2.3.0/quickstart-guide.html), +[ProteinMPNN 1.2.0](https://docs.nvidia.com/nim/bionemo/proteinmpnn/1.2.0/quickstart-guide.html), +and [Boltz2 1.10.0](https://docs.nvidia.com/nim/bionemo/boltz2/1.10.0/getting-started.html) +guides; check the selected release's API/cache contract when changing versions. +Hosted prediction URLs additionally include `/v1`. GPU memory depends on the +model, profile, and input length; check each support matrix and run the stages +sequentially if the models do not fit together. + +## Launch pattern + +Run the following Bash block after setting the three image variables. It checks +every digest before launching any service. Each container receives only its own +cache directory and exposes an unauthenticated API on the host's loopback address. + +```bash +( +set +x +set -euo pipefail +: "${NGC_API_KEY:?Set NGC_API_KEY}" +: "${RFD_IMAGE:?Set the RFdiffusion repository digest}" +: "${PMPNN_IMAGE:?Set the ProteinMPNN repository digest}" +: "${BOLTZ2_IMAGE:?Set the Boltz2 repository digest}" + +require_digest() { + local image_ref="$1" repository="$2" digest + digest="${image_ref#"$repository@sha256:"}" + if [[ "$image_ref" != "$repository@sha256:$digest" || ! "$digest" =~ ^[a-f0-9]{64}$ ]]; then + printf 'Expected an immutable image digest for %s\n' "$repository" >&2 + exit 1 + fi +} +require_digest "$RFD_IMAGE" nvcr.io/nim/ipd/rfdiffusion +require_digest "$PMPNN_IMAGE" nvcr.io/nim/ipd/proteinmpnn +require_digest "$BOLTZ2_IMAGE" nvcr.io/nim/mit/boltz2 + +mkdir -p "$HOME/nimcache_rfd" "$HOME/nimcache_pmpnn" "$HOME/nimcache_boltz2" +chmod 700 "$HOME/nimcache_rfd" "$HOME/nimcache_pmpnn" "$HOME/nimcache_boltz2" +docker run -d --name rfdiffusion --gpus device=0 --shm-size=4g \ + --user "$(id -u):$(id -g)" -e NGC_API_KEY \ + -v "$HOME/nimcache_rfd:/opt/nim/.cache" -p 127.0.0.1:8081:8000 "$RFD_IMAGE" +docker run -d --name proteinmpnn --gpus device=0 --shm-size=4g \ + --user "$(id -u):$(id -g)" -e NGC_API_KEY \ + -v "$HOME/nimcache_pmpnn:/home/nvs/.cache/nim" -p 127.0.0.1:8082:8000 "$PMPNN_IMAGE" +docker run -d --name boltz2 --gpus device=0 --shm-size=8g \ + --user "$(id -u):$(id -g)" -e NGC_API_KEY \ + -v "$HOME/nimcache_boltz2:/opt/nim/.cache" -p 127.0.0.1:8083:8000 "$BOLTZ2_IMAGE" +) +``` + +The three caches are owned by the invoking user and mode 700; the containers run +with that user's UID/GID. For releases requiring a fixed container user, arrange +access to these exact directories for that UID before launch. Do not widen +permissions on unrelated caches. First startup downloads weights and can take +several minutes. Check readiness with local GET requests that send no credentials +or protein data and bypass configured HTTP proxies: + +```bash +curl --noproxy '*' -fsS http://127.0.0.1:8081/v1/health/ready +curl --noproxy '*' -fsS http://127.0.0.1:8082/v1/health/ready +curl --noproxy '*' -fsS http://127.0.0.1:8083/v1/health/ready +``` + +## GPU profile selection + +If startup reports `NIMProfileIDNotFound` or no compatible profiles, inspect the +selected release's supported GPUs and available profiles. For releases exposing +`list-model-profiles`, reuse the already validated image digest: + +```bash +docker run --rm --gpus device=0 -e NGC_API_KEY "${BOLTZ2_IMAGE:?Set the validated Boltz2 digest}" list-model-profiles +``` + +Select a profile documented for that GPU and add `-e NIM_MODEL_PROFILE` to the +corresponding launch command after setting the chosen profile in the environment. +Matching compute capability alone does not establish engine compatibility. +Record the profile with the image digest. Then use the local prediction paths +above in the pipeline and omit hosted authentication headers. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/manifest.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/manifest.md new file mode 100755 index 00000000..f4e6a7cd --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/manifest.md @@ -0,0 +1,137 @@ +# Run Manifest & Directory Convention + +A campaign writes one `manifest.json` (machine state) and one `candidates.csv` +(human/spreadsheet view) under a run directory. The manifest is the backbone for +ranking, resumability, validation, and the final report. + +## Directory layout + +```text +runs/_/ +├── manifest.json # campaign state (see schema below) +├── candidates.csv # flat table, regenerated from the manifest +├── backbones/ # RFdiffusion output PDBs +├── sequences/ # ProteinMPNN mfasta files +└── complexes/ # Boltz2/OpenFold3 co-folded .cif files +``` + +## Schema (`manifest.json`) + +```json +{ + "schema_version": "1.0", + "campaign": "protein-binder-design", + "created": "2026-06-12T18:00:00+00:00", + "run_dir": "runs/_", + "target": { "name": "", "pdb_id": "", "chain": "" }, + "mode": "hosted", + "params": { "n_backbones": 100, "seqs_per_backbone": 8, "binder_len": "60-90" }, + "filters": { "iptm_min": 0.8, "binder_plddt_min": 80.0, "self_consistency_rmsd_max": 2.0 }, + "stages": [ { "stage": "rfdiffusion", "ts": "...", "n": 100 } ], + "candidates": [ + { + "id": "bb003_seq02", + "backbone_id": "bb003", + "sequence": "....", + "scores": { "proteinmpnn_nll": 1.02, "iptm": 0.84, "binder_plddt": 86.2, "self_consistency_rmsd": 1.4 }, + "artifacts": { "backbone_pdb": "backbones/bb003.pdb", "complex_cif": "complexes/bb003_seq02.cif" }, + "passed_filter": true, + "is_control": false, + "control_type": null, + "created": "..." + } + ] +} +``` + +## Usage (`scripts/manifest.py`) + +```python +import sys; sys.path.insert(0, "scripts") +from manifest import Manifest + +m = Manifest.create(run_dir="runs/demo", target={"name": "X", "chain": "A"}, mode="hosted") +m.log_stage("rfdiffusion", n=100) +m.upsert_candidate("bb003_seq02", backbone_id="bb003", sequence="MKT...") +m.set_scores("bb003_seq02", iptm=0.84, binder_plddt=86.2, self_consistency_rmsd=1.4) +m.add_artifact("bb003_seq02", "complex_cif", "complexes/bb003_seq02.cif") +m.apply_filters() +ranked = m.rank(by="iptm", passed_only=True) +m.to_csv(candidates=ranked) + +# resume a campaign later +m2 = Manifest.load("runs/demo") +``` + +`apply_filters()` requires every applicable enabled metric to be present and finite. Missing +or invalid scores cannot pass. Set a threshold to `null` only when deliberately +disabling that criterion for the campaign, and disclose the override. A composite +confidence field does not satisfy an ipTM requirement. Sequence-only controls +without `artifacts.backbone_pdb` skip only RMSD, with the reason saved under +`filter_exemptions`. `filter_metrics` records the criteria actually evaluated. + +### Boltz2-only filtering and ranking + +Use a separate `boltz2_confidence_min` filter when the co-fold response has no +explicit ipTM. Resolve its cutoff with the campaign inputs before inference; +there is no default composite-confidence threshold. The numbers below are +**synthetic bookkeeping examples**, including the illustrative 0.8 cutoff. +Replace scores with values extracted from the saved prediction, and choose the +cutoff for the real campaign using `references/validation.md` and save its source +in `params.confidence_selection`. If no cutoff is supplied, the default route is +explicit ipTM from OpenFold3; if that is unavailable, record +`selection_policy_unresolved` and stop before design inference. Keep binder +pLDDT/RMSD criteria enabled for designs. + +```python +m = Manifest.create( + run_dir="runs/boltz2_demo", + target={"name": "synthetic example"}, + params={"rank_by": "boltz2_confidence"}, + filters={ + "iptm_min": None, + "boltz2_confidence_min": 0.8, + "binder_plddt_min": 80.0, + "self_consistency_rmsd_max": 2.0, + }, +) +m.set_scores("example", boltz2_confidence=0.9, binder_plddt=86.2, self_consistency_rmsd=1.4) +m.apply_filters() +ranked = m.rank(by=m.data["params"]["rank_by"], passed_only=True) +m.to_csv(candidates=ranked) +``` + +This candidate can pass without an `iptm` value; its composite confidence remains +in `boltz2_confidence`. A missing or non-finite enabled metric still fails. +Label the metric and cutoff in the report, and apply the same profile to controls. + +`to_csv()` with no candidate list exports the full audit table in insertion +order. Pass the list returned by `rank()` to export ranked survivors, including +an empty list when none pass. `rank()` does not mutate the manifest; calling +`to_csv()` without that list afterward still writes all entries. Keep the audit +in `all_candidates.csv` and the selected list in `candidates.csv`. An empty +selected list produces only the CSV header; never replace it with all candidates. +The manifest always retains all candidates and +controls. `rank()` excludes missing, nonnumeric, boolean, and non-finite scores. + +## Resumability + +Load `Manifest.load(run_dir)` and retain its filters, mode, parameters, and ranking +metric. `m.required_metrics(cid)` lists the applicable enabled metrics; +`m.missing_scores(cid)` lists missing/non-finite values. A finite failing score is +complete and should not trigger another prediction. + +| Stage | Completed evidence | Resume action when missing | +|---|---|---| +| Backbone (designs only) | nonempty `artifacts.backbone_pdb` file | resolve saved backbone by `backbone_id`; regenerate only if genuinely absent | +| Sequence | nonempty candidate `sequence` plus saved MPNN response for designs | parse saved response before making another MPNN call | +| Co-fold | nonempty `artifacts.complex_cif` or `complex_pdb`, plus saved response | reuse a saved complex/response; otherwise co-fold this candidate | +| Metrics | `missing_scores(cid) == []` | extract from saved response/structure; run `metrics.py` for design RMSD | +| Filtering/report | current `filter_metrics`, `passed_filter`, CSV | rerun deterministic filtering and export; no inference | + +Resolve artifact paths as recorded (the examples include the run directory), and +verify files exist and are nonempty before skipping a stage. Sequence-only controls +do not require a designed backbone or its RMSD. An explicit `failure_reason` from +a failed request remains visible; retry only after the cause or retry decision is +recorded. Checkpoint every successful or failed expensive stage with the mutation +helpers. Preserve all candidates, including incomplete and failed ones. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/offline.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/offline.md new file mode 100644 index 00000000..03f45844 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/offline.md @@ -0,0 +1,106 @@ +# Offline bookkeeping and saved responses + +Use these helpers for explicitly synthetic software exercises. They never read +credentials or submit inference. Outputs establish software behavior only, with +no biological validity, scientific calibration or endpoint coverage. Real +campaign response parsing must first establish the provider's actual schema and +confidence encoding. + +## Score fixtures + +From the skill directory: + +```bash +python3 scripts/offline_bookkeeping.py /workspace/input/bookkeeping.json --output-dir /workspace/runs/bookkeeping +``` + +The input has `target`, optional `provenance`, and `profiles`. Each profile has +`name`, `rank_by`, `filters`, and `candidates` with `id`, `scores`, and control +flags. Names must be unique; a profile name is a single directory component. +One profile writes directly to `--output-dir`; multiple profiles create named +subdirectories. The destination must be empty. Existing scores, explicit nulls, +invalid strings and absent keys are preserved. Unknown filters are rejected. + +Each run contains `manifest.json`, `all_candidates.csv`, `candidates.csv`, and +`summary.json`. The manifest is saved and reloaded before filtering. The audit +retains controls and incomplete candidates. The ranked file contains only +passing non-controls and retains a header when no design passes. Summary counts +exclude controls from the design denominator and disclose missing-score IDs and +a provisional-fraction flag. A finite below-threshold score is complete. + +Check the exit status and inspect saved files before reporting their counts and +ranked IDs. A script on disk or an intended path is not an executed result. + +## Recovering cached metrics + +```bash +python3 scripts/refold_cache.py /workspace/input/resume_cache/manifest.json --output-dir /workspace/output/resumed +``` + +This copies the source run to a new destination, preserves failed stages and +existing finite scores, imports missing required metrics from supported cached +responses, and regenerates exports and the summary. The destination must not +exist or be inside the source run. Relative artifact paths resolve within the +copied run. Missing predictions remain blocked; this command never retries an +endpoint. Completed low-scoring candidates are preserved, not resubmitted. + +The fixture format extends the normal manifest with +`target.structure: {path, sha256}` and a per-candidate `prediction` binding: + +```json +{ + "provider": "boltz2", + "candidate_id": "candidate-1", + "sequence_sha256": "", + "target_sha256": "", + "response": {"path": "responses/candidate-1.json", "sha256": ""}, + "sample_index": 1, + "complex_sha256": "", + "binder_chain": "B", + "target_chain": "A", + "plddt_scale": "0-100", + "backbone": {"path": "backbone.pdb", "sha256": ""}, + "backbone_chain": "D" +} +``` + +These are local provenance fields, not fields promised by a model endpoint. +Digests establish consistency with recorded inputs; they do not authenticate a +model or prove inference occurred. Record bindings when the original request +and response are saved; do not invent them from a convenient cached file. +A changed candidate, sequence, target, response or selected sample blocks the +import. Preserve conflicting evidence and record the reason. + +- **Boltz2:** pair `structures[sample_index]` with the same index in + `confidence_scores`, retained only as `boltz2_confidence`. Require equal array + lengths. Do not select a higher-confidence sample independently of coordinates. +- **OpenFold3:** match `outputs[].input_id` to the candidate, then select + `structures_with_scores[sample_index]`. Only `iptm_score` supplies ipTM; + `confidence_score` and `complex_plddt_score` cannot replace it. +- **Binder pLDDT:** fixtures explicitly encode it in CA B factors with + `plddt_scale` of `0-1` or `0-100`. Average the binder only and normalize to + 0–100. Experimental PDB B factors need not encode pLDDT. mmCIF uses + `label_asym_id` to match the request, which can differ from author chain IDs. + Require the complete binder length and both chain roles. +- **Missing RMSD:** verify the backbone digest and align complete, explicitly + identified binder CA sets with `metrics.ca_rmsd_from_structures()`. Unequal + lengths fail; do not truncate or compare the target instead. + +An incomplete candidate receives scores only after all missing applicable +metrics and bindings validate. Finite existing values are never overwritten. +Recovered candidates record `score_provenance` (the full binding) and +`artifacts.complex` (the selected structure). `cache_status` is `complete`, +`recovered`, or `blocked`; `cache_reason` explains blocked work. Historical +`failure_reason` and stage records remain intact. A provisional passing fraction +still includes incomplete design candidates in its denominator. + +## Target preparation + +`pdb_utils.py` extracts the first model, chooses one CA per author residue, and +keeps insertion codes as distinct map keys. A blank alternate location takes +precedence; otherwise highest occupancy wins, with a lexicographic tie break. +Sequence indices follow CA order, not arithmetic offsets from the first author +number. Specify a chain when author numbering overlaps across chains. +`remap_to_seq_index()` raises for absent hotspots: record the missing IDs and +block that handoff. Verify insertion-code support for a downstream endpoint +before emitting a request containing such a hotspot. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/pipeline.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/pipeline.md new file mode 100755 index 00000000..0863505b --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/pipeline.md @@ -0,0 +1,235 @@ +# Binder Design Pipeline — Orchestration & Handoff Contracts + +This is the detailed orchestration for the `protein-binder-design` workflow. +The agent reasons over these steps and delegates each NIM call to the atomic +skill. Deterministic glue (parsing, remapping, RMSD, manifest) uses the bundled +`scripts/`. + +## 0. Setup + +- Reuse an existing hosted/local choice, or resolve it once if unspecified. +- For hosted calls, read `NGC_API_KEY` or `NVIDIA_API_KEY` from the environment + and stop if neither is set. Do not substitute the agent's `OPENAI_API_KEY` or + print environment variables or authorization headers. +- Wrap every hosted command: + `bash scripts/hosted_env.sh python3 your_nim_request.py` (use the helper's + absolute path outside the skill directory). This exports the chosen key as + `NGC_API_KEY` for RFdiffusion, ProteinMPNN, Boltz2, OpenFold3, and optional MSA + subprocesses. The wrapper selects credentials and launches the command together; + it works in a fresh tool shell and disables tracing. Local + inference uses no authentication header and does not need this helper. +- Create a run directory and manifest: + +```python +import sys; sys.path.insert(0, "scripts") +from manifest import Manifest +m = Manifest.create( + run_dir="runs/_", + target={"name": "", "pdb_id": "", "chain": ""}, + mode="hosted", + params={"n_backbones": 100, "seqs_per_backbone": 8, "binder_len": "60-90", "rank_by": "iptm"}, +) +``` + +Before inference, select the scoring route. The default is a refolder returning +explicit ipTM (OpenFold3) with the manifest's default filters. For Boltz2-only +output without ipTM, require a user-selected or separately calibrated confidence +cutoff and its rationale first (`references/validation.md`). Save that decision: + +```python +m.data["filters"].update(iptm_min=None, boltz2_confidence_min=campaign_cutoff) +m.data["params"]["rank_by"] = "boltz2_confidence" +m.data["params"]["confidence_selection"] = {"source": cutoff_source, "rationale": cutoff_rationale} +m.save() +``` + +This retains the pLDDT/RMSD criteria. The complete Boltz2 manifest example is in +`references/manifest.md`; reuse its saved filters and ranking metric on resume. +If no justified cutoff or explicit-ipTM refolder is available, stop before design +inference with `selection_policy_unresolved`. Never use the synthetic example's +cutoff as an implicit scientific default. + +### Request failures and partial batches + +Use the atomic skill's request client with an explicit per-request timeout (up +to 1200 seconds for inference), at most three retries for HTTP 429/5xx, and bounded +backoff (10, 20, 40 seconds; honor Retry-After up to 120 seconds). Count retries +against the campaign's call/time budget and stop when it is exhausted. A timed-out +generation POST may have run remotely: query its returned job ID if available; +otherwise record `outcome_unknown` and require an explicit retry decision to avoid +duplicate generation charges. Do not retry HTTP 400/401/403 unchanged. + +Save each successful response and update its candidate before the next request. +On exhausted errors, set `failure_reason`, log the failed stage and attempt count +with `m.log_stage()`, and continue independent candidates within the budget. Never +fill missing scores with zeros or drop failed candidates from the denominator. +Abort the batch on missing/invalid credentials. Reuse successful artifacts on +resume; retry failed calls only after the cause or retry decision is recorded. + +## 1. Target prep + +- Obtain the target structure (experimental PDB, or predict with `openfold2-nim` + / `openfold3-nim` / `boltz2-nim` if none exists). +- Identify epitope/hotspot residues (from literature, the registry, or the user) + in **PDB author numbering**. +- Remap to 1-based sequence indices for tools that need them: + +```python +from pdb_utils import remap_to_seq_index +target_pdb = open(".pdb").read() +seq_idx = remap_to_seq_index(target_pdb, chain="", author_resnums=[]) +``` + +- RFdiffusion `hotspot_res` instead uses chain+author strings, e.g. + `["E453", "E455", "E456", "E486"]` (no remap needed there). +- Preserve insertion codes and sparse author numbering: `501` and `501A` are + distinct residues. Missing author IDs block the handoff. Verify endpoint + support before requesting an insertion-coded hotspot. Select the target + chain, one model and one alternate CA location per residue. +- Optional: build a target MSA with `msa-search-nim` if you will fold/co-fold + the target with evolutionary context. + +### Human-in-the-loop gate +Before generating backbones, resolve any missing target chain, epitope/hotspot +set, binder length range, number of backbones, or sequences per backbone. Treat +the user's supplied choices as resolved; ask only for missing values or a +proposed change in campaign scope or deployment. + +## 2. Backbones — `rfdiffusion-nim` + +Binder design mode: pass the target `input_pdb`, a contig combining the target +segment and a generated binder segment, and `hotspot_res`. + +```python +# delegate the actual request to the rfdiffusion-nim skill +payload = { + "input_pdb": target_pdb, + "contigs": "E1-200/0 60-90", # keep target E1-200, chain break, generate 60-90 aa binder + "hotspot_res": ["E453", "E455", "E456", "E486", "E489", "E493", "E501"], + "diffusion_steps": 50, +} +# -> result["output_pdb"] is one backbone +``` + +Generate N backbones (loop with distinct seeds / repeated calls). Save each and +register it: + +```python +m.upsert_candidate("bb003", backbone_id="bb003") +m.add_artifact("bb003", "backbone_pdb", "runs/.../backbones/bb003.pdb") +``` + +## 3. Sequences — `proteinmpnn-nim` + +For each backbone, design k sequences. Redesign the **binder chain only** +(`input_pdb_chains=[binder_chain]`) so the target chain stays fixed. RFdiffusion +may renumber/rename chains, so re-read the backbone PDB to get the binder chain +ID and length first (`pdb_utils.py`). **Drop the native/WT row** from `mfasta` +and pair scores only with designed rows. + +```python +payload = { + "input_pdb": backbone_pdb, + "input_pdb_chains": [binder_chain], # redesign binder, keep target fixed + "num_seq_per_target": 8, + "sampling_temp": [0.1, 0.2], + "use_soluble_model": True, # for soluble binders +} +# parse result["mfasta"]: keep headers without 'native'/'wt'; zip with result["scores"] +m.upsert_candidate("bb003_seq02", backbone_id="bb003", sequence=designed_seq) +m.set_scores("bb003_seq02", proteinmpnn_nll=score) +``` + +## 4. Co-fold + score — `boltz2-nim` or `openfold3-nim` + +Co-fold each designed binder **with the target** as a 2-chain complex. Use the +binder sequence + target sequence (and target MSA if built). + +- `openfold3-nim` returns an explicit `iptm_score` (interface) and pLDDT. +- `boltz2-nim` returns `confidence_scores`; retain it as complex confidence. + It is not an ipTM substitute. Populate `iptm` only from an explicitly identified + interface metric; otherwise leave it missing and use the explicitly selected + `boltz2_confidence_min` filter and `boltz2_confidence` ranking route from setup. + Store the scalar confidence for the same returned sample as the saved complex. + +Record the sample index, chain roles, candidate/sequence identity, and digests of +its target, response and selected complex when saving the response. Verify those +bindings before importing missing metrics on resume. A complete low score does +not need recomputation. Derive binder pLDDT only from explicitly identified +per-residue binder confidence with its scale established; a whole-complex mean +cannot substitute. Keep it missing when the response lacks that evidence. +The synthetic cache format and recovery command are in `references/offline.md`. + +```python +# delegate to boltz2-nim / openfold3-nim +polymers = [ + {"id": "A", "molecule_type": "protein", "sequence": designed_seq}, # binder (single-seq MSA is standard for de novo) + {"id": "B", "molecule_type": "protein", "sequence": target_seq}, # target (+ MSA) +] +m.set_scores("bb003_seq02", iptm=iptm, binder_plddt=plddt, boltz2_confidence=conf) +m.add_artifact("bb003_seq02", "complex_cif", "runs/.../complexes/bb003_seq02.cif") +``` + +### Cost funnel +Co-folding is the expensive stage. Co-fold a **capped shortlist** first (e.g. +best ProteinMPNN NLL per backbone), review, then expand. Reserve high +`diffusion_samples` / `recycling_steps` for the final survivors. + +## 5. Self-consistency RMSD + +Compare the RFdiffusion backbone to the predicted binder chain (from the +co-folded complex). Low RMSD = the sequence is predicted to fold back into the +designed backbone. + +```python +from metrics import ca_rmsd_from_structures +rmsd = ca_rmsd_from_structures( + backbone_pdb, predicted_complex_cif, + backbone_chain=backbone_binder_chain, # verified from the RFdiffusion PDB + predicted_chain="A", # binder polymer ID in the request above + predicted_format="cif", +) +m.set_scores("bb003_seq02", self_consistency_rmsd=rmsd) +``` + +`pdb_utils.structure_ca_coords()` reads mmCIF with biotite using label chain IDs +(the request polymer IDs). It also accepts PDB. Missing chains or unequal CA +counts fail explicitly; do not truncate structures to force an RMSD. The CLI is +`python3 scripts/metrics.py backbone.pdb complex.cif --backbone-chain --predicted-chain `. +For sequence-only controls without a designed backbone, RMSD is not applicable; +`Manifest.apply_filters()` records the exemption and still tests confidence/pLDDT. + +## 6. Filter + rank + report + +```python +from pathlib import Path + +m.apply_filters() # uses the saved campaign criteria +rank_by = m.data["params"].get("rank_by", "iptm") +ranked = m.rank(by=rank_by, descending=True, passed_only=True, include_controls=False) +top = ranked[:20] +run_dir = Path(m.data["run_dir"]) +m.to_csv(run_dir / "all_candidates.csv") # full audit, including controls and failures +m.to_csv(candidates=ranked) # ranked survivors; controls excluded +print(m.summary()) # {n_candidates, n_passed, n_controls} +``` + +`rank()` does not modify the manifest. Always pass its returned list to the +ranked export, including when empty; `to_csv()` alone exports every entry in +insertion order. The complete audit is a separate file. All applicable enabled +filters must pass before ranking, even for a design with the highest interface +score. Disclose missing scores and a provisional passing fraction. + +Produce a short report: target + epitope, params, success rate, the top +designs with their scores and artifact paths, and how they compare to controls +(`references/validation.md`). + +## Branching summary + +- **No target structure** → predict it first (`openfold2/3-nim` or `boltz2-nim`). +- **Target needs evolutionary context** → `msa-search-nim` before co-folding. +- **Interface metric** → prefer OpenFold3 `iptm_score`; Boltz2 `confidence_scores` + uses its own explicit filter and ranking route. Label the selected metric and + cutoff in the report, and compare controls under that same route. +- **Binder MSA** → keep single-sequence for de novo binders (standard); do not + fabricate a binder MSA. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/validation.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/validation.md new file mode 100755 index 00000000..2eae3eed --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/validation.md @@ -0,0 +1,113 @@ +# Binder Design Validation + +All outputs are in-silico, so validation means computational benchmarking. +Never report only top scores — report a **success rate** and compare against +controls. + +## Metrics + +| Metric | Source | Pass guide | Meaning | +|---|---|---|---| +| Interface confidence (ipTM) | Explicit ipTM output, e.g. OpenFold3 `iptm_score` | ≥ 0.8 | predicted interface quality | +| Composite complex confidence | Boltz2 `confidence_scores` | campaign-selected `boltz2_confidence_min` | separate filter/ranking route when ipTM is unavailable | +| Binder pLDDT | OpenFold3 / Boltz2 | ≥ 80 | binder fold confidence | +| Self-consistency RMSD | `scripts/metrics.py` (Kabsch CA-RMSD) | ≤ 2.0 Å | designed backbone vs predicted | +| Sequence quality (NLL) | ProteinMPNN `scores` | lower better | sequence–backbone compatibility | + +There is no protein–protein affinity NIM, so ipTM + self-consistency RMSD are +the binder proxy (the Bennett et al. 2023 filter pattern). + +For a Boltz2-only campaign without explicit ipTM, save `iptm_min: null`, a chosen +`boltz2_confidence_min`, and `params.rank_by: boltz2_confidence` before inference. +Keep pLDDT/RMSD requirements; rank with that saved metric. This profile has its +own success rate and must be labeled as composite-confidence selection, not as +passing the default ipTM filter. `references/manifest.md` has executable usage. + +## Controls + +Run controls through the **same co-folding model and settings** so confidence +and pLDDT distributions are comparable. They do not need RFdiffusion/ProteinMPNN. + +- **Negative controls** — scrambled binder sequences (preserve composition): + +```python +import sys; sys.path.insert(0, "scripts") +from controls import make_scrambled_controls +negs = make_scrambled_controls([designed_seq], n=5, seed=42) +# co-fold each, then register with is_control=True, control_type="scrambled" +m.upsert_candidate("ctrl_neg_01", is_control=True, control_type="scrambled", sequence=negs[0]) +``` + +- **Positive controls** — published binder sequences for the same target + (`assets/targets.json` → `published_binders`), co-folded the same way and + marked `control_type="published"`. + +The bundled registry is a template and contains no published binder sequences. +Use cited user inputs or verify exact sequences against a primary paper/structure +and register the target, binder sequence, and source with `scripts/registry.py`. +If none can be sourced, record `positive_controls_unavailable`; return an +**uncalibrated computational screen** with negative controls and disclose that the +positive-control comparison was not performed. Do not invent a positive or label +the missing comparison as successful. If calibrated benchmarking was requested, +that portion remains blocked. + +Sequence-only controls have no RFdiffusion backbone. Leave their self-consistency +RMSD absent, and do not attach a parent's backbone merely to satisfy the filter. +`apply_filters()` records `filter_exemptions.self_consistency_rmsd` for these +controls while retaining confidence and pLDDT requirements. A control with a real +reference `artifacts.backbone_pdb` is subject to the RMSD filter. Compare control +confidence/pLDDT distributions separately from the design-only RMSD distribution. + +### Choosing a Boltz2 composite-confidence cutoff + +The default route uses explicit ipTM and the existing ipTM threshold. Without a +composite cutoff, prefer OpenFold3; if unavailable, stop before design inference +with `selection_policy_unresolved`. No universally validated composite-confidence +cutoff is bundled here, and the synthetic 0.8 example is not a recommendation. + +For a Boltz2-only campaign, reuse the user's supplied cutoff and record its source +and limitations, or perform a separate calibration with sourced positives and +scrambled negatives against the same target, model/version, and folding settings. +Plot/report both score distributions, choose a cutoff under a stated false-positive +tolerance, and assess it on held-out controls; insufficient controls or inseparable +distributions cannot establish a calibrated cutoff. Obtain the campaign decision +before screening the design pool. Save the cutoff, control IDs, settings, rationale, +and calibration limitations in `params.confidence_selection`; never tune the cutoff +on the design pool just to obtain passers. A user-selected cutoff without calibration +supports explicitly labeled exploratory screening only. + +## Success rate + +The metric the field actually quotes — fraction of designs passing the filter: + +```python +s = m.summary() +success_rate = s["n_passed"] / s["n_candidates"] if s["n_candidates"] else None +``` + +Controls are excluded from both counts. A candidate needs finite values for +every enabled filter to pass. Report the number still missing required scores; +until scoring finishes, label the ratio as provisional. With no candidates, +report the rate as unavailable, not zero percent. + +Use it to compare pipeline configs (diffusion steps, sampling temperature, +sequences/backbone) rather than over-interpreting any single design. + +## Published comparison + +Re-score literature winners through your exact pipeline and check your top +designs land in the same confidence regime. Compare RMSD only when the control has +a real reference backbone. Absolute scores are not comparable +across pipelines — only same-pipeline comparisons are meaningful. + +Benchmark targets come from the target registry (`assets/targets.json`) — add your own +with `scripts/registry.py`. Always confirm epitope residues against the cited structure +before use; registry entries flag illustrative residue lists. + +## Caveats + +- In-silico triage, not experimental validation. Prefer relative ranking and + distribution separation over absolute claims. +- ipTM can be optimistic; corroborate with self-consistency RMSD and ProteinMPNN + NLL before prioritizing. +- Keep all artifacts, payloads, and the manifest together for reproducibility. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements-dev.txt b/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements-dev.txt new file mode 100644 index 00000000..a2637ad3 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements-dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt +pytest==9.1.1 +pytest-cov==7.1.0 diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt b/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt new file mode 100644 index 00000000..2faa4494 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt @@ -0,0 +1,4 @@ +# Tested helper environment (Python 3.12+); NumPy/Biotite match the Tier 3 image. +numpy==2.5.1 +biotite==1.7.1 +requests==2.34.2 diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/controls.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/controls.py new file mode 100644 index 00000000..ca29f023 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/controls.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Negative controls for binder validation: composition-preserving scrambles. + +Usage: Import scramble_sequence or make_scrambled_controls; no command-line API. +Arguments: Uppercase canonical amino-acid sequences, nonnegative count n, integer seed. +Output: A shuffled sequence or list of sequences with unchanged composition. +Exit codes: Not applicable to this library; invalid inputs raise ValueError. +""" +from __future__ import annotations + +import random + +AMINO_ACIDS = frozenset("ACDEFGHIKLMNPQRSTVWY") +DEFAULT_CONTROL_COUNT = 5 +MAX_SCRAMBLE_SEED = 2 ** 31 - 1 + + +def _validate_sequence(seq): + if not isinstance(seq, str) or not seq or set(seq) - AMINO_ACIDS: + raise ValueError("sequence must be a nonempty uppercase canonical amino-acid string") + + +def _validate_seed(seed): + if not isinstance(seed, int) or isinstance(seed, bool): + raise ValueError("seed must be an integer") + + +def scramble_sequence(seq, seed=0): + """Return a shuffled sequence; a shuffle can coincide with the input.""" + _validate_sequence(seq) + _validate_seed(seed) + rng = random.Random(seed) + chars = list(seq) + rng.shuffle(chars) + return "".join(chars) + + +def make_scrambled_controls(seqs, n=DEFAULT_CONTROL_COUNT, seed=0): + """Draw controls cyclically from a sequence list; an empty list yields none.""" + if not isinstance(n, int) or isinstance(n, bool) or n < 0: + raise ValueError("n must be a nonnegative integer") + if not isinstance(seqs, (list, tuple)): + raise ValueError("seqs must be a list or tuple of amino-acid sequences") + _validate_seed(seed) + for seq in seqs: + _validate_sequence(seq) + rng = random.Random(seed) + if not seqs: + return [] + out = [] + for i in range(n): + base = seqs[i % len(seqs)] + out.append(scramble_sequence(base, seed=rng.randint(0, MAX_SCRAMBLE_SEED))) + return out diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/hosted_env.sh b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/hosted_env.sh new file mode 100644 index 00000000..8eef03ad --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/hosted_env.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +# Run a hosted command with its credential in the child's environment: +# bash scripts/hosted_env.sh python your_nim_request.py +# Sourcing remains supported for existing callers. +set +x # Never trace credential assignments, even when invoked with bash -x. +if [ -n "${NGC_API_KEY:-}" ]; then + export NGC_API_KEY +elif [ -n "${NVIDIA_API_KEY:-}" ]; then + export NGC_API_KEY="$NVIDIA_API_KEY" +else + printf '%s\n' 'Set NGC_API_KEY or NVIDIA_API_KEY for hosted NIM access.' >&2 + if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then + return 1 + fi + exit 1 +fi + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + if [[ $# -eq 0 ]]; then + printf '%s\n' 'Usage: bash hosted_env.sh COMMAND [ARG ...]' >&2 + exit 2 + fi + exec "$@" +fi diff --git a/workflows/generative-protein-binder-design/protein-binder-design/scripts/manifest.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/manifest.py similarity index 64% rename from workflows/generative-protein-binder-design/protein-binder-design/scripts/manifest.py rename to workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/manifest.py index c29c519f..2807deaa 100644 --- a/workflows/generative-protein-binder-design/protein-binder-design/scripts/manifest.py +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/manifest.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 """Run manifest for protein-binder-design campaigns. @@ -5,11 +6,18 @@ A campaign manifest is a single JSON file that records every candidate's lineage, scores, artifacts, and filter status. It is the backbone for ranking, resumability, validation, and the final report. No third-party dependencies. + +Usage: Import Manifest and call create(run_dir, target) or load(path). +Arguments: Campaign paths, target/parameter/filter dictionaries, candidate IDs and scores. +Output: Mutation methods save manifest.json; to_csv writes the selected candidate table. +Exit codes: Not applicable to this library; invalid data and I/O errors raise exceptions. """ from __future__ import annotations import csv import json +import math +import operator from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -22,11 +30,22 @@ "self_consistency_rmsd_max": 2.0, } +FILTER_METRICS = ( + ("iptm_min", "iptm", operator.ge), + ("boltz2_confidence_min", "boltz2_confidence", operator.ge), + ("binder_plddt_min", "binder_plddt", operator.ge), + ("self_consistency_rmsd_max", "self_consistency_rmsd", operator.le), +) + def _now() -> str: return datetime.now(timezone.utc).isoformat() +def _finite_score(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + class Manifest: """Read/write wrapper around a campaign ``manifest.json``.""" @@ -54,7 +73,7 @@ def create( "target": target, "mode": mode, "params": params or {}, - "filters": dict(filters) if filters else dict(DEFAULT_FILTERS), + "filters": dict(filters) if filters is not None else dict(DEFAULT_FILTERS), "stages": [], "candidates": [], } @@ -117,20 +136,45 @@ def add_artifact(self, cid: str, key: str, path: str | Path) -> dict[str, Any]: return c # ---- analysis -------------------------------------------------------- + def required_metrics(self, cid: str) -> list[str]: + """Enabled metrics applicable to a candidate, including on resume. + + Sequence-only controls have no designed backbone to compare against. + Their interface confidence and pLDDT still use the campaign thresholds. + """ + candidate = self._find(cid) + if candidate is None: + raise KeyError(cid) + sequence_control = candidate.get("is_control") and not candidate.get("artifacts", {}).get("backbone_pdb") + return [metric for threshold, metric, _ in FILTER_METRICS + if self.data["filters"].get(threshold) is not None + and not (sequence_control and metric == "self_consistency_rmsd")] + + def missing_scores(self, cid: str) -> list[str]: + """Return absent or invalid required scores; low finite scores are complete.""" + required = self.required_metrics(cid) + scores = self._find(cid).get("scores", {}) + return [metric for metric in required if not _finite_score(scores.get(metric))] + def apply_filters(self) -> None: + """Require all applicable metrics; record control-only RMSD exemptions.""" f = self.data["filters"] for c in self.data["candidates"]: s = c.get("scores", {}) checks = [] - if f.get("iptm_min") is not None and s.get("iptm") is not None: - checks.append(s["iptm"] >= f["iptm_min"]) - if f.get("binder_plddt_min") is not None and s.get("binder_plddt") is not None: - checks.append(s["binder_plddt"] >= f["binder_plddt_min"]) - if ( - f.get("self_consistency_rmsd_max") is not None - and s.get("self_consistency_rmsd") is not None - ): - checks.append(s["self_consistency_rmsd"] <= f["self_consistency_rmsd_max"]) + required = self.required_metrics(c["id"]) + c["filter_metrics"] = required + c["filter_exemptions"] = {} + for threshold, metric, compare in FILTER_METRICS: + if f.get(threshold) is None: + continue + if metric not in required: + c["filter_exemptions"][metric] = "sequence-only control: no designed backbone" + continue + score = s.get(metric) + checks.append( + _finite_score(score) and compare(score, f[threshold]) + ) c["passed_filter"] = bool(checks) and all(checks) self.save() @@ -146,17 +190,24 @@ def rank( cands = [c for c in cands if not c.get("is_control")] if passed_only: cands = [c for c in cands if c.get("passed_filter")] - cands = [c for c in cands if c.get("scores", {}).get(by) is not None] + cands = [c for c in cands if _finite_score(c.get("scores", {}).get(by))] return sorted(cands, key=lambda c: c["scores"][by], reverse=descending) - def to_csv(self, path: str | Path | None = None) -> Path: + def to_csv( + self, + path: str | Path | None = None, + *, + candidates: list[dict[str, Any]] | None = None, + ) -> Path: + """Export all candidates, or a supplied ranked list in its given order.""" path = Path(path) if path else Path(self.data["run_dir"]) / "candidates.csv" + candidates = self.data["candidates"] if candidates is None else candidates score_keys = sorted({k for c in self.data["candidates"] for k in c.get("scores", {})}) cols = ["id", "backbone_id", "is_control", "control_type", "passed_filter"] + score_keys with open(path, "w", newline="") as fh: w = csv.writer(fh) w.writerow(cols) - for c in self.data["candidates"]: + for c in candidates: row = [ c.get("id"), c.get("backbone_id"), diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/metrics.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/metrics.py new file mode 100644 index 00000000..6bc45ad4 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/metrics.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Deterministic RMSD metrics (NumPy; Biotite for mmCIF input). + +Usage: python3 metrics.py backbone.pdb prediction.cif --backbone-chain A --predicted-chain B +Arguments: Two structure paths and explicit binder chain IDs (mmCIF label IDs). +Output: CA-RMSD in angstroms on stdout; no files are written. +Exit codes: 0 success; 1 unavailable RMSD/input failure; 2 invalid CLI arguments. + Library functions raise ValueError for invalid coordinates. +""" +from __future__ import annotations + +import numpy as np + +COORDINATE_DIMENSIONS = 3 + + +def kabsch_rmsd(p, q): + """Minimal RMSD after optimal superposition of two (N, 3) coord sets.""" + p = np.asarray(p, dtype=float) + q = np.asarray(q, dtype=float) + if p.shape != q.shape or p.ndim != 2 or p.shape[1] != COORDINATE_DIMENSIONS: + raise ValueError(f"coordinate shape mismatch: {p.shape} vs {q.shape}") + if p.shape[0] == 0: + raise ValueError("no coordinates provided") + if not np.isfinite(p).all() or not np.isfinite(q).all(): + raise ValueError("non-finite coordinates cannot be aligned") + pc = p - p.mean(axis=0) + qc = q - q.mean(axis=0) + h = pc.T @ qc + u, _, vt = np.linalg.svd(h) + d = np.sign(np.linalg.det(vt.T @ u.T)) + rot = vt.T @ np.diag([1.0, 1.0, d]) @ u.T + p_rot = pc @ rot.T + return float(np.sqrt(np.sum((p_rot - qc) ** 2) / p.shape[0])) + + +def ca_rmsd_from_pdb(pdb_a, pdb_b, chain_a=None, chain_b=None): + """CA-RMSD between equal-length PDB chains; missing residues are an error.""" + import os + import sys + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from pdb_utils import ca_coords + + a = ca_coords(pdb_a, chain_a) + b = ca_coords(pdb_b, chain_b) + if not a or not b: + raise ValueError("no CA atoms found for RMSD") + return kabsch_rmsd(a, b) + + +def ca_rmsd_from_structures(backbone_pdb, predicted_text, *, backbone_chain, + predicted_chain, predicted_format="cif"): + """Compare explicit binder chains, using mmCIF label IDs for predictions. + + Both chains must contain the full binder in residue order. Do not silently + truncate missing residues or compare against the target chain. + """ + from pdb_utils import structure_ca_coords + + designed = structure_ca_coords(backbone_pdb, backbone_chain, format="pdb") + predicted = structure_ca_coords(predicted_text, predicted_chain, format=predicted_format) + return kabsch_rmsd(designed, predicted) + + +if __name__ == "__main__": + import argparse + from pathlib import Path + + parser = argparse.ArgumentParser(description="Binder CA-RMSD from a PDB backbone and PDB/mmCIF prediction.") + parser.add_argument("backbone", type=Path) + parser.add_argument("prediction", type=Path) + parser.add_argument("--backbone-chain", required=True) + parser.add_argument("--predicted-chain", required=True) + args = parser.parse_args() + try: + print(ca_rmsd_from_structures( + args.backbone.read_text(), args.prediction.read_text(), + backbone_chain=args.backbone_chain, predicted_chain=args.predicted_chain, + predicted_format=args.prediction.suffix.lstrip("."), + )) + except (ValueError, OSError, ImportError) as exc: + parser.exit(1, f"RMSD unavailable: {exc}\n") diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/offline_bookkeeping.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/offline_bookkeeping.py new file mode 100644 index 00000000..e57d4c04 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/offline_bookkeeping.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Import synthetic score fixtures and export audited, reloaded manifests offline. + +Usage: python3 offline_bookkeeping.py fixture.json --output-dir DIR +Arguments: JSON with target and profiles; a new or empty output directory. +Output: Per-profile manifest.json, all_candidates.csv, candidates.csv, summary.json; + JSON summaries on stdout. Multiple profiles use named subdirectories. +Exit codes: 0 success; 1 invalid fixture or I/O failure; 2 invalid CLI arguments. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import math +from pathlib import Path + +from manifest import FILTER_METRICS, Manifest + + +def export_run(manifest: Manifest) -> dict: + """Filter saved state and write both exports, including an empty ranked list.""" + manifest.save() + manifest = Manifest.load(manifest.path) + manifest.apply_filters() + ranked = manifest.rank(by=manifest.data["params"]["rank_by"], passed_only=True) + root = manifest.path.parent + manifest.to_csv(root / "all_candidates.csv") + manifest.to_csv(root / "candidates.csv", candidates=ranked) + pending = { + c["id"]: manifest.missing_scores(c["id"]) + for c in manifest.data["candidates"] + if manifest.missing_scores(c["id"]) + } + summary = { + "label": "simulated offline bookkeeping", + "live_inference": False, + **manifest.summary(), + "ranked_ids": [c["id"] for c in ranked], + "missing_scores": pending, + "passing_fraction_provisional": any( + c["id"] in pending + for c in manifest.data["candidates"] + if not c.get("is_control") + ), + } + (root / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + return summary + + +def run_fixture(input_path: Path, output_dir: Path) -> dict: + """One profile writes directly to output_dir; multiple profiles get subdirs. + + Refuse existing manifests: import is distinct from resuming a real campaign. + All profiles are validated before creating outputs. Scores stay uncoerced. + """ + fixture = json.loads(Path(input_path).read_text()) + profiles = fixture["profiles"] + if not profiles: + raise ValueError("fixture must contain at least one profile") + plans, names = [], set() + for profile in profiles: + name = profile["name"] + if ( + not isinstance(name, str) + or name in {"", ".", ".."} + or "/" in name + or "\\" in name + ): + raise ValueError("profile names must be single directory names") + if name in names: + raise ValueError(f"duplicate profile: {name}") + names.add(name) + ids = [c["id"] for c in profile["candidates"]] + if any(not isinstance(cid, str) or not cid for cid in ids) or len(ids) != len( + set(ids) + ): + raise ValueError(f"candidate IDs must be unique nonempty strings in {name}") + unknown = set(profile["filters"]) - {item[0] for item in FILTER_METRICS} + if unknown: + raise ValueError(f"unsupported filters: {sorted(unknown)}") + if any( + value is not None + and (type(value) not in (int, float) or not math.isfinite(value)) + for value in profile["filters"].values() + ): + raise ValueError("filter thresholds must be finite numbers or null") + if any( + type(c.get("is_control", False)) is not bool for c in profile["candidates"] + ): + raise ValueError("is_control flags must be booleans") + if not isinstance(profile["rank_by"], str) or not profile["rank_by"]: + raise ValueError("rank_by must name a score") + root = Path(output_dir) / name if len(profiles) > 1 else Path(output_dir) + if root.exists() and any(root.iterdir()): + raise FileExistsError(f"output directory is not empty: {root}") + plans.append((profile, root)) + results = {} + for profile, root in plans: + manifest = Manifest.create( + root, + copy.deepcopy(fixture["target"]), + mode="offline-bookkeeping", + params={"rank_by": profile["rank_by"]}, + filters=copy.deepcopy(profile["filters"]), + ) + manifest.data["provenance"] = fixture.get("provenance") + for candidate in profile["candidates"]: + record = manifest.upsert_candidate(candidate["id"]) + record.update( + copy.deepcopy(candidate) + ) # Preserve explicit nulls and absent score keys. + results[profile["name"]] = export_run(manifest) + return results + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input", type=Path, help="synthetic JSON fixture containing target and profiles" + ) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + try: + result = run_fixture(args.input, args.output_dir) + except (ValueError, KeyError, TypeError, OSError) as exc: + parser.exit(1, f"Offline import failed: {exc}\n") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/pdb_utils.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/pdb_utils.py new file mode 100644 index 00000000..73c4c903 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/pdb_utils.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Dependency-free PDB parsing helpers for binder-design handoffs. + +Covers the fragile glue between NIM steps: +- extract a chain, keep ATOM records +- one-letter sequence from CA atoms +- map PDB author residue numbers -> 1-based sequence index (hotspot/pocket remap) +- CA coordinates for RMSD + +Usage: Import extract_chain, sequence, remap_to_seq_index, or structure_ca_coords. +Arguments: Structure text, explicit chain ID, and author residue IDs where needed. +Output: Selected PDB text, sequence, residue indices, or CA coordinate tuples. +Exit codes: Not applicable to this library; missing/ambiguous residues or chains + raise KeyError/ValueError. mmCIF input requires Biotite. +""" +from __future__ import annotations + +THREE_TO_ONE = { + "ALA": "A", "ARG": "R", "ASN": "N", "ASP": "D", "CYS": "C", "GLN": "Q", + "GLU": "E", "GLY": "G", "HIS": "H", "ILE": "I", "LEU": "L", "LYS": "K", + "MET": "M", "PHE": "F", "PRO": "P", "SER": "S", "THR": "T", "TRP": "W", + "TYR": "Y", "VAL": "V", "MSE": "M", "SEC": "U", "PYL": "O", +} + +# Fixed-width PDB ATOM columns, expressed as zero-based offsets/slices. +ATOM_NAME = slice(12, 16) +ALTLOC = 16 +RESIDUE_NAME = slice(17, 20) +CHAIN_ID = 21 +AUTHOR_RESIDUE_ID = slice(22, 27) +AUTHOR_RESIDUE_NUMBER = slice(22, 26) +INSERTION_CODE = 26 +X_COORD = slice(30, 38) +Y_COORD = slice(38, 46) +Z_COORD = slice(46, 54) +OCCUPANCY = slice(54, 60) + + +def _first_model_lines(pdb_text): + for line in pdb_text.splitlines(): + if line.startswith("ENDMDL"): + break + yield line + + +def _iter_atom_lines(pdb_text, chain=None): + for line in _first_model_lines(pdb_text): + if not line.startswith("ATOM"): + continue + if len(line) < Z_COORD.stop: + continue + if chain is not None and line[CHAIN_ID] != chain: + continue + yield line + + +def extract_chain(pdb_text, chain): + """First model and chosen CA conformer; retain chain and author numbering.""" + keep = [] + selected = {line[AUTHOR_RESIDUE_ID]: line for line in _selected_ca_lines(pdb_text, chain)} + for line in _first_model_lines(pdb_text): + if line.startswith(("ATOM", "HETATM", "TER")) and len(line) > CHAIN_ID and line[CHAIN_ID] == chain: + choice = selected.get(line[AUTHOR_RESIDUE_ID]) + if line.startswith("ATOM") and choice: + if line[ATOM_NAME].strip() == "CA" and line != choice: + continue + if choice[ALTLOC].strip() and line[ALTLOC] not in (" ", choice[ALTLOC]): + continue + keep.append(line) + return "\n".join(keep) + + +def _selected_ca_lines(pdb_text, chain=None): + chosen = {} + for line in _iter_atom_lines(pdb_text, chain): + if line[ATOM_NAME].strip() != "CA": + continue + key = (line[CHAIN_ID], line[AUTHOR_RESIDUE_ID]) + occupancy = float(line[OCCUPANCY].strip() or 0) if len(line) >= OCCUPANCY.stop else 0 + priority = (bool(line[ALTLOC].strip()), -occupancy, line[ALTLOC]) + if key not in chosen or priority < chosen[key][0]: + chosen[key] = (priority, line) + return [line for _, line in chosen.values()] + + +def ca_residues(pdb_text, chain=None): + """First model, one CA per author residue, preserving insertion codes. + + Prefer a blank alternate location, otherwise highest occupancy, then the + lexicographically first altloc on a tie. Residue order is file order. + """ + out = [] + for line in _selected_ca_lines(pdb_text, chain): + res_name = line[RESIDUE_NAME].strip() + res_seq = int(line[AUTHOR_RESIDUE_NUMBER]) + icode = line[INSERTION_CODE].strip() + x, y, z = (float(line[field]) for field in (X_COORD, Y_COORD, Z_COORD)) + out.append((res_name, res_seq, icode, (x, y, z))) + return out + + +def sequence(pdb_text, chain=None): + """One-letter sequence from CA atoms (unknown residues -> 'X').""" + return "".join(THREE_TO_ONE.get(r[0], "X") for r in ca_residues(pdb_text, chain)) + + +def residue_index_map(pdb_text, chain=None): + """Map PDB author residue id -> 1-based sequence index (CA order). + + '501' and '501A' are distinct keys. Never alias an insertion residue to a + bare number: that silently moves a hotspot. Specify a chain when numbering + overlaps across chains. + """ + mapping = {} + for i, (_, res_seq, icode, _) in enumerate(ca_residues(pdb_text, chain), start=1): + key = f"{res_seq}{icode}" + if key in mapping: + raise ValueError(f"ambiguous author residue {key!r}; specify one chain") + mapping[key] = i + return mapping + + +def remap_to_seq_index(pdb_text, chain, author_resnums): + """Convert PDB author residue numbers to 1-based sequence indices.""" + mapping = residue_index_map(pdb_text, chain) + out, missing = [], [] + for a in author_resnums: + key = str(a) + if key in mapping: + out.append(mapping[key]) + else: + missing.append(key) + if missing: + raise KeyError(f"residues not found in chain {chain!r}: {missing}") + return out + + +def ca_coords(pdb_text, chain=None): + """List of (x, y, z) for CA atoms in chain order.""" + return [r[3] for r in ca_residues(pdb_text, chain)] + + +def structure_ca_coords(structure_text, chain, *, format="pdb"): + """Extract one chain's C-alpha coordinates from model 1 of PDB or mmCIF. + + mmCIF uses label chain IDs (the polymer IDs in the prediction request) and + requires biotite. PDB-only callers retain the dependency-free parser. + """ + if format.lower() == "pdb": + # Only the first model: concatenating models corrupts RMSD matching. + coords = ca_coords(structure_text.split("ENDMDL", 1)[0], chain) + elif format.lower() in {"cif", "mmcif"}: + from io import StringIO + from biotite.structure.io import pdbx + + atoms = pdbx.get_structure( + pdbx.CIFFile.read(StringIO(structure_text)), model=1, + use_author_fields=False, + ) + coords = atoms.coord[(atoms.chain_id == chain) & (atoms.atom_name == "CA")].tolist() + else: + raise ValueError(f"unsupported structure format: {format}") + if not coords: + raise ValueError(f"no CA atoms found for chain {chain!r}") + return coords diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/refold_cache.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/refold_cache.py new file mode 100644 index 00000000..3a0da8bb --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/refold_cache.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Reconcile a synthetic campaign with saved responses; never submit inference. + +The cache binding contract is documented in references/offline.md. This is an +offline fixture/recovery adapter, not an endpoint client or a provenance attestor. + +Usage: python3 refold_cache.py manifest.json --output-dir DIR +Arguments: A synthetic offline-bookkeeping manifest and a new directory outside its run. +Output: Copied run with verified metrics or per-candidate blocked reasons, both CSVs, + summary.json, recovered complexes, and a JSON summary on stdout. +Exit codes: 0 recovery/export completed (inspect blocked candidates); 1 invalid run + or I/O failure; 2 invalid CLI arguments. No inference requests are submitted. +""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import math +import shutil +from pathlib import Path + +from biotite import InvalidFileError + +from manifest import Manifest +from offline_bookkeeping import export_run + +PLDDT_PERCENT_MAX = 100 +ARTIFACT_ID_HEX_LENGTH = 16 + + +class CacheError(ValueError): + """A saved response cannot safely supply scores for this candidate.""" + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def local_file(root: Path, name: str) -> Path: + path = (root / name).resolve() + if not path.is_relative_to(root.resolve()): + raise CacheError("artifact_outside_run") + return path + + +def checked_bytes(root: Path, record: dict) -> bytes: + try: + value = local_file(root, record["path"]).read_bytes() + except FileNotFoundError as exc: + raise CacheError("missing_artifact") from exc + if digest(value) != record["sha256"]: + raise CacheError("artifact_digest_mismatch") + return value + + +def finite(value): + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + ) + + +def binder_plddt(text, format, chain, sequence_length, scale): + """Mean CA confidence for an explicitly selected binder chain and scale.""" + from biotite.structure.io import pdb, pdbx + + if format == "pdb": + atoms = pdb.PDBFile.read(io.StringIO(text)).get_structure( + model=1, extra_fields=["b_factor"] + ) + elif format in {"cif", "mmcif"}: + atoms = pdbx.get_structure( + pdbx.CIFFile.read(io.StringIO(text)), + model=1, + use_author_fields=False, + extra_fields=["b_factor"], + ) + else: + raise CacheError("unsupported_structure_format") + selected = atoms[(atoms.chain_id == chain) & (atoms.atom_name == "CA")] + if len(selected) != sequence_length or not len(selected): + raise CacheError("binder_length_mismatch") + if scale not in {"0-1", "0-100"}: + raise CacheError("plddt_scale_unresolved") + values = [float(v) for v in selected.b_factor] + upper = 1 if scale == "0-1" else PLDDT_PERCENT_MAX + if any(not math.isfinite(v) or not 0 <= v <= upper for v in values): + raise CacheError("invalid_plddt") + return sum(values) / len(values) * (PLDDT_PERCENT_MAX / upper) + + +def extract_scores(manifest: Manifest, candidate: dict, root: Path): + binding = candidate.get("prediction") + if not binding: + raise CacheError("missing_prediction") + target = checked_bytes(root, manifest.data["target"]["structure"]) + if binding["candidate_id"] != candidate["id"]: + raise CacheError("candidate_mismatch") + if binding["target_sha256"] != digest(target): + raise CacheError("target_mismatch") + if binding["sequence_sha256"] != digest(candidate["sequence"].encode()): + raise CacheError("sequence_mismatch") + if binding["binder_chain"] == binding["target_chain"]: + raise CacheError("chain_roles_overlap") + raw = checked_bytes(root, binding["response"]) + response = json.loads(raw) + index = binding["sample_index"] + if not isinstance(index, int) or isinstance(index, bool) or index < 0: + raise CacheError("invalid_sample_index") + scores = {} + if binding["provider"] == "boltz2": + structures = response["structures"] + confidences = response.get("confidence_scores", []) + if len(confidences) != len(structures): + raise CacheError("sample_count_mismatch") + if index >= len(structures): + raise CacheError("invalid_sample_index") + sample = structures[index] + scores["boltz2_confidence"] = confidences[index] + elif binding["provider"] == "openfold3": + outputs = [ + o for o in response["outputs"] if o.get("input_id") == candidate["id"] + ] + if len(outputs) != 1: + raise CacheError("candidate_mismatch") + samples = outputs[0]["structures_with_scores"] + if index >= len(samples): + raise CacheError("invalid_sample_index") + sample = samples[index] + if "iptm_score" in sample: + scores["iptm"] = sample["iptm_score"] + else: + raise CacheError("unsupported_provider") + if any(not finite(v) or not 0 <= v <= 1 for v in scores.values()): + raise CacheError("invalid_confidence") + text, format = sample["structure"], sample["format"] + if digest(text.encode()) != binding["complex_sha256"]: + raise CacheError("sample_digest_mismatch") + # Both chains must be present; confidence cannot establish a missing complex. + from pdb_utils import structure_ca_coords + + structure_ca_coords(text, binding["target_chain"], format=format) + scores["binder_plddt"] = binder_plddt( + text, + format, + binding["binder_chain"], + len(candidate["sequence"]), + binding["plddt_scale"], + ) + if "self_consistency_rmsd" in manifest.missing_scores(candidate["id"]): + from metrics import ca_rmsd_from_structures + + backbone = checked_bytes(root, binding["backbone"]) + scores["self_consistency_rmsd"] = ca_rmsd_from_structures( + backbone.decode(), + text, + backbone_chain=binding["backbone_chain"], + predicted_chain=binding["binder_chain"], + predicted_format=format, + ) + return scores, text, format, binding + + +def recover(input_manifest: Path, output_dir: Path) -> dict: + """Copy an interrupted fixture; preserve complete results and failed stages.""" + input_manifest, output_dir = ( + Path(input_manifest).resolve(), + Path(output_dir).resolve(), + ) + if input_manifest.name != "manifest.json": + raise ValueError("input must be named manifest.json") + if output_dir.is_relative_to(input_manifest.parent): + raise ValueError("output must be outside the source run") + if Manifest.load(input_manifest).data.get("mode") != "offline-bookkeeping": + raise ValueError( + "this helper accepts explicitly synthetic offline-bookkeeping runs only" + ) + # Copy once; refusing an existing destination prevents accidental overwrites. + shutil.copytree(input_manifest.parent, output_dir) + manifest = Manifest.load(output_dir) + manifest.data["run_dir"] = str(output_dir) + manifest.data["mode"] = "offline-bookkeeping" + manifest.data["source_manifest_sha256"] = digest(input_manifest.read_bytes()) + for candidate in manifest.data["candidates"]: + missing = manifest.missing_scores(candidate["id"]) + if not missing: + candidate["cache_status"] = "complete" + candidate.pop("cache_reason", None) + continue + try: + scores, structure, format, binding = extract_scores( + manifest, candidate, output_dir + ) + updates = {key: value for key, value in scores.items() if key in missing} + if set(missing) - updates.keys(): + raise CacheError("missing_required_scores") + # Commit metrics only after all bindings and required scores validate. + complexes = output_dir / "recovered_complexes" + complexes.mkdir(exist_ok=True) + name = digest(candidate["id"].encode())[:ARTIFACT_ID_HEX_LENGTH] + ( + ".pdb" if format == "pdb" else ".cif" + ) + path = complexes / name + path.write_text(structure) + candidate["scores"].update(updates) + candidate.setdefault("artifacts", {})["complex"] = str( + path.relative_to(output_dir) + ) + candidate["score_provenance"] = dict(binding) + candidate["cache_status"] = "recovered" + candidate.pop("cache_reason", None) + except ( + CacheError, + KeyError, + IndexError, + TypeError, + ValueError, + OSError, + AttributeError, + InvalidFileError, + ) as exc: + candidate["cache_status"] = "blocked" + candidate["cache_reason"] = str(exc) + return export_run(manifest) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + try: + result = recover(args.manifest, args.output_dir) + except (ValueError, KeyError, TypeError, OSError) as exc: + parser.exit(1, f"Cache recovery failed: {exc}\n") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/workflows/generative-protein-binder-design/protein-binder-design/scripts/registry.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/registry.py similarity index 61% rename from workflows/generative-protein-binder-design/protein-binder-design/scripts/registry.py rename to workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/registry.py index cfa644bd..f4e150c7 100644 --- a/workflows/generative-protein-binder-design/protein-binder-design/scripts/registry.py +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/scripts/registry.py @@ -1,6 +1,14 @@ +#!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 -"""Benchmark target registry loader (reads assets/targets.json).""" +"""Benchmark target registry loader (default: the empty assets/targets.json template). + +Usage: Import load_registry or get_target; no command-line API. +Arguments: Optional registry JSON path; get_target also takes a target name or alias. +Output: Parsed registry or matching target dictionary; no files are written. +Exit codes: Not applicable to this library; missing targets raise KeyError, + malformed JSON and unreadable paths raise their standard Python exceptions. +""" from __future__ import annotations import json diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_controls.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_controls.py new file mode 100644 index 00000000..9c0ea2c7 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_controls.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Check reproducibility, composition, and input boundaries for negative controls.""" + +import sys +import unittest +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from controls import make_scrambled_controls, scramble_sequence # noqa: E402 + + +class ScrambledControlTests(unittest.TestCase): + def test_controls_preserve_each_source_composition_and_repeat_by_seed(self): + sources = ["ACDEFGHIKLMNPQRSTVWY", "AACCGGTT"] + controls = make_scrambled_controls(sources, n=7, seed=42) + self.assertEqual(len(controls), 7) + self.assertEqual(controls, make_scrambled_controls(sources, n=7, seed=42)) + for index, control in enumerate(controls): + self.assertEqual(Counter(control), Counter(sources[index % 2])) + self.assertEqual(sources, ["ACDEFGHIKLMNPQRSTVWY", "AACCGGTT"]) + + def test_zero_count_and_empty_source_set_produce_no_controls(self): + self.assertEqual(make_scrambled_controls(["ACDE"], n=0), []) + self.assertEqual(make_scrambled_controls([]), []) + self.assertEqual(scramble_sequence("AAAA"), "AAAA") + + def test_invalid_sequences_cannot_be_sent_as_negative_controls(self): + for seq in ("", "acde", "AC X", "AX", "AC*", None, ["A", "C"]): + with self.subTest(seq=seq), self.assertRaises(ValueError): + scramble_sequence(seq) + with self.subTest(seq=seq), self.assertRaises(ValueError): + make_scrambled_controls(["ACDE", seq]) + + def test_invalid_count_seed_or_collection_is_rejected(self): + for count in (-1, 1.5, True, "5"): + with self.subTest(count=count), self.assertRaises(ValueError): + make_scrambled_controls(["ACDE"], n=count) + for seed in (None, True, 0.5, "42"): + with self.subTest(seed=seed), self.assertRaises(ValueError): + scramble_sequence("ACDE", seed=seed) + with self.subTest(seed=seed), self.assertRaises(ValueError): + make_scrambled_controls(["ACDE"], seed=seed) + for sources in ("ACDE", None, {"ACDE"}): + with self.subTest(sources=sources), self.assertRaises(ValueError): + make_scrambled_controls(sources) + + +if __name__ == "__main__": + unittest.main() diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_hosted_env.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_hosted_env.py new file mode 100644 index 00000000..b1f563f8 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_hosted_env.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Check that delegated processes inherit the selected hosted credential.""" +import os +import subprocess +import sys +import unittest +from pathlib import Path + + +HELPER = Path(__file__).resolve().parents[1] / "scripts" / "hosted_env.sh" + + +class HostedEnvironmentTests(unittest.TestCase): + def run_child(self, keys, expected="", legacy=False): + # Isolate from real credentials; these values are synthetic test inputs. + env = {"PATH": os.defpath, **keys} + child = [sys.executable, "-c", + "import os, sys; assert os.environ['NGC_API_KEY'] == sys.argv[1]; print('child-ran')", + expected] + command = ["bash", str(HELPER), *child] + if legacy: + command = [ + "bash", "--noprofile", "--norc", "-c", + 'source "$1" && "$2" -c "$3" "$4"', + "test-hosted-env", str(HELPER), sys.executable, + "import os, sys; assert os.environ['NGC_API_KEY'] == sys.argv[1]; print('child-ran')", + expected, + ] + return subprocess.run( + command, + env=env, shell=False, text=True, capture_output=True, + ) + + def test_selected_key_reaches_delegated_process_without_logging(self): + for keys, expected in ( + ({"NGC_API_KEY": "synthetic-ngc"}, "synthetic-ngc"), + ({"NVIDIA_API_KEY": "synthetic-fallback"}, "synthetic-fallback"), + ({"NGC_API_KEY": "", "NVIDIA_API_KEY": "synthetic-fallback"}, "synthetic-fallback"), + ({"NGC_API_KEY": "synthetic-ngc", "NVIDIA_API_KEY": "synthetic-fallback"}, "synthetic-ngc"), + ): + with self.subTest(keys=sorted(keys)): + result = self.run_child(keys, expected) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "child-ran\n") + self.assertEqual(result.stderr, "") + + def test_missing_nim_key_stops_before_delegated_call(self): + for keys in ({}, {"OPENAI_API_KEY": "synthetic-agent-key"}, {"NGC_API_KEY": "", "NVIDIA_API_KEY": ""}): + with self.subTest(keys=sorted(keys)): + result = self.run_child(keys) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + self.assertIn("Set NGC_API_KEY or NVIDIA_API_KEY", result.stderr) + self.assertNotIn("synthetic-agent-key", result.stderr) + + def test_legacy_source_still_exports_to_child(self): + result = self.run_child({"NVIDIA_API_KEY": "synthetic-fallback"}, "synthetic-fallback", legacy=True) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_wrapper_preserves_arguments_and_failure_code_without_tracing_secret(self): + result = subprocess.run( + ["bash", "-x", str(HELPER), sys.executable, "-c", + "import sys; assert sys.argv[1] == 'two words'; sys.exit(7)", "two words"], + env={"PATH": os.defpath, "NVIDIA_API_KEY": "synthetic-secret"}, + shell=False, text=True, capture_output=True, + ) + self.assertEqual(result.returncode, 7) + self.assertNotIn("synthetic-secret", result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_manifest.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_manifest.py new file mode 100644 index 00000000..7de7c759 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_manifest.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Offline bookkeeping tests; numeric fixtures are not scientific predictions.""" +import csv +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +_SPEC = importlib.util.spec_from_file_location( + "binder_manifest", Path(__file__).resolve().parents[1] / "scripts" / "manifest.py" +) +_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_MODULE) +Manifest = _MODULE.Manifest + + +class ManifestFilterTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.manifest = Manifest.create(self.temp.name, {"name": "bookkeeping fixture"}) + + def add(self, cid, **scores): + return self.manifest.set_scores(cid, **scores) + + def test_each_required_score_must_be_present(self): + full = {"iptm": 0.85, "binder_plddt": 85, "self_consistency_rmsd": 1.5} + for missing in full: + with self.subTest(missing=missing): + candidate = self.add(missing, **{k: v for k, v in full.items() if k != missing}) + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + + def test_boundaries_and_failures(self): + passing = self.add("boundary", iptm=0.8, binder_plddt=80, self_consistency_rmsd=2.0) + for cid, scores in ( + ("low_interface", dict(iptm=0.79, binder_plddt=90, self_consistency_rmsd=1.0)), + ("low_plddt", dict(iptm=0.9, binder_plddt=79, self_consistency_rmsd=1.0)), + ("high_rmsd", dict(iptm=0.9, binder_plddt=90, self_consistency_rmsd=2.01)), + ): + self.add(cid, **scores) + self.manifest.apply_filters() + self.assertTrue(passing["passed_filter"]) + self.assertEqual([c["id"] for c in self.manifest.rank(passed_only=True)], ["boundary"]) + + def test_invalid_values_cannot_pass(self): + for value in (float("inf"), float("-inf"), float("nan"), True, "0.9", None): + for metric in ("iptm", "binder_plddt", "self_consistency_rmsd"): + with self.subTest(value=value, metric=metric): + candidate = self.add("invalid", iptm=0.9, binder_plddt=90, self_consistency_rmsd=1.0) + candidate["scores"][metric] = value + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + + def test_explicitly_disabled_metric_is_optional(self): + self.manifest.data["filters"]["self_consistency_rmsd_max"] = None + candidate = self.add("disabled", iptm=0.8, binder_plddt=80) + self.manifest.apply_filters() + self.assertTrue(candidate["passed_filter"]) + + def test_no_scores_or_no_enabled_filters_cannot_pass(self): + candidate = self.add("empty") + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + self.manifest.data["filters"] = {} + self.add("empty", iptm=0.9, binder_plddt=90, self_consistency_rmsd=1.0) + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + + def test_composite_confidence_does_not_replace_iptm(self): + candidate = self.add("composite", boltz2_confidence=0.99, binder_plddt=90, self_consistency_rmsd=1.0) + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + + def test_resume_and_summary_do_not_count_incomplete_as_passed(self): + self.add("done", iptm=0.85, binder_plddt=85, self_consistency_rmsd=1.5) + self.add("pending", iptm=0.85) + self.add("control", iptm=0.85, binder_plddt=85, self_consistency_rmsd=1.5) + self.manifest.upsert_candidate("control", is_control=True, control_type="published") + self.manifest.apply_filters() + loaded = Manifest.load(self.temp.name) + self.assertEqual(loaded.summary(), {"n_candidates": 2, "n_passed": 1, "n_controls": 1}) + loaded.set_scores("pending", binder_plddt=85, self_consistency_rmsd=1.5) + loaded.apply_filters() + self.assertEqual(loaded.summary()["n_passed"], 2) + self.assertEqual(loaded.data["candidates"][0], self.manifest.data["candidates"][0]) + + def test_boltz2_profile_survives_resume_and_exports_ranked_candidates(self): + self.manifest.data["filters"] = { + "iptm_min": None, + "boltz2_confidence_min": 0.8, + "binder_plddt_min": 80, + "self_consistency_rmsd_max": 2.0, + } + self.manifest.data["params"]["rank_by"] = "boltz2_confidence" + for cid, confidence in (("boundary", 0.8), ("low", 0.79), ("best", 0.9), ("control", 0.99)): + self.add(cid, boltz2_confidence=confidence, binder_plddt=85, self_consistency_rmsd=1.0) + self.add("incomplete", boltz2_confidence=0.95) + self.manifest.upsert_candidate("control", is_control=True, control_type="scrambled") + self.manifest.apply_filters() + + loaded = Manifest.load(self.temp.name) + ranked = loaded.rank(by=loaded.data["params"]["rank_by"], passed_only=True) + self.assertEqual([c["id"] for c in ranked], ["best", "boundary"]) + self.assertEqual(loaded.summary(), {"n_candidates": 4, "n_passed": 2, "n_controls": 1}) + self.assertTrue(all("iptm" not in c["scores"] for c in loaded.data["candidates"])) + with loaded.to_csv(candidates=ranked).open() as stream: + rows = list(csv.DictReader(stream)) + self.assertEqual([row["id"] for row in rows], ["best", "boundary"]) + with loaded.to_csv(candidates=[]).open() as stream: + self.assertEqual(list(csv.DictReader(stream)), []) + + def test_boltz2_missing_or_invalid_confidence_cannot_pass_or_rank(self): + self.manifest.data["filters"] = {"boltz2_confidence_min": 0.8} + for value in (None, float("inf"), float("-inf"), float("nan"), True, "0.9"): + with self.subTest(value=value): + candidate = self.add("invalid") + candidate["scores"]["boltz2_confidence"] = value + self.manifest.apply_filters() + self.assertFalse(candidate["passed_filter"]) + self.assertEqual(self.manifest.rank(by="boltz2_confidence"), []) + + def test_create_respects_explicit_empty_filters(self): + manifest = Manifest.create(self.temp.name, {"name": "bookkeeping fixture"}, filters={}) + candidate = manifest.set_scores("no_filters", iptm=0.9, binder_plddt=90, self_consistency_rmsd=1.0) + manifest.apply_filters() + self.assertEqual(manifest.data["filters"], {}) + self.assertFalse(candidate["passed_filter"]) + + def test_sequence_controls_skip_only_inapplicable_rmsd(self): + control = self.manifest.upsert_candidate("control", is_control=True, control_type="published") + self.add("control", iptm=0.85, binder_plddt=85) + design = self.add("design", iptm=0.85, binder_plddt=85) + self.manifest.apply_filters() + self.assertTrue(control["passed_filter"]) + self.assertFalse(design["passed_filter"]) + self.assertIn("self_consistency_rmsd", control["filter_exemptions"]) + self.assertEqual(self.manifest.missing_scores("design"), ["self_consistency_rmsd"]) + self.assertEqual(self.manifest.missing_scores("control"), []) + self.add("control", iptm=0.2) + self.manifest.apply_filters() + self.assertFalse(control["passed_filter"]) + + def test_control_with_reference_backbone_requires_rmsd(self): + control = self.manifest.upsert_candidate("control", is_control=True) + self.manifest.add_artifact("control", "backbone_pdb", "reference.pdb") + self.add("control", iptm=0.85, binder_plddt=85) + self.manifest.apply_filters() + self.assertFalse(control["passed_filter"]) + self.assertEqual(control["filter_exemptions"], {}) + self.add("control", self_consistency_rmsd=1.0) + self.manifest.apply_filters() + self.assertTrue(control["passed_filter"]) + + def test_resume_distinguishes_missing_metrics_from_completed_low_scores(self): + self.add("low", iptm=0.1, binder_plddt=85, self_consistency_rmsd=1.0) + self.add("incomplete", iptm=0.9, binder_plddt=float("nan")) + loaded = Manifest.load(self.temp.name) + self.assertEqual(loaded.missing_scores("low"), []) + self.assertEqual(loaded.missing_scores("incomplete"), ["binder_plddt", "self_consistency_rmsd"]) + with self.assertRaises(KeyError): + loaded.missing_scores("unknown") + + def test_evaluation_fixtures_export_only_ranked_survivors_after_resume(self): + fixtures = Path(__file__).resolve().parents[1] / "evals" / "files" + expectations = { + "bookkeeping.json": (["best", "boundary"], 4, 2), + "boltz2_only.json": (["binder_a", "binder_b"], 4, 2), + "mixed_thresholds.json": (["binder_a", "binder_b"], 6, 2), + "no_survivors.json": ([], 4, 0), + } + for filename, (expected_ids, n_candidates, n_passed) in expectations.items(): + fixture = json.loads((fixtures / filename).read_text()) + for profile in fixture["profiles"]: + with self.subTest(fixture=filename, profile=profile["name"]): + run_dir = Path(self.temp.name) / Path(filename).stem / profile["name"] + manifest = Manifest.create( + run_dir, fixture["target"], + filters=profile["filters"], params={"rank_by": profile["rank_by"]}, + ) + for entry in profile["candidates"]: + manifest.upsert_candidate( + entry["id"], is_control=entry.get("is_control", False), + control_type=entry.get("control_type"), + ) + manifest.set_scores(entry["id"], **entry["scores"]) + + loaded = Manifest.load(run_dir) + loaded.apply_filters() + ranked = loaded.rank( + by=loaded.data["params"]["rank_by"], descending=True, + passed_only=True, include_controls=False, + ) + loaded.to_csv(run_dir / "all_candidates.csv") + loaded.to_csv(run_dir / "candidates.csv", candidates=ranked) + + with (run_dir / "candidates.csv").open() as stream: + reader = csv.DictReader(stream) + self.assertIn("id", reader.fieldnames) + rows = list(reader) + self.assertEqual([row["id"] for row in rows], expected_ids) + self.assertTrue(all(row["is_control"] == "False" for row in rows)) + self.assertTrue(all(row["passed_filter"] == "True" for row in rows)) + with (run_dir / "all_candidates.csv").open() as stream: + audit_rows = list(csv.DictReader(stream)) + self.assertEqual( + [row["id"] for row in audit_rows], + [entry["id"] for entry in profile["candidates"]], + ) + saved = Manifest.load(run_dir) + self.assertEqual(saved.summary(), { + "n_candidates": n_candidates, "n_passed": n_passed, "n_controls": 1, + }) + self.assertEqual(saved.data["filters"], profile["filters"]) + self.assertEqual(saved.data["params"]["rank_by"], profile["rank_by"]) + for entry, candidate in zip(profile["candidates"], saved.data["candidates"]): + self.assertEqual(candidate["id"], entry["id"]) + self.assertEqual(candidate["scores"], entry["scores"]) + self.assertEqual(candidate["is_control"], entry.get("is_control", False)) + + +if __name__ == "__main__": + unittest.main() diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_metrics.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_metrics.py new file mode 100644 index 00000000..c2f73081 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_metrics.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Offline geometric fixtures for explicit PDB/mmCIF binder-chain handoffs.""" +import io +import sys +import unittest +from pathlib import Path + +import biotite.structure.io.pdb as pdb +import biotite.structure.io.pdbx as pdbx + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from metrics import ca_rmsd_from_pdb, ca_rmsd_from_structures, kabsch_rmsd + + +def pdb_text(chains): + lines = [] + for chain, coords in chains: + for resid, (x, y, z) in enumerate(coords, 1): + lines.append(f"ATOM {len(lines)+1:5d} CA ALA {chain}{resid:4d} " + f"{x:8.3f}{y:8.3f}{z:8.3f}{1.0:6.2f}{80.0:6.2f} C ") + return "\n".join([*lines, "END", ""]) + + +class StructureMetricTests(unittest.TestCase): + def setUp(self): + self.coords = [(0, 0, 0), (4, 0, 0), (0, 3, 0)] + self.backbone = pdb_text([("D", self.coords)]) + rotated = [(8, 7, 0), (8, 11, 0), (5, 7, 0)] + atoms = pdb.PDBFile.read(io.StringIO(pdb_text([("T", [(0, 0, 0)]), ("Y", rotated)]))).get_structure(model=1) + cif = pdbx.CIFFile() + pdbx.set_structure(cif, atoms) + # Prediction label IDs differ from author IDs: the request named binder B. + cif.block["atom_site"]["label_asym_id"] = pdbx.CIFColumn(["A", "B", "B", "B"]) + stream = io.StringIO() + cif.write(stream) + self.prediction = stream.getvalue() + + def test_cif_uses_requested_binder_chain_and_aligns_rigid_rotation(self): + value = ca_rmsd_from_structures(self.backbone, self.prediction, + backbone_chain="D", predicted_chain="B") + self.assertAlmostEqual(value, 0.0, places=6) + + def test_missing_or_wrong_chain_does_not_produce_a_partial_rmsd(self): + for chain in ("Y", "absent", "A"): + with self.subTest(chain=chain), self.assertRaises(ValueError): + ca_rmsd_from_structures(self.backbone, self.prediction, + backbone_chain="D", predicted_chain=chain) + + def test_pdb_length_mismatch_is_not_silently_truncated(self): + with self.assertRaisesRegex(ValueError, "shape mismatch"): + ca_rmsd_from_pdb(self.backbone, pdb_text([("B", self.coords[:2])]), "D", "B") + + def test_nonfinite_coordinates_fail_before_alignment(self): + for value in (float("nan"), float("inf")): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, "non-finite"): + kabsch_rmsd(self.coords, [(value, 0, 0), *self.coords[1:]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_offline.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_offline.py new file mode 100644 index 00000000..74e804e1 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_offline.py @@ -0,0 +1,474 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 +"""Execute offline workflows, then challenge the independent artifact verifier.""" + +import copy +import importlib.util +import json +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +SKILL = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SKILL / "scripts")) +from offline_bookkeeping import run_fixture # noqa: E402 - skill scripts are not an installed package +from pdb_utils import ( # noqa: E402 - skill scripts are not an installed package + ca_coords, + extract_chain, + remap_to_seq_index, + residue_index_map, + sequence, +) +from refold_cache import digest, recover # noqa: E402 - skill scripts are not an installed package + +spec = importlib.util.spec_from_file_location( + "artifact_grader", SKILL / "evals/grader.py" +) +grader = importlib.util.module_from_spec(spec) +spec.loader.exec_module(grader) +CASES = { + case["id"]: case + for case in json.loads((SKILL / "evals/evals.json").read_text())["evals"] +} + + +def write_json(path, data): + path.write_text(json.dumps(data, indent=2) + "\n") + + +def prepare_target(workspace): + """Exercise the public PDB helpers; grading uses independently pinned results.""" + source = workspace / "input/target_prep" + request = json.loads((source / "request.json").read_text()) + text = (source / request["structure"]).read_text() + chain = request["chain"] + result = { + "chain": chain, + "sequence": sequence(text, chain), + "author_to_sequence": residue_index_map(text, chain), + "simulated": True, + "live_inference": False, + "requests": {}, + } + for item in request["requests"]: + missing = [r for r in item["hotspots"] if r not in result["author_to_sequence"]] + if missing: + result["requests"][item["id"]] = { + "status": "blocked", + "missing_author_residues": missing, + } + else: + result["requests"][item["id"]] = { + "status": "ready", + "hotspot_res": [chain + r for r in item["hotspots"]], + "sequence_indices": remap_to_seq_index(text, chain, item["hotspots"]), + } + output = workspace / "output/target-prep" + output.mkdir(parents=True) + (output / "target.pdb").write_text(extract_chain(text, chain)) + write_json(output / "preparation.json", result) + + +class OfflineWorkflowTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.workspace = Path(self.temp.name) + + def stage(self, case): + for item in CASES[case]["files"]: + source = SKILL / item + destination = ( + self.workspace / "input" / source.relative_to(SKILL / "evals/files") + ) + destination.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, destination) + else: + shutil.copy2(source, destination) + + def execute(self, case): + if case in grader.BOOKKEEPING: + filename, destination, _ = grader.BOOKKEEPING[case] + output = self.workspace / "output" / (destination or "bookkeeping") + run_fixture(self.workspace / "input" / filename, output) + elif case in grader.CACHE_CASES: + for name in grader.CACHE_CASES[case]: + destination = grader.CACHE_RUNS[name][0] + recover( + self.workspace / "input" / name / "manifest.json", + self.workspace / "output" / destination, + ) + else: + prepare_target(self.workspace) + + def assert_grade(self, case, expected): + result = grader.grade(case, self.workspace) + failures = [ + c["message"] + for c in result["details"]["artifact_correctness"]["checks"] + if not c["passed"] + ] + self.assertEqual( + result["custom_metrics"]["artifact_correctness"], + expected, + "\n".join(failures), + ) + + def test_all_cases_require_artifacts_and_pass_after_real_execution(self): + for case in CASES: + with self.subTest(case=case), tempfile.TemporaryDirectory() as root: + self.workspace = Path(root) + self.stage(case) + before = { + p: digest(p.read_bytes()) + for p in (self.workspace / "input").rglob("*") + if p.is_file() + } + self.assert_grade(case, 0) + self.execute(case) + self.assert_grade(case, 1) + self.assertEqual(before, {p: digest(p.read_bytes()) for p in before}) + + def test_original_scores_are_not_coerced_and_absent_keys_stay_absent(self): + case = "offline-mixed-threshold-ranking" + self.stage(case) + self.execute(case) + data = json.loads( + (self.workspace / "output/mixed-thresholds/manifest.json").read_text() + ) + rows = {c["id"]: c for c in data["candidates"]} + self.assertEqual(rows["invalid_score"]["scores"]["iptm"], "0.95") + self.assertNotIn("self_consistency_rmsd", rows["pending"]["scores"]) + self.assertFalse(rows["invalid_score"]["passed_filter"]) + + def test_bookkeeping_verifier_accepts_equivalent_minimal_manifest(self): + case = "offline-boltz2-ranking" + self.stage(case) + self.execute(case) + path = self.workspace / "output/boltz2-only/manifest.json" + data = json.loads(path.read_text()) + # A baseline need not reproduce helper-specific timestamps or schema extras. + minimal = {key: data[key] for key in ("target", "filters", "candidates")} + minimal["rank_by"] = data["params"]["rank_by"] + for candidate in minimal["candidates"]: + for key in ( + "created", + "backbone_id", + "filter_metrics", + "filter_exemptions", + "passed_filter", + ): + candidate.pop(key, None) + write_json(path, minimal) + self.assert_grade(case, 1) + + def test_target_verifier_accepts_equivalent_pdb_writers(self): + case = "offline-target-preparation" + self.stage(case) + self.execute(case) + path = self.workspace / "output/target-prep/target.pdb" + atoms = [ + line for line in path.read_text().splitlines() if line.startswith("ATOM") + ] + rewritten = [] + for serial, line in enumerate(atoms, 101): + # Change serials, atom-name alignment and numeric formatting; clear + # altLoc after choosing B, and omit unused trailing PDB fields. + xyz = "".join( + f"{float(line[start : start + 8]):8.2f}" for start in (30, 38, 46) + ) + rewritten.append(f"ATOM {serial:5d} CA {line[17:30]}{xyz}") + for model_records in (False, True): + with self.subTest(model_records=model_records): + text = "\n".join(rewritten) + "\n" + if model_records: + text = "MODEL 1\n" + text + "ENDMDL\n" + path.write_text(text + "END\n") + self.assert_grade(case, 1) + + def test_target_verifier_rejects_duplicate_conformers_and_wrong_structure(self): + case = "offline-target-preparation" + self.stage(case) + self.execute(case) + path = self.workspace / "output/target-prep/target.pdb" + atoms = [ + line for line in path.read_text().splitlines() if line.startswith("ATOM") + ] + source = ( + (self.workspace / "input/target_prep/target.pdb").read_text().splitlines() + ) + alternative = next( + line for line in source if line.startswith("ATOM") and line[16] == "A" + ) + changed_serial = atoms[1][:6] + " 999" + atoms[1][11:] + mutations = { + "both conformers": atoms[:2] + [alternative] + atoms[2:], + "same atom, new serial": atoms + [changed_serial], + "wrong conformer": atoms[:1] + [alternative] + atoms[2:], + "missing atom": atoms[:-1], + "wrong chain": [atoms[0][:21] + "A" + atoms[0][22:]] + atoms[1:], + "lost insertion code": atoms[:2] + + [atoms[2][:26] + " " + atoms[2][27:]] + + atoms[3:], + "wrong residue": [atoms[0][:17] + "VAL" + atoms[0][20:]] + atoms[1:], + "wrong coordinates": [atoms[0][:30] + " 1.000" + atoms[0][38:]] + + atoms[1:], + "non-finite coordinates": [atoms[0][:30] + " nan" + atoms[0][38:]] + + atoms[1:], + "wrong order": list(reversed(atoms)), + "second model": ["MODEL 2"] + atoms + ["ENDMDL"], + "two models": ["MODEL 1"] + + atoms + + ["ENDMDL", "MODEL 2"] + + atoms + + ["ENDMDL"], + "outside model": ["MODEL 1"] + atoms + ["ENDMDL", atoms[0]], + } + for name, lines in mutations.items(): + with self.subTest(name=name): + path.write_text("\n".join(lines) + "\nEND\n") + self.assert_grade(case, 0) + + def test_csv_mutations_are_rejected_even_with_correct_manifest_and_summary(self): + case = "offline-boltz2-ranking" + self.stage(case) + self.execute(case) + path = self.workspace / "output/boltz2-only/candidates.csv" + original = path.read_text() + lines = original.splitlines(keepends=True) + for text in ( + lines[0], + lines[0] + "".join(reversed(lines[1:])), + original.replace("0.89", "0.99"), + original + lines[1], + ): + with self.subTest(text=text): + path.write_text(text) + self.assert_grade(case, 0) + path.write_text(original) + self.assert_grade(case, 1) + + def test_empty_rank_must_be_a_header_only_file(self): + case = "offline-empty-ranking" + self.stage(case) + self.execute(case) + root = self.workspace / "output/no-survivors" + shutil.copy2(root / "all_candidates.csv", root / "candidates.csv") + self.assert_grade(case, 0) + + def test_agent_cannot_redefine_fixture_truth(self): + case = "offline-manifest" + self.stage(case) + self.execute(case) + path = self.workspace / "input/bookkeeping.json" + path.write_text(path.read_text().replace('"iptm_min": 0.8', '"iptm_min": 0.1')) + self.assert_grade(case, 0) + + def test_supplied_score_preservation_is_exact(self): + case = "offline-boltz2-ranking" + self.stage(case) + self.execute(case) + path = self.workspace / "output/boltz2-only/manifest.json" + data = json.loads(path.read_text()) + data["candidates"][0]["scores"]["boltz2_confidence"] += 1e-9 + write_json(path, data) + self.assert_grade(case, 0) + + def test_resume_preserves_finite_low_scores_and_only_fills_missing_metrics(self): + case = "offline-cache-resume" + self.stage(case) + self.execute(case) + data = json.loads((self.workspace / "output/resumed/manifest.json").read_text()) + rows = {c["id"]: c for c in data["candidates"]} + self.assertEqual(rows["below"]["cache_status"], "complete") + self.assertEqual(rows["recoverable"]["scores"]["iptm"], 0.86) + self.assertAlmostEqual( + rows["recoverable"]["scores"]["binder_plddt"], 84, places=5 + ) + self.assertAlmostEqual( + rows["recoverable"]["scores"]["self_consistency_rmsd"], 0, places=6 + ) + self.assertEqual( + rows["failed"]["failure_reason"], "HTTP 503: retries exhausted" + ) + self.assertEqual(rows["missing"]["scores"], {}) + + def test_recovered_scores_and_provenance_are_checked_independently(self): + case = "offline-cache-resume" + self.stage(case) + self.execute(case) + path = self.workspace / "output/resumed/manifest.json" + original = json.loads(path.read_text()) + for kind in ("old_score", "sample", "failure", "reason", "decision"): + data = copy.deepcopy(original) + rows = {c["id"]: c for c in data["candidates"]} + if kind == "old_score": + rows["recoverable"]["scores"]["iptm"] = 0.98 + elif kind == "sample": + rows["recoverable"]["score_provenance"]["sample_index"] = 1 + elif kind == "failure": + rows["failed"].pop("failure_reason") + elif kind == "reason": + rows["missing"]["cache_reason"] = "unknown" + else: + rows["below"]["cache_status"] = "blocked" + write_json(path, data) + with self.subTest(kind=kind): + self.assert_grade(case, 0) + + def test_saved_selected_complex_bytes_are_checked(self): + case = "offline-refold-extraction" + self.stage(case) + self.execute(case) + root = self.workspace / "output/raw-refolds/boltz2" + data = json.loads((root / "manifest.json").read_text()) + row = next(c for c in data["candidates"] if c["id"] == "selected") + path = root / row["artifacts"]["complex"] + path.write_text(path.read_text() + "# changed\n") + self.assert_grade(case, 0) + + def test_all_provenance_mismatches_block_import(self): + case = "offline-provenance-rejection" + self.stage(case) + self.execute(case) + data = json.loads( + (self.workspace / "output/provenance/manifest.json").read_text() + ) + for candidate in data["candidates"]: + if candidate["id"] != "valid": + self.assertEqual(candidate["cache_status"], "blocked") + self.assertEqual(candidate["scores"], {}) + self.assertNotIn("score_provenance", candidate) + + def test_invalid_sample_index_is_blocked_without_importing_partial_scores(self): + case = "offline-refold-extraction" + self.stage(case) + path = self.workspace / "input/raw_refolds/boltz2/manifest.json" + original = json.loads(path.read_text()) + for index in (-1, True, 10): + data = copy.deepcopy(original) + data["candidates"][0]["prediction"]["sample_index"] = index + write_json(path, data) + output = self.workspace / str(index) + recover(path, output) + result = json.loads((output / "manifest.json").read_text())["candidates"][0] + self.assertEqual(result["cache_reason"], "invalid_sample_index") + self.assertEqual(result["scores"], {}) + + def test_existing_output_is_not_overwritten(self): + case = "offline-cache-resume" + self.stage(case) + self.execute(case) + with self.assertRaises(FileExistsError): + self.execute(case) + output = self.workspace / "output/resumed/manifest.json" + self.assertTrue(output.is_file()) + + def test_bookkeeping_validates_all_profiles_before_writing(self): + source = self.workspace / "fixture.json" + fixture = json.loads((SKILL / "evals/files/bookkeeping.json").read_text()) + fixture["profiles"][1]["filters"]["binder_plddt_min"] = "80" + write_json(source, fixture) + output = self.workspace / "output" + with self.assertRaisesRegex(ValueError, "finite numbers"): + run_fixture(source, output) + self.assertFalse(output.exists()) + + def test_real_campaign_is_not_relabelled_as_synthetic(self): + case = "offline-cache-resume" + self.stage(case) + path = self.workspace / "input/resume_cache/manifest.json" + data = json.loads(path.read_text()) + data["mode"] = "hosted" + write_json(path, data) + with self.assertRaisesRegex(ValueError, "synthetic"): + self.execute(case) + self.assertFalse((self.workspace / "output/resumed").exists()) + + def test_malformed_structure_blocks_one_candidate_and_continues(self): + case = "offline-refold-extraction" + self.stage(case) + root = self.workspace / "input/raw_refolds/openfold3" + manifest = json.loads((root / "manifest.json").read_text()) + candidate = manifest["candidates"][0] + path = root / candidate["prediction"]["response"]["path"] + data = json.loads(path.read_text()) + data["outputs"][0]["structures_with_scores"][1]["structure"] = "not a PDB" + write_json(path, data) + candidate["prediction"]["response"]["sha256"] = digest(path.read_bytes()) + candidate["prediction"]["complex_sha256"] = digest(b"not a PDB") + write_json(root / "manifest.json", manifest) + recover(root / "manifest.json", self.workspace / "output/broken") + result = json.loads( + (self.workspace / "output/broken/manifest.json").read_text() + ) + self.assertEqual(result["candidates"][0]["cache_status"], "blocked") + self.assertEqual(result["candidates"][0]["scores"], {}) + self.assertEqual(result["candidates"][2]["cache_status"], "recovered") + + def test_standalone_grader_runs_with_only_grader_and_entry_files(self): + case = "offline-boltz2-ranking" + self.stage(case) + self.execute(case) + tests, verifier = self.workspace / "tests", self.workspace / "verifier" + tests.mkdir() + shutil.copy2(SKILL / "evals/grader.py", tests / "grader.py") + write_json(tests / "entry.json", {"id": case}) + parent_reward = self.workspace / "parent-reward.json" + # A parent-only override must not redirect the standalone verifier's + # output. The child needs only these three explicit fixture paths. + with mock.patch.dict("os.environ", {"HARBOR_REWARD_JSON": str(parent_reward)}): + env = { + "HARBOR_TESTS_DIR": str(tests), + "HARBOR_VERIFIER_DIR": str(verifier), + "BINDER_EVAL_WORKSPACE": str(self.workspace), + } + subprocess.run( + [sys.executable, "-I", str(tests / "grader.py")], + env=env, shell=False, check=True, + ) + self.assertFalse(parent_reward.exists()) + self.assertEqual( + json.loads((verifier / "reward.json").read_text())["custom_metrics"], + {"artifact_correctness": 1.0}, + ) + + +class TargetMappingTests(unittest.TestCase): + def setUp(self): + self.text = (SKILL / "evals/files/target_prep/target.pdb").read_text() + + def test_insertion_residue_does_not_overwrite_bare_author_number(self): + self.assertEqual( + residue_index_map(self.text, "E"), {"10": 1, "42": 2, "42A": 3, "77": 4} + ) + self.assertEqual(remap_to_seq_index(self.text, "E", [42, "42A", 77]), [2, 3, 4]) + with self.assertRaises(KeyError): + remap_to_seq_index(self.text, "E", [43]) + + def test_first_model_and_highest_occupancy_altloc_only(self): + self.assertEqual(sequence(self.text, "E"), "AGST") + self.assertEqual( + ca_coords(self.text, "E"), [(0, 0, 0), (4, 0, 0), (4, 3, 0), (8, 3, 0)] + ) + self.assertNotIn("VAL", extract_chain(self.text, "E")) + saved_ca = [ + line + for line in extract_chain(self.text, "E").splitlines() + if line[12:16].strip() == "CA" + ] + self.assertEqual(len(saved_ca), 4) + self.assertEqual(saved_ca[1][16], "B") + + def test_overlapping_author_numbers_require_explicit_chain(self): + with self.assertRaisesRegex(ValueError, "ambiguous"): + residue_index_map(self.text) + + +if __name__ == "__main__": + unittest.main() diff --git a/workflows/generative-protein-binder-design/README.md b/workflows/generative-protein-binder-design/README.md index 4c4dd9ca..d4852758 100644 --- a/workflows/generative-protein-binder-design/README.md +++ b/workflows/generative-protein-binder-design/README.md @@ -7,7 +7,7 @@ and ranks by interface metrics (ipTM / ipSAE / pLDDT / apo↔holo RMSD): - **[`complexa-binder-design/`](complexa-binder-design/SKILL.md)** — NVIDIA **Proteina-Complexa** reward-guided co-design (binder sequence + structure together) with an AF2-reward gate, validated independently with Boltz2 / OpenFold3. -- **[`protein-binder-design/`](protein-binder-design/SKILL.md)** — **RFdiffusion + +- **[`protein-binder-design/`](../../workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md)** — **RFdiffusion + ProteinMPNN + Boltz2 / OpenFold3** BioNeMo NIM orchestration (backbones → sequences → co-fold → filter). diff --git a/workflows/generative-protein-binder-design/protein-binder-design/SKILL.md b/workflows/generative-protein-binder-design/protein-binder-design/SKILL.md deleted file mode 100755 index 5cf456d7..00000000 --- a/workflows/generative-protein-binder-design/protein-binder-design/SKILL.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -name: protein-binder-design -description: > - Orchestrate an end-to-end de novo protein binder design campaign against a protein target by composing BioNeMo NIM skills. Use for binder design, minibinder design, de novo binders, RFdiffusion + ProteinMPNN + Boltz2/OpenFold3 pipelines, epitope/hotspot-targeted design, in-silico binder validation, and ranking designs by interface confidence. -license: Apache-2.0 -compatibility: "numpy>=1.24; requests>=2.28" -allowed-tools: Bash, Read, Write, AskUserQuestion ---- - -# Protein Binder Design (workflow) - -Run a de novo binder design campaign by composing atomic NIM skills. This skill -owns orchestration, handoff contracts, filtering, validation, and the run -manifest. It does NOT duplicate per-NIM API details — defer those to each -atomic skill's `SKILL.md`. - -## Composed skills - -| Step | Skill | Owns | -|---|---|---| -| Backbones | `rfdiffusion-nim` | binder backbone PDBs (contigs + hotspots) | -| Sequences | `proteinmpnn-nim` | sequences for each backbone | -| Co-fold / score | `boltz2-nim` or `openfold3-nim` | binder–target complex + confidence / ipTM | -| MSA (optional) | `msa-search-nim` | target A3M for higher-quality folding | - -The atomic NIM skills are recommended companions (one per NIM, from the BioNeMo -NIM skill set). They are **not required**: `references/pipeline.md` carries the -concrete request shape for every NIM call, so an agent with NIM access can follow -this skill standalone. For endpoints/auth see **Configuration** below. - -## Pipeline - -1. **Target prep** — get the target PDB + epitope; map epitope/hotspot author - residue numbers to RFdiffusion `hotspot_res` strings; optionally build a - target MSA with `msa-search-nim`. -2. **Backbones** (`rfdiffusion-nim`) — binder contig + `hotspot_res`; N backbones. -3. **Sequences** (`proteinmpnn-nim`) — k sequences per backbone; drop the - native/WT row from `mfasta`. -4. **Co-fold + score** (`boltz2-nim` / `openfold3-nim`) — co-fold binder+target; - collect interface confidence (ipTM) and binder pLDDT. -5. **Self-consistency** — CA-RMSD between the RFdiffusion backbone and the - predicted binder (`scripts/metrics.py`). -6. **Filter + rank** — apply thresholds; rank survivors; write manifest + CSV. - -Full handoff contracts, branching, and the cost funnel: `references/pipeline.md`. - -## Handoff contracts (the fragile glue) - -- RFdiffusion `output_pdb` → ProteinMPNN `input_pdb` (inline PDB text). -- ProteinMPNN `mfasta` → Boltz2 binder polymer `sequence` (exclude the - native/WT row; pair scores only with designed rows). -- Epitope author residue numbers → 1-based sequence indices: remap with - `scripts/pdb_utils.py:remap_to_seq_index`. RFdiffusion `hotspot_res` uses - chain+author strings like `"A50"`; Boltz2 pocket/contacts use 1-based indices. -- Boltz2 complex `.cif` → binder chain → self-consistency RMSD vs the backbone. - -## Run manifest (reproducibility backbone) - -Every campaign writes `manifest.json` (+ `candidates.csv`) under a run dir via -`scripts/manifest.py`. It records lineage, params, scores, artifacts, filter -status, and controls — enabling ranking, resumability, validation, and the -final report. Schema and usage: `references/manifest.md`. - -## Filters (defaults) - -- ipTM ≥ 0.8, binder pLDDT ≥ 80, self-consistency RMSD ≤ 2.0 Å. -- Override per campaign and record overrides in the manifest `filters`. - -## Validation - -Always run controls and report a **success rate**, not just top scores. -Negative controls via `scripts/controls.py` (scrambled sequences); positive -controls = published binders re-scored through the same pipeline. Benchmark -targets live in `assets/targets.json` (`scripts/registry.py`). Methodology and -metric definitions: `references/validation.md`. - -## Human-in-the-loop + cost - -- Confirm target, epitope/hotspots, binder length range, and hosted-vs-local - with the user before generating backbones (AskUserQuestion). -- Co-folding is the expensive stage: co-fold a capped shortlist, review, then - expand. State hosted vs local once and reuse it across all NIM calls. - -## Responsible use - -De novo binder design is dual-use. Decline requests aimed at enhancing pathogen -fitness, toxin potency, or bioweapon function; keep designs to legitimate -research and therapeutic intent. - -## Configuration (NIM access) - -Each composed NIM is reached over HTTP; choose **hosted** or **local** once and -reuse it for every call: - -- **Hosted** (managed): base URL `https://health.api.nvidia.com/v1/...` per NIM at - [build.nvidia.com](https://build.nvidia.com); set `NVIDIA_API_KEY` (sent as - `Authorization: Bearer`). Read keys from the env — never hardcode them. -- **Local** (self-hosted NGC containers): point each NIM at its local URL - (e.g. `http://localhost:8000/...`); local NIMs need no auth header. To **launch** the - NIMs yourself (docker run per NIM, persistent caches, health checks, and the GPU - **profile‑selection gotcha** — some NIMs (e.g. Boltz2) need `NIM_MODEL_PROFILE` pinned - on GPUs that have no bundled profile, while others (RFdiffusion/ProteinMPNN) auto‑select - by compute capability): see **`references/local-nim-setup.md`**. - -Per-NIM paths, request/response schemas, and worked `curl`/Python examples live in -`references/pipeline.md`. - -## Scripts - -- `scripts/manifest.py` — campaign manifest (create / load / score / filter / rank / CSV). -- `scripts/pdb_utils.py` — PDB parse, chain extract, sequence, residue remap, CA coords. -- `scripts/metrics.py` — Kabsch CA-RMSD for self-consistency. -- `scripts/controls.py` — scrambled negative controls. -- `scripts/registry.py` + `assets/targets.json` — **example** benchmark target - registry (illustrative epitopes — verify against the cited structure before a - real campaign). Replace with your own targets. diff --git a/workflows/generative-protein-binder-design/protein-binder-design/evals/evals.json b/workflows/generative-protein-binder-design/protein-binder-design/evals/evals.json deleted file mode 100644 index 1dadc67b..00000000 --- a/workflows/generative-protein-binder-design/protein-binder-design/evals/evals.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "skill_name": "protein-binder-design", - "evals": [ - { - "id": "1", - "prompt": "Design 50 de novo mini-binders against PD-L1 (use the target registry entry) at its defined interface epitope. Generate backbones, design 8 sequences each, co-fold the shortlist against the target, and rank by interface confidence. Save a run manifest and a ranked CSV.", - "expected_output": "An orchestration that composes the rfdiffusion-nim, proteinmpnn-nim, and boltz2-nim/openfold3-nim skills in order, remaps epitope residues, writes a run manifest via scripts/manifest.py, applies the ipTM/pLDDT/RMSD filter, and exports a ranked candidates.csv.", - "files": [], - "assertions": [ - {"id": "composes-rfdiffusion", "description": "Uses RFdiffusion for binder backbones", "check": "Plan invokes the rfdiffusion-nim skill with a binder contig and hotspot_res"}, - {"id": "composes-proteinmpnn", "description": "Uses ProteinMPNN for sequence design", "check": "Plan invokes the proteinmpnn-nim skill and drops the native/WT row from mfasta"}, - {"id": "composes-cofold", "description": "Co-folds binder with target", "check": "Plan invokes boltz2-nim or openfold3-nim to co-fold the binder with the target and collects ipTM or confidence"}, - {"id": "writes-manifest", "description": "Writes a run manifest", "check": "Plan uses scripts/manifest.py (Manifest.create, set_scores, apply_filters, to_csv)"}, - {"id": "applies-filter", "description": "Applies default filter thresholds", "check": "Plan filters on ipTM >= 0.8, binder pLDDT >= 80, self-consistency RMSD <= 2.0"}, - {"id": "remaps-epitope", "description": "Remaps epitope residues", "check": "Plan remaps epitope author numbering using scripts/pdb_utils.py before use"} - ] - }, - { - "id": "2", - "prompt": "For my binder campaign against PD-L1, include proper controls and tell me the success rate.", - "expected_output": "Adds scrambled-sequence negative controls and published-binder positive controls through the same co-folding pipeline, marks them is_control in the manifest, and reports success rate as n_passed / n_candidates.", - "files": [], - "assertions": [ - {"id": "negative-controls", "description": "Generates scrambled negative controls", "check": "Plan uses scripts/controls.py to scramble binder sequences and marks them is_control with control_type scrambled"}, - {"id": "positive-controls", "description": "Uses published binders as positive controls", "check": "Plan re-scores published binders from the registry through the same pipeline as positive controls"}, - {"id": "success-rate", "description": "Reports a success rate", "check": "Plan reports success rate as n_passed divided by n_candidates from manifest.summary()"}, - {"id": "same-pipeline", "description": "Controls run through identical pipeline", "check": "Controls are co-folded and scored with the same settings as the designs"} - ] - }, - { - "id": "3", - "prompt": "Continue my interrupted binder run in runs/rbd_binders and only process candidates that are missing scores.", - "expected_output": "Loads the existing manifest with Manifest.load, identifies candidates lacking the needed scores/artifacts, and resumes without re-running completed stages.", - "files": [], - "assertions": [ - {"id": "loads-manifest", "description": "Loads the existing manifest", "check": "Plan calls Manifest.load on the run directory"}, - {"id": "resumes-incomplete", "description": "Skips completed candidates", "check": "Plan only processes candidates missing required scores or artifacts"}, - {"id": "no-recompute", "description": "Does not recompute finished stages", "check": "Plan avoids re-running stages already recorded in the manifest"} - ] - } - ] -} diff --git a/workflows/generative-protein-binder-design/protein-binder-design/references/local-nim-setup.md b/workflows/generative-protein-binder-design/protein-binder-design/references/local-nim-setup.md deleted file mode 100644 index 434fec95..00000000 --- a/workflows/generative-protein-binder-design/protein-binder-design/references/local-nim-setup.md +++ /dev/null @@ -1,80 +0,0 @@ -# Running the BioNeMo NIMs locally (self-hosted) - -Use this when you want to self-host RFdiffusion, ProteinMPNN, and a co-folder (Boltz2) -instead of the managed `build.nvidia.com` endpoints — e.g. to avoid rate limits or to -keep a campaign self-contained on one node. The pipeline logic is unchanged; only the -base URL changes (no `Authorization` header for local NIMs). - -## Prerequisites - -- Docker with the NVIDIA container runtime (`docker run --gpus ...` works). -- An NGC API key in `NGC_API_KEY` (used to pull images and download model weights). -- `docker login nvcr.io -u '$oauthtoken' -p "$NGC_API_KEY"` once. - -## Images - -| NIM | Image | Idle GPU | Serves | -|---|---|---|---| -| RFdiffusion | `nvcr.io/nim/ipd/rfdiffusion:latest` | ~24 GB | `:8000/v1/biology/ipd/rfdiffusion/generate` | -| ProteinMPNN | `nvcr.io/nim/ipd/proteinmpnn:latest` | ~1.5 GB | `:8000/v1/biology/ipd/proteinmpnn/predict` | -| Boltz2 | `nvcr.io/nim/mit/boltz2:latest` | ~8 GB | `:8000/biology/mit/boltz2/predict` | - -All three co-fit on one ≥48 GB GPU (~33 GB idle together). - -## Launch pattern - -Give each NIM its own persistent cache (so weights download once), a name, and a port. -Mount the cache at `/opt/nim/.cache` and make it writable: - -```bash -mkdir -p ~/nimcache_rfd ~/nimcache_pmpnn ~/nimcache_boltz2 && chmod 700 ~/nimcache_* -docker run -d --name rfdiffusion --gpus device=0 --shm-size=4g \ - -e NGC_API_KEY -v ~/nimcache_rfd:/opt/nim/.cache -p 8081:8000 \ - nvcr.io/nim/ipd/rfdiffusion:latest -docker run -d --name proteinmpnn --gpus device=0 --shm-size=4g \ - -e NGC_API_KEY -v ~/nimcache_pmpnn:/opt/nim/.cache -p 8082:8000 \ - nvcr.io/nim/ipd/proteinmpnn:latest -docker run -d --name boltz2 --gpus device=0 --shm-size=8g \ - -e NGC_API_KEY -v ~/nimcache_boltz2:/opt/nim/.cache -p 8083:8000 \ - nvcr.io/nim/mit/boltz2:latest -``` - -Wait for readiness (first start downloads weights — minutes): - -```bash -curl -fsS http://localhost:8081/v1/health/ready && echo RFD_OK -curl -fsS http://localhost:8082/v1/health/ready && echo PMPNN_OK -curl -fsS http://localhost:8083/v1/health/ready && echo BOLTZ2_OK -``` - -If the NIMs and your client share a user-defined docker network, reach them by container -name instead of published ports (e.g. `http://rfdiffusion:8000/...`). - -## GPU profile selection - -Most of these NIMs **auto-select** a profile by compute capability (SM) and just work on -a supported GPU. Some NIMs match profiles by **exact GPU model**, so on a GPU that has no -bundled profile the container exits early with `NIMProfileIDNotFound` / "0 profiles found". -If that happens, list the bundled profiles and pin the one that matches **your** GPU's -compute capability: - -```bash -# 1) list the profiles this NIM ships and their tags (gpu / compute capability / precision / backend) -docker run --rm --gpus device=0 -e NGC_API_KEY \ - list-model-profiles -# 2) choose the profile whose tags match YOUR GPU (compute capability first), then pin it: -docker run -d --name --gpus device=0 --shm-size=8g \ - -e NGC_API_KEY -e NIM_MODEL_PROFILE= \ - -v ~/nimcache_:/opt/nim/.cache -p 8083:8000 -``` - -A TRT-optimized engine built for one GPU generally loads on another of the **same compute -capability** (you may see a benign cross-device warning). Always select the profile for the -hardware you are running on — **do not copy a `NIM_MODEL_PROFILE` id from another machine**, -and check the NIM's support matrix for your exact GPU. - -## Then point the pipeline at local - -Set the local base URLs and drop the `Authorization` header. Request/response shapes for -every NIM are in `references/pipeline.md` — only the URL/auth changes between hosted and -local. diff --git a/workflows/generative-protein-binder-design/protein-binder-design/references/manifest.md b/workflows/generative-protein-binder-design/protein-binder-design/references/manifest.md deleted file mode 100755 index 0f520e00..00000000 --- a/workflows/generative-protein-binder-design/protein-binder-design/references/manifest.md +++ /dev/null @@ -1,71 +0,0 @@ -# Run Manifest & Directory Convention - -A campaign writes one `manifest.json` (machine state) and one `candidates.csv` -(human/spreadsheet view) under a run directory. The manifest is the backbone for -ranking, resumability, validation, and the final report. - -## Directory layout - -```text -runs/_/ -├── manifest.json # campaign state (see schema below) -├── candidates.csv # flat table, regenerated from the manifest -├── backbones/ # RFdiffusion output PDBs -├── sequences/ # ProteinMPNN mfasta files -└── complexes/ # Boltz2/OpenFold3 co-folded .cif files -``` - -## Schema (`manifest.json`) - -```json -{ - "schema_version": "1.0", - "campaign": "protein-binder-design", - "created": "2026-06-12T18:00:00+00:00", - "run_dir": "runs/_", - "target": { "name": "", "pdb_id": "", "chain": "" }, - "mode": "hosted", - "params": { "n_backbones": 100, "seqs_per_backbone": 8, "binder_len": "60-90" }, - "filters": { "iptm_min": 0.8, "binder_plddt_min": 80.0, "self_consistency_rmsd_max": 2.0 }, - "stages": [ { "stage": "rfdiffusion", "ts": "...", "n": 100 } ], - "candidates": [ - { - "id": "bb003_seq02", - "backbone_id": "bb003", - "sequence": "....", - "scores": { "proteinmpnn_nll": 1.02, "iptm": 0.84, "binder_plddt": 86.2, "self_consistency_rmsd": 1.4 }, - "artifacts": { "backbone_pdb": "backbones/bb003.pdb", "complex_cif": "complexes/bb003_seq02.cif" }, - "passed_filter": true, - "is_control": false, - "control_type": null, - "created": "..." - } - ] -} -``` - -## Usage (`scripts/manifest.py`) - -```python -import sys; sys.path.insert(0, "scripts") -from manifest import Manifest - -m = Manifest.create(run_dir="runs/demo", target={"name": "X", "chain": "A"}, mode="hosted") -m.log_stage("rfdiffusion", n=100) -m.upsert_candidate("bb003_seq02", backbone_id="bb003", sequence="MKT...") -m.set_scores("bb003_seq02", iptm=0.84, binder_plddt=86.2, self_consistency_rmsd=1.4) -m.add_artifact("bb003_seq02", "complex_cif", "complexes/bb003_seq02.cif") -m.apply_filters() -top = m.rank(by="iptm", passed_only=True) -m.to_csv() - -# resume a campaign later -m2 = Manifest.load("runs/demo") -``` - -## Resumability - -Because state lives in `manifest.json`, an interrupted campaign resumes by -loading the manifest and skipping candidates that already have the needed -scores/artifacts. Long campaigns should checkpoint after each expensive stage -(`m.save()` is called automatically by the mutation helpers). diff --git a/workflows/generative-protein-binder-design/protein-binder-design/references/pipeline.md b/workflows/generative-protein-binder-design/protein-binder-design/references/pipeline.md deleted file mode 100755 index d2af20a6..00000000 --- a/workflows/generative-protein-binder-design/protein-binder-design/references/pipeline.md +++ /dev/null @@ -1,151 +0,0 @@ -# Binder Design Pipeline — Orchestration & Handoff Contracts - -This is the detailed orchestration for the `protein-binder-design` workflow. -The agent reasons over these steps and delegates each NIM call to the atomic -skill. Deterministic glue (parsing, remapping, RMSD, manifest) uses the bundled -`scripts/`. - -## 0. Setup - -- Decide hosted vs local **once** (ask the user) and reuse for every NIM call. -- Create a run directory and manifest: - -```python -import sys; sys.path.insert(0, "scripts") -from manifest import Manifest -m = Manifest.create( - run_dir="runs/_", - target={"name": "", "pdb_id": "", "chain": ""}, - mode="hosted", - params={"n_backbones": 100, "seqs_per_backbone": 8, "binder_len": "60-90"}, -) -``` - -## 1. Target prep - -- Obtain the target structure (experimental PDB, or predict with `openfold2-nim` - / `openfold3-nim` / `boltz2-nim` if none exists). -- Identify epitope/hotspot residues (from literature, the registry, or the user) - in **PDB author numbering**. -- Remap to 1-based sequence indices for tools that need them: - -```python -from pdb_utils import remap_to_seq_index -target_pdb = open(".pdb").read() -seq_idx = remap_to_seq_index(target_pdb, chain="", author_resnums=[]) -``` - -- RFdiffusion `hotspot_res` instead uses chain+author strings, e.g. - `["E453", "E455", "E456", "E486"]` (no remap needed there). -- Optional: build a target MSA with `msa-search-nim` if you will fold/co-fold - the target with evolutionary context. - -### Human-in-the-loop gate -Before generating backbones, confirm with the user: target chain, epitope/ -hotspot set, binder length range, number of backbones, sequences per backbone. - -## 2. Backbones — `rfdiffusion-nim` - -Binder design mode: pass the target `input_pdb`, a contig combining the target -segment and a generated binder segment, and `hotspot_res`. - -```python -# delegate the actual request to the rfdiffusion-nim skill -payload = { - "input_pdb": target_pdb, - "contigs": "E1-200/0 60-90", # keep target E1-200, chain break, generate 60-90 aa binder - "hotspot_res": ["E453", "E455", "E456", "E486", "E489", "E493", "E501"], - "diffusion_steps": 50, -} -# -> result["output_pdb"] is one backbone -``` - -Generate N backbones (loop with distinct seeds / repeated calls). Save each and -register it: - -```python -m.upsert_candidate("bb003", backbone_id="bb003") -m.add_artifact("bb003", "backbone_pdb", "runs/.../backbones/bb003.pdb") -``` - -## 3. Sequences — `proteinmpnn-nim` - -For each backbone, design k sequences. Redesign the **binder chain only** -(`input_pdb_chains=[binder_chain]`) so the target chain stays fixed. RFdiffusion -may renumber/rename chains, so re-read the backbone PDB to get the binder chain -ID and length first (`pdb_utils.py`). **Drop the native/WT row** from `mfasta` -and pair scores only with designed rows. - -```python -payload = { - "input_pdb": backbone_pdb, - "input_pdb_chains": [binder_chain], # redesign binder, keep target fixed - "num_seq_per_target": 8, - "sampling_temp": [0.1, 0.2], - "use_soluble_model": True, # for soluble binders -} -# parse result["mfasta"]: keep headers without 'native'/'wt'; zip with result["scores"] -m.upsert_candidate("bb003_seq02", backbone_id="bb003", sequence=designed_seq) -m.set_scores("bb003_seq02", proteinmpnn_nll=score) -``` - -## 4. Co-fold + score — `boltz2-nim` or `openfold3-nim` - -Co-fold each designed binder **with the target** as a 2-chain complex. Use the -binder sequence + target sequence (and target MSA if built). - -- `openfold3-nim` returns an explicit `iptm_score` (interface) and pLDDT. -- `boltz2-nim` returns `confidence_scores`; use it as the complex confidence. - -```python -# delegate to boltz2-nim / openfold3-nim -polymers = [ - {"id": "A", "molecule_type": "protein", "sequence": designed_seq}, # binder (single-seq MSA is standard for de novo) - {"id": "B", "molecule_type": "protein", "sequence": target_seq}, # target (+ MSA) -] -m.set_scores("bb003_seq02", iptm=iptm, binder_plddt=plddt, boltz2_confidence=conf) -m.add_artifact("bb003_seq02", "complex_cif", "runs/.../complexes/bb003_seq02.cif") -``` - -### Cost funnel -Co-folding is the expensive stage. Co-fold a **capped shortlist** first (e.g. -best ProteinMPNN NLL per backbone), review, then expand. Reserve high -`diffusion_samples` / `recycling_steps` for the final survivors. - -## 5. Self-consistency RMSD - -Compare the RFdiffusion backbone to the predicted binder chain (from the -co-folded complex). Low RMSD = the sequence is predicted to fold back into the -designed backbone. - -```python -from metrics import ca_rmsd_from_pdb -# extract the binder chain from the predicted complex, then: -rmsd = ca_rmsd_from_pdb(predicted_binder_pdb, backbone_pdb) -m.set_scores("bb003_seq02", self_consistency_rmsd=rmsd) -``` - -(Convert mmCIF→PDB or parse CA atoms from the binder chain; `pdb_utils` reads -PDB ATOM records.) - -## 6. Filter + rank + report - -```python -m.apply_filters() # uses manifest filters (ipTM/pLDDT/RMSD) -top = m.rank(by="iptm", descending=True, passed_only=True)[:20] -m.to_csv() # candidates.csv next to manifest.json -print(m.summary()) # {n_candidates, n_passed, n_controls} -``` - -Produce a short report: target + epitope, params, success rate, the top -designs with their scores and artifact paths, and how they compare to controls -(`references/validation.md`). - -## Branching summary - -- **No target structure** → predict it first (`openfold2/3-nim` or `boltz2-nim`). -- **Target needs evolutionary context** → `msa-search-nim` before co-folding. -- **Interface metric** → prefer OpenFold3 `iptm_score`; Boltz2 `confidence_scores` - is the fallback complex-confidence signal. -- **Binder MSA** → keep single-sequence for de novo binders (standard); do not - fabricate a binder MSA. diff --git a/workflows/generative-protein-binder-design/protein-binder-design/references/validation.md b/workflows/generative-protein-binder-design/protein-binder-design/references/validation.md deleted file mode 100755 index a1e9441b..00000000 --- a/workflows/generative-protein-binder-design/protein-binder-design/references/validation.md +++ /dev/null @@ -1,69 +0,0 @@ -# Binder Design Validation - -All outputs are in-silico, so validation means computational benchmarking. -Never report only top scores — report a **success rate** and compare against -controls. - -## Metrics - -| Metric | Source | Pass guide | Meaning | -|---|---|---|---| -| Interface confidence (ipTM) | OpenFold3 `iptm_score` · Boltz2 `confidence_scores` | ≥ 0.8 | predicted interface quality | -| Binder pLDDT | OpenFold3 / Boltz2 | ≥ 80 | binder fold confidence | -| Self-consistency RMSD | `scripts/metrics.py` (Kabsch CA-RMSD) | ≤ 2.0 Å | designed backbone vs predicted | -| Sequence quality (NLL) | ProteinMPNN `scores` | lower better | sequence–backbone compatibility | - -There is no protein–protein affinity NIM, so ipTM + self-consistency RMSD are -the binder proxy (the Bennett et al. 2023 filter pattern). - -## Controls - -Run controls through the **identical** pipeline so score distributions are -comparable. - -- **Negative controls** — scrambled binder sequences (preserve composition): - -```python -import sys; sys.path.insert(0, "scripts") -from controls import make_scrambled_controls -negs = make_scrambled_controls([designed_seq], n=5, seed=42) -# co-fold each, then register with is_control=True, control_type="scrambled" -m.upsert_candidate("ctrl_neg_01", is_control=True, control_type="scrambled", sequence=negs[0]) -``` - -- **Positive controls** — published binder sequences for the same target - (`assets/targets.json` → `published_binders`), co-folded the same way and - marked `control_type="published"`. - -A working pipeline separates designed/published positives from scrambled -negatives in the ipTM and RMSD distributions. - -## Success rate - -The metric the field actually quotes — fraction of designs passing the filter: - -```python -s = m.summary() -success_rate = s["n_passed"] / max(s["n_candidates"], 1) -``` - -Use it to compare pipeline configs (diffusion steps, sampling temperature, -sequences/backbone) rather than over-interpreting any single design. - -## Published comparison - -Re-score literature winners through your exact pipeline and check your top -designs land in the same ipTM/RMSD regime. Absolute scores are not comparable -across pipelines — only same-pipeline comparisons are meaningful. - -Benchmark targets come from the target registry (`assets/targets.json`) — add your own -with `scripts/registry.py`. Always confirm epitope residues against the cited structure -before use; registry entries flag illustrative residue lists. - -## Caveats - -- In-silico triage, not experimental validation. Prefer relative ranking and - distribution separation over absolute claims. -- ipTM can be optimistic; corroborate with self-consistency RMSD and ProteinMPNN - NLL before prioritizing. -- Keep all artifacts, payloads, and the manifest together for reproducibility. diff --git a/workflows/generative-protein-binder-design/protein-binder-design/scripts/controls.py b/workflows/generative-protein-binder-design/protein-binder-design/scripts/controls.py deleted file mode 100644 index e3cc6bc1..00000000 --- a/workflows/generative-protein-binder-design/protein-binder-design/scripts/controls.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 -"""Negative controls for binder validation: composition-preserving scrambles.""" -from __future__ import annotations - -import random - - -def scramble_sequence(seq, seed=0): - """Return a shuffled sequence (same amino-acid composition, new order).""" - rng = random.Random(seed) - chars = list(seq) - rng.shuffle(chars) - return "".join(chars) - - -def make_scrambled_controls(seqs, n=5, seed=0): - """Generate ``n`` scrambled negative controls drawn from ``seqs``.""" - rng = random.Random(seed) - if not seqs: - return [] - out = [] - for i in range(n): - base = seqs[i % len(seqs)] - out.append(scramble_sequence(base, seed=rng.randint(0, 2 ** 31 - 1))) - return out diff --git a/workflows/generative-protein-binder-design/protein-binder-design/scripts/metrics.py b/workflows/generative-protein-binder-design/protein-binder-design/scripts/metrics.py deleted file mode 100644 index 0f6acde6..00000000 --- a/workflows/generative-protein-binder-design/protein-binder-design/scripts/metrics.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 -"""Deterministic structural metrics for binder design (numpy only).""" -from __future__ import annotations - -import numpy as np - - -def kabsch_rmsd(p, q): - """Minimal RMSD after optimal superposition of two (N, 3) coord sets.""" - p = np.asarray(p, dtype=float) - q = np.asarray(q, dtype=float) - if p.shape != q.shape or p.ndim != 2 or p.shape[1] != 3: - raise ValueError(f"coordinate shape mismatch: {p.shape} vs {q.shape}") - if p.shape[0] == 0: - raise ValueError("no coordinates provided") - pc = p - p.mean(axis=0) - qc = q - q.mean(axis=0) - h = pc.T @ qc - u, _, vt = np.linalg.svd(h) - d = np.sign(np.linalg.det(vt.T @ u.T)) - rot = vt.T @ np.diag([1.0, 1.0, d]) @ u.T - p_rot = pc @ rot.T - return float(np.sqrt(np.sum((p_rot - qc) ** 2) / p.shape[0])) - - -def ca_rmsd_from_pdb(pdb_a, pdb_b, chain_a=None, chain_b=None): - """CA-RMSD between two structures (by chain). Truncates to common length.""" - import os - import sys - sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - from pdb_utils import ca_coords - - a = ca_coords(pdb_a, chain_a) - b = ca_coords(pdb_b, chain_b) - n = min(len(a), len(b)) - if n == 0: - raise ValueError("no CA atoms found for RMSD") - return kabsch_rmsd(a[:n], b[:n]) diff --git a/workflows/generative-protein-binder-design/protein-binder-design/scripts/pdb_utils.py b/workflows/generative-protein-binder-design/protein-binder-design/scripts/pdb_utils.py deleted file mode 100644 index 74445711..00000000 --- a/workflows/generative-protein-binder-design/protein-binder-design/scripts/pdb_utils.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 OR CC-BY-4.0 -"""Dependency-free PDB parsing helpers for binder-design handoffs. - -Covers the fragile glue between NIM steps: -- extract a chain, keep ATOM records -- one-letter sequence from CA atoms -- map PDB author residue numbers -> 1-based sequence index (hotspot/pocket remap) -- CA coordinates for RMSD -""" -from __future__ import annotations - -THREE_TO_ONE = { - "ALA": "A", "ARG": "R", "ASN": "N", "ASP": "D", "CYS": "C", "GLN": "Q", - "GLU": "E", "GLY": "G", "HIS": "H", "ILE": "I", "LEU": "L", "LYS": "K", - "MET": "M", "PHE": "F", "PRO": "P", "SER": "S", "THR": "T", "TRP": "W", - "TYR": "Y", "VAL": "V", "MSE": "M", "SEC": "U", "PYL": "O", -} - - -def _iter_atom_lines(pdb_text, chain=None): - for line in pdb_text.splitlines(): - if not line.startswith("ATOM"): - continue - if len(line) < 54: - continue - if chain is not None and line[21] != chain: - continue - yield line - - -def extract_chain(pdb_text, chain): - """Return PDB text containing only records for ``chain``.""" - keep = [] - for line in pdb_text.splitlines(): - if line.startswith(("ATOM", "HETATM", "TER")) and len(line) > 21 and line[21] == chain: - keep.append(line) - return "\n".join(keep) - - -def ca_residues(pdb_text, chain=None): - """Ordered list of (resName, resSeq, iCode, (x, y, z)) for CA atoms.""" - out = [] - for line in _iter_atom_lines(pdb_text, chain): - if line[12:16].strip() != "CA": - continue - res_name = line[17:20].strip() - res_seq = int(line[22:26]) - icode = line[26].strip() - x = float(line[30:38]); y = float(line[38:46]); z = float(line[46:54]) - out.append((res_name, res_seq, icode, (x, y, z))) - return out - - -def sequence(pdb_text, chain=None): - """One-letter sequence from CA atoms (unknown residues -> 'X').""" - return "".join(THREE_TO_ONE.get(r[0], "X") for r in ca_residues(pdb_text, chain)) - - -def residue_index_map(pdb_text, chain=None): - """Map PDB author residue id -> 1-based sequence index (CA order). - - Keys are stored both as the bare author number ('501') and, when an - insertion code is present, as number+icode ('501A'). - """ - mapping = {} - for i, (_, res_seq, icode, _) in enumerate(ca_residues(pdb_text, chain), start=1): - mapping[str(res_seq)] = i - if icode: - mapping[f"{res_seq}{icode}"] = i - return mapping - - -def remap_to_seq_index(pdb_text, chain, author_resnums): - """Convert PDB author residue numbers to 1-based sequence indices.""" - mapping = residue_index_map(pdb_text, chain) - out, missing = [], [] - for a in author_resnums: - key = str(a) - if key in mapping: - out.append(mapping[key]) - else: - missing.append(key) - if missing: - raise KeyError(f"residues not found in chain {chain!r}: {missing}") - return out - - -def ca_coords(pdb_text, chain=None): - """List of (x, y, z) for CA atoms in chain order.""" - return [r[3] for r in ca_residues(pdb_text, chain)] diff --git a/workflows/generative-protein-binder-design/workflow-rules.mdc b/workflows/generative-protein-binder-design/workflow-rules.mdc index 1a10fff3..6ecf2274 100644 --- a/workflows/generative-protein-binder-design/workflow-rules.mdc +++ b/workflows/generative-protein-binder-design/workflow-rules.mdc @@ -26,7 +26,7 @@ and ranks by interface metrics (ipTM / ipSAE / pLDDT / apo↔holo RMSD). (binder sequence + structure together) with an AF2-reward gate, validated independently with Boltz2 / OpenFold3. Prefer when the user wants Proteina-Complexa specifically, or reward-guided test-time search (best-of-n / beam / FK-steering / MCTS). -- **`protein-binder-design/`** — **RFdiffusion + ProteinMPNN + Boltz2 / OpenFold3** BioNeMo +- **[protein-binder-design](../../workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md)** — **RFdiffusion + ProteinMPNN + Boltz2 / OpenFold3** BioNeMo NIM orchestration (backbones → sequences → co-fold → filter). Prefer the classic diffusion + inverse-folding NIM pipeline. From 7f6d91830d7f3174c8edfb6d62a6fa8a3177176f Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Mon, 12 Oct 2026 17:43:04 -0700 Subject: [PATCH 2/9] Run protein binder design evaluations without a custom Docker image Signed-off-by: Ohad Mosafi --- .github/workflows/sync-skills.yml | 2 +- CONTRIBUTING.md | 12 ++++++------ catalog-exceptions.yml | 2 +- nim-skills/proteinmpnn-nim/SKILL.md | 2 -- scripts/plugin_sync.py | 4 ++-- .../skills/protein-binder-design/README.md | 2 +- .../config/skillspector-baseline.yml | 2 +- .../skills/protein-binder-design/evals/README.md | 13 +++++++++++-- .../skills/protein-binder-design/evals/config.yml | 9 +++++++++ .../evals/environment/Dockerfile | 5 ----- .../skills/protein-binder-design/requirements.txt | 2 +- .../skills/proteinmpnn-nim/SKILL.md | 2 -- .../protein-binder-design/README.md | 2 +- .../config/skillspector-baseline.yml | 2 +- .../protein-binder-design/evals/README.md | 13 +++++++++++-- .../protein-binder-design/evals/config.yml | 9 +++++++++ .../evals/environment/Dockerfile | 5 ----- .../protein-binder-design/requirements.txt | 2 +- 18 files changed, 56 insertions(+), 34 deletions(-) delete mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/environment/Dockerfile delete mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/environment/Dockerfile diff --git a/.github/workflows/sync-skills.yml b/.github/workflows/sync-skills.yml index 1ddd0e9c..8dc70429 100644 --- a/.github/workflows/sync-skills.yml +++ b/.github/workflows/sync-skills.yml @@ -4,7 +4,7 @@ # Sync skills from source repos into the bionemo-agent-toolkit catalog. # Adapted from NVIDIA/skills' sync-skills.yml, STRIPPED to the aggregation # core (no signing / compliance / drift / contact-resolution / tracker steps — -# NVCARPS is deferred; the review PR is the v1 gate). Adapted for GROUPED +# automated validation is deferred; the review PR is the v1 gate). Adapted for GROUPED # catalog dirs: catalog_dir already includes its group prefix # (e.g. library-skills/nvmolkit-usage), so the rsync target is $catalog_dir, not # skills/$catalog_dir. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5033dcfe..7f549fa1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,11 +56,11 @@ copy directly; run `python scripts/plugin_sync.py --write` after changing a source skill. The root `plugins -> skills` symlink is a temporary compatibility path for existing plugin consumers. -The `protein-binder-design` source skill uses `workflow-skills/` because NVCarps -treats the `workflows/` prefix as a whole-workflow validation target. This lets -NVCarps validate the generated skill individually while plugin-sync and security -checks cover its source. Both roots remain registered so existing workflow -sources can stay in `workflows/` until they are migrated. +The `protein-binder-design` source skill uses `workflow-skills/` because CI treats +the `workflows/` prefix as a whole-workflow validation target. This lets CI +validate the generated skill individually while plugin-sync and security checks +cover its source. Both roots remain registered so existing workflow sources can +stay in `workflows/` until they are migrated. ### Onboarding a source repo @@ -95,7 +95,7 @@ the same conventions `NVIDIA/skills` uses, so a compliant repo is ingestible by - **SRC-8 · No `AGENTS.md` / `CLAUDE.md` inside a skill dir** (auto-loaded by some harnesses). - **SRC-9 · No stray non-skill files** in the skills container. -**Compliance — enforced by `NVIDIA/skills`; deferred here (NVCARPS):** +**Compliance — enforced by `NVIDIA/skills`; deferred here:** - **SRC-10 · Co-located `evals/evals.json`** per skill + a `BENCHMARK.md` report. Eval definitions for a **sourced** skill must live under `evals/` **in the source component repo** — never authored into the catalog copy here. The sync diff --git a/catalog-exceptions.yml b/catalog-exceptions.yml index 255e2ae0..9409d043 100644 --- a/catalog-exceptions.yml +++ b/catalog-exceptions.yml @@ -18,7 +18,7 @@ exceptions: owner: trvachov component: Workflows - dir: workflow-skills - reason: Native workflow skills migrated out of workflows/ for individual skill validation in NVCarps. + reason: Native workflow skills migrated out of workflows/ for individual skill validation in CI. owner: trvachov component: Workflows diff --git a/nim-skills/proteinmpnn-nim/SKILL.md b/nim-skills/proteinmpnn-nim/SKILL.md index 6d6568c4..a041dc14 100644 --- a/nim-skills/proteinmpnn-nim/SKILL.md +++ b/nim-skills/proteinmpnn-nim/SKILL.md @@ -12,8 +12,6 @@ permissions: # ProteinMPNN NIM - - Design protein sequences for a supplied backbone PDB. Use this guide for first-pass hosted/local usage; load supplemental files only when needed: diff --git a/scripts/plugin_sync.py b/scripts/plugin_sync.py index 13f98341..df284cb5 100644 --- a/scripts/plugin_sync.py +++ b/scripts/plugin_sync.py @@ -12,7 +12,7 @@ 1. COVERAGE — every distributable source skill is listed in ``skills.sh.json``. 2. FRESHNESS — for every listed skill, the aggregate skill folder is an exact - copy of the source skill folder (including ``evals/`` — NVCARPS + copy of the source skill folder (including ``evals/`` — Tier 3 requires the eval dataset in the validated payload). Modes: @@ -50,7 +50,7 @@ JUNK = {".DS_Store", "__pycache__", ".skillsource.json"} # Subdirectories of a skill that are stripped from the aggregate payload. # Empty: the aggregate is a full copy. ``evals/`` is intentionally kept because -# NVCARPS Tier 3 (live agent eval) validates the aggregate and requires the +# Tier 3 (live agent eval) validates the aggregate and requires the # eval dataset there — stripping it makes Tier 3 skip → coverage invalid → gate block. STRIP_FROM_AGGREGATE: set[str] = set() # Generated artifacts the NVSkills signing/eval pipeline writes into the aggregate diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md index a984980c..150d2c6b 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md @@ -43,7 +43,7 @@ These tests use synthetic fixtures and credentials; they make no model calls. ## Tier 1 validation -NVCARPS selects `config/skillspector-baseline.yml` before the repository-wide +CI selects `config/skillspector-baseline.yml` before the repository-wide baseline. Include this file, the requirements files, and the tests when committing the skill; synchronize the aggregate copy with the repository's plugin sync tool. The baseline records reviewed false positives as exact source fingerprints for diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/config/skillspector-baseline.yml b/skills/bionemo-agent-toolkit/skills/protein-binder-design/config/skillspector-baseline.yml index 161a053a..fe9cceef 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/config/skillspector-baseline.yml +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/config/skillspector-baseline.yml @@ -1,4 +1,4 @@ -# Reviewed 2026-10-07 against SkillSpector 2.11.2, the scanner used by NVCARPS. +# Reviewed 2026-10-07 against SkillSpector 2.11.2, the scanner used in CI. # Exact fingerprints bind the complete file contents and scanner version. # File edits or a scanner upgrade invalidate these exceptions and require review. # No rule-wide suppressions; credential harvesting is fixed in code, not waived. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md index 5377e065..4f1c7a67 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md @@ -20,8 +20,17 @@ inputs are staged in both conditions; equivalent implementations are accepted. | `offline-provenance-rejection` | `provenance_cache/`, `cache_contract.md` | Reject candidate, sequence, target, sample and response-file mismatches before scoring. | Every case stages identical inputs for the with/without-skill conditions. -`environment/Dockerfile` supplies the same pinned NumPy and Biotite versions to -both conditions during environment setup, before the offline task starts. +`harbor.pre_agent_setup` in `config.yml` installs the same pinned NumPy 2.5.1 +and Biotite 1.7.1 wheels in both conditions, then checks their imports and versions +before the agent starts. Setup uses the sandbox's `python3` interpreter, which +also runs the workflow helpers, and needs permission to install its packages. +There is no custom `evals/environment/Dockerfile`, so Astra uses its existing +sandbox image without invoking the managed image builder or requiring its token. +The standard sandbox must provide Python 3.12+, pip, and package-index access +for setup. Setup downloads packages; the subsequent exercises +make no network calls. Normal sandbox and agent API access are still required. +Installation or import failures stop environment preflight before agent trials. +These offline cases need no GPU, NIM credentials, or changes to the CI service. The original prompts and fixtures are preserved; environment and verifier changes should still be recorded when comparing with historical reports. The added prompts request campaign outputs without giving the expected order; diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml index e54e6139..dfff230a 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml @@ -2,6 +2,15 @@ schema_version: 1 harbor: task_source: evals_json + # Use the existing sandbox image; install identical CPU dependencies in both + # conditions before the agent starts. + pre_agent_setup: + - >- + python3 -m pip install --disable-pip-version-check --no-cache-dir + --only-binary=:all: --timeout 15 --retries 1 + numpy==2.5.1 biotite==1.7.1 + - >- + python3 -c "import numpy, biotite; assert numpy.__version__ == '2.5.1'; assert biotite.__version__ == '1.7.1'" agents: claude-code: model: aws/anthropic/bedrock-claude-opus-5 diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/environment/Dockerfile b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/environment/Dockerfile deleted file mode 100644 index 323dfaad..00000000 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/environment/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.12-slim - -# Shared by both conditions, before the offline agent task starts. Pin the -# versions used by the local regression suite; no NIM clients or credentials. -RUN python3 -m pip install --no-cache-dir numpy==2.5.1 biotite==1.7.1 diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt b/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt index 2faa4494..3bf0c707 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt @@ -1,4 +1,4 @@ -# Tested helper environment (Python 3.12+); NumPy/Biotite match the Tier 3 image. +# Tested helper environment (Python 3.12+); NumPy/Biotite match Tier 3 pre-agent setup. numpy==2.5.1 biotite==1.7.1 requests==2.34.2 diff --git a/skills/bionemo-agent-toolkit/skills/proteinmpnn-nim/SKILL.md b/skills/bionemo-agent-toolkit/skills/proteinmpnn-nim/SKILL.md index 6d6568c4..a041dc14 100644 --- a/skills/bionemo-agent-toolkit/skills/proteinmpnn-nim/SKILL.md +++ b/skills/bionemo-agent-toolkit/skills/proteinmpnn-nim/SKILL.md @@ -12,8 +12,6 @@ permissions: # ProteinMPNN NIM - - Design protein sequences for a supplied backbone PDB. Use this guide for first-pass hosted/local usage; load supplemental files only when needed: diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/README.md index a984980c..150d2c6b 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/README.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/README.md @@ -43,7 +43,7 @@ These tests use synthetic fixtures and credentials; they make no model calls. ## Tier 1 validation -NVCARPS selects `config/skillspector-baseline.yml` before the repository-wide +CI selects `config/skillspector-baseline.yml` before the repository-wide baseline. Include this file, the requirements files, and the tests when committing the skill; synchronize the aggregate copy with the repository's plugin sync tool. The baseline records reviewed false positives as exact source fingerprints for diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/config/skillspector-baseline.yml b/workflow-skills/generative-protein-binder-design/protein-binder-design/config/skillspector-baseline.yml index 161a053a..fe9cceef 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/config/skillspector-baseline.yml +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/config/skillspector-baseline.yml @@ -1,4 +1,4 @@ -# Reviewed 2026-10-07 against SkillSpector 2.11.2, the scanner used by NVCARPS. +# Reviewed 2026-10-07 against SkillSpector 2.11.2, the scanner used in CI. # Exact fingerprints bind the complete file contents and scanner version. # File edits or a scanner upgrade invalidate these exceptions and require review. # No rule-wide suppressions; credential harvesting is fixed in code, not waived. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md index 5377e065..4f1c7a67 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md @@ -20,8 +20,17 @@ inputs are staged in both conditions; equivalent implementations are accepted. | `offline-provenance-rejection` | `provenance_cache/`, `cache_contract.md` | Reject candidate, sequence, target, sample and response-file mismatches before scoring. | Every case stages identical inputs for the with/without-skill conditions. -`environment/Dockerfile` supplies the same pinned NumPy and Biotite versions to -both conditions during environment setup, before the offline task starts. +`harbor.pre_agent_setup` in `config.yml` installs the same pinned NumPy 2.5.1 +and Biotite 1.7.1 wheels in both conditions, then checks their imports and versions +before the agent starts. Setup uses the sandbox's `python3` interpreter, which +also runs the workflow helpers, and needs permission to install its packages. +There is no custom `evals/environment/Dockerfile`, so Astra uses its existing +sandbox image without invoking the managed image builder or requiring its token. +The standard sandbox must provide Python 3.12+, pip, and package-index access +for setup. Setup downloads packages; the subsequent exercises +make no network calls. Normal sandbox and agent API access are still required. +Installation or import failures stop environment preflight before agent trials. +These offline cases need no GPU, NIM credentials, or changes to the CI service. The original prompts and fixtures are preserved; environment and verifier changes should still be recorded when comparing with historical reports. The added prompts request campaign outputs without giving the expected order; diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml index e54e6139..dfff230a 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml @@ -2,6 +2,15 @@ schema_version: 1 harbor: task_source: evals_json + # Use the existing sandbox image; install identical CPU dependencies in both + # conditions before the agent starts. + pre_agent_setup: + - >- + python3 -m pip install --disable-pip-version-check --no-cache-dir + --only-binary=:all: --timeout 15 --retries 1 + numpy==2.5.1 biotite==1.7.1 + - >- + python3 -c "import numpy, biotite; assert numpy.__version__ == '2.5.1'; assert biotite.__version__ == '1.7.1'" agents: claude-code: model: aws/anthropic/bedrock-claude-opus-5 diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/environment/Dockerfile b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/environment/Dockerfile deleted file mode 100644 index 323dfaad..00000000 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/environment/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.12-slim - -# Shared by both conditions, before the offline agent task starts. Pin the -# versions used by the local regression suite; no NIM clients or credentials. -RUN python3 -m pip install --no-cache-dir numpy==2.5.1 biotite==1.7.1 diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt b/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt index 2faa4494..3bf0c707 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt @@ -1,4 +1,4 @@ -# Tested helper environment (Python 3.12+); NumPy/Biotite match the Tier 3 image. +# Tested helper environment (Python 3.12+); NumPy/Biotite match Tier 3 pre-agent setup. numpy==2.5.1 biotite==1.7.1 requests==2.34.2 From dbbfcd4d311eb4d9ee15b30d8a506d56a123892b Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Mon, 12 Oct 2026 22:14:52 -0700 Subject: [PATCH 3/9] Fix binder design Tier 1 scan blockers and Tier 3 metric declaration Signed-off-by: Ohad Mosafi --- nim-skills/proteinmpnn-nim/SKILL.md | 2 ++ .../complexa-binder-design/scripts/validate_binders.py | 2 +- .../skills/protein-binder-design/evals/README.md | 5 ++++- .../skills/protein-binder-design/evals/config.yml | 2 ++ skills/bionemo-agent-toolkit/skills/proteinmpnn-nim/SKILL.md | 2 ++ .../protein-binder-design/evals/README.md | 5 ++++- .../protein-binder-design/evals/config.yml | 2 ++ .../complexa-binder-design/scripts/validate_binders.py | 2 +- 8 files changed, 18 insertions(+), 4 deletions(-) diff --git a/nim-skills/proteinmpnn-nim/SKILL.md b/nim-skills/proteinmpnn-nim/SKILL.md index a041dc14..6d6568c4 100644 --- a/nim-skills/proteinmpnn-nim/SKILL.md +++ b/nim-skills/proteinmpnn-nim/SKILL.md @@ -12,6 +12,8 @@ permissions: # ProteinMPNN NIM + + Design protein sequences for a supplied backbone PDB. Use this guide for first-pass hosted/local usage; load supplemental files only when needed: diff --git a/skills/bionemo-agent-toolkit/skills/complexa-binder-design/scripts/validate_binders.py b/skills/bionemo-agent-toolkit/skills/complexa-binder-design/scripts/validate_binders.py index d10f334b..4903c168 100644 --- a/skills/bionemo-agent-toolkit/skills/complexa-binder-design/scripts/validate_binders.py +++ b/skills/bionemo-agent-toolkit/skills/complexa-binder-design/scripts/validate_binders.py @@ -308,7 +308,7 @@ def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--run-dir", required=True, type=Path) - ap.add_argument("--hotspots", type=Path, help="hotspots.json (omit => unconditioned)") + ap.add_argument("--hotspots", type=Path, help="hotspots.json; default: unconditioned validation") ap.add_argument("--target-chain", default="A") ap.add_argument("--binder-chain", default="B") ap.add_argument("--endpoint", choices=["hosted", "local"], default="hosted") diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md index 4f1c7a67..712d6e09 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md @@ -55,7 +55,10 @@ python3 -m unittest discover -s tests -v ## Deterministic artifact checks `grading.mode: aces_plus_custom` runs the existing ACES judges and the standalone -`grader.py`. The verifier reads actual manifests, CSVs, selected complex files +`grader.py`. `config.yml` declares its `artifact_correctness` output under +`grading.custom_metrics` so the evaluator can seal the reward contract before +starting agents. The evaluator computes the schema digest from that declaration +and the grader. The verifier reads actual manifests, CSVs, selected complex files and summaries. It pins input digests, checks score types and absence, ranking order, control exclusion, preserved state, source identity and artifact bytes. It imports no skill helpers. Tests exercise all eight workflows and deliberately diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml index dfff230a..f6ee993e 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml @@ -19,3 +19,5 @@ harbor: grading: mode: aces_plus_custom + custom_metrics: + - artifact_correctness diff --git a/skills/bionemo-agent-toolkit/skills/proteinmpnn-nim/SKILL.md b/skills/bionemo-agent-toolkit/skills/proteinmpnn-nim/SKILL.md index a041dc14..6d6568c4 100644 --- a/skills/bionemo-agent-toolkit/skills/proteinmpnn-nim/SKILL.md +++ b/skills/bionemo-agent-toolkit/skills/proteinmpnn-nim/SKILL.md @@ -12,6 +12,8 @@ permissions: # ProteinMPNN NIM + + Design protein sequences for a supplied backbone PDB. Use this guide for first-pass hosted/local usage; load supplemental files only when needed: diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md index 4f1c7a67..712d6e09 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md @@ -55,7 +55,10 @@ python3 -m unittest discover -s tests -v ## Deterministic artifact checks `grading.mode: aces_plus_custom` runs the existing ACES judges and the standalone -`grader.py`. The verifier reads actual manifests, CSVs, selected complex files +`grader.py`. `config.yml` declares its `artifact_correctness` output under +`grading.custom_metrics` so the evaluator can seal the reward contract before +starting agents. The evaluator computes the schema digest from that declaration +and the grader. The verifier reads actual manifests, CSVs, selected complex files and summaries. It pins input digests, checks score types and absence, ranking order, control exclusion, preserved state, source identity and artifact bytes. It imports no skill helpers. Tests exercise all eight workflows and deliberately diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml index dfff230a..f6ee993e 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml @@ -19,3 +19,5 @@ harbor: grading: mode: aces_plus_custom + custom_metrics: + - artifact_correctness diff --git a/workflows/generative-protein-binder-design/complexa-binder-design/scripts/validate_binders.py b/workflows/generative-protein-binder-design/complexa-binder-design/scripts/validate_binders.py index d10f334b..4903c168 100644 --- a/workflows/generative-protein-binder-design/complexa-binder-design/scripts/validate_binders.py +++ b/workflows/generative-protein-binder-design/complexa-binder-design/scripts/validate_binders.py @@ -308,7 +308,7 @@ def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--run-dir", required=True, type=Path) - ap.add_argument("--hotspots", type=Path, help="hotspots.json (omit => unconditioned)") + ap.add_argument("--hotspots", type=Path, help="hotspots.json; default: unconditioned validation") ap.add_argument("--target-chain", default="A") ap.add_argument("--binder-chain", default="B") ap.add_argument("--endpoint", choices=["hosted", "local"], default="hosted") From 388af1fe683e2463e39c8138031e5d1907172e5b Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Tue, 13 Oct 2026 10:21:11 -0700 Subject: [PATCH 4/9] Isolate PBD Tier 3 dependencies Signed-off-by: Ohad Mosafi --- .../scripts/validate_binders.py | 2 +- .../protein-binder-design/evals/README.md | 28 +++++--- .../protein-binder-design/evals/config.yml | 11 +-- .../protein-binder-design/evals/evals.json | 40 ++++++----- .../evals/files/setup_eval_env.py | 68 +++++++++++++++++++ .../tests/test_eval_environment.py | 44 ++++++++++++ .../protein-binder-design/evals/README.md | 28 +++++--- .../protein-binder-design/evals/config.yml | 11 +-- .../protein-binder-design/evals/evals.json | 40 ++++++----- .../evals/files/setup_eval_env.py | 68 +++++++++++++++++++ .../tests/test_eval_environment.py | 44 ++++++++++++ .../scripts/validate_binders.py | 2 +- 12 files changed, 318 insertions(+), 68 deletions(-) create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py create mode 100644 skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py create mode 100644 workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py diff --git a/skills/bionemo-agent-toolkit/skills/complexa-binder-design/scripts/validate_binders.py b/skills/bionemo-agent-toolkit/skills/complexa-binder-design/scripts/validate_binders.py index 4903c168..d10f334b 100644 --- a/skills/bionemo-agent-toolkit/skills/complexa-binder-design/scripts/validate_binders.py +++ b/skills/bionemo-agent-toolkit/skills/complexa-binder-design/scripts/validate_binders.py @@ -308,7 +308,7 @@ def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--run-dir", required=True, type=Path) - ap.add_argument("--hotspots", type=Path, help="hotspots.json; default: unconditioned validation") + ap.add_argument("--hotspots", type=Path, help="hotspots.json (omit => unconditioned)") ap.add_argument("--target-chain", default="A") ap.add_argument("--binder-chain", default="B") ap.add_argument("--endpoint", choices=["hosted", "local"], default="hosted") diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md index 712d6e09..684550da 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md @@ -2,8 +2,9 @@ ## Active coverage: offline software workflows -`evals.json` runs eight offline exercises. The four original prompts, assertions, -expected outputs and score fixtures remain unchanged for historical comparison. +`evals.json` runs eight offline exercises. The original task requests, assertions, +expected outputs and score fixtures remain unchanged. Each prompt additionally +identifies the same prepared Python interpreter for both conditions. Four additional cases require target mapping, saved-response interpretation, recovery decisions and provenance checks. Their output contracts and all required inputs are staged in both conditions; equivalent implementations are accepted. @@ -20,19 +21,28 @@ inputs are staged in both conditions; equivalent implementations are accepted. | `offline-provenance-rejection` | `provenance_cache/`, `cache_contract.md` | Reject candidate, sequence, target, sample and response-file mismatches before scoring. | Every case stages identical inputs for the with/without-skill conditions. -`harbor.pre_agent_setup` in `config.yml` installs the same pinned NumPy 2.5.1 -and Biotite 1.7.1 wheels in both conditions, then checks their imports and versions -before the agent starts. Setup uses the sandbox's `python3` interpreter, which -also runs the workflow helpers, and needs permission to install its packages. +`harbor.pre_agent_setup` in `config.yml` runs the staged `setup_eval_env.py` in +both conditions. It selects an installed Python 3.12+, creates the writable +`/workspace/pbd-eval-venv` virtual environment, installs the same pinned NumPy +2.5.1 and Biotite 1.7.1 wheels, and checks their imports before the agent starts. +Every prompt specifies `/workspace/pbd-eval-venv/bin/python` for Python commands. +Setup leaves system Python packages untouched and needs no root privileges. There is no custom `evals/environment/Dockerfile`, so Astra uses its existing sandbox image without invoking the managed image builder or requiring its token. -The standard sandbox must provide Python 3.12+, pip, and package-index access +The standard sandbox must provide Python 3.12+ with venv/ensurepip, a writable +workspace, and package-index access for setup. Setup downloads packages; the subsequent exercises make no network calls. Normal sandbox and agent API access are still required. Installation or import failures stop environment preflight before agent trials. +Setup reports the chosen interpreter and the phase that failed. If preflight +fails, inspect its healthcheck stdout and +stderr to distinguish a missing pip module, an installation-permission error, +package-index access, an incompatible wheel, or an import error. The summary's +`HealthcheckError` alone does not identify which condition failed. Preserve this +diagnostic evidence before retrying; changing model settings cannot repair setup. These offline cases need no GPU, NIM credentials, or changes to the CI service. -The original prompts and fixtures are preserved; environment and verifier -changes should still be recorded when comparing with historical reports. +The task requests and fixtures are preserved; the interpreter instruction, +environment and verifier changes must be recorded in historical comparisons. The added prompts request campaign outputs without giving the expected order; the assertions check saved artifacts, not use of a particular helper. All sequences, toy coordinates, numeric inputs, saved responses and control labels are synthetic. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml index f6ee993e..b050442e 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/config.yml @@ -2,15 +2,10 @@ schema_version: 1 harbor: task_source: evals_json - # Use the existing sandbox image; install identical CPU dependencies in both - # conditions before the agent starts. + # Use the existing sandbox image; install identical CPU dependencies into a + # writable virtual environment in both conditions before the agent starts. pre_agent_setup: - - >- - python3 -m pip install --disable-pip-version-check --no-cache-dir - --only-binary=:all: --timeout 15 --retries 1 - numpy==2.5.1 biotite==1.7.1 - - >- - python3 -c "import numpy, biotite; assert numpy.__version__ == '2.5.1'; assert biotite.__version__ == '1.7.1'" + - python3 /workspace/input/setup_eval_env.py agents: claude-code: model: aws/anthropic/bedrock-claude-opus-5 diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/evals.json b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/evals.json index b8711b6f..7aad6167 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/evals.json +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/evals.json @@ -3,10 +3,11 @@ "evals": [ { "id": "offline-manifest", - "prompt": "Run an offline software bookkeeping exercise using /workspace/input/bookkeeping.json. All scores, controls, and thresholds in this file are synthetic; do not call model APIs or describe these numbers as scientific predictions. For each profile, create a separate run directory and manifest using the supplied target, filters, ranking metric, candidates, scores, and control flags. Use the bundled manifest helper if available, or implement equivalent bookkeeping. Save and reload each manifest, apply the supplied filters, exclude controls from rankings and success counts, and export both all_candidates.csv and ranked candidates.csv. Show the resulting counts and ranked IDs for each profile. Leave absent scores absent, including ipTM for the Boltz2 profile. Label all results as simulated bookkeeping, disclose the pending candidate, and explain that these results provide no live inference coverage.", + "prompt": "Run an offline software bookkeeping exercise using /workspace/input/bookkeeping.json. All scores, controls, and thresholds in this file are synthetic; do not call model APIs or describe these numbers as scientific predictions. For each profile, create a separate run directory and manifest using the supplied target, filters, ranking metric, candidates, scores, and control flags. Use the bundled manifest helper if available, or implement equivalent bookkeeping. Save and reload each manifest, apply the supplied filters, exclude controls from rankings and success counts, and export both all_candidates.csv and ranked candidates.csv. Show the resulting counts and ranked IDs for each profile. Leave absent scores absent, including ipTM for the Boltz2 profile. Label all results as simulated bookkeeping, disclose the pending candidate, and explain that these results provide no live inference coverage. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "expected_output": "Two saved and reloaded manifests and two CSVs per profile. Each profile has four non-control candidates, two passing candidates, and one control. Both rankings contain best then boundary. Below and pending fail; the control is excluded from rankings and counts. The Boltz2 profile retains boltz2_confidence without inventing ipTM. The simulated 2/4 passing fraction is provisional because pending lacks required scores. No model calls or scientific validation claims.", "files": [ - "evals/files/bookkeeping.json" + "evals/files/bookkeeping.json", + "evals/files/setup_eval_env.py" ], "assertions": [ "[executes-bookkeeping] Executes the bookkeeping and produces a manifest and both CSV artifacts per profile; a proposed plan alone does not pass.", @@ -19,10 +20,11 @@ }, { "id": "offline-boltz2-ranking", - "prompt": "Review my protein binder campaign in /workspace/input/boltz2_only.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/boltz2-only. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "prompt": "Review my protein binder campaign in /workspace/input/boltz2_only.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/boltz2-only. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "expected_output": "The supplied composite-confidence cutoff is 0.73; ipTM remains disabled and absent. Ranked IDs are binder_a then binder_b. Counts are n_candidates=4, n_passed=2, n_controls=1; the audit retains all five entries. The sequence-only control is exempt only from RMSD. Pending lacks RMSD and low_confidence fails. The 2/4 fraction is provisional and synthetic.", "files": [ - "evals/files/boltz2_only.json" + "evals/files/boltz2_only.json", + "evals/files/setup_eval_env.py" ], "assertions": [ "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/boltz2-only; a plan alone does not pass.", @@ -35,10 +37,11 @@ }, { "id": "offline-mixed-threshold-ranking", - "prompt": "Review my protein binder campaign in /workspace/input/mixed_thresholds.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/mixed-thresholds. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "prompt": "Review my protein binder campaign in /workspace/input/mixed_thresholds.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/mixed-thresholds. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "expected_output": "All supplied thresholds are enforced. Ranked IDs are binder_a then binder_b, despite higher ipTM on the control and designs failing other criteria. Counts are n_candidates=6, n_passed=2, n_controls=1; all seven entries remain in the audit. The string-valued interface score stays invalid rather than being coerced; pending lacks RMSD. Results are synthetic and 2/6 is provisional.", "files": [ - "evals/files/mixed_thresholds.json" + "evals/files/mixed_thresholds.json", + "evals/files/setup_eval_env.py" ], "assertions": [ "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/mixed-thresholds; a plan alone does not pass.", @@ -51,10 +54,11 @@ }, { "id": "offline-empty-ranking", - "prompt": "Review my protein binder campaign in /workspace/input/no_survivors.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/no-survivors. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "prompt": "Review my protein binder campaign in /workspace/input/no_survivors.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/no-survivors. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "expected_output": "No design passes every applicable filter. candidates.csv has a header and zero data rows even though the synthetic control passes. Counts are n_candidates=4, n_passed=0, n_controls=1; all five entries remain in the audit and manifest. Pending lacks pLDDT and RMSD. The 0/4 fraction is provisional; thresholds are not relaxed to manufacture a shortlist.", "files": [ - "evals/files/no_survivors.json" + "evals/files/no_survivors.json", + "evals/files/setup_eval_env.py" ], "assertions": [ "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/no-survivors; a plan alone does not pass.", @@ -67,9 +71,10 @@ }, { "id": "offline-target-preparation", - "prompt": "Prepare the supplied target for a binder workflow from /workspace/input/target_prep/request.json. Follow /workspace/input/target_prep/CONTRACT.md and write the preparation artifacts under /workspace/output/target-prep. Resolve each requested hotspot handoff independently and report which can proceed. Work offline using these synthetic inputs; do not run inference.", + "prompt": "Prepare the supplied target for a binder workflow from /workspace/input/target_prep/request.json. Follow /workspace/input/target_prep/CONTRACT.md and write the preparation artifacts under /workspace/output/target-prep. Resolve each requested hotspot handoff independently and report which can proceed. Work offline using these synthetic inputs; do not run inference. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "files": [ - "evals/files/target_prep/" + "evals/files/target_prep/", + "evals/files/setup_eval_env.py" ], "expected_output": "Target chain E from model 1 gives AGST, author map 10:1,42:2,42A:3,77:4. Present hotspots produce E42,E77 and sequence indices 2,4. Missing author 43 blocks that request without a nearest-residue guess. Target PDB retains chain E and author numbering.", "assertions": [ @@ -83,10 +88,11 @@ }, { "id": "offline-refold-extraction", - "prompt": "Reconcile the saved refold results under /workspace/input/raw_refolds/ into two output runs /workspace/output/raw-refolds/boltz2 and /workspace/output/raw-refolds/openfold3. Follow /workspace/input/cache_contract.md, preserve the campaign policies, and report the resulting rankings and any blocked candidates. The responses are synthetic offline fixtures; make no model calls.", + "prompt": "Reconcile the saved refold results under /workspace/input/raw_refolds/ into two output runs /workspace/output/raw-refolds/boltz2 and /workspace/output/raw-refolds/openfold3. Follow /workspace/input/cache_contract.md, preserve the campaign policies, and report the resulting rankings and any blocked candidates. The responses are synthetic offline fixtures; make no model calls. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "files": [ "evals/files/raw_refolds/", - "evals/files/cache_contract.md" + "evals/files/cache_contract.md", + "evals/files/setup_eval_env.py" ], "expected_output": "Both rankings contain selected only. Boltz2 selected uses sample 1 confidence 0.81 and binder pLDDT 82; ipTM absent. OpenFold3 selected uses sample 1 ipTM 0.83 and binder pLDDT 86. poor_binder fails pLDDT 60 in both. Boltz2 incomplete is blocked for length mismatch; OpenFold3 composite_only is blocked for missing explicit ipTM. Counts 3/1/1 and 3/1/0, respectively; both fractions provisional.", "assertions": [ @@ -100,10 +106,11 @@ }, { "id": "offline-cache-resume", - "prompt": "Finish the recoverable offline work in /workspace/input/resume_cache/manifest.json and write the resumed run under /workspace/output/resumed. Use /workspace/input/cache_contract.md. Preserve the original run and its completed results and failure history, reconcile available cached responses, and replace stale exports. Report what remains blocked. All inputs are synthetic; do not submit inference or retries.", + "prompt": "Finish the recoverable offline work in /workspace/input/resume_cache/manifest.json and write the resumed run under /workspace/output/resumed. Use /workspace/input/cache_contract.md. Preserve the original run and its completed results and failure history, reconcile available cached responses, and replace stale exports. Report what remains blocked. All inputs are synthetic; do not submit inference or retries. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "files": [ "evals/files/resume_cache/", - "evals/files/cache_contract.md" + "evals/files/cache_contract.md", + "evals/files/setup_eval_env.py" ], "expected_output": "done and below are complete and unchanged; a finite low score is not pending. recoverable retains ipTM 0.86, gains binder pLDDT 84 and aligned RMSD approximately zero from cache. missing and failed stay blocked without invented metrics. Failure history and existing artifacts remain unchanged. Counts 5/2/1; ranked IDs done then recoverable; stale control/missing rows removed; 2/5 provisional.", "assertions": [ @@ -117,10 +124,11 @@ }, { "id": "offline-provenance-rejection", - "prompt": "Audit and reconcile /workspace/input/provenance_cache/manifest.json under /workspace/output/provenance. Follow /workspace/input/cache_contract.md. Decide which saved responses can supply scores to their candidates and produce the manifest, exports, summary and rejection reasons. Preserve the supplied evidence. These are synthetic offline fixtures; do not call models or repair provenance by guessing.", + "prompt": "Audit and reconcile /workspace/input/provenance_cache/manifest.json under /workspace/output/provenance. Follow /workspace/input/cache_contract.md. Decide which saved responses can supply scores to their candidates and produce the manifest, exports, summary and rejection reasons. Preserve the supplied evidence. These are synthetic offline fixtures; do not call models or repair provenance by guessing. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "files": [ "evals/files/provenance_cache/", - "evals/files/cache_contract.md" + "evals/files/cache_contract.md", + "evals/files/setup_eval_env.py" ], "expected_output": "Only valid is recovered and ranked, with boltz2_confidence 0.9 and binder_plddt 84. Reject candidate_swap, target_swap, sequence_swap, sample_swap and response_tampered for their respective candidate/target/sequence/selected-sample/file hash mismatch; leave all their scores empty and retain original evidence. Counts 6/1/0; 1/6 provisional; no ipTM invented.", "assertions": [ diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py new file mode 100644 index 00000000..a41d11ce --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Install the same isolated CPU environment before either evaluation condition. + +Usage: python3 /workspace/input/setup_eval_env.py +Arguments: none. Requires an installed Python 3.12+ with venv/ensurepip and index access. +Output: /workspace/pbd-eval-venv; setup diagnostics on stdout/stderr. +Exit codes: 0 for verified imports; 1 for failed setup. +""" + +from pathlib import Path +import shutil +import subprocess +import sys + +ENVIRONMENT = Path("/workspace/pbd-eval-venv") +DEPENDENCIES = ("numpy==2.5.1", "biotite==1.7.1") +IMPORT_CHECK = ( + "import numpy, biotite; " + "assert numpy.__version__ == '2.5.1'; " + "assert biotite.__version__ == '1.7.1'; " + "print('Evaluation imports verified:', numpy.__version__, biotite.__version__)" +) + + +def select_python(): + """Use an installed compatible interpreter without downloading Python.""" + if sys.version_info >= (3, 12): + return sys.executable + for name in ("python3.12", "python3.13", "python3.14"): + executable = shutil.which(name) + if executable: + result = subprocess.run( + [executable, "-c", "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)"], + shell=False, check=False, timeout=5, + ) + if result.returncode == 0: + return executable + raise RuntimeError("An installed Python 3.12+ is required for the pinned evaluation wheels") + + +def setup(environment=ENVIRONMENT): + """Create a writable venv; never modify the sandbox's system site-packages.""" + phase = "interpreter selection" + try: + executable = select_python() + print(f"Evaluation base interpreter: {executable}", flush=True) + phase = "virtual environment creation (requires venv/ensurepip)" + subprocess.run([executable, "-m", "venv", str(environment)], shell=False, check=True, timeout=30) + python = environment / "bin/python" + phase = "dependency installation" + subprocess.run( + [str(python), "-m", "pip", "install", "--disable-pip-version-check", + "--no-input", "--only-binary=:all:", "--timeout", "15", "--retries", "1", + *DEPENDENCIES], + shell=False, check=True, timeout=70, + ) + phase = "dependency import verification" + subprocess.run([str(python), "-c", IMPORT_CHECK], shell=False, check=True, timeout=10) + print(f"Use {python} for Python commands in this evaluation.", flush=True) + except (OSError, RuntimeError, subprocess.SubprocessError) as exc: + print(f"Evaluation setup failed during {phase}: {exc}", file=sys.stderr, flush=True) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(setup()) diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py new file mode 100644 index 00000000..65eb32c4 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Ensure failed dependency setup cannot be reported as a ready evaluation.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import subprocess +import unittest +from unittest.mock import patch + +SKILL = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("eval_setup", SKILL / "evals/files/setup_eval_env.py") +setup = importlib.util.module_from_spec(spec) +spec.loader.exec_module(setup) + + +class EvaluationEnvironmentTests(unittest.TestCase): + def test_install_failure_stops_before_import_check(self): + errors = io.StringIO() + with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=[None, subprocess.CalledProcessError(1, ["pip"])]) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(setup.setup(Path("/workspace/test-venv")), 1) + self.assertEqual(run.call_count, 2) + self.assertIn("dependency installation", errors.getvalue()) + + def test_missing_venv_support_fails_before_install(self): + errors = io.StringIO() + with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["venv"])) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(setup.setup(), 1) + run.assert_called_once() + self.assertIn("venv/ensurepip", errors.getvalue()) + + def test_old_default_python_uses_an_installed_compatible_one(self): + with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value="/opt/bin/python3.12"), patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)): + self.assertEqual(setup.select_python(), "/opt/bin/python3.12") + + def test_no_compatible_python_is_an_explicit_failure(self): + with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value=None), self.assertRaisesRegex(RuntimeError, "Python 3.12"): + setup.select_python() + + +if __name__ == "__main__": + unittest.main() diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md index 712d6e09..684550da 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md @@ -2,8 +2,9 @@ ## Active coverage: offline software workflows -`evals.json` runs eight offline exercises. The four original prompts, assertions, -expected outputs and score fixtures remain unchanged for historical comparison. +`evals.json` runs eight offline exercises. The original task requests, assertions, +expected outputs and score fixtures remain unchanged. Each prompt additionally +identifies the same prepared Python interpreter for both conditions. Four additional cases require target mapping, saved-response interpretation, recovery decisions and provenance checks. Their output contracts and all required inputs are staged in both conditions; equivalent implementations are accepted. @@ -20,19 +21,28 @@ inputs are staged in both conditions; equivalent implementations are accepted. | `offline-provenance-rejection` | `provenance_cache/`, `cache_contract.md` | Reject candidate, sequence, target, sample and response-file mismatches before scoring. | Every case stages identical inputs for the with/without-skill conditions. -`harbor.pre_agent_setup` in `config.yml` installs the same pinned NumPy 2.5.1 -and Biotite 1.7.1 wheels in both conditions, then checks their imports and versions -before the agent starts. Setup uses the sandbox's `python3` interpreter, which -also runs the workflow helpers, and needs permission to install its packages. +`harbor.pre_agent_setup` in `config.yml` runs the staged `setup_eval_env.py` in +both conditions. It selects an installed Python 3.12+, creates the writable +`/workspace/pbd-eval-venv` virtual environment, installs the same pinned NumPy +2.5.1 and Biotite 1.7.1 wheels, and checks their imports before the agent starts. +Every prompt specifies `/workspace/pbd-eval-venv/bin/python` for Python commands. +Setup leaves system Python packages untouched and needs no root privileges. There is no custom `evals/environment/Dockerfile`, so Astra uses its existing sandbox image without invoking the managed image builder or requiring its token. -The standard sandbox must provide Python 3.12+, pip, and package-index access +The standard sandbox must provide Python 3.12+ with venv/ensurepip, a writable +workspace, and package-index access for setup. Setup downloads packages; the subsequent exercises make no network calls. Normal sandbox and agent API access are still required. Installation or import failures stop environment preflight before agent trials. +Setup reports the chosen interpreter and the phase that failed. If preflight +fails, inspect its healthcheck stdout and +stderr to distinguish a missing pip module, an installation-permission error, +package-index access, an incompatible wheel, or an import error. The summary's +`HealthcheckError` alone does not identify which condition failed. Preserve this +diagnostic evidence before retrying; changing model settings cannot repair setup. These offline cases need no GPU, NIM credentials, or changes to the CI service. -The original prompts and fixtures are preserved; environment and verifier -changes should still be recorded when comparing with historical reports. +The task requests and fixtures are preserved; the interpreter instruction, +environment and verifier changes must be recorded in historical comparisons. The added prompts request campaign outputs without giving the expected order; the assertions check saved artifacts, not use of a particular helper. All sequences, toy coordinates, numeric inputs, saved responses and control labels are synthetic. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml index f6ee993e..b050442e 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/config.yml @@ -2,15 +2,10 @@ schema_version: 1 harbor: task_source: evals_json - # Use the existing sandbox image; install identical CPU dependencies in both - # conditions before the agent starts. + # Use the existing sandbox image; install identical CPU dependencies into a + # writable virtual environment in both conditions before the agent starts. pre_agent_setup: - - >- - python3 -m pip install --disable-pip-version-check --no-cache-dir - --only-binary=:all: --timeout 15 --retries 1 - numpy==2.5.1 biotite==1.7.1 - - >- - python3 -c "import numpy, biotite; assert numpy.__version__ == '2.5.1'; assert biotite.__version__ == '1.7.1'" + - python3 /workspace/input/setup_eval_env.py agents: claude-code: model: aws/anthropic/bedrock-claude-opus-5 diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/evals.json b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/evals.json index b8711b6f..7aad6167 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/evals.json +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/evals.json @@ -3,10 +3,11 @@ "evals": [ { "id": "offline-manifest", - "prompt": "Run an offline software bookkeeping exercise using /workspace/input/bookkeeping.json. All scores, controls, and thresholds in this file are synthetic; do not call model APIs or describe these numbers as scientific predictions. For each profile, create a separate run directory and manifest using the supplied target, filters, ranking metric, candidates, scores, and control flags. Use the bundled manifest helper if available, or implement equivalent bookkeeping. Save and reload each manifest, apply the supplied filters, exclude controls from rankings and success counts, and export both all_candidates.csv and ranked candidates.csv. Show the resulting counts and ranked IDs for each profile. Leave absent scores absent, including ipTM for the Boltz2 profile. Label all results as simulated bookkeeping, disclose the pending candidate, and explain that these results provide no live inference coverage.", + "prompt": "Run an offline software bookkeeping exercise using /workspace/input/bookkeeping.json. All scores, controls, and thresholds in this file are synthetic; do not call model APIs or describe these numbers as scientific predictions. For each profile, create a separate run directory and manifest using the supplied target, filters, ranking metric, candidates, scores, and control flags. Use the bundled manifest helper if available, or implement equivalent bookkeeping. Save and reload each manifest, apply the supplied filters, exclude controls from rankings and success counts, and export both all_candidates.csv and ranked candidates.csv. Show the resulting counts and ranked IDs for each profile. Leave absent scores absent, including ipTM for the Boltz2 profile. Label all results as simulated bookkeeping, disclose the pending candidate, and explain that these results provide no live inference coverage. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "expected_output": "Two saved and reloaded manifests and two CSVs per profile. Each profile has four non-control candidates, two passing candidates, and one control. Both rankings contain best then boundary. Below and pending fail; the control is excluded from rankings and counts. The Boltz2 profile retains boltz2_confidence without inventing ipTM. The simulated 2/4 passing fraction is provisional because pending lacks required scores. No model calls or scientific validation claims.", "files": [ - "evals/files/bookkeeping.json" + "evals/files/bookkeeping.json", + "evals/files/setup_eval_env.py" ], "assertions": [ "[executes-bookkeeping] Executes the bookkeeping and produces a manifest and both CSV artifacts per profile; a proposed plan alone does not pass.", @@ -19,10 +20,11 @@ }, { "id": "offline-boltz2-ranking", - "prompt": "Review my protein binder campaign in /workspace/input/boltz2_only.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/boltz2-only. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "prompt": "Review my protein binder campaign in /workspace/input/boltz2_only.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/boltz2-only. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "expected_output": "The supplied composite-confidence cutoff is 0.73; ipTM remains disabled and absent. Ranked IDs are binder_a then binder_b. Counts are n_candidates=4, n_passed=2, n_controls=1; the audit retains all five entries. The sequence-only control is exempt only from RMSD. Pending lacks RMSD and low_confidence fails. The 2/4 fraction is provisional and synthetic.", "files": [ - "evals/files/boltz2_only.json" + "evals/files/boltz2_only.json", + "evals/files/setup_eval_env.py" ], "assertions": [ "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/boltz2-only; a plan alone does not pass.", @@ -35,10 +37,11 @@ }, { "id": "offline-mixed-threshold-ranking", - "prompt": "Review my protein binder campaign in /workspace/input/mixed_thresholds.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/mixed-thresholds. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "prompt": "Review my protein binder campaign in /workspace/input/mixed_thresholds.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/mixed-thresholds. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "expected_output": "All supplied thresholds are enforced. Ranked IDs are binder_a then binder_b, despite higher ipTM on the control and designs failing other criteria. Counts are n_candidates=6, n_passed=2, n_controls=1; all seven entries remain in the audit. The string-valued interface score stays invalid rather than being coerced; pending lacks RMSD. Results are synthetic and 2/6 is provisional.", "files": [ - "evals/files/mixed_thresholds.json" + "evals/files/mixed_thresholds.json", + "evals/files/setup_eval_env.py" ], "assertions": [ "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/mixed-thresholds; a plan alone does not pass.", @@ -51,10 +54,11 @@ }, { "id": "offline-empty-ranking", - "prompt": "Review my protein binder campaign in /workspace/input/no_survivors.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/no-survivors. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim.", + "prompt": "Review my protein binder campaign in /workspace/input/no_survivors.json. Produce manifest.json, a complete all_candidates.csv audit, and ranked candidates.csv under /workspace/output/no-survivors. Preserve the supplied policy, scores, and control flags; save and reload the manifest before reporting counts and ranked IDs. All inputs are synthetic software fixtures. Work offline and make no live inference or scientific validation claim. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "expected_output": "No design passes every applicable filter. candidates.csv has a header and zero data rows even though the synthetic control passes. Counts are n_candidates=4, n_passed=0, n_controls=1; all five entries remain in the audit and manifest. Pending lacks pLDDT and RMSD. The 0/4 fraction is provisional; thresholds are not relaxed to manufacture a shortlist.", "files": [ - "evals/files/no_survivors.json" + "evals/files/no_survivors.json", + "evals/files/setup_eval_env.py" ], "assertions": [ "[executes-export] Creates and reloads the manifest and writes both CSV artifacts under /workspace/output/no-survivors; a plan alone does not pass.", @@ -67,9 +71,10 @@ }, { "id": "offline-target-preparation", - "prompt": "Prepare the supplied target for a binder workflow from /workspace/input/target_prep/request.json. Follow /workspace/input/target_prep/CONTRACT.md and write the preparation artifacts under /workspace/output/target-prep. Resolve each requested hotspot handoff independently and report which can proceed. Work offline using these synthetic inputs; do not run inference.", + "prompt": "Prepare the supplied target for a binder workflow from /workspace/input/target_prep/request.json. Follow /workspace/input/target_prep/CONTRACT.md and write the preparation artifacts under /workspace/output/target-prep. Resolve each requested hotspot handoff independently and report which can proceed. Work offline using these synthetic inputs; do not run inference. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "files": [ - "evals/files/target_prep/" + "evals/files/target_prep/", + "evals/files/setup_eval_env.py" ], "expected_output": "Target chain E from model 1 gives AGST, author map 10:1,42:2,42A:3,77:4. Present hotspots produce E42,E77 and sequence indices 2,4. Missing author 43 blocks that request without a nearest-residue guess. Target PDB retains chain E and author numbering.", "assertions": [ @@ -83,10 +88,11 @@ }, { "id": "offline-refold-extraction", - "prompt": "Reconcile the saved refold results under /workspace/input/raw_refolds/ into two output runs /workspace/output/raw-refolds/boltz2 and /workspace/output/raw-refolds/openfold3. Follow /workspace/input/cache_contract.md, preserve the campaign policies, and report the resulting rankings and any blocked candidates. The responses are synthetic offline fixtures; make no model calls.", + "prompt": "Reconcile the saved refold results under /workspace/input/raw_refolds/ into two output runs /workspace/output/raw-refolds/boltz2 and /workspace/output/raw-refolds/openfold3. Follow /workspace/input/cache_contract.md, preserve the campaign policies, and report the resulting rankings and any blocked candidates. The responses are synthetic offline fixtures; make no model calls. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "files": [ "evals/files/raw_refolds/", - "evals/files/cache_contract.md" + "evals/files/cache_contract.md", + "evals/files/setup_eval_env.py" ], "expected_output": "Both rankings contain selected only. Boltz2 selected uses sample 1 confidence 0.81 and binder pLDDT 82; ipTM absent. OpenFold3 selected uses sample 1 ipTM 0.83 and binder pLDDT 86. poor_binder fails pLDDT 60 in both. Boltz2 incomplete is blocked for length mismatch; OpenFold3 composite_only is blocked for missing explicit ipTM. Counts 3/1/1 and 3/1/0, respectively; both fractions provisional.", "assertions": [ @@ -100,10 +106,11 @@ }, { "id": "offline-cache-resume", - "prompt": "Finish the recoverable offline work in /workspace/input/resume_cache/manifest.json and write the resumed run under /workspace/output/resumed. Use /workspace/input/cache_contract.md. Preserve the original run and its completed results and failure history, reconcile available cached responses, and replace stale exports. Report what remains blocked. All inputs are synthetic; do not submit inference or retries.", + "prompt": "Finish the recoverable offline work in /workspace/input/resume_cache/manifest.json and write the resumed run under /workspace/output/resumed. Use /workspace/input/cache_contract.md. Preserve the original run and its completed results and failure history, reconcile available cached responses, and replace stale exports. Report what remains blocked. All inputs are synthetic; do not submit inference or retries. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "files": [ "evals/files/resume_cache/", - "evals/files/cache_contract.md" + "evals/files/cache_contract.md", + "evals/files/setup_eval_env.py" ], "expected_output": "done and below are complete and unchanged; a finite low score is not pending. recoverable retains ipTM 0.86, gains binder pLDDT 84 and aligned RMSD approximately zero from cache. missing and failed stay blocked without invented metrics. Failure history and existing artifacts remain unchanged. Counts 5/2/1; ranked IDs done then recoverable; stale control/missing rows removed; 2/5 provisional.", "assertions": [ @@ -117,10 +124,11 @@ }, { "id": "offline-provenance-rejection", - "prompt": "Audit and reconcile /workspace/input/provenance_cache/manifest.json under /workspace/output/provenance. Follow /workspace/input/cache_contract.md. Decide which saved responses can supply scores to their candidates and produce the manifest, exports, summary and rejection reasons. Preserve the supplied evidence. These are synthetic offline fixtures; do not call models or repair provenance by guessing.", + "prompt": "Audit and reconcile /workspace/input/provenance_cache/manifest.json under /workspace/output/provenance. Follow /workspace/input/cache_contract.md. Decide which saved responses can supply scores to their candidates and produce the manifest, exports, summary and rejection reasons. Preserve the supplied evidence. These are synthetic offline fixtures; do not call models or repair provenance by guessing. Evaluation runtime: use /workspace/pbd-eval-venv/bin/python for Python commands; the required NumPy and Biotite packages are already installed there.", "files": [ "evals/files/provenance_cache/", - "evals/files/cache_contract.md" + "evals/files/cache_contract.md", + "evals/files/setup_eval_env.py" ], "expected_output": "Only valid is recovered and ranked, with boltz2_confidence 0.9 and binder_plddt 84. Reject candidate_swap, target_swap, sequence_swap, sample_swap and response_tampered for their respective candidate/target/sequence/selected-sample/file hash mismatch; leave all their scores empty and retain original evidence. Counts 6/1/0; 1/6 provisional; no ipTM invented.", "assertions": [ diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py new file mode 100644 index 00000000..a41d11ce --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Install the same isolated CPU environment before either evaluation condition. + +Usage: python3 /workspace/input/setup_eval_env.py +Arguments: none. Requires an installed Python 3.12+ with venv/ensurepip and index access. +Output: /workspace/pbd-eval-venv; setup diagnostics on stdout/stderr. +Exit codes: 0 for verified imports; 1 for failed setup. +""" + +from pathlib import Path +import shutil +import subprocess +import sys + +ENVIRONMENT = Path("/workspace/pbd-eval-venv") +DEPENDENCIES = ("numpy==2.5.1", "biotite==1.7.1") +IMPORT_CHECK = ( + "import numpy, biotite; " + "assert numpy.__version__ == '2.5.1'; " + "assert biotite.__version__ == '1.7.1'; " + "print('Evaluation imports verified:', numpy.__version__, biotite.__version__)" +) + + +def select_python(): + """Use an installed compatible interpreter without downloading Python.""" + if sys.version_info >= (3, 12): + return sys.executable + for name in ("python3.12", "python3.13", "python3.14"): + executable = shutil.which(name) + if executable: + result = subprocess.run( + [executable, "-c", "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)"], + shell=False, check=False, timeout=5, + ) + if result.returncode == 0: + return executable + raise RuntimeError("An installed Python 3.12+ is required for the pinned evaluation wheels") + + +def setup(environment=ENVIRONMENT): + """Create a writable venv; never modify the sandbox's system site-packages.""" + phase = "interpreter selection" + try: + executable = select_python() + print(f"Evaluation base interpreter: {executable}", flush=True) + phase = "virtual environment creation (requires venv/ensurepip)" + subprocess.run([executable, "-m", "venv", str(environment)], shell=False, check=True, timeout=30) + python = environment / "bin/python" + phase = "dependency installation" + subprocess.run( + [str(python), "-m", "pip", "install", "--disable-pip-version-check", + "--no-input", "--only-binary=:all:", "--timeout", "15", "--retries", "1", + *DEPENDENCIES], + shell=False, check=True, timeout=70, + ) + phase = "dependency import verification" + subprocess.run([str(python), "-c", IMPORT_CHECK], shell=False, check=True, timeout=10) + print(f"Use {python} for Python commands in this evaluation.", flush=True) + except (OSError, RuntimeError, subprocess.SubprocessError) as exc: + print(f"Evaluation setup failed during {phase}: {exc}", file=sys.stderr, flush=True) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(setup()) diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py new file mode 100644 index 00000000..65eb32c4 --- /dev/null +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Ensure failed dependency setup cannot be reported as a ready evaluation.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import subprocess +import unittest +from unittest.mock import patch + +SKILL = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("eval_setup", SKILL / "evals/files/setup_eval_env.py") +setup = importlib.util.module_from_spec(spec) +spec.loader.exec_module(setup) + + +class EvaluationEnvironmentTests(unittest.TestCase): + def test_install_failure_stops_before_import_check(self): + errors = io.StringIO() + with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=[None, subprocess.CalledProcessError(1, ["pip"])]) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(setup.setup(Path("/workspace/test-venv")), 1) + self.assertEqual(run.call_count, 2) + self.assertIn("dependency installation", errors.getvalue()) + + def test_missing_venv_support_fails_before_install(self): + errors = io.StringIO() + with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["venv"])) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(setup.setup(), 1) + run.assert_called_once() + self.assertIn("venv/ensurepip", errors.getvalue()) + + def test_old_default_python_uses_an_installed_compatible_one(self): + with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value="/opt/bin/python3.12"), patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)): + self.assertEqual(setup.select_python(), "/opt/bin/python3.12") + + def test_no_compatible_python_is_an_explicit_failure(self): + with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value=None), self.assertRaisesRegex(RuntimeError, "Python 3.12"): + setup.select_python() + + +if __name__ == "__main__": + unittest.main() diff --git a/workflows/generative-protein-binder-design/complexa-binder-design/scripts/validate_binders.py b/workflows/generative-protein-binder-design/complexa-binder-design/scripts/validate_binders.py index 4903c168..d10f334b 100644 --- a/workflows/generative-protein-binder-design/complexa-binder-design/scripts/validate_binders.py +++ b/workflows/generative-protein-binder-design/complexa-binder-design/scripts/validate_binders.py @@ -308,7 +308,7 @@ def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--run-dir", required=True, type=Path) - ap.add_argument("--hotspots", type=Path, help="hotspots.json; default: unconditioned validation") + ap.add_argument("--hotspots", type=Path, help="hotspots.json (omit => unconditioned)") ap.add_argument("--target-chain", default="A") ap.add_argument("--binder-chain", default="B") ap.add_argument("--endpoint", choices=["hosted", "local"], default="hosted") From 8c5a306c3532271798d184a3eeade5df2fbe768a Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Tue, 13 Oct 2026 14:56:44 -0700 Subject: [PATCH 5/9] Preserve PBD Tier 3 setup logs and distinguish preflight failures Signed-off-by: Ohad Mosafi --- .../protein-binder-design/evals/README.md | 30 ++++-- .../evals/files/setup_eval_env.py | 62 +++++++++-- .../tests/test_eval_environment.py | 102 +++++++++++++++++- .../protein-binder-design/evals/README.md | 30 ++++-- .../evals/files/setup_eval_env.py | 62 +++++++++-- .../tests/test_eval_environment.py | 102 +++++++++++++++++- 6 files changed, 348 insertions(+), 40 deletions(-) diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md index 684550da..8164a95a 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md @@ -34,12 +34,30 @@ workspace, and package-index access for setup. Setup downloads packages; the subsequent exercises make no network calls. Normal sandbox and agent API access are still required. Installation or import failures stop environment preflight before agent trials. -Setup reports the chosen interpreter and the phase that failed. If preflight -fails, inspect its healthcheck stdout and -stderr to distinguish a missing pip module, an installation-permission error, -package-index access, an incompatible wheel, or an import error. The summary's -`HealthcheckError` alone does not identify which condition failed. Preserve this -diagnostic evidence before retrying; changing model settings cannot repair setup. +Setup reports the chosen interpreter and the phase that failed. Harbor may omit +healthcheck stdout/stderr from its job trace, but it logs the command's exit code +as `Healthcheck failed (rc=...)`. The setup helper uses distinct failure codes: + +| Exit code | Failed phase | Check | +| --- | --- | --- | +| 10 | Interpreter selection | An installed Python 3.12+ is discoverable. | +| 11 | Virtual environment creation | venv/ensurepip is installed and the workspace is writable. | +| 12 | Dependency installation | pip can reach the package index and install the pinned wheels within the timeout. | +| 13 | Import verification | The installed NumPy and Biotite versions import successfully. | + +All four failures stop preflight; only verified imports return zero. These codes +identify the failing phase, not the underlying cause. The setup helper saves the +bootstrap interpreter version, phase messages, and child-process stdout/stderr +to `/logs/artifacts/pbd-eval-setup.log`. Harbor collects that directory into the +trial's `artifacts/` directory, including on preflight failure. Inspect this file +under the retained `_harbor-preflight/environment-preflight-*//artifacts/` +directory to distinguish permissions, index access, wheel compatibility and +import errors. The CI service must retain these Harbor artifacts separately from +the summary report; the summary does not embed the setup transcript. If artifact +storage is unwritable, setup still runs with console diagnostics and the same +failure codes. A generic `rc=1` may mean the helper did not +start or an unexpected error occurred; do not infer a dependency failure from +that code alone. Changing model settings cannot repair environment setup. These offline cases need no GPU, NIM credentials, or changes to the CI service. The task requests and fixtures are preserved; the interpreter instruction, environment and verifier changes must be recorded in historical comparisons. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py index a41d11ce..fd8b0495 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py @@ -4,17 +4,24 @@ Usage: python3 /workspace/input/setup_eval_env.py Arguments: none. Requires an installed Python 3.12+ with venv/ensurepip and index access. -Output: /workspace/pbd-eval-venv; setup diagnostics on stdout/stderr. -Exit codes: 0 for verified imports; 1 for failed setup. +Output: /workspace/pbd-eval-venv; diagnostics in /logs/artifacts/pbd-eval-setup.log. +Exit codes: 0 ready; 10 interpreter; 11 venv; 12 installation; 13 imports. """ +from contextlib import ExitStack from pathlib import Path import shutil import subprocess import sys ENVIRONMENT = Path("/workspace/pbd-eval-venv") +SETUP_LOG = Path("/logs/artifacts/pbd-eval-setup.log") DEPENDENCIES = ("numpy==2.5.1", "biotite==1.7.1") +# Harbor logs the exit code even when healthcheck stdout/stderr is unavailable. +EXIT_INTERPRETER = 10 +EXIT_VENV = 11 +EXIT_INSTALL = 12 +EXIT_IMPORT = 13 IMPORT_CHECK = ( "import numpy, biotite; " "assert numpy.__version__ == '2.5.1'; " @@ -39,30 +46,65 @@ def select_python(): raise RuntimeError("An installed Python 3.12+ is required for the pinned evaluation wheels") -def setup(environment=ENVIRONMENT): +def setup(environment=ENVIRONMENT, log=None): """Create a writable venv; never modify the sandbox's system site-packages.""" + def report(message, *, error=False): + print(message, file=sys.stderr if error else sys.stdout, flush=True) + if log is not None: + try: + print(message, file=log, flush=True) + except OSError as exc: + print(f"Could not write setup diagnostic: {exc}", file=sys.stderr, flush=True) + phase = "interpreter selection" + exit_code = EXIT_INTERPRETER try: + report(f"Bootstrap interpreter: {sys.executable} (Python {sys.version.split()[0]})") executable = select_python() - print(f"Evaluation base interpreter: {executable}", flush=True) + report(f"Evaluation base interpreter: {executable}") phase = "virtual environment creation (requires venv/ensurepip)" - subprocess.run([executable, "-m", "venv", str(environment)], shell=False, check=True, timeout=30) + exit_code = EXIT_VENV + report(f"Starting {phase}") + subprocess.run( + [executable, "-m", "venv", str(environment)], shell=False, check=True, timeout=30, + stdout=log, stderr=subprocess.STDOUT if log is not None else None, + ) python = environment / "bin/python" phase = "dependency installation" + exit_code = EXIT_INSTALL + report(f"Starting {phase}") subprocess.run( [str(python), "-m", "pip", "install", "--disable-pip-version-check", "--no-input", "--only-binary=:all:", "--timeout", "15", "--retries", "1", *DEPENDENCIES], shell=False, check=True, timeout=70, + stdout=log, stderr=subprocess.STDOUT if log is not None else None, ) phase = "dependency import verification" - subprocess.run([str(python), "-c", IMPORT_CHECK], shell=False, check=True, timeout=10) - print(f"Use {python} for Python commands in this evaluation.", flush=True) + exit_code = EXIT_IMPORT + report(f"Starting {phase}") + subprocess.run( + [str(python), "-c", IMPORT_CHECK], shell=False, check=True, timeout=10, + stdout=log, stderr=subprocess.STDOUT if log is not None else None, + ) + report(f"Use {python} for Python commands in this evaluation.") except (OSError, RuntimeError, subprocess.SubprocessError) as exc: - print(f"Evaluation setup failed during {phase}: {exc}", file=sys.stderr, flush=True) - return 1 + report(f"Evaluation setup failed during {phase}: {exc}", error=True) + return exit_code return 0 +def main(): + """Retain subprocess output even when Harbor omits healthcheck output.""" + with ExitStack() as stack: + try: + SETUP_LOG.parent.mkdir(parents=True, exist_ok=True) + log = stack.enter_context(SETUP_LOG.open("w", encoding="utf-8")) + except OSError as exc: + print(f"Could not create setup log {SETUP_LOG}: {exc}", file=sys.stderr, flush=True) + log = None + return setup(log=log) + + if __name__ == "__main__": - sys.exit(setup()) + sys.exit(main()) diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py index 65eb32c4..4983a612 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py @@ -7,11 +7,13 @@ import io from pathlib import Path import subprocess +import sys +import tempfile import unittest from unittest.mock import patch -SKILL = Path(__file__).resolve().parents[1] -spec = importlib.util.spec_from_file_location("eval_setup", SKILL / "evals/files/setup_eval_env.py") +SOURCE_ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("eval_setup", SOURCE_ROOT / "evals/files/setup_eval_env.py") setup = importlib.util.module_from_spec(spec) spec.loader.exec_module(setup) @@ -20,17 +22,51 @@ class EvaluationEnvironmentTests(unittest.TestCase): def test_install_failure_stops_before_import_check(self): errors = io.StringIO() with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=[None, subprocess.CalledProcessError(1, ["pip"])]) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): - self.assertEqual(setup.setup(Path("/workspace/test-venv")), 1) + self.assertEqual(setup.setup(Path("/workspace/test-venv")), setup.EXIT_INSTALL) self.assertEqual(run.call_count, 2) self.assertIn("dependency installation", errors.getvalue()) def test_missing_venv_support_fails_before_install(self): errors = io.StringIO() with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["venv"])) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): - self.assertEqual(setup.setup(), 1) + self.assertEqual(setup.setup(), setup.EXIT_VENV) run.assert_called_once() self.assertIn("venv/ensurepip", errors.getvalue()) + def test_interpreter_failure_is_distinct_and_runs_no_install_commands(self): + errors = io.StringIO() + with ( + patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.12+ unavailable")), + patch.object(setup.subprocess, "run") as run, + contextlib.redirect_stderr(errors), + ): + self.assertEqual(setup.setup(), setup.EXIT_INTERPRETER) + run.assert_not_called() + self.assertIn("interpreter selection", errors.getvalue()) + + def test_import_failure_is_distinct_from_installation_failure(self): + errors, output = io.StringIO(), io.StringIO() + with ( + patch.object(setup, "select_python", return_value="/python3.12"), + patch.object(setup.subprocess, "run", side_effect=[None, None, subprocess.CalledProcessError(1, ["python"])]) as run, + contextlib.redirect_stderr(errors), + contextlib.redirect_stdout(output), + ): + self.assertEqual(setup.setup(), setup.EXIT_IMPORT) + self.assertEqual(run.call_count, 3) + self.assertIn("dependency import verification", errors.getvalue()) + self.assertNotIn("for Python commands", output.getvalue()) + + def test_success_requires_all_setup_phases(self): + with ( + patch.object(setup, "select_python", return_value="/python3.12"), + patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run, + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(setup.setup(), 0) + self.assertEqual(run.call_count, 3) + self.assertEqual(run.call_args.args[0][-1], setup.IMPORT_CHECK) + def test_old_default_python_uses_an_installed_compatible_one(self): with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value="/opt/bin/python3.12"), patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)): self.assertEqual(setup.select_python(), "/opt/bin/python3.12") @@ -39,6 +75,64 @@ def test_no_compatible_python_is_an_explicit_failure(self): with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value=None), self.assertRaisesRegex(RuntimeError, "Python 3.12"): setup.select_python() + def test_failed_install_retains_real_child_stdout_and_stderr(self): + run_process = subprocess.run + + def simulate_install(command, **kwargs): + if command[1:3] == ["-m", "venv"]: + return subprocess.CompletedProcess(command, 0) + return run_process( + [sys.executable, "-c", "import sys; print('package download started'); " + "print('package index unavailable', file=sys.stderr); sys.exit(1)"], + **kwargs, + ) + + with tempfile.TemporaryDirectory() as directory: + log_path = Path(directory) / "artifacts/pbd-eval-setup.log" + with ( + patch.object(setup, "SETUP_LOG", log_path), + patch.object(setup, "select_python", return_value=sys.executable), + patch.object(setup.subprocess, "run", side_effect=simulate_install) as run, + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(setup.main(), setup.EXIT_INSTALL) + transcript = log_path.read_text() + self.assertEqual(run.call_count, 2) + self.assertIn("Bootstrap interpreter:", transcript) + self.assertIn("package download started", transcript) + self.assertIn("package index unavailable", transcript) + self.assertIn("Evaluation setup failed during dependency installation", transcript) + self.assertNotIn("Starting dependency import verification", transcript) + + def test_missing_interpreter_reason_is_retained_without_starting_subprocesses(self): + with tempfile.TemporaryDirectory() as directory: + log_path = Path(directory) / "setup.log" + with ( + patch.object(setup, "SETUP_LOG", log_path), + patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.12+ unavailable")), + patch.object(setup.subprocess, "run") as run, + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(setup.main(), setup.EXIT_INTERPRETER) + self.assertIn("Python 3.12+ unavailable", log_path.read_text()) + run.assert_not_called() + + def test_unwritable_log_does_not_replace_the_setup_failure(self): + with tempfile.TemporaryDirectory() as directory: + occupied_path = Path(directory) / "file" + occupied_path.write_text("already a file") + errors = io.StringIO() + with ( + patch.object(setup, "SETUP_LOG", occupied_path / "setup.log"), + patch.object(setup, "setup", return_value=setup.EXIT_VENV) as run_setup, + contextlib.redirect_stderr(errors), + ): + self.assertEqual(setup.main(), setup.EXIT_VENV) + run_setup.assert_called_once_with(log=None) + self.assertIn("Could not create setup log", errors.getvalue()) + if __name__ == "__main__": unittest.main() diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md index 684550da..8164a95a 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md @@ -34,12 +34,30 @@ workspace, and package-index access for setup. Setup downloads packages; the subsequent exercises make no network calls. Normal sandbox and agent API access are still required. Installation or import failures stop environment preflight before agent trials. -Setup reports the chosen interpreter and the phase that failed. If preflight -fails, inspect its healthcheck stdout and -stderr to distinguish a missing pip module, an installation-permission error, -package-index access, an incompatible wheel, or an import error. The summary's -`HealthcheckError` alone does not identify which condition failed. Preserve this -diagnostic evidence before retrying; changing model settings cannot repair setup. +Setup reports the chosen interpreter and the phase that failed. Harbor may omit +healthcheck stdout/stderr from its job trace, but it logs the command's exit code +as `Healthcheck failed (rc=...)`. The setup helper uses distinct failure codes: + +| Exit code | Failed phase | Check | +| --- | --- | --- | +| 10 | Interpreter selection | An installed Python 3.12+ is discoverable. | +| 11 | Virtual environment creation | venv/ensurepip is installed and the workspace is writable. | +| 12 | Dependency installation | pip can reach the package index and install the pinned wheels within the timeout. | +| 13 | Import verification | The installed NumPy and Biotite versions import successfully. | + +All four failures stop preflight; only verified imports return zero. These codes +identify the failing phase, not the underlying cause. The setup helper saves the +bootstrap interpreter version, phase messages, and child-process stdout/stderr +to `/logs/artifacts/pbd-eval-setup.log`. Harbor collects that directory into the +trial's `artifacts/` directory, including on preflight failure. Inspect this file +under the retained `_harbor-preflight/environment-preflight-*//artifacts/` +directory to distinguish permissions, index access, wheel compatibility and +import errors. The CI service must retain these Harbor artifacts separately from +the summary report; the summary does not embed the setup transcript. If artifact +storage is unwritable, setup still runs with console diagnostics and the same +failure codes. A generic `rc=1` may mean the helper did not +start or an unexpected error occurred; do not infer a dependency failure from +that code alone. Changing model settings cannot repair environment setup. These offline cases need no GPU, NIM credentials, or changes to the CI service. The task requests and fixtures are preserved; the interpreter instruction, environment and verifier changes must be recorded in historical comparisons. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py index a41d11ce..fd8b0495 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py @@ -4,17 +4,24 @@ Usage: python3 /workspace/input/setup_eval_env.py Arguments: none. Requires an installed Python 3.12+ with venv/ensurepip and index access. -Output: /workspace/pbd-eval-venv; setup diagnostics on stdout/stderr. -Exit codes: 0 for verified imports; 1 for failed setup. +Output: /workspace/pbd-eval-venv; diagnostics in /logs/artifacts/pbd-eval-setup.log. +Exit codes: 0 ready; 10 interpreter; 11 venv; 12 installation; 13 imports. """ +from contextlib import ExitStack from pathlib import Path import shutil import subprocess import sys ENVIRONMENT = Path("/workspace/pbd-eval-venv") +SETUP_LOG = Path("/logs/artifacts/pbd-eval-setup.log") DEPENDENCIES = ("numpy==2.5.1", "biotite==1.7.1") +# Harbor logs the exit code even when healthcheck stdout/stderr is unavailable. +EXIT_INTERPRETER = 10 +EXIT_VENV = 11 +EXIT_INSTALL = 12 +EXIT_IMPORT = 13 IMPORT_CHECK = ( "import numpy, biotite; " "assert numpy.__version__ == '2.5.1'; " @@ -39,30 +46,65 @@ def select_python(): raise RuntimeError("An installed Python 3.12+ is required for the pinned evaluation wheels") -def setup(environment=ENVIRONMENT): +def setup(environment=ENVIRONMENT, log=None): """Create a writable venv; never modify the sandbox's system site-packages.""" + def report(message, *, error=False): + print(message, file=sys.stderr if error else sys.stdout, flush=True) + if log is not None: + try: + print(message, file=log, flush=True) + except OSError as exc: + print(f"Could not write setup diagnostic: {exc}", file=sys.stderr, flush=True) + phase = "interpreter selection" + exit_code = EXIT_INTERPRETER try: + report(f"Bootstrap interpreter: {sys.executable} (Python {sys.version.split()[0]})") executable = select_python() - print(f"Evaluation base interpreter: {executable}", flush=True) + report(f"Evaluation base interpreter: {executable}") phase = "virtual environment creation (requires venv/ensurepip)" - subprocess.run([executable, "-m", "venv", str(environment)], shell=False, check=True, timeout=30) + exit_code = EXIT_VENV + report(f"Starting {phase}") + subprocess.run( + [executable, "-m", "venv", str(environment)], shell=False, check=True, timeout=30, + stdout=log, stderr=subprocess.STDOUT if log is not None else None, + ) python = environment / "bin/python" phase = "dependency installation" + exit_code = EXIT_INSTALL + report(f"Starting {phase}") subprocess.run( [str(python), "-m", "pip", "install", "--disable-pip-version-check", "--no-input", "--only-binary=:all:", "--timeout", "15", "--retries", "1", *DEPENDENCIES], shell=False, check=True, timeout=70, + stdout=log, stderr=subprocess.STDOUT if log is not None else None, ) phase = "dependency import verification" - subprocess.run([str(python), "-c", IMPORT_CHECK], shell=False, check=True, timeout=10) - print(f"Use {python} for Python commands in this evaluation.", flush=True) + exit_code = EXIT_IMPORT + report(f"Starting {phase}") + subprocess.run( + [str(python), "-c", IMPORT_CHECK], shell=False, check=True, timeout=10, + stdout=log, stderr=subprocess.STDOUT if log is not None else None, + ) + report(f"Use {python} for Python commands in this evaluation.") except (OSError, RuntimeError, subprocess.SubprocessError) as exc: - print(f"Evaluation setup failed during {phase}: {exc}", file=sys.stderr, flush=True) - return 1 + report(f"Evaluation setup failed during {phase}: {exc}", error=True) + return exit_code return 0 +def main(): + """Retain subprocess output even when Harbor omits healthcheck output.""" + with ExitStack() as stack: + try: + SETUP_LOG.parent.mkdir(parents=True, exist_ok=True) + log = stack.enter_context(SETUP_LOG.open("w", encoding="utf-8")) + except OSError as exc: + print(f"Could not create setup log {SETUP_LOG}: {exc}", file=sys.stderr, flush=True) + log = None + return setup(log=log) + + if __name__ == "__main__": - sys.exit(setup()) + sys.exit(main()) diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py index 65eb32c4..4983a612 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py @@ -7,11 +7,13 @@ import io from pathlib import Path import subprocess +import sys +import tempfile import unittest from unittest.mock import patch -SKILL = Path(__file__).resolve().parents[1] -spec = importlib.util.spec_from_file_location("eval_setup", SKILL / "evals/files/setup_eval_env.py") +SOURCE_ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("eval_setup", SOURCE_ROOT / "evals/files/setup_eval_env.py") setup = importlib.util.module_from_spec(spec) spec.loader.exec_module(setup) @@ -20,17 +22,51 @@ class EvaluationEnvironmentTests(unittest.TestCase): def test_install_failure_stops_before_import_check(self): errors = io.StringIO() with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=[None, subprocess.CalledProcessError(1, ["pip"])]) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): - self.assertEqual(setup.setup(Path("/workspace/test-venv")), 1) + self.assertEqual(setup.setup(Path("/workspace/test-venv")), setup.EXIT_INSTALL) self.assertEqual(run.call_count, 2) self.assertIn("dependency installation", errors.getvalue()) def test_missing_venv_support_fails_before_install(self): errors = io.StringIO() with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["venv"])) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): - self.assertEqual(setup.setup(), 1) + self.assertEqual(setup.setup(), setup.EXIT_VENV) run.assert_called_once() self.assertIn("venv/ensurepip", errors.getvalue()) + def test_interpreter_failure_is_distinct_and_runs_no_install_commands(self): + errors = io.StringIO() + with ( + patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.12+ unavailable")), + patch.object(setup.subprocess, "run") as run, + contextlib.redirect_stderr(errors), + ): + self.assertEqual(setup.setup(), setup.EXIT_INTERPRETER) + run.assert_not_called() + self.assertIn("interpreter selection", errors.getvalue()) + + def test_import_failure_is_distinct_from_installation_failure(self): + errors, output = io.StringIO(), io.StringIO() + with ( + patch.object(setup, "select_python", return_value="/python3.12"), + patch.object(setup.subprocess, "run", side_effect=[None, None, subprocess.CalledProcessError(1, ["python"])]) as run, + contextlib.redirect_stderr(errors), + contextlib.redirect_stdout(output), + ): + self.assertEqual(setup.setup(), setup.EXIT_IMPORT) + self.assertEqual(run.call_count, 3) + self.assertIn("dependency import verification", errors.getvalue()) + self.assertNotIn("for Python commands", output.getvalue()) + + def test_success_requires_all_setup_phases(self): + with ( + patch.object(setup, "select_python", return_value="/python3.12"), + patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run, + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(setup.setup(), 0) + self.assertEqual(run.call_count, 3) + self.assertEqual(run.call_args.args[0][-1], setup.IMPORT_CHECK) + def test_old_default_python_uses_an_installed_compatible_one(self): with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value="/opt/bin/python3.12"), patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)): self.assertEqual(setup.select_python(), "/opt/bin/python3.12") @@ -39,6 +75,64 @@ def test_no_compatible_python_is_an_explicit_failure(self): with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value=None), self.assertRaisesRegex(RuntimeError, "Python 3.12"): setup.select_python() + def test_failed_install_retains_real_child_stdout_and_stderr(self): + run_process = subprocess.run + + def simulate_install(command, **kwargs): + if command[1:3] == ["-m", "venv"]: + return subprocess.CompletedProcess(command, 0) + return run_process( + [sys.executable, "-c", "import sys; print('package download started'); " + "print('package index unavailable', file=sys.stderr); sys.exit(1)"], + **kwargs, + ) + + with tempfile.TemporaryDirectory() as directory: + log_path = Path(directory) / "artifacts/pbd-eval-setup.log" + with ( + patch.object(setup, "SETUP_LOG", log_path), + patch.object(setup, "select_python", return_value=sys.executable), + patch.object(setup.subprocess, "run", side_effect=simulate_install) as run, + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(setup.main(), setup.EXIT_INSTALL) + transcript = log_path.read_text() + self.assertEqual(run.call_count, 2) + self.assertIn("Bootstrap interpreter:", transcript) + self.assertIn("package download started", transcript) + self.assertIn("package index unavailable", transcript) + self.assertIn("Evaluation setup failed during dependency installation", transcript) + self.assertNotIn("Starting dependency import verification", transcript) + + def test_missing_interpreter_reason_is_retained_without_starting_subprocesses(self): + with tempfile.TemporaryDirectory() as directory: + log_path = Path(directory) / "setup.log" + with ( + patch.object(setup, "SETUP_LOG", log_path), + patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.12+ unavailable")), + patch.object(setup.subprocess, "run") as run, + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(setup.main(), setup.EXIT_INTERPRETER) + self.assertIn("Python 3.12+ unavailable", log_path.read_text()) + run.assert_not_called() + + def test_unwritable_log_does_not_replace_the_setup_failure(self): + with tempfile.TemporaryDirectory() as directory: + occupied_path = Path(directory) / "file" + occupied_path.write_text("already a file") + errors = io.StringIO() + with ( + patch.object(setup, "SETUP_LOG", occupied_path / "setup.log"), + patch.object(setup, "setup", return_value=setup.EXIT_VENV) as run_setup, + contextlib.redirect_stderr(errors), + ): + self.assertEqual(setup.main(), setup.EXIT_VENV) + run_setup.assert_called_once_with(log=None) + self.assertIn("Could not create setup log", errors.getvalue()) + if __name__ == "__main__": unittest.main() From 27b19c6a1dd423361bd2221aeddc42a17a937834 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Tue, 13 Oct 2026 23:05:17 +0000 Subject: [PATCH 6/9] Support managed sandbox Python versions in PBD evaluations --- .../skills/protein-binder-design/README.md | 2 +- .../skills/protein-binder-design/SKILL.md | 2 +- .../protein-binder-design/evals/README.md | 12 ++++--- .../evals/files/setup_eval_env.py | 26 ++++++++------ .../protein-binder-design/requirements.txt | 6 ++-- .../tests/test_eval_environment.py | 34 ++++++++++++++++--- .../protein-binder-design/README.md | 2 +- .../protein-binder-design/SKILL.md | 2 +- .../protein-binder-design/evals/README.md | 12 ++++--- .../evals/files/setup_eval_env.py | 26 ++++++++------ .../protein-binder-design/requirements.txt | 6 ++-- .../tests/test_eval_environment.py | 34 ++++++++++++++++--- 12 files changed, 114 insertions(+), 50 deletions(-) diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md index 150d2c6b..fc26a2a6 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/README.md @@ -30,7 +30,7 @@ protein-binder-design/ - Access to the BioNeMo NIMs you intend to use (RFdiffusion, ProteinMPNN, Boltz2 and/or OpenFold3, optional MSA-Search) — hosted at [build.nvidia.com](https://build.nvidia.com) or self-hosted via NGC. -- Python ≥ 3.12 for the pinned helper environment. Install dependencies with +- Python 3.10-3.13 for the pinned helper environment. Install dependencies with `python3 -m pip install -r requirements.txt`; mmCIF parsing and cached refold recovery require Biotite. Manifest and PDB-only helpers use the standard library. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md index 4a212df6..0b54f619 100755 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md @@ -28,7 +28,7 @@ design campaign. Offline bookkeeping stays offline. ## Prerequisites -- Python 3.12+; install [requirements.txt](requirements.txt) for RMSD, mmCIF, +- Python 3.10-3.13; install [requirements.txt](requirements.txt) for RMSD, mmCIF, cache recovery, and Python HTTP examples. Manifest and PDB-only helpers use the standard library. Install [requirements-dev.txt](requirements-dev.txt) to run `python3 -m pytest tests` with coverage. diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md index 8164a95a..892e30ad 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md @@ -22,14 +22,18 @@ inputs are staged in both conditions; equivalent implementations are accepted. Every case stages identical inputs for the with/without-skill conditions. `harbor.pre_agent_setup` in `config.yml` runs the staged `setup_eval_env.py` in -both conditions. It selects an installed Python 3.12+, creates the writable +both conditions. It selects an installed Python 3.10-3.13, creates the writable `/workspace/pbd-eval-venv` virtual environment, installs the same pinned NumPy -2.5.1 and Biotite 1.7.1 wheels, and checks their imports before the agent starts. +2.2.6 and Biotite 1.2.0 wheels, and checks their imports before the agent starts. +These releases provide wheels for Python 3.10 and 3.11 as well as 3.12 and 3.13; +the previous NumPy 2.5.1/Biotite 1.7.1 pins required Python 3.12+. Managed sandbox +images may provide older Python versions, so requiring 3.12 prevented preflight +from reaching dependency installation. The same pins apply to both conditions. Every prompt specifies `/workspace/pbd-eval-venv/bin/python` for Python commands. Setup leaves system Python packages untouched and needs no root privileges. There is no custom `evals/environment/Dockerfile`, so Astra uses its existing sandbox image without invoking the managed image builder or requiring its token. -The standard sandbox must provide Python 3.12+ with venv/ensurepip, a writable +The standard sandbox must provide Python 3.10-3.13 with venv/ensurepip, a writable workspace, and package-index access for setup. Setup downloads packages; the subsequent exercises make no network calls. Normal sandbox and agent API access are still required. @@ -40,7 +44,7 @@ as `Healthcheck failed (rc=...)`. The setup helper uses distinct failure codes: | Exit code | Failed phase | Check | | --- | --- | --- | -| 10 | Interpreter selection | An installed Python 3.12+ is discoverable. | +| 10 | Interpreter selection | An installed Python 3.10-3.13 is discoverable. | | 11 | Virtual environment creation | venv/ensurepip is installed and the workspace is writable. | | 12 | Dependency installation | pip can reach the package index and install the pinned wheels within the timeout. | | 13 | Import verification | The installed NumPy and Biotite versions import successfully. | diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py index fd8b0495..82c37048 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py @@ -3,7 +3,7 @@ """Install the same isolated CPU environment before either evaluation condition. Usage: python3 /workspace/input/setup_eval_env.py -Arguments: none. Requires an installed Python 3.12+ with venv/ensurepip and index access. +Arguments: none. Requires Python 3.10-3.13 with venv/ensurepip and index access. Output: /workspace/pbd-eval-venv; diagnostics in /logs/artifacts/pbd-eval-setup.log. Exit codes: 0 ready; 10 interpreter; 11 venv; 12 installation; 13 imports. """ @@ -16,7 +16,8 @@ ENVIRONMENT = Path("/workspace/pbd-eval-venv") SETUP_LOG = Path("/logs/artifacts/pbd-eval-setup.log") -DEPENDENCIES = ("numpy==2.5.1", "biotite==1.7.1") +# Both wheels support the older Python interpreters in managed sandboxes. +DEPENDENCIES = ("numpy==2.2.6", "biotite==1.2.0") # Harbor logs the exit code even when healthcheck stdout/stderr is unavailable. EXIT_INTERPRETER = 10 EXIT_VENV = 11 @@ -24,26 +25,29 @@ EXIT_IMPORT = 13 IMPORT_CHECK = ( "import numpy, biotite; " - "assert numpy.__version__ == '2.5.1'; " - "assert biotite.__version__ == '1.7.1'; " + "assert numpy.__version__ == '2.2.6'; " + "assert biotite.__version__ == '1.2.0'; " "print('Evaluation imports verified:', numpy.__version__, biotite.__version__)" ) def select_python(): """Use an installed compatible interpreter without downloading Python.""" - if sys.version_info >= (3, 12): + if (3, 10) <= sys.version_info[:2] < (3, 14): return sys.executable - for name in ("python3.12", "python3.13", "python3.14"): + for name in ("python3.13", "python3.12", "python3.11", "python3.10"): executable = shutil.which(name) if executable: - result = subprocess.run( - [executable, "-c", "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)"], - shell=False, check=False, timeout=5, - ) + try: + result = subprocess.run( + [executable, "-c", "import sys; sys.exit(0 if (3, 10) <= sys.version_info[:2] < (3, 14) else 1)"], + shell=False, check=False, timeout=5, + ) + except (OSError, subprocess.SubprocessError): + continue if result.returncode == 0: return executable - raise RuntimeError("An installed Python 3.12+ is required for the pinned evaluation wheels") + raise RuntimeError("An installed Python 3.10-3.13 is required for the pinned evaluation wheels") def setup(environment=ENVIRONMENT, log=None): diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt b/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt index 3bf0c707..5c52b877 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/requirements.txt @@ -1,4 +1,4 @@ -# Tested helper environment (Python 3.12+); NumPy/Biotite match Tier 3 pre-agent setup. -numpy==2.5.1 -biotite==1.7.1 +# Tested helper environment (Python 3.10-3.13); matches Tier 3 pre-agent setup. +numpy==2.2.6 +biotite==1.2.0 requests==2.34.2 diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py index 4983a612..9acfa175 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py @@ -36,7 +36,7 @@ def test_missing_venv_support_fails_before_install(self): def test_interpreter_failure_is_distinct_and_runs_no_install_commands(self): errors = io.StringIO() with ( - patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.12+ unavailable")), + patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.10-3.13 unavailable")), patch.object(setup.subprocess, "run") as run, contextlib.redirect_stderr(errors), ): @@ -67,12 +67,36 @@ def test_success_requires_all_setup_phases(self): self.assertEqual(run.call_count, 3) self.assertEqual(run.call_args.args[0][-1], setup.IMPORT_CHECK) + def test_supported_default_python_does_not_require_a_newer_interpreter(self): + for version in ((3, 10), (3, 11), (3, 12), (3, 13)): + with ( + self.subTest(version=version), + patch.object(setup.sys, "version_info", version), + patch.object(setup.shutil, "which") as find_python, + patch.object(setup.subprocess, "run") as run, + ): + self.assertEqual(setup.select_python(), sys.executable) + find_python.assert_not_called() + run.assert_not_called() + def test_old_default_python_uses_an_installed_compatible_one(self): - with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value="/opt/bin/python3.12"), patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)): + with patch.object(setup.sys, "version_info", (3, 9)), patch.object(setup.shutil, "which", return_value="/opt/bin/python3.12"), patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)): + self.assertEqual(setup.select_python(), "/opt/bin/python3.12") + + def test_newer_python_without_pinned_wheels_uses_a_supported_interpreter(self): + with patch.object(setup.sys, "version_info", (3, 14)), patch.object(setup.shutil, "which", return_value="/opt/bin/python3.13"), patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)): + self.assertEqual(setup.select_python(), "/opt/bin/python3.13") + + def test_broken_candidate_does_not_hide_an_available_interpreter(self): + with ( + patch.object(setup.sys, "version_info", (3, 9)), + patch.object(setup.shutil, "which", side_effect=["/broken/python3.13", "/opt/bin/python3.12"]), + patch.object(setup.subprocess, "run", side_effect=[OSError("unavailable"), subprocess.CompletedProcess([], 0)]), + ): self.assertEqual(setup.select_python(), "/opt/bin/python3.12") def test_no_compatible_python_is_an_explicit_failure(self): - with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value=None), self.assertRaisesRegex(RuntimeError, "Python 3.12"): + with patch.object(setup.sys, "version_info", (3, 9)), patch.object(setup.shutil, "which", return_value=None), self.assertRaisesRegex(RuntimeError, "Python 3.10-3.13"): setup.select_python() def test_failed_install_retains_real_child_stdout_and_stderr(self): @@ -110,13 +134,13 @@ def test_missing_interpreter_reason_is_retained_without_starting_subprocesses(se log_path = Path(directory) / "setup.log" with ( patch.object(setup, "SETUP_LOG", log_path), - patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.12+ unavailable")), + patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.10-3.13 unavailable")), patch.object(setup.subprocess, "run") as run, contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()), ): self.assertEqual(setup.main(), setup.EXIT_INTERPRETER) - self.assertIn("Python 3.12+ unavailable", log_path.read_text()) + self.assertIn("Python 3.10-3.13 unavailable", log_path.read_text()) run.assert_not_called() def test_unwritable_log_does_not_replace_the_setup_failure(self): diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/README.md index 150d2c6b..fc26a2a6 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/README.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/README.md @@ -30,7 +30,7 @@ protein-binder-design/ - Access to the BioNeMo NIMs you intend to use (RFdiffusion, ProteinMPNN, Boltz2 and/or OpenFold3, optional MSA-Search) — hosted at [build.nvidia.com](https://build.nvidia.com) or self-hosted via NGC. -- Python ≥ 3.12 for the pinned helper environment. Install dependencies with +- Python 3.10-3.13 for the pinned helper environment. Install dependencies with `python3 -m pip install -r requirements.txt`; mmCIF parsing and cached refold recovery require Biotite. Manifest and PDB-only helpers use the standard library. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md index 4a212df6..0b54f619 100755 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md @@ -28,7 +28,7 @@ design campaign. Offline bookkeeping stays offline. ## Prerequisites -- Python 3.12+; install [requirements.txt](requirements.txt) for RMSD, mmCIF, +- Python 3.10-3.13; install [requirements.txt](requirements.txt) for RMSD, mmCIF, cache recovery, and Python HTTP examples. Manifest and PDB-only helpers use the standard library. Install [requirements-dev.txt](requirements-dev.txt) to run `python3 -m pytest tests` with coverage. diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md index 8164a95a..892e30ad 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md @@ -22,14 +22,18 @@ inputs are staged in both conditions; equivalent implementations are accepted. Every case stages identical inputs for the with/without-skill conditions. `harbor.pre_agent_setup` in `config.yml` runs the staged `setup_eval_env.py` in -both conditions. It selects an installed Python 3.12+, creates the writable +both conditions. It selects an installed Python 3.10-3.13, creates the writable `/workspace/pbd-eval-venv` virtual environment, installs the same pinned NumPy -2.5.1 and Biotite 1.7.1 wheels, and checks their imports before the agent starts. +2.2.6 and Biotite 1.2.0 wheels, and checks their imports before the agent starts. +These releases provide wheels for Python 3.10 and 3.11 as well as 3.12 and 3.13; +the previous NumPy 2.5.1/Biotite 1.7.1 pins required Python 3.12+. Managed sandbox +images may provide older Python versions, so requiring 3.12 prevented preflight +from reaching dependency installation. The same pins apply to both conditions. Every prompt specifies `/workspace/pbd-eval-venv/bin/python` for Python commands. Setup leaves system Python packages untouched and needs no root privileges. There is no custom `evals/environment/Dockerfile`, so Astra uses its existing sandbox image without invoking the managed image builder or requiring its token. -The standard sandbox must provide Python 3.12+ with venv/ensurepip, a writable +The standard sandbox must provide Python 3.10-3.13 with venv/ensurepip, a writable workspace, and package-index access for setup. Setup downloads packages; the subsequent exercises make no network calls. Normal sandbox and agent API access are still required. @@ -40,7 +44,7 @@ as `Healthcheck failed (rc=...)`. The setup helper uses distinct failure codes: | Exit code | Failed phase | Check | | --- | --- | --- | -| 10 | Interpreter selection | An installed Python 3.12+ is discoverable. | +| 10 | Interpreter selection | An installed Python 3.10-3.13 is discoverable. | | 11 | Virtual environment creation | venv/ensurepip is installed and the workspace is writable. | | 12 | Dependency installation | pip can reach the package index and install the pinned wheels within the timeout. | | 13 | Import verification | The installed NumPy and Biotite versions import successfully. | diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py index fd8b0495..82c37048 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py @@ -3,7 +3,7 @@ """Install the same isolated CPU environment before either evaluation condition. Usage: python3 /workspace/input/setup_eval_env.py -Arguments: none. Requires an installed Python 3.12+ with venv/ensurepip and index access. +Arguments: none. Requires Python 3.10-3.13 with venv/ensurepip and index access. Output: /workspace/pbd-eval-venv; diagnostics in /logs/artifacts/pbd-eval-setup.log. Exit codes: 0 ready; 10 interpreter; 11 venv; 12 installation; 13 imports. """ @@ -16,7 +16,8 @@ ENVIRONMENT = Path("/workspace/pbd-eval-venv") SETUP_LOG = Path("/logs/artifacts/pbd-eval-setup.log") -DEPENDENCIES = ("numpy==2.5.1", "biotite==1.7.1") +# Both wheels support the older Python interpreters in managed sandboxes. +DEPENDENCIES = ("numpy==2.2.6", "biotite==1.2.0") # Harbor logs the exit code even when healthcheck stdout/stderr is unavailable. EXIT_INTERPRETER = 10 EXIT_VENV = 11 @@ -24,26 +25,29 @@ EXIT_IMPORT = 13 IMPORT_CHECK = ( "import numpy, biotite; " - "assert numpy.__version__ == '2.5.1'; " - "assert biotite.__version__ == '1.7.1'; " + "assert numpy.__version__ == '2.2.6'; " + "assert biotite.__version__ == '1.2.0'; " "print('Evaluation imports verified:', numpy.__version__, biotite.__version__)" ) def select_python(): """Use an installed compatible interpreter without downloading Python.""" - if sys.version_info >= (3, 12): + if (3, 10) <= sys.version_info[:2] < (3, 14): return sys.executable - for name in ("python3.12", "python3.13", "python3.14"): + for name in ("python3.13", "python3.12", "python3.11", "python3.10"): executable = shutil.which(name) if executable: - result = subprocess.run( - [executable, "-c", "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)"], - shell=False, check=False, timeout=5, - ) + try: + result = subprocess.run( + [executable, "-c", "import sys; sys.exit(0 if (3, 10) <= sys.version_info[:2] < (3, 14) else 1)"], + shell=False, check=False, timeout=5, + ) + except (OSError, subprocess.SubprocessError): + continue if result.returncode == 0: return executable - raise RuntimeError("An installed Python 3.12+ is required for the pinned evaluation wheels") + raise RuntimeError("An installed Python 3.10-3.13 is required for the pinned evaluation wheels") def setup(environment=ENVIRONMENT, log=None): diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt b/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt index 3bf0c707..5c52b877 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/requirements.txt @@ -1,4 +1,4 @@ -# Tested helper environment (Python 3.12+); NumPy/Biotite match Tier 3 pre-agent setup. -numpy==2.5.1 -biotite==1.7.1 +# Tested helper environment (Python 3.10-3.13); matches Tier 3 pre-agent setup. +numpy==2.2.6 +biotite==1.2.0 requests==2.34.2 diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py index 4983a612..9acfa175 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py @@ -36,7 +36,7 @@ def test_missing_venv_support_fails_before_install(self): def test_interpreter_failure_is_distinct_and_runs_no_install_commands(self): errors = io.StringIO() with ( - patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.12+ unavailable")), + patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.10-3.13 unavailable")), patch.object(setup.subprocess, "run") as run, contextlib.redirect_stderr(errors), ): @@ -67,12 +67,36 @@ def test_success_requires_all_setup_phases(self): self.assertEqual(run.call_count, 3) self.assertEqual(run.call_args.args[0][-1], setup.IMPORT_CHECK) + def test_supported_default_python_does_not_require_a_newer_interpreter(self): + for version in ((3, 10), (3, 11), (3, 12), (3, 13)): + with ( + self.subTest(version=version), + patch.object(setup.sys, "version_info", version), + patch.object(setup.shutil, "which") as find_python, + patch.object(setup.subprocess, "run") as run, + ): + self.assertEqual(setup.select_python(), sys.executable) + find_python.assert_not_called() + run.assert_not_called() + def test_old_default_python_uses_an_installed_compatible_one(self): - with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value="/opt/bin/python3.12"), patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)): + with patch.object(setup.sys, "version_info", (3, 9)), patch.object(setup.shutil, "which", return_value="/opt/bin/python3.12"), patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)): + self.assertEqual(setup.select_python(), "/opt/bin/python3.12") + + def test_newer_python_without_pinned_wheels_uses_a_supported_interpreter(self): + with patch.object(setup.sys, "version_info", (3, 14)), patch.object(setup.shutil, "which", return_value="/opt/bin/python3.13"), patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)): + self.assertEqual(setup.select_python(), "/opt/bin/python3.13") + + def test_broken_candidate_does_not_hide_an_available_interpreter(self): + with ( + patch.object(setup.sys, "version_info", (3, 9)), + patch.object(setup.shutil, "which", side_effect=["/broken/python3.13", "/opt/bin/python3.12"]), + patch.object(setup.subprocess, "run", side_effect=[OSError("unavailable"), subprocess.CompletedProcess([], 0)]), + ): self.assertEqual(setup.select_python(), "/opt/bin/python3.12") def test_no_compatible_python_is_an_explicit_failure(self): - with patch.object(setup.sys, "version_info", (3, 10)), patch.object(setup.shutil, "which", return_value=None), self.assertRaisesRegex(RuntimeError, "Python 3.12"): + with patch.object(setup.sys, "version_info", (3, 9)), patch.object(setup.shutil, "which", return_value=None), self.assertRaisesRegex(RuntimeError, "Python 3.10-3.13"): setup.select_python() def test_failed_install_retains_real_child_stdout_and_stderr(self): @@ -110,13 +134,13 @@ def test_missing_interpreter_reason_is_retained_without_starting_subprocesses(se log_path = Path(directory) / "setup.log" with ( patch.object(setup, "SETUP_LOG", log_path), - patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.12+ unavailable")), + patch.object(setup, "select_python", side_effect=RuntimeError("Python 3.10-3.13 unavailable")), patch.object(setup.subprocess, "run") as run, contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()), ): self.assertEqual(setup.main(), setup.EXIT_INTERPRETER) - self.assertIn("Python 3.12+ unavailable", log_path.read_text()) + self.assertIn("Python 3.10-3.13 unavailable", log_path.read_text()) run.assert_not_called() def test_unwritable_log_does_not_replace_the_setup_failure(self): From 21958b741db27d2e3e4b62240f2df1d964a4aedc Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Wed, 14 Oct 2026 10:33:11 -0700 Subject: [PATCH 7/9] Use installed uv for PBD evaluation environment setup Signed-off-by: Ohad Mosafi --- .../protein-binder-design/evals/README.md | 16 +++-- .../evals/files/setup_eval_env.py | 31 +++++++--- .../tests/test_eval_environment.py | 58 ++++++++++++++++++- .../protein-binder-design/evals/README.md | 16 +++-- .../evals/files/setup_eval_env.py | 31 +++++++--- .../tests/test_eval_environment.py | 58 ++++++++++++++++++- 6 files changed, 182 insertions(+), 28 deletions(-) diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md index 892e30ad..67f259d0 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md @@ -31,11 +31,17 @@ images may provide older Python versions, so requiring 3.12 prevented preflight from reaching dependency installation. The same pins apply to both conditions. Every prompt specifies `/workspace/pbd-eval-venv/bin/python` for Python commands. Setup leaves system Python packages untouched and needs no root privileges. +When `uv` is already installed on `PATH`, setup uses it to create the environment +and install wheels without requiring Python's `venv`, `ensurepip`, or `pip` +modules. It selects the installed interpreter explicitly and disables Python +downloads. Without `uv`, setup uses `python -m venv` and the environment's pip. +Both paths use the same package pins and import checks, bounded network retries, +and subprocess timeouts. Setup does not download or install `uv`. There is no custom `evals/environment/Dockerfile`, so Astra uses its existing sandbox image without invoking the managed image builder or requiring its token. -The standard sandbox must provide Python 3.10-3.13 with venv/ensurepip, a writable -workspace, and package-index access -for setup. Setup downloads packages; the subsequent exercises +The standard sandbox must provide Python 3.10-3.13 plus either `uv` or +`venv`/`ensurepip`, a writable workspace, and package-index access for setup. +Setup downloads packages; the subsequent exercises make no network calls. Normal sandbox and agent API access are still required. Installation or import failures stop environment preflight before agent trials. Setup reports the chosen interpreter and the phase that failed. Harbor may omit @@ -45,8 +51,8 @@ as `Healthcheck failed (rc=...)`. The setup helper uses distinct failure codes: | Exit code | Failed phase | Check | | --- | --- | --- | | 10 | Interpreter selection | An installed Python 3.10-3.13 is discoverable. | -| 11 | Virtual environment creation | venv/ensurepip is installed and the workspace is writable. | -| 12 | Dependency installation | pip can reach the package index and install the pinned wheels within the timeout. | +| 11 | Virtual environment creation | The workspace is writable; either uv is on PATH or Python has venv/ensurepip. | +| 12 | Dependency installation | The selected installer can reach the package index and install the pinned wheels within the timeout. | | 13 | Import verification | The installed NumPy and Biotite versions import successfully. | All four failures stop preflight; only verified imports return zero. These codes diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py index 82c37048..a12ea5f4 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py @@ -3,13 +3,14 @@ """Install the same isolated CPU environment before either evaluation condition. Usage: python3 /workspace/input/setup_eval_env.py -Arguments: none. Requires Python 3.10-3.13 with venv/ensurepip and index access. +Arguments: none. Requires Python 3.10-3.13, uv or venv/ensurepip, and index access. Output: /workspace/pbd-eval-venv; diagnostics in /logs/artifacts/pbd-eval-setup.log. Exit codes: 0 ready; 10 interpreter; 11 venv; 12 installation; 13 imports. """ from contextlib import ExitStack from pathlib import Path +import os import shutil import subprocess import sys @@ -66,22 +67,36 @@ def report(message, *, error=False): report(f"Bootstrap interpreter: {sys.executable} (Python {sys.version.split()[0]})") executable = select_python() report(f"Evaluation base interpreter: {executable}") - phase = "virtual environment creation (requires venv/ensurepip)" + python = environment / "bin/python" + uv = shutil.which("uv") + if uv: + report(f"Environment installer: {uv} (does not require venv/ensurepip)") + create = [uv, "venv", "--python", executable, "--no-python-downloads", + "--no-config", "--no-cache", "--allow-existing", str(environment)] + install = [uv, "pip", "install", "--python", str(python), + "--no-python-downloads", "--no-config", "--no-cache", + "--only-binary=:all:", *DEPENDENCIES] + process_env = {**os.environ, "UV_HTTP_TIMEOUT": "15", "UV_HTTP_RETRIES": "1", + "UV_NO_PROGRESS": "1"} + else: + report("Environment installer: Python venv/ensurepip and pip (uv not installed)") + create = [executable, "-m", "venv", str(environment)] + install = [str(python), "-m", "pip", "install", "--disable-pip-version-check", + "--no-input", "--only-binary=:all:", "--timeout", "15", "--retries", "1", + *DEPENDENCIES] + process_env = None + phase = "virtual environment creation" exit_code = EXIT_VENV report(f"Starting {phase}") subprocess.run( - [executable, "-m", "venv", str(environment)], shell=False, check=True, timeout=30, + create, shell=False, check=True, timeout=30, env=process_env, stdout=log, stderr=subprocess.STDOUT if log is not None else None, ) - python = environment / "bin/python" phase = "dependency installation" exit_code = EXIT_INSTALL report(f"Starting {phase}") subprocess.run( - [str(python), "-m", "pip", "install", "--disable-pip-version-check", - "--no-input", "--only-binary=:all:", "--timeout", "15", "--retries", "1", - *DEPENDENCIES], - shell=False, check=True, timeout=70, + install, shell=False, check=True, timeout=70, env=process_env, stdout=log, stderr=subprocess.STDOUT if log is not None else None, ) phase = "dependency import verification" diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py index 9acfa175..ba0bca90 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py @@ -19,6 +19,12 @@ class EvaluationEnvironmentTests(unittest.TestCase): + def setUp(self): + # Exercise the stdlib fallback independently of tools on the test host. + self.find_tool = patch.object(setup.shutil, "which", return_value=None) + self.find_tool.start() + self.addCleanup(self.find_tool.stop) + def test_install_failure_stops_before_import_check(self): errors = io.StringIO() with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=[None, subprocess.CalledProcessError(1, ["pip"])]) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): @@ -31,7 +37,57 @@ def test_missing_venv_support_fails_before_install(self): with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["venv"])) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): self.assertEqual(setup.setup(), setup.EXIT_VENV) run.assert_called_once() - self.assertIn("venv/ensurepip", errors.getvalue()) + self.assertIn("virtual environment creation", errors.getvalue()) + + def test_uv_installs_into_the_isolated_environment_without_python_bootstrap(self): + environment = Path("/workspace/test-venv") + python = str(environment / "bin/python") + with ( + patch.object(setup, "select_python", return_value="/python3.11"), + patch.object(setup.shutil, "which", return_value="/usr/local/bin/uv"), + patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run, + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(setup.setup(environment), 0) + create, install, verify = run.call_args_list + self.assertEqual(create.args[0][0:2], ["/usr/local/bin/uv", "venv"]) + self.assertEqual(create.args[0][-1], str(environment)) + self.assertEqual(install.args[0][0:5], ["/usr/local/bin/uv", "pip", "install", "--python", python]) + for command in (create, install): + self.assertIn("--no-python-downloads", command.args[0]) + self.assertIn("--no-cache", command.args[0]) + self.assertFalse(command.kwargs["shell"]) + self.assertEqual(command.kwargs["env"]["UV_HTTP_RETRIES"], "1") + self.assertEqual(command.kwargs["env"]["UV_HTTP_TIMEOUT"], "15") + self.assertIn("--only-binary=:all:", install.args[0]) + self.assertEqual(install.args[0][-2:], list(setup.DEPENDENCIES)) + self.assertEqual(verify.args[0], [python, "-c", setup.IMPORT_CHECK]) + + def test_uv_creation_failure_stops_before_install(self): + errors = io.StringIO() + with ( + patch.object(setup, "select_python", return_value="/python3.11"), + patch.object(setup.shutil, "which", return_value="/usr/local/bin/uv"), + patch.object(setup.subprocess, "run", side_effect=PermissionError("workspace is not writable")) as run, + contextlib.redirect_stderr(errors), + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(setup.setup(), setup.EXIT_VENV) + run.assert_called_once() + self.assertIn("workspace is not writable", errors.getvalue()) + + def test_uv_install_failure_stops_before_import_check(self): + output = io.StringIO() + with ( + patch.object(setup, "select_python", return_value="/python3.11"), + patch.object(setup.shutil, "which", return_value="/usr/local/bin/uv"), + patch.object(setup.subprocess, "run", side_effect=[None, subprocess.CalledProcessError(1, ["uv", "pip"])]) as run, + contextlib.redirect_stdout(output), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(setup.setup(), setup.EXIT_INSTALL) + self.assertEqual(run.call_count, 2) + self.assertNotIn("Starting dependency import verification", output.getvalue()) def test_interpreter_failure_is_distinct_and_runs_no_install_commands(self): errors = io.StringIO() diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md index 892e30ad..67f259d0 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md @@ -31,11 +31,17 @@ images may provide older Python versions, so requiring 3.12 prevented preflight from reaching dependency installation. The same pins apply to both conditions. Every prompt specifies `/workspace/pbd-eval-venv/bin/python` for Python commands. Setup leaves system Python packages untouched and needs no root privileges. +When `uv` is already installed on `PATH`, setup uses it to create the environment +and install wheels without requiring Python's `venv`, `ensurepip`, or `pip` +modules. It selects the installed interpreter explicitly and disables Python +downloads. Without `uv`, setup uses `python -m venv` and the environment's pip. +Both paths use the same package pins and import checks, bounded network retries, +and subprocess timeouts. Setup does not download or install `uv`. There is no custom `evals/environment/Dockerfile`, so Astra uses its existing sandbox image without invoking the managed image builder or requiring its token. -The standard sandbox must provide Python 3.10-3.13 with venv/ensurepip, a writable -workspace, and package-index access -for setup. Setup downloads packages; the subsequent exercises +The standard sandbox must provide Python 3.10-3.13 plus either `uv` or +`venv`/`ensurepip`, a writable workspace, and package-index access for setup. +Setup downloads packages; the subsequent exercises make no network calls. Normal sandbox and agent API access are still required. Installation or import failures stop environment preflight before agent trials. Setup reports the chosen interpreter and the phase that failed. Harbor may omit @@ -45,8 +51,8 @@ as `Healthcheck failed (rc=...)`. The setup helper uses distinct failure codes: | Exit code | Failed phase | Check | | --- | --- | --- | | 10 | Interpreter selection | An installed Python 3.10-3.13 is discoverable. | -| 11 | Virtual environment creation | venv/ensurepip is installed and the workspace is writable. | -| 12 | Dependency installation | pip can reach the package index and install the pinned wheels within the timeout. | +| 11 | Virtual environment creation | The workspace is writable; either uv is on PATH or Python has venv/ensurepip. | +| 12 | Dependency installation | The selected installer can reach the package index and install the pinned wheels within the timeout. | | 13 | Import verification | The installed NumPy and Biotite versions import successfully. | All four failures stop preflight; only verified imports return zero. These codes diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py index 82c37048..a12ea5f4 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py @@ -3,13 +3,14 @@ """Install the same isolated CPU environment before either evaluation condition. Usage: python3 /workspace/input/setup_eval_env.py -Arguments: none. Requires Python 3.10-3.13 with venv/ensurepip and index access. +Arguments: none. Requires Python 3.10-3.13, uv or venv/ensurepip, and index access. Output: /workspace/pbd-eval-venv; diagnostics in /logs/artifacts/pbd-eval-setup.log. Exit codes: 0 ready; 10 interpreter; 11 venv; 12 installation; 13 imports. """ from contextlib import ExitStack from pathlib import Path +import os import shutil import subprocess import sys @@ -66,22 +67,36 @@ def report(message, *, error=False): report(f"Bootstrap interpreter: {sys.executable} (Python {sys.version.split()[0]})") executable = select_python() report(f"Evaluation base interpreter: {executable}") - phase = "virtual environment creation (requires venv/ensurepip)" + python = environment / "bin/python" + uv = shutil.which("uv") + if uv: + report(f"Environment installer: {uv} (does not require venv/ensurepip)") + create = [uv, "venv", "--python", executable, "--no-python-downloads", + "--no-config", "--no-cache", "--allow-existing", str(environment)] + install = [uv, "pip", "install", "--python", str(python), + "--no-python-downloads", "--no-config", "--no-cache", + "--only-binary=:all:", *DEPENDENCIES] + process_env = {**os.environ, "UV_HTTP_TIMEOUT": "15", "UV_HTTP_RETRIES": "1", + "UV_NO_PROGRESS": "1"} + else: + report("Environment installer: Python venv/ensurepip and pip (uv not installed)") + create = [executable, "-m", "venv", str(environment)] + install = [str(python), "-m", "pip", "install", "--disable-pip-version-check", + "--no-input", "--only-binary=:all:", "--timeout", "15", "--retries", "1", + *DEPENDENCIES] + process_env = None + phase = "virtual environment creation" exit_code = EXIT_VENV report(f"Starting {phase}") subprocess.run( - [executable, "-m", "venv", str(environment)], shell=False, check=True, timeout=30, + create, shell=False, check=True, timeout=30, env=process_env, stdout=log, stderr=subprocess.STDOUT if log is not None else None, ) - python = environment / "bin/python" phase = "dependency installation" exit_code = EXIT_INSTALL report(f"Starting {phase}") subprocess.run( - [str(python), "-m", "pip", "install", "--disable-pip-version-check", - "--no-input", "--only-binary=:all:", "--timeout", "15", "--retries", "1", - *DEPENDENCIES], - shell=False, check=True, timeout=70, + install, shell=False, check=True, timeout=70, env=process_env, stdout=log, stderr=subprocess.STDOUT if log is not None else None, ) phase = "dependency import verification" diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py index 9acfa175..ba0bca90 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py @@ -19,6 +19,12 @@ class EvaluationEnvironmentTests(unittest.TestCase): + def setUp(self): + # Exercise the stdlib fallback independently of tools on the test host. + self.find_tool = patch.object(setup.shutil, "which", return_value=None) + self.find_tool.start() + self.addCleanup(self.find_tool.stop) + def test_install_failure_stops_before_import_check(self): errors = io.StringIO() with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=[None, subprocess.CalledProcessError(1, ["pip"])]) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): @@ -31,7 +37,57 @@ def test_missing_venv_support_fails_before_install(self): with patch.object(setup, "select_python", return_value="/python3.12"), patch.object(setup.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["venv"])) as run, contextlib.redirect_stderr(errors), contextlib.redirect_stdout(io.StringIO()): self.assertEqual(setup.setup(), setup.EXIT_VENV) run.assert_called_once() - self.assertIn("venv/ensurepip", errors.getvalue()) + self.assertIn("virtual environment creation", errors.getvalue()) + + def test_uv_installs_into_the_isolated_environment_without_python_bootstrap(self): + environment = Path("/workspace/test-venv") + python = str(environment / "bin/python") + with ( + patch.object(setup, "select_python", return_value="/python3.11"), + patch.object(setup.shutil, "which", return_value="/usr/local/bin/uv"), + patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run, + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(setup.setup(environment), 0) + create, install, verify = run.call_args_list + self.assertEqual(create.args[0][0:2], ["/usr/local/bin/uv", "venv"]) + self.assertEqual(create.args[0][-1], str(environment)) + self.assertEqual(install.args[0][0:5], ["/usr/local/bin/uv", "pip", "install", "--python", python]) + for command in (create, install): + self.assertIn("--no-python-downloads", command.args[0]) + self.assertIn("--no-cache", command.args[0]) + self.assertFalse(command.kwargs["shell"]) + self.assertEqual(command.kwargs["env"]["UV_HTTP_RETRIES"], "1") + self.assertEqual(command.kwargs["env"]["UV_HTTP_TIMEOUT"], "15") + self.assertIn("--only-binary=:all:", install.args[0]) + self.assertEqual(install.args[0][-2:], list(setup.DEPENDENCIES)) + self.assertEqual(verify.args[0], [python, "-c", setup.IMPORT_CHECK]) + + def test_uv_creation_failure_stops_before_install(self): + errors = io.StringIO() + with ( + patch.object(setup, "select_python", return_value="/python3.11"), + patch.object(setup.shutil, "which", return_value="/usr/local/bin/uv"), + patch.object(setup.subprocess, "run", side_effect=PermissionError("workspace is not writable")) as run, + contextlib.redirect_stderr(errors), + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(setup.setup(), setup.EXIT_VENV) + run.assert_called_once() + self.assertIn("workspace is not writable", errors.getvalue()) + + def test_uv_install_failure_stops_before_import_check(self): + output = io.StringIO() + with ( + patch.object(setup, "select_python", return_value="/python3.11"), + patch.object(setup.shutil, "which", return_value="/usr/local/bin/uv"), + patch.object(setup.subprocess, "run", side_effect=[None, subprocess.CalledProcessError(1, ["uv", "pip"])]) as run, + contextlib.redirect_stdout(output), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(setup.setup(), setup.EXIT_INSTALL) + self.assertEqual(run.call_count, 2) + self.assertNotIn("Starting dependency import verification", output.getvalue()) def test_interpreter_failure_is_distinct_and_runs_no_install_commands(self): errors = io.StringIO() From c016c3690dccb9ee6f634bc3f433398bafde5a51 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Tue, 27 Oct 2026 10:33:06 -0700 Subject: [PATCH 8/9] Fix PBD evaluation setup in pipless sandboxes Signed-off-by: Ohad Mosafi --- .../skills/protein-binder-design/SKILL.md | 2 +- .../protein-binder-design/evals/README.md | 29 +++-- .../evals/files/setup_eval_env.py | 90 +++++++++++--- .../tests/test_eval_environment.py | 111 ++++++++++++++++++ .../protein-binder-design/SKILL.md | 2 +- .../protein-binder-design/evals/README.md | 29 +++-- .../evals/files/setup_eval_env.py | 90 +++++++++++--- .../tests/test_eval_environment.py | 111 ++++++++++++++++++ 8 files changed, 416 insertions(+), 48 deletions(-) diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md index 0b54f619..bcc7b378 100755 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md @@ -3,7 +3,7 @@ name: protein-binder-design description: > Use when designing RFdiffusion/ProteinMPNN binders or validating and resuming binder campaigns. Not for code or report review. license: Apache-2.0 -compatibility: "Python>=3.12 with requirements.txt; Bash for hosted_env.sh" +compatibility: "Python 3.10-3.13 with requirements.txt; Bash for hosted_env.sh" metadata: author: "NVIDIA BioNeMo " tags: [protein-design, binder-design, bionemo] diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md index 67f259d0..04d59a89 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/README.md @@ -34,13 +34,21 @@ Setup leaves system Python packages untouched and needs no root privileges. When `uv` is already installed on `PATH`, setup uses it to create the environment and install wheels without requiring Python's `venv`, `ensurepip`, or `pip` modules. It selects the installed interpreter explicitly and disables Python -downloads. Without `uv`, setup uses `python -m venv` and the environment's pip. -Both paths use the same package pins and import checks, bounded network retries, -and subprocess timeouts. Setup does not download or install `uv`. +downloads. Without `uv`, setup uses `python -m venv --without-pip`, then downloads +the universal pip 25.3 wheel from its fixed PyPI URL. It verifies the published +SHA-256 digest and a size limit before running the wheel with the virtual +environment's Python. This works when the sandbox omits both `ensurepip` and +`pip`; it does not install either into system Python. Installer files are +temporary and removed on success or failure. Both paths use the same package +pins and import checks, bounded network retries, and subprocess timeouts. +The installer subprocesses receive an explicit list of path, proxy, certificate, +and package-index settings; agent/model credentials and `PYTHONPATH` are not +forwarded. Setup does not download or install `uv`. There is no custom `evals/environment/Dockerfile`, so Astra uses its existing sandbox image without invoking the managed image builder or requiring its token. The standard sandbox must provide Python 3.10-3.13 plus either `uv` or -`venv`/`ensurepip`, a writable workspace, and package-index access for setup. +`venv`, a writable workspace, and package-index access for setup. The pip +fallback also needs HTTPS access to `files.pythonhosted.org`. Setup downloads packages; the subsequent exercises make no network calls. Normal sandbox and agent API access are still required. Installation or import failures stop environment preflight before agent trials. @@ -51,11 +59,12 @@ as `Healthcheck failed (rc=...)`. The setup helper uses distinct failure codes: | Exit code | Failed phase | Check | | --- | --- | --- | | 10 | Interpreter selection | An installed Python 3.10-3.13 is discoverable. | -| 11 | Virtual environment creation | The workspace is writable; either uv is on PATH or Python has venv/ensurepip. | -| 12 | Dependency installation | The selected installer can reach the package index and install the pinned wheels within the timeout. | +| 11 | Virtual environment creation | The workspace is writable; either uv is on PATH or Python has venv. | +| 12 | Installer bootstrap or dependency installation | The pip wheel passes its digest check, and the selected installer can reach the package index and install the pinned wheels within the timeout. | | 13 | Import verification | The installed NumPy and Biotite versions import successfully. | -All four failures stop preflight; only verified imports return zero. These codes +All four failures stop preflight; only verified imports in an isolated Python +environment return zero. These codes identify the failing phase, not the underlying cause. The setup helper saves the bootstrap interpreter version, phase messages, and child-process stdout/stderr to `/logs/artifacts/pbd-eval-setup.log`. Harbor collects that directory into the @@ -90,6 +99,12 @@ Run the bundled regression tests without model access: python3 -m unittest discover -s tests -v ``` +The environment tests include a real Linux interpreter/stdlib fixture with +`pip` and `ensurepip` absent. They execute virtual-environment creation and stop +at the download boundary, so a regression to pip-seeded venv creation fails +without needing network access. The installer tests also reject altered and +oversized bootstrap wheels before execution and check credential isolation. + ## Deterministic artifact checks `grading.mode: aces_plus_custom` runs the existing ACES judges and the standalone diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py index a12ea5f4..f8479bba 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/evals/files/setup_eval_env.py @@ -3,35 +3,81 @@ """Install the same isolated CPU environment before either evaluation condition. Usage: python3 /workspace/input/setup_eval_env.py -Arguments: none. Requires Python 3.10-3.13, uv or venv/ensurepip, and index access. +Arguments: none. Requires Python 3.10-3.13, uv or venv, and package-download access. Output: /workspace/pbd-eval-venv; diagnostics in /logs/artifacts/pbd-eval-setup.log. Exit codes: 0 ready; 10 interpreter; 11 venv; 12 installation; 13 imports. """ from contextlib import ExitStack from pathlib import Path +import hashlib import os import shutil import subprocess import sys +import tempfile +import urllib.request ENVIRONMENT = Path("/workspace/pbd-eval-venv") SETUP_LOG = Path("/logs/artifacts/pbd-eval-setup.log") # Both wheels support the older Python interpreters in managed sandboxes. DEPENDENCIES = ("numpy==2.2.6", "biotite==1.2.0") +# A universal wheel supplies pip when the sandbox omits pip/ensurepip. Verify +# the published PyPI digest before executing it with the isolated interpreter. +# https://pypi.org/project/pip/25.3/#files +PIP_WHEEL_URL = ( + "https://files.pythonhosted.org/packages/44/3c/" + "d717024885424591d5376220b5e836c2d5293ce2011523c9de23ff7bf068/" + "pip-25.3-py3-none-any.whl" +) +PIP_WHEEL_SHA256 = "9655943313a94722b7774661c21049070f6bbb0a1516bf02f7c8d5d9201514cd" +PIP_WHEEL_MAX_BYTES = 2_000_000 # Harbor logs the exit code even when healthcheck stdout/stderr is unavailable. EXIT_INTERPRETER = 10 EXIT_VENV = 11 EXIT_INSTALL = 12 EXIT_IMPORT = 13 IMPORT_CHECK = ( - "import numpy, biotite; " + "import sys, numpy, biotite; " + "assert sys.prefix != sys.base_prefix, 'Evaluation Python is not isolated'; " "assert numpy.__version__ == '2.2.6'; " "assert biotite.__version__ == '1.2.0'; " "print('Evaluation imports verified:', numpy.__version__, biotite.__version__)" ) +def installer_environment(): + """Forward installation settings, not agent or model-service credentials.""" + names = ( + "PATH", "HOME", "TMPDIR", "TMP", "TEMP", "LANG", "LC_ALL", "LD_LIBRARY_PATH", + "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "all_proxy", "no_proxy", + "SSL_CERT_FILE", "SSL_CERT_DIR", "REQUESTS_CA_BUNDLE", + "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_CERT", "PIP_CLIENT_CERT", + "UV_INDEX_URL", "UV_EXTRA_INDEX_URL", "UV_NATIVE_TLS", + ) + environment = {} + for name in names: + value = os.environ.get(name) + if value is not None: + environment[name] = value + environment.update(UV_HTTP_TIMEOUT="15", UV_HTTP_RETRIES="1", UV_NO_PROGRESS="1") + return environment + + +def download_pip_wheel(directory): + """Fetch one pinned wheel over verified HTTPS; reject unexpected bytes.""" + with urllib.request.urlopen(PIP_WHEEL_URL, timeout=10) as response: + contents = response.read(PIP_WHEEL_MAX_BYTES + 1) + if len(contents) > PIP_WHEEL_MAX_BYTES: + raise RuntimeError("Pinned pip wheel exceeded the download size limit") + if hashlib.sha256(contents).hexdigest() != PIP_WHEEL_SHA256: + raise RuntimeError("Pinned pip wheel SHA-256 mismatch; refusing to execute it") + wheel = Path(directory) / "pip-25.3-py3-none-any.whl" + wheel.write_bytes(contents) + return wheel + + def select_python(): """Use an installed compatible interpreter without downloading Python.""" if (3, 10) <= sys.version_info[:2] < (3, 14): @@ -69,6 +115,7 @@ def report(message, *, error=False): report(f"Evaluation base interpreter: {executable}") python = environment / "bin/python" uv = shutil.which("uv") + process_env = installer_environment() if uv: report(f"Environment installer: {uv} (does not require venv/ensurepip)") create = [uv, "venv", "--python", executable, "--no-python-downloads", @@ -76,34 +123,45 @@ def report(message, *, error=False): install = [uv, "pip", "install", "--python", str(python), "--no-python-downloads", "--no-config", "--no-cache", "--only-binary=:all:", *DEPENDENCIES] - process_env = {**os.environ, "UV_HTTP_TIMEOUT": "15", "UV_HTTP_RETRIES": "1", - "UV_NO_PROGRESS": "1"} else: - report("Environment installer: Python venv/ensurepip and pip (uv not installed)") - create = [executable, "-m", "venv", str(environment)] - install = [str(python), "-m", "pip", "install", "--disable-pip-version-check", - "--no-input", "--only-binary=:all:", "--timeout", "15", "--retries", "1", - *DEPENDENCIES] - process_env = None + report("Environment installer: Python venv and verified pip wheel (uv not installed)") + # Minimal sandbox distributions can include venv but omit ensurepip. + # Creating the environment must not depend on pip already existing. + create = [executable, "-m", "venv", "--without-pip", str(environment)] phase = "virtual environment creation" exit_code = EXIT_VENV report(f"Starting {phase}") subprocess.run( - create, shell=False, check=True, timeout=30, env=process_env, + create, shell=False, check=True, timeout=20, env=process_env, stdout=log, stderr=subprocess.STDOUT if log is not None else None, ) phase = "dependency installation" exit_code = EXIT_INSTALL - report(f"Starting {phase}") - subprocess.run( - install, shell=False, check=True, timeout=70, env=process_env, - stdout=log, stderr=subprocess.STDOUT if log is not None else None, - ) + with ExitStack() as installers: + if not uv: + phase = "installer download and verification" + report(f"Starting {phase}") + installer_dir = installers.enter_context( + tempfile.TemporaryDirectory(prefix="pbd-eval-installer-"), + ) + wheel = download_pip_wheel(installer_dir) + install = [str(python), str(wheel / "pip"), "install", + "--disable-pip-version-check", "--no-input", "--no-cache-dir", + "--only-binary=:all:", "--timeout", "15", "--retries", "1", + *DEPENDENCIES] + phase = "dependency installation" + exit_code = EXIT_INSTALL + report(f"Starting {phase}") + subprocess.run( + install, shell=False, check=True, timeout=60, env=process_env, + stdout=log, stderr=subprocess.STDOUT if log is not None else None, + ) phase = "dependency import verification" exit_code = EXIT_IMPORT report(f"Starting {phase}") subprocess.run( [str(python), "-c", IMPORT_CHECK], shell=False, check=True, timeout=10, + env=process_env, stdout=log, stderr=subprocess.STDOUT if log is not None else None, ) report(f"Use {python} for Python commands in this evaluation.") diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py index ba0bca90..83d74bc0 100644 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/tests/test_eval_environment.py @@ -3,11 +3,15 @@ """Ensure failed dependency setup cannot be reported as a ready evaluation.""" import contextlib +import hashlib import importlib.util import io +import os from pathlib import Path +import shutil import subprocess import sys +import sysconfig import tempfile import unittest from unittest.mock import patch @@ -24,6 +28,113 @@ def setUp(self): self.find_tool = patch.object(setup.shutil, "which", return_value=None) self.find_tool.start() self.addCleanup(self.find_tool.stop) + # Unit tests never download or execute an installer. The complete setup + # is also exercised separately with real minimal Python distributions. + self.download = patch.object( + setup, "download_pip_wheel", return_value=Path("/verified/pip.whl"), + ) + self.download.start() + self.addCleanup(self.download.stop) + + def test_installer_receives_only_installation_settings(self): + with patch.dict(os.environ, { + "OPENAI_API_KEY": "synthetic-agent-secret", + "NGC_API_KEY": "synthetic-nim-secret", + "AWS_SECRET_ACCESS_KEY": "synthetic-cloud-secret", + "PYTHONPATH": "/unrelated/agent/code", + "HTTPS_PROXY": "http://proxy.invalid:8080", + "PIP_INDEX_URL": "https://packages.invalid/simple", + }): + environment = setup.installer_environment() + for key in ("OPENAI_API_KEY", "NGC_API_KEY", "AWS_SECRET_ACCESS_KEY", "PYTHONPATH"): + self.assertNotIn(key, environment) + self.assertEqual(environment["HTTPS_PROXY"], "http://proxy.invalid:8080") + self.assertEqual(environment["PIP_INDEX_URL"], "https://packages.invalid/simple") + + @unittest.skipUnless(sys.platform.startswith("linux"), "minimal Linux sandbox regression") + def test_real_venv_creation_without_ensurepip_or_pip(self): + # Copy the executable and link its stdlib without pip/ensurepip, matching + # a minimal distro. The venv subprocess is real; stop at the subsequent + # download boundary so this regression test needs no network access. + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) / "python" + executable = base / "bin/python3" + executable.parent.mkdir(parents=True) + shutil.copyfile(sys.executable, executable) + executable.chmod(0o755) + stdlib = Path(sysconfig.get_path("stdlib")) + library = base / "lib" / stdlib.name + library.mkdir(parents=True) + for source in stdlib.iterdir(): + if source.name not in {"ensurepip", "site-packages", "__pycache__"}: + (library / source.name).symlink_to(source, target_is_directory=source.is_dir()) + for source in (Path(sys.base_prefix) / "lib").glob("libpython*.so*"): + (base / "lib" / source.name).symlink_to(source) + environment = Path(directory) / "eval-venv" + with ( + patch.object(setup, "select_python", return_value=str(executable)), + patch.object(setup, "download_pip_wheel", side_effect=RuntimeError("offline download boundary")), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()) as errors, + ): + self.assertEqual(setup.setup(environment), setup.EXIT_INSTALL) + self.assertIn("offline download boundary", errors.getvalue()) + subprocess.run([ + str(environment / "bin/python"), "-I", "-c", + "import sys, importlib.util; assert sys.prefix != sys.base_prefix; " + "assert importlib.util.find_spec('ensurepip') is None; " + "assert importlib.util.find_spec('pip') is None", + ], check=True, timeout=10) + + def test_pipless_environment_uses_the_verified_installer_with_its_own_python(self): + environment = Path("/workspace/test-venv") + with ( + patch.object(setup, "select_python", return_value="/python3.12"), + patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run, + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(setup.setup(environment), 0) + create, install, verify = run.call_args_list + self.assertIn("--without-pip", create.args[0]) + self.assertEqual(install.args[0][:3], [str(environment / "bin/python"), "/verified/pip.whl/pip", "install"]) + self.assertIn("--no-cache-dir", install.args[0]) + self.assertEqual(install.args[0][-2:], list(setup.DEPENDENCIES)) + self.assertEqual(verify.args[0][0], str(environment / "bin/python")) + + def test_unverified_installer_never_starts_installation_or_imports(self): + self.download.stop() + with ( + patch.object(setup, "select_python", return_value="/python3.12"), + patch.object(setup.urllib.request, "urlopen", return_value=io.BytesIO(b"corrupt wheel")), + patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run, + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()) as errors, + ): + self.assertEqual(setup.setup(), setup.EXIT_INSTALL) + run.assert_called_once() + self.assertIn("SHA-256 mismatch", errors.getvalue()) + + def test_installer_download_requires_the_exact_published_digest(self): + self.download.stop() + contents = b"wheel fixture" + with tempfile.TemporaryDirectory() as directory: + with ( + patch.object(setup.urllib.request, "urlopen", return_value=io.BytesIO(contents)), + patch.object(setup, "PIP_WHEEL_SHA256", hashlib.sha256(contents).hexdigest()), + ): + wheel = setup.download_pip_wheel(directory) + self.assertEqual(wheel.read_bytes(), contents) + + def test_oversized_installer_is_rejected_without_writing_a_wheel(self): + self.download.stop() + with tempfile.TemporaryDirectory() as directory: + with ( + patch.object(setup.urllib.request, "urlopen", return_value=io.BytesIO(b"too large")), + patch.object(setup, "PIP_WHEEL_MAX_BYTES", 4), + self.assertRaisesRegex(RuntimeError, "size limit"), + ): + setup.download_pip_wheel(directory) + self.assertEqual(list(Path(directory).iterdir()), []) def test_install_failure_stops_before_import_check(self): errors = io.StringIO() diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md index 0b54f619..bcc7b378 100755 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md @@ -3,7 +3,7 @@ name: protein-binder-design description: > Use when designing RFdiffusion/ProteinMPNN binders or validating and resuming binder campaigns. Not for code or report review. license: Apache-2.0 -compatibility: "Python>=3.12 with requirements.txt; Bash for hosted_env.sh" +compatibility: "Python 3.10-3.13 with requirements.txt; Bash for hosted_env.sh" metadata: author: "NVIDIA BioNeMo " tags: [protein-design, binder-design, bionemo] diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md index 67f259d0..04d59a89 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/README.md @@ -34,13 +34,21 @@ Setup leaves system Python packages untouched and needs no root privileges. When `uv` is already installed on `PATH`, setup uses it to create the environment and install wheels without requiring Python's `venv`, `ensurepip`, or `pip` modules. It selects the installed interpreter explicitly and disables Python -downloads. Without `uv`, setup uses `python -m venv` and the environment's pip. -Both paths use the same package pins and import checks, bounded network retries, -and subprocess timeouts. Setup does not download or install `uv`. +downloads. Without `uv`, setup uses `python -m venv --without-pip`, then downloads +the universal pip 25.3 wheel from its fixed PyPI URL. It verifies the published +SHA-256 digest and a size limit before running the wheel with the virtual +environment's Python. This works when the sandbox omits both `ensurepip` and +`pip`; it does not install either into system Python. Installer files are +temporary and removed on success or failure. Both paths use the same package +pins and import checks, bounded network retries, and subprocess timeouts. +The installer subprocesses receive an explicit list of path, proxy, certificate, +and package-index settings; agent/model credentials and `PYTHONPATH` are not +forwarded. Setup does not download or install `uv`. There is no custom `evals/environment/Dockerfile`, so Astra uses its existing sandbox image without invoking the managed image builder or requiring its token. The standard sandbox must provide Python 3.10-3.13 plus either `uv` or -`venv`/`ensurepip`, a writable workspace, and package-index access for setup. +`venv`, a writable workspace, and package-index access for setup. The pip +fallback also needs HTTPS access to `files.pythonhosted.org`. Setup downloads packages; the subsequent exercises make no network calls. Normal sandbox and agent API access are still required. Installation or import failures stop environment preflight before agent trials. @@ -51,11 +59,12 @@ as `Healthcheck failed (rc=...)`. The setup helper uses distinct failure codes: | Exit code | Failed phase | Check | | --- | --- | --- | | 10 | Interpreter selection | An installed Python 3.10-3.13 is discoverable. | -| 11 | Virtual environment creation | The workspace is writable; either uv is on PATH or Python has venv/ensurepip. | -| 12 | Dependency installation | The selected installer can reach the package index and install the pinned wheels within the timeout. | +| 11 | Virtual environment creation | The workspace is writable; either uv is on PATH or Python has venv. | +| 12 | Installer bootstrap or dependency installation | The pip wheel passes its digest check, and the selected installer can reach the package index and install the pinned wheels within the timeout. | | 13 | Import verification | The installed NumPy and Biotite versions import successfully. | -All four failures stop preflight; only verified imports return zero. These codes +All four failures stop preflight; only verified imports in an isolated Python +environment return zero. These codes identify the failing phase, not the underlying cause. The setup helper saves the bootstrap interpreter version, phase messages, and child-process stdout/stderr to `/logs/artifacts/pbd-eval-setup.log`. Harbor collects that directory into the @@ -90,6 +99,12 @@ Run the bundled regression tests without model access: python3 -m unittest discover -s tests -v ``` +The environment tests include a real Linux interpreter/stdlib fixture with +`pip` and `ensurepip` absent. They execute virtual-environment creation and stop +at the download boundary, so a regression to pip-seeded venv creation fails +without needing network access. The installer tests also reject altered and +oversized bootstrap wheels before execution and check credential isolation. + ## Deterministic artifact checks `grading.mode: aces_plus_custom` runs the existing ACES judges and the standalone diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py index a12ea5f4..f8479bba 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/evals/files/setup_eval_env.py @@ -3,35 +3,81 @@ """Install the same isolated CPU environment before either evaluation condition. Usage: python3 /workspace/input/setup_eval_env.py -Arguments: none. Requires Python 3.10-3.13, uv or venv/ensurepip, and index access. +Arguments: none. Requires Python 3.10-3.13, uv or venv, and package-download access. Output: /workspace/pbd-eval-venv; diagnostics in /logs/artifacts/pbd-eval-setup.log. Exit codes: 0 ready; 10 interpreter; 11 venv; 12 installation; 13 imports. """ from contextlib import ExitStack from pathlib import Path +import hashlib import os import shutil import subprocess import sys +import tempfile +import urllib.request ENVIRONMENT = Path("/workspace/pbd-eval-venv") SETUP_LOG = Path("/logs/artifacts/pbd-eval-setup.log") # Both wheels support the older Python interpreters in managed sandboxes. DEPENDENCIES = ("numpy==2.2.6", "biotite==1.2.0") +# A universal wheel supplies pip when the sandbox omits pip/ensurepip. Verify +# the published PyPI digest before executing it with the isolated interpreter. +# https://pypi.org/project/pip/25.3/#files +PIP_WHEEL_URL = ( + "https://files.pythonhosted.org/packages/44/3c/" + "d717024885424591d5376220b5e836c2d5293ce2011523c9de23ff7bf068/" + "pip-25.3-py3-none-any.whl" +) +PIP_WHEEL_SHA256 = "9655943313a94722b7774661c21049070f6bbb0a1516bf02f7c8d5d9201514cd" +PIP_WHEEL_MAX_BYTES = 2_000_000 # Harbor logs the exit code even when healthcheck stdout/stderr is unavailable. EXIT_INTERPRETER = 10 EXIT_VENV = 11 EXIT_INSTALL = 12 EXIT_IMPORT = 13 IMPORT_CHECK = ( - "import numpy, biotite; " + "import sys, numpy, biotite; " + "assert sys.prefix != sys.base_prefix, 'Evaluation Python is not isolated'; " "assert numpy.__version__ == '2.2.6'; " "assert biotite.__version__ == '1.2.0'; " "print('Evaluation imports verified:', numpy.__version__, biotite.__version__)" ) +def installer_environment(): + """Forward installation settings, not agent or model-service credentials.""" + names = ( + "PATH", "HOME", "TMPDIR", "TMP", "TEMP", "LANG", "LC_ALL", "LD_LIBRARY_PATH", + "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "all_proxy", "no_proxy", + "SSL_CERT_FILE", "SSL_CERT_DIR", "REQUESTS_CA_BUNDLE", + "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_CERT", "PIP_CLIENT_CERT", + "UV_INDEX_URL", "UV_EXTRA_INDEX_URL", "UV_NATIVE_TLS", + ) + environment = {} + for name in names: + value = os.environ.get(name) + if value is not None: + environment[name] = value + environment.update(UV_HTTP_TIMEOUT="15", UV_HTTP_RETRIES="1", UV_NO_PROGRESS="1") + return environment + + +def download_pip_wheel(directory): + """Fetch one pinned wheel over verified HTTPS; reject unexpected bytes.""" + with urllib.request.urlopen(PIP_WHEEL_URL, timeout=10) as response: + contents = response.read(PIP_WHEEL_MAX_BYTES + 1) + if len(contents) > PIP_WHEEL_MAX_BYTES: + raise RuntimeError("Pinned pip wheel exceeded the download size limit") + if hashlib.sha256(contents).hexdigest() != PIP_WHEEL_SHA256: + raise RuntimeError("Pinned pip wheel SHA-256 mismatch; refusing to execute it") + wheel = Path(directory) / "pip-25.3-py3-none-any.whl" + wheel.write_bytes(contents) + return wheel + + def select_python(): """Use an installed compatible interpreter without downloading Python.""" if (3, 10) <= sys.version_info[:2] < (3, 14): @@ -69,6 +115,7 @@ def report(message, *, error=False): report(f"Evaluation base interpreter: {executable}") python = environment / "bin/python" uv = shutil.which("uv") + process_env = installer_environment() if uv: report(f"Environment installer: {uv} (does not require venv/ensurepip)") create = [uv, "venv", "--python", executable, "--no-python-downloads", @@ -76,34 +123,45 @@ def report(message, *, error=False): install = [uv, "pip", "install", "--python", str(python), "--no-python-downloads", "--no-config", "--no-cache", "--only-binary=:all:", *DEPENDENCIES] - process_env = {**os.environ, "UV_HTTP_TIMEOUT": "15", "UV_HTTP_RETRIES": "1", - "UV_NO_PROGRESS": "1"} else: - report("Environment installer: Python venv/ensurepip and pip (uv not installed)") - create = [executable, "-m", "venv", str(environment)] - install = [str(python), "-m", "pip", "install", "--disable-pip-version-check", - "--no-input", "--only-binary=:all:", "--timeout", "15", "--retries", "1", - *DEPENDENCIES] - process_env = None + report("Environment installer: Python venv and verified pip wheel (uv not installed)") + # Minimal sandbox distributions can include venv but omit ensurepip. + # Creating the environment must not depend on pip already existing. + create = [executable, "-m", "venv", "--without-pip", str(environment)] phase = "virtual environment creation" exit_code = EXIT_VENV report(f"Starting {phase}") subprocess.run( - create, shell=False, check=True, timeout=30, env=process_env, + create, shell=False, check=True, timeout=20, env=process_env, stdout=log, stderr=subprocess.STDOUT if log is not None else None, ) phase = "dependency installation" exit_code = EXIT_INSTALL - report(f"Starting {phase}") - subprocess.run( - install, shell=False, check=True, timeout=70, env=process_env, - stdout=log, stderr=subprocess.STDOUT if log is not None else None, - ) + with ExitStack() as installers: + if not uv: + phase = "installer download and verification" + report(f"Starting {phase}") + installer_dir = installers.enter_context( + tempfile.TemporaryDirectory(prefix="pbd-eval-installer-"), + ) + wheel = download_pip_wheel(installer_dir) + install = [str(python), str(wheel / "pip"), "install", + "--disable-pip-version-check", "--no-input", "--no-cache-dir", + "--only-binary=:all:", "--timeout", "15", "--retries", "1", + *DEPENDENCIES] + phase = "dependency installation" + exit_code = EXIT_INSTALL + report(f"Starting {phase}") + subprocess.run( + install, shell=False, check=True, timeout=60, env=process_env, + stdout=log, stderr=subprocess.STDOUT if log is not None else None, + ) phase = "dependency import verification" exit_code = EXIT_IMPORT report(f"Starting {phase}") subprocess.run( [str(python), "-c", IMPORT_CHECK], shell=False, check=True, timeout=10, + env=process_env, stdout=log, stderr=subprocess.STDOUT if log is not None else None, ) report(f"Use {python} for Python commands in this evaluation.") diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py index ba0bca90..83d74bc0 100644 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/tests/test_eval_environment.py @@ -3,11 +3,15 @@ """Ensure failed dependency setup cannot be reported as a ready evaluation.""" import contextlib +import hashlib import importlib.util import io +import os from pathlib import Path +import shutil import subprocess import sys +import sysconfig import tempfile import unittest from unittest.mock import patch @@ -24,6 +28,113 @@ def setUp(self): self.find_tool = patch.object(setup.shutil, "which", return_value=None) self.find_tool.start() self.addCleanup(self.find_tool.stop) + # Unit tests never download or execute an installer. The complete setup + # is also exercised separately with real minimal Python distributions. + self.download = patch.object( + setup, "download_pip_wheel", return_value=Path("/verified/pip.whl"), + ) + self.download.start() + self.addCleanup(self.download.stop) + + def test_installer_receives_only_installation_settings(self): + with patch.dict(os.environ, { + "OPENAI_API_KEY": "synthetic-agent-secret", + "NGC_API_KEY": "synthetic-nim-secret", + "AWS_SECRET_ACCESS_KEY": "synthetic-cloud-secret", + "PYTHONPATH": "/unrelated/agent/code", + "HTTPS_PROXY": "http://proxy.invalid:8080", + "PIP_INDEX_URL": "https://packages.invalid/simple", + }): + environment = setup.installer_environment() + for key in ("OPENAI_API_KEY", "NGC_API_KEY", "AWS_SECRET_ACCESS_KEY", "PYTHONPATH"): + self.assertNotIn(key, environment) + self.assertEqual(environment["HTTPS_PROXY"], "http://proxy.invalid:8080") + self.assertEqual(environment["PIP_INDEX_URL"], "https://packages.invalid/simple") + + @unittest.skipUnless(sys.platform.startswith("linux"), "minimal Linux sandbox regression") + def test_real_venv_creation_without_ensurepip_or_pip(self): + # Copy the executable and link its stdlib without pip/ensurepip, matching + # a minimal distro. The venv subprocess is real; stop at the subsequent + # download boundary so this regression test needs no network access. + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) / "python" + executable = base / "bin/python3" + executable.parent.mkdir(parents=True) + shutil.copyfile(sys.executable, executable) + executable.chmod(0o755) + stdlib = Path(sysconfig.get_path("stdlib")) + library = base / "lib" / stdlib.name + library.mkdir(parents=True) + for source in stdlib.iterdir(): + if source.name not in {"ensurepip", "site-packages", "__pycache__"}: + (library / source.name).symlink_to(source, target_is_directory=source.is_dir()) + for source in (Path(sys.base_prefix) / "lib").glob("libpython*.so*"): + (base / "lib" / source.name).symlink_to(source) + environment = Path(directory) / "eval-venv" + with ( + patch.object(setup, "select_python", return_value=str(executable)), + patch.object(setup, "download_pip_wheel", side_effect=RuntimeError("offline download boundary")), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()) as errors, + ): + self.assertEqual(setup.setup(environment), setup.EXIT_INSTALL) + self.assertIn("offline download boundary", errors.getvalue()) + subprocess.run([ + str(environment / "bin/python"), "-I", "-c", + "import sys, importlib.util; assert sys.prefix != sys.base_prefix; " + "assert importlib.util.find_spec('ensurepip') is None; " + "assert importlib.util.find_spec('pip') is None", + ], check=True, timeout=10) + + def test_pipless_environment_uses_the_verified_installer_with_its_own_python(self): + environment = Path("/workspace/test-venv") + with ( + patch.object(setup, "select_python", return_value="/python3.12"), + patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run, + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(setup.setup(environment), 0) + create, install, verify = run.call_args_list + self.assertIn("--without-pip", create.args[0]) + self.assertEqual(install.args[0][:3], [str(environment / "bin/python"), "/verified/pip.whl/pip", "install"]) + self.assertIn("--no-cache-dir", install.args[0]) + self.assertEqual(install.args[0][-2:], list(setup.DEPENDENCIES)) + self.assertEqual(verify.args[0][0], str(environment / "bin/python")) + + def test_unverified_installer_never_starts_installation_or_imports(self): + self.download.stop() + with ( + patch.object(setup, "select_python", return_value="/python3.12"), + patch.object(setup.urllib.request, "urlopen", return_value=io.BytesIO(b"corrupt wheel")), + patch.object(setup.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as run, + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()) as errors, + ): + self.assertEqual(setup.setup(), setup.EXIT_INSTALL) + run.assert_called_once() + self.assertIn("SHA-256 mismatch", errors.getvalue()) + + def test_installer_download_requires_the_exact_published_digest(self): + self.download.stop() + contents = b"wheel fixture" + with tempfile.TemporaryDirectory() as directory: + with ( + patch.object(setup.urllib.request, "urlopen", return_value=io.BytesIO(contents)), + patch.object(setup, "PIP_WHEEL_SHA256", hashlib.sha256(contents).hexdigest()), + ): + wheel = setup.download_pip_wheel(directory) + self.assertEqual(wheel.read_bytes(), contents) + + def test_oversized_installer_is_rejected_without_writing_a_wheel(self): + self.download.stop() + with tempfile.TemporaryDirectory() as directory: + with ( + patch.object(setup.urllib.request, "urlopen", return_value=io.BytesIO(b"too large")), + patch.object(setup, "PIP_WHEEL_MAX_BYTES", 4), + self.assertRaisesRegex(RuntimeError, "size limit"), + ): + setup.download_pip_wheel(directory) + self.assertEqual(list(Path(directory).iterdir()), []) def test_install_failure_stops_before_import_check(self): errors = io.StringIO() From 2b9320676b9d3954bcf3d205c68fb9bdd8dd5b34 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Tue, 27 Oct 2026 18:44:57 -0700 Subject: [PATCH 9/9] report judge refusals and diagnose invalid JSON replies Signed-off-by: Ohad Mosafi --- .../skills/protein-binder-design/SKILL.md | 21 +++++++++--- .../references/manifest.md | 3 ++ .../references/pipeline.md | 34 ++++++++++++++++--- .../references/validation.md | 26 ++++++++++++-- .../protein-binder-design/SKILL.md | 21 +++++++++--- .../references/manifest.md | 3 ++ .../references/pipeline.md | 34 ++++++++++++++++--- .../references/validation.md | 26 ++++++++++++-- 8 files changed, 148 insertions(+), 20 deletions(-) diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md index bcc7b378..fc6b8815 100755 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/SKILL.md @@ -126,7 +126,7 @@ flags unchanged: ```python from pathlib import Path -m.apply_filters() +m.apply_filters() # also saves filter results to manifest.json rank_by = m.data["params"].get("rank_by", "iptm") ranked = m.rank(by=rank_by, descending=True, passed_only=True, include_controls=False) run_dir = Path(m.data["run_dir"]) @@ -136,7 +136,9 @@ print(m.summary()) # controls excluded from candidate/ print([candidate["id"] for candidate in ranked]) ``` -`rank()` returns a new list; it does not reorder or shrink the manifest. +`apply_filters()` saves `passed_filter`, `filter_metrics`, and `filter_exemptions` +before returning. No extra save is needed for those changes. `rank()` returns a +new list; it does not reorder or shrink the manifest. `to_csv()` without `candidates=ranked` writes **all** entries in insertion order. Pass an empty ranked list through unchanged: zero survivors means a header-only `candidates.csv`, while the audit CSV and manifest retain every entry. A high @@ -202,6 +204,8 @@ controls = sourced published binders re-scored through the same pipeline. The bundled `assets/targets.json` is an empty control template, not a positive-control dataset. Use user-provided sequences with citations, or retrieve and verify an exact sequence from a primary publication or structure before registering it. +Record the target, source location, exact sequence and verification evidence +using the [positive-control checklist](references/validation.md#positive-control-provenance). If none is available, record `positive_controls_unavailable`, complete the computational screen with negatives, and label it **uncalibrated screening**; do not claim positive-control benchmarking or calibrated binder success. A @@ -214,8 +218,12 @@ metric definitions: `references/validation.md`. - Reuse target, epitope/hotspots, binder length range, and hosted-vs-local choices already supplied by the user. Ask only for unresolved inputs before generating backbones. -- Co-folding is the expensive stage: co-fold a capped shortlist, review, then - expand. State hosted vs local once and reuse it across all NIM calls. +- Before live co-folding, save `params.validation_budget` with + `initial_shortlist`, `max_refold_calls`, and `max_elapsed_seconds`. Reuse supplied + limits and ask only for missing limits; count controls and retries in the totals. + Select and review each batch using the [cost funnel](references/pipeline.md#cost-funnel), + then expand only within the saved budget. Report unscored work when a limit is + reached. State hosted vs local once and reuse it across all NIM calls. ## Responsible use @@ -268,6 +276,11 @@ Per-NIM paths, request/response schemas, and worked `curl`/Python examples live `references/manifest.md` to execute only missing stages. A finite below-threshold score is complete, not a reason to repeat inference. Reuse saved predictions for missing metrics; preserve completed candidates and explicit failure records. + Offline requests never retry inference: a previously failed candidate without + a usable cached response stays blocked. For authorized live work, follow the + [bounded retry policy](references/pipeline.md#request-failures-and-partial-batches), + record the retry decision and remaining budget, and append new attempts without + overwriting earlier failures. ## Available Scripts diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/manifest.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/manifest.md index f4e6a7cd..7286554c 100755 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/manifest.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/manifest.md @@ -69,6 +69,9 @@ disabling that criterion for the campaign, and disclose the override. A composit confidence field does not satisfy an ipTM requirement. Sequence-only controls without `artifacts.backbone_pdb` skip only RMSD, with the reason saved under `filter_exemptions`. `filter_metrics` records the criteria actually evaluated. +The method calls `save()` before returning, so these fields and `passed_filter` +are persisted in `manifest.json` even when no candidate passes. `rank()` and +`to_csv()` do not change that saved candidate set. ### Boltz2-only filtering and ranking diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/pipeline.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/pipeline.md index 0863505b..14d158f0 100755 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/pipeline.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/pipeline.md @@ -171,9 +171,35 @@ m.add_artifact("bb003_seq02", "complex_cif", "runs/.../complexes/bb003_seq02.cif ``` ### Cost funnel -Co-folding is the expensive stage. Co-fold a **capped shortlist** first (e.g. -best ProteinMPNN NLL per backbone), review, then expand. Reserve high -`diffusion_samples` / `recycling_steps` for the final survivors. +Before live co-folding, save positive integer counts and a finite positive time +limit in `params.validation_budget`: `initial_shortlist` (candidate count), +`max_refold_calls` (all co-fold requests, +including controls and retries), and `max_elapsed_seconds` (wall time for this +co-folding stage). Reuse limits already supplied by the user or saved campaign; +resolve missing limits before submitting calls. Save the remaining allowance on +interruption so resume does not reset an exhausted budget. +Reserve required control requests and retry allowance before selecting a design +batch. If the budget cannot cover the requested controls, report that limitation +and resolve the scope before inference. Cap request timeouts and retry waits by +the remaining wall-time allowance. + +Use the user's selection rule when supplied. Otherwise order each backbone's +sequences by ascending finite `proteinmpnn_nll`, break ties by candidate ID, and +take one sequence per backbone in backbone-ID order before taking a second. +Stop the first batch at `initial_shortlist` or the remaining call allowance, +whichever is smaller. Record selected IDs and the selection rule. Missing NLL +does not mean zero: recover it from the saved MPNN response or leave that candidate +pending selection. This is a sampling rule, not a confidence or binding cutoff. + +After each batch, verify response identity, required metrics, artifact integrity, +and control results, then report missing metrics and the passing fraction. Resolve +systematic response or provenance failures before expanding. Low finite scores +remain completed results; do not resubmit them to obtain passers. Continue the +next batch only within the authorized candidate pool and remaining call/time +allowance. Stop on completion or an exhausted limit, retain pending candidates, +and obtain a new budget decision before further inference. Reserve high +`diffusion_samples` / `recycling_steps` for explicitly selected final survivors +and count those requests against the same limits. ## 5. Self-consistency RMSD @@ -204,7 +230,7 @@ For sequence-only controls without a designed backbone, RMSD is not applicable; ```python from pathlib import Path -m.apply_filters() # uses the saved campaign criteria +m.apply_filters() # applies saved criteria and persists filter results rank_by = m.data["params"].get("rank_by", "iptm") ranked = m.rank(by=rank_by, descending=True, passed_only=True, include_controls=False) top = ranked[:20] diff --git a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/validation.md b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/validation.md index 2eae3eed..7b7a7139 100755 --- a/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/validation.md +++ b/skills/bionemo-agent-toolkit/skills/protein-binder-design/references/validation.md @@ -43,14 +43,36 @@ m.upsert_candidate("ctrl_neg_01", is_control=True, control_type="scrambled", seq marked `control_type="published"`. The bundled registry is a template and contains no published binder sequences. -Use cited user inputs or verify exact sequences against a primary paper/structure -and register the target, binder sequence, and source with `scripts/registry.py`. +Use cited user inputs or verify exact sequences against a primary paper/structure, +then record the target, binder sequence, and source in a user-supplied registry +JSON file. `scripts/registry.py` only loads and looks up entries; it does not write +them or verify their authenticity. If none can be sourced, record `positive_controls_unavailable`; return an **uncalibrated computational screen** with negative controls and disclose that the positive-control comparison was not performed. Do not invent a positive or label the missing comparison as successful. If calibrated benchmarking was requested, that portion remains blocked. +### Positive-control provenance + +Before labeling a sequence as a published positive control: + +1. Check that the primary source reports binding to the campaign's target and + record the target construct or accession. A model prediction alone is not + evidence of measured binding. +2. Match the exact sequence to its source location: supplementary table/record, + or structure accession, chain and residue range. Record tags, mutations and + truncations; an unexplained sequence difference prevents verified-control use. +3. Save the source URL/accession, source location, sequence SHA-256, target identity, + and how the match was checked in a `positive_control_verification` stage via + `m.log_stage()`. Bind this record to the control's candidate ID and keep the + original citation or supplied evidence with the campaign. + +This is a source-verification step performed by the operator or agent, not an +automatic guarantee from the registry loader. An unavailable source or unresolved +identity match stays `positive_control_unverified`; disclose it and do not count +that sequence as a verified positive benchmark. + Sequence-only controls have no RFdiffusion backbone. Leave their self-consistency RMSD absent, and do not attach a parent's backbone merely to satisfy the filter. `apply_filters()` records `filter_exemptions.self_consistency_rmsd` for these diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md index bcc7b378..fc6b8815 100755 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/SKILL.md @@ -126,7 +126,7 @@ flags unchanged: ```python from pathlib import Path -m.apply_filters() +m.apply_filters() # also saves filter results to manifest.json rank_by = m.data["params"].get("rank_by", "iptm") ranked = m.rank(by=rank_by, descending=True, passed_only=True, include_controls=False) run_dir = Path(m.data["run_dir"]) @@ -136,7 +136,9 @@ print(m.summary()) # controls excluded from candidate/ print([candidate["id"] for candidate in ranked]) ``` -`rank()` returns a new list; it does not reorder or shrink the manifest. +`apply_filters()` saves `passed_filter`, `filter_metrics`, and `filter_exemptions` +before returning. No extra save is needed for those changes. `rank()` returns a +new list; it does not reorder or shrink the manifest. `to_csv()` without `candidates=ranked` writes **all** entries in insertion order. Pass an empty ranked list through unchanged: zero survivors means a header-only `candidates.csv`, while the audit CSV and manifest retain every entry. A high @@ -202,6 +204,8 @@ controls = sourced published binders re-scored through the same pipeline. The bundled `assets/targets.json` is an empty control template, not a positive-control dataset. Use user-provided sequences with citations, or retrieve and verify an exact sequence from a primary publication or structure before registering it. +Record the target, source location, exact sequence and verification evidence +using the [positive-control checklist](references/validation.md#positive-control-provenance). If none is available, record `positive_controls_unavailable`, complete the computational screen with negatives, and label it **uncalibrated screening**; do not claim positive-control benchmarking or calibrated binder success. A @@ -214,8 +218,12 @@ metric definitions: `references/validation.md`. - Reuse target, epitope/hotspots, binder length range, and hosted-vs-local choices already supplied by the user. Ask only for unresolved inputs before generating backbones. -- Co-folding is the expensive stage: co-fold a capped shortlist, review, then - expand. State hosted vs local once and reuse it across all NIM calls. +- Before live co-folding, save `params.validation_budget` with + `initial_shortlist`, `max_refold_calls`, and `max_elapsed_seconds`. Reuse supplied + limits and ask only for missing limits; count controls and retries in the totals. + Select and review each batch using the [cost funnel](references/pipeline.md#cost-funnel), + then expand only within the saved budget. Report unscored work when a limit is + reached. State hosted vs local once and reuse it across all NIM calls. ## Responsible use @@ -268,6 +276,11 @@ Per-NIM paths, request/response schemas, and worked `curl`/Python examples live `references/manifest.md` to execute only missing stages. A finite below-threshold score is complete, not a reason to repeat inference. Reuse saved predictions for missing metrics; preserve completed candidates and explicit failure records. + Offline requests never retry inference: a previously failed candidate without + a usable cached response stays blocked. For authorized live work, follow the + [bounded retry policy](references/pipeline.md#request-failures-and-partial-batches), + record the retry decision and remaining budget, and append new attempts without + overwriting earlier failures. ## Available Scripts diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/manifest.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/manifest.md index f4e6a7cd..7286554c 100755 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/manifest.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/manifest.md @@ -69,6 +69,9 @@ disabling that criterion for the campaign, and disclose the override. A composit confidence field does not satisfy an ipTM requirement. Sequence-only controls without `artifacts.backbone_pdb` skip only RMSD, with the reason saved under `filter_exemptions`. `filter_metrics` records the criteria actually evaluated. +The method calls `save()` before returning, so these fields and `passed_filter` +are persisted in `manifest.json` even when no candidate passes. `rank()` and +`to_csv()` do not change that saved candidate set. ### Boltz2-only filtering and ranking diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/pipeline.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/pipeline.md index 0863505b..14d158f0 100755 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/pipeline.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/pipeline.md @@ -171,9 +171,35 @@ m.add_artifact("bb003_seq02", "complex_cif", "runs/.../complexes/bb003_seq02.cif ``` ### Cost funnel -Co-folding is the expensive stage. Co-fold a **capped shortlist** first (e.g. -best ProteinMPNN NLL per backbone), review, then expand. Reserve high -`diffusion_samples` / `recycling_steps` for the final survivors. +Before live co-folding, save positive integer counts and a finite positive time +limit in `params.validation_budget`: `initial_shortlist` (candidate count), +`max_refold_calls` (all co-fold requests, +including controls and retries), and `max_elapsed_seconds` (wall time for this +co-folding stage). Reuse limits already supplied by the user or saved campaign; +resolve missing limits before submitting calls. Save the remaining allowance on +interruption so resume does not reset an exhausted budget. +Reserve required control requests and retry allowance before selecting a design +batch. If the budget cannot cover the requested controls, report that limitation +and resolve the scope before inference. Cap request timeouts and retry waits by +the remaining wall-time allowance. + +Use the user's selection rule when supplied. Otherwise order each backbone's +sequences by ascending finite `proteinmpnn_nll`, break ties by candidate ID, and +take one sequence per backbone in backbone-ID order before taking a second. +Stop the first batch at `initial_shortlist` or the remaining call allowance, +whichever is smaller. Record selected IDs and the selection rule. Missing NLL +does not mean zero: recover it from the saved MPNN response or leave that candidate +pending selection. This is a sampling rule, not a confidence or binding cutoff. + +After each batch, verify response identity, required metrics, artifact integrity, +and control results, then report missing metrics and the passing fraction. Resolve +systematic response or provenance failures before expanding. Low finite scores +remain completed results; do not resubmit them to obtain passers. Continue the +next batch only within the authorized candidate pool and remaining call/time +allowance. Stop on completion or an exhausted limit, retain pending candidates, +and obtain a new budget decision before further inference. Reserve high +`diffusion_samples` / `recycling_steps` for explicitly selected final survivors +and count those requests against the same limits. ## 5. Self-consistency RMSD @@ -204,7 +230,7 @@ For sequence-only controls without a designed backbone, RMSD is not applicable; ```python from pathlib import Path -m.apply_filters() # uses the saved campaign criteria +m.apply_filters() # applies saved criteria and persists filter results rank_by = m.data["params"].get("rank_by", "iptm") ranked = m.rank(by=rank_by, descending=True, passed_only=True, include_controls=False) top = ranked[:20] diff --git a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/validation.md b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/validation.md index 2eae3eed..7b7a7139 100755 --- a/workflow-skills/generative-protein-binder-design/protein-binder-design/references/validation.md +++ b/workflow-skills/generative-protein-binder-design/protein-binder-design/references/validation.md @@ -43,14 +43,36 @@ m.upsert_candidate("ctrl_neg_01", is_control=True, control_type="scrambled", seq marked `control_type="published"`. The bundled registry is a template and contains no published binder sequences. -Use cited user inputs or verify exact sequences against a primary paper/structure -and register the target, binder sequence, and source with `scripts/registry.py`. +Use cited user inputs or verify exact sequences against a primary paper/structure, +then record the target, binder sequence, and source in a user-supplied registry +JSON file. `scripts/registry.py` only loads and looks up entries; it does not write +them or verify their authenticity. If none can be sourced, record `positive_controls_unavailable`; return an **uncalibrated computational screen** with negative controls and disclose that the positive-control comparison was not performed. Do not invent a positive or label the missing comparison as successful. If calibrated benchmarking was requested, that portion remains blocked. +### Positive-control provenance + +Before labeling a sequence as a published positive control: + +1. Check that the primary source reports binding to the campaign's target and + record the target construct or accession. A model prediction alone is not + evidence of measured binding. +2. Match the exact sequence to its source location: supplementary table/record, + or structure accession, chain and residue range. Record tags, mutations and + truncations; an unexplained sequence difference prevents verified-control use. +3. Save the source URL/accession, source location, sequence SHA-256, target identity, + and how the match was checked in a `positive_control_verification` stage via + `m.log_stage()`. Bind this record to the control's candidate ID and keep the + original citation or supplied evidence with the campaign. + +This is a source-verification step performed by the operator or agent, not an +automatic guarantee from the registry loader. An unavailable source or unresolved +identity match stays `positive_control_unverified`; disclose it and do not count +that sequence as a verified positive benchmark. + Sequence-only controls have no RFdiffusion backbone. Leave their self-consistency RMSD absent, and do not attach a parent's backbone merely to satisfy the filter. `apply_filters()` records `filter_exemptions.self_consistency_rmsd` for these