Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions examples/coding_audit/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# Operator coding audit

This example collects and scores bounded coding-task evidence. It belongs to the
operator workflow, separate from the installed OpenEnv validator and its RFC 008
checks. Acceptance here does not establish full RFC 008 conformance, containment,
served-model revision attestation, or training benefit.

From a repository checkout, install the tested SDK and run the example:

```bash
uv sync --group dev
uv pip install 'huggingface_hub==1.33.0'
PYTHONPATH=src:envs uv run --no-sync python -m examples.coding_audit \
--task /path/to/task \
--config examples/coding_audit/config.json \
--output /path/to/new-audit-directory \
--json
```

The output directory is required and must be new. It retains `report.json`,
`evidence.json`, `config.json`, raw evidence artifacts, and `SHA256SUMS`. Exit codes
are `0` for accepted evidence, `1` for rejected or incomplete evidence, and `2` for
configuration or input failure. `--json` prints the report as well.

`scoring.py` provides pure `assess_coding_audit` and `validate_audit_config`
functions. `runner.py` provides `normalize_coding_config`, `coding_budget`,
`validate_coding_task`, and `run_coding_validation`. `hf.py` contains
`HFTaskBackend`, the bounded Sandbox and Inference Providers implementation.
The schemas bind task, verifier, immutable image references and independent execution identities.

Models may declare an explicit boolean `enable_thinking`, forwarded through the
HF client's `extra_body.chat_template_kwargs`. It is omitted by default. A
candidate-only override leaves the target protocol unchanged and is recorded
alongside request parameters. Support depends on the selected provider/model.
Candidate generation streams bounded content and reasoning; only a complete
`stop` response with validated usage can produce an executable candidate. A
`length` response remains rejected even when its terminal usage is retained.
Recognized Bash/sh Markdown fences are parsed before execution; the raw response
and both raw/script hashes are retained. Other language fences are rejected.

Candidate generation can opt into public task API context by adding
`"context_paths": ["environment/source/package/api.py"]` to the `generator`
model configuration. Only those declared regular UTF-8 files are read; paths must
be unique, normalized, and below `environment/source/`. Up to 32 files and 65,536
combined bytes are allowed. Linked files or parents, hidden or private paths,
NUL bytes, invalid UTF-8 and oversized context are rejected before inference.
The existing message and token budgets still apply. No source files are added by
default, and `target.context_paths` is rejected.

The candidate prompt labels each source path and its runtime filename relative to
the configured workdir, with `environment/source/` removed. Each generation record
retains `candidate_context` entries containing `path`, `runtime_path`, `sha256`,
and `size`, bound to the frozen task and checksummed evidence. This option does
not enumerate task directories or include verifier, reference or specification
files.

The HF backend launches paid services when invoked without a supplied test
backend. Use reviewed tasks and distinct execution and inference credentials.
The Hub pipeline's cloud worker packages these production example modules with
its frozen source and task bundle; its tests stay outside the bundle.

Run the offline suite with:

```bash
PYTHONPATH=.:src:envs uv run pytest examples/coding_audit -q
```
1 change: 1 addition & 0 deletions examples/coding_audit/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Operator-owned coding audits, separate from core RFC 008 validation."""
63 changes: 63 additions & 0 deletions examples/coding_audit/__main__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
"""Run a bounded operator coding audit and retain its evidence bundle."""

import argparse
import signal
import sys
import threading
from contextlib import contextmanager
from pathlib import Path

from openenv.validation.runtime.json_values import dumps, loads

from .runner import run_coding_validation


@contextmanager
def _sigterm_as_interrupt():
if threading.current_thread() is not threading.main_thread():
yield
return

def interrupt(signum, frame):
raise KeyboardInterrupt

previous = signal.signal(signal.SIGTERM, interrupt)
try:
yield
finally:
signal.signal(signal.SIGTERM, previous)


def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--task", type=Path, required=True)
parser.add_argument("--config", type=Path, required=True)
parser.add_argument(
"--output",
type=Path,
required=True,
help="New directory for the report, evidence and checksummed artifacts",
)
parser.add_argument("--json", action="store_true", dest="json_output")
args = parser.parse_args(argv)
try:
config = loads(args.config.read_text())
with _sigterm_as_interrupt():
report = run_coding_validation(args.task, config, args.output)
except (OSError, ValueError) as error:
print(
f"Coding audit configuration/input failure ({type(error).__name__})",
file=sys.stderr,
)
return 2
if args.json_output:
print(dumps(report))
else:
print(f"Coding audit: {report['verdict'].upper()} ({report['profile']})")
for name, check in report["checks"].items():
print(f" {check['status'].upper():5s} {name}")
return 0 if report["accepted"] else 1


if __name__ == "__main__":
raise SystemExit(main())
83 changes: 83 additions & 0 deletions examples/coding_audit/config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
{
"schema_version": 1,
"profile": "harbor-coding-audit-v1",
"target": {
"model_id": "Qwen/Qwen3.8-27B",
"revision": "37561816e66b5b9153b1b64c10968c18c1e6f4a0",
"provider": "deepinfra",
"supports_seed": false,
"supports_tools": true,
"temperature": 0.7,
"max_steps": 16,
"max_tokens": 2048,
"max_total_tokens": 16384
},
"execution": {
"agent_image": "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20",
"verifier_image": "docker.io/library/python@sha256:48d884813cf44e927cf64c5529dd630dd3b18338c6dc727991eef9b71e934a20",
"flavor": "cpu-basic",
"timeout_seconds": 120,
"workdir": "/app",
"submission_paths": [
"/app/answer.txt"
],
"private_paths": [
"/tests",
"/solution"
],
"verifier_command": [
"/bin/bash",
"/tests/test.sh"
],
"reward_path": "/logs/verifier/reward.txt",
"reward_key": "reward"
},
"validation": {
"reference_repeats": 3,
"target_attempts": 8,
"reward_min": 0,
"reward_max": 1,
"oracle_tolerance": 0,
"floor_margin": 0.1,
"wrong_answer_max": 0,
"required_negative_cases": [
"wrong-value",
"empty-answer",
"near-miss"
]
},
"generation": {
"candidate_count": 2
},
"negative_cases": [
{
"id": "wrong-value",
"script": "mkdir -p /app; printf '0\\n' > /app/answer.txt"
},
{
"id": "empty-answer",
"script": "mkdir -p /app; : > /app/answer.txt"
},
{
"id": "near-miss",
"script": "mkdir -p /app; printf '41\\n' > /app/answer.txt"
}
],
"seeds": [
1001,
1002,
1003,
1004,
1005,
1006,
1007,
1008
],
"cloud": {
"results_repo": "example-org/private-audit-results",
"flavor": "cpu-basic",
"timeout_seconds": 1800,
"max_sandboxes": 64,
"max_inference_calls": 160
}
}
Loading