Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions envs/coding_env/server/transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,19 @@ class CodeSafetyTransform(Transform):

def __init__(self, penalty: float = -1.0):
self.penalty = penalty
# \b stops a match inside a longer name such as my_open( or model.eval(
# style names; (?<![\w.]) keeps eval( and exec( from matching inside
# identifiers or method calls. popen/Popen are listed because \bopen(
# no longer catches them.
self.dangerous_patterns = [
r"import\s+os",
r"import\s+subprocess",
r"eval\(",
r"exec\(",
r"\bimport\s+os\b",
r"\bimport\s+subprocess\b",
r"(?<![\w.])eval\(",
r"(?<![\w.])exec\(",
r"__import__",
r"open\(",
r"\bopen\(",
r"\bpopen\(",
r"\bPopen\(",
]

def __call__(self, observation: Observation) -> Observation:
Expand Down
58 changes: 57 additions & 1 deletion tests/envs/test_python_codeact_rewards.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,9 @@
pytest.importorskip("smolagents", reason="smolagents is not installed")


from envs.coding_env.models import CodeAction
from envs.coding_env.models import CodeAction, CodeObservation
from envs.coding_env.server.python_codeact_env import PythonCodeActEnv
from envs.coding_env.server.transforms import CodeSafetyTransform


# ============================================================================
Expand Down Expand Up @@ -270,3 +271,58 @@ def test_multiline_code_with_mixed_patterns(env):
# Should be flagged as dangerous even with safe code mixed in
assert obs.reward < 0, "Code with dangerous import should have negative reward"
assert "safety_violation" in obs.metadata


# ============================================================================
# CodeSafetyTransform boundaries
# ============================================================================


def _safety(code):
obs = CodeObservation(metadata={"last_code": code})
return CodeSafetyTransform()(obs)


class TestCodeSafetyTransform:
"""The safety penalty must hit dangerous calls, not names that contain one."""

@pytest.mark.parametrize(
"code",
[
"ast.literal_eval(s)",
"urllib.request.urlopen(u)",
"import osmnx",
"import os_helpers",
"model.eval()",
'df.eval("a + b")',
],
ids=["literal_eval", "urlopen", "osmnx", "os_helpers", "model.eval", "df.eval"],
)
def test_names_containing_a_pattern_are_not_penalized(self, code):
obs = _safety(code)

assert obs.reward == 0.0
assert "safety_violation" not in obs.metadata

@pytest.mark.parametrize(
"code",
[
"import os",
"import os.path",
"import os, sys",
"import subprocess",
"eval('1+1')",
"exec('x=1')",
"__import__('os')",
"open('f.txt')",
"io.open('f')",
"os.open('f', 0)",
"os.popen('ls')",
"subprocess.Popen(['ls'])",
],
)
def test_dangerous_code_is_penalized(self, code):
obs = _safety(code)

assert obs.reward == -1.0
assert "safety_violation" in obs.metadata