Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions src/evalopt_graph/kernel.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,10 +157,23 @@ def __post_init__(self) -> None:
raise ValueError("observed_at must be a string")

def rows(name: str, values: Iterable[Iterable[str]], width: int) -> tuple[tuple[str, ...], ...]:
normalized = tuple(tuple(row) for row in values)
if isinstance(values, str | bytes):
raise ValueError(f"{name} entries require {width} string fields")
normalized: list[tuple[str, ...]] = []
try:
iterator = iter(values)
except TypeError as exc:
raise ValueError(f"{name} entries require {width} string fields") from exc
for row in iterator:
if isinstance(row, str | bytes):
raise ValueError(f"{name} entries require {width} string fields")
try:
normalized.append(tuple(row))
except TypeError as exc:
raise ValueError(f"{name} entries require {width} string fields") from exc
if any(len(row) != width or not all(isinstance(item, str) for item in row) for row in normalized):
raise ValueError(f"{name} entries require {width} string fields")
return normalized
return tuple(normalized)
Comment thread
coderabbitai[bot] marked this conversation as resolved.

object.__setattr__(self, "gate_results", rows("gate_results", self.gate_results, 2))
object.__setattr__(self, "contradictions", rows("contradictions", self.contradictions, 3))
Expand Down Expand Up @@ -217,10 +230,10 @@ def to_dict(self) -> dict[str, Any]:
def from_dict(cls, value: Mapping[str, Any]) -> AcceptanceInput:
return cls(
observed_at=str(value.get("observed_at", "")),
gate_results=tuple(tuple(item) for item in value.get("gate_results", ())),
gate_results=value.get("gate_results", ()),
criteria=tuple(value.get("criteria", ())),
claims=tuple(ClaimRecord(**item) for item in value.get("claims", ())),
contradictions=tuple(tuple(item) for item in value.get("contradictions", ())),
contradictions=value.get("contradictions", ()),
attestations=tuple(EvidenceAttestation.from_dict(item) for item in value.get("attestations", ())),
assessments=tuple(SupportAssessment.from_dict(item) for item in value.get("assessments", ())),
tests_weakened=value.get("tests_weakened", False),
Expand Down
45 changes: 45 additions & 0 deletions tests/test_kernel.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,51 @@ def test_nested_input_rows_are_copied_before_evaluation():
assert before.status == "ACCEPTED"


@pytest.mark.parametrize(
("field", "bad_row"),
[
("gate_results", "ok"),
("gate_results", b"ok"),
("contradictions", "abc"),
("contradictions", b"abc"),
],
)
def test_acceptance_input_rows_reject_scalar_rows_instead_of_splitting_characters(field, bad_row):
with pytest.raises(ValueError, match=f"{field} entries"):
kernel.AcceptanceInput(observed_at=NOW, **{field: (bad_row,)})

with pytest.raises(ValueError, match=f"{field} entries"):
kernel.AcceptanceInput.from_dict(
{
"schema_version": "evalopt.acceptance-input.v1",
"observed_at": NOW,
"gate_results": [],
"criteria": [],
"claims": [],
"contradictions": [],
"attestations": [],
"assessments": [],
"tests_weakened": False,
"evaluator_score": None,
field: [bad_row],
}
)


@pytest.mark.parametrize(
("field", "bad_value"),
[
("gate_results", None),
("gate_results", 1),
("contradictions", None),
("contradictions", 1),
],
)
def test_acceptance_input_rows_reject_non_iterable_row_containers(field, bad_value):
with pytest.raises(ValueError, match=f"{field} entries"):
kernel.AcceptanceInput(observed_at=NOW, **{field: bad_value})


def test_contradiction_enums_are_normalized_and_unknown_values_rejected():
policy, input_ = _input(contradictions=(("x1", "HIGH", "RESOLVED"),))

Expand Down