Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 22 additions & 13 deletions evaluation_function/evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,49 +5,58 @@
from .correction import analyze_fsa_correction

def validate_fsa(value: str | dict) -> FSA:
"""Parse a FSA from JSON string or dict."""
if isinstance(value, str):
return FSA.model_validate_json(value)
return FSA.model_validate(value)

def evaluation_function(payload: Any) -> LFResult:
def evaluation_function(
response: Any = None,
answer: Any = None,
params: Any = None # Temp workaround: treat params as Any
) -> LFResult:
"""
Evaluate a student's FSA response against the expected answer.
Temporary FSA evaluation function.

Args:
payload: dict with keys 'response', 'answer', 'params' (front-end may wrap everything)
response: Student's FSA (may be None if frontend wraps everything in params)
answer: Expected FSA (may be None)
params: Additional parameters (or full payload if frontend wraps everything here)

Returns:
LFResult
LFResult with is_correct and feedback_items
"""
try:
# Extract response/answer from the payload
raw_response = payload.get("response") or payload.get("params", {}).get("response")
raw_answer = payload.get("answer") or payload.get("params", {}).get("answer")
params = payload.get("params", {})
# Extract student and expected FSAs from whatever is present
raw_response = response or getattr(params, "response", None) or params.get("response", None)
raw_answer = answer or getattr(params, "answer", None) or params.get("answer", None)
extra_params = params.get("params") or params

if raw_response is None or raw_answer is None:
raise ValueError("Missing response or answer in payload")
raise ValueError("Missing FSA data: response or answer is None")

# Parse FSAs
student_fsa = validate_fsa(raw_response)
expected_fsa = validate_fsa(raw_answer)

require_minimal = params.get("require_minimal", False)
require_minimal = extra_params.get("require_minimal", False) if isinstance(extra_params, dict) else False

# Run correction
# Run correction pipeline
result: Result = analyze_fsa_correction(student_fsa, expected_fsa, require_minimal)

# Convert to LFResult
# Return LFResult
return LFResult(
is_correct=result.is_correct,
feedback_items=[("feedback", result.feedback)]
)

except Exception as e:
# Always return LFResult with raw payload for debugging
return LFResult(
is_correct=False,
feedback_items=[(
"error",
f"Invalid FSA format: {str(e)}\n\npayload received:\n{payload}"
f"Invalid FSA format: {str(e)}\n\n"
f"response: {response}\nanswer: {answer}\nparams: {params}"
)]
)