diff --git a/evaluation_function/correction/correction.py b/evaluation_function/correction/correction.py index 9a8a0a6..bf5207a 100644 --- a/evaluation_function/correction/correction.py +++ b/evaluation_function/correction/correction.py @@ -33,12 +33,11 @@ def _check_minimality(fsa: FSA) -> Tuple[bool, Optional[ValidationError]]: try: minimized = hopcroft_minimization(fsa) if len(minimized.states) < len(fsa.states): - diff = len(fsa.states) - len(minimized.states) return False, ValidationError( - message=f"Your FSA works correctly, but it's not minimal! You have {len(fsa.states)} states, but only {len(minimized.states)} are needed. You could remove {diff} state(s).", + message=f"FSA is not minimal: has {len(fsa.states)} states but can be reduced to {len(minimized.states)}", code=ErrorCode.NOT_MINIMAL, severity="error", - suggestion="Look for states that behave identically (same transitions and acceptance) - these can be merged into one" + suggestion="Minimize your FSA by merging equivalent states" ) return True, None except Exception: @@ -70,11 +69,9 @@ def _build_feedback( hints = [e.suggestion for e in all_errors if e.suggestion] if structural_info: if structural_info.unreachable_states: - unreachable = ", ".join(structural_info.unreachable_states) - hints.append(f"Tip: States {{{unreachable}}} can't be reached from your start state - you might want to remove them or add transitions to them") + hints.append("Consider removing unreachable states") if structural_info.dead_states: - dead = ", ".join(structural_info.dead_states) - hints.append(f"Tip: States {{{dead}}} can never lead to acceptance - this might be intentional (trap states) or a bug") + hints.append("Dead states can never lead to acceptance") # Build language comparison language = LanguageComparison(are_equivalent=len(equivalence_errors) == 0) @@ -95,20 +92,17 @@ def _summarize_errors(errors: List[ValidationError]) -> str: for error in errors: msg = error.message.lower() if "alphabet" in msg: - error_types.add("alphabet issue") - elif "states" in msg and ("many" in msg or "few" in msg or "needed" in msg): - error_types.add("incorrect number of states") - elif "accepting" in msg or "accept" in msg: - error_types.add("accepting states issue") - elif "transition" in msg or "reading" in msg: - error_types.add("transition issue") + error_types.add("alphabet mismatch") + elif "state" in msg and "count" in msg: + error_types.add("state count mismatch") + elif "accepting" in msg or "incorrectly marked" in msg: + error_types.add("acceptance error") + elif "transition" in msg: + error_types.add("transition error") - if len(error_types) == 1: - issue = list(error_types)[0] - return f"Almost there! Your FSA has an {issue}. Check the details below." - elif error_types: - return f"Your FSA doesn't quite match the expected language. Issues found: {', '.join(error_types)}" - return f"Your FSA doesn't accept the correct language. Found {len(errors)} issue(s) to fix." + if error_types: + return f"Languages differ: {', '.join(error_types)}" + return f"Languages differ: {len(errors)} issue(s)" # ============================================================================= @@ -140,11 +134,7 @@ def analyze_fsa_correction( # Step 1: Validate student FSA structure student_errors = is_valid_fsa(student_fsa) if student_errors: - num_errors = len(student_errors) - if num_errors == 1: - summary = "Your FSA has a structural problem that needs to be fixed first. See the details below." - else: - summary = f"Your FSA has {num_errors} structural problems that need to be fixed first. See the details below." + summary = "FSA has structural errors" return Result( is_correct=False, feedback=summary, @@ -156,7 +146,7 @@ def analyze_fsa_correction( if expected_errors: return Result( is_correct=False, - feedback="Oops! There's an issue with the expected answer. Please contact your instructor." + feedback="Internal error: expected FSA is invalid" ) # Step 3: Check minimality if required @@ -172,18 +162,15 @@ def analyze_fsa_correction( equivalence_errors = fsas_accept_same_language(student_fsa, expected_fsa) if not equivalence_errors and not validation_errors: - # Success message with some stats - state_count = len(student_fsa.states) - feedback = f"Correct! Your FSA with {state_count} state(s) accepts exactly the right language. Well done!" return Result( is_correct=True, - feedback=feedback, - fsa_feedback=_build_feedback("Your FSA is correct!", [], [], structural_info) + feedback="Correct! FSA accepts the expected language.", + fsa_feedback=_build_feedback("FSA is correct", [], [], structural_info) ) # Build result with errors is_correct = len(equivalence_errors) == 0 and len(validation_errors) == 0 - summary = _summarize_errors(equivalence_errors) if equivalence_errors else "Your FSA has some issues to address." + summary = _summarize_errors(equivalence_errors) if equivalence_errors else "FSA has issues" return Result( is_correct=is_correct, diff --git a/evaluation_function/preview.py b/evaluation_function/preview.py index 8331ec0..a47bcac 100755 --- a/evaluation_function/preview.py +++ b/evaluation_function/preview.py @@ -1,290 +1,30 @@ -""" -Preview function for FSA validation. - -The preview function validates student FSA responses BEFORE submission. -It catches clear structural errors early, preventing students from submitting -invalid FSAs for full evaluation. - -Validation checks performed: -1. Parse check - Is the response a valid FSA structure? -2. Structural validation - Are states, initial, accept states, and transitions valid? -3. Warnings - Unreachable states, dead states, non-determinism (if applicable) -""" - -from typing import Any, List, Dict +from typing import Any from lf_toolkit.preview import Result, Params, Preview -from .schemas import FSA, ValidationError -from .validation.validation import ( - is_valid_fsa, - is_deterministic, - find_unreachable_states, - find_dead_states, - get_structured_info_of_fsa, -) - - -def parse_fsa(value: Any) -> FSA: - """ - Parse an FSA from various input formats. - - Args: - value: FSA as dict or JSON string - - Returns: - Parsed FSA object - - Raises: - ValueError: If the input cannot be parsed as a valid FSA +def preview_function(response: Any, params: Params) -> Result: """ - if value is None: - raise ValueError("No FSA provided") - - if isinstance(value, str): - # Try to parse as JSON string - return FSA.model_validate_json(value) - elif isinstance(value, dict): - return FSA.model_validate(value) - else: - raise ValueError(f"Expected FSA as dict or JSON string, got {type(value).__name__}") + Function used to preview a student response. + --- + The handler function passes three arguments to preview_function(): + - `response` which are the answers provided by the student. + - `params` which are any extra parameters that may be useful, + e.g., error tolerances. -def format_errors_for_preview(errors: List[ValidationError], max_errors: int = 5) -> str: - """ - Format validation errors into a human-readable string for preview feedback. - - Args: - errors: List of ValidationError objects - max_errors: Maximum number of errors to show (to avoid overwhelming the user) - - Returns: - Formatted error string - """ - if not errors: - return "" - - # Separate errors by severity - critical_errors = [e for e in errors if e.severity == "error"] - warnings = [e for e in errors if e.severity == "warning"] - - lines = [] - - if critical_errors: - if len(critical_errors) == 1: - lines.append("There's an issue with your FSA that needs to be fixed:") - else: - lines.append(f"There are {len(critical_errors)} issues with your FSA that need to be fixed:") - lines.append("") - - for i, err in enumerate(critical_errors[:max_errors], 1): - lines.append(f" {i}. {err.message}") - if err.suggestion: - lines.append(f" >> {err.suggestion}") - lines.append("") - - if len(critical_errors) > max_errors: - lines.append(f" ... and {len(critical_errors) - max_errors} more issue(s)") - - if warnings: - if lines: - lines.append("") - lines.append("Some things to consider (not blocking, but worth checking):") - lines.append("") - for i, warn in enumerate(warnings[:max_errors], 1): - lines.append(f" - {warn.message}") - if warn.suggestion: - lines.append(f" >> {warn.suggestion}") - - if len(warnings) > max_errors: - lines.append(f" ... and {len(warnings) - max_errors} more suggestion(s)") - - return "\n".join(lines) + The output of this function is what is returned as the API response + and therefore must be JSON-encodable. It must also conform to the + response schema. + Any standard python library may be used, as well as any package + available on pip (provided it is added to requirements.txt). -def errors_to_dict_list(errors: List[ValidationError]) -> List[Dict]: - """ - Convert ValidationError objects to dictionaries for JSON serialization. + The way you wish to structure you code (all in this function, or + split into many) is entirely up to you. """ - return [ - { - "message": e.message, - "code": e.code.value if hasattr(e.code, 'value') else str(e.code), - "severity": e.severity, - "highlight": e.highlight.model_dump() if e.highlight else None, - "suggestion": e.suggestion - } - for e in errors - ] - -def preview_function(response: Any, params: Params) -> Result: - """ - Validate a student's FSA response before submission. - - This function performs structural validation to catch clear errors early, - preventing students from submitting obviously invalid FSAs for evaluation. - - Args: - response: Student's FSA response (dict or JSON string) - params: Extra parameters: - - require_deterministic (bool): Whether to require DFA (default: False) - - show_warnings (bool): Whether to show warnings (default: True) - - Returns: - Result with: - - preview.latex: FSA summary if valid - - preview.feedback: Error/warning messages if any - - preview.sympy: Structured validation data (errors, warnings, info) - """ - # Extract params with defaults - require_deterministic = False - show_warnings = True - - if hasattr(params, 'get'): - require_deterministic = params.get("require_deterministic", False) - show_warnings = params.get("show_warnings", True) - elif isinstance(params, dict): - require_deterministic = params.get("require_deterministic", False) - show_warnings = params.get("show_warnings", True) - try: - # Step 1: Parse the FSA - fsa = parse_fsa(response) - + return Result(preview=Preview(sympy=response)) + except FeedbackException as e: + return Result(preview=Preview(feedback=str(e))) except Exception as e: - # Failed to parse - this is a critical error - error_msg = str(e) - - # Make error message more user-friendly - if "validation error" in error_msg.lower(): - if "states" in error_msg.lower(): - feedback = "Your FSA is missing the 'states' list. Every FSA needs a set of states to define!" - elif "alphabet" in error_msg.lower(): - feedback = "Your FSA is missing the 'alphabet'. What symbols should your automaton recognize?" - elif "initial_state" in error_msg.lower(): - feedback = "Your FSA needs an initial state - this is where processing begins!" - elif "transitions" in error_msg.lower(): - feedback = "There's an issue with your transitions. Each transition needs a from_state, to_state, and symbol." - else: - feedback = f"Your FSA structure isn't quite right: {error_msg}" - elif "json" in error_msg.lower(): - feedback = "Couldn't read your FSA data. Make sure it's properly formatted." - elif "no fsa" in error_msg.lower() or "none" in error_msg.lower(): - feedback = "No FSA provided! Please build your automaton before checking." - else: - feedback = f"There's a problem with your FSA format: {error_msg}" - - return Result( - preview=Preview( - feedback=feedback, - sympy={ - "valid": False, - "parse_error": True, - "errors": [{"message": feedback, "code": "PARSE_ERROR", "severity": "error"}] - } - ) - ) - - # Step 2: Structural validation - all_errors: List[ValidationError] = [] - - # Run structural validation (states, initial, accept, transitions) - structural_errors = is_valid_fsa(fsa) - all_errors.extend(structural_errors) - - # If there are structural errors, don't proceed with other checks - if structural_errors: - feedback = "Your FSA has some issues that need to be fixed before submission.\n\n" - feedback += format_errors_for_preview(all_errors) - return Result( - preview=Preview( - feedback=feedback, - sympy={ - "valid": False, - "errors": errors_to_dict_list(all_errors), - "num_states": len(fsa.states), - "num_transitions": len(fsa.transitions) - } - ) - ) - - # Step 3: Additional checks (determinism, unreachable states, dead states) - warnings: List[ValidationError] = [] - - # Check determinism if required - if require_deterministic: - det_errors = is_deterministic(fsa) - if det_errors: - all_errors.extend(det_errors) - - # Check for warnings (unreachable/dead states) - if show_warnings: - unreachable = find_unreachable_states(fsa) - dead = find_dead_states(fsa) - warnings.extend(unreachable) - warnings.extend(dead) - - # Get structural info - try: - info = get_structured_info_of_fsa(fsa) - info_dict = info.model_dump() - except Exception: - info_dict = { - "num_states": len(fsa.states), - "num_transitions": len(fsa.transitions), - "is_deterministic": len(is_deterministic(fsa)) == 0 - } - - # Step 4: Build response - has_errors = len(all_errors) > 0 - has_warnings = len(warnings) > 0 - - if has_errors: - # Critical errors - cannot submit - feedback = "Hold on! Your FSA has issues that need to be addressed.\n\n" - feedback += format_errors_for_preview(all_errors + warnings) - return Result( - preview=Preview( - feedback=feedback, - sympy={ - "valid": False, - "errors": errors_to_dict_list(all_errors), - "warnings": errors_to_dict_list(warnings), - **info_dict - } - ) - ) - - # Build success message - state_word = "state" if len(fsa.states) == 1 else "states" - trans_word = "transition" if len(fsa.transitions) == 1 else "transitions" - - fsa_type = "DFA (Deterministic)" if info_dict.get("is_deterministic") else "NFA (Non-deterministic)" - - summary = f"{fsa_type} with {len(fsa.states)} {state_word} and {len(fsa.transitions)} {trans_word}" - alphabet_str = ", ".join(f"'{s}'" for s in fsa.alphabet) - - if has_warnings: - # Valid but with warnings - warning_feedback = format_errors_for_preview(warnings) - feedback = f"Looking good! Your FSA is structurally valid.\n\n" - feedback += f"Summary: {summary}\n" - feedback += f"Alphabet: {{{alphabet_str}}}\n\n" - feedback += warning_feedback - else: - feedback = f"Great! Your FSA is structurally valid and ready for submission.\n\n" - feedback += f"Summary: {summary}\n" - feedback += f"Alphabet: {{{alphabet_str}}}" - - return Result( - preview=Preview( - latex=summary, # Short summary for display - feedback=feedback, - sympy={ - "valid": True, - "errors": [], - "warnings": errors_to_dict_list(warnings), - **info_dict - } - ) - ) + return Result(preview=Preview(feedback=str(e))) diff --git a/evaluation_function/validation/validation.py b/evaluation_function/validation/validation.py index bd8b6d8..0471cda 100644 --- a/evaluation_function/validation/validation.py +++ b/evaluation_function/validation/validation.py @@ -1,3 +1,4 @@ +from itertools import product from typing import Dict, List, Set from collections import deque @@ -20,10 +21,10 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: if not states: errors.append( ValidationError( - message="Your FSA needs at least one state to work. Every automaton must have states to process input!", + message="The FSA has no states defined", code=ErrorCode.EMPTY_STATES, severity="error", - suggestion="Start by adding a state - this will be your starting point for the automaton" + suggestion="Add at least one state to the FSA" ) ) return errors # Early return since other checks depend on states @@ -32,10 +33,10 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: if not alphabet: errors.append( ValidationError( - message="Your FSA needs an alphabet - the set of symbols it can read. Without an alphabet, there's nothing to process!", + message="The alphabet is empty", code=ErrorCode.EMPTY_ALPHABET, severity="error", - suggestion="Define the input symbols your FSA should recognize (e.g., 'a', 'b', '0', '1')" + suggestion="Add at least one symbol to the alphabet" ) ) @@ -43,14 +44,14 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: if fsa.initial_state not in states: errors.append( ValidationError( - message=f"Oops! Your initial state '{fsa.initial_state}' doesn't exist in your FSA. The initial state must be one of your defined states.", + message=f"The initial state '{fsa.initial_state}' is not defined in the FSA", code=ErrorCode.INVALID_INITIAL, severity="error", highlight=ElementHighlight( type="initial_state", state_id=fsa.initial_state ), - suggestion=f"Either add '{fsa.initial_state}' to your states, or choose an existing state as the initial state" + suggestion="Include the initial state in your FSA or change your initial state" ) ) @@ -59,14 +60,14 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: if acc not in states: errors.append( ValidationError( - message=f"The accepting state '{acc}' isn't in your FSA. Accepting states must be part of your state set.", + message=f"The accept state '{acc}' is not defined in the FSA", code=ErrorCode.INVALID_ACCEPT, severity="error", highlight=ElementHighlight( type="accept_state", state_id=acc ), - suggestion=f"Either add '{acc}' to your states, or remove it from accepting states" + suggestion="Include the accept state in your FSA or change your accept state" ) ) @@ -75,7 +76,7 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: if t.from_state not in states: errors.append( ValidationError( - message=f"This transition starts from '{t.from_state}', but that state doesn't exist in your FSA.", + message=f"The source state '{t.from_state}' in transition '{t.symbol}' is not defined", code=ErrorCode.INVALID_TRANSITION_SOURCE, severity="error", highlight=ElementHighlight( @@ -84,13 +85,13 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: to_state=t.to_state, symbol=t.symbol ), - suggestion=f"Add '{t.from_state}' to your states, or update this transition to start from an existing state" + suggestion=f"Add state '{t.from_state}' to the FSA or change the transition source" ) ) if t.to_state not in states: errors.append( ValidationError( - message=f"This transition goes to '{t.to_state}', but that state doesn't exist in your FSA.", + message=f"The destination state '{t.to_state}' in transition '{t.symbol}' is not defined", code=ErrorCode.INVALID_TRANSITION_DEST, severity="error", highlight=ElementHighlight( @@ -99,13 +100,13 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: to_state=t.to_state, symbol=t.symbol ), - suggestion=f"Add '{t.to_state}' to your states, or update this transition to go to an existing state" + suggestion=f"Add state '{t.to_state}' to the FSA or change the transition destination" ) ) if t.symbol not in alphabet: errors.append( ValidationError( - message=f"The symbol '{t.symbol}' in this transition isn't in your alphabet. Transitions can only use symbols from the alphabet.", + message=f"The transition symbol '{t.symbol}' is not in the alphabet", code=ErrorCode.INVALID_SYMBOL, severity="error", highlight=ElementHighlight( @@ -114,7 +115,7 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: to_state=t.to_state, symbol=t.symbol ), - suggestion=f"Either add '{t.symbol}' to your alphabet, or change this transition to use an existing symbol" + suggestion=f"Add symbol '{t.symbol}' to the alphabet or change the transition symbol" ) ) @@ -139,7 +140,7 @@ def is_deterministic(fsa: FSA) -> List[ValidationError]: if key in seen: errors.append( ValidationError( - message=f"Your FSA has multiple transitions from state '{t.from_state}' when reading '{t.symbol}'. In a DFA, each state can only have one transition per symbol.", + message=f"Non-deterministic: multiple transitions from '{t.from_state}' on symbol '{t.symbol}'", code=ErrorCode.DUPLICATE_TRANSITION, severity="error", highlight=ElementHighlight( @@ -148,7 +149,7 @@ def is_deterministic(fsa: FSA) -> List[ValidationError]: to_state=t.to_state, symbol=t.symbol ), - suggestion=f"Keep only one transition from '{t.from_state}' on '{t.symbol}', or if you meant to create an NFA, that's also valid!" + suggestion="Remove duplicate transitions or convert to NFA if nondeterminism is intended" ) ) seen.add(key) @@ -168,7 +169,7 @@ def is_complete(fsa: FSA) -> List[ValidationError]: errors.extend(det_errors) errors.append( ValidationError( - message="We can only check completeness for deterministic FSAs. Please fix the determinism issues first.", + message="Cannot check completeness for non-deterministic FSA", code=ErrorCode.NOT_DETERMINISTIC, severity="error" ) @@ -184,7 +185,7 @@ def is_complete(fsa: FSA) -> List[ValidationError]: if (state, symbol) not in transition_keys: errors.append( ValidationError( - message=f"State '{state}' is missing a transition for symbol '{symbol}'. A complete DFA needs transitions for every symbol from every state.", + message=f"Missing transition from state '{state}' on symbol '{symbol}' to make the FSA complete", code=ErrorCode.MISSING_TRANSITION, severity="error", highlight=ElementHighlight( @@ -192,7 +193,7 @@ def is_complete(fsa: FSA) -> List[ValidationError]: state_id=state, symbol=symbol ), - suggestion=f"Add a transition from '{state}' when reading '{symbol}' - it can go to any state, including a 'trap' state" + suggestion=f"Add a transition from state '{state}' on symbol '{symbol}'" ) ) return errors @@ -226,14 +227,14 @@ def find_unreachable_states(fsa: FSA) -> List[ValidationError]: if state not in visited: errors.append( ValidationError( - message=f"State '{state}' can never be reached! There's no path from your initial state to this state.", + message=f"State '{state}' is unreachable from the initial state", code=ErrorCode.UNREACHABLE_STATE, - severity="warning", + severity="warning", # Changed to warning as it's not always an error highlight=ElementHighlight( type="state", state_id=state ), - suggestion=f"Connect '{state}' to your FSA by adding a transition to it, or remove it if it's not needed" + suggestion=f"Add a transition to state '{state}' from a reachable state, or remove it if unnecessary" ) ) return errors @@ -252,14 +253,14 @@ def find_dead_states(fsa: FSA) -> List[ValidationError]: if state != fsa.initial_state or state not in fsa.accept_states: errors.append( ValidationError( - message=f"Your FSA has no accepting states, so no input string can ever be accepted! This means the language is empty.", + message=f"State '{state}' cannot reach any accepting state (no accept states defined)", code=ErrorCode.DEAD_STATE, severity="warning", highlight=ElementHighlight( type="state", state_id=state ), - suggestion="If you want your FSA to accept some strings, mark at least one state as accepting" + suggestion="Add at least one accept state to the FSA" ) ) return errors @@ -284,14 +285,14 @@ def find_dead_states(fsa: FSA) -> List[ValidationError]: if state not in reachable_to_accept: errors.append( ValidationError( - message=f"State '{state}' is a dead end - once you enter it, you can never reach an accepting state. This is often called a 'trap state'.", + message=f"State '{state}' is dead (cannot reach any accepting state)", code=ErrorCode.DEAD_STATE, - severity="warning", + severity="warning", # Changed to warning as it's not always an error highlight=ElementHighlight( type="state", state_id=state ), - suggestion=f"This might be intentional (to reject certain inputs), or you could add a path from '{state}' to an accepting state" + suggestion=f"Add a transition from state '{state}' to a state that can reach an accept state, or make state '{state}' accepting" ) ) return errors @@ -416,6 +417,9 @@ def get_structured_info_of_fsa(fsa: FSA) -> StructuralInfo: """ Get structured information about the FSA including properties and analysis. """ + # Get validation errors first + validation_errors = is_valid_fsa(fsa) + # Check determinism - returns boolean det_errors = is_deterministic(fsa) is_deterministic_bool = len(det_errors) == 0 @@ -456,44 +460,25 @@ def are_isomorphic(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: errors = [] # 1. Alphabet Check (Mandatory) if set(fsa1.alphabet) != set(fsa2.alphabet): - student_only = set(fsa1.alphabet) - set(fsa2.alphabet) - expected_only = set(fsa2.alphabet) - set(fsa1.alphabet) - - msg_parts = ["Your alphabet doesn't match what's expected."] - if student_only: - msg_parts.append(f"You have extra symbols: {student_only}") - if expected_only: - msg_parts.append(f"You're missing symbols: {expected_only}") - errors.append( ValidationError( - message=" ".join(msg_parts), + message="The alphabet of your FSA does not match the required alphabet.", code=ErrorCode.LANGUAGE_MISMATCH, severity="error", - suggestion="Make sure your alphabet contains exactly the symbols needed for this language" + suggestion=f"Your alphabet: {set(fsa1.alphabet)}. Expected: {set(fsa2.alphabet)}." ) ) # 2. Basic Structural Check (State Count) if len(fsa1.states) != len(fsa2.states): - if len(fsa1.states) > len(fsa2.states): - errors.append( - ValidationError( - message=f"Your FSA has {len(fsa1.states)} states, but the minimal solution only needs {len(fsa2.states)}. You might have redundant states.", - code=ErrorCode.LANGUAGE_MISMATCH, - severity="error", - suggestion="Look for states that behave identically and could be merged, or check for unreachable states" - ) - ) - else: - errors.append( - ValidationError( - message=f"Your FSA has {len(fsa1.states)} states, but at least {len(fsa2.states)} are needed. You might be missing some states.", - code=ErrorCode.LANGUAGE_MISMATCH, - severity="error", - suggestion="Think about what different 'situations' your FSA needs to remember - each usually needs its own state" - ) + errors.append( + ValidationError( + message=f"FSA structure mismatch: expected {len(fsa2.states)} states, but found {len(fsa1.states)}.", + code=ErrorCode.LANGUAGE_MISMATCH, + severity="error", + suggestion="Verify if you have unnecessary states or if you have minimized your FSA." ) + ) # 3. State Mapping Initialization mapping: Dict[str, str] = {fsa1.initial_state: fsa2.initial_state} @@ -512,26 +497,16 @@ def are_isomorphic(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: # 4. Check Acceptance Parity if (s1 in accept1) != (s2 in accept2): - if s2 in accept2: - errors.append( - ValidationError( - message=f"State '{s1}' should be an accepting state, but it's not marked as one. Strings that end here should be accepted!", - code=ErrorCode.LANGUAGE_MISMATCH, - severity="error", - highlight=ElementHighlight(type="state", state_id=s1), - suggestion=f"Mark state '{s1}' as an accepting state (add it to your accept states)" - ) - ) - else: - errors.append( - ValidationError( - message=f"State '{s1}' is marked as accepting, but it shouldn't be. Strings that end here should be rejected!", - code=ErrorCode.LANGUAGE_MISMATCH, - severity="error", - highlight=ElementHighlight(type="state", state_id=s1), - suggestion=f"Remove state '{s1}' from your accepting states" - ) + expected_type = "accepting" if s2 in accept2 else "non-accepting" + errors.append( + ValidationError( + message=f"State '{s1}' is incorrectly marked. It should be an {expected_type} state.", + code=ErrorCode.LANGUAGE_MISMATCH, + severity="error", + highlight=ElementHighlight(type="state", state_id=s1), + suggestion=f"Toggle the 'accept' status of state '{s1}'." ) + ) # 5. Check Transitions for every symbol in the shared alphabet for symbol in fsa1.alphabet: @@ -540,26 +515,15 @@ def are_isomorphic(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: # Missing Transition Check if (dest1 is None) != (dest2 is None): - if dest1 is None: - errors.append( - ValidationError( - message=f"State '{s1}' is missing a transition for symbol '{symbol}'. What should happen when you read '{symbol}' here?", - code=ErrorCode.LANGUAGE_MISMATCH, - severity="error", - highlight=ElementHighlight(type="state", state_id=s1, symbol=symbol), - suggestion=f"Add a transition from '{s1}' on '{symbol}' to handle this input" - ) - ) - else: - errors.append( - ValidationError( - message=f"State '{s1}' has an unexpected transition on '{symbol}'. This transition might not be needed.", - code=ErrorCode.LANGUAGE_MISMATCH, - severity="error", - highlight=ElementHighlight(type="state", state_id=s1, symbol=symbol), - suggestion=f"Review if the transition from '{s1}' on '{symbol}' is correct" - ) + errors.append( + ValidationError( + message=f"Missing or extra transition from state '{s1}' on symbol '{symbol}'.", + code=ErrorCode.LANGUAGE_MISMATCH, + severity="error", + highlight=ElementHighlight(type="state", state_id=s1, symbol=symbol), + suggestion="Ensure your DFA is complete and follows the transition logic." ) + ) if dest1 is not None: if dest1 not in mapping: @@ -572,7 +536,7 @@ def are_isomorphic(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: if mapping[dest1] != dest2: errors.append( ValidationError( - message=f"When in state '{s1}' and reading '{symbol}', you go to '{dest1}', but that leads to incorrect behavior. Check where this transition should go!", + message=f"Transition from '{s1}' on '{symbol}' leads to the wrong state.", code=ErrorCode.LANGUAGE_MISMATCH, severity="error", highlight=ElementHighlight( @@ -581,7 +545,7 @@ def are_isomorphic(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: to_state=dest1, symbol=symbol ), - suggestion=f"Think about what state the FSA should be in after reading '{symbol}' from '{s1}' - try tracing through some example strings" + suggestion="Check if this transition should point to a different state." ) ) diff --git a/test_local.py b/test_local.py index 01f3d2b..04804eb 100644 --- a/test_local.py +++ b/test_local.py @@ -448,268 +448,14 @@ def run_test(test_num, test_name, response_data, answer_data, params=None): } ) -# ============================================================================= -# PREVIEW FUNCTION TESTS -# ============================================================================= - -print("\n") -print("#" * 70) -print("# PREVIEW FUNCTION TESTS (Pre-submission Validation)") -print("#" * 70) -print() - -from evaluation_function.preview import preview_function -from lf_toolkit.preview import Params as PreviewParams - - -def run_preview_test(test_num, test_name, response_data, params=None): - """Helper function to run a preview test case""" - print("=" * 70) - print(f"Preview Test {test_num}: {test_name}") - print("=" * 70) - - if params is None: - params = PreviewParams() - - try: - result = preview_function(response_data, params) - - # Handle both Result object and dict - if hasattr(result, 'to_dict'): - result_dict = result.to_dict() - elif isinstance(result, dict): - result_dict = result - else: - result_dict = {'preview': result} - - # Extract preview data - preview_data = result_dict.get('preview', {}) - if hasattr(preview_data, 'model_dump'): - preview_data = preview_data.model_dump() - - sympy_data = preview_data.get('sympy', {}) - is_valid = sympy_data.get('valid', False) if sympy_data else False - - status = "[VALID]" if is_valid else "[INVALID]" - print(f"{status} FSA is valid: {is_valid}") - - feedback = preview_data.get('feedback', '') - if feedback: - print(f" Feedback:\n {str(feedback).replace(chr(10), chr(10) + ' ')}") - - if sympy_data and sympy_data.get('errors'): - print(f" Errors: {len(sympy_data['errors'])}") - if sympy_data and sympy_data.get('warnings'): - print(f" Warnings: {len(sympy_data['warnings'])}") - - print() - return result_dict - except Exception as e: - print(f"[ERROR] {e}") - import traceback - traceback.print_exc() - print() - return None - - -# Preview Test P1: Valid DFA -run_preview_test( - "P1", "Valid DFA - should pass", - response_data={ - "states": ["q0", "q1"], - "alphabet": ["a", "b"], - "transitions": [ - {"from_state": "q0", "to_state": "q1", "symbol": "a"}, - {"from_state": "q1", "to_state": "q1", "symbol": "a"}, - {"from_state": "q1", "to_state": "q1", "symbol": "b"} - ], - "initial_state": "q0", - "accept_states": ["q1"] - } -) - -# Preview Test P2: Invalid initial state -run_preview_test( - "P2", "Invalid initial state - should fail", - response_data={ - "states": ["q0", "q1"], - "alphabet": ["a", "b"], - "transitions": [ - {"from_state": "q0", "to_state": "q1", "symbol": "a"} - ], - "initial_state": "q99", # Does not exist - "accept_states": ["q1"] - } -) - -# Preview Test P3: Invalid accept state -run_preview_test( - "P3", "Invalid accept state - should fail", - response_data={ - "states": ["q0", "q1"], - "alphabet": ["a", "b"], - "transitions": [ - {"from_state": "q0", "to_state": "q1", "symbol": "a"} - ], - "initial_state": "q0", - "accept_states": ["q99"] # Does not exist - } -) - -# Preview Test P4: Transition references non-existent state -run_preview_test( - "P4", "Transition to non-existent state - should fail", - response_data={ - "states": ["q0", "q1"], - "alphabet": ["a", "b"], - "transitions": [ - {"from_state": "q0", "to_state": "q99", "symbol": "a"} # q99 doesn't exist - ], - "initial_state": "q0", - "accept_states": ["q1"] - } -) - -# Preview Test P5: Transition with invalid symbol -run_preview_test( - "P5", "Transition with symbol not in alphabet - should fail", - response_data={ - "states": ["q0", "q1"], - "alphabet": ["a", "b"], - "transitions": [ - {"from_state": "q0", "to_state": "q1", "symbol": "c"} # 'c' not in alphabet - ], - "initial_state": "q0", - "accept_states": ["q1"] - } -) - -# Preview Test P6: FSA with unreachable states (warning) -run_preview_test( - "P6", "FSA with unreachable states - valid with warning", - response_data={ - "states": ["q0", "q1", "q2", "q3"], - "alphabet": ["a", "b"], - "transitions": [ - {"from_state": "q0", "to_state": "q1", "symbol": "a"}, - {"from_state": "q1", "to_state": "q1", "symbol": "a"}, - {"from_state": "q1", "to_state": "q1", "symbol": "b"}, - {"from_state": "q2", "to_state": "q3", "symbol": "a"} # q2, q3 unreachable - ], - "initial_state": "q0", - "accept_states": ["q1"] - } -) - -# Preview Test P7: FSA with dead states (warning) -run_preview_test( - "P7", "FSA with dead states - valid with warning", - response_data={ - "states": ["q0", "q1", "dead"], - "alphabet": ["a", "b"], - "transitions": [ - {"from_state": "q0", "to_state": "q1", "symbol": "a"}, - {"from_state": "q0", "to_state": "dead", "symbol": "b"}, - {"from_state": "q1", "to_state": "q1", "symbol": "a"}, - {"from_state": "q1", "to_state": "q1", "symbol": "b"}, - {"from_state": "dead", "to_state": "dead", "symbol": "a"}, - {"from_state": "dead", "to_state": "dead", "symbol": "b"} - ], - "initial_state": "q0", - "accept_states": ["q1"] # "dead" can never reach accept - } -) - -# Preview Test P8: Not a valid FSA structure (parse error) -run_preview_test( - "P8", "Invalid structure - missing required fields", - response_data={ - "states": ["q0"], - # Missing alphabet, transitions, initial_state, accept_states - } -) - -# Preview Test P9: Empty states list -run_preview_test( - "P9", "Empty states list - should fail", - response_data={ - "states": [], - "alphabet": ["a"], - "transitions": [], - "initial_state": "q0", - "accept_states": [] - } -) - -# Preview Test P10: NFA (valid, non-deterministic) -run_preview_test( - "P10", "Valid NFA - non-deterministic allowed", - response_data={ - "states": ["q0", "q1", "q2"], - "alphabet": ["a", "b"], - "transitions": [ - {"from_state": "q0", "to_state": "q1", "symbol": "a"}, - {"from_state": "q0", "to_state": "q2", "symbol": "a"}, # Non-deterministic - {"from_state": "q1", "to_state": "q1", "symbol": "b"}, - {"from_state": "q2", "to_state": "q2", "symbol": "b"} - ], - "initial_state": "q0", - "accept_states": ["q1"] - } -) - -# Preview Test P11: Epsilon transitions -run_preview_test( - "P11", "Epsilon NFA - epsilon transitions", - response_data={ - "states": ["q0", "q1", "q2"], - "alphabet": ["a", "b"], - "transitions": [ - {"from_state": "q0", "to_state": "q1", "symbol": "a"}, - {"from_state": "q1", "to_state": "q2", "symbol": "b"} - ], - "initial_state": "q0", - "accept_states": ["q2"] - } -) - -# Preview Test P12: Null/None response -run_preview_test( - "P12", "Null response - should fail gracefully", - response_data=None -) - -# Preview Test P13: String response (JSON) -import json -run_preview_test( - "P13", "JSON string response - should parse", - response_data=json.dumps({ - "states": ["q0", "q1"], - "alphabet": ["a"], - "transitions": [ - {"from_state": "q0", "to_state": "q1", "symbol": "a"}, - {"from_state": "q1", "to_state": "q1", "symbol": "a"} - ], - "initial_state": "q0", - "accept_states": ["q1"] - }) -) - print("=" * 70) -print("ALL EVALUATION + PREVIEW TESTS COMPLETED!") +print("ALL TESTS COMPLETED!") print("=" * 70) print("\nRun with: python test_local.py") -print("\nEvaluation tests cover:") +print("These tests cover:") print(" - Basic DFA equivalence") print(" - Non-deterministic FSAs (NFAs)") -print(" - Epsilon transitions") +print(" - Epsilon transitions (ε-NFAs)") print(" - Edge cases (empty language, single state, unreachable states)") print(" - Complex patterns (ending with 'ab', divisibility by 3)") -print("\nPreview tests cover:") -print(" - Valid FSA validation") -print(" - Invalid initial/accept states") -print(" - Invalid transitions (state/symbol)") -print(" - Unreachable and dead states (warnings)") -print(" - Parse errors (invalid structure, null input)") -print(" - NFA and epsilon transition support") -print(" - JSON string input parsing") +print(" - Validation errors")