From b07241ec981fa63357fae853e79bef1d7d1c2249 Mon Sep 17 00:00:00 2001 From: Sachin Adlakha Date: Mon, 6 Oct 2025 18:11:39 -0400 Subject: [PATCH] Checkpoint: timeout on console and gracefull ending and release of console locks, with attempt to fix Error: Unknown Error --- server/command/hybrid_repl_thread.py | 132 ++++++++++++++++++++++----- server/command/ide_cmd.py | 4 +- server/config.py | 2 +- 3 files changed, 112 insertions(+), 26 deletions(-) diff --git a/server/command/hybrid_repl_thread.py b/server/command/hybrid_repl_thread.py index 8701c913..73e6d026 100644 --- a/server/command/hybrid_repl_thread.py +++ b/server/command/hybrid_repl_thread.py @@ -454,7 +454,7 @@ def run(self): os.unlink(wrapper_path) except: pass - + # Ensure process is terminated if self.p: try: @@ -465,51 +465,111 @@ def run(self): self.p.kill() except: pass + + # Always release execution lock in finally to ensure cleanup + self._release_execution_lock() def monitor_timeout(self): """Monitor execution time and kill if exceeds limit""" - # Disable timeout monitoring for now - it's interfering with input() - # TODO: Implement smarter timeout that pauses during input wait - return - + # Smart timeout that handles input() calls properly + # Only monitor during script execution, not REPL + consecutive_high_cpu = 0 + last_cpu_time = 0 + + # Track output flooding + output_count = 0 + output_window_start = time.time() + OUTPUT_FLOOD_THRESHOLD = 1000 # lines per second + OUTPUT_WINDOW_SIZE = 1 # 1 second window + while self.alive and self.p and not self.repl_mode: - elapsed = time.time() - self.start_time - - # Check if timeout exceeded (add grace period for REPL transition) - if elapsed > self.cpu_time_limit + 5: - print(f"[TIMEOUT] Process {self.cmd_id} exceeded time limit ({self.cpu_time_limit}s)") - self.response_to_client(1, { - 'stdout': f'\n[TIMEOUT] Execution time limit exceeded ({self.cpu_time_limit} seconds)\n' - }) - self.kill() - break - - # Also monitor memory usage if psutil available try: if self.p and self.p.pid: process = psutil.Process(self.p.pid) + + # Get CPU time (actual computation time, not wall clock) + cpu_times = process.cpu_times() + current_cpu_time = cpu_times.user + cpu_times.system + + # Check if process is actively computing + cpu_delta = current_cpu_time - last_cpu_time + last_cpu_time = current_cpu_time + + # If CPU usage is high for consecutive checks, it's likely an infinite loop + if cpu_delta > 0.8: # Using more than 80% CPU in the last second + consecutive_high_cpu += 1 + else: + consecutive_high_cpu = 0 # Reset if CPU usage drops (likely waiting for input) + + # Check total CPU time consumed + if current_cpu_time > self.cpu_time_limit: + print(f"[TIMEOUT] Process {self.cmd_id} exceeded CPU time limit ({self.cpu_time_limit}s)") + self.response_to_client(0, { + 'stdout': f'\nConsole ending: Timeout 1 minute\n' + }) + # Mark as had error to prevent REPL from opening + self.had_error = True + # Release execution lock + self._release_execution_lock() + # Send error signal to update UI (button state) + self.response_to_client(4000, {'error': 'Execution timeout exceeded'}) + self.kill() + break + + # Also check for sustained high CPU (infinite loop detection) + if consecutive_high_cpu >= 10: # 10 seconds of continuous high CPU + print(f"[INFINITE LOOP] Process {self.cmd_id} detected in infinite loop") + # Send message to user console FIRST + self.response_to_client(0, { + 'stdout': f'\n\nConsole ending: Timeout 1 minute (infinite loop detected)\n' + }) + # Mark as had error to prevent REPL from opening + self.had_error = True + # Release execution lock + self._release_execution_lock() + # Send error signal to update UI (button state) + self.response_to_client(4000, {'error': 'Infinite loop detected'}) + # Then kill the process + self.kill() + break + + # Memory check memory_mb = process.memory_info().rss / (1024 * 1024) - if memory_mb > self.memory_limit_mb: print(f"[MEMORY] Process {self.cmd_id} exceeded memory limit ({self.memory_limit_mb}MB)") - self.response_to_client(1, { + self.response_to_client(0, { 'stdout': f'\n[MEMORY LIMIT] Memory usage exceeded ({self.memory_limit_mb}MB)\n' }) + # Mark as had error to prevent REPL from opening + self.had_error = True + # Release execution lock + self._release_execution_lock() + # Send error signal to update UI (button state) + self.response_to_client(4000, {'error': 'Memory limit exceeded'}) self.kill() break - except: - # psutil not available or process already dead + + except (psutil.NoSuchProcess, psutil.AccessDenied): + # Process already terminated or access denied + break + except Exception as e: + print(f"[TIMEOUT-MONITOR] Error monitoring process: {e}") + # Continue monitoring even if there's an error pass - + time.sleep(1) def read_output(self): """Read output from subprocess""" buffer = "" - + + # Track output rate for flood detection + self.output_line_count = 0 + self.output_window_start = time.time() + print(f"[HYBRID-REPL] Starting output reader for cmd_id: {self.cmd_id}") - + while self.alive and self.p and self.p.poll() is None: try: # Read character by character to handle input prompts without newlines @@ -609,6 +669,30 @@ def read_output(self): # Strip trailing whitespace only, preserve intentional leading spaces clean_line = line.rstrip() print(f"[HYBRID-REPL] Sending line: {repr(clean_line)}") + + # Track output rate for flood detection + self.output_line_count += 1 + current_time = time.time() + + # Reset window if needed + if current_time - self.output_window_start > 1: + self.output_line_count = 1 + self.output_window_start = current_time + + # Check for output flooding (too many lines per second) + if self.output_line_count > 100: # More than 100 lines/second + print(f"[OUTPUT FLOOD] Process {self.cmd_id} generating excessive output") + self.response_to_client(0, { + 'stdout': f'\n\nConsole ending: Excessive output detected (infinite print loop)\n' + }) + self.had_error = True + # Release execution lock + self._release_execution_lock() + # Send error signal to update UI (button state) + self.response_to_client(4000, {'error': 'Excessive output detected'}) + self.kill() + return # Exit the read_output function + self.response_to_client(0, {'stdout': clean_line}) # Keep the incomplete line in buffer buffer = lines[-1] diff --git a/server/command/ide_cmd.py b/server/command/ide_cmd.py index ffb266b4..3f856adf 100644 --- a/server/command/ide_cmd.py +++ b/server/command/ide_cmd.py @@ -681,7 +681,9 @@ async def run_python_program(self, client, cmd_id, data): lock_acquired = execution_lock_manager.acquire_execution_lock(username, file_path, cmd_id, timeout=2.0) if not lock_acquired: print(f"[BACKEND-DEBUG] Could not acquire execution lock for user {username}, file {file_path}, cmd_id: {cmd_id}") - await response(client, cmd_id, -1, 'You already have this file running. Please wait for it to complete.') + await response(client, cmd_id, -1, { + 'error': 'Script is already running. Please wait for it to complete or click Stop first.' + }) return try: diff --git a/server/config.py b/server/config.py index a44468ce..e640ae9b 100644 --- a/server/config.py +++ b/server/config.py @@ -29,7 +29,7 @@ class Config: # Resource limits MAX_CONCURRENT_EXECUTIONS = int(os.getenv('MAX_CONCURRENT_EXECUTIONS', 60)) MAX_PROCESSES_PER_USER = int(os.getenv('MAX_PROCESSES_PER_USER', 2)) # Max 2 processes per user - EXECUTION_TIMEOUT = int(os.getenv('EXECUTION_TIMEOUT', 30)) # seconds + EXECUTION_TIMEOUT = int(os.getenv('EXECUTION_TIMEOUT', 60)) # seconds - 1 minute timeout MEMORY_LIMIT_MB = int(os.getenv('MEMORY_LIMIT_MB', 128)) MAX_PROCESS_AGE = int(os.getenv('MAX_PROCESS_AGE', 1800)) # 30 minutes MAX_REPL_AGE = int(os.getenv('MAX_REPL_AGE', 3600)) # 60 minutes