fix(worker-ops): preserve queue tracking and prevent scheduler recovery stalls

.github/workflows/e2e_test.yml

@@ -18,16 +18,16 @@ jobs:
         with:
           token: ${{ secrets.GH_PAT }}
           submodules: recursive
-      
+
       - name: Install Flow CLI
         run: sh -ci "$(curl -fsSL https://raw.githubusercontent.com/onflow/flow-cli/master/install.sh)"
-      
+
       - name: Update PATH
         run: echo "$HOME/.local/bin" >> $GITHUB_PATH
-      
+
       - name: Verify Flow CLI Installation
         run: flow version
-      
+
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
@@ -44,12 +44,12 @@ jobs:
         run: |
           chmod +x ./local/setup_and_run_emulator.sh
           chmod +x ./local/deploy_full_stack.sh
-      
+
       - name: Setup and Run Emulator
         run: |
           ./local/setup_and_run_emulator.sh &
           sleep 80  # Wait for the emulator to be fully up
-      
+
       - name: Deploy Full Stack
         run: |
           set -o pipefail
@@ -120,45 +120,45 @@ jobs:
             --legacy
         env:
           AMOUNT: 10000000000000000000
-      
+
       - name: Process Create Request
         run: |
           flow transactions send ./cadence/transactions/process_requests.cdc 0 10 --signer emulator-flow-yield-vaults --compute-limit 9999
-      
+
       - name: Verify YieldVault Creation
         run: |
           echo "=== Verifying YieldVault Creation ==="
-          
+
           # Check yieldvault details using the account-level script
           YIELDVAULT_CHECK=$(flow scripts execute ./cadence/scripts/check_yieldvault_details.cdc 0x045a1763c93006ca)
           echo "$YIELDVAULT_CHECK"
-          
+
           # Verify that we have at least one EVM address with yieldvaults
           if echo "$YIELDVAULT_CHECK" | grep -q '"totalEVMAddresses": 1'; then
             echo "✅ EVM address registered"
           else
             echo "❌ No EVM addresses found"
             exit 1
           fi
-          
+
           # Verify that we have at least one yieldvault created
           if echo "$YIELDVAULT_CHECK" | grep -q '"totalMappedYieldVaults": 1'; then
             echo "✅ YieldVault created successfully"
           else
             echo "❌ No yieldvaults found"
             exit 1
           fi
-          
+
           # Verify the specific EVM address has the yieldvault
           if echo "$YIELDVAULT_CHECK" | grep -q '6813eb9362372eef6200f3b1dbc3f819671cba69'; then
             echo "✅ YieldVault mapped to correct EVM address"
           else
             echo "❌ EVM address mapping not found"
             exit 1
           fi
-          
+
           echo "✅ Test 1 Passed: Basic yieldvault creation verified"
-      
+
       # === TEST 2: FULL YIELDVAULT LIFECYCLE ===
       - name: Test 2 - Deposit Additional Funds (20 FLOW)
         run: |
@@ -175,26 +175,26 @@ jobs:
             --legacy
         env:
           AMOUNT: 20000000000000000000
-      
+
       - name: Process Deposit Request
         run: |
           flow transactions send ./cadence/transactions/process_requests.cdc 0 10 --signer emulator-flow-yield-vaults --compute-limit 9999
-      
+
       - name: Verify Deposit
         run: |
           echo "Verifying deposit (should still have 1 yieldvault with more balance)..."
-          
+
           YIELDVAULT_CHECK=$(flow scripts execute ./cadence/scripts/check_yieldvault_details.cdc 0x045a1763c93006ca)
           echo "$YIELDVAULT_CHECK"
-          
+
           # Should still have 1 yieldvault
           if echo "$YIELDVAULT_CHECK" | grep -q '"totalMappedYieldVaults": 1'; then
             echo "✅ Still has 1 yieldvault after deposit"
           else
             echo "❌ YieldVault count changed unexpectedly"
             exit 1
           fi
-      
+
       - name: Test 2 - Withdraw Half (15 FLOW)
         run: |
           echo "Step 2: Withdrawing 15 FLOW..."
@@ -204,26 +204,26 @@ jobs:
             --rpc-url http://localhost:8545 \
             --broadcast \
             --legacy
-      
+
       - name: Process Withdraw Request
         run: |
           flow transactions send ./cadence/transactions/process_requests.cdc 0 10 --signer emulator-flow-yield-vaults --compute-limit 9999
-      
+
       - name: Verify Withdrawal
         run: |
           echo "Verifying withdrawal (should still have 1 yieldvault with less balance)..."
-          
+
           YIELDVAULT_CHECK=$(flow scripts execute ./cadence/scripts/check_yieldvault_details.cdc 0x045a1763c93006ca)
           echo "$YIELDVAULT_CHECK"
-          
+
           # Should still have 1 yieldvault
           if echo "$YIELDVAULT_CHECK" | grep -q '"totalMappedYieldVaults": 1'; then
             echo "✅ Still has 1 yieldvault after withdrawal"
           else
             echo "❌ YieldVault count changed unexpectedly"
             exit 1
           fi
-      
+
       - name: Test 2 - Close YieldVault
         run: |
           echo "Step 3: Closing yieldvault (withdrawing remaining funds)..."
@@ -233,18 +233,18 @@ jobs:
             --rpc-url http://localhost:8545 \
             --broadcast \
             --legacy
-      
+
       - name: Process Close Request
         run: |
           flow transactions send ./cadence/transactions/process_requests.cdc 0 10 --signer emulator-flow-yield-vaults --compute-limit 9999
-      
+
       - name: Verify YieldVault Closed
         run: |
           echo "Verifying yieldvault was closed..."
-          
+
           YIELDVAULT_CHECK=$(flow scripts execute ./cadence/scripts/check_yieldvault_details.cdc 0x045a1763c93006ca)
           echo "$YIELDVAULT_CHECK"
-          
+
           # After closing, should have 0 yieldvaults or the yieldvault should be marked as closed
           if echo "$YIELDVAULT_CHECK" | grep -q '"totalMappedYieldVaults": 0'; then
             echo "✅ YieldVault successfully closed and removed"
@@ -254,9 +254,9 @@ jobs:
             echo "⚠️ YieldVault may still exist but should be in closed state"
             # Don't fail here as the close transaction succeeded
           fi
-          
+
           echo "✅ Test 2 Passed: Full yieldvault lifecycle completed"
-      
+
       # === FINAL SUMMARY ===
       - name: Test Summary
         run: |

.github/workflows/unit_tests.yml

@@ -17,19 +17,19 @@ jobs:
         with:
           token: ${{ secrets.GH_PAT }}
           submodules: recursive
-      
+
       - name: Install Flow CLI
         run: sh -ci "$(curl -fsSL https://raw.githubusercontent.com/onflow/flow-cli/master/install.sh)"
-      
+
       - name: Update PATH
         run: echo "$HOME/.local/bin" >> $GITHUB_PATH
-      
+
       - name: Verify Flow CLI Installation
         run: flow version
-      
+
       - name: Make test script executable
         run: chmod +x ./local/run_cadence_tests.sh
-      
+
       - name: Run Cadence Tests
         run: ./local/run_cadence_tests.sh
 

.github/workflows/worker_tests.yml

@@ -0,0 +1,86 @@
+name: Worker Operations CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  worker-tests:
+    name: Worker Operations Tests
+    runs-on: ubuntu-latest
+    steps:
+      # === COMMON SETUP ===
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_PAT }}
+          submodules: recursive
+
+      - name: Install Flow CLI
+        run: sh -ci "$(curl -fsSL https://raw.githubusercontent.com/onflow/flow-cli/master/install.sh)"
+
+      - name: Update PATH
+        run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Verify Flow CLI Installation
+        run: flow version
+
+      - name: Initialize submodules
+        run: git submodule update --init --recursive
+
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+        with:
+          version: stable
+
+      - name: Install Solidity dependencies
+        working-directory: ./solidity
+        run: forge install --no-git
+
+      - name: Make scripts executable
+        run: |
+          chmod +x ./local/setup_and_run_emulator.sh
+          chmod +x ./local/deploy_full_stack.sh
+          chmod +x ./local/run_worker_tests.sh
+
+      # === INFRASTRUCTURE SETUP ===
+      - name: Setup and Run Emulator
+        run: |
+          ./local/setup_and_run_emulator.sh &
+          sleep 80  # Wait for the emulator to be fully up
+
+      - name: Deploy Full Stack
+        run: ./local/deploy_full_stack.sh
+
+      - name: Detect Strategy Identifier
+        run: |
+          echo "Detecting supported strategy identifier..."
+          YIELDVAULT_CHECK=$(flow scripts execute ./cadence/scripts/check_yieldvault_details.cdc 0x045a1763c93006ca)
+          echo "$YIELDVAULT_CHECK"
+
+          SUPPORTED_STRATEGIES=$(echo "$YIELDVAULT_CHECK" | grep -oE '"supportedStrategies": \[[^]]*\]' || true)
+          if [ -z "$SUPPORTED_STRATEGIES" ]; then
+            echo "❌ Could not parse supported strategy list"
+            exit 1
+          fi
+
+          STRATEGY_LIST=$(echo "$SUPPORTED_STRATEGIES" | sed -E 's/^"supportedStrategies": \[(.*)\]$/\1/' | tr -d '"' | tr ',' '\n' | sed 's/^ *//;s/ *$//' | sed '/^$/d')
+          STRATEGY_IDENTIFIER=$(echo "$STRATEGY_LIST" | grep 'TracerStrategy' | head -n 1 || true)
+          if [ -z "$STRATEGY_IDENTIFIER" ]; then
+            STRATEGY_IDENTIFIER=$(echo "$STRATEGY_LIST" | head -n 1)
+          fi
+
+          if [ -z "$STRATEGY_IDENTIFIER" ]; then
+            echo "❌ No supported strategy identifier found"
+            exit 1
+          fi
+
+          echo "Using strategy identifier: $STRATEGY_IDENTIFIER"
+          echo "STRATEGY_IDENTIFIER=$STRATEGY_IDENTIFIER" >> $GITHUB_ENV
+
+      # === RUN WORKER TESTS ===
+      - name: Run Worker Tests
+        run: ./local/run_worker_tests.sh

.gitignore

@@ -1,3 +1,5 @@
+.DS_Store
+
 imports
 db
 

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "lib/FlowYieldVaults"]
 	path = lib/FlowYieldVaults
-	url = https://github.com/onflow/FlowYieldVaults.git
+	url = https://github.com/onflow/FlowYieldVaults
 [submodule "solidity/lib/forge-std"]
 	path = solidity/lib/forge-std
 	url = https://github.com/foundry-rs/forge-std

CLAUDE.md

@@ -46,25 +46,26 @@ flow deps install --skip-alias --skip-deployments  # Install dependencies
 ### Cross-VM Request Flow
 
 1. **EVM User** calls `FlowYieldVaultsRequests.sol` (creates request, escrows funds)
-2. **FlowYieldVaultsTransactionHandler.cdc** triggers `Worker.processRequests()` on schedule
+2. **FlowYieldVaultsEVMWorkerOps.cdc** SchedulerHandler schedules WorkerHandlers to process requests
 3. **FlowYieldVaultsEVM.cdc** Worker fetches pending requests via `getPendingRequestsUnpacked()`
-4. **Two-phase commit**: `startProcessing()` marks PROCESSING and deducts balance, `completeProcessing()` marks COMPLETED/FAILED (refunds credited to `claimableRefunds` on failure)
+4. **Two-phase commit**: `startProcessingBatch()` marks PROCESSING and deducts balance, `completeProcessing()` marks COMPLETED/FAILED (refunds credited to `claimableRefunds` on failure)
 
 ### Contract Components
 
 | Contract                                | Location             | Purpose                             |
 | --------------------------------------- | -------------------- | ----------------------------------- |
 | `FlowYieldVaultsRequests.sol`           | `solidity/src/`      | EVM request queue + fund escrow     |
 | `FlowYieldVaultsEVM.cdc`                | `cadence/contracts/` | Cadence worker processing requests  |
-| `FlowYieldVaultsTransactionHandler.cdc` | `cadence/contracts/` | Auto-scheduler with adaptive delays |
+| `FlowYieldVaultsEVMWorkerOps.cdc`       | `cadence/contracts/` | SchedulerHandler + WorkerHandler orchestration |
 
 ### Key Design Patterns
 
 - **COA Bridge**: Cadence Owned Account bridges funds between EVM and Cadence via FlowEVMBridge
 - **Sentinel Values**: `NATIVE_FLOW = 0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF`, `NO_YIELDVAULT_ID = type(uint64).max`
-- **Ownership Tracking**: Parallel mappings on both EVM (`userOwnsYieldVault`) and Cadence (`yieldVaultRegistry`) for O(1) lookups
-- **Adaptive Scheduling**: TransactionHandler adjusts delay based on pending count (3s for >10, 5s for >=5, 7s for >=1, 30s idle)
-- **Dynamic Execution Effort**: `baseEffortPerRequest * maxRequestsPerTx + baseOverhead`
+- **Ownership Tracking**: Parallel mappings on both EVM (`userOwnsYieldVault`) and Cadence (`yieldVaultOwnershipLookup`) for O(1) lookups
+- **Scheduler/Worker Split**: SchedulerHandler runs at fixed interval, schedules WorkerHandlers for individual requests
+- **Batch Preprocessing**: SchedulerHandler validates requests before scheduling workers; invalid requests fail early
+- **Crash Recovery**: SchedulerHandler monitors WorkerHandler transactions and marks panicked requests as FAILED
 
 ### Request Types (must stay synchronized between contracts)
 
@@ -114,7 +115,42 @@ flow deps install --skip-alias --skip-deployments  # Install dependencies
 | --------------------------------- | -------------------------------------------- |
 | FlowYieldVaultsRequests (EVM)     | `0xF633C9dBf1a3964a895fCC4CA4404B6f8BA8141d` |
 | FlowYieldVaultsEVM (Cadence)      | `df111ffc5064198a`                           |
-| FlowYieldVaultsTransactionHandler | `df111ffc5064198a`                           |
+| FlowYieldVaultsEVMWorkerOps       | `df111ffc5064198a`                           |
+
+## Blockchain Execution Model (Critical for Code Review)
+
+When reviewing this codebase, keep these fundamental blockchain properties in mind:
+
+### Transaction Atomicity
+
+**All blockchain transactions are atomic.** If any operation within a transaction panics/reverts, ALL state changes made during that transaction are rolled back completely. There is no "partial completion" scenario.
+
+- In Cadence: `panic()` reverts all state changes in the transaction
+- In Solidity: `revert()` or failed `require()` reverts all state changes
+- This is **by design** and is the correct way to ensure data consistency
+
+Therefore, patterns like:
+```cadence
+// This is SAFE - if processRequest panics, the remove never happened
+scheduledRequests.remove(key: requestId)
+processResult = worker.processRequest(request)  // if this panics, the line above reverts too
+```
+
+### Sequential Execution (No On-Chain Race Conditions)
+
+**Blockchain transactions execute one at a time in a deterministic order.** There is no parallel execution within the same blockchain execution environment.
+
+- Transactions are ordered within blocks and executed sequentially
+- Two transactions cannot "race" against each other simultaneously
+- What might look like a "race condition" is actually just transaction ordering, which is well-defined behavior
+
+This means scenarios like "Transaction A completes but Transaction B sees stale state" are **impossible** within the same execution context. By the time Transaction B executes, Transaction A has either fully committed or fully reverted.
+
+### Implications for This Codebase
+
+1. **WorkerHandler/SchedulerHandler coordination** is safe because they run in separate transactions that execute sequentially
+2. **Panic-based error handling** in `processRequest()` is the correct pattern - it ensures atomicity across Cadence and EVM state
+3. **State removal before vs after processing** doesn't create race conditions - if processing fails, the entire transaction (including removal) reverts
 
 ## Dependencies