diff --git a/cluster/scripts/node-restore.sh b/cluster/scripts/node-restore.sh index 8417504cac..1bdfd4d51c 100755 --- a/cluster/scripts/node-restore.sh +++ b/cluster/scripts/node-restore.sh @@ -100,6 +100,36 @@ function down() { done } +# Adapted from gc-pod-reaper. +# This is necessary because wait_down might block on pods that failed initializing +# e.g., (maybe) because GCP failed to schedule a pod due to Out of Resources +function kill_dead_pods() { + local -r namespace=$1 + local -r bad_pods=$( + kubectl get pods -n "$namespace" -o json | \ + jq -r '.items[] | + select( + (.status.phase == "Unknown" and .status.reason == "ContainerStatusUnknown") or + (.status.reason == "Evicted") or + (.status.containerStatuses[]?.state.terminated? | .reason == "Error" and .exitCode == 137) or + (.status.initContainerStatuses[]?.state.waiting?.reason == "ContainerStatusUnknown") + ) | .metadata.name' | sort -u + ); + if [ -z "$bad_pods" ]; then + echo "No bad pods found in $namespace. Skipping."; + return + fi + echo "Found bad pods in $namespace: $bad_pods"; + for pod_name in $bad_pods; do + echo "Attempting to delete pod $namespace/$pod_name"; + if kubectl delete pod -n "$namespace" "$pod_name" --ignore-not-found=true --timeout=180s; then + echo "Successfully deleted $namespace/$pod_name"; + else + echo "Failed to delete $namespace/$pod_name"; + fi + done +} + function wait_down() { local -r namespace=$1 local -r component=$2 @@ -469,6 +499,8 @@ function main() { down "$namespace" "$component" "$migration_id" done + kill_dead_pods "$namespace" + for component in "${components[@]}"; do wait_down "$namespace" "$component" "$migration_id" done