From 92af2554e4abe896fe5f1463240552f4f012a080 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oriol=20Mu=C3=B1oz?= <oriol.munoz@digitalasset.com>
Date: Wed, 26 Aug 2026 16:47:12 +0000
Subject: [PATCH 1/3] Kill dead pods before node restore
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

[static]

Signed-off-by: Oriol Muñoz <oriol.munoz@digitalasset.com>
---
 cluster/scripts/node-restore.sh | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/cluster/scripts/node-restore.sh b/cluster/scripts/node-restore.sh
index 8417504cac..6d661fed44 100755
--- a/cluster/scripts/node-restore.sh
+++ b/cluster/scripts/node-restore.sh
@@ -100,6 +100,37 @@ function down() {
   done
 }
 
+# Adapted from gc-pod-reaper.
+# This is necessary because wait_down might block on pods that failed initializing
+# e.g., (maybe) because GCP failed to schedule a pod due to Out of Resources
+function kill_dead_pods() {
+  local -r namespace=$1
+  local -r bad_pods=$(
+    kubectl get pods -n "$namespace" -o json | \
+    jq -r '.items[] |
+        select(
+        (.status.phase == "Unknown" and .status.reason == "ContainerStatusUnknown") or
+        (.status.reason == "Evicted") or
+        (.status.containerStatuses[]?.state.terminated? | .reason == "Error" and .exitCode == 137) or
+        (.status.initContainerStatuses[]?.state.waiting?.reason == "ContainerStatusUnknown")
+      ) | .metadata.name' | sort -u
+  );
+  if [ -z "$bad_pods" ]; then
+      echo "No bad pods found in $namespace. Skipping.";
+      return
+  fi
+  echo "Found bad pods in $namespace: $bad_pods";
+  for pod_name in $bad_pods; do
+      echo "Attempting to delete pod $namespace/$pod_name";
+      kubectl delete pod -n "$namespace" "$pod_name"
+      if [ $? -eq 0 ]; then
+          echo "Successfully deleted $namespace/$pod_name";
+      else
+          echo "Failed to delete $namespace/$pod_name";
+      fi
+  done
+}
+
 function wait_down() {
   local -r namespace=$1
   local -r component=$2
@@ -469,6 +500,8 @@ function main() {
     down "$namespace" "$component" "$migration_id"
   done
 
+  kill_dead_pods "$namespace"
+
   for component in "${components[@]}"; do
     wait_down "$namespace" "$component" "$migration_id"
   done

From 73fcb011acd5275e42b39f0a759a40d6c5ee50cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oriol=20Mu=C3=B1oz?= <oriol.munoz@digitalasset.com>
Date: Wed, 26 Aug 2026 16:49:37 +0000
Subject: [PATCH 2/3] shellcheck [static]
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Oriol Muñoz <oriol.munoz@digitalasset.com>
---
 cluster/scripts/node-restore.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cluster/scripts/node-restore.sh b/cluster/scripts/node-restore.sh
index 6d661fed44..acade88461 100755
--- a/cluster/scripts/node-restore.sh
+++ b/cluster/scripts/node-restore.sh
@@ -122,8 +122,7 @@ function kill_dead_pods() {
   echo "Found bad pods in $namespace: $bad_pods";
   for pod_name in $bad_pods; do
       echo "Attempting to delete pod $namespace/$pod_name";
-      kubectl delete pod -n "$namespace" "$pod_name"
-      if [ $? -eq 0 ]; then
+      if kubectl delete pod -n "$namespace" "$pod_name"; then
           echo "Successfully deleted $namespace/$pod_name";
       else
           echo "Failed to delete $namespace/$pod_name";

From 357aefceeea706078cabba0d41f3d490ef7bed93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oriol=20Mu=C3=B1oz?= <oriol.munoz@digitalasset.com>
Date: Wed, 26 Aug 2026 17:01:31 +0000
Subject: [PATCH 3/3] --ignore-not-found=true --timeout=180s [static]
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Oriol Muñoz <oriol.munoz@digitalasset.com>
---
 cluster/scripts/node-restore.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cluster/scripts/node-restore.sh b/cluster/scripts/node-restore.sh
index acade88461..1bdfd4d51c 100755
--- a/cluster/scripts/node-restore.sh
+++ b/cluster/scripts/node-restore.sh
@@ -122,7 +122,7 @@ function kill_dead_pods() {
   echo "Found bad pods in $namespace: $bad_pods";
   for pod_name in $bad_pods; do
       echo "Attempting to delete pod $namespace/$pod_name";
-      if kubectl delete pod -n "$namespace" "$pod_name"; then
+      if kubectl delete pod -n "$namespace" "$pod_name" --ignore-not-found=true --timeout=180s; then
           echo "Successfully deleted $namespace/$pod_name";
       else
           echo "Failed to delete $namespace/$pod_name";