shim: handle connection-closed errors during kill after live migration

After HCS live migration completes, FinalizeSourceLM tries to stop the
source sandbox via containerd. The shim's Kill call fails with
'ttrpc: closed' because the VM has already departed and the connection
(ttrpc transport or GCS bridge) is dead. This error propagates up as a
StopSourceVMFailure, even though the VM is already gone.

Three fixes:

1. exec_hcs: Treat ttrpc.ErrClosed and net.ErrClosed as successful kill.
   If the connection is dead, the process is effectively gone. The
   isConnectionClosed helper covers both the ttrpc transport path
   (containerd-to-shim) and the GCS bridge path (shim-to-guest vsock).

2. uvm: Make CloseCtx idempotent via sync.Once. Three goroutines can
   race to close the UVM (waitForHostExit, waitInitExit, KillExec
   timer). Without idempotency, the second caller panics on
   double-close of outputProcessingDone channel.

3. task_hcs: Route KillExec 30s timeout through closeHost instead of
   calling ht.host.Close() directly. The direct call bypassed
   closeHostOnce, skipping TaskExitEventTopic publish and ht.closed
   channel close.

Fixes: #61773098
Signed-off-by: Shreyansh Sancheti <shsancheti@microsoft.com>

Author: Shreyansh Sancheti

cmd/containerd-shim-runhcs-v1/exec_hcs.go

@@ -4,6 +4,8 @@ package main
 
 import (
 	"context"
+	stderrors "errors"
+	"net"
 	"sync"
 	"time"
 
@@ -12,6 +14,7 @@ import (
 	containerd_v1_types "github.com/containerd/containerd/api/types/task"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/runtime"
+	"github.com/containerd/ttrpc"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -304,7 +307,7 @@ func (he *hcsExec) Kill(ctx context.Context, signal uint32) error {
 					signalDelivered, deliveryErr := he.p.Process.Signal(ctx, options)
 
 					if deliveryErr != nil {
-						if !hcs.IsAlreadyStopped(deliveryErr) {
+						if !hcs.IsAlreadyStopped(deliveryErr) && !isConnectionClosed(deliveryErr) {
 							// Process is not already stopped and there was a signal delivery error to this process
 							log.G(ctx).WithField("err", deliveryErr).Errorf("Error in delivering signal %d, to pid: %d", signal, he.pid)
 						}
@@ -328,6 +331,13 @@ func (he *hcsExec) Kill(ctx context.Context, signal uint32) error {
 				// an already dead process.
 				return nil
 			}
+			if isConnectionClosed(err) {
+				// The connection to the compute system or GCS is closed.
+				// This happens during live migration when the VM has already departed.
+				// The process is effectively gone, so treat this as a successful kill.
+				log.G(ctx).WithError(err).Warn("connection closed during kill; treating as successful")
+				return nil
+			}
 			return err
 		}
 		if !delivered {
@@ -341,6 +351,13 @@ func (he *hcsExec) Kill(ctx context.Context, signal uint32) error {
 	}
 }
 
+// isConnectionClosed reports whether err indicates that the underlying connection
+// to the compute system or GCS is closed. This covers both the ttrpc transport
+// (containerd ↔ shim) and the GCS bridge (shim ↔ guest VM over vsock).
+func isConnectionClosed(err error) bool {
+	return err == ttrpc.ErrClosed || stderrors.Is(err, net.ErrClosed)
+}
+
 func (he *hcsExec) ResizePty(ctx context.Context, width, height uint32) error {
 	he.sl.Lock()
 	defer he.sl.Unlock()

cmd/containerd-shim-runhcs-v1/task_hcs.go

@@ -458,9 +458,9 @@ func (ht *hcsTask) KillExec(ctx context.Context, eid string, signal uint32, all
 			case <-execExited:
 				t.Stop()
 			case <-t.C:
-				// Safe to call multiple times if called previously on
-				// successful shutdown.
-				ht.host.Close()
+				// Route through closeHost to ensure proper event publishing and
+				// that the close is guarded by closeHostOnce.
+				ht.closeHost(ctx)
 			}
 		}()
 	}

internal/uvm/create.go

@@ -214,14 +214,19 @@ func (uvm *UtilityVM) Close() error { return uvm.CloseCtx(context.Background())
 //
 // The context is used for all operations, including waits, so timeouts/cancellations may prevent
 // proper uVM cleanup.
-func (uvm *UtilityVM) CloseCtx(ctx context.Context) (err error) {
+func (uvm *UtilityVM) CloseCtx(ctx context.Context) error {
+	uvm.closeOnce.Do(func() {
+		uvm.closeErr = uvm.closeCtxInternal(ctx)
+	})
+	return uvm.closeErr
+}
+
+func (uvm *UtilityVM) closeCtxInternal(ctx context.Context) (err error) {
 	ctx, span := oc.StartSpan(ctx, "uvm::Close")
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(trace.StringAttribute(logfields.UVMID, uvm.id))
 
-	// TODO: check if uVM already closed
-
 	windows.Close(uvm.vmmemProcess)
 
 	if uvm.hcsSystem != nil {

internal/uvm/types.go

@@ -148,6 +148,10 @@ type UtilityVM struct {
 
 	// confidentialUVMOptions hold confidential UVM specific options
 	confidentialUVMOptions *ConfidentialOptions
+
+	// closeOnce ensures CloseCtx is idempotent and safe to call from multiple goroutines.
+	closeOnce sync.Once
+	closeErr  error
 }
 
 func (uvm *UtilityVM) ScratchEncryptionEnabled() bool {