Config.ps1

###############################################################################################################################
# Shared configuration for all mongodb-flasharray-backup scripts.
# Dot-source this file at the top of each script:
#   . "$PSScriptRoot/Config.ps1"
#
# All values are loaded from a .env file in the same directory as this script.
# Copy .env.example to .env and fill in your values before running any script.
#
# Cluster topology and FlashArray volume mappings are discovered at runtime from authoritative
# sources (Ops Manager API and FlashArray SCSI serial numbers). .env values are used only for
# credentials and as a fallback when live discovery is unavailable.
###############################################################################################################################

#region --- Load .env ---

$EnvFile = Join-Path $PSScriptRoot '.env'
if (-not (Test-Path $EnvFile)) {
    throw ".env file not found at '$EnvFile'. Copy .env.example to .env and fill in your values."
}

$EnvVars = @{}
Get-Content $EnvFile | Where-Object { $_ -match '^\s*[^#\s]' } | ForEach-Object {
    $Parts = $_ -split '=', 2
    if ($Parts.Count -eq 2) { $EnvVars[$Parts[0].Trim()] = $Parts[1].Trim() }
}

function Get-EnvVar ([string]$Key, [switch]$Optional) {
    if ($EnvVars.ContainsKey($Key)) { return $EnvVars[$Key] }
    if ($Optional) { return $null }
    throw ".env is missing required key: $Key"
}

#endregion

#region --- MongoDB cluster topology ---

$MongoshPath = Get-EnvVar 'MONGOSH_PATH'
$MongosHost  = Get-EnvVar 'MONGOS_HOST'
$MongosPort  = [int](Get-EnvVar 'MONGOS_PORT')
$SshUser     = Get-EnvVar 'SSH_USER'

# CLUSTER_NODES is optional - used only as a fallback when Ops Manager is unreachable.
# The authoritative source is the Ops Manager /hosts API (see Get-ClusterNodes below).
$ClusterNodesRaw      = Get-EnvVar 'CLUSTER_NODES' -Optional
$ClusterNodesFallback = if ($ClusterNodesRaw) { $ClusterNodesRaw -split ',' } else { $null }

# SSH options: never prompt interactively, fail fast on auth/host-key issues, and multiplex
# concurrent ssh invocations onto a single TCP connection per remote host (ControlMaster).
# Without multiplexing, a load driver (e.g. Start-InsertLoad.ps1) that spawns one ssh per
# write batch can saturate sshd's MaxStartups on the mongos node and cause "Connection closed
# by ... port 22" failures in any concurrent script (snapshot baseline capture, oplog dump,
# etc.). With ControlMaster=auto + ControlPersist, the second ssh to a given host reuses the
# existing TCP/auth session - the server sees one connection regardless of script concurrency.
# %C is a hash of (user,host,port,user@local) - keeps ControlPath short and stable per peer.
$SshOpts = @(
    '-o', 'BatchMode=yes',
    '-o', 'StrictHostKeyChecking=no',
    '-o', 'ConnectTimeout=15',
    '-o', 'ControlMaster=auto',
    '-o', 'ControlPath=/tmp/ssh-mux-%C',
    '-o', 'ControlPersist=60s'
)

#endregion

#region --- MongoDB database tools ---

$MongoToolsBase   = Get-EnvVar 'MONGO_TOOLS_BASE'
$MongodumpPath    = "$MongoToolsBase/mongodump"
$MongorestorePath = "$MongoToolsBase/mongorestore"

#endregion

#region --- Pure Storage FlashArray ---

$FaEndpoint   = Get-EnvVar 'FA_ENDPOINT'
$FaSecurePass = ConvertTo-SecureString (Get-EnvVar 'FA_PASSWORD') -AsPlainText -Force
$FaCred       = [System.Management.Automation.PSCredential]::new((Get-EnvVar 'FA_USERNAME'), $FaSecurePass)

# Protection group name — must exist on every FlashArray before running the snapshot script.
# Run Initialize-ProtectionGroups.ps1 once to create it. Snapshots are named:
#   <ProtectionGroupName>.<SnapshotTag>                  (the PG snapshot itself)
#   <ProtectionGroupName>.<SnapshotTag>.<VolumeName>     (the per-volume member snapshot used for restore)
$ProtectionGroupName = Get-EnvVar 'FA_PROTECTION_GROUP'
$ClusterName         = Get-EnvVar 'FA_CLUSTER_NAME'

#endregion

#region --- Ops Manager ---

# Base URL is constructed from host + pinned API version so the version is explicit
# and independently lockable without editing a compound URL string.
$OmHost            = (Get-EnvVar 'OM_BASE_URL').TrimEnd('/')
$OmApiVersion      = Get-EnvVar 'OM_API_VERSION'
$OpsManagerBaseUrl = "$OmHost/api/public/$OmApiVersion"
$GroupId           = Get-EnvVar 'OM_GROUP_ID'
$ClusterId         = Get-EnvVar 'OM_CLUSTER_ID'
$OmPublicKey       = Get-EnvVar 'OM_PUBLIC_KEY'
$OmPrivateKey      = Get-EnvVar 'OM_PRIVATE_KEY'

#endregion

#region --- Shared helpers ---

# New-ScriptLock: atomically acquires a per-script lock file at $LockPath. Writes pid/host/started
# into the file so a stale-lock detector can recognize and clean up locks left by crashed runs.
# Throws if a live process is already holding the lock; clears and re-acquires if the PID is dead.
function New-ScriptLock {
    param([Parameter(Mandatory)] [string]$LockPath)

    if (Test-Path $LockPath) {
        $LockedPid = (Get-Content $LockPath -ErrorAction SilentlyContinue |
            Where-Object { $_ -match '^pid=' } |
            ForEach-Object { ($_ -split '=', 2)[1].Trim() } |
            Select-Object -First 1)
        if ($LockedPid -and ($LockedPid -as [int])) {
            if (Get-Process -Id ([int]$LockedPid) -ErrorAction SilentlyContinue) {
                throw "Lock held by live PID ${LockedPid} (lock file: ${LockPath}). Another run is in progress."
            }
            Write-Host "  Stale lock detected (PID ${LockedPid} not running) - removing $LockPath" -ForegroundColor DarkYellow
            Remove-Item -Path $LockPath -Force -ErrorAction SilentlyContinue
        } else {
            throw "Lock file exists but has no parseable PID: $LockPath. Inspect and delete manually if no script is running."
        }
    }

    # FileMode.CreateNew is atomic - throws IOException if the file already exists, so two
    # concurrent starts can't both win the race past Test-Path above.
    try {
        $Stream = [System.IO.File]::Open($LockPath, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None)
    } catch {
        throw "Could not acquire lock at ${LockPath}: $($_.Exception.Message). Another run may have started concurrently."
    }
    try {
        $Content = "pid=$PID`nhost=$([System.Net.Dns]::GetHostName())`nstarted=$((Get-Date).ToUniversalTime().ToString('o'))`n"
        $Bytes = [System.Text.Encoding]::UTF8.GetBytes($Content)
        $Stream.Write($Bytes, 0, $Bytes.Length)
    } finally {
        $Stream.Dispose()
    }
}

# Remove-ScriptLock: idempotent counterpart to New-ScriptLock. Safe to call from finally blocks.
function Remove-ScriptLock {
    param([Parameter(Mandatory)] [string]$LockPath)
    if (Test-Path $LockPath) { Remove-Item -Path $LockPath -Force -ErrorAction SilentlyContinue }
}

# Invoke-ParallelOrThrow: runs a script block in parallel across $InputObjects.
# Each iteration must return a hashtable with at minimum: @{ Success=$true/$false; Message=... }
# plus a Key field named either 'Key' or 'Node' (both are checked). Throws if any iteration failed.
# Returns the array of result hashtables.
function Invoke-ParallelOrThrow {
    param(
        [Parameter(Mandatory)] [object[]]$InputObjects,
        [Parameter(Mandatory)] [scriptblock]$ScriptBlock,
        [Parameter(Mandatory)] [string]$StepName,
        [int]$ThrottleLimit = 10
    )
    $results  = $InputObjects | ForEach-Object -Parallel $ScriptBlock -ThrottleLimit $ThrottleLimit
    $failures = @($results | Where-Object { -not $_.Success })
    if ($failures.Count -gt 0) {
        $msg = ($failures | ForEach-Object {
            $id = if ($_.Key) { $_.Key } else { $_.Node }
            "  [$id] $($_.Message)"
        }) -join "`n"
        throw "$StepName failed on $($failures.Count) item(s):`n$msg"
    }
    return $results
}

#endregion

#region --- HTTP Digest auth for Ops Manager API ---

# -Authentication Digest is not supported on macOS in PS7. Implements the
# standard two-step Digest challenge/response flow (RFC 2617 / qop=auth) manually.
# These functions are defined here so both Snapshot.ps1 and Restore.ps1 share the same implementation.

function Get-MD5Hash ([string]$PlainText) {
    $Bytes = [System.Security.Cryptography.MD5]::Create().ComputeHash(
        [System.Text.Encoding]::UTF8.GetBytes($PlainText)
    )
    return ($Bytes | ForEach-Object { $_.ToString('x2') }) -join ''
}

function Invoke-OmApi {
    param(
        [string]$Method = 'GET',
        [string]$Path,
        [object]$Body = $null,
        # PathPrefix controls what is inserted between the base URL and $Path.
        # Default is 'backup/third_party/' for the third-party backup API.
        # Pass '' to reach the public API root (e.g. /groups/{id}/hosts).
        [string]$PathPrefix = 'backup/third_party/'
    )
    $Uri = "$OpsManagerBaseUrl/$PathPrefix$Path"

    $BodyParams = @{}
    if ($Body) {
        $BodyParams.Body        = ($Body | ConvertTo-Json -Depth 10 -Compress)
        $BodyParams.ContentType = 'application/json'
    } elseif ($Method -ne 'GET') {
        # Ops Manager requires Content-Type: application/json on all POST/DELETE
        # even when the body is empty
        $BodyParams.Body        = '{}'
        $BodyParams.ContentType = 'application/json'
    }

    # Step 1: Probe to get Digest challenge - server returns 401
    # Include Content-Type on the probe for non-GET requests to avoid 415
    $ProbeHeaders = @{ Accept = 'application/json' }
    if ($Method -ne 'GET') { $ProbeHeaders['Content-Type'] = 'application/json' }
    $Probe = Invoke-WebRequest -Method $Method -Uri $Uri `
        -Headers $ProbeHeaders @BodyParams `
        -SkipHttpErrorCheck -TimeoutSec 30 -ErrorAction Stop
    if ($Probe.StatusCode -ne 401) {
        throw "Expected 401 Digest challenge from $Uri but got HTTP $($Probe.StatusCode): $($Probe.Content)"
    }
    $Challenge = $Probe.Headers['WWW-Authenticate']

    # Step 2: Parse challenge fields
    $realm  = [regex]::Match($Challenge, 'realm="([^"]*)"').Groups[1].Value
    $nonce  = [regex]::Match($Challenge, 'nonce="([^"]*)"').Groups[1].Value
    $qop    = [regex]::Match($Challenge, 'qop="?([^",\s]*)"?').Groups[1].Value
    $opaque = [regex]::Match($Challenge, 'opaque="([^"]*)"').Groups[1].Value

    # Step 3: Compute Digest response (RFC 2617 / qop=auth)
    $UriPath = ([uri]$Uri).PathAndQuery
    $HA1     = Get-MD5Hash "$OmPublicKey`:$realm`:$OmPrivateKey"
    $HA2     = Get-MD5Hash "$Method`:$UriPath"
    $nc      = '00000001'
    $cnonce  = [System.Guid]::NewGuid().ToString('N').Substring(0, 8)

    if ($qop -in @('auth', 'auth-int')) {
        $DigestResponse = Get-MD5Hash "$HA1`:$nonce`:$nc`:$cnonce`:$qop`:$HA2"
        $AuthHeader = "Digest username=`"$OmPublicKey`", realm=`"$realm`", nonce=`"$nonce`", uri=`"$UriPath`", qop=$qop, nc=$nc, cnonce=`"$cnonce`", response=`"$DigestResponse`""
    } else {
        $DigestResponse = Get-MD5Hash "$HA1`:$nonce`:$HA2"
        $AuthHeader = "Digest username=`"$OmPublicKey`", realm=`"$realm`", nonce=`"$nonce`", uri=`"$UriPath`", response=`"$DigestResponse`""
    }
    if ($opaque) { $AuthHeader += ", opaque=`"$opaque`"" }

    # Step 4: Authenticated request
    return Invoke-RestMethod -Method $Method -Uri $Uri `
        -Headers @{ Accept = 'application/json'; Authorization = $AuthHeader } `
        @BodyParams -TimeoutSec 30 -ErrorAction Stop
}

# Retries transient errors (network blips, 5xx). Does NOT retry permanent 4xx errors.
function Invoke-OmApiWithRetry {
    param(
        [string]$Method = 'GET',
        [string]$Path,
        [object]$Body = $null,
        [int]$MaxAttempts = 5,
        [int]$BackoffSec  = 5
    )
    for ($Attempt = 1; $Attempt -le $MaxAttempts; $Attempt++) {
        try {
            return Invoke-OmApi -Method $Method -Path $Path -Body $Body
        } catch {
            $Msg = $_.Exception.Message
            # Don't retry on permanent client errors
            if ($Msg -match '\b(400|401|403|404|409|415|422)\b') { throw }
            if ($Attempt -eq $MaxAttempts) { throw }
            Write-Host "    Transient error on attempt ${Attempt}: $Msg - retrying in ${BackoffSec}s" -ForegroundColor DarkYellow
            Start-Sleep -Seconds $BackoffSec
        }
    }
}

#endregion

#region --- Mongo shell + snapshot-state helpers ---

# Mongosh helper — runs JavaScript against the mongos router over SSH
function Invoke-Mongos ([string]$Eval) {
    $Out = ssh @SshOpts "${SshUser}@${MongosHost}" `
        "$MongoshPath --quiet --eval '$Eval' mongodb://${MongosHost}:${MongosPort} 2>/dev/null"
    if ($LASTEXITCODE -ne 0) { throw "mongosh failed (exit $LASTEXITCODE): $Out" }
    return ($Out | Out-String).Trim()
}
# Shared mongosh JavaScript snippets. Centralized here so Snapshot, Replay, and Tailer all
# emit the same shape and so a fix in one place propagates to every caller.
#   $ListShardsJs - listShards reducer; emits [{shardId, rsHosts, host}, ...] where host is
#                   the first member of the replica-set host list (suitable for SSH targeting).
#   $OplogTopJs   - reads the most recent oplog entry on the connected node and prints
#                   {t, i}. The "\$natural" backslash is required so PowerShell does not
#                   interpolate $natural; mongosh receives the literal $natural.
$ListShardsJs = 'var shards=db.adminCommand({listShards:1}).shards; var out=[]; for(var i=0;i<shards.length;i++){var s=shards[i]; out.push({shardId:s._id,rsHosts:s.host,host:s.host.split("/")[1].split(",")[0]});} print(JSON.stringify(out));'
$OplogTopJs   = 'var latest=db.getSiblingDB("local").oplog.rs.find().sort({"\$natural":-1}).limit(1).toArray()[0]; print(JSON.stringify({t:latest.ts.t,i:latest.ts.i}));'

# Invoke-MongoshJs: runs a mongosh --eval expression on a remote host via SSH and returns
# the raw stdout. Centralizes the ssh+mongosh+stderr-suppression pattern used in 10+ call
# sites across Snapshot, Restore, and Replay scripts. Throws on non-zero exit (after any
# requested retries). Caller is responsible for parsing the returned string (digit, JSON,
# free text). Designed for the non-parallel main-script path; the tailer's parallel-block
# call site stays inline because functions are not visible across runspaces.
function Invoke-MongoshJs {
    param(
        [Parameter(Mandatory)] [string]$SshTarget,
        [Parameter(Mandatory)] [string]$Uri,
        [Parameter(Mandatory)] [string]$Js,
        [int]$MaxAttempts = 1,
        [string]$Context  = 'mongosh'
    )
    $LastExit = 0
    $Raw      = $null
    for ($Attempt = 1; $Attempt -le $MaxAttempts; $Attempt++) {
        $Raw = ssh @SshOpts "${SshUser}@${SshTarget}" "$MongoshPath --quiet --eval '$Js' $Uri 2>/dev/null"
        $LastExit = $LASTEXITCODE
        if ($LastExit -eq 0) { return $Raw }
        if ($Attempt -lt $MaxAttempts) {
            $SleepSec = [int][math]::Pow(2, $Attempt - 1)
            Write-Host "    $Context attempt $Attempt failed (exit $LastExit) - retrying in ${SleepSec}s ..." -ForegroundColor Yellow
            Start-Sleep -Seconds $SleepSec
        }
    }
    throw "$Context failed after $MaxAttempts attempt(s) (exit $LastExit, output: $Raw)"
}

# Wait-OmSnapshotState: polls the third-party backup API until the snapshot reaches
# $TargetState (typically READY or FINISHED), aborting on FAILED/FAILING or after a
# timeout. Used by New-MongoSnapshot.ps1's STEP 4 and STEP 7.
function Wait-OmSnapshotState {
    param(
        [Parameter(Mandatory)] [string]$SnapshotId,
        [Parameter(Mandatory)] [string]$TargetState,
        [int]$TimeoutMinutes  = 150,
        [int]$PollIntervalSec = 10
    )
    $State    = ''
    $Deadline = (Get-Date).AddMinutes($TimeoutMinutes)
    while ($State -ne $TargetState) {
        if ((Get-Date) -gt $Deadline) {
            throw "Snapshot $SnapshotId timed out waiting for $TargetState state."
        }
        if ($State -in @('FAILED', 'FAILING')) {
            throw "Snapshot $SnapshotId entered $State state - aborting."
        }
        # Check before sleeping so we catch an immediate transition without paying a full interval.
        $StatusResponse = Invoke-OmApiWithRetry -Path "group/$GroupId/clusters/$ClusterId/snapshot/$SnapshotId"
        $State = $StatusResponse.state
        Write-Host "  $(Get-Date -Format 'HH:mm:ss')  state = $State" -ForegroundColor Cyan
        if ($State -ne $TargetState) { Start-Sleep -Seconds $PollIntervalSec }
    }
}

#endregion

#region --- Runtime topology discovery ---

# Get-ClusterNodes: Returns the hostnames of all nodes belonging to $ClusterId in $GroupId.
# Queries the Ops Manager /hosts API first (authoritative). Falls back to CLUSTER_NODES from .env
# if OM is unreachable and that key is present. Throws if neither source can provide nodes.
#
# For a sharded cluster, OM's /hosts endpoint tags each host with its shard-level clusterId
# (REPLICA_SET / CONFIG_SERVER_REPLICA_SET), NOT the top-level SHARDED_REPLICA_SET clusterId.
# We first query /clusters to collect all child cluster IDs that share the same clusterName as
# $ClusterId, then filter /hosts by any of those IDs.
function Get-ClusterNodes {
    try {
        # Resolve the clusterName for $ClusterId and collect all sibling/child cluster IDs.
        $ClustersResponse = Invoke-OmApi -Path "groups/$GroupId/clusters" -PathPrefix ''
        $ParentCluster = $ClustersResponse.results | Where-Object { $_.id -eq $ClusterId } | Select-Object -First 1
        if (-not $ParentCluster) {
            throw "ClusterId '$ClusterId' not found in group '$GroupId'."
        }
        $OmClusterName = $ParentCluster.clusterName
        # Collect IDs of all clusters in the group that share this clusterName (parent + all shards).
        $AllClusterIds = @($ClustersResponse.results |
            Where-Object { $_.clusterName -eq $OmClusterName } |
            ForEach-Object { $_.id })

        # The /hosts endpoint returns all agents in the group; filter to hosts whose clusterId
        # matches any of the cluster IDs collected above (shard or config RS members).
        $HostsResponse = Invoke-OmApi -Path "groups/$GroupId/hosts" -PathPrefix ''
        $Nodes = @($HostsResponse.results |
            Where-Object { $AllClusterIds -contains $_.clusterId } |
            ForEach-Object { $_.hostname } |
            Sort-Object -Unique)
        if ($Nodes.Count -gt 0) {
            Write-Host "  Cluster nodes discovered from Ops Manager ($($Nodes.Count)): $($Nodes -join ', ')" -ForegroundColor Cyan
            return $Nodes
        }
        Write-Host "  WARNING: OM returned 0 hosts for clusterId=$ClusterId - falling back to .env" -ForegroundColor Yellow
    } catch {
        Write-Host "  WARNING: OM node discovery failed ($($_.Exception.Message)) - falling back to .env" -ForegroundColor Yellow
    }
    if (-not $ClusterNodesFallback) {
        throw "Could not discover cluster nodes from Ops Manager and CLUSTER_NODES is not set in .env"
    }
    Write-Host "  Using CLUSTER_NODES from .env ($($ClusterNodesFallback.Count) nodes): $($ClusterNodesFallback -join ', ')" -ForegroundColor DarkYellow
    return $ClusterNodesFallback
}

# Resolve-FaContextNames: Returns the short names of all fleet FlashArrays that have $PgName as a
# protection group. Called after connecting to the gateway ($FA) so it uses that session.
# Enumerates all fleet members via Get-Pfa2FleetMember, filters out FlashBlades (which throw
# "Cross-product requests not supported" errors), then checks PG existence on each FlashArray.
# Throws if no arrays are found with the PG.
function Resolve-FaContextNames {
    param(
        [Parameter(Mandatory)] [object]$FA,
        [Parameter(Mandatory)] [string]$PgName
    )
    # Get all fleet members with minimal retries (fast path)
    $FleetMembers = $null
    for ($attempt = 1; $attempt -le 2 -and -not $FleetMembers; $attempt++) {
        try {
            $FleetMembers = Get-Pfa2FleetMember -Array $FA -ErrorAction Stop
            break
        } catch {
            if ($attempt -lt 2) {
                Write-Host "  WARNING: Get-Pfa2FleetMember attempt $attempt failed: $($_.Exception.Message). Retrying in 2s..." -ForegroundColor Yellow
                Start-Sleep -Seconds 2
            } else { throw }
        }
    }
    $AllMemberNames = @($FleetMembers.Member.Name)
    Write-Host "  Fleet members discovered ($($AllMemberNames.Count)): $($AllMemberNames -join ', ')" -ForegroundColor Cyan

    # Filter out FlashBlades by attempting Get-Pfa2Array (FlashBlades throw "Cross-product" error)
    $FlashArrayNames = [System.Collections.Generic.List[String]]::new()
    foreach ($MemberName in $AllMemberNames) {
        $ArrayInfo = Get-Pfa2Array -Array $FA -ContextName @($MemberName) -ErrorAction SilentlyContinue
        if ($ArrayInfo -and $ArrayInfo.Os -eq 'Purity//FA') {
            $FlashArrayNames.Add($MemberName)
        } else {
            Write-Host "    ${MemberName}: not a FlashArray (skipped)" -ForegroundColor DarkGray
        }
    }
    Write-Host "  FlashArrays in fleet ($($FlashArrayNames.Count)): $($FlashArrayNames -join ', ')" -ForegroundColor Cyan

    # Check which FlashArrays have the protection group (no retries - fast and reliable)
    $ContextsWithPG = [System.Collections.Generic.List[String]]::new()
    foreach ($ArrayName in $FlashArrayNames) {
        $Pg = Get-Pfa2ProtectionGroup -Array $FA -ContextName @($ArrayName) -Name $PgName -ErrorAction SilentlyContinue
        if ($Pg) {
            $ContextsWithPG.Add($ArrayName)
            Write-Host "    ${ArrayName}: PG '$PgName' present" -ForegroundColor Cyan
        } else {
            Write-Host "    ${ArrayName}: PG '$PgName' NOT found" -ForegroundColor DarkGray
        }
    }
    if ($ContextsWithPG.Count -eq 0) {
        throw "No FlashArrays found with protection group '$PgName'. Run Initialize-ProtectionGroups.ps1 first."
    }
    Write-Host "  Resolved $($ContextsWithPG.Count) FlashArray(s) with PG." -ForegroundColor Green
    return $ContextsWithPG
}

# Get-FaSnapshotTags: Read metadata tags from a PG snapshot, trying each array in $ContextNames
# in order. Returns an ordered hashtable of tag Key -> Value from the first array that returns
# at least one tag for the given snapshot. Returns an empty hashtable if no array has tags.
function Get-FaSnapshotTags {
    param(
        [Parameter(Mandatory)] [object]   $FA,
        [Parameter(Mandatory)] [string[]] $ContextNames,
        [Parameter(Mandatory)] [string]   $SnapshotName
    )
    foreach ($CtxName in $ContextNames) {
        $Tags = Get-Pfa2ProtectionGroupSnapshotTag -Array $FA `
            -ContextName  @($CtxName) `
            -ResourceName $SnapshotName `
            -ErrorAction  SilentlyContinue
        if ($Tags -and $Tags.Count -gt 0) {
            $Map = [ordered]@{}
            foreach ($T in $Tags) { $Map[$T.Key] = $T.Value }
            Write-Host "  Snapshot tags loaded from $CtxName ($($Tags.Count) tags)" -ForegroundColor Cyan
            return $Map
        }
    }
    Write-Host "  WARNING: No snapshot tags found on any of $($ContextNames.Count) array(s) for '$SnapshotName'" -ForegroundColor Yellow
    return [ordered]@{}
}

# Resolve-NodeToArrayVolumeMap: For each cluster node, SSHes to the node to read the SCSI serial
# number of the block device backing /data/mongo, then queries each fleet array to find which
# array owns a volume with that serial. Returns an ordered hashtable:
#   node -> @{ ShortName = <array-short-name>; VolumeName = <fa-volume-name> }
# Throws immediately if any node's volume cannot be resolved. Serial lookup is done serially
# (not in parallel) because each node SSH + each FA query must succeed before proceeding.
function Resolve-NodeToArrayVolumeMap {
    param(
        [Parameter(Mandatory)] [object]$FA,
        [Parameter(Mandatory)] [string[]]$Nodes,
        [Parameter(Mandatory)] [string]$SshUserParam,
        [Parameter(Mandatory)] [string[]]$SshOptsParam,
        [Parameter(Mandatory)] [string[]]$ContextNames
    )
    $Map = [ordered]@{}
    foreach ($Node in $Nodes) {
        # Get the SCSI serial of the block device backing /data/mongo.
        # Primary path: findmnt gives the partition, lsblk PKNAME maps to parent disk, lsblk SERIAL reads the FA serial.
        # Fallback (volume not mounted - e.g. mid-restore recovery): scan all disks for a FA-format serial
        # (24+ uppercase hex chars, NAA WWN format). Each node has exactly one data volume so this is unambiguous.
        $Cmd = 'p=$(findmnt -no SOURCE /data/mongo 2>/dev/null); if [ -n "$p" ]; then pk=$(lsblk -no PKNAME "$p" 2>/dev/null); lsblk -no SERIAL "/dev/$pk" 2>/dev/null | head -1; else lsblk -dno SERIAL 2>/dev/null | grep -E "^[0-9a-fA-F]{20,}$" | head -1; fi'
        # Pure FlashArray volume serials are 24-character NAA-format hex strings (the SCSI page-80
        # serial reflects the volume's WWN). Enforce a minimum length so a truncated/garbled
        # output (e.g. a stray "0" or "ok") cannot pass validation and trigger an empty filter
        # that would silently match the wrong volume.
        $Serial = $null
        for ($attempt = 1; $attempt -le 3 -and -not $Serial; $attempt++) {
            $RawSerial = @(ssh @SshOptsParam "${SshUserParam}@${Node}" $Cmd 2>/dev/null)
            $Serial = ($RawSerial | Where-Object { $_ -is [string] -and $_.Trim() -match '^[0-9a-fA-F]{20,}$' } | Select-Object -Last 1)?.Trim()?.ToLower()
            if (-not $Serial -and $attempt -lt 3) { Start-Sleep -Milliseconds 1000 }
        }
        if (-not $Serial -or $Serial -notmatch '^[0-9a-f]{20,}$') {
            throw "Could not read FA volume serial from $Node (got: '$Serial'). Verify /data/mongo is mounted and the block device is a Pure Storage pRDM."
        }
        Write-Host "  $Node serial: $Serial" -ForegroundColor Cyan

        # Query each context array to find which one owns the volume with this serial.
        # FA stores serials as uppercase hex; apply ToUpper() in the filter string.
        $Found = $false
        foreach ($CtxName in $ContextNames) {
            # Single attempt with filter - API calls are fast and reliable
            $Vol = Get-Pfa2Volume -Array $FA -ContextName @($CtxName) -Filter "serial='$($Serial.ToUpper())'" -ErrorAction SilentlyContinue
            if ($Vol) {
                $Map[$Node] = @{ ShortName = $CtxName; VolumeName = $Vol.Name }
                Write-Host "    -> $CtxName / $($Vol.Name)" -ForegroundColor Green
                $Found = $true
                break
            }
        }
        if (-not $Found) {
            throw "No FlashArray volume with serial '$Serial' found across any fleet array for node $Node. Verify the pRDM is presented from the expected array."
        }
    }
    return $Map
}

#endregion