From db524e423e3426eb1ec4dbf01eeb297f903a94b8 Mon Sep 17 00:00:00 2001 From: Will Godbe Date: Wed, 5 Aug 2026 09:01:17 -0700 Subject: [PATCH 1/5] Enable Helix Job Monitor Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .config/dotnet-tools.json | 12 ++++++++++++ azure-pipelines-internal-tests.yml | 3 +++ azure-pipelines-public.yml | 1 + eng/Version.Details.props | 2 ++ eng/Version.Details.xml | 4 ++++ eng/helix.proj | 1 + 6 files changed, 23 insertions(+) create mode 100644 .config/dotnet-tools.json diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 00000000000..d104beb316e --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "microsoft.dotnet.helix.jobmonitor": { + "version": "11.0.0-beta.26401.101", + "commands": [ + "dotnet-helix-job-monitor" + ] + } + } +} diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index da4325a67e7..67b72f81415 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -645,6 +645,9 @@ extends: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + - template: /eng/common/core-templates/job/helix-job-monitor.yml@self + parameters: + helixAccessToken: $(HelixApiAccessToken) - stage: validate displayName: Validate dependsOn: build diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index fd68cf6c873..e1e953a7b5d 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -509,6 +509,7 @@ stages: env: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - template: /eng/common/core-templates/job/helix-job-monitor.yml - stage: validate displayName: Validate dependsOn: build diff --git a/eng/Version.Details.props b/eng/Version.Details.props index 142cdd226f9..2dd071adb12 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -8,6 +8,7 @@ This file should be imported by eng/Versions.props 11.0.0-beta.26401.101 11.0.0-beta.26401.101 + 11.0.0-beta.26401.101 11.0.0-beta.26401.101 11.0.0-rc.1.26401.101 11.0.0-rc.1.26401.101 @@ -29,6 +30,7 @@ This file should be imported by eng/Versions.props $(MicrosoftDotNetArcadeSdkPackageVersion) $(MicrosoftDotNetBuildTasksTemplatingPackageVersion) + $(MicrosoftDotNetHelixJobMonitorPackageVersion) $(MicrosoftDotNetHelixSdkPackageVersion) $(MicrosoftExtensionsCachingMemoryPackageVersion) $(MicrosoftExtensionsConfigurationPackageVersion) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5fa787159b9..a822ba32a54 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -68,6 +68,10 @@ https://github.com/dotnet/dotnet 5da2de22f7c6e2ebe7ed4c16c614b00183145d3c + + https://github.com/dotnet/dotnet + 5da2de22f7c6e2ebe7ed4c16c614b00183145d3c + https://github.com/dotnet/dotnet 5da2de22f7c6e2ebe7ed4c16c614b00183145d3c diff --git a/eng/helix.proj b/eng/helix.proj index 3dfca72e27d..8d341f9724e 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -11,6 +11,7 @@ $(NETCoreSdkVersion) true + true true $(RepoRoot)/test/EFCore.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.SqlServer.HierarchyId.Tests/*.csproj;$(RepoRoot)/test/EFCore.CrossStore.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.OData.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.AspNet.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.VisualBasic.FunctionalTests/*.vbproj;$(RepoRoot)/test/EFCore.FSharp.FunctionalTests/*.fsproj $(RepoRoot)/test/EFCore.Cosmos.FunctionalTests/*.csproj From 34bf52554862a3d21addcb1645a38b9cf0b4e63e Mon Sep 17 00:00:00 2001 From: Will Godbe Date: Wed, 5 Aug 2026 10:34:07 -0700 Subject: [PATCH 2/5] Validate Helix monitor result Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- azure-pipelines-internal-tests.yml | 10 ++++++++++ azure-pipelines-public.yml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index 67b72f81415..d09e30640eb 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -673,8 +673,18 @@ extends: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] + HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | + $helixJobMonitorResult = "$(HelixJobMonitorResult)" + Write-Host "Helix Job Monitor result: $helixJobMonitorResult" + + if ($helixJobMonitorResult -ne 'Succeeded') + { + Write-Error "Helix Job Monitor did not succeed: $helixJobMonitorResult." + exit 1 + } + $groupResults = @{ Windows = @("$(WindowsResult)", "$(HelixWindowsResult)") Linux = @("$(LinuxResult)", "$(HelixUbuntuResult)") diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index e1e953a7b5d..bc7715b8c2d 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -534,8 +534,18 @@ stages: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] + HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | + $helixJobMonitorResult = "$(HelixJobMonitorResult)" + Write-Host "Helix Job Monitor result: $helixJobMonitorResult" + + if ($helixJobMonitorResult -ne 'Succeeded') + { + Write-Error "Helix Job Monitor did not succeed: $helixJobMonitorResult." + exit 1 + } + $groupResults = @{ Windows = @("$(WindowsResult)", "$(HelixWindowsResult)") Linux = @("$(LinuxResult)", "$(HelixUbuntuResult)") From 4abe9f1e1e1c2f2a229b6bb6af5b399e17dad20c Mon Sep 17 00:00:00 2001 From: Will Godbe Date: Tue, 15 Sep 2026 15:15:57 -0700 Subject: [PATCH 3/5] Skip Helix monitor when no Helix jobs are selected Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- azure-pipelines-internal-tests.yml | 7 ++++--- azure-pipelines-public.yml | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index 9763b800657..e406b52eba2 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -663,9 +663,10 @@ extends: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) - - template: /eng/common/core-templates/job/helix-job-monitor.yml@self - parameters: - helixAccessToken: $(HelixApiAccessToken) + - ${{ if or(eq(parameters.jobs, '[]'), contains(parameters.jobs, '"Helix_')) }}: + - template: /eng/common/core-templates/job/helix-job-monitor.yml@self + parameters: + helixAccessToken: $(HelixApiAccessToken) - stage: validate displayName: Validate dependsOn: build diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index f8cacc25cd9..89078385670 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -527,7 +527,8 @@ stages: env: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) - - template: /eng/common/core-templates/job/helix-job-monitor.yml + - ${{ if or(eq(parameters.jobs, '[]'), contains(parameters.jobs, '"Helix_')) }}: + - template: /eng/common/core-templates/job/helix-job-monitor.yml - stage: validate displayName: Validate dependsOn: build From 3677be42534efa1c1df6252a93895bf33cdf4f85 Mon Sep 17 00:00:00 2001 From: Will Godbe Date: Thu, 17 Sep 2026 15:59:55 -0700 Subject: [PATCH 4/5] Preserve grouped Helix validation Use the Job Monitor's per-queue failed-work-item attachments so grouped validation and selective retries retain their existing semantics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- azure-pipelines-internal-tests.yml | 11 +- azure-pipelines-public.yml | 12 +- eng/aggregate-azdo-tests.ps1 | 190 +++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 20 deletions(-) diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index e406b52eba2..718e5c5d3ad 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -667,6 +667,7 @@ extends: - template: /eng/common/core-templates/job/helix-job-monitor.yml@self parameters: helixAccessToken: $(HelixApiAccessToken) + continueOnError: true - stage: validate displayName: Validate dependsOn: build @@ -693,18 +694,8 @@ extends: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] - HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | - $helixJobMonitorResult = "$(HelixJobMonitorResult)" - Write-Host "Helix Job Monitor result: $helixJobMonitorResult" - - if ($helixJobMonitorResult -ne 'Succeeded') - { - Write-Error "Helix Job Monitor did not succeed: $helixJobMonitorResult." - exit 1 - } - $jobResults = @{ Windows = "$(WindowsResult)" Helix_Windows = "$(HelixWindowsResult)" diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index 89078385670..d64df895e7e 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -529,6 +529,8 @@ stages: SYSTEM_ACCESSTOKEN: $(System.AccessToken) - ${{ if or(eq(parameters.jobs, '[]'), contains(parameters.jobs, '"Helix_')) }}: - template: /eng/common/core-templates/job/helix-job-monitor.yml + parameters: + continueOnError: true - stage: validate displayName: Validate dependsOn: build @@ -554,18 +556,8 @@ stages: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] - HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | - $helixJobMonitorResult = "$(HelixJobMonitorResult)" - Write-Host "Helix Job Monitor result: $helixJobMonitorResult" - - if ($helixJobMonitorResult -ne 'Succeeded') - { - Write-Error "Helix Job Monitor did not succeed: $helixJobMonitorResult." - exit 1 - } - $jobResults = @{ Windows = "$(WindowsResult)" Helix_Windows = "$(HelixWindowsResult)" diff --git a/eng/aggregate-azdo-tests.ps1 b/eng/aggregate-azdo-tests.ps1 index 884b92f46f2..2255a670858 100644 --- a/eng/aggregate-azdo-tests.ps1 +++ b/eng/aggregate-azdo-tests.ps1 @@ -20,6 +20,21 @@ $jobDisplayNames = @{ Helix_Ubuntu_SqlServer = 'Helix Ubuntu SQL Server' Helix_Ubuntu_Cosmos = 'Helix Ubuntu Cosmos' Helix_Ubuntu = 'Helix Ubuntu' + HelixJobMonitor = 'Monitor Helix Jobs' +} + +# Test runs uploaded by the Helix Job Monitor are named after their target queue. +# Public queue names add ".Open"; normalize that suffix so the same map works for internal builds. +$helixQueueNames = @{ + Helix_Windows = 'Windows.10.Amd64' + Helix_Windows_SqlServer = 'Windows.11.Amd64.Client' + Helix_Windows_Arm64 = 'Windows.11.Arm64' + Helix_Windows_Cosmos = 'Windows.Server2025.Amd64' + Helix_macOS_x64 = 'OSX.15.Amd64' + Helix_macOS_ARM64 = 'OSX.15.ARM64' + Helix_Ubuntu_SqlServer = 'Ubuntu.2204.Amd64.XL@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-sqlserver-amd64' + Helix_Ubuntu_Cosmos = 'Ubuntu.2204.Amd64.XL' + Helix_Ubuntu = 'Ubuntu.2204.Amd64' } # A group succeeds when at least one of its jobs succeeds. Jobs may participate in multiple groups. @@ -71,10 +86,173 @@ function Get-FailedGroups([hashtable]$resultsByJob) return $failed } +function Get-AzureDevOpsApiContext +{ + if ([string]::IsNullOrEmpty($env:SYSTEM_ACCESSTOKEN)) + { + throw 'SYSTEM_ACCESSTOKEN is required to get Helix test results.' + } + + if ([string]::IsNullOrEmpty($env:SYSTEM_COLLECTIONURI) -or + [string]::IsNullOrEmpty($env:SYSTEM_TEAMPROJECT)) + { + throw 'SYSTEM_COLLECTIONURI and SYSTEM_TEAMPROJECT are required to get Helix test results.' + } + + $project = [Uri]::EscapeDataString($env:SYSTEM_TEAMPROJECT) + @{ + ApiBaseUri = "$($env:SYSTEM_COLLECTIONURI.TrimEnd('/'))/$project/_apis" + Headers = @{ Authorization = "Bearer $env:SYSTEM_ACCESSTOKEN" } + } +} + +function Get-NormalizedHelixQueueName([string]$queueName) +{ + return $queueName -replace '\.Open(?=@|$)', '' +} + +function Invoke-AzureDevOpsRestMethod([string]$uri, [hashtable]$headers) +{ + $maxAttempts = 3 + + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) + { + try + { + return Invoke-RestMethod ` + -Uri $uri ` + -Headers $headers ` + -ErrorAction Stop + } + catch + { + $statusCode = 0 + if ($null -ne $_.Exception.Response) + { + $statusCode = [int]$_.Exception.Response.StatusCode + } + + if ($attempt -eq $maxAttempts -or $statusCode -notin @(408, 429, 500, 502, 503, 504)) + { + throw + } + + $delay = [Math]::Pow(2, $attempt) + Write-Warning "Azure DevOps request failed with HTTP $statusCode. Retrying in $delay seconds." + Start-Sleep -Seconds $delay + } + } +} + +function Get-AzureDevOpsTestRuns([int]$buildId, [hashtable]$apiContext) +{ + $buildUri = [Uri]::EscapeDataString("vstfs:///Build/Build/$buildId") + $pageSize = 1000 + $skip = 0 + $testRuns = @() + + do + { + $uri = "$($apiContext.ApiBaseUri)/test/runs?buildUri=$buildUri&%24top=$pageSize&%24skip=$skip&api-version=7.1" + $response = Invoke-AzureDevOpsRestMethod $uri $apiContext.Headers + $page = @($response.value) + $testRuns += $page + $skip += $page.Count + } + while ($page.Count -eq $pageSize) + + return $testRuns +} + +function Set-HelixJobResults( + [hashtable]$resultsByJob, + [string[]]$jobNames, + [int]$buildId) +{ + $helixJobNames = @($jobNames | Where-Object { $_ -like 'Helix_*' }) + if ($helixJobNames.Count -eq 0) + { + return + } + + foreach ($jobName in $helixJobNames) + { + if (-not $resultsByJob.ContainsKey($jobName)) + { + throw "Missing result for Helix job '$jobName'." + } + + if (-not $helixQueueNames.ContainsKey($jobName)) + { + throw "Missing Helix queue mapping for job '$jobName'." + } + } + + $apiContext = Get-AzureDevOpsApiContext + $testRuns = @(Get-AzureDevOpsTestRuns $buildId $apiContext) + + foreach ($jobName in $helixJobNames) + { + if ($resultsByJob[$jobName] -eq 'Skipped') + { + continue + } + + if ($resultsByJob[$jobName] -notin @('Succeeded', 'SucceededWithIssues')) + { + Write-Warning "Helix submission job '$jobName' did not succeed: $($resultsByJob[$jobName])." + $resultsByJob[$jobName] = 'Failed' + continue + } + + $queueName = $helixQueueNames[$jobName] + # Job Monitor retries create a new run with the same queue name containing only the retried work items. + $run = @($testRuns + | Where-Object { (Get-NormalizedHelixQueueName $_.name) -eq $queueName } + | Sort-Object id -Descending)[0] + + if ($null -eq $run) + { + Write-Warning "No completed Helix test run was found for job '$jobName' and queue '$queueName'." + $resultsByJob[$jobName] = 'Failed' + continue + } + + if ($run.state -ne 'Completed') + { + throw "Helix test run $($run.id) for job '$jobName' did not complete; its state is '$($run.state)'." + } + + $attachmentsResponse = Invoke-AzureDevOpsRestMethod ` + "$($apiContext.ApiBaseUri)/test/runs/$($run.id)/attachments?api-version=7.1" ` + $apiContext.Headers + $failedWorkItemsAttachment = @($attachmentsResponse.value + | Where-Object { $_.fileName -eq 'helix-failed-workitems.json' }) + + if ($failedWorkItemsAttachment.Count -gt 0) + { + $resultsByJob[$jobName] = 'Failed' + } + else + { + $resultsByJob[$jobName] = 'Succeeded' + } + + Write-Host " $jobName ($($run.name)): $($resultsByJob[$jobName])" + } +} + $jobAttempt = 1 [void][int]::TryParse($env:SYSTEM_JOBATTEMPT, [ref]$jobAttempt) $stageAttempt = 1 [void][int]::TryParse($env:SYSTEM_STAGEATTEMPT, [ref]$stageAttempt) +$buildId = 0 +if (-not [int]::TryParse($env:BUILD_BUILDID, [ref]$buildId) -or $buildId -le 0) +{ + throw "BUILD_BUILDID must contain a valid build ID; received '$env:BUILD_BUILDID'." +} + +Set-HelixJobResults $JobResults @($helixQueueNames.Keys) $buildId $failedGroups = @(Get-FailedGroups $JobResults) # Retrying validation queues a child build containing the distinct jobs needed by all failed groups. @@ -146,6 +324,18 @@ if (($jobAttempt -gt 1 -or $stageAttempt -gt 1) -and $failedGroups.Count -gt 0) $JobResults[$jobName] = $record.result } + $helixJobsToRetry = @($jobsToRetry | Where-Object { $_ -like 'Helix_*' }) + if ($helixJobsToRetry.Count -gt 0) + { + $monitorRecord = Get-JobRecord $timeline 'HelixJobMonitor' + if ($null -eq $monitorRecord) + { + throw 'Could not find the Helix Job Monitor timeline record for the retry build.' + } + + Set-HelixJobResults $JobResults $helixJobsToRetry $retryBuild.id + } + $failedGroups = @(Get-FailedGroups $JobResults) } From 8747e32cfe9e52dccf2e4b0dfe24f2f5a2e79194 Mon Sep 17 00:00:00 2001 From: Will Godbe Date: Tue, 22 Sep 2026 14:31:57 -0700 Subject: [PATCH 5/5] Fix Helix monitor result aggregation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .config/dotnet-tools.json | 2 +- azure-pipelines-internal-tests.yml | 3 + azure-pipelines-public.yml | 2 + eng/aggregate-azdo-tests.ps1 | 213 ++++++++++++++++++++++------- 4 files changed, 172 insertions(+), 48 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 6f804df1b4c..83101982217 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dotnet.helix.jobmonitor": { - "version": "12.0.0-beta.26461.114", + "version": "12.0.0-beta.26470.102", "commands": [ "dotnet-helix-job-monitor" ] diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index 20d5e0cda7c..e7d76ef2e18 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -725,6 +725,7 @@ extends: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] + HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | $jobResults = @{ @@ -741,9 +742,11 @@ extends: Helix_macOS_ARM64 = "$(HelixMacOsArm64Result)" Helix_Windows_Arm64 = "$(HelixWindowsArm64Result)" Helix_Ubuntu_Cosmos = "$(HelixUbuntuCosmosResult)" + HelixJobMonitor = "$(HelixJobMonitorResult)" } & eng/aggregate-azdo-tests.ps1 -JobResults $jobResults displayName: Evaluate grouped job outcomes env: + HELIX_ACCESSTOKEN: $(HelixApiAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index 7e6480c84ef..fc30306fa4d 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -556,6 +556,7 @@ stages: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] + HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | $jobResults = @{ @@ -572,6 +573,7 @@ stages: Helix_macOS_ARM64 = "$(HelixMacOsArm64Result)" Helix_Windows_Arm64 = "$(HelixWindowsArm64Result)" Helix_Ubuntu_Cosmos = "$(HelixUbuntuCosmosResult)" + HelixJobMonitor = "$(HelixJobMonitorResult)" } & eng/aggregate-azdo-tests.ps1 -JobResults $jobResults diff --git a/eng/aggregate-azdo-tests.ps1 b/eng/aggregate-azdo-tests.ps1 index 2255a670858..e8b318b288a 100644 --- a/eng/aggregate-azdo-tests.ps1 +++ b/eng/aggregate-azdo-tests.ps1 @@ -23,20 +23,6 @@ $jobDisplayNames = @{ HelixJobMonitor = 'Monitor Helix Jobs' } -# Test runs uploaded by the Helix Job Monitor are named after their target queue. -# Public queue names add ".Open"; normalize that suffix so the same map works for internal builds. -$helixQueueNames = @{ - Helix_Windows = 'Windows.10.Amd64' - Helix_Windows_SqlServer = 'Windows.11.Amd64.Client' - Helix_Windows_Arm64 = 'Windows.11.Arm64' - Helix_Windows_Cosmos = 'Windows.Server2025.Amd64' - Helix_macOS_x64 = 'OSX.15.Amd64' - Helix_macOS_ARM64 = 'OSX.15.ARM64' - Helix_Ubuntu_SqlServer = 'Ubuntu.2204.Amd64.XL@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-sqlserver-amd64' - Helix_Ubuntu_Cosmos = 'Ubuntu.2204.Amd64.XL' - Helix_Ubuntu = 'Ubuntu.2204.Amd64' -} - # A group succeeds when at least one of its jobs succeeds. Jobs may participate in multiple groups. $groupJobs = @{ Windows = @('Windows', 'Helix_Windows') @@ -47,6 +33,11 @@ $groupJobs = @{ SqlServer = @('Windows_SqlServer', 'Helix_Windows_SqlServer', 'Helix_Ubuntu_SqlServer') } +$helixJobNames = @($groupJobs.Values + | ForEach-Object { $_ } + | Where-Object { $_ -like 'Helix_*' } + | Select-Object -Unique) + function Get-JobRecord($timeline, [string]$jobName) { if (-not $jobDisplayNames.ContainsKey($jobName)) @@ -100,18 +91,34 @@ function Get-AzureDevOpsApiContext } $project = [Uri]::EscapeDataString($env:SYSTEM_TEAMPROJECT) + $collectionUri = [Uri]$env:SYSTEM_COLLECTIONURI + $vstmrHost = if ($collectionUri.Host -eq 'dev.azure.com') + { + 'vstmr.dev.azure.com' + } + elseif ($collectionUri.Host.EndsWith('.visualstudio.com')) + { + $collectionUri.Host.Insert($collectionUri.Host.IndexOf('.'), '.vstmr') + } + else + { + $collectionUri.Host + } + + $vstmrCollectionUri = [UriBuilder]::new($collectionUri) + $vstmrCollectionUri.Host = $vstmrHost + @{ ApiBaseUri = "$($env:SYSTEM_COLLECTIONURI.TrimEnd('/'))/$project/_apis" + VstmrApiBaseUri = "$($vstmrCollectionUri.Uri.ToString().TrimEnd('/'))/$project/_apis" Headers = @{ Authorization = "Bearer $env:SYSTEM_ACCESSTOKEN" } } } -function Get-NormalizedHelixQueueName([string]$queueName) -{ - return $queueName -replace '\.Open(?=@|$)', '' -} - -function Invoke-AzureDevOpsRestMethod([string]$uri, [hashtable]$headers) +function Invoke-RestMethodWithRetry( + [string]$uri, + [hashtable]$headers, + [string]$serviceName) { $maxAttempts = 3 @@ -138,12 +145,28 @@ function Invoke-AzureDevOpsRestMethod([string]$uri, [hashtable]$headers) } $delay = [Math]::Pow(2, $attempt) - Write-Warning "Azure DevOps request failed with HTTP $statusCode. Retrying in $delay seconds." + Write-Warning "$serviceName request failed with HTTP $statusCode. Retrying in $delay seconds." Start-Sleep -Seconds $delay } } } +function Invoke-AzureDevOpsRestMethod([string]$uri, [hashtable]$headers) +{ + return Invoke-RestMethodWithRetry $uri $headers 'Azure DevOps' +} + +function Invoke-HelixRestMethod([string]$uri) +{ + $headers = @{} + if (-not [string]::IsNullOrEmpty($env:HELIX_ACCESSTOKEN)) + { + $headers.Authorization = "token $env:HELIX_ACCESSTOKEN" + } + + return Invoke-RestMethodWithRetry $uri $headers 'Helix' +} + function Get-AzureDevOpsTestRuns([int]$buildId, [hashtable]$apiContext) { $buildUri = [Uri]::EscapeDataString("vstfs:///Build/Build/$buildId") @@ -164,6 +187,66 @@ function Get-AzureDevOpsTestRuns([int]$buildId, [hashtable]$apiContext) return $testRuns } +function Get-HelixJobNameFromTestRun([int]$runId, [hashtable]$apiContext) +{ + $run = Invoke-AzureDevOpsRestMethod ` + "$($apiContext.VstmrApiBaseUri)/testresults/runs/$runId`?includeTags=true&api-version=7.1-preview.1" ` + $apiContext.Headers + $tag = @($run.tags | Where-Object { $_.name -match '^helixjob(?[0-9a-f]{32})$' })[0] + + if ($null -eq $tag) + { + return $null + } + + return [Guid]::ParseExact($tag.name.Substring('helixjob'.Length), 'N').ToString('D') +} + +function Get-HelixTestRuns([int]$buildId, [hashtable]$apiContext) +{ + $testRuns = @(Get-AzureDevOpsTestRuns $buildId $apiContext) + $helixTestRuns = @() + + foreach ($testRun in $testRuns) + { + if ($testRun.state -ne 'Completed') + { + continue + } + + $helixJobName = Get-HelixJobNameFromTestRun $testRun.id $apiContext + if ([string]::IsNullOrEmpty($helixJobName)) + { + continue + } + + $helixJob = Invoke-HelixRestMethod "https://helix.dot.net/api/2019-06-17/jobs/$helixJobName" + $phaseName = $helixJob.Properties.'System.PhaseName' + if ([string]::IsNullOrEmpty($phaseName)) + { + Write-Warning "Helix job '$helixJobName' for test run $($testRun.id) has no System.PhaseName property." + continue + } + + $stageAttempt = 0 + [void][int]::TryParse($helixJob.Properties.'System.StageAttempt', [ref]$stageAttempt) + $jobAttempt = 0 + [void][int]::TryParse($helixJob.Properties.'System.JobAttempt', [ref]$jobAttempt) + + $helixTestRuns += @{ + HelixJobName = $helixJobName + JobAttempt = $jobAttempt + LogicalJobName = $helixJob.Properties.jobName + PhaseName = $phaseName + QueueId = $helixJob.QueueId + StageAttempt = $stageAttempt + TestRun = $testRun + } + } + + return $helixTestRuns +} + function Set-HelixJobResults( [hashtable]$resultsByJob, [string[]]$jobNames, @@ -181,15 +264,16 @@ function Set-HelixJobResults( { throw "Missing result for Helix job '$jobName'." } + } - if (-not $helixQueueNames.ContainsKey($jobName)) - { - throw "Missing Helix queue mapping for job '$jobName'." - } + if (-not $resultsByJob.ContainsKey('HelixJobMonitor')) + { + throw "Missing result for job 'HelixJobMonitor'." } $apiContext = Get-AzureDevOpsApiContext - $testRuns = @(Get-AzureDevOpsTestRuns $buildId $apiContext) + $helixTestRuns = @(Get-HelixTestRuns $buildId $apiContext) + $missingRuns = @() foreach ($jobName in $helixJobNames) { @@ -205,40 +289,74 @@ function Set-HelixJobResults( continue } - $queueName = $helixQueueNames[$jobName] - # Job Monitor retries create a new run with the same queue name containing only the retried work items. - $run = @($testRuns - | Where-Object { (Get-NormalizedHelixQueueName $_.name) -eq $queueName } - | Sort-Object id -Descending)[0] + $phaseRuns = @($helixTestRuns | Where-Object { $_.PhaseName -eq $jobName }) - if ($null -eq $run) + if ($phaseRuns.Count -eq 0) { - Write-Warning "No completed Helix test run was found for job '$jobName' and queue '$queueName'." - $resultsByJob[$jobName] = 'Failed' + $missingRuns += $jobName continue } - if ($run.state -ne 'Completed') + $latestStageAttempt = ($phaseRuns.StageAttempt | Measure-Object -Maximum).Maximum + $phaseRuns = @($phaseRuns | Where-Object { $_.StageAttempt -eq $latestStageAttempt }) + $latestJobAttempt = ($phaseRuns.JobAttempt | Measure-Object -Maximum).Maximum + $phaseRuns = @($phaseRuns | Where-Object { $_.JobAttempt -eq $latestJobAttempt }) + + foreach ($runInfo in $phaseRuns) { - throw "Helix test run $($run.id) for job '$jobName' did not complete; its state is '$($run.state)'." + $runInfo['StreamName'] = if (-not [string]::IsNullOrEmpty($runInfo.LogicalJobName)) + { + "$($runInfo.QueueId)|$($runInfo.LogicalJobName)" + } + elseif (-not [string]::IsNullOrEmpty($runInfo.QueueId)) + { + $runInfo.QueueId + } + else + { + $runInfo.HelixJobName + } } - $attachmentsResponse = Invoke-AzureDevOpsRestMethod ` - "$($apiContext.ApiBaseUri)/test/runs/$($run.id)/attachments?api-version=7.1" ` - $apiContext.Headers - $failedWorkItemsAttachment = @($attachmentsResponse.value - | Where-Object { $_.fileName -eq 'helix-failed-workitems.json' }) + # A phase may submit multiple logical Helix jobs. Monitor retries preserve each stream's + # queue and logical job name, and create a newer test run for that stream. + $authoritativeRuns = @($phaseRuns + | Group-Object StreamName + | ForEach-Object { @($_.Group | Sort-Object { $_.TestRun.id } -Descending)[0] }) + $resultsByJob[$jobName] = 'Succeeded' - if ($failedWorkItemsAttachment.Count -gt 0) + foreach ($runInfo in $authoritativeRuns) { - $resultsByJob[$jobName] = 'Failed' + $run = $runInfo.TestRun + $attachmentsResponse = Invoke-AzureDevOpsRestMethod ` + "$($apiContext.ApiBaseUri)/test/runs/$($run.id)/attachments?api-version=7.1" ` + $apiContext.Headers + $failedWorkItemsAttachment = @($attachmentsResponse.value + | Where-Object { $_.fileName -eq 'helix-failed-workitems.json' }) + + if ($failedWorkItemsAttachment.Count -gt 0) + { + $resultsByJob[$jobName] = 'Failed' + } + + Write-Host " $jobName ($($run.name), Helix job $($runInfo.HelixJobName)): $($resultsByJob[$jobName])" + } + } + + if ($missingRuns.Count -gt 0) + { + if ($resultsByJob.HelixJobMonitor -eq 'Succeeded') + { + Write-Warning "No completed monitor test run was found for $($missingRuns -join ', '), but the Helix Job Monitor succeeded. Treating those Helix jobs as successful." + foreach ($jobName in $missingRuns) + { + $resultsByJob[$jobName] = 'Succeeded' + } } else { - $resultsByJob[$jobName] = 'Succeeded' + throw "The Helix Job Monitor result was '$($resultsByJob.HelixJobMonitor)' and it did not publish completed test runs for: $($missingRuns -join ', ')." } - - Write-Host " $jobName ($($run.name)): $($resultsByJob[$jobName])" } } @@ -252,7 +370,7 @@ if (-not [int]::TryParse($env:BUILD_BUILDID, [ref]$buildId) -or $buildId -le 0) throw "BUILD_BUILDID must contain a valid build ID; received '$env:BUILD_BUILDID'." } -Set-HelixJobResults $JobResults @($helixQueueNames.Keys) $buildId +Set-HelixJobResults $JobResults $helixJobNames $buildId $failedGroups = @(Get-FailedGroups $JobResults) # Retrying validation queues a child build containing the distinct jobs needed by all failed groups. @@ -333,6 +451,7 @@ if (($jobAttempt -gt 1 -or $stageAttempt -gt 1) -and $failedGroups.Count -gt 0) throw 'Could not find the Helix Job Monitor timeline record for the retry build.' } + $JobResults.HelixJobMonitor = $monitorRecord.result Set-HelixJobResults $JobResults $helixJobsToRetry $retryBuild.id }