diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 00000000000..83101982217 --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "microsoft.dotnet.helix.jobmonitor": { + "version": "12.0.0-beta.26470.102", + "commands": [ + "dotnet-helix-job-monitor" + ] + } + } +} diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index 169fa6a754f..e7d76ef2e18 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -694,6 +694,11 @@ extends: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + - ${{ if or(eq(parameters.jobs, '[]'), contains(parameters.jobs, '"Helix_')) }}: + - template: /eng/common/core-templates/job/helix-job-monitor.yml@self + parameters: + helixAccessToken: $(HelixApiAccessToken) + continueOnError: true - stage: validate displayName: Validate dependsOn: build @@ -720,6 +725,7 @@ extends: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] + HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | $jobResults = @{ @@ -736,9 +742,11 @@ extends: Helix_macOS_ARM64 = "$(HelixMacOsArm64Result)" Helix_Windows_Arm64 = "$(HelixWindowsArm64Result)" Helix_Ubuntu_Cosmos = "$(HelixUbuntuCosmosResult)" + HelixJobMonitor = "$(HelixJobMonitorResult)" } & eng/aggregate-azdo-tests.ps1 -JobResults $jobResults displayName: Evaluate grouped job outcomes env: + HELIX_ACCESSTOKEN: $(HelixApiAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index edc480c492e..fc30306fa4d 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -527,6 +527,10 @@ stages: env: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - ${{ if or(eq(parameters.jobs, '[]'), contains(parameters.jobs, '"Helix_')) }}: + - template: /eng/common/core-templates/job/helix-job-monitor.yml + parameters: + continueOnError: true - stage: validate displayName: Validate dependsOn: build @@ -552,6 +556,7 @@ stages: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] + HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | $jobResults = @{ @@ -568,6 +573,7 @@ stages: Helix_macOS_ARM64 = "$(HelixMacOsArm64Result)" Helix_Windows_Arm64 = "$(HelixWindowsArm64Result)" Helix_Ubuntu_Cosmos = "$(HelixUbuntuCosmosResult)" + HelixJobMonitor = "$(HelixJobMonitorResult)" } & eng/aggregate-azdo-tests.ps1 -JobResults $jobResults diff --git a/eng/Version.Details.props b/eng/Version.Details.props index e5474a51834..cf9fa38ea01 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -8,6 +8,7 @@ This file should be imported by eng/Versions.props 12.0.0-beta.26470.102 12.0.0-beta.26470.102 + 12.0.0-beta.26470.102 12.0.0-beta.26470.102 12.0.0-alpha.1.26470.102 12.0.0-alpha.1.26470.102 @@ -29,6 +30,7 @@ This file should be imported by eng/Versions.props $(MicrosoftDotNetArcadeSdkPackageVersion) $(MicrosoftDotNetBuildTasksTemplatingPackageVersion) + $(MicrosoftDotNetHelixJobMonitorPackageVersion) $(MicrosoftDotNetHelixSdkPackageVersion) $(MicrosoftExtensionsCachingMemoryPackageVersion) $(MicrosoftExtensionsConfigurationPackageVersion) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index edfb27edf41..b8625a490c1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -68,6 +68,10 @@ https://github.com/dotnet/dotnet 66551619c00bee615342142059fbd8b9cad2e1af + + https://github.com/dotnet/dotnet + 66551619c00bee615342142059fbd8b9cad2e1af + https://github.com/dotnet/dotnet 66551619c00bee615342142059fbd8b9cad2e1af diff --git a/eng/aggregate-azdo-tests.ps1 b/eng/aggregate-azdo-tests.ps1 index 884b92f46f2..e8b318b288a 100644 --- a/eng/aggregate-azdo-tests.ps1 +++ b/eng/aggregate-azdo-tests.ps1 @@ -20,6 +20,7 @@ $jobDisplayNames = @{ Helix_Ubuntu_SqlServer = 'Helix Ubuntu SQL Server' Helix_Ubuntu_Cosmos = 'Helix Ubuntu Cosmos' Helix_Ubuntu = 'Helix Ubuntu' + HelixJobMonitor = 'Monitor Helix Jobs' } # A group succeeds when at least one of its jobs succeeds. Jobs may participate in multiple groups. @@ -32,6 +33,11 @@ $groupJobs = @{ SqlServer = @('Windows_SqlServer', 'Helix_Windows_SqlServer', 'Helix_Ubuntu_SqlServer') } +$helixJobNames = @($groupJobs.Values + | ForEach-Object { $_ } + | Where-Object { $_ -like 'Helix_*' } + | Select-Object -Unique) + function Get-JobRecord($timeline, [string]$jobName) { if (-not $jobDisplayNames.ContainsKey($jobName)) @@ -71,10 +77,300 @@ function Get-FailedGroups([hashtable]$resultsByJob) return $failed } +function Get-AzureDevOpsApiContext +{ + if ([string]::IsNullOrEmpty($env:SYSTEM_ACCESSTOKEN)) + { + throw 'SYSTEM_ACCESSTOKEN is required to get Helix test results.' + } + + if ([string]::IsNullOrEmpty($env:SYSTEM_COLLECTIONURI) -or + [string]::IsNullOrEmpty($env:SYSTEM_TEAMPROJECT)) + { + throw 'SYSTEM_COLLECTIONURI and SYSTEM_TEAMPROJECT are required to get Helix test results.' + } + + $project = [Uri]::EscapeDataString($env:SYSTEM_TEAMPROJECT) + $collectionUri = [Uri]$env:SYSTEM_COLLECTIONURI + $vstmrHost = if ($collectionUri.Host -eq 'dev.azure.com') + { + 'vstmr.dev.azure.com' + } + elseif ($collectionUri.Host.EndsWith('.visualstudio.com')) + { + $collectionUri.Host.Insert($collectionUri.Host.IndexOf('.'), '.vstmr') + } + else + { + $collectionUri.Host + } + + $vstmrCollectionUri = [UriBuilder]::new($collectionUri) + $vstmrCollectionUri.Host = $vstmrHost + + @{ + ApiBaseUri = "$($env:SYSTEM_COLLECTIONURI.TrimEnd('/'))/$project/_apis" + VstmrApiBaseUri = "$($vstmrCollectionUri.Uri.ToString().TrimEnd('/'))/$project/_apis" + Headers = @{ Authorization = "Bearer $env:SYSTEM_ACCESSTOKEN" } + } +} + +function Invoke-RestMethodWithRetry( + [string]$uri, + [hashtable]$headers, + [string]$serviceName) +{ + $maxAttempts = 3 + + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) + { + try + { + return Invoke-RestMethod ` + -Uri $uri ` + -Headers $headers ` + -ErrorAction Stop + } + catch + { + $statusCode = 0 + if ($null -ne $_.Exception.Response) + { + $statusCode = [int]$_.Exception.Response.StatusCode + } + + if ($attempt -eq $maxAttempts -or $statusCode -notin @(408, 429, 500, 502, 503, 504)) + { + throw + } + + $delay = [Math]::Pow(2, $attempt) + Write-Warning "$serviceName request failed with HTTP $statusCode. Retrying in $delay seconds." + Start-Sleep -Seconds $delay + } + } +} + +function Invoke-AzureDevOpsRestMethod([string]$uri, [hashtable]$headers) +{ + return Invoke-RestMethodWithRetry $uri $headers 'Azure DevOps' +} + +function Invoke-HelixRestMethod([string]$uri) +{ + $headers = @{} + if (-not [string]::IsNullOrEmpty($env:HELIX_ACCESSTOKEN)) + { + $headers.Authorization = "token $env:HELIX_ACCESSTOKEN" + } + + return Invoke-RestMethodWithRetry $uri $headers 'Helix' +} + +function Get-AzureDevOpsTestRuns([int]$buildId, [hashtable]$apiContext) +{ + $buildUri = [Uri]::EscapeDataString("vstfs:///Build/Build/$buildId") + $pageSize = 1000 + $skip = 0 + $testRuns = @() + + do + { + $uri = "$($apiContext.ApiBaseUri)/test/runs?buildUri=$buildUri&%24top=$pageSize&%24skip=$skip&api-version=7.1" + $response = Invoke-AzureDevOpsRestMethod $uri $apiContext.Headers + $page = @($response.value) + $testRuns += $page + $skip += $page.Count + } + while ($page.Count -eq $pageSize) + + return $testRuns +} + +function Get-HelixJobNameFromTestRun([int]$runId, [hashtable]$apiContext) +{ + $run = Invoke-AzureDevOpsRestMethod ` + "$($apiContext.VstmrApiBaseUri)/testresults/runs/$runId`?includeTags=true&api-version=7.1-preview.1" ` + $apiContext.Headers + $tag = @($run.tags | Where-Object { $_.name -match '^helixjob(?[0-9a-f]{32})$' })[0] + + if ($null -eq $tag) + { + return $null + } + + return [Guid]::ParseExact($tag.name.Substring('helixjob'.Length), 'N').ToString('D') +} + +function Get-HelixTestRuns([int]$buildId, [hashtable]$apiContext) +{ + $testRuns = @(Get-AzureDevOpsTestRuns $buildId $apiContext) + $helixTestRuns = @() + + foreach ($testRun in $testRuns) + { + if ($testRun.state -ne 'Completed') + { + continue + } + + $helixJobName = Get-HelixJobNameFromTestRun $testRun.id $apiContext + if ([string]::IsNullOrEmpty($helixJobName)) + { + continue + } + + $helixJob = Invoke-HelixRestMethod "https://helix.dot.net/api/2019-06-17/jobs/$helixJobName" + $phaseName = $helixJob.Properties.'System.PhaseName' + if ([string]::IsNullOrEmpty($phaseName)) + { + Write-Warning "Helix job '$helixJobName' for test run $($testRun.id) has no System.PhaseName property." + continue + } + + $stageAttempt = 0 + [void][int]::TryParse($helixJob.Properties.'System.StageAttempt', [ref]$stageAttempt) + $jobAttempt = 0 + [void][int]::TryParse($helixJob.Properties.'System.JobAttempt', [ref]$jobAttempt) + + $helixTestRuns += @{ + HelixJobName = $helixJobName + JobAttempt = $jobAttempt + LogicalJobName = $helixJob.Properties.jobName + PhaseName = $phaseName + QueueId = $helixJob.QueueId + StageAttempt = $stageAttempt + TestRun = $testRun + } + } + + return $helixTestRuns +} + +function Set-HelixJobResults( + [hashtable]$resultsByJob, + [string[]]$jobNames, + [int]$buildId) +{ + $helixJobNames = @($jobNames | Where-Object { $_ -like 'Helix_*' }) + if ($helixJobNames.Count -eq 0) + { + return + } + + foreach ($jobName in $helixJobNames) + { + if (-not $resultsByJob.ContainsKey($jobName)) + { + throw "Missing result for Helix job '$jobName'." + } + } + + if (-not $resultsByJob.ContainsKey('HelixJobMonitor')) + { + throw "Missing result for job 'HelixJobMonitor'." + } + + $apiContext = Get-AzureDevOpsApiContext + $helixTestRuns = @(Get-HelixTestRuns $buildId $apiContext) + $missingRuns = @() + + foreach ($jobName in $helixJobNames) + { + if ($resultsByJob[$jobName] -eq 'Skipped') + { + continue + } + + if ($resultsByJob[$jobName] -notin @('Succeeded', 'SucceededWithIssues')) + { + Write-Warning "Helix submission job '$jobName' did not succeed: $($resultsByJob[$jobName])." + $resultsByJob[$jobName] = 'Failed' + continue + } + + $phaseRuns = @($helixTestRuns | Where-Object { $_.PhaseName -eq $jobName }) + + if ($phaseRuns.Count -eq 0) + { + $missingRuns += $jobName + continue + } + + $latestStageAttempt = ($phaseRuns.StageAttempt | Measure-Object -Maximum).Maximum + $phaseRuns = @($phaseRuns | Where-Object { $_.StageAttempt -eq $latestStageAttempt }) + $latestJobAttempt = ($phaseRuns.JobAttempt | Measure-Object -Maximum).Maximum + $phaseRuns = @($phaseRuns | Where-Object { $_.JobAttempt -eq $latestJobAttempt }) + + foreach ($runInfo in $phaseRuns) + { + $runInfo['StreamName'] = if (-not [string]::IsNullOrEmpty($runInfo.LogicalJobName)) + { + "$($runInfo.QueueId)|$($runInfo.LogicalJobName)" + } + elseif (-not [string]::IsNullOrEmpty($runInfo.QueueId)) + { + $runInfo.QueueId + } + else + { + $runInfo.HelixJobName + } + } + + # A phase may submit multiple logical Helix jobs. Monitor retries preserve each stream's + # queue and logical job name, and create a newer test run for that stream. + $authoritativeRuns = @($phaseRuns + | Group-Object StreamName + | ForEach-Object { @($_.Group | Sort-Object { $_.TestRun.id } -Descending)[0] }) + $resultsByJob[$jobName] = 'Succeeded' + + foreach ($runInfo in $authoritativeRuns) + { + $run = $runInfo.TestRun + $attachmentsResponse = Invoke-AzureDevOpsRestMethod ` + "$($apiContext.ApiBaseUri)/test/runs/$($run.id)/attachments?api-version=7.1" ` + $apiContext.Headers + $failedWorkItemsAttachment = @($attachmentsResponse.value + | Where-Object { $_.fileName -eq 'helix-failed-workitems.json' }) + + if ($failedWorkItemsAttachment.Count -gt 0) + { + $resultsByJob[$jobName] = 'Failed' + } + + Write-Host " $jobName ($($run.name), Helix job $($runInfo.HelixJobName)): $($resultsByJob[$jobName])" + } + } + + if ($missingRuns.Count -gt 0) + { + if ($resultsByJob.HelixJobMonitor -eq 'Succeeded') + { + Write-Warning "No completed monitor test run was found for $($missingRuns -join ', '), but the Helix Job Monitor succeeded. Treating those Helix jobs as successful." + foreach ($jobName in $missingRuns) + { + $resultsByJob[$jobName] = 'Succeeded' + } + } + else + { + throw "The Helix Job Monitor result was '$($resultsByJob.HelixJobMonitor)' and it did not publish completed test runs for: $($missingRuns -join ', ')." + } + } +} + $jobAttempt = 1 [void][int]::TryParse($env:SYSTEM_JOBATTEMPT, [ref]$jobAttempt) $stageAttempt = 1 [void][int]::TryParse($env:SYSTEM_STAGEATTEMPT, [ref]$stageAttempt) +$buildId = 0 +if (-not [int]::TryParse($env:BUILD_BUILDID, [ref]$buildId) -or $buildId -le 0) +{ + throw "BUILD_BUILDID must contain a valid build ID; received '$env:BUILD_BUILDID'." +} + +Set-HelixJobResults $JobResults $helixJobNames $buildId $failedGroups = @(Get-FailedGroups $JobResults) # Retrying validation queues a child build containing the distinct jobs needed by all failed groups. @@ -146,6 +442,19 @@ if (($jobAttempt -gt 1 -or $stageAttempt -gt 1) -and $failedGroups.Count -gt 0) $JobResults[$jobName] = $record.result } + $helixJobsToRetry = @($jobsToRetry | Where-Object { $_ -like 'Helix_*' }) + if ($helixJobsToRetry.Count -gt 0) + { + $monitorRecord = Get-JobRecord $timeline 'HelixJobMonitor' + if ($null -eq $monitorRecord) + { + throw 'Could not find the Helix Job Monitor timeline record for the retry build.' + } + + $JobResults.HelixJobMonitor = $monitorRecord.result + Set-HelixJobResults $JobResults $helixJobsToRetry $retryBuild.id + } + $failedGroups = @(Get-FailedGroups $JobResults) } diff --git a/eng/helix.proj b/eng/helix.proj index 26377954ba2..db580a8d51b 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -11,6 +11,7 @@ $(NETCoreSdkVersion) true + true true $(RepoRoot)/test/EFCore.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.SqlServer.HierarchyId.Tests/*.csproj;$(RepoRoot)/test/EFCore.CrossStore.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.OData.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.AspNet.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.VisualBasic.FunctionalTests/*.vbproj;$(RepoRoot)/test/EFCore.FSharp.FunctionalTests/*.fsproj $(RepoRoot)/test/EFCore.Cosmos.FunctionalTests/*.csproj