From b6e4bcb365d4126b03804f5c2757f986927499dc Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 14 Sep 2026 09:25:59 -0700 Subject: [PATCH 01/15] Update markdown links following samples update (#7353) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .portal-docs/docker-hub/README.sdk.md | 2 +- .portal-docs/mar/README.sdk.portal.md | 2 +- README.sdk.md | 2 +- eng/readme-templates/Use.sdk.md | 2 +- samples/kubernetes/graceful-shutdown/README.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.portal-docs/docker-hub/README.sdk.md b/.portal-docs/docker-hub/README.sdk.md index ca05e56b50..deaa876f9a 100644 --- a/.portal-docs/docker-hub/README.sdk.md +++ b/.portal-docs/docker-hub/README.sdk.md @@ -25,7 +25,7 @@ The [.NET Docker samples](https://github.com/dotnet/dotnet-docker/blob/main/samp ## Building .NET Apps with Docker -* [.NET Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/ConsoleApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/ConsoleApp/Dockerfile) builds, tests, and runs the sample. It includes and builds multiple projects. +* [.NET Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/MultiProjectApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/MultiProjectApp/Dockerfile) builds, tests, and runs the sample. It includes and builds multiple projects. * [ASP.NET Core Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/AspNetCoreRazorApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/AspNetCoreRazorApp/Dockerfile) demonstrates using Docker with an ASP.NET Core Web App. ## Develop .NET Apps in a Container diff --git a/.portal-docs/mar/README.sdk.portal.md b/.portal-docs/mar/README.sdk.portal.md index f0ee132ad6..c67cac0052 100644 --- a/.portal-docs/mar/README.sdk.portal.md +++ b/.portal-docs/mar/README.sdk.portal.md @@ -43,7 +43,7 @@ The [.NET Docker samples](https://github.com/dotnet/dotnet-docker/blob/main/samp ### Building .NET Apps with Docker -* [.NET Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/ConsoleApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/ConsoleApp/Dockerfile) builds, tests, and runs the sample. It includes and builds multiple projects. +* [.NET Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/MultiProjectApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/MultiProjectApp/Dockerfile) builds, tests, and runs the sample. It includes and builds multiple projects. * [ASP.NET Core Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/AspNetCoreRazorApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/AspNetCoreRazorApp/Dockerfile) demonstrates using Docker with an ASP.NET Core Web App. ### Develop .NET Apps in a Container diff --git a/README.sdk.md b/README.sdk.md index f63bdc257c..c4acd96cb3 100644 --- a/README.sdk.md +++ b/README.sdk.md @@ -27,7 +27,7 @@ The [.NET Docker samples](https://github.com/dotnet/dotnet-docker/blob/main/samp ### Building .NET Apps with Docker -* [.NET Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/ConsoleApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/ConsoleApp/Dockerfile) builds, tests, and runs the sample. It includes and builds multiple projects. +* [.NET Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/MultiProjectApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/MultiProjectApp/Dockerfile) builds, tests, and runs the sample. It includes and builds multiple projects. * [ASP.NET Core Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/AspNetCoreRazorApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/AspNetCoreRazorApp/Dockerfile) demonstrates using Docker with an ASP.NET Core Web App. ### Develop .NET Apps in a Container diff --git a/eng/readme-templates/Use.sdk.md b/eng/readme-templates/Use.sdk.md index cfb4402c17..b47de2a40f 100644 --- a/eng/readme-templates/Use.sdk.md +++ b/eng/readme-templates/Use.sdk.md @@ -4,7 +4,7 @@ readme-host: Moniker of the site that will host the readme }}{{ARGS["top-header"]}}# Building .NET Apps with Docker -* [.NET Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/ConsoleApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/ConsoleApp/Dockerfile) builds, tests, and runs the sample. It includes and builds multiple projects. +* [.NET Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/MultiProjectApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/MultiProjectApp/Dockerfile) builds, tests, and runs the sample. It includes and builds multiple projects. * [ASP.NET Core Docker Sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/AspNetCoreRazorApp/README.md) - This [sample](https://github.com/dotnet/dotnet-docker/blob/main/samples/AspNetCoreRazorApp/Dockerfile) demonstrates using Docker with an ASP.NET Core Web App. {{ARGS["top-header"]}}# Develop .NET Apps in a Container diff --git a/samples/kubernetes/graceful-shutdown/README.md b/samples/kubernetes/graceful-shutdown/README.md index 823294a028..4433a74b49 100644 --- a/samples/kubernetes/graceful-shutdown/README.md +++ b/samples/kubernetes/graceful-shutdown/README.md @@ -2,7 +2,7 @@ Enable graceful shutdown of an app, particularly with long-running code. This pattern is described in more depth in [Graceful shutdown of ASP.NET-based applications](graceful-shutdown.md). -The [following pattern](https://github.com/dotnet/dotnet-docker/blob/main/samples/AspNetCoreRazorApp/AspNetCoreRazorApp/Program.cs#L30-L55) can be used: +The [following pattern](https://github.com/dotnet/dotnet-docker/blob/main/samples/AspNetCoreRazorApp/AspNetCoreRazorApp/Program.cs#L36-L52) can be used: ```csharp CancellationTokenSource cancellation = new(); From a3578fff5cd9703acf5b18c7d01d754049a10eed Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:02:04 -0700 Subject: [PATCH 02/15] [nightly] Update dependencies from syft (#7340) --- manifest.versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.versions.json b/manifest.versions.json index 8f29a7c680..32d91af9a4 100644 --- a/manifest.versions.json +++ b/manifest.versions.json @@ -265,7 +265,7 @@ "sdk|11.0|minor-tag": "$(dotnet|11.0|minor-tag)", "syft|repo": "anchore/syft", - "syft|version": "v1.51.0", + "syft|version": "v1.51.1", "syft|tag": "$(syft|version)-debug" } } From 45fa75f89f530e4fa33df3feb3ca74fc3939f1be Mon Sep 17 00:00:00 2001 From: "net-containers-bot[bot]" <307772199+net-containers-bot[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:17:08 -0700 Subject: [PATCH 03/15] [nightly] Update common Docker engineering infrastructure with latest (#7335) --- eng/docker-tools/templates/variables/docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml index 46549a7668..abc02dff3a 100644 --- a/eng/docker-tools/templates/variables/docker-images.yml +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:b2ee4d5f90beae94be5bdde2840d8e1cc52a8b67db50513b49e1c7f640f38045 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:79ee50949017dfce2212d66aa473e4361351b2aefda2c5df1dab566f20b68001 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner From e583983875af920109f380700a1c58868e10cd3a Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 18 Sep 2026 09:17:26 -0700 Subject: [PATCH 04/15] [nightly] Update common Docker engineering infrastructure with latest (#7357) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74c5ebc2-2880-411c-97ea-f8e26f28f6be --- eng/docker-tools/CHANGELOG.md | 9 +++++++++ eng/docker-tools/DEV-GUIDE.md | 16 ++++++++++++++++ eng/docker-tools/templates/jobs/build-images.yml | 3 +++ .../templates/stages/build-and-test.yml | 9 +++++++++ .../templates/stages/dotnet/build-and-test.yml | 2 ++ .../stages/dotnet/build-test-publish-repo.yml | 2 ++ eng/docker-tools/templates/steps/init-common.yml | 6 ++++++ .../templates/steps/init-imagebuilder.yml | 5 +++++ .../templates/variables/docker-images.yml | 2 +- 9 files changed, 53 insertions(+), 1 deletion(-) diff --git a/eng/docker-tools/CHANGELOG.md b/eng/docker-tools/CHANGELOG.md index 0936f6feb6..09e796d6f2 100644 --- a/eng/docker-tools/CHANGELOG.md +++ b/eng/docker-tools/CHANGELOG.md @@ -4,6 +4,15 @@ All breaking changes and new features in `eng/docker-tools` will be documented i --- +## 2026-08-10: Pre-ImageBuilder build customization + +Build pipeline templates now accept `customPreImageBuilderBuildSteps`. These steps run after +ImageBuilder is available but before repository content is copied into the Linux ImageBuilder +image. Repositories can use the hook to stage files into Docker build contexts, such as shared +`eng/common` content required by Dockerfiles. + +--- + ## 2026-06-11: Configurable per-registry referrer-lookup rate limit - Issue: [#2141](https://github.com/dotnet/docker-tools/issues/2141) diff --git a/eng/docker-tools/DEV-GUIDE.md b/eng/docker-tools/DEV-GUIDE.md index 9d4eaee7c4..ba45f30771 100644 --- a/eng/docker-tools/DEV-GUIDE.md +++ b/eng/docker-tools/DEV-GUIDE.md @@ -389,6 +389,22 @@ To force a rebuild regardless of cache state, set the `noCache` parameter to `tr ## Common Customization Patterns +### Pattern: Staging Files into Docker Build Contexts + +Use `customPreImageBuilderBuildSteps` to modify repository content immediately before the +repository is copied into the Linux ImageBuilder image. For example, a repository can stage +shared `eng/common` files next to Dockerfiles whose build contexts cannot access the repository +root: + +```yaml +customPreImageBuilderBuildSteps: +- powershell: ./eng/Stage-EngCommon.ps1 + displayName: Stage eng/common in Docker Build Contexts +``` + +The steps run once per Linux build job, after ImageBuilder is available and before the +`Dockerfile.WithRepo` image is built. + ### Pattern: Adding Build Arguments Pass Dockerfile `ARG` values via ImageBuilder: diff --git a/eng/docker-tools/templates/jobs/build-images.yml b/eng/docker-tools/templates/jobs/build-images.yml index 7327b6d697..a77c02cb7c 100644 --- a/eng/docker-tools/templates/jobs/build-images.yml +++ b/eng/docker-tools/templates/jobs/build-images.yml @@ -10,6 +10,8 @@ parameters: # Custom steps that run after ImageBuilder is set up but before the build starts. # Use for build-specific initialization (e.g., setting variables, additional setup). customBuildInitSteps: [] + # Custom steps that modify repository content before it is copied into the ImageBuilder image. + customPreImageBuilderBuildSteps: [] publishConfig: null versionsRepoRef: "" noCache: false @@ -41,6 +43,7 @@ jobs: versionsRepoRef: ${{ parameters.versionsRepoRef }} cleanupDocker: true customInitSteps: ${{ parameters.customInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} - ${{ parameters.customBuildInitSteps }} - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self parameters: diff --git a/eng/docker-tools/templates/stages/build-and-test.yml b/eng/docker-tools/templates/stages/build-and-test.yml index a03804aa89..6d602fe401 100644 --- a/eng/docker-tools/templates/stages/build-and-test.yml +++ b/eng/docker-tools/templates/stages/build-and-test.yml @@ -13,6 +13,8 @@ parameters: # Custom steps that run after ImageBuilder is set up but before the build starts. # Use for build-specific initialization (e.g., setting variables, additional setup). customBuildInitSteps: [] + # Custom steps that modify repository content before it is copied into the ImageBuilder image. + customPreImageBuilderBuildSteps: [] customTestInitSteps: [] sourceBuildPipelineRunId: "" # When true, the Post-Build stage runs even if the Build stage failed (succeededOrFailed). @@ -113,6 +115,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -128,6 +131,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -143,6 +147,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -158,6 +163,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -173,6 +179,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -188,6 +195,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -203,6 +211,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} diff --git a/eng/docker-tools/templates/stages/dotnet/build-and-test.yml b/eng/docker-tools/templates/stages/dotnet/build-and-test.yml index 5b7e23379b..81eee2c9e7 100644 --- a/eng/docker-tools/templates/stages/dotnet/build-and-test.yml +++ b/eng/docker-tools/templates/stages/dotnet/build-and-test.yml @@ -29,6 +29,7 @@ parameters: linuxArmBuildJobTimeout: 60 windowsAmdBuildJobTimeout: 60 customBuildInitSteps: [] + customPreImageBuilderBuildSteps: [] # Test parameters testMatrixType: platformVersionedOs @@ -58,6 +59,7 @@ stages: testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps}} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} customInitSteps: ${{ parameters.customInitSteps }} customTestInitSteps: ${{ parameters.customTestInitSteps }} windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} diff --git a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml index 2c924ef0d7..9102d994cb 100644 --- a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml @@ -21,6 +21,7 @@ parameters: linuxArmBuildJobTimeout: 60 windowsAmdBuildJobTimeout: 60 customBuildInitSteps: [] + customPreImageBuilderBuildSteps: [] # Test parameters testMatrixType: platformVersionedOs @@ -60,6 +61,7 @@ stages: linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} customInitSteps: ${{ parameters.customInitSteps }} # Test sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/docker-tools/templates/steps/init-common.yml b/eng/docker-tools/templates/steps/init-common.yml index eda572618c..25d22a7b1e 100644 --- a/eng/docker-tools/templates/steps/init-common.yml +++ b/eng/docker-tools/templates/steps/init-common.yml @@ -34,6 +34,11 @@ parameters: type: stepList default: [] +# Custom steps that modify repository content before it is copied into the ImageBuilder image. +- name: customPreImageBuilderBuildSteps + type: stepList + default: [] + # Registry and authentication configuration for publishing images. # Contains server URLs, repo prefixes, subscriptions, and resource groups. # When null, build/publish steps that require registry access will be skipped. @@ -245,3 +250,4 @@ steps: publishConfig: ${{ parameters.publishConfig }} condition: ${{ parameters.condition }} customInitSteps: ${{ parameters.customInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} diff --git a/eng/docker-tools/templates/steps/init-imagebuilder.yml b/eng/docker-tools/templates/steps/init-imagebuilder.yml index b85c62b6c4..1e4f57a1f8 100644 --- a/eng/docker-tools/templates/steps/init-imagebuilder.yml +++ b/eng/docker-tools/templates/steps/init-imagebuilder.yml @@ -21,6 +21,10 @@ parameters: type: stepList default: [] +- name: customPreImageBuilderBuildSteps + type: stepList + default: [] + steps: # Custom ImageBuilder setup (e.g., bootstrap from source) - ${{ if gt(length(parameters.customInitSteps), 0) }}: @@ -67,6 +71,7 @@ steps: # The withrepo image layers the checked-out repository into the ImageBuilder # container at /repo, so ImageBuilder can access manifests and Dockerfiles - ${{ if eq(parameters.dockerClientOS, 'linux') }}: + - ${{ parameters.customPreImageBuilderBuildSteps }} - script: >- docker build -t $(imageNames.imageBuilder.withrepo) diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml index abc02dff3a..5bbb32dd5a 100644 --- a/eng/docker-tools/templates/variables/docker-images.yml +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:79ee50949017dfce2212d66aa473e4361351b2aefda2c5df1dab566f20b68001 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:c790fdb4e4861ebe7d9f0eb52bf9c91a07843ee98c27c00fd62028ff3d4586f7 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner From 4bd95c15c72d9a2508624a4d0532729e23404195 Mon Sep 17 00:00:00 2001 From: "net-containers-bot[bot]" <307772199+net-containers-bot[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 10:02:45 -0700 Subject: [PATCH 05/15] [nightly] Update common Docker engineering infrastructure with latest (#7358) Co-authored-by: net-containers-bot[bot] <307772199+net-containers-bot[bot]@users.noreply.github.com> --- eng/docker-tools/templates/variables/docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml index 5bbb32dd5a..09c3567f78 100644 --- a/eng/docker-tools/templates/variables/docker-images.yml +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:c790fdb4e4861ebe7d9f0eb52bf9c91a07843ee98c27c00fd62028ff3d4586f7 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:152c17472c4b93b531b784b029033c3c6128f14dba1e71f8d717028d4b914ec8 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner From ff4f5b5c6d43dbdd2f2cbd1349e9293cc7dd9200 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Tue, 22 Sep 2026 07:48:49 -0700 Subject: [PATCH 06/15] Rework update-dependencies (#7362) Architecture - Removed monolithic `SpecificCommand` that all updates used to flow through - Added new `ManifestVariables` component that centralizes updating manifest variables while preserving formatting (was previously managed through M.DN.VersionTools) - Tools are now first class - all updatable components are treated the same - Converted PowerShell scripts for updating .NET Monitor into a new C# command - Removed dead PowerShell scripts for updating Aspire - Removed dead code for fetching .NET SHA512 hashes Behavior - GitHub pull requests now use GitHub App authentication - Pull requests now include version in the title/commit - fixes https://github.com/dotnet/dotnet-docker/issues/5489 - This was up for debate in the past, but this change will save me 30 seconds - 1 minute per automated pull request. In order to have a descriptive commit message, I typically have to look at the pull request's code, copy the precise version of the new component, and then manually write the commit message when squash-merging the PR. - Automation no longer force pushes away maintainer commits Command line - **Commands are constructed automatically** for each dependency based on capabilities expressed through interfaces - **Adopted Microsoft.Extensions.Configuration for static configuration** in order to make command line code simpler - Upgraded `System.CommandLine` from 2.0.0-beta.5 to GA - Removed obsolete `System.CommandLine.Hosting` library - Removed obsolete `System.CommandLine.NamingConventionBinder` library - binding is now manual The command line interface has changed. Before, commands were set up ad-hoc/manually and could differ in structure or arguments. Now, most commands are constructed automatically. - Typical structure: `update-dependencies ` - Example: `update-dependencies dotnet channel 1234` Formatting - Changed namespace from `Dotnet.Docker` to `Microsoft.DotNet.Docker.UpdateDependencies` - fixes https://github.com/dotnet/docker-tools/issues/1430 - Adopted file-scoped namespaces everywhere - Removed all unused usings Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CONTRIBUTING.md | 36 +- eng/DependencyManagement.psm1 | 20 - eng/Get-AspireDropVersions.ps1 | 42 -- eng/Get-DropVersions.ps1 | 354 ----------- eng/Get-MonitorDropVersions.ps1 | 27 - eng/Set-DotnetVersions.ps1 | 146 ----- .../pipelines/sync-internal-release.yml | 12 +- .../update-dependencies-internal.yml | 15 +- .../pipelines/update-dependencies.yml | 71 ++- eng/pipelines/steps/get-bar-token.yml | 19 + .../steps/update-dependencies-specific.yml | 51 -- .../update-dotnet-dependencies-specific.yml | 92 --- eng/pipelines/update-dependencies-monitor.yml | 52 +- .../update-dependencies-official.yml | 10 +- eng/update-dependencies/AGENTS.md | 61 ++ .../AspireBuildUpdaterService.cs | 113 ---- eng/update-dependencies/AzdoAuthProvider.cs | 57 +- eng/update-dependencies/AzdoHttpClient.cs | 22 +- eng/update-dependencies/BaseCommand.cs | 39 -- eng/update-dependencies/BaseUrlUpdater.cs | 109 ---- eng/update-dependencies/BuildAssetService.cs | 10 +- eng/update-dependencies/BuildExtensions.cs | 27 - eng/update-dependencies/BuildLabelService.cs | 2 +- eng/update-dependencies/BuildRepo.cs | 15 - eng/update-dependencies/ChangeMode.cs | 20 - eng/update-dependencies/ChecksumHelper.cs | 14 +- eng/update-dependencies/ChiselUpdater.cs | 98 --- .../ChiselVersionSource.cs | 15 - .../Commands/CreatePullRequestOptions.cs | 63 ++ .../Commands/DependencyCommand.cs | 35 ++ .../Commands/FromBuildCommand.cs | 33 ++ .../Commands/FromBuildOptions.cs | 22 + .../Commands/FromChannelCommand.cs | 29 + .../Commands/FromChannelOptions.cs | 22 + .../Commands/FromGitHubReleaseCommand.cs | 27 + .../Commands/FromPipelineBuildCommand.cs | 32 + .../Commands/FromPipelineBuildOptions.cs | 22 + .../FromStagingPipelineCommand.cs | 166 +++--- .../FromStagingPipelineOptions.cs | 66 +-- .../Commands/FromVersionCommand.cs | 29 + .../Commands/FromVersionOptions.cs | 25 + .../{ => Commands}/ICommand.cs | 5 +- .../Commands/PositiveIdArgument.cs | 21 + .../SyncInternalReleaseCommand.cs | 45 +- .../Commands/SyncInternalReleaseOptions.cs | 23 + .../ComponentVersionChannel.cs | 15 - .../ComponentVersionInfo.cs | 9 - .../CreatePullRequestOptions.cs | 55 -- .../CreatePullRequestOptionsExtensions.cs | 71 --- .../DependencyInjectionExtensions.cs | 45 -- .../DependencyUpdateRunner.cs | 133 +++++ eng/update-dependencies/DirectoryStack.cs | 2 +- eng/update-dependencies/Dockerfile | 36 -- .../DockerfileShaUpdater.cs | 428 ------------- .../EmptyDependencyUpdater.cs | 12 - eng/update-dependencies/EnvironmentService.cs | 3 +- eng/update-dependencies/ErrorTraceListener.cs | 33 -- eng/update-dependencies/FromBuildCommand.cs | 41 -- eng/update-dependencies/FromBuildOptions.cs | 27 - eng/update-dependencies/FromChannelCommand.cs | 47 -- eng/update-dependencies/FromChannelOptions.cs | 33 -- .../FromComponentCommand.cs | 65 -- .../FromComponentOptions.cs | 55 -- eng/update-dependencies/Git/GitRemoteInfo.cs | 2 +- eng/update-dependencies/Git/GitRepoHelper.cs | 6 +- .../Git/GitRepoHelperFactory.cs | 2 +- .../Git/GitValidationExtensions.cs | 2 +- eng/update-dependencies/Git/IGitRepoHelper.cs | 5 +- .../Git/ILocalGitRepoHelper.cs | 5 +- .../Git/IRemoteGitRepoHelper.cs | 2 +- .../Git/InvalidBranchException.cs | 4 +- .../Git/LocalGitRepoHelper.cs | 6 +- .../Git/PullRequestCreationInfo.cs | 2 +- .../Git/RemoteGitRepoFactory.cs | 3 +- .../Git/RemoteGitRepoHelper.cs | 2 +- eng/update-dependencies/GitHubHelper.cs | 28 - eng/update-dependencies/GitHubReleaseInfo.cs | 12 - .../GitHubReleaseUpdaterBase.cs | 58 -- .../GitHubReleaseUrlUpdater.cs | 35 -- .../GitHubReleaseVersionUpdater.cs | 23 - eng/update-dependencies/GitRemote.cs | 4 +- .../IBuildUpdaterService.cs | 11 - .../IDependencyVersionSource.cs | 26 - eng/update-dependencies/IManifestVariables.cs | 16 - eng/update-dependencies/IOptions.cs | 13 - eng/update-dependencies/JsonHelper.cs | 14 - eng/update-dependencies/ManifestHelper.cs | 96 +-- eng/update-dependencies/ManifestVariables.cs | 305 +++++++++- eng/update-dependencies/MinGitUpdater.cs | 68 --- .../Model/Release/ReleaseConfig.cs | 4 +- .../PipelineArtifactProvider.cs | 35 +- eng/update-dependencies/PipelinesService.cs | 2 +- eng/update-dependencies/ProductCommits.cs | 2 +- eng/update-dependencies/Program.cs | 256 ++++---- eng/update-dependencies/README.md | 69 +-- eng/update-dependencies/ReleaseState.cs | 2 +- .../RocksToolboxUpdater.cs | 29 - eng/update-dependencies/ScriptRunner.cs | 73 +++ .../ScriptRunnerUpdater.cs | 68 --- eng/update-dependencies/SpecificCommand.cs | 560 ------------------ .../SpecificCommandOptions.cs | 126 ---- eng/update-dependencies/SyftUpdater.cs | 26 - .../Sync/DotNetVersionExtensions.cs | 2 +- .../Sync/IInternalVersionsService.cs | 2 +- .../Sync/InternalStageContainers.cs | 2 +- .../Sync/InternalVersionsService.cs | 2 +- .../Sync/SyncInternalReleaseOptions.cs | 22 - eng/update-dependencies/Tools.cs | 35 -- .../UpdateDependenciesConfiguration.cs | 47 ++ .../Updaters/AspireUpdater.cs | 95 +++ .../Updaters/BaseUrlUpdater.cs | 50 ++ .../Updaters/ChiselUpdater.cs | 73 +++ .../Updaters/DependencyUpdate.cs | 39 ++ .../Updaters/DotNetUpdater.cs | 135 +++++ .../Updaters/IBarBuildUpdater.cs | 11 + .../Updaters/IBarChannelUpdater.cs | 9 + .../Updaters/IGitHubReleaseUpdater.cs | 9 + .../Updaters/IPipelineBuildUpdater.cs | 9 + .../Updaters/IStagingPipelineUpdater.cs | 16 + eng/update-dependencies/Updaters/IUpdater.cs | 10 + .../Updaters/IVersionUpdater.cs | 9 + .../Updaters/MinGitUpdater.cs | 67 +++ .../Updaters/MonitorUpdater.cs | 156 +++++ .../{ => Updaters}/NuGetConfigUpdater.cs | 87 +-- .../Updaters/RocksToolboxUpdater.cs | 33 ++ .../Updaters/SyftUpdater.cs | 33 ++ eng/update-dependencies/VariableHelper.cs | 43 +- eng/update-dependencies/VariableUpdateInfo.cs | 13 - eng/update-dependencies/VariableUpdater.cs | 31 - .../VariableUpdaterBase.cs | 34 -- eng/update-dependencies/VersionHelper.cs | 6 +- eng/update-dependencies/VersionType.cs | 14 - eng/update-dependencies/VersionUpdater.cs | 106 ---- .../VmrBuildUpdaterService.cs | 65 -- eng/update-dependencies/appsettings.json | 13 + .../update-dependencies.csproj | 21 +- .../BuildLabelServiceTests.cs | 2 +- .../DotNetUpdaterTests.cs | 91 +++ .../FromStagingPipelineCommandTests.cs | 54 +- .../InternalStagingBuildsTests.cs | 2 +- .../LocalGitRepoHelperTests.cs | 2 +- .../ManifestVariablesTests.cs | 190 ++++++ .../MonitorUpdaterTests.cs | 70 +++ .../NuGetConfigUpdaterTests.cs | 46 ++ .../SyncInternalReleaseTests.cs | 61 +- .../VersionHelperTests.cs | 19 + 146 files changed, 2787 insertions(+), 4375 deletions(-) delete mode 100644 eng/Get-AspireDropVersions.ps1 delete mode 100644 eng/Get-DropVersions.ps1 delete mode 100644 eng/Get-MonitorDropVersions.ps1 delete mode 100644 eng/Set-DotnetVersions.ps1 create mode 100644 eng/pipelines/steps/get-bar-token.yml delete mode 100644 eng/pipelines/steps/update-dependencies-specific.yml delete mode 100644 eng/pipelines/steps/update-dotnet-dependencies-specific.yml create mode 100644 eng/update-dependencies/AGENTS.md delete mode 100644 eng/update-dependencies/AspireBuildUpdaterService.cs delete mode 100644 eng/update-dependencies/BaseCommand.cs delete mode 100644 eng/update-dependencies/BaseUrlUpdater.cs delete mode 100644 eng/update-dependencies/BuildExtensions.cs delete mode 100644 eng/update-dependencies/BuildRepo.cs delete mode 100644 eng/update-dependencies/ChangeMode.cs delete mode 100644 eng/update-dependencies/ChiselUpdater.cs delete mode 100644 eng/update-dependencies/ChiselVersionSource.cs create mode 100644 eng/update-dependencies/Commands/CreatePullRequestOptions.cs create mode 100644 eng/update-dependencies/Commands/DependencyCommand.cs create mode 100644 eng/update-dependencies/Commands/FromBuildCommand.cs create mode 100644 eng/update-dependencies/Commands/FromBuildOptions.cs create mode 100644 eng/update-dependencies/Commands/FromChannelCommand.cs create mode 100644 eng/update-dependencies/Commands/FromChannelOptions.cs create mode 100644 eng/update-dependencies/Commands/FromGitHubReleaseCommand.cs create mode 100644 eng/update-dependencies/Commands/FromPipelineBuildCommand.cs create mode 100644 eng/update-dependencies/Commands/FromPipelineBuildOptions.cs rename eng/update-dependencies/{ => Commands}/FromStagingPipelineCommand.cs (70%) rename eng/update-dependencies/{ => Commands}/FromStagingPipelineOptions.cs (60%) create mode 100644 eng/update-dependencies/Commands/FromVersionCommand.cs create mode 100644 eng/update-dependencies/Commands/FromVersionOptions.cs rename eng/update-dependencies/{ => Commands}/ICommand.cs (72%) create mode 100644 eng/update-dependencies/Commands/PositiveIdArgument.cs rename eng/update-dependencies/{Sync => Commands}/SyncInternalReleaseCommand.cs (86%) create mode 100644 eng/update-dependencies/Commands/SyncInternalReleaseOptions.cs delete mode 100644 eng/update-dependencies/ComponentVersionChannel.cs delete mode 100644 eng/update-dependencies/ComponentVersionInfo.cs delete mode 100644 eng/update-dependencies/CreatePullRequestOptions.cs delete mode 100644 eng/update-dependencies/CreatePullRequestOptionsExtensions.cs delete mode 100644 eng/update-dependencies/DependencyInjectionExtensions.cs create mode 100644 eng/update-dependencies/DependencyUpdateRunner.cs delete mode 100644 eng/update-dependencies/Dockerfile delete mode 100644 eng/update-dependencies/DockerfileShaUpdater.cs delete mode 100644 eng/update-dependencies/EmptyDependencyUpdater.cs delete mode 100644 eng/update-dependencies/ErrorTraceListener.cs delete mode 100644 eng/update-dependencies/FromBuildCommand.cs delete mode 100644 eng/update-dependencies/FromBuildOptions.cs delete mode 100644 eng/update-dependencies/FromChannelCommand.cs delete mode 100644 eng/update-dependencies/FromChannelOptions.cs delete mode 100644 eng/update-dependencies/FromComponentCommand.cs delete mode 100644 eng/update-dependencies/FromComponentOptions.cs delete mode 100644 eng/update-dependencies/GitHubHelper.cs delete mode 100644 eng/update-dependencies/GitHubReleaseInfo.cs delete mode 100644 eng/update-dependencies/GitHubReleaseUpdaterBase.cs delete mode 100644 eng/update-dependencies/GitHubReleaseUrlUpdater.cs delete mode 100644 eng/update-dependencies/GitHubReleaseVersionUpdater.cs delete mode 100644 eng/update-dependencies/IBuildUpdaterService.cs delete mode 100644 eng/update-dependencies/IDependencyVersionSource.cs delete mode 100644 eng/update-dependencies/IManifestVariables.cs delete mode 100644 eng/update-dependencies/IOptions.cs delete mode 100644 eng/update-dependencies/JsonHelper.cs delete mode 100644 eng/update-dependencies/MinGitUpdater.cs delete mode 100644 eng/update-dependencies/RocksToolboxUpdater.cs create mode 100644 eng/update-dependencies/ScriptRunner.cs delete mode 100644 eng/update-dependencies/ScriptRunnerUpdater.cs delete mode 100644 eng/update-dependencies/SpecificCommand.cs delete mode 100644 eng/update-dependencies/SpecificCommandOptions.cs delete mode 100644 eng/update-dependencies/SyftUpdater.cs delete mode 100644 eng/update-dependencies/Sync/SyncInternalReleaseOptions.cs delete mode 100644 eng/update-dependencies/Tools.cs create mode 100644 eng/update-dependencies/UpdateDependenciesConfiguration.cs create mode 100644 eng/update-dependencies/Updaters/AspireUpdater.cs create mode 100644 eng/update-dependencies/Updaters/BaseUrlUpdater.cs create mode 100644 eng/update-dependencies/Updaters/ChiselUpdater.cs create mode 100644 eng/update-dependencies/Updaters/DependencyUpdate.cs create mode 100644 eng/update-dependencies/Updaters/DotNetUpdater.cs create mode 100644 eng/update-dependencies/Updaters/IBarBuildUpdater.cs create mode 100644 eng/update-dependencies/Updaters/IBarChannelUpdater.cs create mode 100644 eng/update-dependencies/Updaters/IGitHubReleaseUpdater.cs create mode 100644 eng/update-dependencies/Updaters/IPipelineBuildUpdater.cs create mode 100644 eng/update-dependencies/Updaters/IStagingPipelineUpdater.cs create mode 100644 eng/update-dependencies/Updaters/IUpdater.cs create mode 100644 eng/update-dependencies/Updaters/IVersionUpdater.cs create mode 100644 eng/update-dependencies/Updaters/MinGitUpdater.cs create mode 100644 eng/update-dependencies/Updaters/MonitorUpdater.cs rename eng/update-dependencies/{ => Updaters}/NuGetConfigUpdater.cs (68%) create mode 100644 eng/update-dependencies/Updaters/RocksToolboxUpdater.cs create mode 100644 eng/update-dependencies/Updaters/SyftUpdater.cs delete mode 100644 eng/update-dependencies/VariableUpdateInfo.cs delete mode 100644 eng/update-dependencies/VariableUpdater.cs delete mode 100644 eng/update-dependencies/VariableUpdaterBase.cs delete mode 100644 eng/update-dependencies/VersionType.cs delete mode 100644 eng/update-dependencies/VersionUpdater.cs delete mode 100644 eng/update-dependencies/VmrBuildUpdaterService.cs create mode 100644 eng/update-dependencies/appsettings.json create mode 100644 tests/UpdateDependencies.Tests/DotNetUpdaterTests.cs create mode 100644 tests/UpdateDependencies.Tests/ManifestVariablesTests.cs create mode 100644 tests/UpdateDependencies.Tests/MonitorUpdaterTests.cs create mode 100644 tests/UpdateDependencies.Tests/NuGetConfigUpdaterTests.cs create mode 100644 tests/UpdateDependencies.Tests/VersionHelperTests.cs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bfc08bfef1..289952dacf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -190,33 +190,23 @@ When adding or removing Dockerfiles, it is important to update the `manifest.jso ### Updating Product Versions -Updating the product versions (e.g. .NET runtime, ASP.NET runtime, PowerShell, etc.) contained within the images is typically performed by automation. All of the product version information is stored in the [`manifest.versions.json`](https://github.com/dotnet/dotnet-docker/blob/main/manifest.versions.json) file. The Dockerfile templates reference the product versions numbers and checksums from this file. Updating a product version involves updating the `manifest.versions.json` and regenerating the Dockerfiles. If there are cases where you need to update a product version, you can use the [update-dependencies](https://github.com/dotnet/dotnet-docker/tree/main/eng/update-dependencies) tool. The tool will do the following: +All of the product version information is stored in [`manifest.versions.json`]. +The Dockerfile templates reference the product versions from this file and use +them as template variables. Updating a product version involves updating +`manifest.versions.json` and regenerating the Dockerfiles. -1. Update the product versions and checksums stored in `manifest.versions.json` -1. Regenerate the Dockerfiles -1. Update the tags listing in the readmes +Updating product versions is typically performed by automation. +The [update-dependencies] tool can update the versions of many image components +from many different sources. It can be run locally or in a pipeline. To run it +locally, use `dotnet run --project eng/update-dependencies -- `. -Use the [new .NET version release lifecycle](eng/developer-docs/dotnet-release-lifecycle.md) when planning work for a .NET release. +For more details: -The following examples illustrate how to run `update-dependencies`: +- [README.md](eng/update-dependencies/README.md): Usage guidelines +- [AGENTS.md](eng/update-dependencies/AGENTS.md): Coding guidelines -- Update the 9.0 product versions (uses a helper script for running update-dependencies) - - ``` console - > ./eng/Set-DotnetVersions.ps1 -ProductVersion 9.0 -SdkVersion 9.0.100 -RuntimeVersion 9.0.0 -AspnetVersion 9.0.0 - ``` - -- Update the .NET Monitor version (uses a helper script for running update-dependencies) - - ``` console - > ./eng/Set-DotnetVersions.ps1 -ProductVersion 8.0 -MonitorVersion 8.0.5 - ``` - -- Update the PowerShell version used in the 9.0 images - - ``` console - > dotnet run --project .\eng\update-dependencies\ -- specific 9.0 --product-version powershell=7.5.0 - ``` +[`manifest.versions.json`]: manifest.versions.json +[update-dependencies]: eng/update-dependencies #### Checking Markdown links locally diff --git a/eng/DependencyManagement.psm1 b/eng/DependencyManagement.psm1 index 21c3899ce9..959f4d1a03 100644 --- a/eng/DependencyManagement.psm1 +++ b/eng/DependencyManagement.psm1 @@ -14,26 +14,6 @@ function Get-Branch() { } } -function Get-IsStableBranding([string] $version) { - return $Version.Contains("-servicing") -or $Version.Contains("-rtm") -} - -function Resolve-DotnetProductUrl([string] $akaMsUrl) { - Write-Host "Querying $akaMsUrl" - $response = Invoke-WebRequest -Uri $akaMsUrl -Method Head - $resolvedUrl = $response.BaseResponse.RequestMessage.RequestUri.AbsoluteUri - Write-Host "Resolved URL: $resolvedUrl" - return $resolvedUrl -} - -function Get-ProductReleaseState() { - if ($(Get-Branch) -ieq 'main') { - return 'Release' - } else { - return 'Prerelease' - } -} - function Get-DockerOs() { return docker version -f "{{ .Server.Os }}" } diff --git a/eng/Get-AspireDropVersions.ps1 b/eng/Get-AspireDropVersions.ps1 deleted file mode 100644 index 12223cff94..0000000000 --- a/eng/Get-AspireDropVersions.ps1 +++ /dev/null @@ -1,42 +0,0 @@ -<# -.SYNOPSIS -Returns the various component versions of the latest .NET build. -#> -[cmdletbinding()] -param( - # The release channel to use for determining the latest .NET build. - [Parameter(ParameterSetName = "Channel")] - [string] - $Channel, - - [Parameter(ParameterSetName = "Explicit")] - # Aspire version to target - [string] - $AspireVersion -) - -$ErrorActionPreference = 'Stop' -Import-Module -force $PSScriptRoot/DependencyManagement.psm1 - -if ($Channel) { - # Example channel: '8.0/daily' - $akaMsUrl = "https://aka.ms/dotnet/${Channel}/aspire-dashboard-linux-x64.zip" - $versionSpecificUrl = Resolve-DotnetProductUrl $akaMsUrl - - # Assume the versionSpecificUrl is a string like - # https://ci.dot.net/public/aspire/8.0.0-preview.X.YYYYY.Z/aspire-dashboard-linux-x64.zip - $aspireVersion = $versionSpecificUrl -replace '^.*/aspire/([^/]+)/.*$', '$1' - - if (Get-IsStableBranding $aspireVersion) { - # The stable version for 8.0.0-preview.4.24105.1 is 8.0.0-preview.4 - $aspireVersion = $aspireVersion -replace '^(.*?)-.*$', '$1' - } -} else { - $aspireVersion = $AspireVersion -} - -# Grab the major.minor version from the Aspire version string -$dockerfileVersion = $aspireVersion -replace '^(\d+\.\d+).*$', '$1' - -Write-Output "##vso[task.setvariable variable=dockerfileVersion]$dockerfileVersion" -Write-Output "##vso[task.setvariable variable=aspireVersion]$aspireVersion" diff --git a/eng/Get-DropVersions.ps1 b/eng/Get-DropVersions.ps1 deleted file mode 100644 index 95c175958e..0000000000 --- a/eng/Get-DropVersions.ps1 +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env pwsh - -<# -.SYNOPSIS -Returns the various component versions of the latest .NET build. -#> -[cmdletbinding()] -param( - # The release channel to use for determining the latest .NET build. - [Parameter(ParameterSetName = "Channel")] - [string] - $Channel, - - # The pipeline run ID of the .NET release staging build. - [Parameter(ParameterSetName = "BuildId")] - [string] - $BuildId, - - [Parameter(ParameterSetName = "Explicit")] - # SDK versions to target - [string[]] - $SdkVersions, - - # Whether to target an internal .NET build - [switch] - $UseInternalBuild, - - # Whether to call Set-DotnetVersions with the new versions - [switch] - $UpdateDependencies, - - # PAT used to access the versions repo in AzDO - [string] - $AzdoVersionsRepoInfoAccessToken, - - # PAT used to access internal AzDO build artifacts - [string] - $InternalAccessToken -) - -Import-Module -force $PSScriptRoot/DependencyManagement.psm1 - -function GetSdkVersionInfo([string]$sdkUrl) { - New-Item -Path $tempDir -ItemType Directory -Force | Out-Null - - try { - # Download the SDK - Write-Host "Downloading SDK from $sdkUrl" - $sdkOutPath = "$tempDir/sdk.zip" - Invoke-WebRequest -Uri $sdkUrl -OutFile $sdkOutPath - - # Extract .version file from SDK zip - $zipFile = [IO.Compression.ZipFile]::OpenRead($sdkOutPath) - try { - $zipFile.Entries | Where-Object { $_.FullName -like "sdk/*/.version" } | ForEach-Object { - [IO.Compression.ZipFileExtensions]::ExtractToFile($_, "$tempDir/$($_.Name)") - } - } - finally { - $zipFile.Dispose() - } - - # Get the commit SHA from the .version file. Example contents: - # - # Unstable version: - # f54f69f9aade0a00936aca343c3eb493d5edbc05 - # 8.0.100-rtm.23512.13 - # win-x64 - # 8.0.100-rtm.23512.13 - # - # Stable version: - # eb26aacfecb08dd87b418f4cfaf7faa10eb1900f - # 7.0.401 - # win-x64 - # 7.0.401-servicing.23425.30 - # - $versionInfoText = $(Get-Content "$tempDir/.version") - - $commitSha = $versionInfoText[0].Trim() - $shortVersion = $versionInfoText[1].Trim() - $rid = $versionInfoText[2].Trim() - $fullVersion = $versionInfoText[3].Trim() - $isStableVersion = $fullVersion -ne $shortVersion - - return [PSCustomObject]@{ - CommitSha = $commitSha - Version = $fullVersion - IsStableVersion = $isStableVersion - Rid = $rid - } - } - finally { - Remove-Item $tempDir -Force -Recurse -ErrorAction Ignore - } -} - -function ResolveSdkUrl([string]$sdkVersion, [bool]$useStableBranding) { - if ($useStableBranding) { - $sdkStableVersion = ($sdkVersion -split "-")[0] - } - else { - $sdkStableVersion = $sdkVersion - } - - $zipFile = "dotnet-sdk-$sdkStableVersion-win-x64.zip" - - $containerVersion = $sdkVersion.Replace(".", "-") - - if ($UseInternalBuild) { - $sdkUrl = "https://dotnetstage.blob.core.windows.net/$containerVersion-internal/Sdk/$sdkVersion/$zipFile" - } - else { - $sdkUrl = "https://ci.dot.net/public/Sdk/$sdkVersion/$zipFile" - } - return $sdkUrl -} - -function GetVersionDetails([string]$commitSha, [string]$dockerfileVersion) { - $versionDetailsPath="eng/Version.Details.xml" - - if (([Version]$dockerfileVersion).Major -le 8) { - $repoName = "installer" - $repoId = "c20f712b-f093-40de-9013-d6b084c1ff30" - } - else { - $repoName = "sdk" - $repoId = "7fa5dddb-89e8-4b26-8595-a6d15593e354" - } - - if ($UseInternalBuild) { - $versionDetailsUrl="https://dev.azure.com/dnceng/internal/_apis/git/repositories/$repoId/items?scopePath=/$versionDetailsPath&api-version=6.0&version=$commitSha&versionType=commit" - $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzdoVersionsRepoInfoAccessToken")) - $headers = @{ - "Authorization" = "Basic $base64AccessToken" - } - } - else { - $versionDetailsUrl="https://raw.githubusercontent.com/dotnet/$repoName/$commitSha/$versionDetailsPath" - $headers = @{} - } - - Write-Host "Downloading version details from $versionDetailsUrl" - $versionDetails = [xml](Invoke-RestMethod -Uri $versionDetailsUrl -Headers $headers) - return $versionDetails -} - -function GetDependencyVersion([string]$dependencyName, [xml]$versionDetails) { - $result = Select-Xml -XPath "//ProductDependencies/Dependency[starts-with(@Name,'$dependencyName')]/@Version" -Xml $versionDetails - return $result.Node.Value -} - -function GetVersionInfoFromBuildId([string]$buildId) { - $configFilename = "config.json" - $configPath = Join-Path $tempDir $configFilename - - try { - New-Item -Path $tempDir -ItemType Directory -Force | Out-Null - - $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$InternalAccessToken")) - $headers = @{ - "Authorization" = "Basic $base64AccessToken" - } - - $url = GetArtifactUrl 'drop' - $url = $url.Replace("content?format=zip", "content?format=file&subPath=%2F$configFilename") - - Invoke-WebRequest -OutFile $configPath $url -Headers $headers - - $config = $(Get-Content -Path $configPath | Out-String) | ConvertFrom-Json - - $isStableVersion = Get-IsStableBranding -Version $config.Sdk_Builds[0] - - if ($UseInternalBuild) { - return [PSCustomObject]@{ - DockerfileVersion = $config.Channel - SdkVersion = @($config.Sdk_Builds | Sort-Object -Descending)[0] - RuntimeVersion = $config.Runtime_Build - AspnetVersion = $config.Asp_Build - StableBranding = $isStableVersion - } - } else { - return [PSCustomObject]@{ - DockerfileVersion = $config.Channel - SdkVersion = @($config.Sdks | Sort-Object -Descending)[0] - RuntimeVersion = $config.Runtime - AspnetVersion = $config.Asp - StableBranding = $isStableVersion - } - } - } - catch [System.Management.Automation.CommandNotFoundException] { - Write-Error "Azure CLI is not installed. Please visit https://learn.microsoft.com/cli/azure/install-azure-cli." - Write-Host "Original Exception: $_" - exit 1 - } - finally { - Remove-Item -Force $configPath - } -} - -function GetArtifactUrl([string]$artifactName) { - $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$InternalAccessToken")) - $headers = @{ - "Authorization" = "Basic $base64AccessToken" - } - - $artifactsUrl = "https://dev.azure.com/dnceng/internal/_apis/build/builds/$BuildId/artifacts?api-version=6.0" - $response = Invoke-RestMethod -Uri $artifactsUrl -Method Get -Headers $headers - - $url = $null - foreach ($artifact in $response.value) { - if ($artifact.name -eq $artifactName) { - $url = $artifact.resource.downloadUrl - break - } - } - - if ($url -eq $null) { - Write-Error "Artifact '$artifactName' was not found in build# $BuildId" - exit 1 - } - - return $url -} - -function GetInternalBaseUrl() { - $shippingUrl = GetArtifactUrl 'shipping' - - # Format artifact URL into base-url - return $shippingUrl.Replace("content?format=zip", "content?format=file&subPath=%2Fassets") -} - -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' -Set-StrictMode -Version 2.0 - -$tempDir = Join-Path ([System.IO.Path]::GetTempPath()) -ChildPath "dotnet-docker-get-dropversions" | Join-Path -ChildPath ([System.Guid]::NewGuid()) - -if ($BuildId) { - if (!$InternalAccessToken) { - $InternalAccessToken = az account get-access-token --query accessToken --output tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to obtain access token using Azure CLI" - Write-Error "Please provide 'InternalAccessToken' parameter when using 'BuildId' option" - exit 1 - } - } -} - -if ($UseInternalBuild) { - if ($Channel) - { - $Channel = "internal/$Channel" - } - - if ($BuildId) { - $internalBaseUrl = GetInternalBaseUrl - } -} - -$sdkVersionInfos = @() - -if ($Channel) { - $sdkFile = "dotnet-sdk-win-x64.zip" - $akaMsUrl = "https://aka.ms/dotnet/$Channel/$sdkFile" - - $sdkUrl = Resolve-DotnetProductUrl $akaMsUrl - $sdkVersionInfos += GetSdkVersionInfo $sdkUrl -} - -foreach ($sdkVersion in $SdkVersions) -{ - $useStableBranding = Get-IsStableBranding -Version $sdkVersion - $sdkUrl = ResolveSdkUrl $sdkVersion $useStableBranding - $sdkVersionInfo = GetSdkVersionInfo $sdkUrl - $sdkVersionInfos += $sdkVersionInfo -} - -$versionInfos = @() -if ($BuildId) { - $versionInfos += GetVersionInfoFromBuildId($BuildId) -} - -foreach ($sdkVersionInfo in $SdkVersionInfos) { - $sdkVersionParts = $sdkVersionInfo.Version -split "\." - $dockerfileVersion = "$($sdkVersionParts[0]).$($sdkVersionParts[1])" - - $versionDetails = GetVersionDetails $sdkVersionInfo.CommitSha $dockerfileVersion - - $runtimeVersion = GetDependencyVersion "VS.Redist.Common.NetCore.SharedFramework.x64" $versionDetails - - $aspnetVersion = GetDependencyVersion "VS.Redist.Common.AspNetCore.SharedFramework.x64" $versionDetails - - if (-not $runtimeVersion) { - Write-Error "Unable to resolve the runtime version" - exit 1 - } - - if (-not $aspnetVersion) { - Write-Error "Unable to resolve the ASP.NET Core runtime version" - exit 1 - } - - Write-Host "Dockerfile version: $dockerfileVersion" - Write-Host "SDK version: $($sdkVersionInfo.Version)" - Write-Host "Runtime version: $runtimeVersion" - Write-Host "ASP.NET Core version: $aspnetVersion" - Write-Host - - $versionInfos += @{ - DockerfileVersion = $dockerfileVersion - SdkVersion = $sdkVersionInfo.Version - RuntimeVersion = $runtimeVersion - AspnetVersion = $aspnetVersion - StableBranding = $sdkVersionInfo.IsStableVersion - } -} - -if ($UpdateDependencies) -{ - $additionalArgs = @{} - - if ($UseInternalBuild) { - $additionalArgs += @{ InternalBaseUrl = "$internalBaseUrl" } - $additionalArgs += @{ InternalAccessToken = "$InternalAccessToken" } - } - - foreach ($versionInfo in $versionInfos) { - Write-Host "Dockerfile version: $($versionInfo.DockerfileVersion)" - Write-Host "SDK version: $($versionInfo.SdkVersion)" - Write-Host "Runtime version: $($versionInfo.RuntimeVersion)" - Write-Host "ASP.NET Core version: $($versionInfo.AspnetVersion)" - Write-Host - - if ($versionInfo.StableBranding) { - $additionalArgs += @{ UseStableBranding = $versionInfo.StableBranding } - } - - $setVersionsScript = Join-Path $PSScriptRoot "Set-DotnetVersions.ps1" - & $setVersionsScript ` - -ProductVersion $versionInfo.DockerfileVersion ` - -RuntimeVersion $versionInfo.RuntimeVersion ` - -AspnetVersion $versionInfo.AspnetVersion ` - -SdkVersion $versionInfo.SdkVersion ` - @additionalArgs - - Write-Host "`r`nDone: Updates for .NET $($versionInfo.RuntimeVersion)/$($versionInfo.SdkVersion)`r`n" - } -} else { - Write-Output "##vso[task.setvariable variable=versionInfos]$($versionInfos | ConvertTo-Json -Compress -AsArray)" - if ($UseInternalBuild) { - Write-Output "##vso[task.setvariable variable=internalBaseUrl]$internalBaseUrl" - } -} diff --git a/eng/Get-MonitorDropVersions.ps1 b/eng/Get-MonitorDropVersions.ps1 deleted file mode 100644 index 3fcb7630ed..0000000000 --- a/eng/Get-MonitorDropVersions.ps1 +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env pwsh - -<# -.SYNOPSIS -Returns the various component versions of the latest .NET Monitor build. -#> -[cmdletbinding()] -param( - # A file containing the latest .NET Monitor build version. - [Parameter(Mandatory=$true)] - [ValidateScript({if (-not (Test-Path -PathType Leaf -Path $(Resolve-Path $_).Path)) { throw "Could not find file $_"} $true})] - [string] - $BuildVersionFilePath -) - -Import-Module -force $PSScriptRoot/DependencyManagement.psm1 - -$monitorVersion = $(Get-Content $BuildVersionFilePath).Trim() - -$versionSplit=$monitorVersion.Split('.', 3) -$majorMinorVersion="$($versionSplit[0]).$($versionSplit[1])" - -$stableBranding = Get-IsStableBranding -Version $monitorVersion - -Write-Output "##vso[task.setvariable variable=monitorMajorMinorVersion]$majorMinorVersion" -Write-Output "##vso[task.setvariable variable=monitorVer]$monitorVersion" -Write-Output "##vso[task.setvariable variable=stableBranding]$stableBranding" diff --git a/eng/Set-DotnetVersions.ps1 b/eng/Set-DotnetVersions.ps1 deleted file mode 100644 index f1593ccd39..0000000000 --- a/eng/Set-DotnetVersions.ps1 +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env pwsh - -<# -.SYNOPSIS -Updates dependencies for the specified .NET version. -#> -[cmdletbinding()] -param( - # The major/minor version of the product (e.g. 6.0). - [Parameter(Mandatory = $true, Position = 0)] - [string] - $ProductVersion, - - # Build version of the SDK - [Parameter(Mandatory = $false, ParameterSetName = 'DotnetSdk')] - [string] - $SdkVersion, - - # Build version of ASP.NET Core - [Parameter(Mandatory = $false, ParameterSetName = 'DotnetSdk')] - [string] - $AspnetVersion, - - # Build version of the .NET runtime - [Parameter(Mandatory = $false, ParameterSetName = 'DotnetSdk')] - [string] - $RuntimeVersion, - - # Build verison of the .NET Monitor tool - [Parameter(Mandatory = $false, ParameterSetName = 'DotnetMonitor')] - [string] - $MonitorVersion, - - # Build verison of the Aspire Dashboard - [Parameter(Mandatory = $false, ParameterSetName = 'DotnetAspireDashboard')] - [string] - $AspireVersion, - - # Use stable branding version numbers to compute paths - [Switch] - $UseStableBranding, - - # When set, only prints out an Azure DevOps variable with the value of the args to pass to update-dependencies. - [string] - $AzdoVariableName, - - # File containing checksums for each product asset; used to override the behavior of locating the checksums from blob storage accounts. - [string] - $ChecksumsFile, - - # The release state of the product assets - [ValidateSet("Prerelease", "Release")] - [string] - $ReleaseState, - - # PAT used to access internal AzDO build artifacts - [string] - $InternalAccessToken, - - # Base Url for internal AzDO build artifacts - [string] - $InternalBaseUrl -) - -Import-Module -force $PSScriptRoot/DependencyManagement.psm1 - -# Use 'specific' command to set product versions to specific values -$updateDepsArgs = @("specific", $ProductVersion) - -if ($SdkVersion) { - $updateDepsArgs += @("--product-version", "sdk=$SdkVersion") -} - -if ($AspnetVersion) { - $updateDepsArgs += @("--product-version", "aspnet=$AspnetVersion", "--product-version", "aspnet-composite=$AspnetVersion") - - if (!$InternalBaseUrl) { - # rpm packages are only needed for 6.0 which isn't supported for internal testing scenarios - $updateDepsArgs += @("--product-version", "aspnet-runtime-targeting-pack=$AspnetVersion") - } -} - -if ($RuntimeVersion) { - $updateDepsArgs += @("--product-version", "runtime=$RuntimeVersion") - - if (!$InternalBaseUrl) { - # rpm packages are only needed for 6.0 which isn't supported for internal testing scenarios - $updateDepsArgs += @("--product-version", "runtime-apphost-pack=$RuntimeVersion", "--product-version", "runtime-targeting-pack=$RuntimeVersion", "--product-version", "runtime-host=$RuntimeVersion", "--product-version", "runtime-hostfxr=$RuntimeVersion", "--product-version", "netstandard-targeting-pack-2.1.0", "--product-version", "runtime-deps-cm.1=$RuntimeVersion", "--product-version", "runtime-deps-cm.2=$RuntimeVersion") - } -} - -if ($MonitorVersion) { - $updateDepsArgs += @("--product-version", "monitor=$MonitorVersion") - - $productMajorVersion = $ProductVersion.Split('.', 2)[0] - if ($productMajorVersion -ge 8) { - $updateDepsArgs += @("--product-version", "monitor-base=$MonitorVersion") - $updateDepsArgs += @("--product-version", "monitor-ext-azureblobstorage=$MonitorVersion") - $updateDepsArgs += @("--product-version", "monitor-ext-s3storage=$MonitorVersion") - } -} - -if ($AspireVersion) { - $updateDepsArgs += @("--product-version", "aspire-dashboard=$AspireVersion") - $productMajorVersion = $ProductVersion.Split('.', 2)[0] -} - -if ($ChecksumsFile) { - $updateDepsArgs += "--checksums-file=$ChecksumsFile" -} - -if ($UseStableBranding) { - $updateDepsArgs += "--stable-branding" -} - -if ($ReleaseState) { - $updateDepsArgs += "--release-state=$ReleaseState" -} - -if ($InternalAccessToken) { - $updateDepsArgs += "--internal-access-token=$InternalAccessToken" -} - -if ($InternalBaseUrl) { - $updateDepsArgs += "--internal-base-url=$InternalBaseUrl" -} - -$versionSourceName = switch ($PSCmdlet.ParameterSetName) { - "DotnetSdk" { "dotnet/sdk" } - "DotnetMonitor" { "dotnet/dotnet-monitor/$ProductVersion" } - "DotnetAspireDashboard" { "dotnet/aspire-dashboard/$ProductVersion" } - default { Write-Error -Message "Unknown version source" -ErrorAction Stop } -} - -if ($versionSourceName) { - $updateDepsArgs += "--version-source-name=$versionSourceName" -} - -$updateDepsArgs += "--source-branch=$(Get-Branch)" - -if ($AzdoVariableName) { - Write-Host "##vso[task.setvariable variable=$AzdoVariableName]$updateDepsArgs" -} -else { - & dotnet run --project $PSScriptRoot/update-dependencies/update-dependencies.csproj @updateDepsArgs -} diff --git a/eng/pipelines/pipelines/sync-internal-release.yml b/eng/pipelines/pipelines/sync-internal-release.yml index 6be3cc0b59..4e13589cde 100644 --- a/eng/pipelines/pipelines/sync-internal-release.yml +++ b/eng/pipelines/pipelines/sync-internal-release.yml @@ -56,15 +56,15 @@ extends: dotnet run --configuration Release --project ./eng/update-dependencies/update-dependencies.csproj -- sync-internal-release - --azdo-organization "$env:SYSTEM_COLLECTIONURI" - --azdo-project "$env:SYSTEM_TEAMPROJECT" - --azdo-repo "$env:BUILD_REPOSITORY_NAME" --source-branch "$sourceBranch" --target-branch "$targetBranch" - --pr-branch-prefix "pr" - --user "$env:DOTNETDOCKERBOT_USERNAME" - --email "$env:DOTNETDOCKERBOT_EMAIL" --staging-storage-account "dotnetstage" env: + UpdateDependencies__PullRequestDestination: AzureDevOps + UpdateDependencies__AzureDevOps__Organization: $(System.CollectionUri) + UpdateDependencies__AzureDevOps__Project: $(System.TeamProject) + UpdateDependencies__AzureDevOps__Repository: $(Build.Repository.Name) + UpdateDependencies__User: $(dotnetDockerBot.userName) + UpdateDependencies__Email: $(dotnetDockerBot.email) # Used to download build artifacts containing .NET version information. SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/pipelines/pipelines/update-dependencies-internal.yml b/eng/pipelines/pipelines/update-dependencies-internal.yml index 31be1fa870..6def40daa6 100644 --- a/eng/pipelines/pipelines/update-dependencies-internal.yml +++ b/eng/pipelines/pipelines/update-dependencies-internal.yml @@ -44,17 +44,18 @@ extends: scriptLocation: "inlineScript" inlineScript: >- dotnet run --project eng/update-dependencies/update-dependencies.csproj -- - from-staging-pipeline + dotnet staging-pipeline "${{ parameters.stageContainers }}" - --mode Remote - --azdo-organization "$(System.CollectionUri)" - --azdo-project "$(System.TeamProject)" - --azdo-repo "$(Build.Repository.Name)" + --submit-pr --internal --staging-storage-account ${{ parameters.stagingStorageAccount }} --target-branch "${{ parameters.targetBranch }}" - --user "$(dotnetDockerBot.userName)" - --email "$(dotnetDockerBot.email)" env: + UpdateDependencies__PullRequestDestination: AzureDevOps + UpdateDependencies__AzureDevOps__Organization: $(System.CollectionUri) + UpdateDependencies__AzureDevOps__Project: $(System.TeamProject) + UpdateDependencies__AzureDevOps__Repository: $(Build.Repository.Name) + UpdateDependencies__User: $(dotnetDockerBot.userName) + UpdateDependencies__Email: $(dotnetDockerBot.email) # Used to download build artifacts from .NET release staging pipeline SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/pipelines/pipelines/update-dependencies.yml b/eng/pipelines/pipelines/update-dependencies.yml index 18bc338b18..49e34c7201 100644 --- a/eng/pipelines/pipelines/update-dependencies.yml +++ b/eng/pipelines/pipelines/update-dependencies.yml @@ -26,10 +26,14 @@ parameters: - name: updateAspire type: boolean default: true -# Additional authentication arguments to pass to update-dependencies -- name: gitHubAuthArgs - type: string - default: "" +# Whether to submit a pull request with the updates instead of applying them locally +- name: submitPullRequest + type: boolean + default: false +# Configuration environment variables passed to update-dependencies +- name: updateEnvironment + type: object + default: {} extends: template: /eng/docker-tools/templates/1es.yml@self @@ -44,48 +48,73 @@ extends: parameters: dependencyName: dotnet updateSteps: + - ${{ if parameters.submitPullRequest }}: + - template: /eng/pipelines/steps/get-bar-token.yml@self - task: AzureCLI@2 displayName: Update .NET inputs: - azureSubscription: "Darc: Maestro Production" + ${{ if parameters.submitPullRequest }}: + azureSubscription: DotnetContainersGitHubBot + ${{ else }}: + azureSubscription: "Darc: Maestro Production" scriptType: "pscore" scriptLocation: "inlineScript" inlineScript: >- dotnet run --project eng/update-dependencies/update-dependencies.csproj -- - from-channel + dotnet channel ${{ parameters.vmrChannel }} - https://github.com/dotnet/dotnet - --version-source-name 'dotnet/dotnet' - ${{ parameters.gitHubAuthArgs }} + --submit-pr ${{ parameters.submitPullRequest }} + env: + ${{ insert }}: ${{ parameters.updateEnvironment }} + ${{ if parameters.submitPullRequest }}: + UpdateDependencies__BarToken: $(barAccessToken) - ${{ if parameters.updateAspire }}: - template: /eng/pipelines/jobs/update-dependency.yml@self parameters: dependencyName: aspire updateSteps: + - ${{ if parameters.submitPullRequest }}: + - template: /eng/pipelines/steps/get-bar-token.yml@self - task: AzureCLI@2 displayName: Update Aspire inputs: - azureSubscription: "Darc: Maestro Production" + ${{ if parameters.submitPullRequest }}: + azureSubscription: DotnetContainersGitHubBot + ${{ else }}: + azureSubscription: "Darc: Maestro Production" scriptType: "pscore" scriptLocation: "inlineScript" inlineScript: >- dotnet run --project eng/update-dependencies/update-dependencies.csproj -- - from-channel + aspire channel ${{ parameters.aspireChannel }} - https://github.com/microsoft/aspire - --version-source-name 'microsoft/aspire' - ${{ parameters.gitHubAuthArgs }} + --submit-pr ${{ parameters.submitPullRequest }} + env: + ${{ insert }}: ${{ parameters.updateEnvironment }} + ${{ if parameters.submitPullRequest }}: + UpdateDependencies__BarToken: $(barAccessToken) - ${{ each tool in parameters.tools }}: - template: /eng/pipelines/jobs/update-dependency.yml@self parameters: dependencyName: ${{ tool }} updateSteps: - - powershell: >- - dotnet run --project eng/update-dependencies/update-dependencies.csproj -- - specific 9.0 - --tool ${{ tool }} - --version-source-name ${{ tool }} - ${{ parameters.gitHubAuthArgs }} - displayName: Update ${{ tool }} + - ${{ if parameters.submitPullRequest }}: + - task: AzureCLI@2 + displayName: Update ${{ tool }} + inputs: + azureSubscription: DotnetContainersGitHubBot + scriptType: pscore + scriptLocation: inlineScript + inlineScript: >- + dotnet run --project eng/update-dependencies/update-dependencies.csproj -- + ${{ tool }} + --submit-pr + env: ${{ parameters.updateEnvironment }} + - ${{ else }}: + - powershell: >- + dotnet run --project eng/update-dependencies/update-dependencies.csproj -- + ${{ tool }} + displayName: Update ${{ tool }} + env: ${{ parameters.updateEnvironment }} diff --git a/eng/pipelines/steps/get-bar-token.yml b/eng/pipelines/steps/get-bar-token.yml new file mode 100644 index 0000000000..ff1c976e9d --- /dev/null +++ b/eng/pipelines/steps/get-bar-token.yml @@ -0,0 +1,19 @@ +steps: +- task: AzureCLI@2 + displayName: Authenticate to BAR + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + # Keep BAR authentication separate from the GitHub App's Key Vault signing identity. + $token = az account get-access-token ` + --resource 54c17f3d-7325-4eca-9db7-f090bfc765a8 ` + --query accessToken ` + --output tsv + + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($token)) { + throw "Failed to acquire a BAR access token." + } + + Write-Host "##vso[task.setvariable variable=barAccessToken;issecret=true]$token" diff --git a/eng/pipelines/steps/update-dependencies-specific.yml b/eng/pipelines/steps/update-dependencies-specific.yml deleted file mode 100644 index ce1fa52514..0000000000 --- a/eng/pipelines/steps/update-dependencies-specific.yml +++ /dev/null @@ -1,51 +0,0 @@ -parameters: - # The customArgsArray parameter is used to specify the configuration for multiple Dockerfile versions. - # This allows for a single branch to be generated for different internal .NET build versions. -- name: customArgsArray - type: string - default: "" -- name: useInternalBuild - type: boolean - default: false -# Token used for internal-build pushes when the internal path is enabled. -# Set the `internalAccessToken` YAML variable to override this if needed. -- name: internalAccessToken - type: string - default: '$(internalAccessToken)' -- name: skipPullRequest - type: boolean - default: false - -steps: -- script: docker build -t update-dependencies -f $(engPath)/update-dependencies/Dockerfile --pull . - displayName: Build Update Dependencies Tool -- script: docker run --name update-dependencies -d -t --entrypoint /bin/sh -v /var/run/docker.sock:/var/run/docker.sock update-dependencies - displayName: Create Update Dependencies container -- powershell: | - if ("${{ parameters.useInternalBuild }}" -eq "true") { - $pat='${{ parameters.internalAccessToken }}' - } else { - $pat="$(BotAccount-dotnet-docker-bot-PAT)" - } - - $credArgs = "--user $(dotnetDockerBot.userName) --email $(dotnetDockerBot.email) --password $pat" - - # Execute update-deps for each of the items in the customArgsArray, representing different Dockerfile versions - # Ensure that the value is treated as an array by wrapping it in an array literal. This deals with the quirk of - # how PowerShell treats a single item as a string instead of an array. - $customArgsArray = @('${{ parameters.customArgsArray }}' | ConvertFrom-Json) - foreach ($customArgs in $customArgsArray) { - # If this is the last iteration, include the credentials to cause a PR to be generated. - # For internal builds this will cause branch to be pushed to AzDO, but no PRs will be generated. - if ($customArgs -eq $customArgsArray[-1] -and "${{ parameters.skipPullRequest }}" -ne "true") { - $customArgs += " $credArgs" - } - - $command = "docker exec update-dependencies update-dependencies $customArgs" - Write-Host "Executing: $command" - Invoke-Expression $command - } - displayName: Run Update Dependencies -- script: docker rm -f update-dependencies - displayName: Remove Update Dependencies container - condition: always() diff --git a/eng/pipelines/steps/update-dotnet-dependencies-specific.yml b/eng/pipelines/steps/update-dotnet-dependencies-specific.yml deleted file mode 100644 index 86760241ab..0000000000 --- a/eng/pipelines/steps/update-dotnet-dependencies-specific.yml +++ /dev/null @@ -1,92 +0,0 @@ -parameters: - useInternalBuild: false - # Comma-delimited list of SDK versions to target (overrides the use of channel var to determine latest version) - sdkVersions: "" - buildId: "" - # Token used for AzdoVersionsRepoInfoAccessToken in the internal-build path. - # Set the `internalAccessToken` YAML variable to override this if needed. - internalAccessToken: '$(internalAccessToken)' - skipPullRequest: false - -steps: -- powershell: | - if ("${{ parameters.sdkVersions }}" -ne "") { - $args = @{ - SdkVersions = "${{ parameters.sdkVersions }}" -split "," - } - } - elseif ("${{ parameters.useInternalBuild }}" -eq "false") { - $args = @{ - Channel = "$(channel)" - } - } - else { - $args = @{ - BuildId = "${{ parameters.buildId }}" - } - } - - if ("${{ parameters.useInternalBuild }}" -eq "true") { - $args["UseInternalBuild"] = $true - $args["InternalAccessToken"] = '$(System.AccessToken)' - $args["AzdoVersionsRepoInfoAccessToken"] = '${{ parameters.internalAccessToken }}' - } - - $(engPath)/Get-DropVersions.ps1 @args - displayName: Get Versions -- powershell: | - Import-Module -force $(engPath)/DependencyManagement.psm1 - - $versionInfos = '$(versionInfos)' | ConvertFrom-Json - - $index=0 - foreach ($versionInfo in $versionInfos) { - $args = @{ - ProductVersion = $versionInfo.DockerfileVersion - RuntimeVersion = $versionInfo.RuntimeVersion - AspnetVersion = $versionInfo.AspnetVersion - SdkVersion = $versionInfo.SdkVersion - AzdoVariableName = "updateDepsArgs-$index" - UseStableBranding = $versionInfo.StableBranding - } - - if ("${{ parameters.useInternalBuild }}" -eq "true") { - $args["InternalAccessToken"] = '$(System.AccessToken)' - $args["InternalBaseUrl"] = '"$(internalBaseUrl)"' - } else { - $args["ReleaseState"] = $(Get-ProductReleaseState) - } - - Write-Host "Executing Set-DotnetVersions.ps1 for $($versionInfo.DockerfileVersion)" - $(engPath)/Set-DotnetVersions.ps1 @args - $index++ - } - displayName: Get update-dependencies args -- powershell: | - Import-Module -force $(engPath)/DependencyManagement.psm1 - - $targetBranch = $(Get-Branch) - if ("${{ parameters.useInternalBuild }}" -eq "true") { - $targetBranch = "staging-${{ parameters.buildId }}-pipeline-$(Build.BuildId)" - } - - $customArgsArray = @() - $index=0 - - # Grab the variables that were set by the multiple calls to Set-DotnetVersions.ps1 and - # add them to an array. This allows us to pass args for multiple Dockerfile versions. - while ([Environment]::GetEnvironmentVariable("UPDATEDEPSARGS-$index") -ne $null) { - $updateDepsArgs = [Environment]::GetEnvironmentVariable("UPDATEDEPSARGS-$index") - $updateDepsArgs = "$updateDepsArgs --org dnceng --project $(System.TeamProject) --repo $(Build.Repository.Name) --target-branch $targetBranch" - $customArgsArray += $updateDepsArgs - $index++ - } - - echo "##vso[task.setvariable variable=customArgsArray]$($customArgsArray | ConvertTo-Json -Compress -AsArray)" - displayName: Set Custom Args -- template: update-dependencies-specific.yml - parameters: - customArgsArray: "$(customArgsArray)" - skipPullRequest: ${{ parameters.skipPullRequest }} - useInternalBuild: ${{ parameters.useInternalBuild }} - internalAccessToken: ${{ parameters.internalAccessToken }} diff --git a/eng/pipelines/update-dependencies-monitor.yml b/eng/pipelines/update-dependencies-monitor.yml index ffb0c0de08..b867f5a479 100644 --- a/eng/pipelines/update-dependencies-monitor.yml +++ b/eng/pipelines/update-dependencies-monitor.yml @@ -23,7 +23,6 @@ parameters: variables: - template: /eng/docker-tools/templates/variables/dotnet/common.yml@self -- template: /eng/docker-tools/templates/variables/dotnet/secrets.yml@self stages: - stage: Monitor @@ -33,29 +32,28 @@ stages: pool: vmImage: $(defaultLinuxAmd64PoolImage) steps: - - download: dotnet-monitor - artifact: Build_Info - displayName: "Download Build Info (Branch: $(resources.pipeline.dotnet-monitor.sourceBranch))" - - pwsh: $(engPath)/Get-MonitorDropVersions.ps1 -BuildVersionFilePath "$(Pipeline.Workspace)/dotnet-monitor/Build_Info/dotnet-monitor.nupkg.buildversion" - displayName: Get Versions - - powershell: | - Import-Module -force $(engPath)/DependencyManagement.psm1 - - $scriptArgs = @{ - ProductVersion = "$(monitorMajorMinorVersion)" - MonitorVersion = "$(monitorVer)" - AzdoVariableName = 'updateDepsArgs' - UseStableBranding = [bool]::Parse("$(stableBranding)") - ReleaseState = $(Get-ProductReleaseState) - } - - $(engPath)/Set-DotnetVersions.ps1 @scriptArgs - displayName: Get update-dependencies args - - powershell: | - $customArgsArray = @("$(updateDepsArgs)") - echo "##vso[task.setvariable variable=customArgsArray]$($customArgsArray | ConvertTo-Json -Compress -AsArray)" - displayName: Set Custom Args - - template: steps/update-dependencies-specific.yml - parameters: - customArgsArray: $(customArgsArray) - skipPullRequest: ${{ parameters.skipPullRequest }} + - checkout: self + fetchDepth: 0 + - template: steps/install-dotnet.yml + - ${{ if not(parameters.skipPullRequest) }}: + - task: AzureCLI@2 + displayName: Update .NET Monitor + inputs: + azureSubscription: DotnetContainersGitHubBot + scriptType: pscore + scriptLocation: inlineScript + inlineScript: >- + dotnet run --project "$(engPath)/update-dependencies/update-dependencies.csproj" -- + monitor pipeline-build "$(resources.pipeline.dotnet-monitor.runID)" + --submit-pr + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + UpdateDependencies__GitHub__AppClientId: $(dotnetContainersBot.gitHubApp.clientId) + UpdateDependencies__GitHub__AppKeyUri: $(dotnetContainersBot.gitHubApp.privateKeyUri) + - ${{ else }}: + - pwsh: >- + dotnet run --project "$(engPath)/update-dependencies/update-dependencies.csproj" -- + monitor pipeline-build "$(resources.pipeline.dotnet-monitor.runID)" + displayName: Update .NET Monitor + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/pipelines/update-dependencies-official.yml b/eng/pipelines/update-dependencies-official.yml index 9800c65d5a..25ea328616 100644 --- a/eng/pipelines/update-dependencies-official.yml +++ b/eng/pipelines/update-dependencies-official.yml @@ -41,8 +41,6 @@ parameters: variables: - template: /eng/docker-tools/templates/variables/dotnet/common.yml@self -# Uses GitHub token to create pull requests -- template: /eng/docker-tools/templates/variables/dotnet/secrets.yml@self extends: template: /eng/pipelines/pipelines/update-dependencies.yml@self @@ -52,8 +50,8 @@ extends: updateAspire: ${{ parameters.updateAspire }} aspireChannel: $(aspireChannel) tools: ${{ parameters.tools }} + submitPullRequest: ${{ not(parameters.skipPullRequest) }} ${{ if ne(parameters.skipPullRequest, true) }}: - gitHubAuthArgs: >- - --user $(dotnetDockerBot.userName) - --email $(dotnetDockerBot.email) - --password $(BotAccount-dotnet-docker-bot-PAT) + updateEnvironment: + UpdateDependencies__GitHub__AppClientId: $(dotnetContainersBot.gitHubApp.clientId) + UpdateDependencies__GitHub__AppKeyUri: $(dotnetContainersBot.gitHubApp.privateKeyUri) diff --git a/eng/update-dependencies/AGENTS.md b/eng/update-dependencies/AGENTS.md new file mode 100644 index 0000000000..001840bc2e --- /dev/null +++ b/eng/update-dependencies/AGENTS.md @@ -0,0 +1,61 @@ +# Instructions for update-dependencies + +For usage, see [README.md](./README.md). + +## Design/Concepts + +- An [Updater](./Updaters) contains all details needed to update one specific + product. It implements `IUpdater` and one or more of the other `I.*Updater` + interfaces. +- [Commands](./Commands) map to the sources for dependency updates. These are + typically implemented as static `Command` factories that bind to command line + arguments. Commands know how to pass the correct values to the corresponding + `I.*Updater` interface. + - If the command fits within the standard updater/command split, it should be + registered in [DependencyCommands.cs](./Commands/DependencyCommand.cs). + The exception is if it's a one-off workflow type of command, like + [SyncInternalReleaseCommand](./Commands/SyncInternalReleaseCommand.cs). + +Example: + +- `FromBuildOptions`: declares the `System.CommandLine` options/arguments for + one specific BAR build ID. +- `IBarBuildUpdater`: declares a method that takes the necessary parameters to + update from a BAR build. +- `FromBuildCommand`: maps `FromBuildOptions` (command line options) to + `IBarBuildUpdater` method parameters. +- `DotnetUpdater`: implements `IBarBuildUpdater`. + +In this way, the tool separates **what to update** from **where versions come +from**. `DotnetUpdater` and `FromBuildCommand` do not need to know about each +other. Adding a new dependency from an existing update source would only +require implementing an `I.*Updater` interface. Implementing a new update +source only requires adding a new updater implementation if there is something +fundamentally different about how the update source maps to the existing +updater interfaces. + +## Command line + +Updaters and commands are automatically composed together in the CLI without any +extra work: `update-dependencies `. + +## Inputs/Configuration + +Inputs come as either [app configuration](./appsettings.json) (via +Microsoft.Extensions.Configuration) or from the command line. + +- Use configuration for things that do not change between runs or between + different commands: Azure DevOps org/project, GitHub repo, secrets, etc. +- Use command line args for inputs that vary between each run, like build IDs, + channels, branch targets, or versions. +- Basically, use command line args for anything that you might want to change + via pipeline parameters/variables without submitting a pull request. + +## Goals to preserve + +- Keep each dependency's update rules together so they do not diverge between + sources. +- Support only combinations that make sense for a dependency. Composition does + not mean every updater must support every command. +- Separate selecting an update from applying it so the same update can work in + a local checkout or a temporary workspace used to prepare a pull request. diff --git a/eng/update-dependencies/AspireBuildUpdaterService.cs b/eng/update-dependencies/AspireBuildUpdaterService.cs deleted file mode 100644 index 1c9219ba22..0000000000 --- a/eng/update-dependencies/AspireBuildUpdaterService.cs +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.DotNet.ProductConstructionService.Client.Models; -using Microsoft.Extensions.Logging; - -namespace Dotnet.Docker; - -internal class AspireBuildUpdaterService( - ILogger logger, - HttpClient httpClient, - Func manifestVariablesFactory -) : IBuildUpdaterService -{ - private readonly ILogger _logger = logger; - private readonly HttpClient _httpClient = httpClient; - private readonly Func _createManifestVariables = manifestVariablesFactory; - - /// - /// Given a BAR (https://aka.ms/bar) build of Aspire, updates the Aspire Dashboard build version. - /// - public async Task UpdateFrom(Build build, CreatePullRequestOptions pullRequestOptions) - { - if (build.GetBuildRepo() != BuildRepo.Aspire) - { - throw new InvalidOperationException( - $"AspireBuildUpdaterService cannot process builds from repo: {build.GitHubRepository}"); - } - - // Read manifest.versions.json early so that we can fail fast in case we have the wrong file path. - var manifestVariables = _createManifestVariables(pullRequestOptions.GetManifestVersionsFilePath()); - var branch = manifestVariables.GetValue("branch"); - var dashboardBaseUrl = manifestVariables.GetValue($"aspire-dashboard|base-url|{branch}"); - - var dashboardAssets = build.Assets.Where(asset => - asset.Name.Contains("aspire-dashboard-linux-x64") - || asset.Name.Contains("aspire-dashboard-linux-arm64")); - - if (!dashboardAssets.Any()) - { - throw new InvalidOperationException($"Could not find aspire-dashboard-linux-* assets in build {build.Id}"); - } - - _logger.LogInformation("Found Aspire build version: {version}", dashboardAssets.First().Version); - - var dashboardChecksums = await Task.WhenAll(dashboardAssets - .Select(asset => GetChecksumAsync(GetDashboardUrl(dashboardBaseUrl, asset)))); - - var dashboardChecksumInfos = dashboardAssets.Zip(dashboardChecksums); - - var manifestVersionsPath = pullRequestOptions.GetManifestVersionsFilePath(); - - var version = dashboardAssets.First().Version; - var majorMinorVersion = VersionHelper.ResolveMajorMinorVersion(version); - - // Known issue: Aspire Dashboard builds are always "preview 1". - // Passing in isStableRelease here keeps us from setting the product - // version to the full build version with the "-preview.1" suffix. - var productVersion = VersionHelper.ResolveProductVersion(version, isStableRelease: true); - - List variableUpdates = - [ - new VariableUpdateInfo("aspire-dashboard|build-version", version), - new VariableUpdateInfo("aspire-dashboard|product-version", productVersion), - new VariableUpdateInfo("aspire-dashboard|fixed-tag", productVersion), - new VariableUpdateInfo("aspire-dashboard|minor-tag", majorMinorVersion.ToString(2)), - new VariableUpdateInfo("aspire-dashboard|major-tag", majorMinorVersion.Major.ToString()), - ]; - - variableUpdates.AddRange(dashboardChecksumInfos - // Filter out null checksums (indicates the checksum was not found above) - .Where(info => info.Second is not null) - .Select(info => - { - var (asset, checksum) = info; - var arch = asset.Name.Contains("arm64") ? "arm64" : "x64"; - // Null-forgiving operator is OK since we filtered out null checksums above. - return new VariableUpdateInfo($"aspire-dashboard|linux|{arch}|sha", checksum!); - })); - - var dependencyUpdaters = variableUpdates - .Select(updateInfo => new VariableUpdater(manifestVersionsPath, updateInfo)); - - var updateDependencies = new SpecificCommand(); - updateDependencies.CustomUpdateInfos.AddRange(variableUpdates); - updateDependencies.CustomUpdaters.AddRange(dependencyUpdaters); - - // Don't pass in any through options, since we calculated all of the variables - // and their new versions to update above. Everything is handled through CustomUpdateInfos. - // Using the SpecificCommand here just allows us to easily create automated pull requests. - var updateDependenciesOptions = SpecificCommandOptions.FromPullRequestOptions(pullRequestOptions); - return await updateDependencies.ExecuteAsync(updateDependenciesOptions); - } - - /// - /// Gets the full download URL to an Aspire Dashboard asset given its base - /// URL and the BAR asset information. - /// - private string GetDashboardUrl(string baseUrl, Asset dashboardAsset) => $"{baseUrl}/{dashboardAsset.Name}"; - - /// - /// Manually compute the checksum of an Aspire Dashboard asset by downloading it from - /// and hashing the contents. - /// - /// - /// Remove once https://github.com/dotnet/dotnet-docker/issues/6568 is completed. - /// - /// - /// Null if the checksum could not be computed for any reason. An error will - /// be logged to the console if this happens. - /// - private Task GetChecksumAsync(string url) => ChecksumHelper.ComputeChecksumShaAsync(_httpClient, url); -} diff --git a/eng/update-dependencies/AzdoAuthProvider.cs b/eng/update-dependencies/AzdoAuthProvider.cs index 73f4932a6f..980f527050 100644 --- a/eng/update-dependencies/AzdoAuthProvider.cs +++ b/eng/update-dependencies/AzdoAuthProvider.cs @@ -1,13 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Azure.Identity; +using Maestro.Common.AzureDevOpsTokens; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.Services.Common; using Microsoft.VisualStudio.Services.WebApi; -namespace Dotnet.Docker; +namespace Microsoft.DotNet.Docker.UpdateDependencies; public interface IAzdoAuthProvider { @@ -22,32 +22,25 @@ public interface IAzdoAuthProvider VssConnection GetVssConnection(string azdoOrg); } -public class AzdoAuthProvider : IAzdoAuthProvider +public class AzdoAuthProvider( + IAzureDevOpsTokenProvider tokenProvider, + IConfiguration configuration) : IAzdoAuthProvider { /// - /// This scope provides access to Azure DevOps Services REST API. + /// Uses the pipeline token for artifact access, falling back to configured Azure DevOps authentication. /// - /// - /// See https://learn.microsoft.com/rest/api/azure/devops/tokens/?view=azure-devops-rest-7.1&tabs=powershell#personal-access-tokens-pats - /// _logger; - private readonly IEnvironmentService _environmentService; - private readonly Lazy _accessToken; - - public AzdoAuthProvider(ILogger logger, IEnvironmentService environmentService) + public string AccessToken { - _logger = logger; - _environmentService = environmentService; - _accessToken = new(GetAccessTokenInternal); + get + { + // Git operations use the token provider directly to retain the service connection identity. + string? pipelineToken = configuration["SYSTEM_ACCESSTOKEN"]; + return !string.IsNullOrWhiteSpace(pipelineToken) + ? pipelineToken + : tokenProvider.GetTokenForAccount("default"); + } } - /// - /// Gets an Azure DevOps REST API access token. - /// - public string AccessToken => _accessToken.Value; - /// /// Gets a connection to Azure DevOps Services. /// @@ -67,24 +60,6 @@ public VssConnection GetVssConnection(string azdoOrg) var connection = new VssConnection(baseUrl, credential); return connection; } - - private string GetAccessTokenInternal() - { - var accessToken = _environmentService.GetSystemAccessToken(); - if (!string.IsNullOrWhiteSpace(accessToken)) - { - return accessToken; - } - - _logger.LogWarning("Environment variable SYSTEM_ACCESSTOKEN was not set." - + " Did you forget to explicitly pass it in to your pipeline step?" - + " See https://learn.microsoft.com/azure/devops/pipelines/build/variables#systemaccesstoken"); - - var credential = new AzureDeveloperCliCredential(); - var requestContext = new Azure.Core.TokenRequestContext([Scope]); - accessToken = credential.GetToken(requestContext).Token; - return accessToken; - } } internal static class AzdoAuthProviderExtensions diff --git a/eng/update-dependencies/AzdoHttpClient.cs b/eng/update-dependencies/AzdoHttpClient.cs index 32fcf243d2..458a956a01 100644 --- a/eng/update-dependencies/AzdoHttpClient.cs +++ b/eng/update-dependencies/AzdoHttpClient.cs @@ -5,28 +5,22 @@ using System.Text; using Microsoft.Extensions.DependencyInjection; -namespace Dotnet.Docker; +namespace Microsoft.DotNet.Docker.UpdateDependencies; -internal class AzdoHttpClient +internal class AzdoHttpClient(IAzdoAuthProvider azdoAuthProvider, HttpClient httpClient) { - private readonly IAzdoAuthProvider _azdoAuthProvider; - private readonly HttpClient _httpClient; - - public AzdoHttpClient(IAzdoAuthProvider azdoAuthProvider, HttpClient httpClient) + public async Task GetAsync(string requestUri, CancellationToken ct = default) { - _azdoAuthProvider = azdoAuthProvider; - _httpClient = httpClient; - - _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + using var request = new HttpRequestMessage(HttpMethod.Get, requestUri); + request.Headers.Authorization = new AuthenticationHeaderValue( "Basic", Convert.ToBase64String( - Encoding.ASCII.GetBytes($":{_azdoAuthProvider.AccessToken}") + Encoding.ASCII.GetBytes($":{azdoAuthProvider.AccessToken}") ) ); - } - public async Task GetAsync(string requestUri, CancellationToken ct = default) => - await _httpClient.GetAsync(requestUri, ct); + return await httpClient.SendAsync(request, ct); + } } internal static class AzdoHttpClientExtensions diff --git a/eng/update-dependencies/BaseCommand.cs b/eng/update-dependencies/BaseCommand.cs deleted file mode 100644 index 402b3e7fc3..0000000000 --- a/eng/update-dependencies/BaseCommand.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.CommandLine; -using System.CommandLine.NamingConventionBinder; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; - -namespace Dotnet.Docker; - -public abstract class BaseCommand() : ICommand where TOptions : IOptions -{ - public abstract Task ExecuteAsync(TOptions options); - - public static Command Create(string name, string description) - { - var command = new Command(name, description); - - foreach (var option in TOptions.Options) - { - command.Add(option); - } - - foreach (var argument in TOptions.Arguments) - { - command.Add(argument); - } - - command.Action = Handler; - return command; - } - - private static BindingHandler Handler => - CommandHandler.Create(async (options, host) => - { - var thisCommand = host.Services.GetRequiredService>(); - return await thisCommand.ExecuteAsync(options); - }); -} diff --git a/eng/update-dependencies/BaseUrlUpdater.cs b/eng/update-dependencies/BaseUrlUpdater.cs deleted file mode 100644 index 1b0dc1b14e..0000000000 --- a/eng/update-dependencies/BaseUrlUpdater.cs +++ /dev/null @@ -1,109 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Data; -using System.Diagnostics; -using System.Linq; -using Microsoft.DotNet.VersionTools.Dependencies; -using Newtonsoft.Json.Linq; - -namespace Dotnet.Docker; - -/// -/// Updates the baseUrl variables in the manifest.versions.json file. -/// -internal class BaseUrlUpdater : FileRegexUpdater -{ - private const string BaseUrlGroupName = "BaseUrlValue"; - private readonly SpecificCommandOptions _options; - private readonly ManifestVariables _manifestVariables; - private readonly string _manifestVariableName; - - /// - /// Creates a new for updating base URLs. - /// If the base URL variable cannot be found in the manifest, the updater - /// won't do anything. - /// - public static IEnumerable CreateUpdaters(ManifestVariables manifestVariables, SpecificCommandOptions options) - { - if (manifestVariables is null) - { - Trace.TraceWarning("BaseUrlUpdater: manifest variables missing - skipping base URL update."); - return []; - } - - var upstreamBranch = manifestVariables.GetValue("branch"); - var baseUrlVarNames = ManifestHelper.GetBaseUrlVariableNames( - dockerfileVersion: options.DockerfileVersion, - branch: upstreamBranch, - versionSourceName: options.VersionSourceName, - sdkOnlyRelease: options.IsSdkOnly); - - IEnumerable updaters = baseUrlVarNames - .SelectMany(variable => CreateUpdater(variable, manifestVariables, options)); - - return updaters; - } - - private static IEnumerable CreateUpdater( - string baseUrlVarName, - ManifestVariables manifestVariables, - SpecificCommandOptions options) - { - var variableHasValue = manifestVariables.HasValue(baseUrlVarName); - - if (!variableHasValue) - { - Trace.TraceWarning($"BaseUrlUpdater: variable '{baseUrlVarName}' not found - skipping base URL update."); - return []; - } - - return [new BaseUrlUpdater(options, manifestVariables, baseUrlVarName)]; - } - - private BaseUrlUpdater( - SpecificCommandOptions options, - ManifestVariables manifestVariables, - string manifestVariableName) - { - Path = options.GetManifestVersionsFilePath(); - VersionGroupName = BaseUrlGroupName; - _options = options; - _manifestVariables = manifestVariables; - _manifestVariableName = manifestVariableName; - - Regex = ManifestHelper.GetManifestVariableRegex(_manifestVariableName, $"(?<{BaseUrlGroupName}>.+)"); - } - - protected override string TryGetDesiredValue(IEnumerable dependencyInfos, out IEnumerable usedDependencyInfos) - { - usedDependencyInfos = Enumerable.Empty(); - - string baseUrlVersionVarName = _manifestVariableName; - string unresolvedBaseUrl = _manifestVariables.Variables[baseUrlVersionVarName]?.ToString() ?? - throw new InvalidOperationException($"Variable with name '{baseUrlVersionVarName}' is missing."); - - if (_options.IsInternal) - { - if (string.IsNullOrEmpty(_options.InternalBaseUrl)) - { - throw new InvalidOperationException("InternalBaseUrl must be set in order to update base url for internal builds"); - } - - unresolvedBaseUrl = _options.InternalBaseUrl; - } - else if (_options.ReleaseState.HasValue) - { - unresolvedBaseUrl = $"$({ManifestHelper.GetBaseUrlVariableName(_options.ReleaseState.Value, _options.TargetBranch)})"; - } - else - { - // Modifying the URL from internal to public is not suppported because it's not possible to know - // what common variable it was originally referencing when it was last public. - } - - return unresolvedBaseUrl; - } -} diff --git a/eng/update-dependencies/BuildAssetService.cs b/eng/update-dependencies/BuildAssetService.cs index 98b132ae87..f250913443 100644 --- a/eng/update-dependencies/BuildAssetService.cs +++ b/eng/update-dependencies/BuildAssetService.cs @@ -1,23 +1,17 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.Linq; -using System.Net.Http; -using System.Threading.Tasks; using Microsoft.DotNet.ProductConstructionService.Client.Models; using Microsoft.Extensions.Logging; -namespace Dotnet.Docker; +namespace Microsoft.DotNet.Docker.UpdateDependencies; internal interface IBuildAssetService { Task GetAssetTextContentsAsync(Asset asset); } -internal class BuildAssetService( - HttpClient httpClient, - ILogger logger) +internal class BuildAssetService(HttpClient httpClient, ILogger logger) : IBuildAssetService { private readonly HttpClient _httpClient = httpClient; diff --git a/eng/update-dependencies/BuildExtensions.cs b/eng/update-dependencies/BuildExtensions.cs deleted file mode 100644 index ee9315d9e8..0000000000 --- a/eng/update-dependencies/BuildExtensions.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.DotNet.ProductConstructionService.Client.Models; - -namespace Dotnet.Docker; - -/// -/// Extensions for .NET build asset registry s. -/// -internal static class BuildExtensions -{ - /// - /// Given a , maps its source repository (either GitHub - /// or Azure Devops) to a supported enum value. - /// - public static BuildRepo GetBuildRepo(this Build build) - { - string repo = build.GitHubRepository ?? build.AzureDevOpsRepository; - return repo switch - { - "https://github.com/dotnet/dotnet" or "https://dev.azure.com/dnceng/internal/_git/dotnet-dotnet" => BuildRepo.Vmr, - "https://github.com/microsoft/aspire" or "https://dev.azure.com/dnceng/internal/_git/microsoft-aspire" => BuildRepo.Aspire, - _ => throw new InvalidOperationException($"Build {build.Id} was from unsupported repository '{repo}'"), - }; - } -} diff --git a/eng/update-dependencies/BuildLabelService.cs b/eng/update-dependencies/BuildLabelService.cs index 1ce40c1710..47f11bed62 100644 --- a/eng/update-dependencies/BuildLabelService.cs +++ b/eng/update-dependencies/BuildLabelService.cs @@ -4,7 +4,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -namespace Dotnet.Docker; +namespace Microsoft.DotNet.Docker.UpdateDependencies; /// /// Service for adding build tags to Azure Pipelines runs. diff --git a/eng/update-dependencies/BuildRepo.cs b/eng/update-dependencies/BuildRepo.cs deleted file mode 100644 index cd6841c187..0000000000 --- a/eng/update-dependencies/BuildRepo.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace Dotnet.Docker; - -/// -/// Represents supported source repositories for builds from the .NET build -/// asset registry. -/// -internal enum BuildRepo -{ - Vmr, - Aspire, - // When adding new repos, also update "BuildExtensions.GetBuildRepo()" -} diff --git a/eng/update-dependencies/ChangeMode.cs b/eng/update-dependencies/ChangeMode.cs deleted file mode 100644 index 5430e83520..0000000000 --- a/eng/update-dependencies/ChangeMode.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace Dotnet.Docker; - -internal enum ChangeMode -{ - /// - /// Run the command locally, making changes directly to the local repo. - /// Do not run any git operations. - /// - Local, - - /// - /// The local repo is not modified. The command clones a remote repo into - /// a local directory and makes changes there. A pull request is submitted - /// with the changes. - /// - Remote, -} diff --git a/eng/update-dependencies/ChecksumHelper.cs b/eng/update-dependencies/ChecksumHelper.cs index 7ad953c154..51c9c1a5ea 100644 --- a/eng/update-dependencies/ChecksumHelper.cs +++ b/eng/update-dependencies/ChecksumHelper.cs @@ -1,23 +1,23 @@ using System.Diagnostics; -using System.IO; -using System.Net.Http; using System.Security.Cryptography; using System.Text; -using System.Threading.Tasks; -namespace Dotnet.Docker; +namespace Microsoft.DotNet.Docker.UpdateDependencies; internal static class ChecksumHelper { - public static async Task ComputeChecksumShaAsync(HttpClient httpClient, string downloadUrl) + public static async Task ComputeChecksumShaAsync( + HttpClient httpClient, + string downloadUrl, + CancellationToken cancellationToken = default) { string? sha = null; - using (HttpResponseMessage response = await httpClient.GetAsync(downloadUrl)) + using (HttpResponseMessage response = await httpClient.GetAsync(downloadUrl, cancellationToken)) { if (response.IsSuccessStatusCode) { - using (Stream httpStream = await response.Content.ReadAsStreamAsync()) + using (Stream httpStream = await response.Content.ReadAsStreamAsync(cancellationToken)) using (SHA512 hash = SHA512.Create()) { byte[] hashedInputBytes = hash.ComputeHash(httpStream); diff --git a/eng/update-dependencies/ChiselUpdater.cs b/eng/update-dependencies/ChiselUpdater.cs deleted file mode 100644 index 204b134d7f..0000000000 --- a/eng/update-dependencies/ChiselUpdater.cs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Net.Http; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.DotNet.VersionTools.Dependencies; - -namespace Dotnet.Docker; - -internal static class ChiselUpdater -{ - public const string ToolName = Repo; - - private const string Owner = "canonical"; - - private const string Repo = "chisel"; - - private static readonly string[] s_supportedArchitectures = ["amd64", "arm", "arm64"]; - - public static IEnumerable GetUpdaters(string manifestVersionsFilePath) => - s_supportedArchitectures - .SelectMany(arch => - [ - new GitHubReleaseUrlUpdater( - manifestVersionsFilePath: manifestVersionsFilePath, - toolName: ToolName, - variableName: GetChiselManifestVariable(ToolName, arch, "url", "latest"), - owner: Owner, - repo: Repo, - assetRegex: GetAssetRegex(arch)), - new ChiselReleaseShaUpdater( - manifestVersionsFilePath, - arch), - ]); - - public static async Task GetBuildInfoAsync() => - new GitHubReleaseInfo( - SimpleName: ToolName, - Release: await GitHubHelper.GetLatestRelease(Owner, Repo)); - - public static string GetChiselManifestVariable(string product, string arch, string type, string dockerfileVersion = "latest") - { - // Workaround for ambiguous method call, will be fixed with https://github.com/dotnet/csharplang/issues/8374 - return string.Join('|', new string[] { product, dockerfileVersion, ToManifestArch(arch), type }); - } - - private static Regex GetAssetRegex(string arch) => new(@"chisel_v\d+\.\d+\.\d+_linux_" + arch + @"\.tar\.gz"); - - private static string ToManifestArch(string arch) => arch == "amd64" ? "x64" : arch; - - private class ChiselReleaseShaUpdater( - string manifestVersionsFilePath, - string arch) - : GitHubReleaseUrlUpdater( - manifestVersionsFilePath, - ChiselUpdater.ToolName, - GetChiselManifestVariable("chisel", arch, ShaFunction, "latest"), - ChiselUpdater.Owner, - ChiselUpdater.Repo, - GetAssetRegex(arch)) - { - private const string ShaFunction = "sha384"; - - private static readonly HttpClient s_httpClient = new(); - - protected override string? GetValue(GitHubReleaseInfo dependencyInfo) - { - string? downloadUrl = base.GetValue(dependencyInfo); - if (downloadUrl is null) - { - return null; - } - - downloadUrl = $"{downloadUrl}.{ShaFunction}"; - return GetChecksumFromUrlAsync(downloadUrl).Result; - } - - private static async Task GetChecksumFromUrlAsync(string downloadUrl) - { - using HttpResponseMessage response = await s_httpClient.GetAsync(downloadUrl); - if (!response.IsSuccessStatusCode) - { - Trace.TraceInformation($"Failed to download {downloadUrl}."); - return null; - } - - // Expected format: - // abcdef1234567890 chisel_v1.0.0_linux_amd64.tar.gz - string content = await response.Content.ReadAsStringAsync(); - string sha = content.Split(" ")[0]; - return sha.ToLowerInvariant(); - } - } -} diff --git a/eng/update-dependencies/ChiselVersionSource.cs b/eng/update-dependencies/ChiselVersionSource.cs deleted file mode 100644 index 5070a5d08a..0000000000 --- a/eng/update-dependencies/ChiselVersionSource.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Threading.Tasks; - -namespace Dotnet.Docker; - -internal sealed class ChiselVersionSource : IDependencyVersionSource -{ - public async Task GetVersionInfoAsync(ComponentVersionChannel channel) - { - GitHubReleaseInfo chiselReleaseInfo = await ChiselUpdater.GetBuildInfoAsync(); - return new ComponentVersionInfo(chiselReleaseInfo.SimpleVersion); - } -} diff --git a/eng/update-dependencies/Commands/CreatePullRequestOptions.cs b/eng/update-dependencies/Commands/CreatePullRequestOptions.cs new file mode 100644 index 0000000000..1f42c5cc25 --- /dev/null +++ b/eng/update-dependencies/Commands/CreatePullRequestOptions.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.CommandLine; + +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; + +public record CreatePullRequestOptions +{ + public string RepoRoot { get; set; } = Directory.GetCurrentDirectory(); + public string SourceBranch { get; set; } = ""; + public string TargetBranch { get; set; } = "nightly"; + public bool SubmitPullRequest { get; set; } + + private static readonly Option s_repoRoot = new("--repo-root") + { + Description = "The root of the dotnet-docker repo to run against (defaults to current working directory)", + DefaultValueFactory = _ => Directory.GetCurrentDirectory(), + }; + private static readonly Option s_submitPullRequest = new("--submit-pr") + { + Description = "Apply updates in a temporary clone and submit a pull request with them." + + " By default, updates are applied to the local repo and no git operations are run.", + }; + private static readonly Option s_sourceBranch = new("--source-branch") + { + Description = "Branch to pull updates from", + DefaultValueFactory = _ => "", + }; + private static readonly Option s_targetBranch = new("--target-branch") + { + Description = "Pull request will be submitted targeting this branch", + DefaultValueFactory = _ => "nightly", + }; + + public static void AddTo(Command command) + { + command.Options.Add(s_repoRoot); + command.Options.Add(s_submitPullRequest); + command.Options.Add(s_sourceBranch); + command.Options.Add(s_targetBranch); + } + + public static CreatePullRequestOptions Bind(ParseResult result) => + Bind(result, new CreatePullRequestOptions()); + + protected static TOptions Bind(ParseResult result, TOptions options) + where TOptions : CreatePullRequestOptions + { + options.RepoRoot = result.GetRequiredValue(s_repoRoot); + options.SubmitPullRequest = result.GetValue(s_submitPullRequest); + options.SourceBranch = result.GetRequiredValue(s_sourceBranch); + options.TargetBranch = result.GetRequiredValue(s_targetBranch); + return options; + } + + public string CreatePullRequestBranchName(string name, string buildId = "") + { + var buildIdSuffix = string.IsNullOrWhiteSpace(buildId) ? string.Empty : $"-{buildId}"; + var sanitizedTargetBranch = TargetBranch.Replace('/', '-'); + return $"{sanitizedTargetBranch}/{name}{buildIdSuffix}"; + } +} diff --git a/eng/update-dependencies/Commands/DependencyCommand.cs b/eng/update-dependencies/Commands/DependencyCommand.cs new file mode 100644 index 0000000000..b634228ef7 --- /dev/null +++ b/eng/update-dependencies/Commands/DependencyCommand.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.CommandLine; +using Microsoft.DotNet.Docker.UpdateDependencies.Updaters; + +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; + +internal static class DependencyCommand +{ + public static Command CreateCliCommand(IServiceProvider services) where TUpdater : class, IUpdater + { + var command = new Command(TUpdater.Name, $"Update {TUpdater.Name}"); + + if (typeof(IBarBuildUpdater).IsAssignableFrom(typeof(TUpdater))) + command.Subcommands.Add(FromBuildCommand.CreateCliCommand(services)); + + if (typeof(IBarChannelUpdater).IsAssignableFrom(typeof(TUpdater))) + command.Subcommands.Add(FromChannelCommand.CreateCliCommand(services)); + + if (typeof(IPipelineBuildUpdater).IsAssignableFrom(typeof(TUpdater))) + command.Subcommands.Add(FromPipelineBuildCommand.CreateCliCommand(services)); + + if (typeof(IStagingPipelineUpdater).IsAssignableFrom(typeof(TUpdater))) + command.Subcommands.Add(FromStagingPipelineCommand.CreateCliCommand(services)); + + if (typeof(IVersionUpdater).IsAssignableFrom(typeof(TUpdater))) + command.Subcommands.Add(FromVersionCommand.CreateCliCommand(services)); + + if (typeof(IGitHubReleaseUpdater).IsAssignableFrom(typeof(TUpdater))) + FromGitHubReleaseCommand.Configure(command, services); + + return command; + } +} diff --git a/eng/update-dependencies/Commands/FromBuildCommand.cs b/eng/update-dependencies/Commands/FromBuildCommand.cs new file mode 100644 index 0000000000..17cc4e3929 --- /dev/null +++ b/eng/update-dependencies/Commands/FromBuildCommand.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.CommandLine; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Docker.UpdateDependencies.Updaters; +using Microsoft.DotNet.ProductConstructionService.Client.Models; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; + +internal static class FromBuildCommand +{ + public static Command CreateCliCommand(IServiceProvider services) where TUpdater : class, IUpdater + { + var command = new Command("build-id", "Update from a specific BAR build"); + FromBuildOptions.AddTo(command); + + command.SetAction(async (result, cancellationToken) => + { + FromBuildOptions options = FromBuildOptions.Bind(result); + var barClient = services.GetRequiredService(); + var updater = (IBarBuildUpdater)services.GetRequiredService(); + var runner = services.GetRequiredService(); + Build build = await barClient.GetBuildAsync(options.Id).WaitAsync(cancellationToken); + DependencyUpdate update = await updater.ResolveFromBarBuildAsync(build, cancellationToken); + + await runner.RunAsync(options, TUpdater.VersionSourceName, update, cancellationToken); + }); + + return command; + } +} diff --git a/eng/update-dependencies/Commands/FromBuildOptions.cs b/eng/update-dependencies/Commands/FromBuildOptions.cs new file mode 100644 index 0000000000..660a36ab16 --- /dev/null +++ b/eng/update-dependencies/Commands/FromBuildOptions.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.CommandLine; + +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; + +internal record FromBuildOptions : CreatePullRequestOptions +{ + public required int Id { get; init; } + + private static readonly PositiveIdArgument s_id = new("The BAR build ID to use as the update source"); + + public static new void AddTo(Command command) + { + CreatePullRequestOptions.AddTo(command); + command.Arguments.Add(s_id); + } + + public static new FromBuildOptions Bind(ParseResult result) => + Bind(result, new FromBuildOptions { Id = result.GetValue(s_id) }); +} diff --git a/eng/update-dependencies/Commands/FromChannelCommand.cs b/eng/update-dependencies/Commands/FromChannelCommand.cs new file mode 100644 index 0000000000..0a650e4bdd --- /dev/null +++ b/eng/update-dependencies/Commands/FromChannelCommand.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.CommandLine; +using Microsoft.DotNet.Docker.UpdateDependencies.Updaters; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; + +internal static class FromChannelCommand +{ + public static Command CreateCliCommand(IServiceProvider services) where TUpdater : class, IUpdater + { + var command = new Command("channel", "Update from the latest build in a BAR channel"); + FromChannelOptions.AddTo(command); + + command.SetAction(async (result, cancellationToken) => + { + FromChannelOptions options = FromChannelOptions.Bind(result); + var updater = (IBarChannelUpdater)services.GetRequiredService(); + var runner = services.GetRequiredService(); + DependencyUpdate update = await updater.ResolveFromBarChannelAsync(options.Channel, cancellationToken); + + await runner.RunAsync(options, TUpdater.VersionSourceName, update, cancellationToken); + }); + + return command; + } +} diff --git a/eng/update-dependencies/Commands/FromChannelOptions.cs b/eng/update-dependencies/Commands/FromChannelOptions.cs new file mode 100644 index 0000000000..34b4f540cb --- /dev/null +++ b/eng/update-dependencies/Commands/FromChannelOptions.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.CommandLine; + +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; + +internal record FromChannelOptions : CreatePullRequestOptions +{ + public required int Channel { get; init; } + + private static readonly PositiveIdArgument s_channel = new("The BAR channel ID to use as the update source"); + + public static new void AddTo(Command command) + { + CreatePullRequestOptions.AddTo(command); + command.Arguments.Add(s_channel); + } + + public static new FromChannelOptions Bind(ParseResult result) => + Bind(result, new FromChannelOptions { Channel = result.GetValue(s_channel) }); +} diff --git a/eng/update-dependencies/Commands/FromGitHubReleaseCommand.cs b/eng/update-dependencies/Commands/FromGitHubReleaseCommand.cs new file mode 100644 index 0000000000..ce6abbbbc8 --- /dev/null +++ b/eng/update-dependencies/Commands/FromGitHubReleaseCommand.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.CommandLine; +using Microsoft.DotNet.Docker.UpdateDependencies.Updaters; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; + +internal static class FromGitHubReleaseCommand +{ + public static void Configure(Command command, IServiceProvider services) + where TUpdater : class, IUpdater + { + CreatePullRequestOptions.AddTo(command); + + command.SetAction(async (result, cancellationToken) => + { + CreatePullRequestOptions options = CreatePullRequestOptions.Bind(result); + var updater = (IGitHubReleaseUpdater)services.GetRequiredService(); + var runner = services.GetRequiredService(); + DependencyUpdate update = await updater.ResolveFromGitHubReleaseAsync(cancellationToken); + + await runner.RunAsync(options, TUpdater.VersionSourceName, update, cancellationToken); + }); + } +} diff --git a/eng/update-dependencies/Commands/FromPipelineBuildCommand.cs b/eng/update-dependencies/Commands/FromPipelineBuildCommand.cs new file mode 100644 index 0000000000..3bfdaf984b --- /dev/null +++ b/eng/update-dependencies/Commands/FromPipelineBuildCommand.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.CommandLine; +using Microsoft.DotNet.Docker.UpdateDependencies.Updaters; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; + +internal static class FromPipelineBuildCommand +{ + public static Command CreateCliCommand(IServiceProvider services) where TUpdater : class, IUpdater + { + var command = new Command("pipeline-build", "Update from an Azure DevOps pipeline run"); + FromPipelineBuildOptions.AddTo(command); + + command.SetAction(async (result, cancellationToken) => + { + FromPipelineBuildOptions options = FromPipelineBuildOptions.Bind(result); + var updater = (IPipelineBuildUpdater)services.GetRequiredService(); + var runner = services.GetRequiredService(); + + DependencyUpdate update = await updater.ResolveFromPipelineBuildAsync( + options.RunId, + cancellationToken); + + await runner.RunAsync(options, TUpdater.VersionSourceName, update, cancellationToken); + }); + + return command; + } +} diff --git a/eng/update-dependencies/Commands/FromPipelineBuildOptions.cs b/eng/update-dependencies/Commands/FromPipelineBuildOptions.cs new file mode 100644 index 0000000000..f5e754c449 --- /dev/null +++ b/eng/update-dependencies/Commands/FromPipelineBuildOptions.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.CommandLine; + +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; + +internal record FromPipelineBuildOptions : CreatePullRequestOptions +{ + public required int RunId { get; init; } + + private static readonly PositiveIdArgument s_runId = new("The Azure DevOps pipeline run ID to use as the update source"); + + public static new void AddTo(Command command) + { + CreatePullRequestOptions.AddTo(command); + command.Arguments.Add(s_runId); + } + + public static new FromPipelineBuildOptions Bind(ParseResult result) => + Bind(result, new FromPipelineBuildOptions { RunId = result.GetValue(s_runId) }); +} diff --git a/eng/update-dependencies/FromStagingPipelineCommand.cs b/eng/update-dependencies/Commands/FromStagingPipelineCommand.cs similarity index 70% rename from eng/update-dependencies/FromStagingPipelineCommand.cs rename to eng/update-dependencies/Commands/FromStagingPipelineCommand.cs index 28c3e9e60e..093a3fa345 100644 --- a/eng/update-dependencies/FromStagingPipelineCommand.cs +++ b/eng/update-dependencies/Commands/FromStagingPipelineCommand.cs @@ -1,14 +1,17 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Dotnet.Docker.Git; -using Dotnet.Docker.Sync; +using System.CommandLine; +using Microsoft.DotNet.Docker.UpdateDependencies.Git; +using Microsoft.DotNet.Docker.UpdateDependencies.Sync; +using Microsoft.DotNet.Docker.UpdateDependencies.Updaters; using Microsoft.DotNet.Docker.Shared; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -namespace Dotnet.Docker; +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; -internal partial class FromStagingPipelineCommand : BaseCommand +internal partial class FromStagingPipelineCommand : ICommand { /// /// Callback that stages all changes and commits them. @@ -21,15 +24,19 @@ internal partial class FromStagingPipelineCommand : BaseCommand _logger; + private readonly UpdateDependenciesConfiguration _configuration; private readonly IPipelineArtifactProvider _pipelineArtifactProvider; private readonly IPipelinesService _pipelinesService; private readonly IInternalVersionsService _internalVersionsService; + private readonly DotNetUpdater _updater; private readonly IEnvironmentService _environmentService; private readonly IBuildLabelService _buildLabelService; private readonly Func> _createGitRepoContextAsync; public FromStagingPipelineCommand( + UpdateDependenciesConfiguration configuration, ILogger logger, + DotNetUpdater updater, IPipelineArtifactProvider pipelineArtifactProvider, IPipelinesService pipelinesService, IInternalVersionsService internalVersionsService, @@ -37,16 +44,35 @@ public FromStagingPipelineCommand( IBuildLabelService buildLabelService, IGitRepoHelperFactory gitRepoHelperFactory) { + _configuration = configuration; _logger = logger; + _updater = updater; _pipelineArtifactProvider = pipelineArtifactProvider; _pipelinesService = pipelinesService; _internalVersionsService = internalVersionsService; _environmentService = environmentService; _buildLabelService = buildLabelService; - _createGitRepoContextAsync = options => GitRepoContext.CreateAsync(_logger, gitRepoHelperFactory, options, _environmentService); + _createGitRepoContextAsync = options => GitRepoContext.CreateAsync( + _logger, + gitRepoHelperFactory, + options, + _environmentService, + configuration); } - public override async Task ExecuteAsync(FromStagingPipelineOptions options) + public static Command CreateCliCommand(IServiceProvider services) + { + var command = new Command("staging-pipeline", "Update .NET from staging pipeline runs"); + FromStagingPipelineOptions.AddTo(command); + command.SetAction((result, _) => + { + FromStagingPipelineOptions options = FromStagingPipelineOptions.Bind(result); + return services.GetRequiredService>().ExecuteAsync(options); + }); + return command; + } + + public async Task ExecuteAsync(FromStagingPipelineOptions options) { var stageContainers = options.GetStageContainerList(); @@ -75,16 +101,11 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) { _logger.LogInformation("Processing stage container: {StageContainer}", stageContainer); - var (commitMessage, prBodySection, exitCode) = await ProcessStageContainerAsync( + var (commitMessage, prBodySection) = await ProcessStageContainerAsync( options, stageContainer, gitRepoContext); - if (exitCode != 0) - { - return exitCode; - } - // Commit changes for this stage container await gitRepoContext.CommitChanges(commitMessage); @@ -107,9 +128,9 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) /// Processes a single stage container and applies the updates. /// /// - /// A tuple containing the commit message, PR body section, and exit code. + /// A tuple containing the commit message and PR body section. /// - private async Task<(string CommitMessage, string PrBodySection, int ExitCode)> ProcessStageContainerAsync( + private async Task<(string CommitMessage, string PrBodySection)> ProcessStageContainerAsync( FromStagingPipelineOptions options, string stageContainer, GitRepoContext gitRepoContext) @@ -118,8 +139,8 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) // Log staging pipeline tags for diagnostic purposes var stagingPipelineTags = await _pipelinesService.GetBuildTagsAsync( - options.AzdoOrganization, - options.AzdoProject, + _configuration.AzureDevOps.Organization, + _configuration.AzureDevOps.Project, stagingPipelineRunId); _logger.LogInformation("Staging pipeline tags: {Tags}", string.Join(", ", stagingPipelineTags)); @@ -140,8 +161,8 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) } var releaseConfig = await _pipelineArtifactProvider.GetReleaseConfigAsync( - options.AzdoOrganization, - options.AzdoProject, + _configuration.AzureDevOps.Organization, + _configuration.AzureDevOps.Project, stagingPipelineRunId); string dotnetProductVersion = VersionHelper.ResolveProductVersion(releaseConfig.RuntimeBuild); @@ -157,83 +178,37 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) stageContainer: stageContainer); } - var productVersions = (options.Internal, releaseConfig.SdkOnly) switch - { - // SDK-only internal/staging release - (true, true) => new Dictionary - { - // SDK-only releases are almost always one-off updates/bug - // fixes on top of an existing release of the Runtime and - // ASP.NET Core. - // - // If the release config tells us that this is an - // SDK-only release, we can assume that we have already - // released the runtime/aspnet versions that it's based on, and - // therefore we shouldn't update them unnecessarily. - { "sdk", VersionHelper.GetHighestSdkVersion(releaseConfig.SdkBuilds) } - }, - // Internal/staging release - (true, false) => new Dictionary - { - { "dotnet", dotnetProductVersion }, - { "runtime", releaseConfig.RuntimeBuild }, - { "aspnet", releaseConfig.AspBuild }, - { "aspnet-composite", releaseConfig.AspBuild }, - { "sdk", VersionHelper.GetHighestSdkVersion(releaseConfig.SdkBuilds) }, - }, - // Public release - whether or not it's an SDK-only release doesn't - // matter because the product versions will end up being the same. - (false, _) => new Dictionary - { - { "dotnet", dotnetProductVersion }, - { "runtime", releaseConfig.Runtime }, - { "aspnet", releaseConfig.Asp }, - { "aspnet-composite", releaseConfig.Asp }, - { "sdk", VersionHelper.GetHighestSdkVersion(releaseConfig.Sdks) }, - } - }; - - _logger.LogInformation( - "Resolved product versions: {productVersions}", - string.Join(", ", productVersions.Select(kv => $"{kv.Key}: {kv.Value}"))); + string sdkVersion = VersionHelper.GetHighestSdkVersion(options.Internal ? releaseConfig.SdkBuilds : releaseConfig.Sdks); + string runtimeVersion = options.Internal ? releaseConfig.RuntimeBuild : releaseConfig.Runtime; + string? aspnetVersion = options.Internal ? releaseConfig.AspBuild : releaseConfig.Asp; // Example build URL: https://dev.azure.com///_build/results?buildId= - var buildUrl = $"{options.AzdoOrganization}/{options.AzdoProject}/_build/results?buildId={stagingPipelineRunId}"; + var buildUrl = $"{_configuration.AzureDevOps.Organization.TrimEnd('/')}/{_configuration.AzureDevOps.Project}/_build/results?buildId={stagingPipelineRunId}"; _logger.LogInformation( "Applying internal build {StageContainer} ({BuildUrl})", stageContainer, buildUrl); - _logger.LogInformation( - "Ignore any git-related logging output below, because git " - + "operations are being managed by a different command."); - - // Run old update-dependencies command using the resolved versions. - // Do not use the old command to submit a pull request. - var updateDependencies = new SpecificCommand(); - var updateDependenciesOptions = new SpecificCommandOptions() - { - RepoRoot = gitRepoContext.LocalRepoPath, - DockerfileVersion = majorMinorVersionString, - ProductVersions = productVersions, - InternalBaseUrl = internalBaseUrl, - }; - var exitCode = await updateDependencies.ExecuteAsync(updateDependenciesOptions); - if (exitCode != 0) - { - _logger.LogError( - "Failed to apply stage container {StageContainer}. " - + "Command exited with code {ExitCode}.", - stageContainer, exitCode); - return (string.Empty, string.Empty, exitCode); - } + await DependencyUpdateRunner.ApplyAsync( + gitRepoContext.LocalRepoPath, + (variables, repoRoot, token) => _updater.UpdateFromStagingPipelineAsync( + variables, repoRoot, releaseConfig, internalBaseUrl, token), + CancellationToken.None); var commitMessage = releaseConfig switch { - { SdkOnly: true } => $"Update .NET {majorMinorVersionString} SDK to {productVersions["sdk"]}", - _ => $"Update .NET {majorMinorVersionString} to {productVersions["sdk"]} SDK / {productVersions["runtime"]} Runtime", + { SdkOnly: true } => $"Update .NET {majorMinorVersionString} SDK to {sdkVersion}", + _ => $"Update .NET {majorMinorVersionString} to {sdkVersion} SDK / {runtimeVersion} Runtime", }; - var newVersionsList = productVersions.Select(kvp => $"- {kvp.Key.ToUpper()}: {kvp.Value}"); + List newVersionsList = []; + if (!options.Internal || !releaseConfig.SdkOnly) + { + newVersionsList.Add($"- DOTNET: {dotnetProductVersion}"); + newVersionsList.Add($"- RUNTIME: {runtimeVersion}"); + newVersionsList.Add($"- ASPNET: {aspnetVersion}"); + newVersionsList.Add($"- ASPNET-COMPOSITE: {aspnetVersion}"); + } + newVersionsList.Add($"- SDK: {sdkVersion}"); var prBodySection = $""" ## .NET {majorMinorVersionString} @@ -244,7 +219,7 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) These versions are from .NET staging pipeline run [{stageContainer}]({buildUrl}). """; - return (commitMessage, prBodySection, 0); + return (commitMessage, prBodySection); } /// @@ -289,22 +264,23 @@ private record GitRepoContext( /// then use to push all commits and create a pull request. /// /// - /// If is , + /// If is false, /// no git operations will be performed. /// public static async Task CreateAsync( ILogger logger, IGitRepoHelperFactory gitRepoFactory, FromStagingPipelineOptions options, - IEnvironmentService environmentService) + IEnvironmentService environmentService, + UpdateDependenciesConfiguration configuration) { CommitChanges commitChanges; PushAndCreatePullRequest pushAndCreatePullRequest; string localRepoPath; - if (options.Mode == ChangeMode.Remote) + if (options.SubmitPullRequest) { - var remoteUrl = options.GetAzdoRepoUrl(); + var remoteUrl = configuration.AzureDevOps.GetRepoUrl(); var targetBranch = options.TargetBranch; var buildId = environmentService.GetBuildId() ?? ""; var stageContainerList = options.GetStageContainerList(); @@ -312,8 +288,10 @@ public static async Task CreateAsync( { throw new ArgumentException("At least one stage container must be provided."); } - var prBranch = options.CreatePrBranchName($"update-deps-int-{stageContainerList[0]}", buildId); - var committer = options.GetCommitterIdentity(); + var prBranch = options.CreatePullRequestBranchName( + $"update-deps-int-{stageContainerList[0]}", + buildId); + var committer = configuration.GetCommitterIdentity(); // Clone the repo and configure git identity for commits var git = await gitRepoFactory.CreateAndCloneAsync(remoteUrl, gitIdentity: committer); @@ -344,18 +322,18 @@ await git.Remote.CreatePullRequestAsync(new( } else { - logger.LogInformation("No git operations will be performed in {Mode} mode.", options.Mode); + logger.LogInformation("No git operations will be performed."); localRepoPath = options.RepoRoot; commitChanges = async (commitMessage) => { - logger.LogInformation("Skipping commit in {Mode} mode.", options.Mode); + logger.LogInformation("Skipping commit."); logger.LogInformation("Commit message: {CommitMessage}", commitMessage); }; pushAndCreatePullRequest = async (prTitle, prBody) => { - logger.LogInformation("Skipping push and pull request creation in {Mode} mode.", options.Mode); + logger.LogInformation("Skipping push and pull request creation."); logger.LogInformation("Pull request title: {PullRequestTitle}", prTitle); logger.LogInformation("Pull request body:\n{PullRequestBody}", prBody); }; diff --git a/eng/update-dependencies/FromStagingPipelineOptions.cs b/eng/update-dependencies/Commands/FromStagingPipelineOptions.cs similarity index 60% rename from eng/update-dependencies/FromStagingPipelineOptions.cs rename to eng/update-dependencies/Commands/FromStagingPipelineOptions.cs index faeefe6987..28f0b3099c 100644 --- a/eng/update-dependencies/FromStagingPipelineOptions.cs +++ b/eng/update-dependencies/Commands/FromStagingPipelineOptions.cs @@ -4,9 +4,9 @@ using System.CommandLine; using System.Text.RegularExpressions; -namespace Dotnet.Docker; +namespace Microsoft.DotNet.Docker.UpdateDependencies.Commands; -internal partial record FromStagingPipelineOptions : CreatePullRequestOptions, IOptions +internal partial record FromStagingPipelineOptions : CreatePullRequestOptions { public const string StagingStorageAccountOptionName = "--staging-storage-account"; public const string InternalOption = "--internal"; @@ -22,52 +22,46 @@ internal partial record FromStagingPipelineOptions : CreatePullRequestOptions, I /// public bool Internal { get; init; } = false; - /// - /// The mode in which to run the command. - /// - public ChangeMode Mode { get; init; } = ChangeMode.Local; - /// /// This Azure Storage Account will be used as a source for the update. /// This should be one of two storage accounts: dotnetstagetest or dotnetstage. /// public string? StagingStorageAccount { get; init; } - public static new List Arguments { get; } = - [ - new Argument("stage-containers") - { - Arity = ArgumentArity.ExactlyOne, - Description = "A comma-delimited list of stage container names to use as a source for the update (e.g., 'stage-1234567,stage-2345678')" - }, - ..CreatePullRequestOptions.Arguments, - ]; - - public static Option StagingStorageAccountOption = new(StagingStorageAccountOptionName) + private static readonly Argument s_stageContainers = new("stage-containers") + { + Description = "A comma-delimited list of stage container names to use as a source for the update (e.g., 'stage-1234567,stage-2345678')", + }; + internal static readonly Option StagingStorageAccountOption = new(StagingStorageAccountOptionName) { Description = "The Azure Storage Account to use as a source for the update." + " This should be one of two storage accounts: dotnetstagetest or dotnetstage." - + " For example: https://dotnetstagetest.blob.core.windows.net/" + + " For example: https://dotnetstagetest.blob.core.windows.net/", + DefaultValueFactory = _ => string.Empty, + }; + private static readonly Option s_internal = new(InternalOption) + { + Description = "Whether or not to use the internal versions of the staged build. When not using an internal" + + " build, Dockerfiles will be updated as if the staged build has already been released. When using an" + + " internal build, Dockerfiles will be updated with internal download links and will only be buildable" + + " by using an internal Azure DevOps access token.", }; - public static new List