diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c6e64b..c67ab1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,3 +65,25 @@ jobs: with: name: runtime-proof-gradle-${{ matrix.gradle-version }}-jdk-${{ matrix.java-version }} path: build/test-results/test/ + + windows-snapshot-test: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - uses: gradle/actions/setup-gradle@v4 + + - name: Verify upload snapshots, cleanup and journal concurrency on Windows + run: ./gradlew.bat test --tests "*CombinedReleaseTest" + + - name: Retain Windows snapshot test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: windows-snapshot-test-results + path: build/test-results/test/ diff --git a/README.md b/README.md index 0b99126..5fe7cf6 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ plugins { // your other plugins... id 'signing' id 'maven-publish' - id 'se.alipsa.nexus-release-plugin' version '2.5.1' + id 'se.alipsa.nexus-release-plugin' version '2.6.0' } publishing { @@ -113,7 +113,7 @@ Example build.gradle: plugins { id 'groovy' id 'maven-publish' - id 'se.alipsa.nexus-release-plugin' version '2.5.1' + id 'se.alipsa.nexus-release-plugin' version '2.6.0' } group = 'se.alipsa' @@ -227,7 +227,7 @@ uses the modules' signing configuration and publishes only the eligible named pu ```groovy plugins { - id 'se.alipsa.nexus-release-plugin' version '2.5.1' + id 'se.alipsa.nexus-release-plugin' version '2.6.0' } nexusReleasePlugin { @@ -290,7 +290,7 @@ The selected publication's module metadata and adjacent signature are cleared be The publishing-script harness runs as part of `check` when Bash is available. Local builds without Bash skip it with a reason; CI requires it with `-PrequireShellHarness=true`. -### Upload a verified combined bundle +### Upload a verified combined bundle (2.6.0+) Run planning and `bundleAll` separately from the upload invocation. After testing and reviewing the saved ZIP, configure the upload task with the ZIP and manifest SHA-256 values diff --git a/combined-releases.md b/combined-releases.md index f0bbf47..7679d76 100644 --- a/combined-releases.md +++ b/combined-releases.md @@ -31,7 +31,7 @@ this plugin is a separate operation. In the root build, list the project paths whose publications you want to consider: ```groovy -plugins { id 'se.alipsa.nexus-release-plugin' version '2.5.1' } +plugins { id 'se.alipsa.nexus-release-plugin' version '2.6.0' } nexusReleasePlugin { userName = providers.gradleProperty('sonatypeUsername') @@ -176,7 +176,7 @@ The source repository must be clean and committed. Generated files belong in ign | `releaseAllGuard` | `sourceDirectory`, `manifestFile`, `versionsJson`; checks revision, clean tree, publication-POM hashes and Git index flags and effective publication versions before staging tasks run. | | `releaseAllPrepareMaven` | `sourceDirectory`, `manifestFile`; returns after source validation when no static Maven component is selected. Otherwise requires `stagingDirectory`, separate `localRepository`, `mavenWorkDirectory`, `verifierFile`, `settingsHelperFile`, `stagingRepositoryId` (default `release-staging`), optional `userSettingsFile` (default `~/.m2/settings.xml`); runs the caller's verifier and Maven `install -Prelease`, never deploy. The settings helper takes source user-settings, staged repository, output-settings and repository ID; it must preserve signing settings without logging secrets. | | `bundleAll` | `sourceDirectory`, `manifestFile`, `stagingDirectory`; writes `bundleFile` and `receiptFile`. Rechecks source, dependency POMs, metadata, artifact policy, nonempty signature files and staged checksums (signature cryptography is not verified). Generates MD5/SHA-1 for artifacts/signatures. | -| `releaseAll` | `sourceDirectory`, `manifestFile`, `bundleFile`, `receiptFile`, `recordFile`, `portalFixture`, repository/Portal URLs and credentials; `resumeId` defaults to an empty string for a new upload, or a saved deployment ID for status-only recovery. Rechecks availability and uploads once. | +| `releaseAll` | `sourceDirectory`, `manifestFile`, `bundleFile`, `receiptFile`, `recordFile`, `portalFixture`, repository/Portal URLs and credentials; `resumeId` defaults to an empty string for a new upload, or a saved deployment ID for status-only recovery. `useExistingBundle` (default `false`, 2.6.0+) uploads a prepared bundle without rebuilding, signing or staging, and requires both `expectedBundleSha256` and `expectedManifestSha256` (64 hexadecimal characters each). Digests are rejected for normal uploads. Rechecks source, manifest and bundle hashes against the receipt and reviewed digests, and publication availability before uploading a verified private ZIP snapshot once. With `resumeId` set, these three properties are ignored. See [verified-bundle uploads](README.md#upload-a-verified-combined-bundle-260). | The Matrix build configures these properties in `gradle/release-all.gradle`. For example, a caller configures the bundle task after registering its own staging/preparation workflow: @@ -213,6 +213,14 @@ planning, checks, signing, staging, bundling, or upload can run. Existing manife bundles, and receipts remain untouched. Rejected graphs cannot be reused from the configuration cache to bypass this check. Separate planning/bundling, standalone upload, and status-only resume retain cache support. + +In advanced (caller-managed) mode, order any candidate POM regeneration before +`releaseAllGuard` and `releaseAllPrepareMaven` using `mustRunAfter` (or `dependsOn` +when regeneration should always be scheduled). Both tasks hash the planned POMs; +running them alongside POM generation without ordering can intermittently fail with +`Stale manifest input: …/pom-default.xml`. Module mode supplies this ordering +automatically. Order `bundleAll` after Maven preparation when scheduling both. + Run source guards before staging, the caller's scoped verification before preparation, then `bundleAll`, then `releaseAll`. Do not upload if any preceding phase fails. The combined tasks support strict configuration-cache reuse; legacy verification tasks can diff --git a/release.md b/release.md index 0511e84..5f9c353 100644 --- a/release.md +++ b/release.md @@ -1,6 +1,34 @@ # Release history ## Ver 2.6.0 (in progress) +- Add verified existing-bundle uploads: configure `releaseAll { useExistingBundle = true }` + with required `expectedBundleSha256` and `expectedManifestSha256` values from retained, + reviewed evidence. Upload without rebuilding, signing or staging. Before upload, validate + digest presence/syntax, source revision and cleanliness, saved POM hashes, manifest and + bundle hashes against the receipt and reviewed digests, and publication availability. + Hash and parse the same manifest bytes, and upload a private ZIP snapshot verified + against the reviewed digest. + Snapshot creation requires a writable bundle directory; cleanup failures warn with the + path to remove manually after upload stops. See [verified-bundle uploads](README.md#upload-a-verified-combined-bundle-260). +- Behaviour change: normal `releaseAll` uploads fail if either expected digest is set, + including through project/user `gradle.properties` or `ORG_GRADLE_PROJECT_*` environment + variables. Clear the digests or enable existing-bundle mode. Requesting `bundleAll` and + `releaseAll` together in existing-bundle mode is rejected, including via dependencies. + Status-only `resumeId` recovery ignores existing-bundle settings and digests. +- Report missing prepared files with a `Prepared release file missing` error. +- Decode Git output as UTF-8 so non-ASCII source paths and guard diagnostics remain + readable with non-UTF-8 JVM defaults, including Windows on JDK 17. +- Fix Windows journal locking during concurrent reads and atomic updates by reading + JSON through NIO with delete sharing and briefly retrying filesystem failures during + atomic replacement on Windows, independently of the display language. Persistent + failures still propagate without changing the old journal. +- Fix (PR #18): order `releaseAllGuard` after planned POM generation in module mode to avoid + intermittent `Stale manifest input: …/pom-default.xml` failures in user builds and CI. + Also order `releaseAllPrepareMaven` after planned POM generation and `bundleAll` after + Maven preparation when scheduled together. Document caller-managed ordering in advanced mode. +- Diagnostic (PR #17): log `Upload evidence ignored` at info level (`--info`) when + `releaseAllGuard` runs without `releaseAll` in the task graph; bundle-only and check-only + runs do not validate upload evidence. - Change the default combined-release bundle name from `bundle.zip` to `-mega-bundle.zip` (e.g. `jmlx-mega-bundle.zip`), which is also used as the Central Portal deployment name. Override with diff --git a/src/main/groovy/se/alipsa/gradle/plugin/release/ModuleReleaseConfigurer.groovy b/src/main/groovy/se/alipsa/gradle/plugin/release/ModuleReleaseConfigurer.groovy index 2ca3bd0..ba4ea66 100644 --- a/src/main/groovy/se/alipsa/gradle/plugin/release/ModuleReleaseConfigurer.groovy +++ b/src/main/groovy/se/alipsa/gradle/plugin/release/ModuleReleaseConfigurer.groovy @@ -73,12 +73,17 @@ final class ModuleReleaseConfigurer { task.unpublishedProjectPaths.set(unpublishedPaths) } + extension.prepareMavenTask.configure { task -> + task.mustRunAfter(model.call().pomTasks) + } + // Resolve only when bundling is scheduled, even if a consumer eagerly realizes tasks. // Finalization prevents repeated dependency queries from registering the repository twice. def staging = project.objects.mapProperty(String, Object) staging.set(project.providers.provider { stagingTasks(project, extension, paths) }) staging.finalizeValueOnRead() extension.bundleAllTask.configure { task -> + task.mustRunAfter(extension.prepareMavenTask) task.dependsOn(extension.guardTask) task.dependsOn(extension.signingCheckTask) task.dependsOn(staging.map { it.tasks }) diff --git a/src/main/groovy/se/alipsa/gradle/plugin/release/ReleaseManifest.groovy b/src/main/groovy/se/alipsa/gradle/plugin/release/ReleaseManifest.groovy index 1f03e97..49f2aa6 100644 --- a/src/main/groovy/se/alipsa/gradle/plugin/release/ReleaseManifest.groovy +++ b/src/main/groovy/se/alipsa/gradle/plugin/release/ReleaseManifest.groovy @@ -10,11 +10,11 @@ class ReleaseManifest { static String gitOutput(File root, String... arguments) { Process process = new ProcessBuilder(['git', '-C', root.absolutePath] + arguments.toList()).start() - StringBuffer output = new StringBuffer() - StringBuffer errors = new StringBuffer() + ByteArrayOutputStream output = new ByteArrayOutputStream() + ByteArrayOutputStream errors = new ByteArrayOutputStream() process.waitForProcessOutput(output, errors) - if (process.exitValue() != 0) throw new IllegalStateException('Git source validation failed: ' + errors.toString().trim()) - output.toString() + if (process.exitValue() != 0) throw new IllegalStateException('Git source validation failed: ' + errors.toString('UTF-8').trim()) + output.toString('UTF-8') } /** Rejects index flags that can hide working-tree changes, preserving arbitrary filenames. */ @@ -45,8 +45,14 @@ class ReleaseManifest { } } - /** Reads the persisted JSON manifest. */ - static Map read(File file) { new JsonSlurper().parse(file) as Map } + /** Reads a complete snapshot, closing the file before parsing. */ + static Map read(File file) { + // NIO opens Windows files with delete sharing, allowing an atomic replacement + // while a reader still holds the old file. JsonSlurper.parse(File) uses a + // FileInputStream, whose Windows handle prevents that replacement. + byte[] snapshot = java.nio.file.Files.readAllBytes(file.toPath()) + new JsonSlurper().parse(snapshot, 'UTF-8') as Map + } /** Atomically replaces a JSON file in its output directory. */ static void write(File file, Map value) { @@ -54,7 +60,29 @@ class ReleaseManifest { File temporary = java.nio.file.Files.createTempFile(file.absoluteFile.parentFile.toPath(), file.name + '.', '.tmp').toFile() try { temporary.setText(JsonOutput.prettyPrint(JsonOutput.toJson(value)), 'UTF-8') - java.nio.file.Files.move(temporary.toPath(), file.toPath(), java.nio.file.StandardCopyOption.ATOMIC_MOVE, java.nio.file.StandardCopyOption.REPLACE_EXISTING) + replaceAtomically(temporary.toPath(), file.toPath()) } finally { temporary.delete() } } + + private static void replaceAtomically(java.nio.file.Path source, java.nio.file.Path target) { + for (int attempt = 0; ; attempt++) { + try { + java.nio.file.Files.move(source, target, java.nio.file.StandardCopyOption.ATOMIC_MOVE, + java.nio.file.StandardCopyOption.REPLACE_EXISTING) + return + } catch (java.nio.file.FileSystemException failure) { + // Windows reports sharing conflicts using localized reason text and + // different FileSystemException subtypes. Retry the atomic move for + // any such failure, with 1.27 seconds of total backoff. Persistent + // failures still propagate without deleting the existing journal. + if (attempt >= 7 || !System.getProperty('os.name').toLowerCase(Locale.ROOT).contains('windows')) throw failure + try { + Thread.sleep(10L << attempt) + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt() + throw interrupted + } + } + } + } } diff --git a/src/test/groovy/se/alipsa/gradle/plugin/release/CharsetTest.groovy b/src/test/groovy/se/alipsa/gradle/plugin/release/CharsetTest.groovy index 83294fb..2626b59 100644 --- a/src/test/groovy/se/alipsa/gradle/plugin/release/CharsetTest.groovy +++ b/src/test/groovy/se/alipsa/gradle/plugin/release/CharsetTest.groovy @@ -2,12 +2,39 @@ package se.alipsa.gradle.plugin.release import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir import org.gradle.api.logging.Logging import java.nio.charset.Charset import java.nio.charset.StandardCharsets import static org.junit.jupiter.api.Assertions.* class CharsetTest { + @TempDir File directory + + @Test @Tag('non-utf8-charset') + void gitPathsAndErrorsUseUtf8() { + assertNotEquals(StandardCharsets.UTF_8, Charset.defaultCharset()) + String name = '\u00e5.txt' + new File(directory, name).setText('original', 'UTF-8') + ReleaseManifest.git(directory, 'init') + ReleaseManifest.git(directory, 'add', '.') + assertTrue(ReleaseManifest.gitOutput(directory, 'ls-files', '-z').contains(name)) + // Feed UTF-8 bytes directly: JDK 17 on Unix also uses the default charset + // when encoding process arguments, independently of output decoding. + def flag = new ProcessBuilder('git', '-C', directory.absolutePath, 'update-index', + '--assume-unchanged', '-z', '--stdin').start() + flag.outputStream.withCloseable { it.write((name + '\u0000').getBytes(StandardCharsets.UTF_8)) } + flag.waitForProcessOutput(new ByteArrayOutputStream(), new ByteArrayOutputStream()) + assertEquals(0, flag.exitValue()) + assertTrue(assertThrows(IllegalStateException) { + ReleaseManifest.checkTrackedFlags(directory) + }.message.contains(name)) + def failure = assertThrows(IllegalStateException) { + ReleaseManifest.git(directory, '-c', 'alias.charset-error=!printf "\\303\\245\\n" >&2; exit 1', 'charset-error') + } + assertTrue(failure.message.contains('\u00e5')) + } + @Test @Tag('non-utf8-charset') void credentialsUseUtf8() { assertNotEquals(StandardCharsets.UTF_8, Charset.defaultCharset(), 'charsetTest must run with a non-UTF-8 default charset') diff --git a/src/test/groovy/se/alipsa/gradle/plugin/release/CombinedReleaseTest.groovy b/src/test/groovy/se/alipsa/gradle/plugin/release/CombinedReleaseTest.groovy index 95bb68a..b4fb092 100644 --- a/src/test/groovy/se/alipsa/gradle/plugin/release/CombinedReleaseTest.groovy +++ b/src/test/groovy/se/alipsa/gradle/plugin/release/CombinedReleaseTest.groovy @@ -2,6 +2,9 @@ package se.alipsa.gradle.plugin.release import com.sun.net.httpserver.HttpServer import org.junit.jupiter.api.Test +import org.junit.jupiter.api.RepeatedTest +import org.junit.jupiter.api.condition.EnabledOnOs +import org.junit.jupiter.api.condition.OS import org.junit.jupiter.api.io.TempDir import org.junit.jupiter.api.parallel.Isolated import java.util.zip.ZipFile @@ -532,7 +535,8 @@ class CombinedReleaseTest { assertTrue(failure.message.contains('fatal:')) } - @Test void concurrentRecordUpdatesKeepValidJournal() { + @RepeatedTest(5) + void concurrentRecordUpdatesKeepValidJournal() { File record = new File(directory, 'journal.json') ReleaseManifest.write(record, [state: 'UPLOADED']) def executor = java.util.concurrent.Executors.newFixedThreadPool(3) @@ -548,6 +552,60 @@ class CombinedReleaseTest { } finally { executor.shutdownNow() } } + @Test + @EnabledOnOs(OS.WINDOWS) + void journalUpdateRetriesUntilWindowsReaderReleasesItsLock() { + File record = new File(directory, 'journal.json') + ReleaseManifest.write(record, [state: 'UPLOADED']) + def executor = java.util.concurrent.Executors.newSingleThreadScheduledExecutor() + new FileInputStream(record).withCloseable { reader -> + def release = executor.schedule({ -> reader.close() } as Runnable, 100, java.util.concurrent.TimeUnit.MILLISECONDS) + try { + ReleaseManifest.write(record, [state: 'AVAILABLE']) + release.get(5, java.util.concurrent.TimeUnit.SECONDS) + assertEquals('AVAILABLE', ReleaseManifest.read(record).state) + assertFalse(directory.listFiles().any { it.name.endsWith('.tmp') }) + } finally { executor.shutdownNow() } + } + } + + @Test + @EnabledOnOs(OS.WINDOWS) + void journalReplacementRetriesWhenItsTemporaryFileIsInUse() { + File source = new File(directory, 'journal.tmp') + File target = new File(directory, 'journal.json') + ReleaseManifest.write(source, [state: 'AVAILABLE']) + ReleaseManifest.write(target, [state: 'UPLOADED']) + // A scanner can open the temporary file between writing and renaming it. + // Invoke the replacement directly to hold that source handle deterministically. + def replace = ReleaseManifest.getDeclaredMethod('replaceAtomically', java.nio.file.Path, java.nio.file.Path) + replace.accessible = true + def executor = java.util.concurrent.Executors.newSingleThreadScheduledExecutor() + new FileInputStream(source).withCloseable { reader -> + def release = executor.schedule({ -> reader.close() } as Runnable, 100, java.util.concurrent.TimeUnit.MILLISECONDS) + try { + replace.invoke(null, source.toPath(), target.toPath()) + release.get(5, java.util.concurrent.TimeUnit.SECONDS) + assertEquals('AVAILABLE', ReleaseManifest.read(target).state) + assertFalse(source.exists()) + } finally { executor.shutdownNow() } + } + } + + @Test + @EnabledOnOs(OS.WINDOWS) + void permanentlyLockedJournalFailsWithoutChangingOriginalAndCleansUp() { + File record = new File(directory, 'journal.json') + ReleaseManifest.write(record, [state: 'UPLOADED']) + new FileInputStream(record).withCloseable { + assertThrows(java.nio.file.AccessDeniedException) { + ReleaseManifest.write(record, [state: 'AVAILABLE']) + } + assertEquals('UPLOADED', ReleaseManifest.read(record).state) + assertFalse(directory.listFiles().any { it.name.endsWith('.tmp') }) + } + } + @Test void parentPomsAreRejectedClearlyAndExternalManagedVersionsCanBeFlattened() { File pom = new File(directory, 'parent.pom') pom.text = pomText('a', 'pom').replace('', 'gparent1') diff --git a/src/test/groovy/se/alipsa/gradle/plugin/release/ModuleReleaseTest.groovy b/src/test/groovy/se/alipsa/gradle/plugin/release/ModuleReleaseTest.groovy index c98b32e..2e0a28e 100644 --- a/src/test/groovy/se/alipsa/gradle/plugin/release/ModuleReleaseTest.groovy +++ b/src/test/groovy/se/alipsa/gradle/plugin/release/ModuleReleaseTest.groovy @@ -318,6 +318,28 @@ tasks.named('generatePomFileForMavenPublication') { } } + @Test void mavenPreparationWaitsForRegeneratedPomsAndPrecedesBundling() { + withServer { String base, List requests -> + fixture(base, "modules = [':core']") + run(':releaseAllPlan') + // Preparation validates the planned POM even without a static Maven component, + // so this exercises its real guard without requiring external Maven helpers. + File pom = new File(root, 'core/build/publications/maven/pom-default.xml') + 2.times { int invocation -> + assertTrue(pom.delete()) + def result = run(':bundleAll', ':releaseAllPrepareMaven', ':core:generatePomFileForMavenPublication') + List tasks = result.tasks*.path + [':core:generatePomFileForMavenPublication', ':releaseAllPrepareMaven', ':bundleAll'].each { path -> + assertTrue(tasks.contains(path), 'Expected task in results: ' + path) + } + assertTrue(tasks.indexOf(':core:generatePomFileForMavenPublication') < tasks.indexOf(':releaseAllPrepareMaven')) + assertTrue(tasks.indexOf(':releaseAllPrepareMaven') < tasks.indexOf(':bundleAll')) + assertTrue(new File(root, BUNDLE).isFile()) + if (invocation == 1) assertTrue(result.output.contains('Reusing configuration cache')) + } + } + } + @Test void changedSelectionCannotReuseStaleStagedArtifactsAfterReplanning() { withServer { String base, List requests -> fixture(base, "modules = [':api', ':core']")