diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c30d0877c..10a4b20c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -455,6 +455,7 @@ jobs: run: | mkdir -p java-natives/linux-x86_64 cp rust/target/release/libsdm_jni.so java-natives/linux-x86_64/ + cp LICENSE NOTICE packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt java-natives/ - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: java-native-linux-x86_64 @@ -509,11 +510,16 @@ jobs: name: native-wheel-test-ops retention-days: 1 path: native-test-ops/ + - name: The test-only Java binding, with its notices + run: | + mkdir -p java-test-ops-native + cp rust/target/release/libsdm_jni.so LICENSE NOTICE \ + packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt java-test-ops-native/ - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: java-test-ops-native retention-days: 1 - path: rust/target/release/libsdm_jni.so + path: java-test-ops-native/ native-wheel-aarch64: name: native-wheel (aarch64, ${{ matrix.build }}) @@ -549,6 +555,7 @@ jobs: run: | mkdir -p java-natives/linux-aarch64 cp rust/target/release/libsdm_jni.so java-natives/linux-aarch64/ + cp LICENSE NOTICE packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt java-natives/ - if: matrix.java == 'shipped' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -1501,6 +1508,7 @@ jobs: library=libsdm_jni.dylib if [ "$RUNNER_OS" = Windows ]; then library=sdm_jni.dll; fi cp "rust/target/release/$library" "java-natives/${{ matrix.platform }}/" + cp LICENSE NOTICE packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt java-natives/ - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: java-native-${{ matrix.platform }} @@ -1508,7 +1516,10 @@ jobs: path: java-natives/ # The one JAR, with every platform's native (the all-natives profile - # refuses a JAR missing one), and its runtime dependencies beside it. + # refuses a JAR missing one). Its third-party runtime dependencies and + # JDBC drivers are listed, not carried: each job that needs them fetches + # them from Maven Central, so the downloadable artifact holds no + # third-party JAR. java-jar: runs-on: ubuntu-latest timeout-minutes: 20 @@ -1542,9 +1553,8 @@ jobs: - name: The JAR's artifact run: | mkdir -p java-jar - cp java/target/structured-data-models-client-${{ needs.configuration.outputs.version }}.jar java/jar-load/JarLoad.java java-jar/ - cp -R java/target/lib java-jar/lib - cp -R java/target/jdbc java-jar/jdbc + cp java/target/structured-data-models-client-${{ needs.configuration.outputs.version }}.jar java/jar-load/JarLoad.java java/jar-load/FetchDependencies.java java-jar/ + python3 scripts/java_dependency_list.py java/target ~/.m2/repository > java-jar/java-dependencies.tsv cp java/pom.xml java-jar/structured-data-models-client-${{ needs.configuration.outputs.version }}.pom python3 - <<'PYCODE' import hashlib, json, os, pathlib @@ -1672,6 +1682,7 @@ jobs: echo "::error::a Rust toolchain is in the container"; exit 1 fi java -version + java jar/FetchDependencies.java jar separator=: if [ "$RUNNER_OS" = Windows ]; then separator=';'; fi java $FLAGS -cp "jar/structured-data-models-client-${{ needs.configuration.outputs.version }}.jar${separator}jar/lib/*" jar/JarLoad.java @@ -1728,6 +1739,8 @@ jobs: with: python-version: '3.12' - uses: ./.github/actions/python-cache + - name: The JAR's dependencies, from Maven Central + run: java jar/FetchDependencies.java jar - name: The S0 cases on the Java host run: | pip install pytest pyyaml jsonschema @@ -1811,7 +1824,7 @@ jobs: # The reference cell's data-library pins, DuckDB 2 and # the drivers the contract loader writes SQLite and Postgres with: # what the runner's capture and the loader read with, never what the - # Java host reads with (its JDBC drivers, in the JAR's artifact). + # Java host reads with (its JDBC drivers, listed in the JAR's artifact). - name: Install the runner's environment run: | printf '%s\n' $CELL_PINS duckdb==2.0.0 > "$RUNNER_TEMP/cell.txt" @@ -1826,6 +1839,8 @@ jobs: unzip -q jar/structured-data-models-client-${{ needs.configuration.outputs.version }}.jar 'com/nvidia/sdm/native/*' -d "$RUNNER_TEMP/java-native" mvn -B -f java/pom.xml -Dsdm.natives="$RUNNER_TEMP/java-native/com/nvidia/sdm/native" \ -Dtest=SchemaTest,PredictTest,PostgresReadTest,PostgresKeylessOrderTest test + - name: The JAR's dependencies, from Maven Central + run: java jar/FetchDependencies.java jar - name: The S1 cases on the Java host run: | python contract/runners/run_java.py --database "$DATABASE" \ diff --git a/java/jar-load/FetchDependencies.java b/java/jar-load/FetchDependencies.java new file mode 100644 index 000000000..4ea7eb939 --- /dev/null +++ b/java/jar-load/FetchDependencies.java @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.HexFormat; +import java.util.List; + +/** + * Fetches the third-party JARs that CI's java-jar artifact lists but does not carry. Each line of + * {@code ROOT/java-dependencies.tsv} is a path under ROOT, a SHA-256 and a Maven Central URL. Run + * in source-file mode: + * + *
java FetchDependencies.java ROOT
+ * + * It exits 0 once every listed JAR is in place with its listed digest, else non-zero. + */ +public final class FetchDependencies { + private static final int ATTEMPTS = 4; + + private FetchDependencies() {} + + public static void main(String[] arguments) throws Exception { + Path root = Path.of(arguments[0]); + HttpClient client = + HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .connectTimeout(Duration.ofSeconds(30)) + .build(); + List lines = Files.readAllLines(root.resolve("java-dependencies.tsv")); + for (String line : lines) { + String[] fields = line.split("\t"); + Path target = root.resolve(fields[0]); + Files.createDirectories(target.getParent()); + Files.write(target, fetch(client, URI.create(fields[2]), fields[1])); + } + System.out.println("fetched " + lines.size() + " dependencies"); + } + + private static byte[] fetch(HttpClient client, URI uri, String sha256) + throws IOException, InterruptedException, NoSuchAlgorithmException { + HttpRequest request = HttpRequest.newBuilder(uri).timeout(Duration.ofMinutes(2)).build(); + String failure = ""; + for (int attempt = 1; attempt <= ATTEMPTS; attempt++) { + try { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() == 200) { + byte[] body = response.body(); + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(body)); + if (!digest.equals(sha256)) { + throw new IllegalStateException(uri + " has SHA-256 " + digest + ", not " + sha256); + } + return body; + } + failure = "HTTP " + response.statusCode(); + } catch (IOException error) { + failure = error.toString(); + } + Thread.sleep(2000L * attempt); + } + throw new IOException(uri + " failed " + ATTEMPTS + " times: " + failure); + } +} diff --git a/scripts/java_dependency_list.py b/scripts/java_dependency_list.py new file mode 100644 index 000000000..86f305960 --- /dev/null +++ b/scripts/java_dependency_list.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""List the third-party JARs a Java build copied, as Maven Central fetches. + +Usage: java_dependency_list.py TARGET REPOSITORY + +TARGET is the build's target directory, whose lib and jdbc directories hold +the copied runtime dependencies and JDBC drivers. REPOSITORY is the local +Maven repository they were resolved into. Each line printed is a path under +the artifact root, the JAR's SHA-256 and its Maven Central URL, which +java/jar-load/FetchDependencies.java reads. A JAR found nowhere in the +repository with the same bytes is an error. +""" + +import hashlib +import sys +from pathlib import Path + +CENTRAL = 'https://repo1.maven.org/maven2/' +DIRECTORIES = ('lib', 'jdbc') + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def lines(target: Path, repository: Path) -> list[str]: + copies: dict[str, list[Path]] = {} + for path in repository.rglob('*.jar'): + copies.setdefault(path.name, []).append(path) + listed = [] + for directory in DIRECTORIES: + for jar in sorted((target / directory).glob('*.jar')): + digest = sha256(jar) + sources = [ + path + for path in copies.get(jar.name, []) + if sha256(path) == digest + ] + if not sources: + raise SystemExit(f'{jar} is not in {repository}') + url = CENTRAL + sources[0].relative_to(repository).as_posix() + listed.append(f'{directory}/{jar.name}\t{digest}\t{url}') + if not listed: + raise SystemExit(f'{target} holds no dependencies') + return listed + + +def main() -> None: + target, repository = (Path(argument) for argument in sys.argv[1:3]) + print('\n'.join(lines(target, repository))) + + +if __name__ == '__main__': + main() diff --git a/tests/test_java_driver_ci.py b/tests/test_java_driver_ci.py index 48018be99..c40bff02d 100644 --- a/tests/test_java_driver_ci.py +++ b/tests/test_java_driver_ci.py @@ -12,6 +12,8 @@ from types import ModuleType from typing import Any +import yaml + ROOT = Path(__file__).resolve().parents[1] CI = ROOT / '.github/workflows/ci.yml' #: Java's S1 legs: JDK, platform, database. @@ -107,9 +109,43 @@ def test_the_java_tests_run_with_postgres_required() -> None: assert job['env']['SDM_POSTGRES_TEST_DSN'].startswith('postgresql://') -def test_the_jar_s_artifact_holds_the_jdbc_drivers() -> None: - job, _ = _jobs()['java-jar'] - assert 'cp -R java/target/jdbc java-jar/jdbc' in _text(job) - pom = (ROOT / 'java/pom.xml').read_text(encoding='utf-8') - for driver in ('duckdb_jdbc', 'sqlite-jdbc', 'postgresql'): - assert driver in pom, driver +def test_the_jar_s_artifact_lists_and_never_carries_third_party_jars() -> None: + jobs = _jobs() + job, _ = jobs['java-jar'] + text = _text(job) + assert 'java_dependency_list.py java/target' in text + assert '> java-jar/java-dependencies.tsv' in text + assert 'target/lib' not in text and 'target/jdbc' not in text + for name, (consumer, _) in jobs.items(): + if 'jar/lib' in _text(consumer) or 'jar/jdbc' in _text(consumer): + assert 'java jar/FetchDependencies.java jar' in _text(consumer), ( + name + ) + + +def test_every_downloadable_native_carries_its_notices() -> None: + workflow = yaml.safe_load(CI.read_text(encoding='utf-8')) + uploaded = set() + for job in workflow['jobs'].values(): + for step in job.get('steps', []): + artifact = str(step.get('with', {}).get('name', '')) + if not str(step.get('uses', '')).startswith( + 'actions/upload-artifact@' + ): + continue + if not artifact.startswith(('java-native', 'java-test-ops-native')): + continue + uploaded.add(artifact) + notices = ( + 'LICENSE NOTICE', + 'packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt ' + + step['with']['path'], + ) + for notice in notices: + assert notice in _text(job), (artifact, notice) + assert uploaded == { + 'java-native-linux-x86_64', + 'java-native-linux-aarch64', + 'java-native-${{ matrix.platform }}', + 'java-test-ops-native', + }