Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,7 @@ jobs:
run: |
mkdir -p java-natives/linux-x86_64
cp rust/target/release/libsdm_jni.so java-natives/linux-x86_64/
cp LICENSE NOTICE packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt java-natives/
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: java-native-linux-x86_64
Expand Down Expand Up @@ -509,11 +510,16 @@ jobs:
name: native-wheel-test-ops
retention-days: 1
path: native-test-ops/
- name: The test-only Java binding, with its notices
run: |
mkdir -p java-test-ops-native
cp rust/target/release/libsdm_jni.so LICENSE NOTICE \
packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt java-test-ops-native/
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: java-test-ops-native
retention-days: 1
path: rust/target/release/libsdm_jni.so
path: java-test-ops-native/

native-wheel-aarch64:
name: native-wheel (aarch64, ${{ matrix.build }})
Expand Down Expand Up @@ -549,6 +555,7 @@ jobs:
run: |
mkdir -p java-natives/linux-aarch64
cp rust/target/release/libsdm_jni.so java-natives/linux-aarch64/
cp LICENSE NOTICE packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt java-natives/
- if: matrix.java == 'shipped'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
Expand Down Expand Up @@ -1501,14 +1508,18 @@ jobs:
library=libsdm_jni.dylib
if [ "$RUNNER_OS" = Windows ]; then library=sdm_jni.dll; fi
cp "rust/target/release/$library" "java-natives/${{ matrix.platform }}/"
cp LICENSE NOTICE packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt java-natives/
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: java-native-${{ matrix.platform }}
retention-days: 1
path: java-natives/

# The one JAR, with every platform's native (the all-natives profile
# refuses a JAR missing one), and its runtime dependencies beside it.
# refuses a JAR missing one). Its third-party runtime dependencies and
# JDBC drivers are listed, not carried: each job that needs them fetches
# them from Maven Central, so the downloadable artifact holds no
# third-party JAR.
java-jar:
runs-on: ubuntu-latest
timeout-minutes: 20
Expand Down Expand Up @@ -1542,9 +1553,8 @@ jobs:
- name: The JAR's artifact
run: |
mkdir -p java-jar
cp java/target/structured-data-models-client-${{ needs.configuration.outputs.version }}.jar java/jar-load/JarLoad.java java-jar/
cp -R java/target/lib java-jar/lib
cp -R java/target/jdbc java-jar/jdbc
cp java/target/structured-data-models-client-${{ needs.configuration.outputs.version }}.jar java/jar-load/JarLoad.java java/jar-load/FetchDependencies.java java-jar/
python3 scripts/java_dependency_list.py java/target ~/.m2/repository > java-jar/java-dependencies.tsv
cp java/pom.xml java-jar/structured-data-models-client-${{ needs.configuration.outputs.version }}.pom
python3 - <<'PYCODE'
import hashlib, json, os, pathlib
Expand Down Expand Up @@ -1672,6 +1682,7 @@ jobs:
echo "::error::a Rust toolchain is in the container"; exit 1
fi
java -version
java jar/FetchDependencies.java jar
separator=:
if [ "$RUNNER_OS" = Windows ]; then separator=';'; fi
java $FLAGS -cp "jar/structured-data-models-client-${{ needs.configuration.outputs.version }}.jar${separator}jar/lib/*" jar/JarLoad.java
Expand Down Expand Up @@ -1728,6 +1739,8 @@ jobs:
with:
python-version: '3.12'
- uses: ./.github/actions/python-cache
- name: The JAR's dependencies, from Maven Central
run: java jar/FetchDependencies.java jar
- name: The S0 cases on the Java host
run: |
pip install pytest pyyaml jsonschema
Expand Down Expand Up @@ -1811,7 +1824,7 @@ jobs:
# The reference cell's data-library pins, DuckDB 2 and
# the drivers the contract loader writes SQLite and Postgres with:
# what the runner's capture and the loader read with, never what the
# Java host reads with (its JDBC drivers, in the JAR's artifact).
# Java host reads with (its JDBC drivers, listed in the JAR's artifact).
- name: Install the runner's environment
run: |
printf '%s\n' $CELL_PINS duckdb==2.0.0 > "$RUNNER_TEMP/cell.txt"
Expand All @@ -1826,6 +1839,8 @@ jobs:
unzip -q jar/structured-data-models-client-${{ needs.configuration.outputs.version }}.jar 'com/nvidia/sdm/native/*' -d "$RUNNER_TEMP/java-native"
mvn -B -f java/pom.xml -Dsdm.natives="$RUNNER_TEMP/java-native/com/nvidia/sdm/native" \
-Dtest=SchemaTest,PredictTest,PostgresReadTest,PostgresKeylessOrderTest test
- name: The JAR's dependencies, from Maven Central
run: java jar/FetchDependencies.java jar
- name: The S1 cases on the Java host
run: |
python contract/runners/run_java.py --database "$DATABASE" \
Expand Down
72 changes: 72 additions & 0 deletions java/jar-load/FetchDependencies.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
// All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.util.HexFormat;
import java.util.List;

/**
* Fetches the third-party JARs that CI's java-jar artifact lists but does not carry. Each line of
* {@code ROOT/java-dependencies.tsv} is a path under ROOT, a SHA-256 and a Maven Central URL. Run
* in source-file mode:
*
* <pre>java FetchDependencies.java ROOT</pre>
*
* It exits 0 once every listed JAR is in place with its listed digest, else non-zero.
*/
public final class FetchDependencies {
private static final int ATTEMPTS = 4;

private FetchDependencies() {}

public static void main(String[] arguments) throws Exception {
Path root = Path.of(arguments[0]);
HttpClient client =
HttpClient.newBuilder()
.followRedirects(HttpClient.Redirect.NORMAL)
.connectTimeout(Duration.ofSeconds(30))
.build();
List<String> lines = Files.readAllLines(root.resolve("java-dependencies.tsv"));
for (String line : lines) {
String[] fields = line.split("\t");
Path target = root.resolve(fields[0]);
Files.createDirectories(target.getParent());
Files.write(target, fetch(client, URI.create(fields[2]), fields[1]));
}
System.out.println("fetched " + lines.size() + " dependencies");
}

private static byte[] fetch(HttpClient client, URI uri, String sha256)
throws IOException, InterruptedException, NoSuchAlgorithmException {
HttpRequest request = HttpRequest.newBuilder(uri).timeout(Duration.ofMinutes(2)).build();
String failure = "";
for (int attempt = 1; attempt <= ATTEMPTS; attempt++) {
try {
HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
if (response.statusCode() == 200) {
byte[] body = response.body();
String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(body));
if (!digest.equals(sha256)) {
throw new IllegalStateException(uri + " has SHA-256 " + digest + ", not " + sha256);
}
return body;
}
failure = "HTTP " + response.statusCode();
} catch (IOException error) {
failure = error.toString();
}
Thread.sleep(2000L * attempt);
}
throw new IOException(uri + " failed " + ATTEMPTS + " times: " + failure);
}
}
58 changes: 58 additions & 0 deletions scripts/java_dependency_list.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/usr/bin/env python
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""List the third-party JARs a Java build copied, as Maven Central fetches.

Usage: java_dependency_list.py TARGET REPOSITORY

TARGET is the build's target directory, whose lib and jdbc directories hold
the copied runtime dependencies and JDBC drivers. REPOSITORY is the local
Maven repository they were resolved into. Each line printed is a path under
the artifact root, the JAR's SHA-256 and its Maven Central URL, which
java/jar-load/FetchDependencies.java reads. A JAR found nowhere in the
repository with the same bytes is an error.
"""

import hashlib
import sys
from pathlib import Path

CENTRAL = 'https://repo1.maven.org/maven2/'
DIRECTORIES = ('lib', 'jdbc')


def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()


def lines(target: Path, repository: Path) -> list[str]:
copies: dict[str, list[Path]] = {}
for path in repository.rglob('*.jar'):
copies.setdefault(path.name, []).append(path)
listed = []
for directory in DIRECTORIES:
for jar in sorted((target / directory).glob('*.jar')):
digest = sha256(jar)
sources = [
path
for path in copies.get(jar.name, [])
if sha256(path) == digest
]
if not sources:
raise SystemExit(f'{jar} is not in {repository}')
url = CENTRAL + sources[0].relative_to(repository).as_posix()
listed.append(f'{directory}/{jar.name}\t{digest}\t{url}')
if not listed:
raise SystemExit(f'{target} holds no dependencies')
return listed


def main() -> None:
target, repository = (Path(argument) for argument in sys.argv[1:3])
print('\n'.join(lines(target, repository)))


if __name__ == '__main__':
main()
48 changes: 42 additions & 6 deletions tests/test_java_driver_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
from types import ModuleType
from typing import Any

import yaml

ROOT = Path(__file__).resolve().parents[1]
CI = ROOT / '.github/workflows/ci.yml'
#: Java's S1 legs: JDK, platform, database.
Expand Down Expand Up @@ -107,9 +109,43 @@ def test_the_java_tests_run_with_postgres_required() -> None:
assert job['env']['SDM_POSTGRES_TEST_DSN'].startswith('postgresql://')


def test_the_jar_s_artifact_holds_the_jdbc_drivers() -> None:
job, _ = _jobs()['java-jar']
assert 'cp -R java/target/jdbc java-jar/jdbc' in _text(job)
pom = (ROOT / 'java/pom.xml').read_text(encoding='utf-8')
for driver in ('duckdb_jdbc', 'sqlite-jdbc', 'postgresql'):
assert driver in pom, driver
def test_the_jar_s_artifact_lists_and_never_carries_third_party_jars() -> None:
jobs = _jobs()
job, _ = jobs['java-jar']
text = _text(job)
assert 'java_dependency_list.py java/target' in text
assert '> java-jar/java-dependencies.tsv' in text
assert 'target/lib' not in text and 'target/jdbc' not in text
for name, (consumer, _) in jobs.items():
if 'jar/lib' in _text(consumer) or 'jar/jdbc' in _text(consumer):
assert 'java jar/FetchDependencies.java jar' in _text(consumer), (
name
)


def test_every_downloadable_native_carries_its_notices() -> None:
workflow = yaml.safe_load(CI.read_text(encoding='utf-8'))
uploaded = set()
for job in workflow['jobs'].values():
for step in job.get('steps', []):
artifact = str(step.get('with', {}).get('name', ''))
if not str(step.get('uses', '')).startswith(
'actions/upload-artifact@'
):
continue
if not artifact.startswith(('java-native', 'java-test-ops-native')):
continue
uploaded.add(artifact)
notices = (
'LICENSE NOTICE',
'packages/structured-data-models-client/THIRD-PARTY-NOTICES-RUST.txt '
+ step['with']['path'],
)
for notice in notices:
assert notice in _text(job), (artifact, notice)
assert uploaded == {
'java-native-linux-x86_64',
'java-native-linux-aarch64',
'java-native-${{ matrix.platform }}',
'java-test-ops-native',
}
Loading