diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57e3ad857..e94fb95e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,17 @@ jobs: run: cargo fmt --all -- --check java-tests: - name: Java tests (debug native build) + name: Java tests (debug native build)${{ matrix.suffix }} + strategy: + fail-fast: false + matrix: + include: + - line: '2.2' + profile: '' + suffix: '' + - line: '1.18' + profile: '-Pflink-1.18' + suffix: ' (Flink 1.18)' runs-on: ubuntu-24.04 timeout-minutes: 150 steps: @@ -64,8 +74,15 @@ jobs: run: | mvn -B -ntp -f dev/flink-suite/agent/pom.xml package python3 -m unittest discover -s dev/flink-suite -p 'test_*.py' - python3 -m unittest discover -s bin -p 'test_check_native_tls.py' + python3 -m unittest discover -s bin -p 'test_check_flink_suite_classpath.py' python3 -m unittest discover -s bin -p 'test_check_native_glibc.py' + python3 -m unittest discover -s bin -p 'test_check_native_tls.py' + + - name: Test Flink 1.18 native and fallback contract selection + if: matrix.line == '1.18' + run: | + mvn -B -ntp -f dev/flink-suite/agent/pom.xml -Dstreamfusion.flink-suite.flink-line=1.18 test + mvn -B -ntp -f dev/flink-suite/agent/pom.xml -Dstreamfusion.flink-suite.flink-line=1.18 -Dstreamfusion.flink-suite.native-rocksdb=true test - uses: dtolnay/rust-toolchain@1.94.0 @@ -77,13 +94,13 @@ jobs: # full JUnit suite. The Kafka decode harnesses use Testcontainers, which the # hosted runners' preinstalled Docker daemon serves directly. - name: Run Java test suite - run: mvn -B -ntp test + run: mvn -B -ntp ${{ matrix.profile }} test - name: Upload portable SQL audit outcomes if: always() uses: actions/upload-artifact@v4 with: - name: portable-sql-audit + name: portable-sql-audit-${{ matrix.line }} path: streamfusion-runtime/target/sql-audit/*.json if-no-files-found: error @@ -109,6 +126,14 @@ jobs: profile: paimon module: streamfusion-paimon format: orc + - name: Paimon Parquet (Flink 1.18) + profile: paimon,flink-1.18 + module: streamfusion-paimon + format: parquet + - name: Paimon ORC (Flink 1.18) + profile: paimon,flink-1.18 + module: streamfusion-paimon + format: orc steps: - uses: actions/checkout@v6 @@ -134,12 +159,22 @@ jobs: SF_PAIMON_FILE_FORMAT: ${{ matrix.format || 'parquet' }} run: > mvn -B -ntp -P${{ matrix.profile }} - -pl :${{ matrix.module }} -am test -Dsf.runtime.tests.skip=true + -pl ${{ matrix.module }} -am test -Dsf.runtime.tests.skip=true image-integration: - name: Flink image integration (optimized native build) + name: Flink image integration (optimized native build)${{ matrix.suffix }} runs-on: ubuntu-22.04 timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - line: '2.2' + profile: image-it + suffix: '' + - line: '1.18' + profile: image-it,flink-1.18 + suffix: ' (Flink 1.18)' steps: - uses: actions/checkout@v6 @@ -166,26 +201,16 @@ jobs: # Host-only is still a production Linux/x86_64 release (including mimalloc); cross-platform # manifest assembly is covered by the release builder, but does not belong on this smoke path. - name: Build optimized core image + env: + FLINK_LINE: ${{ matrix.line }} run: | - bin/build-release.sh --host-only - version="$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version)" - jar tf "streamfusion-kafka/target/streamfusion-kafka-${version}.jar" | grep -q 'native/kafka/linux/x86_64/libstreamfusion_kafka.so' - jar tf "streamfusion-json/target/streamfusion-json-${version}.jar" | grep -q 'native/json/linux/x86_64/libstreamfusion_json.so' - jar tf "streamfusion-csv/target/streamfusion-csv-${version}.jar" | grep -q 'native/csv/linux/x86_64/libstreamfusion_csv.so' - jar tf "streamfusion-raw/target/streamfusion-raw-${version}.jar" | grep -q 'native/raw/linux/x86_64/libstreamfusion_raw.so' - jar tf "streamfusion-avro/target/streamfusion-avro-${version}.jar" | grep -q 'native/avro/linux/x86_64/libstreamfusion_avro.so' - if jar tf "streamfusion-avro-confluent-registry/target/streamfusion-avro-confluent-registry-${version}.jar" | grep -q 'libstreamfusion_avro'; then - exit 1 - fi - jar tf "streamfusion-protobuf/target/streamfusion-protobuf-${version}.jar" | grep -q 'native/protobuf/linux/x86_64/libstreamfusion_protobuf.so' - jar tf "streamfusion-parquet/target/streamfusion-parquet-${version}.jar" | grep -q 'native/parquet/linux/x86_64/libstreamfusion_parquet.so' - jar tf "streamfusion-orc/target/streamfusion-orc-${version}.jar" | grep -q 'native/orc/linux/x86_64/libstreamfusion_orc.so' - bin/check-artifacts.sh --host-only - bin/build-flink-image.sh --tag streamfusion-flink:image-it --load --skip-release-build + bin/build-release.sh --host-only --flink-line "$FLINK_LINE" + bin/check-artifacts.sh --host-only --flink-line "$FLINK_LINE" + bin/build-flink-image.sh --flink-line "$FLINK_LINE" --tag streamfusion-flink:image-it --load --skip-release-build - name: Run Flink Session-cluster image smoke test run: > - mvn -B -ntp -Pimage-it -pl :streamfusion-image-it verify + mvn -B -ntp -P${{ matrix.profile }} -pl streamfusion-image-it verify -Dstreamfusion.image.name=streamfusion-flink:image-it module-tests: @@ -223,6 +248,42 @@ jobs: - name: Parquet tag: streamfusion-parquet package: streamfusion-parquet + - name: Kafka (Flink 1.18) + tag: streamfusion-kafka + package: streamfusion-kafka + profile: '-Pflink-1.18' + - name: JSON (Flink 1.18) + tag: streamfusion-json + package: streamfusion-json + profile: '-Pflink-1.18' + - name: CSV (Flink 1.18) + tag: streamfusion-csv + package: streamfusion-csv + profile: '-Pflink-1.18' + - name: RAW (Flink 1.18) + tag: streamfusion-raw + package: streamfusion-raw + profile: '-Pflink-1.18' + - name: Avro (Flink 1.18) + tag: streamfusion-avro + package: streamfusion-avro + profile: '-Pflink-1.18' + - name: Avro Confluent Registry (Flink 1.18) + tag: streamfusion-avro-confluent + package: streamfusion-avro + profile: '-Pflink-1.18' + - name: Protobuf (Flink 1.18) + tag: streamfusion-protobuf + package: streamfusion-protobuf + profile: '-Pflink-1.18' + - name: ORC (Flink 1.18) + tag: streamfusion-orc + package: streamfusion-orc + profile: '-Pflink-1.18' + - name: Parquet (Flink 1.18) + tag: streamfusion-parquet + package: streamfusion-parquet + profile: '-Pflink-1.18' steps: - uses: actions/checkout@v6 @@ -249,13 +310,50 @@ jobs: - name: Run tagged Java module suite run: > - mvn -B -ntp -pl :streamfusion-runtime test + mvn -B -ntp ${{ matrix.profile }} -pl streamfusion-runtime test -Dgroups="${{ matrix.tag }}" + flink118-artifacts: + name: Flink 1.18 qualified artifacts and loader + runs-on: ubuntu-22.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v6 + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup + df -h / + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + cache: maven + - uses: dtolnay/rust-toolchain@1.94.0 + - uses: Swatinem/rust-cache@v2 + with: + key: ubuntu-22.04-glibc-2.35 + workspaces: native + - name: Build and validate qualified optimized payloads + run: | + bin/build-release.sh --host-only --flink-line 1.18 + bin/check-artifacts.sh --host-only --flink-line 1.18 + bin/package-release.sh --flink-line 1.18 + - name: Execute the packaged loader and mixed-line identity tests + run: > + mvn -B -ntp -Pflink-1.18,universal -pl streamfusion-loader -am test + -Dtest=FlinkPayloadIdentityTest,StreamFusionPlannerLoaderTest + -Dsurefire.failIfNoSpecifiedTests=false + + - uses: actions/upload-artifact@v4 + with: + name: flink1.18-optimized-linux-payloads + path: target/release/streamfusion-flink1.18-*-bin.tar.gz* + if-no-files-found: error + all-tests: name: All CI tests if: ${{ always() }} - needs: [native-tests, java-tests, lake-module-tests, image-integration, module-tests] + needs: [native-tests, java-tests, lake-module-tests, image-integration, module-tests, flink118-artifacts] runs-on: ubuntu-24.04 timeout-minutes: 5 steps: diff --git a/.github/workflows/flink-suite.yml b/.github/workflows/flink-suite.yml index 4868532da..f425052f6 100644 --- a/.github/workflows/flink-suite.yml +++ b/.github/workflows/flink-suite.yml @@ -17,13 +17,24 @@ concurrency: jobs: upstream-suite: - name: ${{ matrix.suite }} suite + name: ${{ matrix.suite }} suite${{ matrix.suffix }} runs-on: ubuntu-24.04 timeout-minutes: 180 strategy: fail-fast: false matrix: suite: [runtime, formats, parquet, orc, kafka, paimon, delta, state] + line: ['2.2', '1.18'] + exclude: + - suite: delta + line: '1.18' + include: + - line: '2.2' + version: '2.2.1' + suffix: '' + - line: '1.18' + version: '1.18.1' + suffix: ' (Flink 1.18)' steps: - uses: actions/checkout@v6 @@ -42,9 +53,11 @@ jobs: - uses: Swatinem/rust-cache@v2 with: - workspaces: native -> ../.flink-suite/streamfusion-source/native/target + workspaces: native -> ../.flink-suite/${{ matrix.line }}/streamfusion-source/native/target - name: Run unchanged upstream suite with StreamFusion + env: + FLINK_VERSION: ${{ matrix.version }} run: | set -o pipefail bin/flink-suite.sh "${{ matrix.suite }}" 2>&1 | tee upstream-suite.log @@ -52,11 +65,11 @@ jobs: - uses: actions/upload-artifact@v4 if: always() with: - name: upstream-flink-${{ matrix.suite }}-${{ github.sha }} + name: upstream-flink-${{ matrix.line }}-${{ matrix.suite }}-${{ github.sha }} path: | upstream-suite.log - .flink-suite/native-execution/ - .flink-suite/diagnostics/ + .flink-suite/**/native-execution/ + .flink-suite/**/diagnostics/ .flink-suite/**/target/surefire-reports/ include-hidden-files: true retention-days: 7 diff --git a/bin/build-flink-image.sh b/bin/build-flink-image.sh index 3be768c2f..dfa461ab1 100755 --- a/bin/build-flink-image.sh +++ b/bin/build-flink-image.sh @@ -12,7 +12,8 @@ Builds a job-neutral StreamFusion Flink base image. --push Build linux/amd64 and linux/arm64, then push a manifest list. --load Build one platform and load it into the local Docker daemon. --platform Platform for --load (default: Docker server platform). - --flink-image Flink base image (default: flink:2.2.1-scala_2.12-java17). + --flink-line Payload line: 2.2 (default) or 1.18. + --flink-image Override the official image selected for the payload line. --skip-release-build Reuse the already-built StreamFusion JARs. EOF exit 64 @@ -20,7 +21,8 @@ EOF script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) repo_root=$(cd "$script_dir/.." && pwd) -flink_image=flink:2.2.1-scala_2.12-java17 +flink_image= +flink_line=2.2 image_tag= mode= platform= @@ -43,6 +45,11 @@ while [ "$#" -gt 0 ]; do platform=$2 shift 2 ;; + --flink-line) + [ "$#" -ge 2 ] || usage + flink_line=$2 + shift 2 + ;; --flink-image) [ "$#" -ge 2 ] || usage flink_image=$2 @@ -60,6 +67,14 @@ done [ -n "$image_tag" ] && [ -n "$mode" ] || usage +artifact_suffix= +case "$flink_line" in + 2.2) default_flink_image=flink:2.2.1-scala_2.12-java17 ;; + 1.18) artifact_suffix=-flink1.18; default_flink_image=flink:1.18.1-scala_2.12-java17 ;; + *) echo "unsupported Flink line: $flink_line" >&2; exit 64 ;; +esac +flink_image=${flink_image:-$default_flink_image} + command -v docker >/dev/null 2>&1 || { echo "Docker with buildx is required." >&2 exit 69 @@ -67,12 +82,12 @@ command -v docker >/dev/null 2>&1 || { docker buildx version >/dev/null if [ "$skip_release_build" = false ]; then - "$repo_root/bin/build-release.sh" --linux-only + "$repo_root/bin/build-release.sh" --linux-only --flink-line "$flink_line" fi artifact_version=$(cd "$repo_root" && mvn -q -DforceStdout help:evaluate -Dexpression=project.version) -loader_jar=$repo_root/streamfusion-loader/target/streamfusion-loader-$artifact_version.jar -core_jar=$repo_root/streamfusion-core/target/streamfusion-core-$artifact_version-runtime.jar +loader_jar=$repo_root/streamfusion-loader/target/streamfusion-loader$artifact_suffix-$artifact_version.jar +core_jar=$repo_root/streamfusion-core/target/streamfusion-core$artifact_suffix-$artifact_version-runtime.jar [ -f "$loader_jar" ] && [ -f "$core_jar" ] || { echo "StreamFusion release JARs are missing; run bin/build-release.sh first." >&2 exit 66 @@ -104,6 +119,8 @@ docker buildx build \ --platform "$platforms" \ --build-arg "FLINK_IMAGE=$flink_image" \ --build-arg "STREAMFUSION_VERSION=$artifact_version" \ + --build-arg "STREAMFUSION_ARTIFACT_SUFFIX=$artifact_suffix" \ + --build-arg "FLINK_LINE=$flink_line" \ --tag "$image_tag" \ --file "$repo_root/docker/flink-base.Dockerfile" \ "$output" \ diff --git a/bin/build-release.sh b/bin/build-release.sh index e9189b55d..69cbacfe6 100755 --- a/bin/build-release.sh +++ b/bin/build-release.sh @@ -2,23 +2,33 @@ set -eu -if [ "$#" -gt 1 ] || { [ "$#" -eq 1 ] && [ "$1" != "--host-only" ] && [ "$1" != "--linux-only" ]; }; then - echo "usage: $0 [--host-only | --linux-only]" >&2 +host_only=false +linux_only=false +flink_line=2.2 +while [ "$#" -gt 0 ]; do + case "$1" in + --host-only) host_only=true; shift ;; + --linux-only) linux_only=true; shift ;; + --flink-line) + if [ "$#" -lt 2 ]; then echo "--flink-line requires 2.2 or 1.18" >&2; exit 64; fi + flink_line=$2; shift 2 ;; + *) echo "usage: $0 [--host-only | --linux-only] [--flink-line 2.2|1.18]" >&2; exit 64 ;; + esac +done +if [ "$host_only" = true ] && [ "$linux_only" = true ]; then + echo "--host-only and --linux-only are mutually exclusive" >&2 exit 64 fi +case "$flink_line" in + 2.2) maven_profiles=dist,universal,release,delta,paimon ;; + 1.18) maven_profiles=dist,universal,release,paimon,flink-1.18 ;; + *) echo "unsupported Flink line: $flink_line" >&2; exit 64 ;; +esac script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) repo_root=$(cd "$script_dir/.." && pwd) native_dir=$repo_root/native stage_dir=$native_dir/target/universal -host_only=false -linux_only=false - -if [ "$#" -eq 1 ] && [ "$1" = "--host-only" ]; then - host_only=true -elif [ "$#" -eq 1 ]; then - linux_only=true -fi host_platform() { case "$(uname -s)" in @@ -203,4 +213,4 @@ fi # platform build. A release always starts from empty Java output directories. The release profile # builds the same source and javadoc attachments as the publish workflow, unsigned, so attachment # failures surface here instead of on the release runner. -(cd "$repo_root" && mvn clean package -Pdist,universal,release,delta,paimon -Dgpg.skip=true -DskipTests) +(cd "$repo_root" && mvn clean package "-P$maven_profiles" -Dgpg.skip=true -DskipTests) diff --git a/bin/check-artifacts.sh b/bin/check-artifacts.sh index 05fac64d5..7aa4e1ad5 100755 --- a/bin/check-artifacts.sh +++ b/bin/check-artifacts.sh @@ -2,14 +2,25 @@ set -eu -if [ "$#" -gt 1 ] || { [ "$#" -eq 1 ] && [ "$1" != "--host-only" ]; }; then - echo "usage: $0 [--host-only]" >&2 - exit 64 -fi - host_only=false -if [ "$#" -eq 1 ]; then - host_only=true +flink_line=2.2 +while [ "$#" -gt 0 ]; do + case "$1" in + --host-only) host_only=true; shift ;; + --flink-line) + if [ "$#" -lt 2 ]; then echo "--flink-line requires 2.2 or 1.18" >&2; exit 64; fi + flink_line=$2; shift 2 ;; + *) echo "usage: $0 [--host-only] [--flink-line 2.2|1.18]" >&2; exit 64 ;; + esac +done +artifact_suffix="" +case "$flink_line" in + 2.2) set -- ;; + 1.18) artifact_suffix=-flink1.18; set -- -Pflink-1.18 ;; + *) echo "unsupported Flink line: $flink_line" >&2; exit 64 ;; +esac + +if [ "$host_only" = true ]; then case "$(uname -s)" in Linux) host_platform=linux; host_extension=so ;; Darwin) host_platform=darwin; host_extension=dylib ;; @@ -24,9 +35,9 @@ fi script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) repo_root=$(cd "$script_dir/.." && pwd) -version=$(cd "$repo_root" && mvn -q -DforceStdout help:evaluate -Dexpression=project.version) -flink_line=$(cd "$repo_root" && mvn -q -DforceStdout help:evaluate -Dexpression=flink.line) -modules="core kafka json csv raw avro avro-confluent-registry protobuf parquet orc delta paimon" +version=$(cd "$repo_root" && mvn "$@" -q -DforceStdout help:evaluate -Dexpression=project.version) +modules="core kafka json csv raw avro avro-confluent-registry protobuf parquet orc paimon" +if [ "$flink_line" = 2.2 ]; then modules="$modules delta"; fi entries=$(mktemp) native_entries=$(mktemp) expected_native_entries=$(mktemp) @@ -69,6 +80,10 @@ assert_native_payload() { diff -u "$expected_native_entries" "$native_entries" >&2 || true exit 1 fi + if [ "$(uname -s)" = Linux ]; then + python3 "$script_dir/check-native-glibc.py" "$jar_file" + python3 "$script_dir/check-native-tls.py" "$jar_file" + fi } assert_no_native_payload() { @@ -82,22 +97,19 @@ assert_no_native_payload() { for suffix in $modules; do module="streamfusion-$suffix" + artifact="${module}${artifact_suffix}" if [ "$suffix" = core ]; then - jar_file="$repo_root/$module/target/$module-$version-runtime.jar" + jar_file="$repo_root/$module/target/$artifact-$version-runtime.jar" else - jar_file="$repo_root/$module/target/$module-$version.jar" + jar_file="$repo_root/$module/target/$artifact-$version.jar" fi if [ ! -f "$jar_file" ]; then echo "missing artifact: $jar_file" >&2 exit 1 fi - assert_flink_identity "$jar_file" "$module" - if [ "$(uname -s)" = Linux ]; then - python3 "$script_dir/check-native-tls.py" "$jar_file" - python3 "$script_dir/check-native-glibc.py" "$jar_file" - fi + assert_flink_identity "$jar_file" "$artifact" if jar tf "$jar_file" | grep -q '^org/slf4j/'; then - echo "$module must use Flink's logging API instead of bundling SLF4J" >&2 + echo "$artifact must use Flink's logging API instead of bundling SLF4J" >&2 exit 1 fi jar tf "$jar_file" | awk -v module="$module" \ @@ -109,9 +121,9 @@ for suffix in $modules; do fi done -core_jar="$repo_root/streamfusion-core/target/streamfusion-core-$version-runtime.jar" -core_main_jar="$repo_root/streamfusion-core/target/streamfusion-core-$version.jar" -assert_flink_identity "$core_main_jar" streamfusion-core +core_jar="$repo_root/streamfusion-core/target/streamfusion-core${artifact_suffix}-$version-runtime.jar" +core_main_jar="$repo_root/streamfusion-core/target/streamfusion-core${artifact_suffix}-$version.jar" +assert_flink_identity "$core_main_jar" "streamfusion-core${artifact_suffix}" assert_native_payload "$core_main_jar" streamfusion-core libstreamfusion "" assert_native_payload "$core_jar" streamfusion-core libstreamfusion "" if jar tf "$core_jar" | grep -Eq '^tech/streamfusion/(kafka|parquet|orc|delta|paimon|format/(json|csv|raw|avro|avroconfluent|protobuf))/'; then @@ -121,30 +133,34 @@ fi for suffix in kafka json csv raw avro protobuf parquet orc paimon; do assert_native_payload \ - "$repo_root/streamfusion-$suffix/target/streamfusion-$suffix-$version.jar" \ + "$repo_root/streamfusion-$suffix/target/streamfusion-$suffix${artifact_suffix}-$version.jar" \ "streamfusion-$suffix" "libstreamfusion_$suffix" "$suffix" done assert_no_native_payload \ - "$repo_root/streamfusion-runtime/target/streamfusion-runtime-$version.jar" \ + "$repo_root/streamfusion-runtime/target/streamfusion-runtime${artifact_suffix}-$version.jar" \ streamfusion-runtime assert_flink_identity \ - "$repo_root/streamfusion-runtime/target/streamfusion-runtime-$version.jar" \ - streamfusion-runtime + "$repo_root/streamfusion-runtime/target/streamfusion-runtime${artifact_suffix}-$version.jar" \ + "streamfusion-runtime${artifact_suffix}" assert_no_native_payload \ - "$repo_root/streamfusion-avro-confluent-registry/target/streamfusion-avro-confluent-registry-$version.jar" \ + "$repo_root/streamfusion-avro-confluent-registry/target/streamfusion-avro-confluent-registry${artifact_suffix}-$version.jar" \ streamfusion-avro-confluent-registry -assert_no_native_payload \ - "$repo_root/streamfusion-delta/target/streamfusion-delta-$version.jar" \ - streamfusion-delta +if [ "$flink_line" = 2.2 ]; then + assert_no_native_payload \ + "$repo_root/streamfusion-delta/target/streamfusion-delta-$version.jar" streamfusion-delta +elif [ -f "$repo_root/streamfusion-delta/target/streamfusion-delta${artifact_suffix}-$version.jar" ]; then + echo "Delta has no verified Flink 1.18 deployment artifact" >&2 + exit 1 +fi -loader_jar="$repo_root/streamfusion-loader/target/streamfusion-loader-$version.jar" +loader_jar="$repo_root/streamfusion-loader/target/streamfusion-loader${artifact_suffix}-$version.jar" if [ ! -f "$loader_jar" ]; then echo "missing artifact: $loader_jar" >&2 exit 1 fi assert_no_native_payload "$loader_jar" streamfusion-loader -assert_flink_identity "$loader_jar" streamfusion-loader +assert_flink_identity "$loader_jar" "streamfusion-loader${artifact_suffix}" if ! unzip -p "$loader_jar" streamfusion-planner.jar | cmp -s - "$core_jar"; then echo "streamfusion-loader does not embed the exact core runtime payload" >&2 exit 1 @@ -157,10 +173,51 @@ if [ -n "$duplicates" ]; then exit 1 fi -confluent_jar="$repo_root/streamfusion-avro-confluent-registry/target/streamfusion-avro-confluent-registry-$version.jar" +confluent_jar="$repo_root/streamfusion-avro-confluent-registry/target/streamfusion-avro-confluent-registry${artifact_suffix}-$version.jar" if jar tf "$confluent_jar" | grep -q 'libstreamfusion_avro'; then echo "the Confluent integration duplicates streamfusion-avro's native library" >&2 exit 1 fi -echo "StreamFusion artifact boundaries are clean for $version" +python3 - "$repo_root" "$version" "$flink_line" "$artifact_suffix" "$modules" <<'PYTHON' +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +root, version, line, suffix, modules = sys.argv[1:] +namespace = {"p": "http://maven.apache.org/POM/4.0.0"} +for module in [*modules.split(), "runtime", "loader"]: + path = Path(root) / f"streamfusion-{module}" / ".flattened-pom.xml" + pom = ET.parse(path).getroot() + def value(element, name): + return element.findtext(f"p:{name}", namespaces=namespace) + identity = (value(pom, "groupId"), value(pom, "artifactId"), value(pom, "version")) + expected = ("tech.streamfusion", f"streamfusion-{module}{suffix}", version) + if identity != expected or "${" in path.read_text(): + raise SystemExit(f"{path}: unresolved or wrong artifact identity {identity}; expected {expected}") + dependencies = { + (value(dep, "groupId"), value(dep, "artifactId")): dep + for dep in pom.findall("p:dependencies/p:dependency", namespace) + } + if module != "loader": + for artifact, scope in [("arrow-vector", "compile"), ("arrow-c-data", "compile"), + ("arrow-memory-unsafe", "runtime")]: + dep = dependencies.get(("org.apache.arrow", artifact)) + if dep is None or (value(dep, "scope") or "compile") != scope: + raise SystemExit(f"{path}: missing inherited {artifact} dependency in {scope} scope") + for (group, artifact), dep in dependencies.items(): + dep_version = value(dep, "version") + if group == "tech.streamfusion" and artifact.startswith("streamfusion-"): + actual_suffix = "-flink1.18" if artifact.endswith("-flink1.18") else "" + if actual_suffix != suffix or dep_version != version: + raise SystemExit(f"{path}: mismatched StreamFusion dependency {artifact}:{dep_version}") + if group == "org.apache.flink" and not artifact.startswith("flink-shaded-"): + expected_version = line + "." + if artifact == "flink-connector-kafka": + if not dep_version.endswith("-" + line): + raise SystemExit(f"{path}: mismatched Kafka connector line {dep_version}") + elif not dep_version.startswith(expected_version): + raise SystemExit(f"{path}: mismatched Flink dependency {artifact}:{dep_version}") +PYTHON + +echo "StreamFusion artifact boundaries and consumer POMs are clean for $version / Flink $flink_line" diff --git a/bin/check-flink-suite-classpath.py b/bin/check-flink-suite-classpath.py new file mode 100644 index 000000000..072e9f4ce --- /dev/null +++ b/bin/check-flink-suite-classpath.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Reject a source-suite classpath containing payloads from another Flink line.""" + +import argparse +import os +from pathlib import Path +import zipfile + + +def validate(classpath, line): + if line not in {"1.18", "2.2"}: + raise ValueError(f"Unsupported Flink line: {line}") + suffix = "-flink1.18" if line == "1.18" else "" + modules = set() + for item in classpath.strip().split(os.pathsep): + path = Path(item) + with zipfile.ZipFile(path) as archive: + try: + manifest = archive.read("META-INF/MANIFEST.MF").decode("utf-8") + except KeyError: + manifest = "" + manifest = manifest.replace("\r\n", "\n").replace("\n ", "") + fields = dict(entry.split(": ", 1) for entry in manifest.splitlines() if ": " in entry) + module = fields.get("StreamFusion-Module") + actual_line = fields.get("StreamFusion-Flink-Line") + if not module and not actual_line and not path.name.startswith("streamfusion-"): + continue + if ( + actual_line != line + or not module + or not module.startswith("streamfusion-") + or (line == "1.18" and not module.endswith(suffix)) + or (line == "2.2" and "-flink" in module) + ): + raise ValueError( + f"{path}: payload identity {module!r} / Flink {actual_line!r}, expected Flink {line}" + ) + if module in modules: + raise ValueError(f"Duplicate StreamFusion payload: {module}") + modules.add(module) + core = "streamfusion-core" + suffix + if core not in modules: + raise ValueError(f"Suite classpath is missing {core}") + return modules + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("classpath", type=Path) + parser.add_argument("line", choices=["1.18", "2.2"]) + args = parser.parse_args() + try: + modules = validate(args.classpath.read_text(), args.line) + except (OSError, ValueError, zipfile.BadZipFile) as error: + parser.exit(2, f"Cannot use the Flink suite classpath: {error}\n") + print(f"Verified {len(modules)} source-suite payloads for Flink {args.line}") + + +if __name__ == "__main__": + main() diff --git a/bin/flink-suite.sh b/bin/flink-suite.sh index 191f52bad..47ba1b33c 100755 --- a/bin/flink-suite.sh +++ b/bin/flink-suite.sh @@ -5,23 +5,52 @@ set -uo pipefail readonly REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" readonly FLINK_VERSION="${FLINK_VERSION:-2.2.1}" readonly FLINK_TAG="release-${FLINK_VERSION}" -readonly KAFKA_CONNECTOR_VERSION="${KAFKA_CONNECTOR_VERSION:-5.0.0}" -readonly KAFKA_CONNECTOR_TAG="v${KAFKA_CONNECTOR_VERSION}" +case "${FLINK_VERSION}" in + 2.2.0|2.2.1) + FLINK_LINE=2.2 + STREAMFUSION_LINE_PROFILES=() + STREAMFUSION_ARTIFACT_SUFFIX="" + KAFKA_DEFAULT_VERSION=5.0.0 + PAIMON_FLINK_PROFILE=flink2 + ;; + 1.18.1) + FLINK_LINE=1.18 + STREAMFUSION_LINE_PROFILES=(-Pflink-1.18) + STREAMFUSION_ARTIFACT_SUFFIX=-flink1.18 + KAFKA_DEFAULT_VERSION=3.2.0 + PAIMON_FLINK_PROFILE=flink1 + ;; + *) echo "Unsupported Flink suite version: ${FLINK_VERSION}" >&2; exit 2 ;; +esac +readonly FLINK_LINE STREAMFUSION_ARTIFACT_SUFFIX KAFKA_DEFAULT_VERSION PAIMON_FLINK_PROFILE +readonly KAFKA_CONNECTOR_VERSION="${KAFKA_CONNECTOR_VERSION:-${KAFKA_DEFAULT_VERSION}}" +# The published 3.2.0 source archive matches the final candidate; no v3.2.0 tag exists. +if [[ "${KAFKA_CONNECTOR_VERSION}" == "3.2.0" ]]; then + KAFKA_CONNECTOR_TAG=v3.2.0-rc1 +else + KAFKA_CONNECTOR_TAG="v${KAFKA_CONNECTOR_VERSION}" +fi +readonly KAFKA_CONNECTOR_TAG readonly PAIMON_VERSION="${PAIMON_VERSION:-2.0.0}" # Paimon publishes its releases from the final release-candidate tag; 2.0.0 is release-2.0.0-rc10. readonly PAIMON_TAG="${PAIMON_TAG:-release-${PAIMON_VERSION}-rc10}" readonly DELTA_VERSION="${DELTA_VERSION:-4.4.0}" -readonly SUITE_ROOT="${FLINK_SUITE_ROOT:-${REPO_ROOT}/.flink-suite}" +readonly SUITE_BASE_ROOT="${FLINK_SUITE_ROOT:-${REPO_ROOT}/.flink-suite}" +readonly SUITE_ROOT="${SUITE_BASE_ROOT}/${FLINK_LINE}" readonly FLINK_ROOT="${SUITE_ROOT}/flink-${FLINK_VERSION}" readonly KAFKA_CONNECTOR_ROOT="${SUITE_ROOT}/flink-connector-kafka-${KAFKA_CONNECTOR_VERSION}" readonly PAIMON_ROOT="${SUITE_ROOT}/paimon-${PAIMON_VERSION}" readonly PAIMON_MODULE="paimon-flink/paimon-flink-common" +readonly PAIMON_LINE_MODULE="paimon-flink/paimon-flink-${FLINK_LINE}" +readonly PAIMON_118_SHARED_TESTS="${REPO_ROOT}/dev/flink-suite/paimon-flink118-shared-tests.txt" +readonly PAIMON_118_LINE_CLASSES="org.apache.paimon.flink.AppendTableITCase,org.apache.paimon.flink.ContinuousFileStoreITCase,org.apache.paimon.flink.RemoveOrphanFilesActionITCase,org.apache.paimon.flink.procedure.ProcedurePositionalArgumentsITCase,org.apache.paimon.flink.iceberg.Flink118IcebergITCase" readonly DELTA_ROOT="${SUITE_ROOT}/delta-${DELTA_VERSION}" readonly DELTA_TEST_POM="${REPO_ROOT}/dev/flink-suite/delta/pom.xml" readonly DELTA_TEST_OUTPUT="${SUITE_ROOT}/delta-tests/target" readonly STREAMFUSION_BUILD_ROOT="${SUITE_ROOT}/streamfusion-source" readonly AGENT_ROOT="${REPO_ROOT}/dev/flink-suite/agent" -readonly AGENT_JAR="${AGENT_ROOT}/target/streamfusion-flink-suite-agent-1.0-SNAPSHOT.jar" +readonly AGENT_OUTPUT="${SUITE_ROOT}/agent/target" +readonly AGENT_JAR="${AGENT_OUTPUT}/streamfusion-flink-suite-agent-1.0-SNAPSHOT.jar" readonly CLASSPATH_FILE="${SUITE_ROOT}/streamfusion-classpath.txt" readonly MAVEN_SETTINGS="${REPO_ROOT}/dev/flink-suite/settings.xml" readonly SUITE_MAVEN_REPO="${SUITE_ROOT}/m2" @@ -32,11 +61,22 @@ readonly UNSHADED_BRIDGE_POM="${SUITE_ROOT}/flink-table-calcite-bridge-${FLINK_V readonly UNSHADED_SQL_PARSER_JAR="${SUITE_ROOT}/flink-sql-parser-${FLINK_VERSION}-unshaded.jar" readonly UNSHADED_SQL_PARSER_POM="${SUITE_ROOT}/flink-sql-parser-${FLINK_VERSION}-effective.pom" readonly SUITE_MODE="${1:-runtime}" +CONTRACT_SUFFIX="${STREAMFUSION_ARTIFACT_SUFFIX}" +NATIVE_STATE_SUITE=false +if [[ "${SUITE_MODE}" == "state" ]]; then NATIVE_STATE_SUITE=true; fi +if [[ "${FLINK_LINE}" == "1.18" && "${SUITE_MODE}" == "state" ]]; then + CONTRACT_SUFFIX="${CONTRACT_SUFFIX}-state" +fi +readonly CONTRACT_FILE="${AGENT_ROOT}/src/main/resources/native-execution${CONTRACT_SUFFIX}.tsv" readonly NATIVE_REPORT_ROOT="${SUITE_ROOT}/native-execution/${SUITE_MODE}" readonly DIAGNOSTIC_ROOT="${SUITE_ROOT}/diagnostics/${SUITE_MODE}" -readonly FLINK_MODULE_CONFIG="-Duser.timezone=UTC -Djava.library.path=${STREAMFUSION_BUILD_ROOT}/native/target/debug --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED -Djunit.platform.reflection.search.useLegacySemantics=true -javaagent:${AGENT_JAR}" +readonly FLINK_MODULE_CONFIG="-Dstreamfusion.flink-suite.native-rocksdb=${NATIVE_STATE_SUITE} -Dstreamfusion.flink-suite.flink-line=${FLINK_LINE} -Duser.timezone=UTC -Djava.library.path=${STREAMFUSION_BUILD_ROOT}/native/target/debug --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED -Djunit.platform.reflection.search.useLegacySemantics=true -javaagent:${AGENT_JAR}" readonly CONNECTOR_MODULE_CONFIG="-XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false -Dio.netty.tryReflectionSetAccessible=true ${FLINK_MODULE_CONFIG}" -readonly PAIMON_BUILD_ARGS=(-Pflink2 "-Dpaimon-flink-common.flink.version=${FLINK_VERSION}" "-Dtest.flink.version=${FLINK_VERSION}" -Dspotless.check.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dmaven.javadoc.skip=true) +# Paimon 2.0 compiles its Flink-1 shared sources against 1.20.1, including APIs +# guarded by version adapters. Execute the compiled tests on the requested host API. +PAIMON_COMPILE_VERSION="${FLINK_VERSION}" +if [[ "${FLINK_LINE}" == "1.18" ]]; then PAIMON_COMPILE_VERSION=1.20.1; fi +readonly PAIMON_BUILD_ARGS=("-P${PAIMON_FLINK_PROFILE}" "-Dtest.flink.main.version=${FLINK_LINE}" "-Dpaimon-flink-common.flink.version=${PAIMON_COMPILE_VERSION}" "-Dtest.flink.version=${FLINK_VERSION}" -Dspotless.check.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dmaven.javadoc.skip=true) readonly FORMAT_MODULES="flink-formats/flink-json,flink-formats/flink-csv,flink-formats/flink-avro,flink-formats/flink-avro-confluent-registry,flink-formats/flink-protobuf" readonly ORC_MODULE="flink-formats/flink-orc" readonly ORC_SQL_TESTS="org.apache.flink.orc.OrcFsStreamingSinkITCase,org.apache.flink.orc.OrcFileSystemITCase" @@ -56,6 +96,15 @@ if [[ -n "${FLINK_SUITE_TEST:-}" ]]; then fi case "${SUITE_MODE}" in + config) + printf '%s\n' "flink.version=${FLINK_VERSION}" "flink.line=${FLINK_LINE}" \ + "kafka.version=${KAFKA_CONNECTOR_VERSION}" "kafka.tag=${KAFKA_CONNECTOR_TAG}" \ + "paimon.profile=${PAIMON_FLINK_PROFILE}" \ + "suite.root=${SUITE_ROOT}" "streamfusion.source=${STREAMFUSION_BUILD_ROOT}" \ + "maven.repo=${SUITE_MAVEN_REPO}" "agent.jar=${AGENT_JAR}" \ + "classpath=${CLASSPATH_FILE}" "contracts=${CONTRACT_FILE}" + exit 0 + ;; runtime) TEST_GOAL="surefire:test@integration-tests" TEST_MODULES="flink-table/flink-table-planner" @@ -112,13 +161,21 @@ case "${SUITE_MODE}" in REPORT_ROOT="${PAIMON_ROOT}/${PAIMON_MODULE}/target/surefire-reports" if [[ -z "${FLINK_SUITE_TEST:-}" ]]; then paimon_selector="${PAIMON_SQL_TESTS}" - for isolated_test in "${PAIMON_ISOLATED_TESTS[@]}"; do - paimon_selector+=",!${isolated_test}" - done + if [[ "${FLINK_LINE}" == "1.18" ]]; then + paimon_selector="$(paste -sd, "${PAIMON_118_SHARED_TESTS}")" + else + for isolated_test in "${PAIMON_ISOLATED_TESTS[@]}"; do + paimon_selector+=",!${isolated_test}" + done + fi TEST_SELECTOR_ARGS=("-Dtest=${paimon_selector}") fi ;; delta) + if [[ "${FLINK_LINE}" != "2.2" ]]; then + echo "Delta acceleration is not admitted on Flink ${FLINK_LINE}; no suite payload is available." >&2 + exit 2 + fi TEST_GOAL="test" TEST_MODULES=":streamfusion-upstream-delta-tests" REPORT_ROOT="${DELTA_TEST_OUTPUT}/surefire-reports" @@ -128,17 +185,30 @@ case "${SUITE_MODE}" in FLINK_SUITE_REUSE_BUILD=true "${BASH_SOURCE[0]}" parquet || exit $? FLINK_SUITE_REUSE_BUILD=true "${BASH_SOURCE[0]}" orc || exit $? FLINK_SUITE_REUSE_BUILD=true "${BASH_SOURCE[0]}" runtime || exit $? + FLINK_SUITE_REUSE_BUILD=true "${BASH_SOURCE[0]}" state || exit $? "${BASH_SOURCE[0]}" paimon || exit $? - "${BASH_SOURCE[0]}" delta || exit $? + if [[ "${FLINK_LINE}" == "2.2" ]]; then + "${BASH_SOURCE[0]}" delta || exit $? + fi "${BASH_SOURCE[0]}" kafka exit $? ;; *) - echo "Usage: $0 [runtime|diagnostic|state|formats|parquet|orc|kafka|paimon|delta|all]" >&2 + echo "Usage: $0 [config|runtime|diagnostic|state|formats|parquet|orc|kafka|paimon|delta|all]" >&2 exit 2 ;; esac +flink_mvn() { + (cd "${FLINK_ROOT}" && ./mvnw "$@") +} + +kafka_mvn() { + (cd "${KAFKA_CONNECTOR_ROOT}" && { + if [[ -x ./mvnw ]]; then ./mvnw "$@"; else mvn "$@"; fi + }) +} + mkdir -p "${SUITE_ROOT}" if [[ ! -d "${FLINK_ROOT}/.git" ]]; then git clone --depth 1 --branch "${FLINK_TAG}" https://github.com/apache/flink.git "${FLINK_ROOT}" || exit $? @@ -221,20 +291,25 @@ if [[ "${FLINK_SUITE_REUSE_BUILD:-false}" == "true" ]]; then echo "Cannot reuse the Paimon-suite build; run bin/flink-suite.sh paimon once without FLINK_SUITE_REUSE_BUILD." >&2 exit 2 fi + if [[ "${SUITE_MODE}" == "paimon" && "${FLINK_LINE}" == "1.18" ]] \ + && [[ ! -f "${PAIMON_ROOT}/${PAIMON_LINE_MODULE}/target/test-classes/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.class" ]]; then + echo "Cannot reuse the Paimon 1.18 suite before its version-specific tests have been compiled." >&2 + exit 2 + fi echo "Reusing the existing Flink suite and StreamFusion build artifacts..." else echo "Building the test-JVM planner injection agent..." - mvn -B -ntp -s "${MAVEN_SETTINGS}" -f "${AGENT_ROOT}/pom.xml" package || exit $? + mvn -B -ntp -s "${MAVEN_SETTINGS}" -f "${AGENT_ROOT}/pom.xml" "-Dsuite.agent.output=${AGENT_OUTPUT}" package || exit $? echo "Building the pinned Flink planner and its reactor dependencies..." - "${FLINK_ROOT}/mvnw" -B -ntp -s "${MAVEN_SETTINGS}" -f "${FLINK_ROOT}/pom.xml" \ + flink_mvn -B -ntp -s "${MAVEN_SETTINGS}" -f "${FLINK_ROOT}/pom.xml" \ -Dmaven.repo.local="${SUITE_MAVEN_REPO}" \ -pl flink-table/flink-table-planner -am -DskipTests -Dfast install || exit $? echo "Installing the untouched planner classes for StreamFusion's source-suite build..." jar --create --file "${UNSHADED_SQL_PARSER_JAR}" \ -C "${FLINK_ROOT}/flink-table/flink-sql-parser/target/classes" . || exit $? - "${FLINK_ROOT}/mvnw" -B -ntp -s "${MAVEN_SETTINGS}" -f "${FLINK_ROOT}/pom.xml" \ + flink_mvn -B -ntp -s "${MAVEN_SETTINGS}" -f "${FLINK_ROOT}/pom.xml" \ -Dmaven.repo.local="${SUITE_MAVEN_REPO}" -Didea.version=streamfusion-suite \ -pl flink-table/flink-sql-parser \ help:effective-pom -Doutput="${UNSHADED_SQL_PARSER_POM}" || exit $? @@ -244,7 +319,7 @@ else -DpomFile="${UNSHADED_SQL_PARSER_POM}" || exit $? jar --create --file "${UNSHADED_BRIDGE_JAR}" \ -C "${FLINK_ROOT}/flink-table/flink-table-calcite-bridge/target/classes" . || exit $? - "${FLINK_ROOT}/mvnw" -B -ntp -s "${MAVEN_SETTINGS}" -f "${FLINK_ROOT}/pom.xml" \ + flink_mvn -B -ntp -s "${MAVEN_SETTINGS}" -f "${FLINK_ROOT}/pom.xml" \ -Dmaven.repo.local="${SUITE_MAVEN_REPO}" -Didea.version=streamfusion-suite \ -pl flink-table/flink-table-calcite-bridge \ help:effective-pom -Doutput="${UNSHADED_BRIDGE_POM}" || exit $? @@ -254,7 +329,7 @@ else -DpomFile="${UNSHADED_BRIDGE_POM}" || exit $? jar --create --file "${UNSHADED_PLANNER_JAR}" \ -C "${FLINK_ROOT}/flink-table/flink-table-planner/target/classes" . || exit $? - "${FLINK_ROOT}/mvnw" -B -ntp -s "${MAVEN_SETTINGS}" -f "${FLINK_ROOT}/pom.xml" \ + flink_mvn -B -ntp -s "${MAVEN_SETTINGS}" -f "${FLINK_ROOT}/pom.xml" \ -Dmaven.repo.local="${SUITE_MAVEN_REPO}" -Didea.version=streamfusion-suite \ -pl flink-table/flink-table-planner \ help:effective-pom -Doutput="${UNSHADED_PLANNER_POM}" || exit $? @@ -282,23 +357,24 @@ else echo "Building and installing StreamFusion and its supported connector/format modules against the source-suite planner..." streamfusion_profiles="paimon" - streamfusion_modules=":streamfusion-core,:streamfusion-kafka,:streamfusion-json,:streamfusion-csv,:streamfusion-raw,:streamfusion-avro,:streamfusion-avro-confluent-registry,:streamfusion-protobuf,:streamfusion-parquet,:streamfusion-orc,:streamfusion-paimon" + streamfusion_modules="streamfusion-core,streamfusion-kafka,streamfusion-json,streamfusion-csv,streamfusion-raw,streamfusion-avro,streamfusion-avro-confluent-registry,streamfusion-protobuf,streamfusion-parquet,streamfusion-orc,streamfusion-paimon" if [[ "${SUITE_MODE}" == "delta" ]]; then streamfusion_profiles+=",delta" - streamfusion_modules+=",:streamfusion-delta" + streamfusion_modules+=",streamfusion-delta" fi mvn -B -ntp -s "${MAVEN_SETTINGS}" -Dmaven.repo.local="${SUITE_MAVEN_REPO}" \ - -Dstreamfusion.flink-source-suite "-P${streamfusion_profiles}" -Dnative.build.skip=true \ + -Dstreamfusion.flink-source-suite "-P${streamfusion_profiles}" "${STREAMFUSION_LINE_PROFILES[@]}" \ + "-Dflink.version=${FLINK_VERSION}" -Dnative.build.skip=true \ -f "${STREAMFUSION_BUILD_ROOT}/pom.xml" \ -pl "${streamfusion_modules}" \ -am -DskipTests clean install || exit $? mvn -B -ntp -s "${MAVEN_SETTINGS}" -Dmaven.repo.local="${SUITE_MAVEN_REPO}" \ - -f "${REPO_ROOT}/dev/flink-suite/classpath-pom.xml" \ + -f "${REPO_ROOT}/dev/flink-suite/classpath-pom.xml" "${STREAMFUSION_LINE_PROFILES[@]}" \ dependency:build-classpath -Dmdep.outputFile="${CLASSPATH_FILE}" || exit $? if [[ "${SUITE_MODE}" == "formats" || "${SUITE_MODE}" == "parquet" || "${SUITE_MODE}" == "orc" ]]; then echo "Compiling the untouched upstream Flink format integration tests..." - "${FLINK_ROOT}/mvnw" -B -ntp -s "${MAVEN_SETTINGS}" -f "${FLINK_ROOT}/pom.xml" \ + flink_mvn -B -ntp -s "${MAVEN_SETTINGS}" -f "${FLINK_ROOT}/pom.xml" \ -Dmaven.repo.local="${SUITE_MAVEN_REPO}" -Didea.version=streamfusion-suite \ -pl "${FORMAT_COMPILE_MODULES}" \ -am -Dfast -DskipTests process-test-classes || exit $? @@ -306,7 +382,7 @@ else if [[ "${SUITE_MODE}" == "kafka" ]]; then echo "Compiling the untouched upstream Kafka connector SQL integration tests..." - "${KAFKA_CONNECTOR_ROOT}/mvnw" -B -ntp -s "${MAVEN_SETTINGS}" \ + kafka_mvn -B -ntp -s "${MAVEN_SETTINGS}" \ -f "${KAFKA_CONNECTOR_ROOT}/pom.xml" -Dmaven.repo.local="${SUITE_MAVEN_REPO}" \ -Dflink.version="${FLINK_VERSION}" -pl flink-connector-kafka \ -DskipTests test-compile || exit $? @@ -317,26 +393,49 @@ else mvn -B -ntp -s "${MAVEN_SETTINGS}" -f "${PAIMON_ROOT}/pom.xml" \ -Dmaven.repo.local="${SUITE_MAVEN_REPO}" "${PAIMON_BUILD_ARGS[@]}" \ -pl "${PAIMON_MODULE}" -am -DskipTests install || exit $? + if [[ "${FLINK_LINE}" == "1.18" ]]; then + # Compile version-specific tests without replacing the released production JAR. + mvn -B -ntp -s "${MAVEN_SETTINGS}" -f "${PAIMON_ROOT}/pom.xml" \ + -Dmaven.repo.local="${SUITE_MAVEN_REPO}" "${PAIMON_BUILD_ARGS[@]}" \ + -pl "${PAIMON_LINE_MODULE}" -DskipTests test-compile || exit $? + fi fi fi +python3 "${REPO_ROOT}/bin/check-flink-suite-classpath.py" "${CLASSPATH_FILE}" "${FLINK_LINE}" || exit $? STREAMFUSION_CLASSPATH="$(tr ':' ',' < "${CLASSPATH_FILE}")" if [[ "${SUITE_MODE}" != "paimon" ]]; then # Paimon's provided connector API is supplied by its own suite. Do not install its optional # planner extension into unrelated Flink test JVMs that do not have that connector. STREAMFUSION_CLASSPATH="$(python3 - "${STREAMFUSION_CLASSPATH}" <<'PY' import sys -print(','.join(p for p in sys.argv[1].split(',') if '/tech/streamfusion/streamfusion-paimon/' not in p)) +print(','.join(p for p in sys.argv[1].split(',') if not any(part == 'streamfusion-paimon' or part.startswith('streamfusion-paimon-flink') for part in p.split('/')))) PY )" fi if [[ "${SUITE_MODE}" == "paimon" ]]; then - # Paimon declares the planner's test-jar before the planner itself, which places stock - # calcite-core ahead of Flink's patched Calcite classes in Surefire's resolved classpath. Drop the - # resolved calcite-core and append it instead, so the planner's copies win as in Flink's own build. + if [[ "${FLINK_LINE}" == "1.18" ]]; then + # Common tests compile against 1.20. The released 1.18 runtime also replaces helpers + # with different host signatures; appending its JAR cannot override target/classes. + PAIMON_RUNTIME_JAR="${SUITE_MAVEN_REPO}/org/apache/paimon/paimon-flink-${FLINK_LINE}/${PAIMON_VERSION}/paimon-flink-${FLINK_LINE}-${PAIMON_VERSION}.jar" + if [[ ! -f "${PAIMON_RUNTIME_JAR}" ]]; then + echo "Missing released Paimon compatibility artifact: ${PAIMON_RUNTIME_JAR}" >&2 + exit 2 + fi + TEST_MODULES=":paimon-flink-common" + fi +fi +if [[ "${SUITE_MODE}" == "kafka" && "${FLINK_LINE}" == "1.18" ]]; then + # Kafka 3.2 otherwise isolates the planner in planner-loader while the agent and native + # planner live in the test JVM. Use the same unshaded planner as the other upstream suites. + STREAMFUSION_CLASSPATH="${UNSHADED_PLANNER_JAR},${STREAMFUSION_CLASSPATH}" +fi +if [[ "${SUITE_MODE}" == "paimon" || ( "${SUITE_MODE}" == "kafka" && "${FLINK_LINE}" == "1.18" ) ]]; then + # These connector fixtures resolve stock Calcite before the planner. Put it after the + # planner's patched classes, preserving the ordering used by Flink's own test build. CALCITE_CORE_JAR="$(find "${SUITE_MAVEN_REPO}/org/apache/calcite/calcite-core" -name 'calcite-core-*.jar' \ ! -name '*-sources.jar' ! -name '*-tests.jar' | head -n 1)" if [[ -z "${CALCITE_CORE_JAR}" ]]; then - echo "The isolated suite repository holds no calcite-core jar for the Paimon suite classpath." >&2 + echo "The isolated suite repository holds no calcite-core jar for the connector suite classpath." >&2 exit 2 fi STREAMFUSION_CLASSPATH="${STREAMFUSION_CLASSPATH},${CALCITE_CORE_JAR}" @@ -385,6 +484,11 @@ if [[ "${SUITE_MODE}" == "kafka" ]]; then -Dflink.version="${FLINK_VERSION}" -Dflink.surefire.baseArgLine="${FLINK_MODULE_CONFIG}" ) + if [[ "${FLINK_LINE}" == "1.18" ]]; then + MAVEN_TEST_ARGS+=( + -Dmaven.test.dependency.excludes=org.apache.flink:flink-table-planner-loader,org.apache.calcite:calcite-core + ) + fi elif [[ "${SUITE_MODE}" == "delta" ]]; then MAVEN_TEST_ARGS+=( -f "${DELTA_TEST_POM}" @@ -395,9 +499,17 @@ elif [[ "${SUITE_MODE}" == "delta" ]]; then -Ddelta.test.jvm.args="${CONNECTOR_MODULE_CONFIG}" ) elif [[ "${SUITE_MODE}" == "paimon" ]]; then + PAIMON_TEST_POM="${PAIMON_ROOT}/pom.xml" + if [[ "${FLINK_LINE}" == "1.18" ]]; then + PAIMON_TEST_POM="${DIAGNOSTIC_ROOT}/paimon-runtime-pom.xml" + python3 "${REPO_ROOT}/dev/flink-suite/prepare_paimon_runtime_pom.py" \ + "${PAIMON_ROOT}/${PAIMON_MODULE}/pom.xml" "${PAIMON_RUNTIME_JAR}" \ + "${PAIMON_TEST_POM}" || exit $? + fi MAVEN_TEST_ARGS+=( - -f "${PAIMON_ROOT}/pom.xml" + -f "${PAIMON_TEST_POM}" "${PAIMON_BUILD_ARGS[@]}" + "-Dpaimon-flink-common.flink.version=${FLINK_VERSION}" -Dflink.forkCount="${FLINK_SUITE_IT_FORKS:-1}" -Djunit.jupiter.execution.timeout.default=10m -Dsurefire.timeout=1800 @@ -420,13 +532,44 @@ if [[ ${#TEST_SELECTOR_ARGS[@]} -gt 0 ]]; then fi MAVEN_TEST_ARGS+=("${TEST_GOAL}") if [[ "${SUITE_MODE}" == "kafka" ]]; then - "${KAFKA_CONNECTOR_ROOT}/mvnw" "${MAVEN_TEST_ARGS[@]}" + kafka_mvn "${MAVEN_TEST_ARGS[@]}" elif [[ "${SUITE_MODE}" == "delta" ]]; then mvn "${MAVEN_TEST_ARGS[@]}" elif [[ "${SUITE_MODE}" == "paimon" ]]; then mvn "${MAVEN_TEST_ARGS[@]}" paimon_status=$? - if [[ -z "${FLINK_SUITE_TEST:-}" ]]; then + if [[ -z "${FLINK_SUITE_TEST:-}" && "${FLINK_LINE}" == "1.18" ]]; then + line_pom="${DIAGNOSTIC_ROOT}/paimon-line-pom.xml" + line_reports="${PAIMON_ROOT}/${PAIMON_LINE_MODULE}/target/surefire-reports" + python3 "${REPO_ROOT}/dev/flink-suite/prepare_paimon_runtime_pom.py" \ + "${PAIMON_ROOT}/${PAIMON_LINE_MODULE}/pom.xml" "${PAIMON_RUNTIME_JAR}" \ + "${line_pom}" || exit $? + mkdir -p "${line_reports}" + find "${line_reports}" -type f -delete + echo "Running Paimon's complete unchanged Flink 1.18 module in separate JVMs..." + line_args=("${MAVEN_TEST_ARGS[@]}") + for index in "${!line_args[@]}"; do + case "${line_args[index]}" in + -f) line_args[index+1]="${line_pom}" ;; + -pl) line_args[index+1]=:streamfusion-upstream-paimon-line-tests ;; + -Dtest=*) line_args[index]="-Dtest=${PAIMON_118_LINE_CLASSES}" ;; + -Dsurefire.failIfNoSpecifiedTests=*) line_args[index]=-Dsurefire.failIfNoSpecifiedTests=true ;; + esac + done + mvn "${line_args[@]}" + line_status=$? + line_summary=("${line_reports}" --contracts "${CONTRACT_FILE}" --process-exit "${line_status}" + --audit-output "${DIAGNOSTIC_ROOT}/paimon-line-execution-audit.json") + while IFS= read -r line_test; do + line_summary+=(--require-test-class "${line_test}") + done < <(printf '%s\n' "${PAIMON_118_LINE_CLASSES}" | tr ',' '\n') + python3 "${REPO_ROOT}/dev/flink-suite/summarize.py" "${line_summary[@]}" + line_status=$? + if [[ ${line_status} -ne 0 ]]; then paimon_status=${line_status}; fi + # Two upstream classes share names with common fixtures; retain both sets of XML. + mkdir -p "${REPORT_ROOT}/flink-1.18" + cp -R "${line_reports}/." "${REPORT_ROOT}/flink-1.18/" || exit $? + elif [[ -z "${FLINK_SUITE_TEST:-}" ]]; then # Cancelled upstream compactors can kill their shared MiniCluster during cleanup. Run each # unchanged standalone compaction test in its own fork to contain that lifecycle race. mkdir -p "${REPORT_ROOT}/shared-cluster" @@ -452,7 +595,7 @@ elif [[ "${SUITE_MODE}" == "paimon" ]]; then # Keep either invocation's nonzero exit status for the common report checks below. (exit "${paimon_status}") else - "${FLINK_ROOT}/mvnw" "${MAVEN_TEST_ARGS[@]}" + flink_mvn "${MAVEN_TEST_ARGS[@]}" fi readonly TEST_STATUS=$? @@ -491,13 +634,13 @@ if [[ "${SUITE_MODE}" == "paimon" && ${TEST_STATUS} -eq 0 ]]; then done fi -SUMMARY_ARGS=("${REPORT_ROOT}" --native-reports "${NATIVE_REPORT_ROOT}" --process-exit "${TEST_STATUS}" +SUMMARY_ARGS=("${REPORT_ROOT}" --contracts "${CONTRACT_FILE}" --native-reports "${NATIVE_REPORT_ROOT}" --process-exit "${TEST_STATUS}" --audit-output "${DIAGNOSTIC_ROOT}/execution-audit.json") if [[ "${SUITE_MODE}" == "runtime" || "${SUITE_MODE}" == "diagnostic" ]]; then - SUMMARY_ARGS+=( - --xfail "org.apache.flink.table.planner.runtime.batch.sql.CalcITCase#testCurrentDate" - --maven-result "${DIAGNOSTIC_ROOT}/maven-result.tsv" - ) + SUMMARY_ARGS+=(--maven-result "${DIAGNOSTIC_ROOT}/maven-result.tsv") + if [[ "${FLINK_LINE}" == "2.2" ]]; then + SUMMARY_ARGS+=(--xfail "org.apache.flink.table.planner.runtime.batch.sql.CalcITCase#testCurrentDate") + fi if [[ -z "${FLINK_SUITE_TEST:-}" ]]; then SUMMARY_ARGS+=(--require-contract-prefix org.apache.flink.) fi @@ -508,5 +651,18 @@ if [[ "${SUITE_MODE}" == "delta" && -z "${FLINK_SUITE_TEST:-}" ]]; then --require-test 'io.delta.flink.sink.sql.FlinkSqlTest#testGroupedAggregationPreservesEachRow' ) fi +if [[ "${SUITE_MODE}" == "state" && -z "${FLINK_SUITE_TEST:-}" ]]; then + for state_test in ${ROCKSDB_STATE_SQL_TESTS//,/ }; do + SUMMARY_ARGS+=(--require-test-class "${state_test}") + if awk -F '\t' -v prefix="${state_test}#" 'index($1, prefix) == 1 { found=1 } END { exit !found }' "${CONTRACT_FILE}"; then + SUMMARY_ARGS+=(--require-contract-prefix "${state_test}#") + fi + done +fi +if [[ "${SUITE_MODE}" == "paimon" && "${FLINK_LINE}" == "1.18" && -z "${FLINK_SUITE_TEST:-}" ]]; then + while IFS= read -r shared_test; do + SUMMARY_ARGS+=(--require-test-method "${shared_test}") + done < "${PAIMON_118_SHARED_TESTS}" +fi python3 "${REPO_ROOT}/dev/flink-suite/summarize.py" "${SUMMARY_ARGS[@]}" exit $? diff --git a/bin/install-flink.sh b/bin/install-flink.sh index c99938afd..0ed585476 100755 --- a/bin/install-flink.sh +++ b/bin/install-flink.sh @@ -2,17 +2,28 @@ set -eu +flink_line=2.2 +if [ "${1:-}" = --flink-line ]; then + if [ "$#" -lt 3 ]; then echo "usage: $0 [--flink-line 2.2|1.18] " >&2; exit 64; fi + flink_line=$2 + shift 2 +fi if [ "$#" -ne 1 ]; then - echo "usage: $0 " >&2 + echo "usage: $0 [--flink-line 2.2|1.18] " >&2 exit 64 fi - flink_home=$1 +artifact_suffix= +case "$flink_line" in + 2.2) ;; + 1.18) artifact_suffix=-flink1.18 ;; + *) echo "unsupported Flink line: $flink_line" >&2; exit 64 ;; +esac script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) repo_root=$(cd "$script_dir/.." && pwd) artifact_version=$(cd "$repo_root" && mvn -q -DforceStdout help:evaluate -Dexpression=project.version) -loader_jar=$repo_root/streamfusion-loader/target/streamfusion-loader-$artifact_version.jar -core_jar=$repo_root/streamfusion-core/target/streamfusion-core-$artifact_version-runtime.jar +loader_jar=$repo_root/streamfusion-loader/target/streamfusion-loader$artifact_suffix-$artifact_version.jar +core_jar=$repo_root/streamfusion-core/target/streamfusion-core$artifact_suffix-$artifact_version-runtime.jar if [ ! -d "$flink_home/lib" ]; then echo "Flink lib directory does not exist: $flink_home/lib" >&2 @@ -24,6 +35,23 @@ if [ ! -f "$loader_jar" ] || [ ! -f "$core_jar" ]; then exit 66 fi +for jar_file in "$loader_jar" "$core_jar"; do + payload_line=$(unzip -p "$jar_file" META-INF/MANIFEST.MF | tr -d '\r' | sed -n 's/^StreamFusion-Flink-Line: //p') + if [ "$payload_line" != "$flink_line" ]; then + echo "Payload $jar_file targets Flink $payload_line, expected $flink_line" >&2 + exit 65 + fi +done +set -- "$flink_home"/lib/flink-dist-*.jar +if [ "$#" -ne 1 ]; then + echo "Expected exactly one Flink distribution JAR in $flink_home/lib" >&2 + exit 65 +fi +case "$flink_line:$(basename "$1")" in + 1.18:flink-dist-1.18.1.jar|2.2:flink-dist-2.2.0.jar|2.2:flink-dist-2.2.1.jar) ;; + *) echo "Flink distribution $(basename "$1") does not match the selected supported line $flink_line" >&2; exit 65 ;; +esac + cp "$loader_jar" "$flink_home/lib/00-streamfusion-loader.jar" cp "$core_jar" "$flink_home/lib/streamfusion-core.jar" diff --git a/bin/package-release.sh b/bin/package-release.sh index cf6da89a3..32fdf6b31 100755 --- a/bin/package-release.sh +++ b/bin/package-release.sh @@ -2,12 +2,34 @@ set -eu +flink_line=2.2 +output_dir= +while [ "$#" -gt 0 ]; do + case "$1" in + --flink-line) + if [ "$#" -lt 2 ]; then echo "--flink-line requires 2.2 or 1.18" >&2; exit 64; fi + flink_line=$2; shift 2 ;; + --*) echo "usage: $0 [--flink-line 2.2|1.18] [output-directory]" >&2; exit 64 ;; + *) + if [ -n "$output_dir" ]; then echo "only one output directory is allowed" >&2; exit 64; fi + output_dir=$1; shift ;; + esac +done +artifact_suffix="" +modules="kafka json csv raw avro avro-confluent-registry protobuf parquet orc paimon" +case "$flink_line" in + 2.2) set --; modules="$modules delta" ;; + 1.18) artifact_suffix=-flink1.18; set -- -Pflink-1.18 ;; + *) echo "unsupported Flink line: $flink_line" >&2; exit 64 ;; +esac + script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) repo_root=$(cd "$script_dir/.." && pwd) -version=$(cd "$repo_root" && mvn -q -DforceStdout help:evaluate -Dexpression=project.version) -output_dir=${1:-"$repo_root/target/release"} +version=$(cd "$repo_root" && mvn "$@" -q -DforceStdout help:evaluate -Dexpression=project.version) +output_dir=${output_dir:-"$repo_root/target/release"} +bundle_name=streamfusion${artifact_suffix}-$version stage_dir=$(mktemp -d) -bundle_dir=$stage_dir/streamfusion-$version +bundle_dir=$stage_dir/$bundle_name cleanup() { rm -rf "$stage_dir" @@ -16,15 +38,15 @@ trap cleanup EXIT HUP INT TERM mkdir -p "$bundle_dir" "$output_dir" cp "$repo_root/LICENSE" "$repo_root/readme.md" "$bundle_dir/" -cp "$repo_root/streamfusion-loader/target/streamfusion-loader-$version.jar" "$bundle_dir/" -cp "$repo_root/streamfusion-core/target/streamfusion-core-$version-runtime.jar" "$bundle_dir/" +cp "$repo_root/streamfusion-loader/target/streamfusion-loader${artifact_suffix}-$version.jar" "$bundle_dir/" +cp "$repo_root/streamfusion-core/target/streamfusion-core${artifact_suffix}-$version-runtime.jar" "$bundle_dir/" -for suffix in kafka json csv raw avro avro-confluent-registry protobuf parquet orc delta paimon; do - cp "$repo_root/streamfusion-$suffix/target/streamfusion-$suffix-$version.jar" "$bundle_dir/" +for suffix in $modules; do + cp "$repo_root/streamfusion-$suffix/target/streamfusion-$suffix${artifact_suffix}-$version.jar" "$bundle_dir/" done -archive=$output_dir/streamfusion-$version-bin.tar.gz -(cd "$stage_dir" && tar -czf "$archive" "streamfusion-$version") +archive=$output_dir/$bundle_name-bin.tar.gz +(cd "$stage_dir" && COPYFILE_DISABLE=1 tar -czf "$archive" "$bundle_name") (cd "$output_dir" && shasum -a 256 "$(basename "$archive")" > "$(basename "$archive").sha256") printf '%s\n' "$archive" diff --git a/bin/test_check_flink_suite_classpath.py b/bin/test_check_flink_suite_classpath.py new file mode 100644 index 000000000..abb820198 --- /dev/null +++ b/bin/test_check_flink_suite_classpath.py @@ -0,0 +1,66 @@ +import importlib.util +import os +from pathlib import Path +import tempfile +import unittest +import zipfile + + +spec = importlib.util.spec_from_file_location( + "check_classpath", Path(__file__).with_name("check-flink-suite-classpath.py") +) +checker = importlib.util.module_from_spec(spec) +spec.loader.exec_module(checker) + + +class SuiteClasspathTest(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.directory = Path(directory.name) + + def jar(self, name, module=None, line=None): + path = self.directory / name + with zipfile.ZipFile(path, "w") as archive: + manifest = "Manifest-Version: 1.0\r\n" + if module: + manifest += f"StreamFusion-Module: {module}\r\n" + if line: + manifest += f"StreamFusion-Flink-Line: {line}\r\n" + archive.writestr("META-INF/MANIFEST.MF", manifest) + return str(path) + + def test_matching_lines_and_unrelated_dependencies(self): + for line, suffix in [("2.2", ""), ("1.18", "-flink1.18")]: + with self.subTest(line=line): + core = "streamfusion-core" + suffix + json = "streamfusion-json" + suffix + paths = [self.jar("core.jar", core, line), self.jar("json.jar", json, line), + self.jar("arrow.jar")] + self.assertEqual({core, json}, checker.validate(os.pathsep.join(paths), line)) + + def test_renaming_a_payload_cannot_hide_its_line(self): + core = self.jar("core.jar", "streamfusion-core-flink1.18", "1.18") + wrong = self.jar("renamed.jar", "streamfusion-json", "2.2") + with self.assertRaisesRegex(ValueError, "expected Flink 1.18"): + checker.validate(os.pathsep.join([core, wrong]), "1.18") + + def test_source_suite_requires_identified_core(self): + with self.assertRaisesRegex(ValueError, "missing streamfusion-core"): + checker.validate(self.jar("arrow.jar"), "2.2") + with self.assertRaisesRegex(ValueError, "payload identity"): + checker.validate(self.jar("streamfusion-core.jar"), "2.2") + + def test_line_and_artifact_identity_must_agree(self): + with self.assertRaisesRegex(ValueError, "payload identity"): + checker.validate(self.jar("core.jar", "streamfusion-core", "1.18"), "1.18") + + def test_duplicate_payloads_are_rejected(self): + paths = [self.jar("a.jar", "streamfusion-core", "2.2"), + self.jar("b.jar", "streamfusion-core", "2.2")] + with self.assertRaisesRegex(ValueError, "Duplicate"): + checker.validate(os.pathsep.join(paths), "2.2") + + +if __name__ == "__main__": + unittest.main() diff --git a/dev/flink-suite/agent/pom.xml b/dev/flink-suite/agent/pom.xml index 2a713b1ab..5b5ae1de1 100644 --- a/dev/flink-suite/agent/pom.xml +++ b/dev/flink-suite/agent/pom.xml @@ -11,6 +11,7 @@ 17 UTF-8 + ${project.basedir}/target 1.17.6 @@ -32,9 +33,22 @@ 5.10.2 test + + org.apache.flink + flink-table-common + 1.18.1 + test + + + org.apache.flink + flink-streaming-java + 1.18.1 + test + + ${suite.agent.output} org.apache.maven.plugins diff --git a/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/LegacyStateBackend.java b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/LegacyStateBackend.java new file mode 100644 index 000000000..1cda9b5bb --- /dev/null +++ b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/LegacyStateBackend.java @@ -0,0 +1,74 @@ +package tech.streamfusion.suite; + +/** Preserves legacy upstream checkpoint storage while selecting the native keyed backend. */ +public final class LegacyStateBackend { + private LegacyStateBackend() {} + + public static Object replace(Object environment, Object backend) { + if (!Boolean.getBoolean("streamfusion.flink-suite.native-rocksdb") || backend == null) { + return backend; + } + String type = backend.getClass().getName(); + if (type.equals("org.apache.flink.runtime.state.memory.MemoryStateBackend") + || type.equals("org.apache.flink.runtime.state.filesystem.FsStateBackend") + || type.equals("org.apache.flink.runtime.state.hashmap.HashMapStateBackend")) { + if (StreamFusionSuiteAgent.reportHeapState()) { + System.err.println("StreamFusion upstream state suite exercised Flink heap backend"); + } + return backend; + } + if (!type.equals("org.apache.flink.contrib.streaming.state.RocksDBStateBackend")) { + return backend; + } + try { + if (environment + .getClass() + .getMethod("isChangelogStateBackendEnabled") + .invoke(environment) + .toString() + .equals("TRUE")) { + return backend; + } + Object checkpointConfig = + environment.getClass().getMethod("getCheckpointConfig").invoke(environment); + Object storage = backend.getClass().getMethod("getCheckpointBackend").invoke(backend); + checkpointConfig + .getClass() + .getMethod( + "setCheckpointStorage", + Class.forName("org.apache.flink.runtime.state.CheckpointStorage")) + .invoke(checkpointConfig, storage); + Class configurationType = Class.forName("org.apache.flink.configuration.Configuration"); + Object configuration = + configurationType + .getConstructor(configurationType) + .newInstance( + environment.getClass().getMethod("getConfiguration").invoke(environment)); + configurationType + .getMethod("setBoolean", String.class, boolean.class) + .invoke( + configuration, + "state.backend.incremental", + backend.getClass().getMethod("isIncrementalCheckpointsEnabled").invoke(backend)); + ClassLoader loader = Thread.currentThread().getContextClassLoader(); + Object factory = + Class.forName("tech.streamfusion.state.RocksDBNativeStateBackendFactory", true, loader) + .getConstructor() + .newInstance(); + Object replacement = + factory + .getClass() + .getMethod( + "createFromConfig", + Class.forName("org.apache.flink.configuration.ReadableConfig"), + ClassLoader.class) + .invoke(factory, configuration, loader); + if (StreamFusionSuiteAgent.reportRocksDBState()) { + System.err.println("StreamFusion upstream state suite installed native RocksDB backend"); + } + return replacement; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("legacy native RocksDB suite backend installation failed", e); + } + } +} diff --git a/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/NativeExecution.java b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/NativeExecution.java index ea31dda73..abdccd6e2 100644 --- a/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/NativeExecution.java +++ b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/NativeExecution.java @@ -29,12 +29,23 @@ public final class NativeExecution { private NativeExecution() {} private static Map> loadContracts() { + String flinkLine = System.getProperty("streamfusion.flink-suite.flink-line", "2.2"); + String resource = + switch (flinkLine) { + case "2.2" -> "/native-execution.tsv"; + case "1.18" -> + Boolean.getBoolean("streamfusion.flink-suite.native-rocksdb") + ? "/native-execution-flink1.18-state.tsv" + : "/native-execution-flink1.18.tsv"; + default -> + throw new IllegalArgumentException( + "Unknown native execution contract line: " + flinkLine); + }; Map> contracts = new LinkedHashMap<>(); try (BufferedReader reader = new BufferedReader( new InputStreamReader( - NativeExecution.class.getResourceAsStream("/native-execution.tsv"), - StandardCharsets.UTF_8))) { + NativeExecution.class.getResourceAsStream(resource), StandardCharsets.UTF_8))) { for (String line; (line = reader.readLine()) != null; ) { if (line.isBlank() || line.startsWith("#")) { continue; @@ -42,7 +53,7 @@ private static Map> loadContracts() { String[] fields = line.split("\t", -1); if (fields.length != 3 || !fields[0].matches("[\\w.$]+#[\\w$]+") - || !fields[1].matches("\\*|\\w+=(?:true|false)") + || !fields[1].matches("\\*|\\w+=\\w+(?:&\\w+=\\w+)*") || !(fields[2].matches("\\w+(?:[+|]\\w+)*") || fields[2].matches("!.+"))) { throw new IllegalStateException("Invalid native execution contract: " + line); } @@ -94,17 +105,44 @@ public static synchronized Scope begin(String test, Object fixture) { return active; } - private static boolean matches(String selector, Object fixture) { - if (selector.equals("*")) { - return true; + static boolean matches(String selector, Object fixture) { + if (selector.equals("*")) return true; + for (String condition : selector.split("&")) { + String[] parts = condition.split("=", 2); + Object value = + parts[0].equals("changelog") ? changelogEnabled(fixture) : field(fixture, parts[0]); + if (!String.valueOf(value).equals(parts[1])) return false; } - String[] parts = selector.split("="); + return true; + } + + private static Object field(Object fixture, String name) { + for (Class type = fixture.getClass(); type != null; type = type.getSuperclass()) { + try { + var field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(fixture); + } catch (NoSuchFieldException ignored) { + // Pinned fixture parameters may belong to the upstream base class. + } catch (ReflectiveOperationException e) { + throw new AssertionError("Cannot read pinned upstream fixture selector " + name, e); + } + } + throw new AssertionError("Cannot read pinned upstream fixture selector " + name); + } + + private static boolean changelogEnabled(Object fixture) { try { - var field = fixture.getClass().getDeclaredField(parts[0]); - field.setAccessible(true); - return Boolean.toString(field.getBoolean(fixture)).equals(parts[1]); + Object environment = field(fixture, "env"); + Object javaEnvironment = environment.getClass().getMethod("getJavaEnv").invoke(environment); + return javaEnvironment + .getClass() + .getMethod("isChangelogStateBackendEnabled") + .invoke(javaEnvironment) + .toString() + .equals("TRUE"); } catch (ReflectiveOperationException e) { - throw new AssertionError("Cannot read pinned upstream fixture selector " + selector, e); + throw new AssertionError("Cannot read pinned upstream changelog configuration", e); } } diff --git a/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/PaimonCatalogFixture.java b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/PaimonCatalogFixture.java new file mode 100644 index 000000000..ee624d381 --- /dev/null +++ b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/PaimonCatalogFixture.java @@ -0,0 +1,49 @@ +package tech.streamfusion.suite; + +import java.util.List; +import java.util.Map; + +/** Preserves the shared Paimon fixture's catalog metadata through Flink 1.18's factory API. */ +public final class PaimonCatalogFixture { + private PaimonCatalogFixture() {} + + public static Object create( + ClassLoader loader, + Map options, + List columns, + List partitions, + List primaryKeys) { + try { + Class constraintType = + Class.forName("org.apache.flink.table.catalog.UniqueConstraint", true, loader); + Object constraint = + primaryKeys.isEmpty() + ? null + : constraintType + .getMethod("primaryKey", String.class, List.class) + .invoke(null, "pk", primaryKeys); + Class resolvedType = + Class.forName("org.apache.flink.table.catalog.ResolvedSchema", true, loader); + Object resolved = + resolvedType + .getConstructor(List.class, List.class, constraintType) + .newInstance(columns, List.of(), constraint); + Class schemaType = Class.forName("org.apache.flink.table.api.Schema", true, loader); + Object builder = schemaType.getMethod("newBuilder").invoke(null); + builder.getClass().getMethod("fromResolvedSchema", resolvedType).invoke(builder, resolved); + Object schema = builder.getClass().getMethod("build").invoke(builder); + Class catalogType = + Class.forName("org.apache.flink.table.catalog.CatalogTable", true, loader); + Object origin = + catalogType + .getMethod("of", schemaType, String.class, List.class, Map.class) + .invoke(null, schema, "a comment", partitions, options); + return Class.forName("org.apache.flink.table.catalog.ResolvedCatalogTable", true, loader) + .getConstructor(catalogType, resolvedType) + .newInstance(origin, resolved); + } catch (ReflectiveOperationException error) { + throw new IllegalStateException( + "Cannot construct the Paimon catalog fixture on Flink 1.18", error); + } + } +} diff --git a/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/PaimonCheckpointFixture.java b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/PaimonCheckpointFixture.java new file mode 100644 index 000000000..bf72fd37c --- /dev/null +++ b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/PaimonCheckpointFixture.java @@ -0,0 +1,82 @@ +package tech.streamfusion.suite; + +import static net.bytebuddy.matcher.ElementMatchers.any; + +import net.bytebuddy.asm.AsmVisitorWrapper; +import net.bytebuddy.jar.asm.MethodVisitor; +import net.bytebuddy.jar.asm.Opcodes; + +/** Uses Flink 1.18's equivalent checkpoint settings without replacing recovery test bodies. */ +public final class PaimonCheckpointFixture { + private static final String CONFIGURATION = "org/apache/flink/configuration/"; + private static final String ENVIRONMENT = "org/apache/flink/streaming/api/environment/"; + + private PaimonCheckpointFixture() {} + + static boolean supports(String name) { + return name.equals("org.apache.paimon.flink.FlinkJobRecoveryITCase") + || name.equals("org.apache.paimon.flink.RescaleBucketITCase"); + } + + public static boolean isRecoveryTest() { + return StackWalker.getInstance() + .walk(frames -> frames.anyMatch(frame -> supports(frame.getClassName()))); + } + + public static void applyRestoreSettings(Object configuration, Object pipeline) { + try { + ClassLoader loader = pipeline.getClass().getClassLoader(); + Class settings = + Class.forName("org.apache.flink.runtime.jobgraph.SavepointRestoreSettings", true, loader); + Class readableConfig = + Class.forName("org.apache.flink.configuration.ReadableConfig", true, loader); + Object restore = + settings.getMethod("fromConfiguration", readableConfig).invoke(null, configuration); + pipeline + .getClass() + .getMethod("setSavepointRestoreSettings", settings) + .invoke(pipeline, restore); + } catch (ReflectiveOperationException error) { + throw new IllegalStateException( + "Cannot apply the Paimon recovery fixture on Flink 1.18", error); + } + } + + static AsmVisitorWrapper adapter() { + return new AsmVisitorWrapper.ForDeclaredMethods() + .method( + any(), + (type, method, visitor, context, pool, writerFlags, readerFlags) -> adapt(visitor)); + } + + static MethodVisitor adapt(MethodVisitor visitor) { + return new MethodVisitor(Opcodes.ASM9, visitor) { + @Override + public void visitLdcInsn(Object value) { + if ("execution.state-recovery.path".equals(value)) { + value = "execution.savepoint.path"; + } + super.visitLdcInsn(value); + } + + @Override + public void visitFieldInsn(int opcode, String owner, String name, String descriptor) { + if (opcode == Opcodes.GETSTATIC) { + if (owner.equals(CONFIGURATION + "CheckpointingOptions") + && name.equals("EXTERNALIZED_CHECKPOINT_RETENTION")) { + owner = ENVIRONMENT + "ExecutionCheckpointingOptions"; + name = "EXTERNALIZED_CHECKPOINT"; + } else if (owner.equals(CONFIGURATION + "ExternalizedCheckpointRetention") + && name.equals("RETAIN_ON_CANCELLATION")) { + owner = ENVIRONMENT + "CheckpointConfig$ExternalizedCheckpointCleanup"; + descriptor = "L" + owner + ";"; + } else if (owner.equals(CONFIGURATION + "StateRecoveryOptions") + && name.equals("SAVEPOINT_IGNORE_UNCLAIMED_STATE")) { + owner = "org/apache/flink/runtime/jobgraph/SavepointConfigOptions"; + } + } + super.visitFieldInsn(opcode, owner, name, descriptor); + } + }; + } +} diff --git a/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/PaimonTestWatch.java b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/PaimonTestWatch.java index ad5a0e639..7129c02f1 100644 --- a/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/PaimonTestWatch.java +++ b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/PaimonTestWatch.java @@ -30,21 +30,13 @@ public static void initialize(PrintStream output) { private PaimonTestWatch(String test, Object fixture) { this.test = test; - String options; - try { - var field = fixture.getClass().getDeclaredField("tableDefaultProperties"); - field.setAccessible(true); - options = String.valueOf(field.get(fixture)); - } catch (ReflectiveOperationException e) { - options = "unavailable: " + e; - } diagnostics.println( "StreamFusion upstream test started " + test + " at " + Instant.now() + "; randomized table defaults " - + options); + + tableDefaults(fixture)); long delay = Long.getLong("streamfusion.flink-suite.diagnostic-delay-seconds", 120L); if (delay <= 0) { throw new IllegalArgumentException("The upstream diagnostic delay must be positive"); @@ -52,6 +44,21 @@ private PaimonTestWatch(String test, Object fixture) { diagnostic = TIMER.schedule(this::dumpThreads, delay, TimeUnit.SECONDS); } + private static String tableDefaults(Object fixture) { + for (Class type = fixture.getClass(); type != null; type = type.getSuperclass()) { + try { + var field = type.getDeclaredField("tableDefaultProperties"); + field.setAccessible(true); + return String.valueOf(field.get(fixture)); + } catch (NoSuchFieldException ignored) { + // Most Paimon fixtures do not randomize table defaults. + } catch (IllegalAccessException failure) { + return "unavailable: " + failure; + } + } + return "none"; + } + public static PaimonTestWatch start(String test, Object fixture) { return new PaimonTestWatch(test, fixture); } diff --git a/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/StreamFusionSuiteAgent.java b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/StreamFusionSuiteAgent.java index 5e55a1740..974c41fb6 100644 --- a/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/StreamFusionSuiteAgent.java +++ b/dev/flink-suite/agent/src/main/java/tech/streamfusion/suite/StreamFusionSuiteAgent.java @@ -2,6 +2,8 @@ import static net.bytebuddy.matcher.ElementMatchers.isAnnotatedWith; import static net.bytebuddy.matcher.ElementMatchers.isConstructor; +import static net.bytebuddy.matcher.ElementMatchers.nameEndsWith; +import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith; import static net.bytebuddy.matcher.ElementMatchers.named; import static net.bytebuddy.matcher.ElementMatchers.namedOneOf; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; @@ -69,6 +71,15 @@ public static void premain(String arguments, Instrumentation instrumentation) { PaimonTestWatch.initialize(System.err); new AgentBuilder.Default() .with(AgentBuilder.Listener.StreamWriting.toSystemError().withTransformationsOnly()) + .type(named("org.apache.paimon.flink.FlinkTestBase")) + .transform( + (builder, type, classLoader, module, protectionDomain) -> + builder.visit( + Advice.to(BuildLegacyPaimonCatalog.class) + .on( + named("createResolvedTable") + .and(takesArguments(4)) + .and(takesArgument(1, java.util.List.class))))) .type( namedOneOf( "org.apache.flink.runtime.minicluster.MiniCluster$TerminatingFatalErrorHandler", @@ -76,16 +87,29 @@ public static void premain(String arguments, Instrumentation instrumentation) { .transform( (builder, type, classLoader, module, protectionDomain) -> builder.visit(Advice.to(ReportClusterFailure.class).on(named("onFatalError")))) - .type(named("org.apache.paimon.flink.PrimaryKeyFileStoreTableITCase")) + .type( + nameStartsWith("org.apache.paimon.flink.") + .and(nameEndsWith("ITCase").or(nameEndsWith("ITCaseBase")))) + .transform( + (builder, type, classLoader, module, protectionDomain) -> { + if (System.getProperty("streamfusion.flink-suite.flink-line", "2.2").equals("1.18") + && PaimonCheckpointFixture.supports(type.getName())) { + builder = builder.visit(PaimonCheckpointFixture.adapter()); + } + return builder.visit( + Advice.to(WatchPaimonTest.class) + .on( + isAnnotatedWith(named("org.junit.jupiter.api.Test")) + .or( + isAnnotatedWith( + named("org.junit.jupiter.params.ParameterizedTest"))))); + }) + .type(named("org.apache.flink.table.planner.delegation.DefaultExecutor")) .transform( (builder, type, classLoader, module, protectionDomain) -> builder.visit( - Advice.to(WatchPaimonTest.class) - .on( - isAnnotatedWith(named("org.junit.jupiter.api.Test")) - .or( - isAnnotatedWith( - named("org.junit.jupiter.params.ParameterizedTest")))))) + Advice.to(ApplyLegacyPaimonRestore.class) + .on(named("createPipeline").and(takesArguments(4))))) .type(named("tech.streamfusion.planner.PhysicalPlanScan")) .transform( (builder, type, classLoader, module, protectionDomain) -> @@ -103,6 +127,7 @@ public static void premain(String arguments, Instrumentation instrumentation) { "tech.streamfusion.operator.NativeAsyncLookupJoinOperator", "tech.streamfusion.operator.NativeFilterOperator", "tech.streamfusion.operator.NativeColumnarGroupAggregateOperator", + "tech.streamfusion.operator.NativeColumnarUpdatingJoinOperator", "tech.streamfusion.operator.NativeColumnarTopNOperator", "tech.streamfusion.operator.NativeWindowOperatorCore", "tech.streamfusion.operator.NativeColumnarGlobalWindowAggregateOperator")) @@ -115,6 +140,9 @@ public static void premain(String arguments, Instrumentation instrumentation) { if (type.getName().endsWith("NativeColumnarTopNOperator")) { return builder.visit(Advice.to(RecordNativeBatch.class).on(named("push"))); } + if (type.getName().endsWith("NativeColumnarUpdatingJoinOperator")) { + return builder.visit(Advice.to(RecordNativeBatch.class).on(named("joinOpen"))); + } if (type.getName().endsWith("NativeWindowOperatorCore")) { return builder.visit( Advice.to(RecordNativeBatch.class) @@ -151,9 +179,13 @@ public static void premain(String arguments, Instrumentation instrumentation) { .type(named(STREAM_EXECUTION_ENVIRONMENT)) .transform( (builder, type, classLoader, module, protectionDomain) -> - builder.visit( - Advice.to(InstallNativeRocksDB.class) - .on(named("configure").and(takesArguments(2))))) + builder + .visit( + Advice.to(InstallNativeRocksDB.class) + .on(named("configure").and(takesArguments(2)))) + .visit( + Advice.to(InstallLegacyNativeRocksDB.class) + .on(named("setStateBackend").and(takesArguments(1))))) .type(named(NATIVE_STATEFUL_OPERATOR)) .transform( (builder, type, classLoader, module, protectionDomain) -> @@ -502,6 +534,17 @@ static void enter(@Advice.Argument(0) Object configuration) { } try { Class configOption = Class.forName("org.apache.flink.configuration.ConfigOption"); + if (System.getProperty("streamfusion.flink-suite.flink-line", "2.2").equals("1.18")) { + Object changelogOption = + Class.forName("org.apache.flink.configuration.StateChangelogOptions") + .getField("ENABLE_STATE_CHANGE_LOG") + .get(null); + if (Boolean.TRUE.equals( + configuration + .getClass() + .getMethod("get", configOption) + .invoke(configuration, changelogOption))) return; + } Object backendOption = Class.forName("org.apache.flink.configuration.StateBackendOptions") .getField("STATE_BACKEND") @@ -536,6 +579,20 @@ static void enter(@Advice.Argument(0) Object configuration) { } } + /** Flink 1.18 upstream fixtures still use the legacy programmatic backend API. */ + public static final class InstallLegacyNativeRocksDB { + @Advice.OnMethodEnter + static void enter( + @Advice.This Object environment, + @Advice.Argument( + value = 0, + readOnly = false, + typing = net.bytebuddy.implementation.bytecode.assign.Assigner.Typing.DYNAMIC) + Object backend) { + backend = LegacyStateBackend.replace(environment, backend); + } + } + /** Proves that an upstream heap-backend case initialized StreamFusion's Rust hot-map state. */ public static final class ReportNativeMemoryState { @@ -620,4 +677,36 @@ static void exit(@Advice.Return(typing = Assigner.Typing.DYNAMIC) Object batch) } } } + + public static final class ApplyLegacyPaimonRestore { + @Advice.OnMethodExit + static void exit(@Advice.Argument(1) Object configuration, @Advice.Return Object pipeline) { + if (System.getProperty("streamfusion.flink-suite.flink-line", "2.2").equals("1.18") + && PaimonCheckpointFixture.isRecoveryTest()) { + PaimonCheckpointFixture.applyRestoreSettings(configuration, pipeline); + } + } + } + + public static final class BuildLegacyPaimonCatalog { + @Advice.OnMethodEnter(skipOn = Advice.OnNonDefaultValue.class) + static Object enter(@Advice.Origin Class fixture, @Advice.AllArguments Object[] arguments) { + if (!System.getProperty("streamfusion.flink-suite.flink-line", "2.2").equals("1.18")) { + return null; + } + return PaimonCatalogFixture.create( + fixture.getClassLoader(), + (java.util.Map) arguments[0], + (java.util.List) arguments[1], + (java.util.List) arguments[2], + (java.util.List) arguments[3]); + } + + @Advice.OnMethodExit + static void exit( + @Advice.Enter Object replacement, + @Advice.Return(readOnly = false, typing = Assigner.Typing.DYNAMIC) Object result) { + if (replacement != null) result = replacement; + } + } } diff --git a/dev/flink-suite/agent/src/main/resources/native-execution-flink1.18-state.tsv b/dev/flink-suite/agent/src/main/resources/native-execution-flink1.18-state.tsv new file mode 100644 index 000000000..1f7be58aa --- /dev/null +++ b/dev/flink-suite/agent/src/main/resources/native-execution-flink1.18-state.tsv @@ -0,0 +1,47 @@ +# Flink release-1.18.1 methods and fixture parameters, read from unchanged source. +# Retracting window TVF and Delta SQL cases are absent on this host line. +# Unchanged upstream methodfixture selectorexpected route. +# * matches every invocation; field=true/false selects a pinned fixture parameter. +# + requires every operator; | separates alternatives; ! requires full fallback and its reason. +org.apache.flink.table.planner.runtime.stream.sql.CalcITCase#testNotIn * NativeFilterOperator|NativeCalcOperator +org.apache.flink.table.planner.runtime.stream.sql.CalcITCase#testLongProjectionList * NativeCalcOperator +org.apache.flink.table.planner.runtime.stream.sql.CalcITCase#testIfFunction * NativeCalcOperator +org.apache.flink.table.planner.runtime.stream.sql.AggregateITCase#testGroupByAgg changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.AggregateITCase#testGroupByAgg changelog=false NativeColumnarGroupAggregateOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithGroupByAndRetract changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithGroupByAndRetract changelog=false NativeColumnarGroupAggregateOperator+NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithVariableTopSize changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithVariableTopSize changelog=false !Top-N: nullable variable rank bounds require Flink's row-access semantics +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testTumbleWindow changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testTumbleWindow splitDistinct=false&changelog=false NativeColumnarWindowAggregateOperator|NativeColumnarLocalWindowAggregateOperator+NativeColumnarGlobalWindowAggregateOperator +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testTumbleWindow splitDistinct=true&changelog=false !window aggregate: attached-window aggregation requires two-phase execution +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testHopWindow changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testHopWindow splitDistinct=false&changelog=false NativeColumnarWindowAggregateOperator|NativeColumnarLocalWindowAggregateOperator+NativeColumnarGlobalWindowAggregateOperator +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testHopWindow splitDistinct=true&changelog=false !window aggregate: attached-window aggregation requires two-phase execution +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testCumulateWindow changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testCumulateWindow splitDistinct=false&changelog=false NativeColumnarWindowAggregateOperator|NativeColumnarLocalWindowAggregateOperator+NativeColumnarGlobalWindowAggregateOperator +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testCumulateWindow splitDistinct=true&changelog=false !window aggregate: attached-window aggregation requires two-phase execution +org.apache.flink.table.planner.runtime.stream.sql.AsyncLookupJoinITCase#testAsyncJoinTemporalTable legacyTableSource=false NativeAsyncLookupJoinOperator +org.apache.flink.table.planner.runtime.stream.sql.AsyncLookupJoinITCase#testAsyncJoinTemporalTable legacyTableSource=true !lookup join: temporal table is not a (non-legacy) table source +org.apache.flink.table.planner.runtime.stream.sql.AsyncLookupJoinITCase#testAsyncJoinTemporalTableWithRetry legacyTableSource=false NativeAsyncLookupJoinOperator +org.apache.flink.table.planner.runtime.stream.sql.AsyncLookupJoinITCase#testAsyncJoinTemporalTableWithRetry legacyTableSource=true !lookup join: temporal table is not a (non-legacy) table source +org.apache.flink.table.planner.runtime.stream.sql.LookupJoinITCase#testJoinTemporalTable legacyTableSource=false NativeLookupJoinOperator +org.apache.flink.table.planner.runtime.stream.sql.LookupJoinITCase#testJoinTemporalTable legacyTableSource=true !lookup join: temporal table is not a (non-legacy) table source + +# Legacy upsert sinks must retain planner-proven keys after native replacement. +org.apache.flink.table.planner.runtime.stream.table.JoinITCase#testJoinKeyEqualsGroupByKey changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.table.JoinITCase#testJoinKeyEqualsGroupByKey changelog=false NativeColumnarUpdatingJoinOperator +org.apache.flink.table.planner.runtime.stream.table.JoinITCase#testJoinKeyNotEqualPrimaryKey changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.table.JoinITCase#testJoinKeyNotEqualPrimaryKey changelog=false NativeColumnarUpdatingJoinOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithUpsertSink changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithUpsertSink changelog=false NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testNestedTopN changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testNestedTopN changelog=false NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testMultipleUnaryTopNAfterAgg changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testMultipleUnaryTopNAfterAgg changelog=false NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTop1WithGroupByCount changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTop1WithGroupByCount changelog=false NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testUnarySortTopNOnString changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testUnarySortTopNOnString changelog=false NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.AggregateITCase#testConstantGroupKeyWithUpsertSink changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.AggregateITCase#testConstantGroupKeyWithUpsertSink changelog=false NativeColumnarGroupAggregateOperator diff --git a/dev/flink-suite/agent/src/main/resources/native-execution-flink1.18.tsv b/dev/flink-suite/agent/src/main/resources/native-execution-flink1.18.tsv new file mode 100644 index 000000000..f5341cd96 --- /dev/null +++ b/dev/flink-suite/agent/src/main/resources/native-execution-flink1.18.tsv @@ -0,0 +1,61 @@ +# Flink release-1.18.1 methods and fixture parameters, read from unchanged source. +# Retracting window TVF and Delta SQL cases are absent on this host line. +# Unchanged upstream methodfixture selectorexpected route. +# * matches every invocation; field=true/false selects a pinned fixture parameter. +# + requires every operator; | separates alternatives; ! requires full fallback and its reason. +org.apache.flink.table.planner.runtime.stream.sql.CalcITCase#testNotIn * NativeFilterOperator|NativeCalcOperator +org.apache.flink.table.planner.runtime.stream.sql.CalcITCase#testLongProjectionList * NativeCalcOperator +org.apache.flink.table.planner.runtime.stream.sql.CalcITCase#testIfFunction * NativeCalcOperator +org.apache.flink.table.planner.runtime.stream.sql.AggregateITCase#testGroupByAgg changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.AggregateITCase#testGroupByAgg state=HEAP&changelog=false NativeColumnarGroupAggregateOperator +org.apache.flink.table.planner.runtime.stream.sql.AggregateITCase#testGroupByAgg state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithGroupByAndRetract changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithGroupByAndRetract state=HEAP&changelog=false NativeColumnarGroupAggregateOperator+NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithGroupByAndRetract state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithVariableTopSize changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithVariableTopSize state=HEAP&changelog=false !Top-N: nullable variable rank bounds require Flink's row-access semantics +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithVariableTopSize state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testTumbleWindow changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testTumbleWindow state=HEAP&splitDistinct=false&changelog=false NativeColumnarWindowAggregateOperator|NativeColumnarLocalWindowAggregateOperator+NativeColumnarGlobalWindowAggregateOperator +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testTumbleWindow state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testTumbleWindow state=HEAP&splitDistinct=true&changelog=false !window aggregate: attached-window aggregation requires two-phase execution +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testHopWindow changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testHopWindow state=HEAP&splitDistinct=false&changelog=false NativeColumnarWindowAggregateOperator|NativeColumnarLocalWindowAggregateOperator+NativeColumnarGlobalWindowAggregateOperator +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testHopWindow state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testHopWindow state=HEAP&splitDistinct=true&changelog=false !window aggregate: attached-window aggregation requires two-phase execution +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testCumulateWindow changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testCumulateWindow state=HEAP&splitDistinct=false&changelog=false NativeColumnarWindowAggregateOperator|NativeColumnarLocalWindowAggregateOperator+NativeColumnarGlobalWindowAggregateOperator +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testCumulateWindow state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.WindowDistinctAggregateITCase#testCumulateWindow state=HEAP&splitDistinct=true&changelog=false !window aggregate: attached-window aggregation requires two-phase execution +org.apache.flink.table.planner.runtime.stream.sql.AsyncLookupJoinITCase#testAsyncJoinTemporalTable legacyTableSource=false NativeAsyncLookupJoinOperator +org.apache.flink.table.planner.runtime.stream.sql.AsyncLookupJoinITCase#testAsyncJoinTemporalTable legacyTableSource=true !lookup join: temporal table is not a (non-legacy) table source +org.apache.flink.table.planner.runtime.stream.sql.AsyncLookupJoinITCase#testAsyncJoinTemporalTableWithRetry legacyTableSource=false NativeAsyncLookupJoinOperator +org.apache.flink.table.planner.runtime.stream.sql.AsyncLookupJoinITCase#testAsyncJoinTemporalTableWithRetry legacyTableSource=true !lookup join: temporal table is not a (non-legacy) table source +org.apache.flink.table.planner.runtime.stream.sql.LookupJoinITCase#testJoinTemporalTable legacyTableSource=false NativeLookupJoinOperator +org.apache.flink.table.planner.runtime.stream.sql.LookupJoinITCase#testJoinTemporalTable legacyTableSource=true !lookup join: temporal table is not a (non-legacy) table source + +# Legacy upsert sinks must retain planner-proven keys after native replacement. +org.apache.flink.table.planner.runtime.stream.table.JoinITCase#testJoinKeyEqualsGroupByKey changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.table.JoinITCase#testJoinKeyEqualsGroupByKey state=HEAP&changelog=false NativeColumnarUpdatingJoinOperator +org.apache.flink.table.planner.runtime.stream.table.JoinITCase#testJoinKeyEqualsGroupByKey state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.table.JoinITCase#testJoinKeyNotEqualPrimaryKey changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.table.JoinITCase#testJoinKeyNotEqualPrimaryKey state=HEAP&changelog=false NativeColumnarUpdatingJoinOperator +org.apache.flink.table.planner.runtime.stream.table.JoinITCase#testJoinKeyNotEqualPrimaryKey state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithUpsertSink changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithUpsertSink state=HEAP&changelog=false NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTopNWithUpsertSink state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testNestedTopN changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testNestedTopN state=HEAP&changelog=false NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testNestedTopN state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testMultipleUnaryTopNAfterAgg changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testMultipleUnaryTopNAfterAgg state=HEAP&changelog=false NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testMultipleUnaryTopNAfterAgg state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTop1WithGroupByCount changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTop1WithGroupByCount state=HEAP&changelog=false NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testTop1WithGroupByCount state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testUnarySortTopNOnString changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testUnarySortTopNOnString state=HEAP&changelog=false NativeColumnarTopNOperator +org.apache.flink.table.planner.runtime.stream.sql.RankITCase#testUnarySortTopNOnString state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend +org.apache.flink.table.planner.runtime.stream.sql.AggregateITCase#testConstantGroupKeyWithUpsertSink changelog=true !state backend: Flink 1.18 changelog state is not verified for native keyed state +org.apache.flink.table.planner.runtime.stream.sql.AggregateITCase#testConstantGroupKeyWithUpsertSink state=HEAP&changelog=false NativeColumnarGroupAggregateOperator +org.apache.flink.table.planner.runtime.stream.sql.AggregateITCase#testConstantGroupKeyWithUpsertSink state=ROCKSDB&changelog=false !state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB backend diff --git a/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/NativeExecutionTest.java b/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/NativeExecutionTest.java index 9dcf33a7a..99f58cab3 100644 --- a/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/NativeExecutionTest.java +++ b/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/NativeExecutionTest.java @@ -1,6 +1,7 @@ package tech.streamfusion.suite; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -146,6 +147,8 @@ private static NativeExecution.Scope begin(String test) { } private static class WindowFixture { + private final String state = "HEAP"; + private final FakeScalaEnvironment env = new FakeScalaEnvironment(false); private final boolean splitDistinct; WindowFixture(boolean splitDistinct) { @@ -156,14 +159,16 @@ private static class WindowFixture { @Test void expectedFallbackRequiresItsSpecificReasonAndNoNativeWork() { NativeExecution.Scope scope = NativeExecution.begin(WINDOW, new WindowFixture(true)); - NativeExecution.fallback("window aggregate: attached-window aggregation requires two-phase execution"); + NativeExecution.fallback( + "window aggregate: attached-window aggregation requires two-phase execution"); NativeExecution.finish(scope); scope = NativeExecution.begin(WINDOW, new WindowFixture(true)); NativeExecution.fallback("different unsupported function"); NativeExecution.Scope wrongReason = scope; assertThrows(AssertionError.class, () -> NativeExecution.finish(wrongReason)); scope = NativeExecution.begin(WINDOW, new WindowFixture(true)); - NativeExecution.fallback("window aggregate: attached-window aggregation requires two-phase execution"); + NativeExecution.fallback( + "window aggregate: attached-window aggregation requires two-phase execution"); Object window = new NativeColumnarWindowAggregateOperator(); NativeExecution.opened(window); NativeExecution.completed(window, 1); @@ -176,6 +181,61 @@ void aChangedUpstreamFixtureFailsInsteadOfSkippingItsContract() { assertThrows(AssertionError.class, () -> NativeExecution.begin(WINDOW, new Object())); } + @Test + void selectorsReadInheritedStateAndConjoinFixtureParameters() { + class BackendFixture { + private final String state = "HEAP"; + } + class WindowBackendFixture extends BackendFixture { + private final boolean splitDistinct = false; + } + Object fixture = new WindowBackendFixture(); + assertTrue(NativeExecution.matches("state=HEAP&splitDistinct=false", fixture)); + assertFalse(NativeExecution.matches("state=ROCKSDB&splitDistinct=false", fixture)); + assertFalse(NativeExecution.matches("state=HEAP&splitDistinct=true", fixture)); + assertThrows(AssertionError.class, () -> NativeExecution.matches("missing=true", fixture)); + } + + @Test + void selectorsObserveTheFixturesActualRandomizedChangelogConfiguration() { + class Fixture { + private final FakeScalaEnvironment env; + + Fixture(boolean enabled) { + env = new FakeScalaEnvironment(enabled); + } + } + assertTrue(NativeExecution.matches("changelog=true", new Fixture(true))); + assertFalse(NativeExecution.matches("changelog=false", new Fixture(true))); + assertTrue(NativeExecution.matches("changelog=false", new Fixture(false))); + assertThrows( + AssertionError.class, () -> NativeExecution.matches("changelog=true", new Object())); + } + + public static final class FakeScalaEnvironment { + private final FakeJavaEnvironment javaEnvironment; + + FakeScalaEnvironment(boolean enabled) { + javaEnvironment = new FakeJavaEnvironment(enabled); + } + + public FakeJavaEnvironment getJavaEnv() { + return javaEnvironment; + } + } + + public static final class FakeJavaEnvironment { + private final boolean enabled; + + FakeJavaEnvironment(boolean enabled) { + this.enabled = enabled; + } + + public String isChangelogStateBackendEnabled() { + return enabled ? "TRUE" : "FALSE"; + } + } + private static class NativeCalcOperator {} private static class NativeColumnarGroupAggregateOperator {} diff --git a/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/PaimonCatalogFixtureTest.java b/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/PaimonCatalogFixtureTest.java new file mode 100644 index 000000000..136f1cdad --- /dev/null +++ b/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/PaimonCatalogFixtureTest.java @@ -0,0 +1,63 @@ +package tech.streamfusion.suite; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedCatalogTable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class PaimonCatalogFixtureTest { + @Test + void currentHostKeepsTheOriginalFixture() { + String key = "streamfusion.flink-suite.flink-line"; + String previous = System.getProperty(key); + try { + System.setProperty(key, "2.2"); + assertNull(StreamFusionSuiteAgent.BuildLegacyPaimonCatalog.enter(getClass(), null)); + } finally { + if (previous == null) System.clearProperty(key); + else System.setProperty(key, previous); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void preservesTheUpstreamFixtureMetadata(boolean keyed) { + var columns = + List.of( + Column.physical("k", DataTypes.INT().notNull()), + Column.physical("values", DataTypes.ARRAY(DataTypes.STRING()))); + var options = Map.of("path", "fixture-table", "bucket", "2"); + var table = + (ResolvedCatalogTable) + PaimonCatalogFixture.create( + getClass().getClassLoader(), + options, + columns, + List.of("k"), + keyed ? List.of("k") : List.of()); + assertEquals(columns, table.getResolvedSchema().getColumns()); + assertEquals(List.of(), table.getResolvedSchema().getWatermarkSpecs()); + assertEquals(options, table.getOptions()); + assertEquals(List.of("k"), table.getPartitionKeys()); + assertEquals("a comment", table.getComment()); + assertEquals( + Schema.newBuilder().fromResolvedSchema(table.getResolvedSchema()).build(), + table.getUnresolvedSchema()); + if (keyed) { + var primaryKey = table.getResolvedSchema().getPrimaryKey().orElseThrow(); + assertEquals("pk", primaryKey.getName()); + assertEquals(List.of("k"), primaryKey.getColumns()); + } else { + assertTrue(table.getResolvedSchema().getPrimaryKey().isEmpty()); + } + } +} diff --git a/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/PaimonCheckpointFixtureTest.java b/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/PaimonCheckpointFixtureTest.java new file mode 100644 index 000000000..8c4b61d1d --- /dev/null +++ b/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/PaimonCheckpointFixtureTest.java @@ -0,0 +1,122 @@ +package tech.streamfusion.suite; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +import net.bytebuddy.jar.asm.ClassWriter; +import net.bytebuddy.jar.asm.MethodVisitor; +import net.bytebuddy.jar.asm.Opcodes; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.jobgraph.SavepointConfigOptions; +import org.apache.flink.streaming.api.environment.CheckpointConfig.ExternalizedCheckpointCleanup; +import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; +import org.junit.jupiter.api.Test; + +class PaimonCheckpointFixtureTest { + @Test + void linksTheMovedFieldsToEquivalentReleasedSettings() throws Exception { + ClassWriter bytecode = new ClassWriter(ClassWriter.COMPUTE_MAXS); + String type = "tech/streamfusion/suite/LegacyRecoveryProbe"; + bytecode.visit(Opcodes.V17, Opcodes.ACC_PUBLIC, type, null, "java/lang/Object", null); + fieldMethod( + bytecode, + "retentionOption", + "CheckpointingOptions", + "EXTERNALIZED_CHECKPOINT_RETENTION", + "ConfigOption"); + fieldMethod( + bytecode, + "retentionValue", + "ExternalizedCheckpointRetention", + "RETAIN_ON_CANCELLATION", + "ExternalizedCheckpointRetention"); + fieldMethod( + bytecode, + "ignoreUnclaimedState", + "StateRecoveryOptions", + "SAVEPOINT_IGNORE_UNCLAIMED_STATE", + "ConfigOption"); + MethodVisitor path = + PaimonCheckpointFixture.adapt( + bytecode.visitMethod( + Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, + "restorePath", + "()Ljava/lang/String;", + null, + null)); + path.visitCode(); + path.visitLdcInsn("execution.state-recovery.path"); + path.visitInsn(Opcodes.ARETURN); + path.visitMaxs(0, 0); + path.visitEnd(); + bytecode.visitEnd(); + byte[] bytes = bytecode.toByteArray(); + Class probe = + new ClassLoader(getClass().getClassLoader()) { + Class loadProbe() { + return defineClass(null, bytes, 0, bytes.length); + } + }.loadProbe(); + + assertSame( + ExecutionCheckpointingOptions.EXTERNALIZED_CHECKPOINT, + probe.getMethod("retentionOption").invoke(null)); + assertSame( + ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION, + probe.getMethod("retentionValue").invoke(null)); + assertSame( + SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE, + probe.getMethod("ignoreUnclaimedState").invoke(null)); + assertEquals( + SavepointConfigOptions.SAVEPOINT_PATH.key(), probe.getMethod("restorePath").invoke(null)); + Configuration configuration = new Configuration(); + configuration.set( + ExecutionCheckpointingOptions.EXTERNALIZED_CHECKPOINT, + ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); + configuration.set(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE, true); + assertEquals( + "RETAIN_ON_CANCELLATION", + configuration.toMap().get("execution.checkpointing.externalized-checkpoint-retention")); + assertEquals("true", configuration.toMap().get("execution.savepoint.ignore-unclaimed-state")); + } + + @Test + void appliesTheRestorePathAndIgnorePolicyToTheLegacyStreamGraph() { + Configuration configuration = new Configuration(); + configuration.set(SavepointConfigOptions.SAVEPOINT_PATH, "file:///checkpoint/chk-1"); + configuration.set(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE, true); + var graph = + new org.apache.flink.streaming.api.graph.StreamGraph( + new org.apache.flink.api.common.ExecutionConfig(), + new org.apache.flink.streaming.api.environment.CheckpointConfig(), + org.apache.flink.runtime.jobgraph.SavepointRestoreSettings.none()); + PaimonCheckpointFixture.applyRestoreSettings(configuration, graph); + assertEquals("file:///checkpoint/chk-1", graph.getSavepointRestoreSettings().getRestorePath()); + assertEquals(true, graph.getSavepointRestoreSettings().allowNonRestoredState()); + PaimonCheckpointFixture.applyRestoreSettings(new Configuration(), graph); + assertSame( + org.apache.flink.runtime.jobgraph.SavepointRestoreSettings.none(), + graph.getSavepointRestoreSettings()); + } + + private static void fieldMethod( + ClassWriter bytecode, String method, String owner, String field, String fieldType) { + MethodVisitor visitor = + PaimonCheckpointFixture.adapt( + bytecode.visitMethod( + Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, + method, + "()Ljava/lang/Object;", + null, + null)); + visitor.visitCode(); + visitor.visitFieldInsn( + Opcodes.GETSTATIC, + "org/apache/flink/configuration/" + owner, + field, + "Lorg/apache/flink/configuration/" + fieldType + ";"); + visitor.visitInsn(Opcodes.ARETURN); + visitor.visitMaxs(0, 0); + visitor.visitEnd(); + } +} diff --git a/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/PaimonTestWatchTest.java b/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/PaimonTestWatchTest.java new file mode 100644 index 000000000..9d8c77b01 --- /dev/null +++ b/dev/flink-suite/agent/src/test/java/tech/streamfusion/suite/PaimonTestWatchTest.java @@ -0,0 +1,63 @@ +package tech.streamfusion.suite; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class PaimonTestWatchTest { + private final ByteArrayOutputStream output = new ByteArrayOutputStream(); + + @BeforeEach + void captureDiagnostics() { + PaimonTestWatch.initialize(new PrintStream(output, true, StandardCharsets.UTF_8)); + } + + @AfterEach + void restoreDiagnostics() { + PaimonTestWatch.initialize(System.err); + } + + @Test + void reportsFixturesWithoutRandomizedDefaults() { + PaimonTestWatch.start("ContinuousFileStoreITCase.testScanFromOldSchema", new Object()) + .finish(null); + String log = output.toString(StandardCharsets.UTF_8); + assertTrue(log.contains("test started ContinuousFileStoreITCase.testScanFromOldSchema at ")); + assertTrue(log.contains("randomized table defaults none")); + assertTrue( + log.contains("test finished ContinuousFileStoreITCase.testScanFromOldSchema after ")); + assertTrue(log.contains("seconds: passed")); + assertFalse(log.contains("unavailable")); + } + + @Test + void retainsInheritedRandomizedDefaults() { + PaimonTestWatch.start("DerivedFixture.testWrite", new DerivedFixture()).finish(null); + assertTrue( + output.toString(StandardCharsets.UTF_8).contains("{sink.writer-coordinator.enabled=true}")); + } + + @Test + void printsTheFailureBeforeTheReportIsWritten() { + PaimonTestWatch.start("ReadWriteTableITCase.testSinkParallelism", new Object()) + .finish(new IllegalStateException("fixture linkage failure")); + String log = output.toString(StandardCharsets.UTF_8); + assertTrue(log.contains("seconds: java.lang.IllegalStateException")); + assertTrue(log.contains("java.lang.IllegalStateException: fixture linkage failure")); + assertTrue(log.contains("at tech.streamfusion.suite.PaimonTestWatchTest.")); + } + + private static class BaseFixture { + private final Map tableDefaultProperties = + Map.of("sink.writer-coordinator.enabled", "true"); + } + + private static final class DerivedFixture extends BaseFixture {} +} diff --git a/dev/flink-suite/classpath-pom.xml b/dev/flink-suite/classpath-pom.xml index db9c57385..0a43e9688 100644 --- a/dev/flink-suite/classpath-pom.xml +++ b/dev/flink-suite/classpath-pom.xml @@ -6,60 +6,73 @@ tech.streamfusion streamfusion-flink-suite-classpath 1.0-SNAPSHOT + + + 5.0.0-2.2 + + + + flink-1.18 + + -flink1.18 + 3.2.0-1.18 + + + tech.streamfusion - streamfusion-core + streamfusion-core${streamfusion.artifact.suffix} 0.1.0-rc3 tech.streamfusion - streamfusion-kafka + streamfusion-kafka${streamfusion.artifact.suffix} 0.1.0-rc3 tech.streamfusion - streamfusion-json + streamfusion-json${streamfusion.artifact.suffix} 0.1.0-rc3 tech.streamfusion - streamfusion-csv + streamfusion-csv${streamfusion.artifact.suffix} 0.1.0-rc3 tech.streamfusion - streamfusion-raw + streamfusion-raw${streamfusion.artifact.suffix} 0.1.0-rc3 tech.streamfusion - streamfusion-avro + streamfusion-avro${streamfusion.artifact.suffix} 0.1.0-rc3 tech.streamfusion - streamfusion-avro-confluent-registry + streamfusion-avro-confluent-registry${streamfusion.artifact.suffix} 0.1.0-rc3 tech.streamfusion - streamfusion-protobuf + streamfusion-protobuf${streamfusion.artifact.suffix} 0.1.0-rc3 tech.streamfusion - streamfusion-parquet + streamfusion-parquet${streamfusion.artifact.suffix} 0.1.0-rc3 tech.streamfusion - streamfusion-orc + streamfusion-orc${streamfusion.artifact.suffix} 0.1.0-rc3 tech.streamfusion - streamfusion-paimon + streamfusion-paimon${streamfusion.artifact.suffix} 0.1.0-rc3 @@ -195,7 +199,7 @@ org.apache.flink flink-connector-kafka - 5.0.0-2.2 + ${flink.connector.kafka.version} provided + + org.apache.maven.plugins + maven-enforcer-plugin + 3.6.3 + + target-flink-line + enforce + + true + StreamFusion dependencies must target Flink ${flink.line}; select matching connector and format releases. + org.apache.flink:* + + org.apache.flink:*:${flink.dependency.range} + org.apache.flink:flink-shaded-* + org.apache.flink:flink-connector-kafka:[${flink.connector.kafka.version}] + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + flink-compatibility-sources + generate-sources + add-source + + ${project.basedir}/../src/main/java-flink${flink.line} + + + + + org.apache.maven.plugins maven-javadoc-plugin @@ -369,6 +408,14 @@ + + org.codehaus.mojo + flatten-maven-plugin + 1.7.3 + + inherited + + org.apache.maven.plugins maven-jar-plugin @@ -424,7 +471,7 @@ org.apache.paimon - paimon-flink-2.2 + paimon-flink-${flink.line} ${paimon.version} @@ -560,6 +607,65 @@ + + flink-1.18 + + 1.18.1 + 1.18 + -flink1.18 + [1.18.0,1.19.0) + 3.2.0-1.18 + 3.4.0 + + 1.32.0 + 3.21.7 + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.7.3 + + resolveCiFriendliesOnly + true + + + + flatten-release-pom + process-resources + + flatten + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.parquet + parquet-hadoop + 1.16.0 + + + org.apache.hadoop + hadoop-common + 3.3.4 + + + org.apache.paimon + paimon-flink-1.18 + ${paimon.version} + + + + + + + @@ -575,7 +681,7 @@ org.codehaus.mojo flatten-maven-plugin - 1.6.0 + 1.7.3 resolveCiFriendliesOnly true @@ -677,7 +783,7 @@ org.apache.calcite calcite-core - 1.36.0 + ${calcite.version} provided @@ -690,14 +796,14 @@ org.codehaus.mojo flatten-maven-plugin - 1.6.0 + 1.7.3 resolveCiFriendliesOnly true - flatten-flink-source-suite-pom + flatten-release-pom process-resources flatten diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/ColumnarPartitionerCompat.java b/src/main/java-flink1.18/tech/streamfusion/compat/ColumnarPartitionerCompat.java new file mode 100644 index 000000000..2eb547593 --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/ColumnarPartitionerCompat.java @@ -0,0 +1,11 @@ +package tech.streamfusion.compat; + +import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner; + +/** + * Flink 1.18 relies on the planner's aligned/recoverable exchange selection instead of an edge + * flag. + */ +public abstract class ColumnarPartitionerCompat extends StreamPartitioner { + protected final void disableUnalignedCheckpoints() {} +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/FileSinkCompat.java b/src/main/java-flink1.18/tech/streamfusion/compat/FileSinkCompat.java new file mode 100644 index 000000000..3ad9dd0cb --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/FileSinkCompat.java @@ -0,0 +1,109 @@ +package tech.streamfusion.compat; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.file.table.FileSystemConnectorOptions; +import org.apache.flink.connector.file.table.stream.PartitionCommitInfo; +import org.apache.flink.connector.file.table.stream.StreamingSink; +import org.apache.flink.core.fs.Path; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig; +import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink; +import tech.streamfusion.operator.NativeFileBulkWriterFactory; +import tech.streamfusion.operator.NativeFileRollingPolicy; +import tech.streamfusion.operator.PartitionedArrowBatch; +import tech.streamfusion.operator.PartitionedBatchBucketAssigner; + +/** Adapts the host file writer's package move without changing its bucket or commit semantics. */ +public final class FileSinkCompat { + private FileSinkCompat() {} + + public static String parquetSchemaShape() { + return "flink1.18"; + } + + public static org.apache.arrow.vector.types.pojo.Schema encoderSchema( + org.apache.flink.table.types.logical.RowType type) { + var schema = tech.streamfusion.arrow.ArrowConversion.toArrowSchema(type); + var fields = new java.util.ArrayList(); + for (int i = 0; i < type.getFieldCount(); i++) { + fields.add(withMapKeyNullability(schema.getFields().get(i), type.getTypeAt(i))); + } + return new org.apache.arrow.vector.types.pojo.Schema(fields); + } + + private static org.apache.arrow.vector.types.pojo.Field withMapKeyNullability( + org.apache.arrow.vector.types.pojo.Field field, + org.apache.flink.table.types.logical.LogicalType type) { + var children = new java.util.ArrayList<>(field.getChildren()); + var metadata = new java.util.HashMap<>(field.getMetadata()); + if (type instanceof org.apache.flink.table.types.logical.MapType + || type instanceof org.apache.flink.table.types.logical.MultisetType) { + var keyType = type.getChildren().get(0); + var valueType = + type instanceof org.apache.flink.table.types.logical.MapType + ? type.getChildren().get(1) + : new org.apache.flink.table.types.logical.IntType(false); + metadata.put("streamfusion:parquet_nullable_map_key", Boolean.toString(keyType.isNullable())); + var entries = children.get(0); + children.set( + 0, + new org.apache.arrow.vector.types.pojo.Field( + entries.getName(), + entries.getFieldType(), + java.util.List.of( + withMapKeyNullability(entries.getChildren().get(0), keyType), + withMapKeyNullability(entries.getChildren().get(1), valueType)))); + } else if (type instanceof org.apache.flink.table.types.logical.ArrayType + || type instanceof org.apache.flink.table.types.logical.RowType) { + for (int i = 0; i < children.size(); i++) { + children.set(i, withMapKeyNullability(children.get(i), type.getChildren().get(i))); + } + } + return new org.apache.arrow.vector.types.pojo.Field( + field.getName(), + new org.apache.arrow.vector.types.pojo.FieldType( + field.isNullable(), field.getType(), field.getDictionary(), metadata), + children); + } + + public static DataStream writer( + Configuration options, + Path location, + NativeFileBulkWriterFactory writerFactory, + DataStream stream, + int parallelism, + List partitionKeys, + boolean parallelismConfigured) { + StreamingFileSink.DefaultBulkFormatBuilder buckets = + StreamingFileSink.forBulkFormat(location, writerFactory) + .withBucketAssigner(new PartitionedBatchBucketAssigner()) + .withRollingPolicy( + new NativeFileRollingPolicy( + options + .get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_FILE_SIZE) + .getBytes(), + options + .get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_ROLLOVER_INTERVAL) + .toMillis(), + options + .get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_INACTIVITY_INTERVAL) + .toMillis())) + // The host's exact naming: a fresh UUID per sink keeps restarted or parallel writers + // from colliding on part-file names within a bucket. + .withOutputFileConfig( + OutputFileConfig.builder().withPartPrefix("part-" + UUID.randomUUID()).build()); + + return StreamingSink.writer( + name -> Optional.empty(), + stream, + options.get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_CHECK_INTERVAL).toMillis(), + buckets, + parallelism, + partitionKeys, + options, + parallelismConfigured); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/FixedSerializerTypeInformation.java b/src/main/java-flink1.18/tech/streamfusion/compat/FixedSerializerTypeInformation.java new file mode 100644 index 000000000..a94e108fc --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/FixedSerializerTypeInformation.java @@ -0,0 +1,15 @@ +package tech.streamfusion.compat; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.TypeSerializer; + +/** Adapts Flink's serializer factory signature for types with an explicit serializer. */ +public abstract class FixedSerializerTypeInformation extends TypeInformation { + @Override + public final TypeSerializer createSerializer(ExecutionConfig config) { + return createSerializer(); + } + + protected abstract TypeSerializer createSerializer(); +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/FlinkCompat.java b/src/main/java-flink1.18/tech/streamfusion/compat/FlinkCompat.java new file mode 100644 index 000000000..7a911abda --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/FlinkCompat.java @@ -0,0 +1,156 @@ +package tech.streamfusion.compat; + +import java.util.List; +import java.util.Map; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.flink.table.planner.plan.logical.WindowSpec; + +/** Planner constructs selected at build time for the target Flink line. */ +public final class FlinkCompat { + private FlinkCompat() {} + + public static final boolean GLOBAL_TTL_EMITS_UNCHANGED = false; + + public static int groupWindowTimeColumn( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGroupWindowAggregate + node) { + return node.getInput() + .getRowType() + .getFieldNames() + .indexOf(node.window().timeAttribute().getName()); + } + + public static final boolean EAGER_ROWTIME_KEEP_FIRST = true; + + public static Long singleStateTtl( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGroupAggregate node) { + return null; + } + + public static Long singleStateTtl( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGlobalGroupAggregate + node) { + return null; + } + + public static Map joinStateTtl(List hints) { + return Map.of(); + } + + public static boolean isSessionWindow(WindowSpec window) { + return false; + } + + public static long sessionGapMillis(WindowSpec window) { + throw new IllegalArgumentException("Flink 1.18 has no session window table function"); + } + + public static boolean isDeltaJoin(RelNode node) { + return false; + } + + public static String unescapeJsonPath(String path) { + return org.apache.flink.table.shaded.com.jayway.jsonpath.internal.Utils.unescape(path); + } + + public static boolean forceDeltaJoin(RelNode node) { + return false; + } + + public static boolean normalizeHasFilter( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize + node) { + return false; + } + + public static boolean normalizeSharesSource( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize + node) { + return false; + } + + public static java.util.Optional watermarkRowtime( + org.apache.flink.table.planner.plan.abilities.source.WatermarkPushDownSpec spec) { + return java.util.Optional.empty(); + } + + public static org.apache.flink.table.planner.plan.logical.WindowingStrategy windowDedupWindowing( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWindowDeduplicate + node) { + if (node instanceof PreparedWindowDeduplicate prepared) return prepared.windowing; + var query = + org.apache.flink.table.planner.plan.metadata.FlinkRelMetadataQuery.reuseOrCreate( + node.getCluster().getMetadataQuery()); + var properties = query.getRelWindowProperties(node.getInput()); + if (properties == null + || properties.getWindowStartColumns().isEmpty() + || properties.getWindowEndColumns().isEmpty()) return null; + return new org.apache.flink.table.planner.plan.logical.WindowAttachedWindowingStrategy( + properties.getWindowSpec(), properties.getTimeAttributeType(), + properties.getWindowStartColumns().nth(0), properties.getWindowEndColumns().nth(0)); + } + + public static RelNode prepareWindowDeduplicate( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWindowDeduplicate + node, + boolean keepLast) { + var windowing = windowDedupWindowing(node); + return windowing == null + ? node + : new PreparedWindowDeduplicate( + node, node.getTraitSet(), node.getInput(), keepLast, windowing); + } + + /** Capture released planner metadata before its input is replaced by native relations. */ + private static final class PreparedWindowDeduplicate + extends org.apache.flink.table.planner.plan.nodes.physical.stream + .StreamPhysicalWindowDeduplicate { + private final boolean keepLast; + private final org.apache.flink.table.planner.plan.logical.WindowingStrategy windowing; + + PreparedWindowDeduplicate( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWindowDeduplicate + node, + org.apache.calcite.plan.RelTraitSet traits, + RelNode input, + boolean keepLast, + org.apache.flink.table.planner.plan.logical.WindowingStrategy windowing) { + super( + node.getCluster(), + traits, + input, + node.partitionKeys(), + node.orderKey(), + keepLast, + windowing); + this.keepLast = keepLast; + this.windowing = windowing; + } + + @Override + public RelNode copy(org.apache.calcite.plan.RelTraitSet traits, List inputs) { + return new PreparedWindowDeduplicate(this, traits, inputs.get(0), keepLast, windowing); + } + } + + public static java.util.List + sinkAbilities( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalSink sink) { + return ((org.apache.flink.table.planner.plan.nodes.exec.common.CommonExecSink) + sink.translateToExecNode()) + .getTableSinkSpec() + .getSinkAbilities(); + } + + public static String expressionClassName( + org.apache.flink.table.planner.codegen.CodeGeneratorContext context) { + return org.apache.flink.table.planner.codegen.CodeGenUtils.newName("FlinkExpressionEvaluator"); + } + + public static org.apache.flink.table.planner.functions.casting.CastRule.Context castContext( + ClassLoader classLoader) { + return org.apache.flink.table.planner.functions.casting.CastRule.Context.create( + false, false, java.time.ZoneId.of("UTC"), classLoader); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/FlinkLookupCompat.java b/src/main/java-flink1.18/tech/streamfusion/compat/FlinkLookupCompat.java new file mode 100644 index 000000000..88261751b --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/FlinkLookupCompat.java @@ -0,0 +1,206 @@ +package tech.streamfusion.compat; + +import java.util.*; +import java.util.concurrent.CompletableFuture; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; +import org.apache.flink.api.common.functions.*; +import org.apache.flink.api.common.functions.util.FunctionUtils; +import org.apache.flink.api.dag.Transformation; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.streaming.api.functions.async.*; +import org.apache.flink.table.catalog.DataTypeFactory; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.conversion.DataStructureConverter; +import org.apache.flink.table.functions.*; +import org.apache.flink.table.planner.codegen.LookupJoinCodeGenerator; +import org.apache.flink.table.planner.delegation.PlannerBase; +import org.apache.flink.table.planner.plan.nodes.exec.utils.TransformationMetadata; +import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalLookupJoin; +import org.apache.flink.table.planner.plan.utils.*; +import org.apache.flink.table.runtime.collector.*; +import org.apache.flink.table.runtime.generated.*; +import org.apache.flink.table.runtime.operators.join.lookup.*; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.types.logical.RowType; + +/** The released host's lookup code generation and lifecycle APIs, selected at build time. */ +public final class FlinkLookupCompat { + private FlinkLookupCompat() {} + + public static LookupKeys lookupKeys(StreamPhysicalLookupJoin join) { + Map keys = new HashMap<>(); + scala.collection.JavaConverters.mapAsJavaMapConverter(join.allLookupKeys()) + .asJava() + .forEach((index, param) -> keys.put((Integer) index, param)); + return new LookupKeys(keys); + } + + public static String unsupportedKeyShape(LookupKeys keys) { + for (LookupJoinUtil.LookupKey key : keys.values().values()) { + if (!(key instanceof LookupJoinUtil.FieldRefLookupKey) + && !(key instanceof LookupJoinUtil.ConstantLookupKey)) { + return "lookup join: unsupported lookup key shape " + key.getClass().getSimpleName(); + } + } + return null; + } + + public static LookupAsyncOptions asyncOptions(StreamPhysicalLookupJoin join) { + if (join.asyncOptions().isEmpty()) return null; + var options = join.asyncOptions().get(); + return new LookupAsyncOptions( + options.asyncBufferCapacity, options.asyncTimeout, false, options.asyncOutputMode); + } + + public static RexNode preFilter(StreamPhysicalLookupJoin join) { + return null; + } + + public static RexNode remainingCondition(StreamPhysicalLookupJoin join) { + var condition = join.remainingCondition(); + return condition.isDefined() ? condition.get() : null; + } + + public static boolean preferCustomShuffle(StreamPhysicalLookupJoin join) { + return false; + } + + public static ExpandedLookupCalc expandCalc(RexProgram calc) { + var expanded = FlinkRexUtil.expandRexProgram(calc); + return new ExpandedLookupCalc( + scala.collection.JavaConverters.seqAsJavaListConverter(expanded._1()).asJava(), + expanded._2().isDefined() ? expanded._2().get() : null); + } + + public static UserDefinedFunction lookupFunction( + RelOptTable table, + LookupKeys keys, + ClassLoader classLoader, + boolean async, + ResultRetryStrategy retry, + boolean customShuffle) { + return LookupJoinUtil.getLookupFunction(table, keys.keySet(), classLoader, async, retry); + } + + public static Transformation customShuffle( + PlannerBase planner, + RelOptTable table, + RowType probeType, + LookupKeys keys, + Transformation rows, + ChangelogMode mode, + TransformationMetadata metadata) { + throw new UnsupportedOperationException("Flink 1.18 has no custom lookup partitioner SPI"); + } + + public static GeneratedAsyncFetcher asyncFetcher( + ReadableConfig config, + ClassLoader classLoader, + DataTypeFactory types, + RowType probeType, + RowType tableType, + RowType resultType, + LookupKeys keys, + AsyncTableFunction function, + String tableName) { + var generated = + LookupJoinCodeGenerator.generateAsyncLookupFunction( + config, + classLoader, + types, + probeType, + tableType, + resultType, + keys.values(), + LookupJoinUtil.getOrderedLookupKeys(keys.keySet()), + function, + tableName); + return new GeneratedAsyncFetcher(generated.tableFunc(), generated.dataType()); + } + + public static GeneratedFunction> syncFetcher( + ReadableConfig config, + ClassLoader classLoader, + DataTypeFactory types, + RowType probeType, + RowType tableType, + RowType resultType, + LookupKeys keys, + TableFunction function, + String tableName, + boolean objectReuse) { + return LookupJoinCodeGenerator.generateSyncLookupFunction( + config, + classLoader, + types, + probeType, + tableType, + resultType, + keys.values(), + LookupJoinUtil.getOrderedLookupKeys(keys.keySet()), + function, + tableName, + objectReuse); + } + + public static LookupJoinRunner syncRunner( + GeneratedFunction> fetcher, + GeneratedFunction> calc, + GeneratedCollector> collector, + ReadableConfig config, + ClassLoader classLoader, + RexNode preFilter, + RowType probeType, + boolean leftOuter, + int rightFields) { + + return calc == null + ? new LookupJoinRunner(fetcher, collector, leftOuter, rightFields) + : new LookupJoinWithCalcRunner(fetcher, calc, collector, leftOuter, rightFields); + } + + public static AsyncLookupJoinRunner asyncRunner( + GeneratedFunction> fetcher, + DataStructureConverter converter, + GeneratedFunction> calc, + GeneratedResultFuture> collector, + ReadableConfig config, + ClassLoader classLoader, + RexNode preFilter, + RowType probeType, + RowDataSerializer rightSerializer, + boolean leftOuter, + int capacity) { + + return calc == null + ? new AsyncLookupJoinRunner( + fetcher, converter, collector, rightSerializer, leftOuter, capacity) + : new AsyncLookupJoinWithCalcRunner( + fetcher, converter, calc, collector, rightSerializer, leftOuter, capacity); + } + + public static void open(Function function) throws Exception { + FunctionUtils.openFunction(function, new org.apache.flink.configuration.Configuration()); + } + + public static boolean preFilter(LookupJoinRunner runner, RowData probe) throws Exception { + return true; + } + + public static ResultFuture resultFuture(CompletableFuture> future) { + return new ResultFuture<>() { + @Override + public void complete(Collection result) { + future.complete(result); + } + + @Override + public void completeExceptionally(Throwable failure) { + future.completeExceptionally(failure); + } + }; + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/FlinkOutput.java b/src/main/java-flink1.18/tech/streamfusion/compat/FlinkOutput.java new file mode 100644 index 000000000..024601bf6 --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/FlinkOutput.java @@ -0,0 +1,40 @@ +package tech.streamfusion.compat; + +import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; +import org.apache.flink.util.OutputTag; + +/** Forwards host control events without changing the record representation. */ +public abstract class FlinkOutput implements Output> { + private final Output delegate; + + protected FlinkOutput(Output delegate) { + this.delegate = delegate; + } + + @Override + public void emitWatermark(Watermark watermark) { + delegate.emitWatermark(watermark); + } + + @Override + public void emitWatermarkStatus(WatermarkStatus status) { + delegate.emitWatermarkStatus(status); + } + + @Override + public void collect(OutputTag tag, StreamRecord record) { + delegate.collect(tag, record); + } + + @Override + public void emitLatencyMarker(LatencyMarker marker) { + delegate.emitLatencyMarker(marker); + } + + @Override + public void close() {} +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/FlinkStateBackendCompat.java b/src/main/java-flink1.18/tech/streamfusion/compat/FlinkStateBackendCompat.java new file mode 100644 index 000000000..1a05ac449 --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/FlinkStateBackendCompat.java @@ -0,0 +1,173 @@ +package tech.streamfusion.compat; + +import java.util.Collection; +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.contrib.streaming.state.EmbeddedRocksDBStateBackend; +import org.apache.flink.core.fs.CloseableRegistry; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.query.TaskKvStateRegistry; +import org.apache.flink.runtime.state.*; +import org.apache.flink.runtime.state.ttl.TtlTimeProvider; + +/** Adapts the host backend factory signature without owning native state or snapshot bytes. */ +public abstract class FlinkStateBackendCompat implements StateBackend { + protected final StateBackend delegate; + + public static String unsupportedNativeStateReason( + org.apache.flink.streaming.api.environment.StreamExecutionEnvironment environment, + ReadableConfig tableConfig) { + boolean changelog = + environment == null + ? tableConfig + .getOptional( + org.apache.flink.configuration.StateChangelogOptions.ENABLE_STATE_CHANGE_LOG) + .orElse(false) + : environment.isChangelogStateBackendEnabled() + == org.apache.flink.util.TernaryBoolean.TRUE; + if (changelog) { + return "state backend: Flink 1.18 changelog state is not verified for native keyed state"; + } + StateBackend backend = environment == null ? null : environment.getStateBackend(); + if (backend instanceof org.apache.flink.runtime.state.hashmap.HashMapStateBackend + || backend instanceof org.apache.flink.runtime.state.memory.MemoryStateBackend + || backend instanceof org.apache.flink.runtime.state.filesystem.FsStateBackend + // The deployed planner and host load the same final backend in separate classloaders. + || (backend != null + && backend + .getClass() + .getName() + .equals(tech.streamfusion.state.RocksDBNativeStateBackend.class.getName()))) { + return null; + } + var option = org.apache.flink.configuration.StateBackendOptions.STATE_BACKEND; + String configured = + environment == null + ? tableConfig.getOptional(option).orElse("hashmap") + : environment + .getConfiguration() + .getOptional(option) + .orElse(tableConfig.getOptional(option).orElse("hashmap")); + if (backend == null + && java.util.Set.of( + "hashmap", + "jobmanager", + "filesystem", + "memory", + "tech.streamfusion.state.RocksDBNativeStateBackendFactory") + .contains(configured)) { + return null; + } + return "state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion" + + " RocksDB backend"; + } + + protected FlinkStateBackendCompat(ReadableConfig config, ClassLoader classLoader) { + delegate = new EmbeddedRocksDBStateBackend().configure(config, classLoader); + } + + protected abstract CheckpointableKeyedStateBackend createNativeKeyedBackend( + KeyedBackendContext context) throws Exception; + + @Override + public boolean useManagedMemory() { + return delegate.useManagedMemory(); + } + + @Override + public final CheckpointableKeyedStateBackend createKeyedStateBackend( + Environment env, + JobID jobId, + String operatorIdentifier, + TypeSerializer keySerializer, + int numberOfKeyGroups, + KeyGroupRange keyGroupRange, + TaskKvStateRegistry kvStateRegistry, + TtlTimeProvider ttlTimeProvider, + MetricGroup metricGroup, + Collection stateHandles, + CloseableRegistry cancelStreamRegistry) + throws Exception { + return createKeyedStateBackend( + env, + jobId, + operatorIdentifier, + keySerializer, + numberOfKeyGroups, + keyGroupRange, + kvStateRegistry, + ttlTimeProvider, + metricGroup, + stateHandles, + cancelStreamRegistry, + 0.0); + } + + @Override + public final CheckpointableKeyedStateBackend createKeyedStateBackend( + Environment env, + JobID jobId, + String operatorIdentifier, + TypeSerializer keySerializer, + int numberOfKeyGroups, + KeyGroupRange keyGroupRange, + TaskKvStateRegistry kvStateRegistry, + TtlTimeProvider ttlTimeProvider, + MetricGroup metricGroup, + Collection stateHandles, + CloseableRegistry cancelStreamRegistry, + double managedMemoryFraction) + throws Exception { + return createNativeKeyedBackend( + new KeyedBackendContext<>( + env, + jobId, + operatorIdentifier, + keySerializer, + numberOfKeyGroups, + keyGroupRange, + stateHandles, + managedMemoryFraction, + handles -> + delegate.createKeyedStateBackend( + env, + jobId, + operatorIdentifier, + keySerializer, + numberOfKeyGroups, + keyGroupRange, + kvStateRegistry, + ttlTimeProvider, + metricGroup, + handles, + cancelStreamRegistry, + managedMemoryFraction), + handles -> + new org.apache.flink.runtime.state.hashmap.HashMapStateBackend() + .createKeyedStateBackend( + env, + jobId, + operatorIdentifier, + keySerializer, + numberOfKeyGroups, + keyGroupRange, + kvStateRegistry, + ttlTimeProvider, + metricGroup, + handles, + cancelStreamRegistry))); + } + + @Override + public final OperatorStateBackend createOperatorStateBackend( + Environment env, + String operatorIdentifier, + Collection stateHandles, + CloseableRegistry cancelStreamRegistry) + throws Exception { + return delegate.createOperatorStateBackend( + env, operatorIdentifier, stateHandles, cancelStreamRegistry); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/FlinkStreamOperator.java b/src/main/java-flink1.18/tech/streamfusion/compat/FlinkStreamOperator.java new file mode 100644 index 000000000..4a302b86b --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/FlinkStreamOperator.java @@ -0,0 +1,13 @@ +package tech.streamfusion.compat; + +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.ChainingStrategy; + +/** Keep columnar operators in the upstream task chain when Flink's topology permits it. */ +public abstract class FlinkStreamOperator extends AbstractStreamOperator { + private static final long serialVersionUID = 1L; + + protected FlinkStreamOperator() { + setChainingStrategy(ChainingStrategy.ALWAYS); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/JsonRuntimeCompat.java b/src/main/java-flink1.18/tech/streamfusion/compat/JsonRuntimeCompat.java new file mode 100644 index 000000000..408caf153 --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/JsonRuntimeCompat.java @@ -0,0 +1,22 @@ +package tech.streamfusion.compat; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonFactory; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.util.BufferRecycler; + +/** The buffer-recycler contract verified for the line's released Jackson runtime. */ +public final class JsonRuntimeCompat { + private JsonRuntimeCompat() {} + + public static final boolean ACCEPTS_ARRAY_ROOTS = false; + + public static final boolean PRESERVES_DECIMAL_SCALE = false; + + public static boolean verifiedFactory(JsonFactory factory) { + return false; + } + + public static void releaseToPool(BufferRecycler recycler) { + // Flink 1.18's Jackson predates recycler pools; verifiedFactory disables native buffer + // emulation. + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/LookupKeys.java b/src/main/java-flink1.18/tech/streamfusion/compat/LookupKeys.java new file mode 100644 index 000000000..63d20566a --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/LookupKeys.java @@ -0,0 +1,11 @@ +package tech.streamfusion.compat; + +import java.util.Map; +import java.util.Set; +import org.apache.flink.table.planner.plan.utils.LookupJoinUtil; + +public record LookupKeys(Map values) { + public Set keySet() { + return values.keySet(); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/ProjectedRowDataCompat.java b/src/main/java-flink1.18/tech/streamfusion/compat/ProjectedRowDataCompat.java new file mode 100644 index 000000000..f2f236f71 --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/ProjectedRowDataCompat.java @@ -0,0 +1,10 @@ +package tech.streamfusion.compat; + +import org.apache.flink.table.data.RowData; + +/** The RowData methods introduced after Flink 1.18, backed by the shared projection. */ +public abstract class ProjectedRowDataCompat implements RowData { + protected abstract RowData sourceRow(); + + protected abstract int sourceIndex(int pos); +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/RocksDBOptionsCompat.java b/src/main/java-flink1.18/tech/streamfusion/compat/RocksDBOptionsCompat.java new file mode 100644 index 000000000..d3913cf6d --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/RocksDBOptionsCompat.java @@ -0,0 +1,37 @@ +package tech.streamfusion.compat; + +import java.util.List; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.MemorySize; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.contrib.streaming.state.RocksDBConfigurableOptions; +import org.apache.flink.contrib.streaming.state.RocksDBOptions; +import org.rocksdb.CompressionType; + +/** The selected Flink line owns option keys and defaults. */ +public final class RocksDBOptionsCompat extends RocksDBConfigurableOptions { + private RocksDBOptionsCompat() {} + + public static final ConfigOption PREDEFINED_OPTIONS = RocksDBOptions.PREDEFINED_OPTIONS; + public static final ConfigOption OPTIONS_FACTORY = RocksDBOptions.OPTIONS_FACTORY; + public static final ConfigOption LOCAL_DIRECTORIES = RocksDBOptions.LOCAL_DIRECTORIES; + public static final ConfigOption USE_MANAGED_MEMORY = RocksDBOptions.USE_MANAGED_MEMORY; + public static final ConfigOption FIX_PER_SLOT_MEMORY_SIZE = + RocksDBOptions.FIX_PER_SLOT_MEMORY_SIZE; + public static final ConfigOption FIX_PER_TM_MEMORY_SIZE = + RocksDBOptions.FIX_PER_TM_MEMORY_SIZE; + public static final ConfigOption WRITE_BUFFER_RATIO = RocksDBOptions.WRITE_BUFFER_RATIO; + + // These backend-wide options were added after 1.18. Preserve the older host defaults. + public static List compressionPerLevel(ReadableConfig config) { + return List.of(); + } + + public static long compactionQueryEntries(ReadableConfig config) { + return 1000L; + } + + public static long periodicCompactionSeconds(ReadableConfig config) { + return 0L; + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/RuntimeCompat.java b/src/main/java-flink1.18/tech/streamfusion/compat/RuntimeCompat.java new file mode 100644 index 000000000..fd2e63e3c --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/RuntimeCompat.java @@ -0,0 +1,51 @@ +package tech.streamfusion.compat; + +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.runtime.operators.coordination.OperatorEventDispatcher; +import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; + +public final class RuntimeCompat { + private RuntimeCompat() {} + + public static org.apache.flink.table.data.RowData copyRow( + org.apache.flink.table.runtime.typeutils.RowDataSerializer serializer, + org.apache.flink.table.data.RowData row) { + // In 1.18, copying a custom ArrayData view can return the serializer's reused buffer. + return serializer.toBinaryRow(row).copy(); + } + + public static int attempt(RuntimeContext context) { + return context.getAttemptNumber(); + } + + public static String taskName(RuntimeContext context) { + return context.getTaskNameWithSubtasks(); + } + + public static int parallelism(RuntimeContext context) { + return context.getNumberOfParallelSubtasks(); + } + + public static int subtask(RuntimeContext context) { + return context.getIndexOfThisSubtask(); + } + + public static StreamOperatorParameters withOutput( + StreamOperatorParameters parameters, Output> output) { + return withOutput(parameters, output, parameters.getOperatorEventDispatcher()); + } + + public static StreamOperatorParameters withOutput( + StreamOperatorParameters parameters, + Output> output, + OperatorEventDispatcher dispatcher) { + return new StreamOperatorParameters<>( + parameters.getContainingTask(), + parameters.getStreamConfig(), + output, + parameters::getProcessingTimeService, + dispatcher); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/StateCompat.java b/src/main/java-flink1.18/tech/streamfusion/compat/StateCompat.java new file mode 100644 index 000000000..de4c5a789 --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/StateCompat.java @@ -0,0 +1,56 @@ +package tech.streamfusion.compat; + +import java.util.List; +import java.util.stream.Stream; +import org.apache.flink.runtime.state.*; +import tech.streamfusion.state.RocksDBNativeKeyedStateBackend; + +/** Preserves the host line's key context and incremental checkpoint ownership APIs. */ +public final class StateCompat { + private StateCompat() {} + + public static void setCurrentKeyAndGroup( + CheckpointableKeyedStateBackend backend, K key, int group) { + if (backend instanceof RocksDBNativeKeyedStateBackend nativeBackend) { + nativeBackend.setCurrentKeyAndKeyGroup(key, group); + } else if (backend + instanceof org.apache.flink.runtime.state.heap.HeapKeyedStateBackend host) { + // The canonical key identifies a partition; its hash is unrelated to that partition's group. + host.getKeyContext().setCurrentKey(key); + host.getKeyContext().setCurrentKeyGroupIndex(group); + } else if (backend instanceof AbstractKeyedStateBackend host + && KeyGroupRangeAssignment.assignToKeyGroup(key, host.getNumberOfKeyGroups()) == group) { + host.setCurrentKey(key); + } else { + throw new IllegalStateException( + "Backend cannot restore an explicit key group: " + backend.getClass().getName()); + } + } + + public static Stream keys( + CheckpointableKeyedStateBackend backend, List states, N namespace) { + return states.stream().flatMap(state -> backend.getKeys(state, namespace)).distinct(); + } + + public static String backendType(CheckpointableKeyedStateBackend backend) { + return backend.getClass().getName(); + } + + public static boolean couldReuse(CheckpointStreamFactory factory, StreamStateHandle handle) { + return true; + } + + public static StreamStateHandle placeholder(StreamStateHandle handle) { + return new PlaceholderStreamStateHandle(handle.getStreamStateHandleID(), handle.getStateSize()); + } + + public static void reused(CheckpointStreamFactory factory, List handles) + throws java.io.IOException { + // 1.18 registers reused placeholders through the shared-state registry, without a factory + // callback. + } + + public static StreamStateHandle metaHandle(IncrementalRemoteKeyedStateHandle handle) { + return handle.getMetaStateHandle(); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/compat/TableErrors.java b/src/main/java-flink1.18/tech/streamfusion/compat/TableErrors.java new file mode 100644 index 000000000..63dbdbdb0 --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/compat/TableErrors.java @@ -0,0 +1,14 @@ +package tech.streamfusion.compat; + +/** Preserves the host's generated SINGLE_VALUE failure type across Flink lines. */ +public final class TableErrors { + private TableErrors() {} + + public static Class exceptionType() { + return RuntimeException.class; + } + + public static void fail(String message) { + throw new RuntimeException(message); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/format/avro/compat/AvroCompat.java b/src/main/java-flink1.18/tech/streamfusion/format/avro/compat/AvroCompat.java new file mode 100644 index 000000000..b0b182345 --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/format/avro/compat/AvroCompat.java @@ -0,0 +1,27 @@ +package tech.streamfusion.format.avro.compat; + +import org.apache.avro.Schema; +import org.apache.flink.formats.avro.AvroToRowDataConverters; +import org.apache.flink.formats.avro.RowDataToAvroConverters; +import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; + +/** Resolves schemas and converters using the timestamp mappings the released format exposes. */ +public final class AvroCompat { + private AvroCompat() {} + + public static Schema schema(LogicalType type, boolean legacy) { + if (!legacy) + throw new IllegalArgumentException("Flink 1.18 has only the legacy Avro timestamp mapping"); + return AvroSchemaConverter.convertToSchema(type); + } + + public static void validateDecoder(RowType type, boolean legacy) { + AvroToRowDataConverters.createRowConverter(type); + } + + public static void validateEncoder(RowType type, boolean legacy) { + RowDataToAvroConverters.createConverter(type); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaFetcherManagerCompat.java b/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaFetcherManagerCompat.java new file mode 100644 index 000000000..fbeaba79a --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaFetcherManagerCompat.java @@ -0,0 +1,30 @@ +package tech.streamfusion.kafka.compat; + +import java.util.function.Supplier; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; +import org.apache.flink.connector.base.source.reader.fetcher.SingleThreadFetcherManager; +import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; +import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue; +import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit; +import tech.streamfusion.operator.NativeSourceRecord; + +/** Preserves the released source reader's queue ownership between its fetcher and consumer. */ +public abstract class KafkaFetcherManagerCompat + extends SingleThreadFetcherManager { + final FutureCompletingBlockingQueue> queue; + + protected KafkaFetcherManagerCompat( + Supplier> supplier, + Configuration config) { + this(new FutureCompletingBlockingQueue<>(), supplier, config); + } + + private KafkaFetcherManagerCompat( + FutureCompletingBlockingQueue> queue, + Supplier> supplier, + Configuration config) { + super(queue, supplier, config); + this.queue = queue; + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaRecordSchemaCompat.java b/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaRecordSchemaCompat.java new file mode 100644 index 000000000..41b454f65 --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaRecordSchemaCompat.java @@ -0,0 +1,10 @@ +package tech.streamfusion.kafka.compat; + +import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema; +import tech.streamfusion.kafka.PreSerializedKafkaRecord; + +/** Retains connector lineage on releases that provide that SPI. */ +public abstract class KafkaRecordSchemaCompat + implements KafkaRecordSerializationSchema { + protected abstract String topic(); +} diff --git a/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaSinkCompat.java b/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaSinkCompat.java new file mode 100644 index 000000000..0c0f66d12 --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaSinkCompat.java @@ -0,0 +1,19 @@ +package tech.streamfusion.kafka.compat; + +import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder; + +/** Transaction naming is a connector option introduced after the 1.18 connector release. */ +public final class KafkaSinkCompat { + private KafkaSinkCompat() {} + + public static String transactionNaming(String option) { + if (option != null) + throw new IllegalArgumentException("Flink 1.18 has no transaction naming strategy option"); + return "DEFAULT"; + } + + public static void setTransactionNaming(KafkaSinkBuilder builder, String strategy) { + if (!"DEFAULT".equals(strategy)) + throw new IllegalArgumentException("Unsupported transaction naming: " + strategy); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaSourceReaderCompat.java b/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaSourceReaderCompat.java new file mode 100644 index 000000000..6de344c2a --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/kafka/compat/KafkaSourceReaderCompat.java @@ -0,0 +1,23 @@ +package tech.streamfusion.kafka.compat; + +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.base.source.reader.RecordEmitter; +import org.apache.flink.connector.base.source.reader.SingleThreadMultiplexSourceReaderBase; +import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit; +import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplitState; +import tech.streamfusion.operator.ArrowBatch; +import tech.streamfusion.operator.NativeSourceRecord; + +/** Supplies the same fetcher-owned queue to the older reader constructor. */ +public abstract class KafkaSourceReaderCompat + extends SingleThreadMultiplexSourceReaderBase< + NativeSourceRecord, ArrowBatch, KafkaPartitionSplit, KafkaPartitionSplitState> { + protected KafkaSourceReaderCompat( + KafkaFetcherManagerCompat fetcher, + RecordEmitter emitter, + Configuration config, + SourceReaderContext context) { + super(fetcher.queue, fetcher, emitter, config, context); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/planner/FlinkPlannerCompat.java b/src/main/java-flink1.18/tech/streamfusion/planner/FlinkPlannerCompat.java new file mode 100644 index 000000000..014f8757e --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/planner/FlinkPlannerCompat.java @@ -0,0 +1,72 @@ +package tech.streamfusion.planner; + +import java.util.List; +import org.apache.calcite.rel.RelNode; +import org.apache.flink.table.planner.calcite.FlinkTypeFactory$; +import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalDeduplicate; +import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalRel; +import org.apache.flink.table.planner.plan.utils.ChangelogPlanUtils; +import tech.streamfusion.operator.RowDataArrowConverter; + +/** Flink 1.18 uses a dedicated physical node instead of a rank-1 node. */ +final class FlinkPlannerCompat { + private FlinkPlannerCompat() {} + + static RelNode prepareForRewrite(RelNode node) { + if (node + instanceof + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalLegacySink + sink) { + return PreparedLegacySink.prepare(sink); + } + if (node + instanceof + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWindowDeduplicate + dedup) { + Boolean keepLast = WindowDeduplicateMatcher.keepLastRow(dedup); + if (keepLast != null) + return tech.streamfusion.compat.FlinkCompat.prepareWindowDeduplicate(dedup, keepLast); + } + return node; + } + + static void addSubstitutions(List> entries) { + entries.add( + Substitution.of( + StreamPhysicalDeduplicate.class, "deduplicate", FlinkPlannerCompat::deduplicate) + .keyedState() + .matching( + node -> + ChangelogPlanUtils.isInsertOnly((StreamPhysicalRel) node.getInput()) + && RowDataArrowConverter.supports( + FlinkTypeFactory$.MODULE$.toLogicalRowType(node.getRowType())) + && (!node.isRowtime() || rowtimeColumn(node) >= 0)) + .reason( + node -> + "deduplication: needs insert-only input, supported Arrow columns and a rowtime" + + " attribute for event-time ordering") + .changelogSafe()); + } + + private static int rowtimeColumn(StreamPhysicalDeduplicate node) { + var fields = node.getInput().getRowType().getFieldList(); + for (int i = 0; i < fields.size(); i++) { + if (FlinkTypeFactory$.MODULE$.isRowtimeIndicatorType(fields.get(i).getType())) return i; + } + return -1; + } + + private static RelNode deduplicate(StreamPhysicalDeduplicate node, PlanContext context) { + int[] keys = node.getUniqueKeys(); + return new StreamPhysicalNativeDeduplicate( + node.getCluster(), + node.getTraitSet(), + context.columnarInput(node.getInput(), keys), + node.getRowType(), + keys, + rowtimeColumn(node), + node.keepLastRow(), + ChangelogPlanUtils.generateUpdateBefore(node), + !node.isRowtime()); + } +} diff --git a/src/main/java-flink1.18/tech/streamfusion/planner/PreparedLegacySink.java b/src/main/java-flink1.18/tech/streamfusion/planner/PreparedLegacySink.java new file mode 100644 index 000000000..7c0e7dc0f --- /dev/null +++ b/src/main/java-flink1.18/tech/streamfusion/planner/PreparedLegacySink.java @@ -0,0 +1,72 @@ +package tech.streamfusion.planner; + +import java.util.List; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.flink.table.planner.plan.nodes.exec.ExecNode; +import org.apache.flink.table.planner.plan.nodes.exec.InputProperty; +import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLegacySink; +import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalLegacySink; +import org.apache.flink.table.planner.plan.utils.ChangelogPlanUtils; +import org.apache.flink.table.planner.plan.utils.UpdatingPlanChecker; +import org.apache.flink.table.planner.utils.ShortcutUtils; +import org.apache.flink.table.runtime.types.LogicalTypeDataTypeConverter; +import org.apache.flink.table.sinks.TableSink; +import org.apache.flink.table.sinks.UpsertStreamTableSink; + +/** Retains host-proven sink keys while the input becomes an Arrow island. */ +final class PreparedLegacySink extends StreamPhysicalLegacySink { + private final String[] upsertKeys; + private final boolean needRetraction; + + static RelNode prepare(StreamPhysicalLegacySink sink) { + if (sink instanceof PreparedLegacySink) return sink; + String[] keys = null; + if (sink.sink() instanceof UpsertStreamTableSink upsert) { + var proven = UpdatingPlanChecker.getUniqueKeyForUpsertSink(sink, upsert); + if (proven.isDefined()) keys = proven.get(); + } + return new PreparedLegacySink( + sink, sink.getTraitSet(), sink.getInput(), keys, !ChangelogPlanUtils.inputInsertOnly(sink)); + } + + @SuppressWarnings("unchecked") + private PreparedLegacySink( + StreamPhysicalLegacySink sink, + RelTraitSet traits, + RelNode input, + String[] upsertKeys, + boolean needRetraction) { + super( + sink.getCluster(), + traits, + input, + sink.hints(), + (TableSink) sink.sink(), + sink.sinkName()); + this.upsertKeys = upsertKeys; + this.needRetraction = needRetraction; + } + + @Override + public String getRelTypeName() { + return "StreamPhysicalLegacySink"; + } + + @Override + public RelNode copy(RelTraitSet traits, List inputs) { + return new PreparedLegacySink(this, traits, inputs.get(0), upsertKeys, needRetraction); + } + + @Override + public ExecNode translateToExecNode() { + return new StreamExecLegacySink<>( + ShortcutUtils.unwrapTableConfig(this), + sink(), + upsertKeys == null ? null : upsertKeys.clone(), + needRetraction, + InputProperty.DEFAULT, + LogicalTypeDataTypeConverter.fromDataTypeToLogicalType(sink().getConsumedDataType()), + getRelDetailedDescription()); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/ColumnarPartitionerCompat.java b/src/main/java-flink2.2/tech/streamfusion/compat/ColumnarPartitionerCompat.java new file mode 100644 index 000000000..5825a7329 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/ColumnarPartitionerCompat.java @@ -0,0 +1,9 @@ +package tech.streamfusion.compat; + +import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner; + +/** + * Flink 1.18 relies on the planner's aligned/recoverable exchange selection instead of an edge + * flag. + */ +public abstract class ColumnarPartitionerCompat extends StreamPartitioner {} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/FileSinkCompat.java b/src/main/java-flink2.2/tech/streamfusion/compat/FileSinkCompat.java new file mode 100644 index 000000000..f238a6599 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/FileSinkCompat.java @@ -0,0 +1,70 @@ +package tech.streamfusion.compat; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.file.table.FileSystemConnectorOptions; +import org.apache.flink.connector.file.table.stream.PartitionCommitInfo; +import org.apache.flink.connector.file.table.stream.StreamingSink; +import org.apache.flink.core.fs.Path; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig; +import org.apache.flink.streaming.api.functions.sink.filesystem.legacy.StreamingFileSink; +import tech.streamfusion.operator.NativeFileBulkWriterFactory; +import tech.streamfusion.operator.NativeFileRollingPolicy; +import tech.streamfusion.operator.PartitionedArrowBatch; +import tech.streamfusion.operator.PartitionedBatchBucketAssigner; + +/** Adapts the host file writer's package move without changing its bucket or commit semantics. */ +public final class FileSinkCompat { + private FileSinkCompat() {} + + public static String parquetSchemaShape() { + return "flink"; + } + + public static org.apache.arrow.vector.types.pojo.Schema encoderSchema( + org.apache.flink.table.types.logical.RowType type) { + var schema = tech.streamfusion.arrow.ArrowConversion.toArrowSchema(type); + return schema; + } + + public static DataStream writer( + Configuration options, + Path location, + NativeFileBulkWriterFactory writerFactory, + DataStream stream, + int parallelism, + List partitionKeys, + boolean parallelismConfigured) { + StreamingFileSink.DefaultBulkFormatBuilder buckets = + StreamingFileSink.forBulkFormat(location, writerFactory) + .withBucketAssigner(new PartitionedBatchBucketAssigner()) + .withRollingPolicy( + new NativeFileRollingPolicy( + options + .get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_FILE_SIZE) + .getBytes(), + options + .get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_ROLLOVER_INTERVAL) + .toMillis(), + options + .get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_INACTIVITY_INTERVAL) + .toMillis())) + // The host's exact naming: a fresh UUID per sink keeps restarted or parallel writers + // from colliding on part-file names within a bucket. + .withOutputFileConfig( + OutputFileConfig.builder().withPartPrefix("part-" + UUID.randomUUID()).build()); + + return StreamingSink.writer( + name -> Optional.empty(), + stream, + options.get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_CHECK_INTERVAL).toMillis(), + buckets, + parallelism, + partitionKeys, + options, + parallelismConfigured); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/FixedSerializerTypeInformation.java b/src/main/java-flink2.2/tech/streamfusion/compat/FixedSerializerTypeInformation.java new file mode 100644 index 000000000..31701dded --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/FixedSerializerTypeInformation.java @@ -0,0 +1,15 @@ +package tech.streamfusion.compat; + +import org.apache.flink.api.common.serialization.SerializerConfig; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.TypeSerializer; + +/** Adapts Flink's serializer factory signature for types with an explicit serializer. */ +public abstract class FixedSerializerTypeInformation extends TypeInformation { + @Override + public final TypeSerializer createSerializer(SerializerConfig config) { + return createSerializer(); + } + + protected abstract TypeSerializer createSerializer(); +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/FlinkCompat.java b/src/main/java-flink2.2/tech/streamfusion/compat/FlinkCompat.java new file mode 100644 index 000000000..4b572889d --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/FlinkCompat.java @@ -0,0 +1,113 @@ +package tech.streamfusion.compat; + +import java.util.List; +import java.util.Map; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.flink.table.planner.plan.logical.WindowSpec; + +/** Planner constructs selected at build time for the target Flink line. */ +public final class FlinkCompat { + private FlinkCompat() {} + + public static final boolean GLOBAL_TTL_EMITS_UNCHANGED = true; + + public static int groupWindowTimeColumn( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGroupWindowAggregate + node) { + return node.window().timeAttribute().getFieldIndex(); + } + + public static final boolean EAGER_ROWTIME_KEEP_FIRST = false; + + public static Long singleStateTtl( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGroupAggregate node) { + return org.apache.flink.table.planner.hint.StateTtlHint.getStateTtlFromHintOnSingleRel( + node.hints()); + } + + public static Long singleStateTtl( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGlobalGroupAggregate + node) { + return org.apache.flink.table.planner.hint.StateTtlHint.getStateTtlFromHintOnSingleRel( + node.hints()); + } + + public static Map joinStateTtl(List hints) { + return org.apache.flink.table.planner.hint.StateTtlHint.getStateTtlFromHintOnBiRel(hints); + } + + public static boolean isSessionWindow(WindowSpec window) { + return window instanceof org.apache.flink.table.planner.plan.logical.SessionWindowSpec; + } + + public static long sessionGapMillis(WindowSpec window) { + return ((org.apache.flink.table.planner.plan.logical.SessionWindowSpec) window) + .getGap() + .toMillis(); + } + + public static boolean isDeltaJoin(RelNode node) { + return node + instanceof + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalDeltaJoin; + } + + public static String unescapeJsonPath(String path) { + return org.apache.flink.shaded.com.jayway.jsonpath.internal.Utils.unescape(path); + } + + public static boolean forceDeltaJoin(RelNode node) { + return org.apache.flink.table.planner.utils.ShortcutUtils.unwrapTableConfig(node) + .get( + org.apache.flink.table.api.config.OptimizerConfigOptions + .TABLE_OPTIMIZER_DELTA_JOIN_STRATEGY) + == org.apache.flink.table.api.config.OptimizerConfigOptions.DeltaJoinStrategy.FORCE; + } + + public static boolean normalizeHasFilter( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize + node) { + return node.filterCondition() != null; + } + + public static boolean normalizeSharesSource( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize + node) { + return node.sourceReused() || node.commonFilter().length > 0; + } + + public static java.util.Optional watermarkRowtime( + org.apache.flink.table.planner.plan.abilities.source.WatermarkPushDownSpec spec) { + return spec.getRowtimeExpr(); + } + + public static org.apache.flink.table.planner.plan.logical.WindowingStrategy windowDedupWindowing( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWindowDeduplicate + node) { + return node.getWindowingStrategy(); + } + + public static java.util.List + sinkAbilities( + org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalSink sink) { + return java.util.Arrays.asList(sink.abilitySpecs()); + } + + public static String expressionClassName( + org.apache.flink.table.planner.codegen.CodeGeneratorContext context) { + return org.apache.flink.table.planner.codegen.CodeGenUtils.newName( + context, "FlinkExpressionEvaluator"); + } + + public static org.apache.flink.table.planner.functions.casting.CastRule.Context castContext( + ClassLoader classLoader) { + return org.apache.flink.table.planner.functions.casting.CastRule.Context.create( + false, + false, + java.time.ZoneId.of("UTC"), + classLoader, + new org.apache.flink.table.planner.codegen.CodeGeneratorContext( + new org.apache.flink.configuration.Configuration(), classLoader)); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/FlinkLookupCompat.java b/src/main/java-flink2.2/tech/streamfusion/compat/FlinkLookupCompat.java new file mode 100644 index 000000000..61b35c276 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/FlinkLookupCompat.java @@ -0,0 +1,224 @@ +package tech.streamfusion.compat; + +import java.util.*; +import java.util.concurrent.CompletableFuture; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; +import org.apache.flink.api.common.functions.*; +import org.apache.flink.api.common.functions.util.FunctionUtils; +import org.apache.flink.api.dag.Transformation; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.streaming.api.functions.async.*; +import org.apache.flink.table.catalog.DataTypeFactory; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.conversion.DataStructureConverter; +import org.apache.flink.table.functions.*; +import org.apache.flink.table.planner.codegen.LookupJoinCodeGenerator; +import org.apache.flink.table.planner.delegation.PlannerBase; +import org.apache.flink.table.planner.plan.nodes.exec.utils.TransformationMetadata; +import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalLookupJoin; +import org.apache.flink.table.planner.plan.utils.*; +import org.apache.flink.table.runtime.collector.*; +import org.apache.flink.table.runtime.generated.*; +import org.apache.flink.table.runtime.operators.join.lookup.*; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.types.logical.RowType; + +/** The released host's lookup code generation and lifecycle APIs, selected at build time. */ +public final class FlinkLookupCompat { + private FlinkLookupCompat() {} + + public static LookupKeys lookupKeys(StreamPhysicalLookupJoin join) { + Map keys = new HashMap<>(); + scala.collection.JavaConverters.mapAsJavaMapConverter(join.allLookupKeys()) + .asJava() + .forEach((index, param) -> keys.put((Integer) index, param)); + return new LookupKeys(keys); + } + + public static String unsupportedKeyShape(LookupKeys keys) { + for (FunctionCallUtil.FunctionParam key : keys.values().values()) { + if (!(key instanceof FunctionCallUtil.FieldRef) + && !(key instanceof FunctionCallUtil.Constant)) { + return "lookup join: unsupported lookup key shape " + key.getClass().getSimpleName(); + } + } + return null; + } + + public static LookupAsyncOptions asyncOptions(StreamPhysicalLookupJoin join) { + if (join.asyncOptions().isEmpty()) return null; + var options = join.asyncOptions().get(); + return new LookupAsyncOptions( + options.asyncBufferCapacity, + options.asyncTimeout, + options.keyOrdered, + options.asyncOutputMode); + } + + public static RexNode preFilter(StreamPhysicalLookupJoin join) { + return join.finalPreFilterCondition().isDefined() ? join.finalPreFilterCondition().get() : null; + } + + public static RexNode remainingCondition(StreamPhysicalLookupJoin join) { + var condition = join.finalRemainingCondition(); + return condition.isDefined() ? condition.get() : null; + } + + public static boolean preferCustomShuffle(StreamPhysicalLookupJoin join) { + return join.preferCustomShuffle(); + } + + public static ExpandedLookupCalc expandCalc(RexProgram calc) { + var expanded = FlinkRexUtil.expandRexProgram(calc); + return new ExpandedLookupCalc( + expanded._1(), expanded._2().isDefined() ? expanded._2().get() : null); + } + + public static UserDefinedFunction lookupFunction( + RelOptTable table, + LookupKeys keys, + ClassLoader classLoader, + boolean async, + ResultRetryStrategy retry, + boolean customShuffle) { + return LookupJoinUtil.getLookupFunction( + table, keys.keySet(), classLoader, async, retry, customShuffle); + } + + public static Transformation customShuffle( + PlannerBase planner, + RelOptTable table, + RowType probeType, + LookupKeys keys, + Transformation rows, + ChangelogMode mode, + TransformationMetadata metadata) { + return LookupJoinUtil.tryApplyCustomShufflePartitioner( + planner, table, probeType, keys.values(), rows, mode, metadata); + } + + private static List orderedKeys(LookupKeys keys) { + List result = new ArrayList<>(); + for (int key : LookupJoinUtil.getOrderedLookupKeys(keys.keySet())) + result.add(keys.values().get(key)); + return result; + } + + public static GeneratedAsyncFetcher asyncFetcher( + ReadableConfig config, + ClassLoader classLoader, + DataTypeFactory types, + RowType probeType, + RowType tableType, + RowType resultType, + LookupKeys keys, + AsyncTableFunction function, + String tableName) { + var generated = + LookupJoinCodeGenerator.generateAsyncLookupFunction( + config, + classLoader, + types, + probeType, + tableType, + resultType, + orderedKeys(keys), + function, + tableName); + return new GeneratedAsyncFetcher(generated.tableFunc(), generated.dataType()); + } + + public static GeneratedFunction> syncFetcher( + ReadableConfig config, + ClassLoader classLoader, + DataTypeFactory types, + RowType probeType, + RowType tableType, + RowType resultType, + LookupKeys keys, + TableFunction function, + String tableName, + boolean objectReuse) { + return LookupJoinCodeGenerator.generateSyncLookupFunction( + config, + classLoader, + types, + probeType, + tableType, + resultType, + orderedKeys(keys), + function, + tableName, + objectReuse); + } + + public static LookupJoinRunner syncRunner( + GeneratedFunction> fetcher, + GeneratedFunction> calc, + GeneratedCollector> collector, + ReadableConfig config, + ClassLoader classLoader, + RexNode preFilter, + RowType probeType, + boolean leftOuter, + int rightFields) { + var filter = + org.apache.flink.table.planner.codegen.FilterCodeGenerator.generateFilterCondition( + config, classLoader, preFilter, probeType); + return calc == null + ? new LookupJoinRunner(fetcher, collector, filter, leftOuter, rightFields) + : new LookupJoinWithCalcRunner(fetcher, calc, collector, filter, leftOuter, rightFields); + } + + public static AsyncLookupJoinRunner asyncRunner( + GeneratedFunction> fetcher, + DataStructureConverter converter, + GeneratedFunction> calc, + GeneratedResultFuture> collector, + ReadableConfig config, + ClassLoader classLoader, + RexNode preFilter, + RowType probeType, + RowDataSerializer rightSerializer, + boolean leftOuter, + int capacity) { + var filter = + org.apache.flink.table.planner.codegen.FilterCodeGenerator.generateFilterCondition( + config, classLoader, preFilter, probeType); + return calc == null + ? new AsyncLookupJoinRunner( + fetcher, converter, collector, filter, rightSerializer, leftOuter, capacity) + : new AsyncLookupJoinWithCalcRunner( + fetcher, converter, calc, collector, filter, rightSerializer, leftOuter, capacity); + } + + public static void open(Function function) throws Exception { + FunctionUtils.openFunction(function, DefaultOpenContext.INSTANCE); + } + + public static boolean preFilter(LookupJoinRunner runner, RowData probe) throws Exception { + return runner.preFilter(FilterCondition.Context.INVALID_CONTEXT, probe); + } + + public static ResultFuture resultFuture(CompletableFuture> future) { + return new ResultFuture<>() { + @Override + public void complete(Collection result) { + future.complete(result); + } + + @Override + public void completeExceptionally(Throwable failure) { + future.completeExceptionally(failure); + } + + @Override + public void complete(CollectionSupplier supplier) { + throw new UnsupportedOperationException(); + } + }; + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/FlinkOutput.java b/src/main/java-flink2.2/tech/streamfusion/compat/FlinkOutput.java new file mode 100644 index 000000000..aeb1c7441 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/FlinkOutput.java @@ -0,0 +1,51 @@ +package tech.streamfusion.compat; + +import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; +import org.apache.flink.util.OutputTag; + +/** Forwards host control events without changing the record representation. */ +public abstract class FlinkOutput implements Output> { + private final Output delegate; + + protected FlinkOutput(Output delegate) { + this.delegate = delegate; + } + + @Override + public void emitWatermark(Watermark watermark) { + delegate.emitWatermark(watermark); + } + + @Override + public void emitWatermarkStatus(WatermarkStatus status) { + delegate.emitWatermarkStatus(status); + } + + @Override + public void collect(OutputTag tag, StreamRecord record) { + delegate.collect(tag, record); + } + + @Override + public void emitLatencyMarker(LatencyMarker marker) { + delegate.emitLatencyMarker(marker); + } + + @Override + public void close() {} + + @Override + public void emitWatermark(org.apache.flink.runtime.event.WatermarkEvent watermark) { + delegate.emitWatermark(watermark); + } + + @Override + public void emitRecordAttributes( + org.apache.flink.streaming.runtime.streamrecord.RecordAttributes attributes) { + delegate.emitRecordAttributes(attributes); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/FlinkStateBackendCompat.java b/src/main/java-flink2.2/tech/streamfusion/compat/FlinkStateBackendCompat.java new file mode 100644 index 000000000..fa3c2a26f --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/FlinkStateBackendCompat.java @@ -0,0 +1,53 @@ +package tech.streamfusion.compat; + +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.runtime.state.*; +import org.apache.flink.state.rocksdb.EmbeddedRocksDBStateBackend; + +/** Adapts the host backend factory signature without owning native state or snapshot bytes. */ +public abstract class FlinkStateBackendCompat implements StateBackend { + protected final StateBackend delegate; + + public static String unsupportedNativeStateReason( + org.apache.flink.streaming.api.environment.StreamExecutionEnvironment environment, + ReadableConfig tableConfig) { + return null; + } + + protected FlinkStateBackendCompat(ReadableConfig config, ClassLoader classLoader) { + delegate = new EmbeddedRocksDBStateBackend().configure(config, classLoader); + } + + protected abstract CheckpointableKeyedStateBackend createNativeKeyedBackend( + KeyedBackendContext context) throws Exception; + + @Override + public boolean useManagedMemory() { + return delegate.useManagedMemory(); + } + + @Override + public final CheckpointableKeyedStateBackend createKeyedStateBackend( + KeyedStateBackendParameters parameters) throws Exception { + return createNativeKeyedBackend( + new KeyedBackendContext<>( + parameters.getEnv(), + parameters.getJobID(), + parameters.getOperatorIdentifier(), + parameters.getKeySerializer(), + parameters.getNumberOfKeyGroups(), + parameters.getKeyGroupRange(), + parameters.getStateHandles(), + parameters.getManagedMemoryFraction(), + handles -> + delegate.createKeyedStateBackend( + new KeyedStateBackendParametersImpl<>(parameters).setStateHandles(handles)), + null)); + } + + @Override + public final OperatorStateBackend createOperatorStateBackend( + OperatorStateBackendParameters parameters) throws Exception { + return delegate.createOperatorStateBackend(parameters); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/FlinkStreamOperator.java b/src/main/java-flink2.2/tech/streamfusion/compat/FlinkStreamOperator.java new file mode 100644 index 000000000..71edf5988 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/FlinkStreamOperator.java @@ -0,0 +1,10 @@ +package tech.streamfusion.compat; + +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; + +/** Keep columnar operators in the upstream task chain when Flink's topology permits it. */ +public abstract class FlinkStreamOperator extends AbstractStreamOperator { + private static final long serialVersionUID = 1L; + + protected FlinkStreamOperator() {} +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/JsonRuntimeCompat.java b/src/main/java-flink2.2/tech/streamfusion/compat/JsonRuntimeCompat.java new file mode 100644 index 000000000..15addd77d --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/JsonRuntimeCompat.java @@ -0,0 +1,28 @@ +package tech.streamfusion.compat; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonFactory; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.util.BufferRecycler; + +/** The buffer-recycler contract verified for the line's released Jackson runtime. */ +public final class JsonRuntimeCompat { + private JsonRuntimeCompat() {} + + public static final boolean ACCEPTS_ARRAY_ROOTS = true; + + public static final boolean PRESERVES_DECIMAL_SCALE = true; + + public static boolean verifiedFactory(JsonFactory factory) { + var version = factory.version(); + return version.getMajorVersion() == 2 + && version.getMinorVersion() == 18 + && version.getPatchLevel() == 2 + && factory._getRecyclerPool() + instanceof + org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.util.JsonRecyclerPools + .ThreadLocalPool; + } + + public static void releaseToPool(BufferRecycler recycler) { + recycler.releaseToPool(); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/LookupKeys.java b/src/main/java-flink2.2/tech/streamfusion/compat/LookupKeys.java new file mode 100644 index 000000000..2943ead24 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/LookupKeys.java @@ -0,0 +1,11 @@ +package tech.streamfusion.compat; + +import java.util.Map; +import java.util.Set; +import org.apache.flink.table.planner.plan.utils.FunctionCallUtil; + +public record LookupKeys(Map values) { + public Set keySet() { + return values.keySet(); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/ProjectedRowDataCompat.java b/src/main/java-flink2.2/tech/streamfusion/compat/ProjectedRowDataCompat.java new file mode 100644 index 000000000..e0bbe42a7 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/ProjectedRowDataCompat.java @@ -0,0 +1,15 @@ +package tech.streamfusion.compat; + +import org.apache.flink.table.data.RowData; + +/** The RowData methods introduced after Flink 1.18, backed by the shared projection. */ +public abstract class ProjectedRowDataCompat implements RowData { + protected abstract RowData sourceRow(); + + protected abstract int sourceIndex(int pos); + + @Override + public org.apache.flink.types.variant.Variant getVariant(int pos) { + return sourceRow().getVariant(sourceIndex(pos)); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/RocksDBOptionsCompat.java b/src/main/java-flink2.2/tech/streamfusion/compat/RocksDBOptionsCompat.java new file mode 100644 index 000000000..aa4e2570a --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/RocksDBOptionsCompat.java @@ -0,0 +1,36 @@ +package tech.streamfusion.compat; + +import java.util.List; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.MemorySize; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.state.rocksdb.RocksDBConfigurableOptions; +import org.apache.flink.state.rocksdb.RocksDBOptions; +import org.rocksdb.CompressionType; + +/** The selected Flink line owns option keys and defaults. */ +public final class RocksDBOptionsCompat extends RocksDBConfigurableOptions { + private RocksDBOptionsCompat() {} + + public static final ConfigOption PREDEFINED_OPTIONS = RocksDBOptions.PREDEFINED_OPTIONS; + public static final ConfigOption OPTIONS_FACTORY = RocksDBOptions.OPTIONS_FACTORY; + public static final ConfigOption LOCAL_DIRECTORIES = RocksDBOptions.LOCAL_DIRECTORIES; + public static final ConfigOption USE_MANAGED_MEMORY = RocksDBOptions.USE_MANAGED_MEMORY; + public static final ConfigOption FIX_PER_SLOT_MEMORY_SIZE = + RocksDBOptions.FIX_PER_SLOT_MEMORY_SIZE; + public static final ConfigOption FIX_PER_TM_MEMORY_SIZE = + RocksDBOptions.FIX_PER_TM_MEMORY_SIZE; + public static final ConfigOption WRITE_BUFFER_RATIO = RocksDBOptions.WRITE_BUFFER_RATIO; + + public static List compressionPerLevel(ReadableConfig config) { + return config.get(COMPRESSION_PER_LEVEL); + } + + public static long compactionQueryEntries(ReadableConfig config) { + return config.get(COMPACT_FILTER_QUERY_TIME_AFTER_NUM_ENTRIES); + } + + public static long periodicCompactionSeconds(ReadableConfig config) { + return config.get(COMPACT_FILTER_PERIODIC_COMPACTION_TIME).getSeconds(); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/RuntimeCompat.java b/src/main/java-flink2.2/tech/streamfusion/compat/RuntimeCompat.java new file mode 100644 index 000000000..2f73a8e15 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/RuntimeCompat.java @@ -0,0 +1,51 @@ +package tech.streamfusion.compat; + +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.runtime.operators.coordination.OperatorEventDispatcher; +import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; + +public final class RuntimeCompat { + private RuntimeCompat() {} + + public static org.apache.flink.table.data.RowData copyRow( + org.apache.flink.table.runtime.typeutils.RowDataSerializer serializer, + org.apache.flink.table.data.RowData row) { + return serializer.copy(row); + } + + public static int attempt(RuntimeContext context) { + return context.getTaskInfo().getAttemptNumber(); + } + + public static String taskName(RuntimeContext context) { + return context.getTaskInfo().getTaskNameWithSubtasks(); + } + + public static int parallelism(RuntimeContext context) { + return context.getTaskInfo().getNumberOfParallelSubtasks(); + } + + public static int subtask(RuntimeContext context) { + return context.getTaskInfo().getIndexOfThisSubtask(); + } + + public static StreamOperatorParameters withOutput( + StreamOperatorParameters parameters, Output> output) { + return withOutput(parameters, output, parameters.getOperatorEventDispatcher()); + } + + public static StreamOperatorParameters withOutput( + StreamOperatorParameters parameters, + Output> output, + OperatorEventDispatcher dispatcher) { + return new StreamOperatorParameters<>( + parameters.getContainingTask(), + parameters.getStreamConfig(), + output, + parameters::getProcessingTimeService, + dispatcher, + parameters.getMailboxExecutor()); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/StateCompat.java b/src/main/java-flink2.2/tech/streamfusion/compat/StateCompat.java new file mode 100644 index 000000000..069180679 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/StateCompat.java @@ -0,0 +1,42 @@ +package tech.streamfusion.compat; + +import java.util.List; +import java.util.stream.Stream; +import org.apache.flink.runtime.state.*; + +/** Preserves the host line's key context and incremental checkpoint ownership APIs. */ +public final class StateCompat { + private StateCompat() {} + + public static void setCurrentKeyAndGroup( + CheckpointableKeyedStateBackend backend, K key, int group) { + backend.setCurrentKeyAndKeyGroup(key, group); + } + + public static Stream keys( + CheckpointableKeyedStateBackend backend, List states, N namespace) { + return backend.getKeys(states, namespace); + } + + public static String backendType(CheckpointableKeyedStateBackend backend) { + return backend.getBackendTypeIdentifier(); + } + + public static boolean couldReuse(CheckpointStreamFactory factory, StreamStateHandle handle) { + return factory.couldReuseStateHandle(handle); + } + + public static StreamStateHandle placeholder(StreamStateHandle handle) { + return new PlaceholderStreamStateHandle( + handle.getStreamStateHandleID(), handle.getStateSize(), false); + } + + public static void reused(CheckpointStreamFactory factory, List handles) + throws java.io.IOException { + factory.reusePreviousStateHandle(handles); + } + + public static StreamStateHandle metaHandle(IncrementalRemoteKeyedStateHandle handle) { + return handle.getMetaDataStateHandle(); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/compat/TableErrors.java b/src/main/java-flink2.2/tech/streamfusion/compat/TableErrors.java new file mode 100644 index 000000000..aecce5b5e --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/compat/TableErrors.java @@ -0,0 +1,14 @@ +package tech.streamfusion.compat; + +/** Preserves the host's generated SINGLE_VALUE failure type across Flink lines. */ +public final class TableErrors { + private TableErrors() {} + + public static Class exceptionType() { + return org.apache.flink.table.api.TableRuntimeException.class; + } + + public static void fail(String message) { + throw new org.apache.flink.table.api.TableRuntimeException(message); + } +} diff --git a/src/main/java/tech/streamfusion/delta/ArrowKernelBatch.java b/src/main/java-flink2.2/tech/streamfusion/delta/ArrowKernelBatch.java similarity index 100% rename from src/main/java/tech/streamfusion/delta/ArrowKernelBatch.java rename to src/main/java-flink2.2/tech/streamfusion/delta/ArrowKernelBatch.java diff --git a/src/main/java/tech/streamfusion/delta/ArrowKernelRows.java b/src/main/java-flink2.2/tech/streamfusion/delta/ArrowKernelRows.java similarity index 100% rename from src/main/java/tech/streamfusion/delta/ArrowKernelRows.java rename to src/main/java-flink2.2/tech/streamfusion/delta/ArrowKernelRows.java diff --git a/src/main/java/tech/streamfusion/delta/ArrowToDeltaRowsOperator.java b/src/main/java-flink2.2/tech/streamfusion/delta/ArrowToDeltaRowsOperator.java similarity index 90% rename from src/main/java/tech/streamfusion/delta/ArrowToDeltaRowsOperator.java rename to src/main/java-flink2.2/tech/streamfusion/delta/ArrowToDeltaRowsOperator.java index f721aba8c..b0323640c 100644 --- a/src/main/java/tech/streamfusion/delta/ArrowToDeltaRowsOperator.java +++ b/src/main/java-flink2.2/tech/streamfusion/delta/ArrowToDeltaRowsOperator.java @@ -3,15 +3,15 @@ import io.delta.flink.sink.Conversions; import io.delta.kernel.types.StructType; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; +import tech.streamfusion.compat.FlinkStreamOperator; import tech.streamfusion.operator.ArrowBatch; /** Emits retained RowData views for Delta bookkeeping without materializing Arrow rows. */ -public final class ArrowToDeltaRowsOperator extends AbstractStreamOperator +public final class ArrowToDeltaRowsOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final RowType rowType; diff --git a/src/main/java/tech/streamfusion/delta/KernelBatchRowDataSerializer.java b/src/main/java-flink2.2/tech/streamfusion/delta/KernelBatchRowDataSerializer.java similarity index 100% rename from src/main/java/tech/streamfusion/delta/KernelBatchRowDataSerializer.java rename to src/main/java-flink2.2/tech/streamfusion/delta/KernelBatchRowDataSerializer.java diff --git a/src/main/java/tech/streamfusion/delta/KernelBatchRowDataTypeInformation.java b/src/main/java-flink2.2/tech/streamfusion/delta/KernelBatchRowDataTypeInformation.java similarity index 79% rename from src/main/java/tech/streamfusion/delta/KernelBatchRowDataTypeInformation.java rename to src/main/java-flink2.2/tech/streamfusion/delta/KernelBatchRowDataTypeInformation.java index 116577598..8c6d2c949 100644 --- a/src/main/java/tech/streamfusion/delta/KernelBatchRowDataTypeInformation.java +++ b/src/main/java-flink2.2/tech/streamfusion/delta/KernelBatchRowDataTypeInformation.java @@ -1,12 +1,12 @@ package tech.streamfusion.delta; -import org.apache.flink.api.common.serialization.SerializerConfig; -import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.table.data.RowData; +import tech.streamfusion.compat.FixedSerializerTypeInformation; /** Ownership-transfer type used only on the chained Arrow-view-to-Delta-writer edge. */ -public final class KernelBatchRowDataTypeInformation extends TypeInformation { +public final class KernelBatchRowDataTypeInformation + extends FixedSerializerTypeInformation { public static final KernelBatchRowDataTypeInformation INSTANCE = new KernelBatchRowDataTypeInformation(); @@ -18,7 +18,9 @@ private KernelBatchRowDataTypeInformation() {} @Override public int getTotalFields() { return 1; } @Override public Class getTypeClass() { return RowData.class; } @Override public boolean isKeyType() { return false; } - @Override public TypeSerializer createSerializer(SerializerConfig config) { + + @Override + public TypeSerializer createSerializer() { return new KernelBatchRowDataSerializer(); } @Override public String toString() { return "KernelBatchRowData"; } diff --git a/src/main/java/tech/streamfusion/delta/NativeDeltaHadoopTable.java b/src/main/java-flink2.2/tech/streamfusion/delta/NativeDeltaHadoopTable.java similarity index 100% rename from src/main/java/tech/streamfusion/delta/NativeDeltaHadoopTable.java rename to src/main/java-flink2.2/tech/streamfusion/delta/NativeDeltaHadoopTable.java diff --git a/src/main/java/tech/streamfusion/delta/NativeDeltaParquetHandler.java b/src/main/java-flink2.2/tech/streamfusion/delta/NativeDeltaParquetHandler.java similarity index 100% rename from src/main/java/tech/streamfusion/delta/NativeDeltaParquetHandler.java rename to src/main/java-flink2.2/tech/streamfusion/delta/NativeDeltaParquetHandler.java diff --git a/src/main/java/tech/streamfusion/delta/NativeDeltaSink.java b/src/main/java-flink2.2/tech/streamfusion/delta/NativeDeltaSink.java similarity index 100% rename from src/main/java/tech/streamfusion/delta/NativeDeltaSink.java rename to src/main/java-flink2.2/tech/streamfusion/delta/NativeDeltaSink.java diff --git a/src/main/java/tech/streamfusion/delta/NativeDeltaSinkWriter.java b/src/main/java-flink2.2/tech/streamfusion/delta/NativeDeltaSinkWriter.java similarity index 100% rename from src/main/java/tech/streamfusion/delta/NativeDeltaSinkWriter.java rename to src/main/java-flink2.2/tech/streamfusion/delta/NativeDeltaSinkWriter.java diff --git a/src/main/java/tech/streamfusion/delta/NativeMoRUpsert.java b/src/main/java-flink2.2/tech/streamfusion/delta/NativeMoRUpsert.java similarity index 100% rename from src/main/java/tech/streamfusion/delta/NativeMoRUpsert.java rename to src/main/java-flink2.2/tech/streamfusion/delta/NativeMoRUpsert.java diff --git a/src/main/java/tech/streamfusion/delta/PartitionedArrowToDeltaRowsOperator.java b/src/main/java-flink2.2/tech/streamfusion/delta/PartitionedArrowToDeltaRowsOperator.java similarity index 89% rename from src/main/java/tech/streamfusion/delta/PartitionedArrowToDeltaRowsOperator.java rename to src/main/java-flink2.2/tech/streamfusion/delta/PartitionedArrowToDeltaRowsOperator.java index c5a2347fe..9c223c79d 100644 --- a/src/main/java/tech/streamfusion/delta/PartitionedArrowToDeltaRowsOperator.java +++ b/src/main/java-flink2.2/tech/streamfusion/delta/PartitionedArrowToDeltaRowsOperator.java @@ -3,16 +3,15 @@ import io.delta.flink.sink.Conversions; import io.delta.kernel.types.StructType; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; +import tech.streamfusion.compat.FlinkStreamOperator; import tech.streamfusion.operator.PartitionedArrowBatch; /** Converts partition-routed Arrow batches into retained Delta row views. */ -public final class PartitionedArrowToDeltaRowsOperator - extends AbstractStreamOperator +public final class PartitionedArrowToDeltaRowsOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final RowType rowType; private transient StructType deltaSchema; diff --git a/src/main/java-flink2.2/tech/streamfusion/format/avro/compat/AvroCompat.java b/src/main/java-flink2.2/tech/streamfusion/format/avro/compat/AvroCompat.java new file mode 100644 index 000000000..b8b468026 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/format/avro/compat/AvroCompat.java @@ -0,0 +1,26 @@ +package tech.streamfusion.format.avro.compat; + +import org.apache.avro.Schema; +import org.apache.flink.formats.avro.AvroToRowDataConverters; +import org.apache.flink.formats.avro.RowDataToAvroConverters; +import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; + +/** Resolves schemas and converters using the timestamp mappings the released format exposes. */ +public final class AvroCompat { + private AvroCompat() {} + + public static Schema schema(LogicalType type, boolean legacy) { + + return AvroSchemaConverter.convertToSchema(type, legacy); + } + + public static void validateDecoder(RowType type, boolean legacy) { + AvroToRowDataConverters.createRowConverter(type, legacy); + } + + public static void validateEncoder(RowType type, boolean legacy) { + RowDataToAvroConverters.createConverter(type, legacy); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaFetcherManagerCompat.java b/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaFetcherManagerCompat.java new file mode 100644 index 000000000..1ce6f3545 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaFetcherManagerCompat.java @@ -0,0 +1,18 @@ +package tech.streamfusion.kafka.compat; + +import java.util.function.Supplier; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.base.source.reader.fetcher.SingleThreadFetcherManager; +import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; +import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit; +import tech.streamfusion.operator.NativeSourceRecord; + +/** Preserves the released source reader's queue ownership between its fetcher and consumer. */ +public abstract class KafkaFetcherManagerCompat + extends SingleThreadFetcherManager { + protected KafkaFetcherManagerCompat( + Supplier> supplier, + Configuration config) { + super(supplier, config); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaRecordSchemaCompat.java b/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaRecordSchemaCompat.java new file mode 100644 index 000000000..d213c1a60 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaRecordSchemaCompat.java @@ -0,0 +1,19 @@ +package tech.streamfusion.kafka.compat; + +import java.util.List; +import java.util.Optional; +import org.apache.flink.connector.kafka.lineage.*; +import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema; +import tech.streamfusion.kafka.PreSerializedKafkaRecord; + +/** Retains connector lineage on releases that provide that SPI. */ +public abstract class KafkaRecordSchemaCompat + implements KafkaRecordSerializationSchema, KafkaDatasetFacetProvider { + protected abstract String topic(); + + @Override + public Optional getKafkaDatasetFacet() { + return Optional.of( + new DefaultKafkaDatasetFacet(DefaultKafkaDatasetIdentifier.ofTopics(List.of(topic())))); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaSinkCompat.java b/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaSinkCompat.java new file mode 100644 index 000000000..c8b422d65 --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaSinkCompat.java @@ -0,0 +1,21 @@ +package tech.streamfusion.kafka.compat; + +import java.util.Locale; +import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder; +import org.apache.flink.connector.kafka.sink.TransactionNamingStrategy; + +/** Transaction naming is a connector option introduced after the 1.18 connector release. */ +public final class KafkaSinkCompat { + private KafkaSinkCompat() {} + + public static String transactionNaming(String option) { + return option == null || "default".equalsIgnoreCase(option) + ? TransactionNamingStrategy.DEFAULT.name() + : TransactionNamingStrategy.valueOf(option.replace('-', '_').toUpperCase(Locale.ROOT)) + .name(); + } + + public static void setTransactionNaming(KafkaSinkBuilder builder, String strategy) { + builder.setTransactionNamingStrategy(TransactionNamingStrategy.valueOf(strategy)); + } +} diff --git a/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaSourceReaderCompat.java b/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaSourceReaderCompat.java new file mode 100644 index 000000000..bcf94a7de --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/kafka/compat/KafkaSourceReaderCompat.java @@ -0,0 +1,23 @@ +package tech.streamfusion.kafka.compat; + +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.base.source.reader.RecordEmitter; +import org.apache.flink.connector.base.source.reader.SingleThreadMultiplexSourceReaderBase; +import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit; +import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplitState; +import tech.streamfusion.operator.ArrowBatch; +import tech.streamfusion.operator.NativeSourceRecord; + +/** Supplies the same fetcher-owned queue to the older reader constructor. */ +public abstract class KafkaSourceReaderCompat + extends SingleThreadMultiplexSourceReaderBase< + NativeSourceRecord, ArrowBatch, KafkaPartitionSplit, KafkaPartitionSplitState> { + protected KafkaSourceReaderCompat( + KafkaFetcherManagerCompat fetcher, + RecordEmitter emitter, + Configuration config, + SourceReaderContext context) { + super(fetcher, emitter, config, context); + } +} diff --git a/src/main/java/tech/streamfusion/planner/DeltaPlannerExtension.java b/src/main/java-flink2.2/tech/streamfusion/planner/DeltaPlannerExtension.java similarity index 100% rename from src/main/java/tech/streamfusion/planner/DeltaPlannerExtension.java rename to src/main/java-flink2.2/tech/streamfusion/planner/DeltaPlannerExtension.java diff --git a/src/main/java/tech/streamfusion/planner/DeltaSinkMatcher.java b/src/main/java-flink2.2/tech/streamfusion/planner/DeltaSinkMatcher.java similarity index 100% rename from src/main/java/tech/streamfusion/planner/DeltaSinkMatcher.java rename to src/main/java-flink2.2/tech/streamfusion/planner/DeltaSinkMatcher.java diff --git a/src/main/java-flink2.2/tech/streamfusion/planner/FlinkPlannerCompat.java b/src/main/java-flink2.2/tech/streamfusion/planner/FlinkPlannerCompat.java new file mode 100644 index 000000000..1986f694d --- /dev/null +++ b/src/main/java-flink2.2/tech/streamfusion/planner/FlinkPlannerCompat.java @@ -0,0 +1,13 @@ +package tech.streamfusion.planner; + +import java.util.List; + +final class FlinkPlannerCompat { + private FlinkPlannerCompat() {} + + static org.apache.calcite.rel.RelNode prepareForRewrite(org.apache.calcite.rel.RelNode node) { + return node; + } + + static void addSubstitutions(List> entries) {} +} diff --git a/src/main/java/tech/streamfusion/planner/NativeDeltaSinkExecNode.java b/src/main/java-flink2.2/tech/streamfusion/planner/NativeDeltaSinkExecNode.java similarity index 100% rename from src/main/java/tech/streamfusion/planner/NativeDeltaSinkExecNode.java rename to src/main/java-flink2.2/tech/streamfusion/planner/NativeDeltaSinkExecNode.java diff --git a/src/main/java/tech/streamfusion/planner/StreamPhysicalNativeDeltaSink.java b/src/main/java-flink2.2/tech/streamfusion/planner/StreamPhysicalNativeDeltaSink.java similarity index 100% rename from src/main/java/tech/streamfusion/planner/StreamPhysicalNativeDeltaSink.java rename to src/main/java-flink2.2/tech/streamfusion/planner/StreamPhysicalNativeDeltaSink.java diff --git a/src/main/java/tech/streamfusion/Native.java b/src/main/java/tech/streamfusion/Native.java index cac74886f..3597bf829 100644 --- a/src/main/java/tech/streamfusion/Native.java +++ b/src/main/java/tech/streamfusion/Native.java @@ -1176,19 +1176,21 @@ public static native long restoreWindowRankerPartitions( long memoryBudgetBytes); /** - * Creates a non-windowed {@code GROUP BY} aggregator and returns an opaque handle. Each input batch - * folds into per-key state and the aggregator exports the changelog rows it produces, with the row - * kinds carried on the {@code $row_kind$} column. Released with {@link #closeGroupAggregator}. + * Creates a non-windowed {@code GROUP BY} aggregator and returns an opaque handle. Each input + * batch folds into per-key state and the aggregator exports the changelog rows it produces, with + * the row kinds carried on the {@code $row_kind$} column. Released with {@link + * #closeGroupAggregator}. * * @param aggregateKinds aggregate codes (see {@link #createTumblingAggregator}) * @param valueTypes per-aggregate value-column types (see {@link #createTumblingAggregator}) - * @param valueColumns per-aggregate value-column index in the input batch ({@code -1} for COUNT(*)) + * @param valueColumns per-aggregate value-column index in the input batch ({@code -1} for + * COUNT(*)) * @param keyColumns grouping-key column indices in the input batch (empty for global aggregation) * @param keyTimestampPrecisions pre-order logical key type descriptors (timestamp precision or * {@code -1}); this lets the native BinaryRow codec preserve nested timestamp layout - * @param countColumns per-aggregate two-phase AVG count-partial column ({@code -1} otherwise): the - * value column is then the local's pre-summed sum partial, and each row bumps the count by this - * column instead of by one + * @param countColumns per-aggregate two-phase AVG count-partial column ({@code -1} otherwise): + * the value column is then the local's pre-summed sum partial, and each row bumps the count + * by this column instead of by one * @param distinctViewColumns per-aggregate two-phase distinct-view column ({@code -1} otherwise): * the column carries a local bundle's distinct (value, count) entries as a list of structs, * folded into the per-key distinct set with multiplicities instead of one value per row @@ -1201,7 +1203,7 @@ public static native long restoreWindowRankerPartitions( * absent, and the unchanged-result suppression is disabled — Flink's TTL'd emission * @param memoryBudgetBytes task off-heap budget (see {@link #createTumblingAggregator}) */ - public static native long createGroupAggregator( + public static long createGroupAggregator( int[] aggregateKinds, int[] valueTypes, int[] valueColumns, @@ -1214,7 +1216,41 @@ public static native long createGroupAggregator( boolean generateUpdateBefore, boolean miniBatch, long stateTtlMillis, - long memoryBudgetBytes); + long memoryBudgetBytes) { + return createGroupAggregatorWithTtlEmission( + aggregateKinds, + valueTypes, + valueColumns, + keyColumns, + keyTimestampPrecisions, + filterColumns, + countColumns, + distinctViewColumns, + recordCountColumn, + generateUpdateBefore, + miniBatch, + stateTtlMillis, + memoryBudgetBytes, + !miniBatch + || countColumns.length == 0 + || tech.streamfusion.compat.FlinkCompat.GLOBAL_TTL_EMITS_UNCHANGED); + } + + private static native long createGroupAggregatorWithTtlEmission( + int[] aggregateKinds, + int[] valueTypes, + int[] valueColumns, + int[] keyColumns, + int[] keyTimestampPrecisions, + int[] filterColumns, + int[] countColumns, + int[] distinctViewColumns, + int recordCountColumn, + boolean generateUpdateBefore, + boolean miniBatch, + long stateTtlMillis, + long memoryBudgetBytes, + boolean emitUnchangedWithTtl); /** * Folds an input batch into per-key state, exporting the changelog rows it produces (grouping keys, @@ -1246,7 +1282,7 @@ public static native byte[][] snapshotGroupAggregatorPartitions( * pre-TTL writer), granting them a full retention from the restore — Flink's enable-TTL * migration. */ - public static native long restoreGroupAggregatorPartitions( + public static long restoreGroupAggregatorPartitions( int[] aggregateKinds, int[] valueTypes, int[] valueColumns, @@ -1261,7 +1297,45 @@ public static native long restoreGroupAggregatorPartitions( long stateTtlMillis, long nowMillis, byte[][] snapshots, - long memoryBudgetBytes); + long memoryBudgetBytes) { + return restoreGroupAggregatorPartitionsWithTtlEmission( + aggregateKinds, + valueTypes, + valueColumns, + keyColumns, + keyTimestampPrecisions, + filterColumns, + countColumns, + distinctViewColumns, + recordCountColumn, + generateUpdateBefore, + miniBatch, + stateTtlMillis, + nowMillis, + snapshots, + memoryBudgetBytes, + !miniBatch + || countColumns.length == 0 + || tech.streamfusion.compat.FlinkCompat.GLOBAL_TTL_EMITS_UNCHANGED); + } + + private static native long restoreGroupAggregatorPartitionsWithTtlEmission( + int[] aggregateKinds, + int[] valueTypes, + int[] valueColumns, + int[] keyColumns, + int[] keyTimestampPrecisions, + int[] filterColumns, + int[] countColumns, + int[] distinctViewColumns, + int recordCountColumn, + boolean generateUpdateBefore, + boolean miniBatch, + long stateTtlMillis, + long nowMillis, + byte[][] snapshots, + long memoryBudgetBytes, + boolean emitUnchangedWithTtl); /** Whether this native build carries direct Rust RocksDB state. */ public static native boolean rocksdbStateAvailable(); @@ -1280,7 +1354,7 @@ public static native boolean rocksdbGroupAggregatorSupported( int[] aggregateKinds, int[] valueTypes); /** Creates a group aggregator backed directly by a Rust-owned RocksDB instance. */ - public static native long createRocksDBGroupAggregator( + public static long createRocksDBGroupAggregator( int[] aggregateKinds, int[] valueTypes, int[] valueColumns, @@ -1304,7 +1378,63 @@ public static native long createRocksDBGroupAggregator( int keyGroupStart, int keyGroupEnd, boolean aligned, - byte[][] restoredPartitions); + byte[][] restoredPartitions) { + return createRocksDBGroupAggregatorWithTtlEmission( + aggregateKinds, + valueTypes, + valueColumns, + keyColumns, + keyTimestampPrecisions, + filterColumns, + countColumns, + distinctViewColumns, + recordCountColumn, + generateUpdateBefore, + miniBatch, + stateTtlMillis, + nowMillis, + memoryBudgetBytes, + databaseDirectory, + maxParallelism, + optionsJson, + sharedResources, + sourceDirectories, + sourceSnapshotTokens, + keyGroupStart, + keyGroupEnd, + aligned, + restoredPartitions, + !miniBatch + || countColumns.length == 0 + || tech.streamfusion.compat.FlinkCompat.GLOBAL_TTL_EMITS_UNCHANGED); + } + + private static native long createRocksDBGroupAggregatorWithTtlEmission( + int[] aggregateKinds, + int[] valueTypes, + int[] valueColumns, + int[] keyColumns, + int[] keyTimestampPrecisions, + int[] filterColumns, + int[] countColumns, + int[] distinctViewColumns, + int recordCountColumn, + boolean generateUpdateBefore, + boolean miniBatch, + long stateTtlMillis, + long nowMillis, + long memoryBudgetBytes, + String databaseDirectory, + int maxParallelism, + String optionsJson, + long sharedResources, + String[] sourceDirectories, + String[] sourceSnapshotTokens, + int keyGroupStart, + int keyGroupEnd, + boolean aligned, + byte[][] restoredPartitions, + boolean emitUnchangedWithTtl); /** Materializes direct RocksDB aggregate state as backend-independent key-group partitions. */ public static native byte[][] snapshotRocksDBGroupAggregatorPartitions(long handle); diff --git a/src/main/java/tech/streamfusion/compat/ExpandedLookupCalc.java b/src/main/java/tech/streamfusion/compat/ExpandedLookupCalc.java new file mode 100644 index 000000000..66df5300d --- /dev/null +++ b/src/main/java/tech/streamfusion/compat/ExpandedLookupCalc.java @@ -0,0 +1,6 @@ +package tech.streamfusion.compat; + +import java.util.List; +import org.apache.calcite.rex.RexNode; + +public record ExpandedLookupCalc(List projection, RexNode filter) {} diff --git a/src/main/java/tech/streamfusion/compat/GeneratedAsyncFetcher.java b/src/main/java/tech/streamfusion/compat/GeneratedAsyncFetcher.java new file mode 100644 index 000000000..3123821d9 --- /dev/null +++ b/src/main/java/tech/streamfusion/compat/GeneratedAsyncFetcher.java @@ -0,0 +1,9 @@ +package tech.streamfusion.compat; + +import org.apache.flink.streaming.api.functions.async.AsyncFunction; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.runtime.generated.GeneratedFunction; +import org.apache.flink.table.types.DataType; + +public record GeneratedAsyncFetcher( + GeneratedFunction> tableFunc, DataType dataType) {} diff --git a/src/main/java/tech/streamfusion/compat/KeyedBackendContext.java b/src/main/java/tech/streamfusion/compat/KeyedBackendContext.java new file mode 100644 index 000000000..5289ea19f --- /dev/null +++ b/src/main/java/tech/streamfusion/compat/KeyedBackendContext.java @@ -0,0 +1,27 @@ +package tech.streamfusion.compat; + +import java.util.Collection; +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.state.CheckpointableKeyedStateBackend; +import org.apache.flink.runtime.state.KeyGroupRange; +import org.apache.flink.runtime.state.KeyedStateHandle; +import org.apache.flink.util.function.FunctionWithException; + +/** Host construction parameters plus a factory that preserves all remaining delegate arguments. */ +public record KeyedBackendContext( + Environment environment, + JobID jobId, + String operatorIdentifier, + TypeSerializer keySerializer, + int numberOfKeyGroups, + KeyGroupRange keyGroupRange, + Collection stateHandles, + double managedMemoryFraction, + FunctionWithException< + Collection, CheckpointableKeyedStateBackend, Exception> + delegateFactory, + FunctionWithException< + Collection, CheckpointableKeyedStateBackend, Exception> + canonicalProjectionFactory) {} diff --git a/src/main/java/tech/streamfusion/compat/LookupAsyncOptions.java b/src/main/java/tech/streamfusion/compat/LookupAsyncOptions.java new file mode 100644 index 000000000..aaa310481 --- /dev/null +++ b/src/main/java/tech/streamfusion/compat/LookupAsyncOptions.java @@ -0,0 +1,9 @@ +package tech.streamfusion.compat; + +import org.apache.flink.streaming.api.datastream.AsyncDataStream; + +public record LookupAsyncOptions( + int asyncBufferCapacity, + long asyncTimeout, + boolean keyOrdered, + AsyncDataStream.OutputMode asyncOutputMode) {} diff --git a/src/main/java/tech/streamfusion/format/EncodeFormat.java b/src/main/java/tech/streamfusion/format/EncodeFormat.java index af7699a32..e3e74dfd5 100644 --- a/src/main/java/tech/streamfusion/format/EncodeFormat.java +++ b/src/main/java/tech/streamfusion/format/EncodeFormat.java @@ -83,6 +83,9 @@ public String openOptions() throws IOException { /** JSON encode options resolved with Flink's json format factory defaults. */ public static EncodeFormat json(Map options) { StringBuilder encoded = new StringBuilder(); + if (!tech.streamfusion.compat.JsonRuntimeCompat.PRESERVES_DECIMAL_SCALE) { + encoded.append("legacy-decimal-nodes=true\n"); + } String timestampFormat = options.getOrDefault("timestamp-format.standard", "SQL"); if ("ISO-8601".equals(timestampFormat)) { encoded.append("timestamp-format=ISO-8601\n"); @@ -136,6 +139,9 @@ public static EncodeFormat csv(Map options) { return null; } StringBuilder encoded = new StringBuilder(); + if (!tech.streamfusion.compat.JsonRuntimeCompat.PRESERVES_DECIMAL_SCALE) { + encoded.append("legacy-decimal-nodes=true\n"); + } String delimiter = options.get("field-delimiter"); if (delimiter != null) { Character unescaped = NativeFormatOptions.unescapedDelimiter(delimiter); diff --git a/src/main/java/tech/streamfusion/format/NativeFormatOptions.java b/src/main/java/tech/streamfusion/format/NativeFormatOptions.java index 6afcdb9b3..1b353bc00 100644 --- a/src/main/java/tech/streamfusion/format/NativeFormatOptions.java +++ b/src/main/java/tech/streamfusion/format/NativeFormatOptions.java @@ -65,6 +65,9 @@ private static String encodeFormat(Map options) { } StringBuilder encoded = new StringBuilder(); if (FormatCodes.isJsonFamily(format)) { + if (!tech.streamfusion.compat.JsonRuntimeCompat.ACCEPTS_ARRAY_ROOTS) { + encoded.append("json.reject-array-roots=true\n"); + } // A missing field is null natively (Flink's default); the fail mode isn't modeled. if ("true".equalsIgnoreCase(option(options, "fail-on-missing-field"))) { return null; @@ -79,7 +82,9 @@ private static String encodeFormat(Map options) { return encoded.toString(); } // The factory validates the value, so returning null for anything else is defensive. - return "ISO-8601".equals(timestampFormat) ? "timestamp-format=ISO-8601\n" : null; + if (!"ISO-8601".equals(timestampFormat)) return null; + encoded.append("timestamp-format=ISO-8601\n"); + return encoded.toString(); } if (!"csv".equals(format)) { return encoded.toString(); diff --git a/src/main/java/tech/streamfusion/format/avro/AvroDecodeGate.java b/src/main/java/tech/streamfusion/format/avro/AvroDecodeGate.java index 1de0a5ef8..59859d99d 100644 --- a/src/main/java/tech/streamfusion/format/avro/AvroDecodeGate.java +++ b/src/main/java/tech/streamfusion/format/avro/AvroDecodeGate.java @@ -1,7 +1,5 @@ package tech.streamfusion.format.avro; -import org.apache.flink.formats.avro.AvroToRowDataConverters; -import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.TimeType; @@ -30,8 +28,10 @@ public static boolean supports(RowType rowType, boolean legacyTimestampMapping) return true; } try { - AvroSchemaConverter.convertToSchema(rowType.copy(false), legacyTimestampMapping); - AvroToRowDataConverters.createRowConverter(rowType, legacyTimestampMapping); + tech.streamfusion.format.avro.compat.AvroCompat.schema( + rowType.copy(false), legacyTimestampMapping); + tech.streamfusion.format.avro.compat.AvroCompat.validateDecoder( + rowType, legacyTimestampMapping); } catch (RuntimeException e) { return false; } diff --git a/src/main/java/tech/streamfusion/format/avro/AvroEncodeGate.java b/src/main/java/tech/streamfusion/format/avro/AvroEncodeGate.java index 12aeded87..05878924a 100644 --- a/src/main/java/tech/streamfusion/format/avro/AvroEncodeGate.java +++ b/src/main/java/tech/streamfusion/format/avro/AvroEncodeGate.java @@ -1,7 +1,5 @@ package tech.streamfusion.format.avro; -import org.apache.flink.formats.avro.RowDataToAvroConverters; -import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.TimeType; @@ -25,8 +23,9 @@ private AvroEncodeGate() {} */ public static boolean supports(RowType rowType, boolean legacyTimestampMapping) { try { - AvroSchemaConverter.convertToSchema(rowType, legacyTimestampMapping); - RowDataToAvroConverters.createConverter(rowType, legacyTimestampMapping); + tech.streamfusion.format.avro.compat.AvroCompat.schema(rowType, legacyTimestampMapping); + tech.streamfusion.format.avro.compat.AvroCompat.validateEncoder( + rowType, legacyTimestampMapping); } catch (RuntimeException e) { return false; } @@ -42,7 +41,8 @@ public static boolean supports(RowType rowType, boolean legacyTimestampMapping) * a planner JVM that has Flink. */ public static String derivedSchema(RowType rowType, boolean legacyTimestampMapping) { - return AvroSchemaConverter.convertToSchema(rowType, legacyTimestampMapping).toString(); + return tech.streamfusion.format.avro.compat.AvroCompat.schema(rowType, legacyTimestampMapping) + .toString(); } private static boolean encodableColumn(LogicalType type) { diff --git a/src/main/java/tech/streamfusion/format/avro/AvroFormatProvider.java b/src/main/java/tech/streamfusion/format/avro/AvroFormatProvider.java index 378ef1b06..3da8f2ffc 100644 --- a/src/main/java/tech/streamfusion/format/avro/AvroFormatProvider.java +++ b/src/main/java/tech/streamfusion/format/avro/AvroFormatProvider.java @@ -1,5 +1,7 @@ package tech.streamfusion.format.avro; +import java.util.Map; +import org.apache.flink.table.types.logical.RowType; import tech.streamfusion.format.EncodeFormat; import tech.streamfusion.format.FormatCodes; import tech.streamfusion.format.NativeFormatContext; @@ -7,9 +9,6 @@ import tech.streamfusion.format.NativeFormatProvider; import tech.streamfusion.format.NativeMessageDecoderFactory; import tech.streamfusion.format.NativeSchemaMessageDecoder; -import java.util.Map; -import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; -import org.apache.flink.table.types.logical.RowType; /** Native provider for Flink's schema-embedded {@code avro} format. */ public final class AvroFormatProvider implements NativeFormatProvider { @@ -44,7 +43,8 @@ public boolean supports(NativeFormatContext context) { /** Flink's {@code avro.timestamp_mapping.legacy}, default true. */ private static boolean legacyTimestampMapping(Map options) { - return !"false".equalsIgnoreCase(NativeFormatOptions.option(options, "timestamp_mapping.legacy")); + return !"false" + .equalsIgnoreCase(NativeFormatOptions.option(options, "timestamp_mapping.legacy")); } @Override @@ -70,11 +70,15 @@ public EncodeFormat encodeFormat(NativeFormatContext context) { public NativeMessageDecoderFactory createDecoder(NativeFormatContext context) { boolean legacy = legacyTimestampMapping(context.options()); String writerSchema = - AvroSchemaConverter.convertToSchema(context.writerType().copy(false), legacy).toString(); + tech.streamfusion.format.avro.compat.AvroCompat.schema( + context.writerType().copy(false), legacy) + .toString(); String readerSchema = context.writerType().equals(context.outputType()) ? "" - : AvroSchemaConverter.convertToSchema(context.outputType().copy(false), legacy).toString(); + : tech.streamfusion.format.avro.compat.AvroCompat.schema( + context.outputType().copy(false), legacy) + .toString(); return () -> new Decoder(writerSchema, readerSchema); } diff --git a/src/main/java/tech/streamfusion/kafka/NativeKafkaSerializationOperator.java b/src/main/java/tech/streamfusion/kafka/NativeKafkaSerializationOperator.java index 4188c0216..48067eeb6 100644 --- a/src/main/java/tech/streamfusion/kafka/NativeKafkaSerializationOperator.java +++ b/src/main/java/tech/streamfusion/kafka/NativeKafkaSerializationOperator.java @@ -1,21 +1,21 @@ package tech.streamfusion.kafka; -import tech.streamfusion.format.EncodeFormat; -import tech.streamfusion.operator.ArrowBatch; -import tech.streamfusion.operator.NativeAllocator; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.c.Data; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.flink.metrics.Counter; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.compat.FlinkStreamOperator; +import tech.streamfusion.format.EncodeFormat; +import tech.streamfusion.operator.ArrowBatch; +import tech.streamfusion.operator.NativeAllocator; /** Encodes each input Arrow batch once and emits the final Kafka value bytes for its rows. */ public final class NativeKafkaSerializationOperator - extends AbstractStreamOperator + extends FlinkStreamOperator implements OneInputStreamOperator { private final EncodeFormat valueFormat; diff --git a/src/main/java/tech/streamfusion/kafka/NativeKafkaSourceFetcherManager.java b/src/main/java/tech/streamfusion/kafka/NativeKafkaSourceFetcherManager.java index ccc0eebb2..468139cc8 100644 --- a/src/main/java/tech/streamfusion/kafka/NativeKafkaSourceFetcherManager.java +++ b/src/main/java/tech/streamfusion/kafka/NativeKafkaSourceFetcherManager.java @@ -1,11 +1,9 @@ package tech.streamfusion.kafka; -import tech.streamfusion.operator.NativeSourceRecord; import java.io.IOException; import java.util.Map; import java.util.function.Supplier; import org.apache.flink.configuration.Configuration; -import org.apache.flink.connector.base.source.reader.fetcher.SingleThreadFetcherManager; import org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher; import org.apache.flink.connector.base.source.reader.fetcher.SplitFetcherTask; import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; @@ -13,10 +11,11 @@ import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.TopicPartition; +import tech.streamfusion.operator.NativeSourceRecord; /** Single-thread native Kafka fetcher manager with Flink's checkpoint-offset commit hook. */ final class NativeKafkaSourceFetcherManager - extends SingleThreadFetcherManager { + extends tech.streamfusion.kafka.compat.KafkaFetcherManagerCompat { NativeKafkaSourceFetcherManager( Supplier> splitReaderSupplier, diff --git a/src/main/java/tech/streamfusion/kafka/NativeKafkaSourceReader.java b/src/main/java/tech/streamfusion/kafka/NativeKafkaSourceReader.java index cf0ea2f79..dbf0541b9 100644 --- a/src/main/java/tech/streamfusion/kafka/NativeKafkaSourceReader.java +++ b/src/main/java/tech/streamfusion/kafka/NativeKafkaSourceReader.java @@ -1,7 +1,5 @@ package tech.streamfusion.kafka; -import tech.streamfusion.operator.ArrowBatch; -import tech.streamfusion.operator.NativeSourceRecord; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -14,7 +12,6 @@ import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.base.source.reader.RecordEmitter; -import org.apache.flink.connector.base.source.reader.SingleThreadMultiplexSourceReaderBase; import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; import org.apache.flink.connector.kafka.source.KafkaSourceOptions; import org.apache.flink.connector.kafka.source.metrics.KafkaSourceReaderMetrics; @@ -24,11 +21,11 @@ import org.apache.kafka.common.TopicPartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import tech.streamfusion.operator.ArrowBatch; +import tech.streamfusion.operator.NativeSourceRecord; /** Kafka source reader retaining Flink's split state and checkpoint offset-commit contract. */ -final class NativeKafkaSourceReader - extends SingleThreadMultiplexSourceReaderBase< - NativeSourceRecord, ArrowBatch, KafkaPartitionSplit, KafkaPartitionSplitState> { +final class NativeKafkaSourceReader extends tech.streamfusion.kafka.compat.KafkaSourceReaderCompat { private static final Logger LOG = LoggerFactory.getLogger(NativeKafkaSourceReader.class); diff --git a/src/main/java/tech/streamfusion/kafka/PreSerializedKafkaRecordSchema.java b/src/main/java/tech/streamfusion/kafka/PreSerializedKafkaRecordSchema.java index 290329537..47c88ec7a 100644 --- a/src/main/java/tech/streamfusion/kafka/PreSerializedKafkaRecordSchema.java +++ b/src/main/java/tech/streamfusion/kafka/PreSerializedKafkaRecordSchema.java @@ -1,17 +1,10 @@ package tech.streamfusion.kafka; -import java.util.List; -import java.util.Optional; -import org.apache.flink.connector.kafka.lineage.DefaultKafkaDatasetFacet; -import org.apache.flink.connector.kafka.lineage.DefaultKafkaDatasetIdentifier; -import org.apache.flink.connector.kafka.lineage.KafkaDatasetFacet; -import org.apache.flink.connector.kafka.lineage.KafkaDatasetFacetProvider; -import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema; import org.apache.kafka.clients.producer.ProducerRecord; /** Hands native-serialized values to Flink's Kafka writer without another serialization pass. */ public final class PreSerializedKafkaRecordSchema - implements KafkaRecordSerializationSchema, KafkaDatasetFacetProvider { + extends tech.streamfusion.kafka.compat.KafkaRecordSchemaCompat { private final String topic; @@ -26,8 +19,7 @@ public ProducerRecord serialize( } @Override - public Optional getKafkaDatasetFacet() { - return Optional.of( - new DefaultKafkaDatasetFacet(DefaultKafkaDatasetIdentifier.ofTopics(List.of(topic)))); + protected String topic() { + return topic; } } diff --git a/src/main/java/tech/streamfusion/operator/AbstractNativeStatefulOperator.java b/src/main/java/tech/streamfusion/operator/AbstractNativeStatefulOperator.java index 3c824debb..f831c6ac8 100644 --- a/src/main/java/tech/streamfusion/operator/AbstractNativeStatefulOperator.java +++ b/src/main/java/tech/streamfusion/operator/AbstractNativeStatefulOperator.java @@ -1,12 +1,5 @@ package tech.streamfusion.operator; -import tech.streamfusion.Native; -import tech.streamfusion.arrow.ArrowConversion; -import tech.streamfusion.planner.NativeConfig; -import tech.streamfusion.state.CanonicalNativeState; -import tech.streamfusion.state.RocksDBNativeKeyedStateBackend; -import tech.streamfusion.state.RocksDBNativeState; -import tech.streamfusion.state.RocksDBNativeStateSupport; import java.util.List; import java.util.function.BooleanSupplier; import java.util.function.LongBinaryOperator; @@ -18,12 +11,18 @@ import org.apache.flink.runtime.state.CheckpointableKeyedStateBackend; import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.table.types.logical.RowType; +import tech.streamfusion.Native; +import tech.streamfusion.arrow.ArrowConversion; +import tech.streamfusion.compat.FlinkStreamOperator; +import tech.streamfusion.state.CanonicalNativeState; +import tech.streamfusion.state.RocksDBNativeKeyedStateBackend; +import tech.streamfusion.state.RocksDBNativeState; +import tech.streamfusion.state.RocksDBNativeStateSupport; /** - * Lifecycle shared by every native operator whose hot state lives in Rust: the handle, its - * task off-heap reservation, and the two ways that state can be checkpointed. + * Lifecycle shared by every native operator whose hot state lives in Rust: the handle, its task + * off-heap reservation, and the two ways that state can be checkpointed. * *

State travels one of two routes, decided once at {@link #initializeState}. On the native * RocksDB backend it checkpoints incrementally through the keyed state backend. Otherwise state @@ -32,11 +31,11 @@ * state remains readable. * *

Subclasses supply only the native calls that differ — create, restore, snapshot, close, and - * the state-size probe — plus optional direct-RocksDB and processing-time-timer hooks. Everything else - * (budget reservation, restore/snapshot plumbing, handle release) is identical across operators and - * lives here. + * the state-size probe — plus optional direct-RocksDB and processing-time-timer hooks. Everything + * else (budget reservation, restore/snapshot plumbing, handle release) is identical across + * operators and lives here. */ -public abstract class AbstractNativeStatefulOperator extends AbstractStreamOperator { +public abstract class AbstractNativeStatefulOperator extends FlinkStreamOperator { private final String stateLabel; private final int[] keyTimestampPrecisions; diff --git a/src/main/java/tech/streamfusion/operator/ArrowBatchTypeInformation.java b/src/main/java/tech/streamfusion/operator/ArrowBatchTypeInformation.java index 26cf302e4..246294120 100644 --- a/src/main/java/tech/streamfusion/operator/ArrowBatchTypeInformation.java +++ b/src/main/java/tech/streamfusion/operator/ArrowBatchTypeInformation.java @@ -1,15 +1,14 @@ package tech.streamfusion.operator; -import org.apache.flink.api.common.serialization.SerializerConfig; -import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; +import tech.streamfusion.compat.FixedSerializerTypeInformation; /** * The stream element type for the columnar edges between native operators: one {@link ArrowBatch} * per record. Declaring this on a native operator's output transformation is what tells Flink to * carry Arrow batches (via {@link ArrowBatchSerializer}) rather than rows on that edge. */ -public final class ArrowBatchTypeInformation extends TypeInformation { +public final class ArrowBatchTypeInformation extends FixedSerializerTypeInformation { public static final ArrowBatchTypeInformation INSTANCE = new ArrowBatchTypeInformation(false); @@ -57,7 +56,7 @@ public boolean isKeyType() { } @Override - public TypeSerializer createSerializer(SerializerConfig config) { + public TypeSerializer createSerializer() { return new ArrowBatchSerializer(zeroCopy); } diff --git a/src/main/java/tech/streamfusion/operator/ArrowBucketRouter.java b/src/main/java/tech/streamfusion/operator/ArrowBucketRouter.java index 2b7b3fd2b..9fc7766ad 100644 --- a/src/main/java/tech/streamfusion/operator/ArrowBucketRouter.java +++ b/src/main/java/tech/streamfusion/operator/ArrowBucketRouter.java @@ -7,10 +7,10 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import tech.streamfusion.Native; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Routes a sink's Arrow batches to Paimon write destinations. Each incoming batch is split natively @@ -26,7 +26,7 @@ * the router keeps the column for its writer to merge by; it trails the table's columns, so the * routing ordinals are the same either way. */ -public class ArrowBucketRouter extends AbstractStreamOperator +public class ArrowBucketRouter extends FlinkStreamOperator implements OneInputStreamOperator { private final int[] partitionColumns; diff --git a/src/main/java/tech/streamfusion/operator/ArrowToRowDataOperator.java b/src/main/java/tech/streamfusion/operator/ArrowToRowDataOperator.java index 6158488da..179d6017a 100644 --- a/src/main/java/tech/streamfusion/operator/ArrowToRowDataOperator.java +++ b/src/main/java/tech/streamfusion/operator/ArrowToRowDataOperator.java @@ -1,28 +1,28 @@ package tech.streamfusion.operator; -import tech.streamfusion.arrow.ArrowConversion; -import tech.streamfusion.arrow.ArrowReader; import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.table.data.RowData; import org.apache.flink.table.runtime.typeutils.RowDataSerializer; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.types.RowKind; -import org.apache.flink.metrics.Counter; +import tech.streamfusion.arrow.ArrowConversion; +import tech.streamfusion.arrow.ArrowReader; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Transpose leaving a columnar region: reads each {@link ArrowBatch} back into rows. Sits where a * native columnar operator feeds a rowwise (host) one, so the Arrow→row conversion happens once at * the boundary. It consumes (and closes) each batch it receives. * - *

The Arrow reader exposes a reusable view backed by the input batch. Chained Flink operators are - * allowed to retain a collected {@code RowData}, and closing this batch invalidates every such view, - * so the boundary deep-copies each row before handing it back to the rowwise runtime. + *

The Arrow reader exposes a reusable view backed by the input batch. Chained Flink operators + * are allowed to retain a collected {@code RowData}, and closing this batch invalidates every such + * view, so the boundary deep-copies each row before handing it back to the rowwise runtime. */ -public class ArrowToRowDataOperator extends AbstractStreamOperator +public class ArrowToRowDataOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final RowType rowType; diff --git a/src/main/java/tech/streamfusion/operator/BucketedArrowBatchTypeInformation.java b/src/main/java/tech/streamfusion/operator/BucketedArrowBatchTypeInformation.java index 476fa4885..acd566df5 100644 --- a/src/main/java/tech/streamfusion/operator/BucketedArrowBatchTypeInformation.java +++ b/src/main/java/tech/streamfusion/operator/BucketedArrowBatchTypeInformation.java @@ -1,14 +1,14 @@ package tech.streamfusion.operator; -import org.apache.flink.api.common.serialization.SerializerConfig; -import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; +import tech.streamfusion.compat.FixedSerializerTypeInformation; /** * The stream element type on the edge from a sink's bucket router to its writer: one * destination-routed {@link BucketedArrowBatch} per record. */ -public final class BucketedArrowBatchTypeInformation extends TypeInformation { +public final class BucketedArrowBatchTypeInformation + extends FixedSerializerTypeInformation { public static final BucketedArrowBatchTypeInformation INSTANCE = new BucketedArrowBatchTypeInformation(); @@ -44,7 +44,7 @@ public boolean isKeyType() { } @Override - public TypeSerializer createSerializer(SerializerConfig config) { + public TypeSerializer createSerializer() { return new BucketedArrowBatchSerializer(); } diff --git a/src/main/java/tech/streamfusion/operator/FilePartitionSplitOperator.java b/src/main/java/tech/streamfusion/operator/FilePartitionSplitOperator.java index e8a087281..4b4ee743f 100644 --- a/src/main/java/tech/streamfusion/operator/FilePartitionSplitOperator.java +++ b/src/main/java/tech/streamfusion/operator/FilePartitionSplitOperator.java @@ -9,7 +9,6 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.flink.connector.file.table.RowDataPartitionComputer; import org.apache.flink.metrics.Counter; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.table.data.RowData; @@ -19,6 +18,7 @@ import org.apache.flink.table.utils.PartitionPathUtils; import tech.streamfusion.Native; import tech.streamfusion.arrow.ArrowConversion; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Routes the sink's Arrow batches to filesystem buckets. An unpartitioned table passes each batch @@ -28,7 +28,7 @@ * off its first row through Flink's own partition-path code — so escaping, null handling, and value * stringification match the host sink by construction. */ -public class FilePartitionSplitOperator extends AbstractStreamOperator +public class FilePartitionSplitOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final RowType rowType; diff --git a/src/main/java/tech/streamfusion/operator/NativeAsyncLookupJoinOperator.java b/src/main/java/tech/streamfusion/operator/NativeAsyncLookupJoinOperator.java index cb6f7afba..7d5c1fc0f 100644 --- a/src/main/java/tech/streamfusion/operator/NativeAsyncLookupJoinOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeAsyncLookupJoinOperator.java @@ -11,12 +11,8 @@ import java.util.concurrent.TimeoutException; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.api.common.functions.DefaultOpenContext; import org.apache.flink.api.common.functions.util.FunctionUtils; -import org.apache.flink.streaming.api.functions.async.CollectionSupplier; -import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.table.data.RowData; @@ -25,6 +21,8 @@ import org.apache.flink.table.types.logical.RowType; import tech.streamfusion.arrow.ArrowConversion; import tech.streamfusion.arrow.ArrowReader; +import tech.streamfusion.compat.FlinkLookupCompat; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Processing-time lookup join against an async connector, columnar in and out. The async @@ -52,7 +50,7 @@ * timeout and the runner's timeout callback. Results retain probe order, which also satisfies the * unordered mode's relaxed output contract. */ -public class NativeAsyncLookupJoinOperator extends AbstractStreamOperator +public class NativeAsyncLookupJoinOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final RichAsyncFunction runner; @@ -89,7 +87,7 @@ public void open() throws Exception { FunctionUtils.setFunctionRuntimeContext(runner, getRuntimeContext()); runnerNeedsClose = true; try { - FunctionUtils.openFunction(runner, DefaultOpenContext.INSTANCE); + FlinkLookupCompat.open(runner); } catch (Exception | Error failure) { try { closeRunner(); @@ -147,7 +145,7 @@ public void processElement(StreamRecord element) throws Exception { (rows, error) -> { if (error != null) failure.completeExceptionally(error); }); - runner.asyncInvoke(probe, adapt(result)); + runner.asyncInvoke(probe, FlinkLookupCompat.resultFuture(result)); } PendingLookup lookup = pending.getFirst(); await(lookup, failure); @@ -184,31 +182,11 @@ private void await(PendingLookup lookup, CompletableFuture> try { completion.get(Math.max(0, remaining), TimeUnit.NANOSECONDS); } catch (TimeoutException timeout) { - runner.timeout(lookup.probe(), adapt(lookup.result())); + runner.timeout(lookup.probe(), FlinkLookupCompat.resultFuture(lookup.result())); completion.get(); } } private record PendingLookup( RowData probe, CompletableFuture> result, long startedNanos) {} - - /** The runner completes each row's joined results through Flink's {@link ResultFuture} shape. */ - private static ResultFuture adapt(CompletableFuture> future) { - return new ResultFuture<>() { - @Override - public void complete(Collection result) { - future.complete(result); - } - - @Override - public void completeExceptionally(Throwable error) { - future.completeExceptionally(error); - } - - @Override - public void complete(CollectionSupplier supplier) { - throw new UnsupportedOperationException(); - } - }; - } } diff --git a/src/main/java/tech/streamfusion/operator/NativeBytesDecodeOperator.java b/src/main/java/tech/streamfusion/operator/NativeBytesDecodeOperator.java index ee87daf48..43319f9e7 100644 --- a/src/main/java/tech/streamfusion/operator/NativeBytesDecodeOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeBytesDecodeOperator.java @@ -1,7 +1,5 @@ package tech.streamfusion.operator; -import tech.streamfusion.format.NativeMessageDecoder; -import tech.streamfusion.format.NativeMessageDecoderFactory; import java.util.List; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; @@ -9,24 +7,26 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.BoundedOneInput; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; -import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; import org.apache.flink.table.types.logical.RowType; +import tech.streamfusion.compat.FlinkStreamOperator; +import tech.streamfusion.format.NativeMessageDecoder; +import tech.streamfusion.format.NativeMessageDecoderFactory; /** - * The shallow ingest path's format-neutral decode core. It turns raw message bodies into typed Arrow - * batches while a format extension supplies the native decoder through the provider SPI. This class - * owns batching, checkpoint flushing, and the Arrow C Data Interface bridge, so connector and format - * artifacts can be installed independently. + * The shallow ingest path's format-neutral decode core. It turns raw message bodies into typed + * Arrow batches while a format extension supplies the native decoder through the provider SPI. This + * class owns batching, checkpoint flushing, and the Arrow C Data Interface bridge, so connector and + * format artifacts can be installed independently. * *

The operator is stateless across batches. It flushes partial batches at end of input, before a * checkpoint barrier, and on a processing-time timer; this preserves Flink's source-checkpoint * contract while bounding low-volume ingest latency. */ -public class NativeBytesDecodeOperator extends AbstractStreamOperator +public class NativeBytesDecodeOperator extends FlinkStreamOperator implements OneInputStreamOperator, BoundedOneInput { private final RowType outputType; diff --git a/src/main/java/tech/streamfusion/operator/NativeCalcOperator.java b/src/main/java/tech/streamfusion/operator/NativeCalcOperator.java index 33129f90e..1ccafd6ac 100644 --- a/src/main/java/tech/streamfusion/operator/NativeCalcOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeCalcOperator.java @@ -6,11 +6,11 @@ import org.apache.arrow.c.Data; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.table.functions.FunctionContext; import tech.streamfusion.Native; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Stateless native Calc, columnar in and out: applies an encoded Calc — an optional condition then @@ -19,7 +19,7 @@ * columns and constants). The Calc is compiled once into a native handle reused across batches; * carrying Arrow lets it chain with other native operators without converting to rows. */ -public class NativeCalcOperator extends AbstractStreamOperator +public class NativeCalcOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final int[] kinds; diff --git a/src/main/java/tech/streamfusion/operator/NativeColumnarDeduplicateOperator.java b/src/main/java/tech/streamfusion/operator/NativeColumnarDeduplicateOperator.java index ab3c24cb0..0a1add344 100644 --- a/src/main/java/tech/streamfusion/operator/NativeColumnarDeduplicateOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeColumnarDeduplicateOperator.java @@ -1,8 +1,5 @@ package tech.streamfusion.operator; -import tech.streamfusion.Native; -import tech.streamfusion.planner.NativeConfig; -import tech.streamfusion.state.RocksDBNativeStateSupport; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.c.Data; @@ -12,8 +9,9 @@ import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.table.runtime.operators.over.AbstractRowTimeUnboundedPrecedingOver; import org.apache.flink.table.types.logical.RowType; +import tech.streamfusion.Native; +import tech.streamfusion.state.RocksDBNativeStateSupport; /** * Columnar append-only keep-first deduplication (`ROW_NUMBER() OVER (PARTITION BY … ORDER BY rowtime @@ -146,9 +144,7 @@ public void open() throws Exception { super.open(); // Flink's RowTimeDeduplicateKeepFirstRowFunction counts every late-dropped row under this // exact name; the native late filter accumulates the total and each push syncs the delta. - numLateRecordsDropped = - getMetricGroup() - .counter(AbstractRowTimeUnboundedPrecedingOver.LATE_ELEMENTS_DROPPED_METRIC_NAME); + numLateRecordsDropped = getMetricGroup().counter("numLateRecordsDropped"); reportedLateDrops = 0; } diff --git a/src/main/java/tech/streamfusion/operator/NativeColumnarExpandOperator.java b/src/main/java/tech/streamfusion/operator/NativeColumnarExpandOperator.java index fc9fee9c2..cb7c310b9 100644 --- a/src/main/java/tech/streamfusion/operator/NativeColumnarExpandOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeColumnarExpandOperator.java @@ -1,26 +1,26 @@ package tech.streamfusion.operator; -import tech.streamfusion.Native; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.c.CDataDictionaryProvider; import org.apache.arrow.c.Data; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.Native; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Stateless GROUPING SETS / CUBE / ROLLUP expansion, columnar in and out: the Arrow-batch analog of - * Flink's generated {@code ExpandFunction}. Each incoming batch is fanned out natively to one output - * row per grouping set — copying the grouped-in columns, nulling the grouped-out ones, and stamping - * the per-set expand id — then forwarded to the downstream native GROUP BY (over the keys plus the - * expand-id column). It does no buffering, so it forwards watermarks unchanged (the default {@link - * AbstractStreamOperator} behavior); carrying Arrow lets it chain with the native aggregate without a - * transpose. + * Flink's generated {@code ExpandFunction}. Each incoming batch is fanned out natively to one + * output row per grouping set — copying the grouped-in columns, nulling the grouped-out ones, and + * stamping the per-set expand id — then forwarded to the downstream native GROUP BY (over the keys + * plus the expand-id column). It does no buffering, so it forwards watermarks unchanged (the + * default {@link AbstractStreamOperator} behavior); carrying Arrow lets it chain with the native + * aggregate without a transpose. */ -public class NativeColumnarExpandOperator extends AbstractStreamOperator +public class NativeColumnarExpandOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final int numExpandRows; diff --git a/src/main/java/tech/streamfusion/operator/NativeColumnarLocalGroupAggregateOperator.java b/src/main/java/tech/streamfusion/operator/NativeColumnarLocalGroupAggregateOperator.java index f48ace57e..892d462fc 100644 --- a/src/main/java/tech/streamfusion/operator/NativeColumnarLocalGroupAggregateOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeColumnarLocalGroupAggregateOperator.java @@ -1,32 +1,32 @@ package tech.streamfusion.operator; -import tech.streamfusion.Native; -import tech.streamfusion.operator.MiniBatchMetrics.FlushReason; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.c.CDataDictionaryProvider; import org.apache.arrow.c.Data; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.Native; +import tech.streamfusion.compat.FlinkStreamOperator; +import tech.streamfusion.operator.MiniBatchMetrics.FlushReason; /** - * Local half of a two-phase non-windowed {@code GROUP BY}, columnar in and out — the Arrow analog of - * Flink's {@code MapBundleOperator} wrapping {@code MiniBatchLocalGroupAggFunction}. It buffers a - * mini-batch of rows into per-key accumulators (held in the native handle) and flushes one partial - * row per key downstream to the native global merge. + * Local half of a two-phase non-windowed {@code GROUP BY}, columnar in and out — the Arrow analog + * of Flink's {@code MapBundleOperator} wrapping {@code MiniBatchLocalGroupAggFunction}. It buffers + * a mini-batch of rows into per-key accumulators (held in the native handle) and flushes one + * partial row per key downstream to the native global merge. * *

Flush is driven exactly like Flink's bundle: the mini-batch marker the {@link * NativeColumnarMiniBatchAssignerOperator} emits arrives as a {@link Watermark} ({@link * #processWatermark}), a size trigger caps the buffer at {@code miniBatchSize} rows, and the buffer * is always drained before a checkpoint ({@link #prepareSnapshotPreBarrier}) and at end of input - * ({@link #finish}). Because it is drained ahead of every barrier the buffer is transient — there is - * no checkpointed state here; the durable state lives in the global half. + * ({@link #finish}). Because it is drained ahead of every barrier the buffer is transient — there + * is no checkpointed state here; the durable state lives in the global half. */ -public class NativeColumnarLocalGroupAggregateOperator extends AbstractStreamOperator +public class NativeColumnarLocalGroupAggregateOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final int[] aggregateKinds; diff --git a/src/main/java/tech/streamfusion/operator/NativeColumnarLocalWindowAggregateOperator.java b/src/main/java/tech/streamfusion/operator/NativeColumnarLocalWindowAggregateOperator.java index 55f644652..7104725c5 100644 --- a/src/main/java/tech/streamfusion/operator/NativeColumnarLocalWindowAggregateOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeColumnarLocalWindowAggregateOperator.java @@ -4,7 +4,6 @@ import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.c.Data; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.api.common.TaskInfo; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import tech.streamfusion.Native; @@ -106,10 +105,11 @@ public NativeColumnarLocalWindowAggregateOperator( @Override public void open() throws Exception { super.open(); - TaskInfo task = getRuntimeContext().getTaskInfo(); + var task = getRuntimeContext(); setCurrentKey( ArrowBatchSubtaskKeySelector.stateKeysForSubtasks( - maxParallelism(), task.getNumberOfParallelSubtasks())[task.getIndexOfThisSubtask()]); + maxParallelism(), tech.streamfusion.compat.RuntimeCompat.parallelism(task))[ + tech.streamfusion.compat.RuntimeCompat.subtask(task)]); } @Override diff --git a/src/main/java/tech/streamfusion/operator/NativeColumnarMiniBatchAssignerOperator.java b/src/main/java/tech/streamfusion/operator/NativeColumnarMiniBatchAssignerOperator.java index a06b72aa3..6832e6f66 100644 --- a/src/main/java/tech/streamfusion/operator/NativeColumnarMiniBatchAssignerOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeColumnarMiniBatchAssignerOperator.java @@ -1,20 +1,21 @@ package tech.streamfusion.operator; import org.apache.flink.api.common.operators.ProcessingTimeService; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.compat.FlinkStreamOperator; /** - * Columnar analog of Flink's {@code ProcTimeMiniBatchAssignerOperator}: it forwards each Arrow batch - * unchanged and, on a processing-time timer every {@code intervalMs} (and lazily when a batch crosses - * an interval boundary), emits a {@link Watermark} that marks a mini-batch boundary. Downstream native - * mini-batch operators — the local GROUP BY aggregate — flush their bundle on that marker, exactly as - * Flink's {@code MapBundleOperator} does, so the whole columnar island shares one mini-batch cadence. - * Carries no per-row work and no state; Arrow in and out keeps it inside the island. + * Columnar analog of Flink's {@code ProcTimeMiniBatchAssignerOperator}: it forwards each Arrow + * batch unchanged and, on a processing-time timer every {@code intervalMs} (and lazily when a batch + * crosses an interval boundary), emits a {@link Watermark} that marks a mini-batch boundary. + * Downstream native mini-batch operators — the local GROUP BY aggregate — flush their bundle on + * that marker, exactly as Flink's {@code MapBundleOperator} does, so the whole columnar island + * shares one mini-batch cadence. Carries no per-row work and no state; Arrow in and out keeps it + * inside the island. */ -public class NativeColumnarMiniBatchAssignerOperator extends AbstractStreamOperator +public class NativeColumnarMiniBatchAssignerOperator extends FlinkStreamOperator implements OneInputStreamOperator, ProcessingTimeService.ProcessingTimeCallback { diff --git a/src/main/java/tech/streamfusion/operator/NativeColumnarRowTimeMiniBatchAssignerOperator.java b/src/main/java/tech/streamfusion/operator/NativeColumnarRowTimeMiniBatchAssignerOperator.java index 367b8cd62..da346c3c5 100644 --- a/src/main/java/tech/streamfusion/operator/NativeColumnarRowTimeMiniBatchAssignerOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeColumnarRowTimeMiniBatchAssignerOperator.java @@ -1,9 +1,9 @@ package tech.streamfusion.operator; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Columnar analog of Flink's {@code RowTimeMiniBatchAssginerOperator}: forwards each Arrow batch @@ -13,7 +13,7 @@ * proc-time assigner it generates nothing itself; the filtered watermark sequence is a pure * function of the input watermarks, so the operator is deterministic. */ -public class NativeColumnarRowTimeMiniBatchAssignerOperator extends AbstractStreamOperator +public class NativeColumnarRowTimeMiniBatchAssignerOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final long intervalMs; diff --git a/src/main/java/tech/streamfusion/operator/NativeColumnarShareOperator.java b/src/main/java/tech/streamfusion/operator/NativeColumnarShareOperator.java index ff9869842..0a06a0d08 100644 --- a/src/main/java/tech/streamfusion/operator/NativeColumnarShareOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeColumnarShareOperator.java @@ -1,8 +1,8 @@ package tech.streamfusion.operator; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Fan-out point for a shared native sub-plan: declares on each passing batch how many chained @@ -13,7 +13,7 @@ * consumer count RisingWave's {@code StreamShare} carries; the sharing itself is Arrow buffer * reference counting, the analog of Arroyo's {@code Arc} clone. */ -public class NativeColumnarShareOperator extends AbstractStreamOperator +public class NativeColumnarShareOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final int consumers; diff --git a/src/main/java/tech/streamfusion/operator/NativeColumnarUnnestOperator.java b/src/main/java/tech/streamfusion/operator/NativeColumnarUnnestOperator.java index 9469ee0aa..3b76ecc23 100644 --- a/src/main/java/tech/streamfusion/operator/NativeColumnarUnnestOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeColumnarUnnestOperator.java @@ -1,24 +1,24 @@ package tech.streamfusion.operator; -import tech.streamfusion.Native; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.c.CDataDictionaryProvider; import org.apache.arrow.c.Data; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.Native; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Stateless INNER UNNEST of an ARRAY column, columnar in and out: the Arrow-batch analog of Flink's * {@code Correlate} over {@code $UNNEST_ROWS$}. Each incoming batch is fanned out natively to one - * output row per array element — the input columns repeated and the element appended — then forwarded. - * It does no buffering, so watermarks pass straight through; carrying Arrow lets it chain with the - * other native operators without a transpose. + * output row per array element — the input columns repeated and the element appended — then + * forwarded. It does no buffering, so watermarks pass straight through; carrying Arrow lets it + * chain with the other native operators without a transpose. */ -public class NativeColumnarUnnestOperator extends AbstractStreamOperator +public class NativeColumnarUnnestOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final int arrayColumn; diff --git a/src/main/java/tech/streamfusion/operator/NativeColumnarWatermarkAssignerOperator.java b/src/main/java/tech/streamfusion/operator/NativeColumnarWatermarkAssignerOperator.java index bc85c038e..e67f5996c 100644 --- a/src/main/java/tech/streamfusion/operator/NativeColumnarWatermarkAssignerOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeColumnarWatermarkAssignerOperator.java @@ -2,10 +2,10 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.flink.api.common.operators.ProcessingTimeService.ProcessingTimeCallback; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Columnar event-time watermark assigner: the Arrow-batch analog of the host's {@link @@ -28,10 +28,9 @@ * parallelism > 1 a window's effective watermark is the min across its input channels, which a * post-shuffle operator cannot reconstruct). Even sorted rows retain these boundaries because a * downstream {@code CURRENT_WATERMARK} can observe each eager emission. A batch without an internal - * emission boundary is forwarded whole. - * Idleness is not modelled for this assigner. + * emission boundary is forwarded whole. Idleness is not modelled for this assigner. */ -public class NativeColumnarWatermarkAssignerOperator extends AbstractStreamOperator +public class NativeColumnarWatermarkAssignerOperator extends FlinkStreamOperator implements OneInputStreamOperator, ProcessingTimeCallback { private final int rowtimeColumn; diff --git a/src/main/java/tech/streamfusion/operator/NativeFileBulkWriterFactory.java b/src/main/java/tech/streamfusion/operator/NativeFileBulkWriterFactory.java index 9bcbeb17a..54742244e 100644 --- a/src/main/java/tech/streamfusion/operator/NativeFileBulkWriterFactory.java +++ b/src/main/java/tech/streamfusion/operator/NativeFileBulkWriterFactory.java @@ -6,7 +6,7 @@ import org.apache.flink.api.common.serialization.BulkWriter; import org.apache.flink.core.fs.FSDataOutputStream; import org.apache.flink.table.types.logical.RowType; -import tech.streamfusion.arrow.ArrowConversion; +import tech.streamfusion.compat.FileSinkCompat; import tech.streamfusion.format.ColumnarFileCodec; /** @@ -53,7 +53,7 @@ public NativeFileBulkWriterFactory( public BulkWriter create(FSDataOutputStream out) throws IOException { ColumnarFileCodec.Encoder encoder = codec.createEncoder( - ArrowConversion.toArrowSchema(rowType), + FileSinkCompat.encoderSchema(rowType), partitionColumns, configKeys, configValues, diff --git a/src/main/java/tech/streamfusion/operator/NativeFilterOperator.java b/src/main/java/tech/streamfusion/operator/NativeFilterOperator.java index a67d0b385..a91d46bd1 100644 --- a/src/main/java/tech/streamfusion/operator/NativeFilterOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeFilterOperator.java @@ -1,6 +1,5 @@ package tech.streamfusion.operator; -import tech.streamfusion.Native; import java.util.ArrayList; import java.util.List; import org.apache.arrow.c.ArrowArray; @@ -11,18 +10,19 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.util.TransferPair; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.table.functions.FunctionContext; +import tech.streamfusion.Native; +import tech.streamfusion.compat.FlinkStreamOperator; /** - * Stateless native filter, columnar in and out: applies the encoded predicate to each incoming Arrow - * batch natively and emits the surviving rows (projected to an input-column subset/reorder) as a - * batch. The predicate is compiled once into a native handle reused across batches. Carrying Arrow - * lets this chain with other native operators without converting to rows between them. + * Stateless native filter, columnar in and out: applies the encoded predicate to each incoming + * Arrow batch natively and emits the surviving rows (projected to an input-column subset/reorder) + * as a batch. The predicate is compiled once into a native handle reused across batches. Carrying + * Arrow lets this chain with other native operators without converting to rows between them. */ -public class NativeFilterOperator extends AbstractStreamOperator +public class NativeFilterOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final int[] projection; diff --git a/src/main/java/tech/streamfusion/operator/NativeJsonRuntime.java b/src/main/java/tech/streamfusion/operator/NativeJsonRuntime.java index 621bb3895..a5ff26e2b 100644 --- a/src/main/java/tech/streamfusion/operator/NativeJsonRuntime.java +++ b/src/main/java/tech/streamfusion/operator/NativeJsonRuntime.java @@ -3,7 +3,7 @@ import java.util.function.Supplier; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonFactory; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.util.BufferRecycler; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.util.JsonRecyclerPools; +import tech.streamfusion.compat.JsonRuntimeCompat; /** * Jackson input-buffer state shared with Flink's SQL/JSON parser on the task thread. Flink's @@ -63,11 +63,7 @@ static boolean probe(Supplier nativeFactory, Supplier } private static boolean verifiedFactory(JsonFactory factory) { - var version = factory.version(); - return version.getMajorVersion() == 2 - && version.getMinorVersion() == 18 - && version.getPatchLevel() == 2 - && factory._getRecyclerPool() instanceof JsonRecyclerPools.ThreadLocalPool; + return JsonRuntimeCompat.verifiedFactory(factory); } public int bufferSize() { @@ -80,6 +76,6 @@ public void release(int requiredSize) { recycler.releaseCharBuffer( BufferRecycler.CHAR_TOKEN_BUFFER, requiredSize > buffer.length ? new char[requiredSize] : buffer); - recycler.releaseToPool(); + JsonRuntimeCompat.releaseToPool(recycler); } } diff --git a/src/main/java/tech/streamfusion/operator/NativeLookupJoinOperator.java b/src/main/java/tech/streamfusion/operator/NativeLookupJoinOperator.java index 9146a123e..106c7a57d 100644 --- a/src/main/java/tech/streamfusion/operator/NativeLookupJoinOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeLookupJoinOperator.java @@ -1,41 +1,40 @@ package tech.streamfusion.operator; -import tech.streamfusion.arrow.ArrowConversion; -import tech.streamfusion.arrow.ArrowReader; -import tech.streamfusion.arrow.ArrowWriter; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.api.common.functions.DefaultOpenContext; import org.apache.flink.api.common.functions.util.FunctionUtils; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.table.data.RowData; -import org.apache.flink.table.runtime.generated.FilterCondition; import org.apache.flink.table.runtime.operators.join.lookup.LookupJoinRunner; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.util.Collector; +import tech.streamfusion.arrow.ArrowConversion; +import tech.streamfusion.arrow.ArrowReader; +import tech.streamfusion.arrow.ArrowWriter; +import tech.streamfusion.compat.FlinkLookupCompat; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Processing-time lookup join, columnar in and out: for each probe {@link ArrowBatch} it reads the * rows and drives Flink's own {@link LookupJoinRunner} — the exact generated pipeline the host's * lookup join executes: pre-filter, key building (field references and constants alike), the - * connector's real {@code LookupFunction}, the optional projection/filter on the dimension table, the - * residual join condition, and LEFT null-padding — gathering the joined rows back into an Arrow batch. - * Byte-identical to the host by construction, since the row-level core is the host's code; - * keeping the operator's boundary Arrow lets the probe-side Calc/source stay in the native island - * rather than the whole query falling back around the lookup. + * connector's real {@code LookupFunction}, the optional projection/filter on the dimension table, + * the residual join condition, and LEFT null-padding — gathering the joined rows back into an Arrow + * batch. Byte-identical to the host by construction, since the row-level core is the + * host's code; keeping the operator's boundary Arrow lets the probe-side Calc/source stay in the + * native island rather than the whole query falling back around the lookup. * *

Only the dimension lookup is row-oriented (as it must be — a per-key point lookup); the probe * batch and the emitted batch are Arrow. This is the synchronous connector path; an async connector * routes to {@link NativeAsyncLookupJoinOperator}. The runner carries generated code (compiled at * open), so the operator works on a distributed task manager, not just the planner's JVM. * - *

The pre-filter runs under {@link FilterCondition.Context#INVALID_CONTEXT}, exactly as Flink's own - * async lookup runner evaluates the same condition — a pre-filter needing the streaming context - * (current watermark / record timestamp) fails loudly rather than silently diverging. + *

The pre-filter runs without streaming context, exactly as Flink's own async lookup runner + * evaluates the same condition — a pre-filter needing the streaming context (current watermark / + * record timestamp) fails loudly rather than silently diverging. */ -public class NativeLookupJoinOperator extends AbstractStreamOperator +public class NativeLookupJoinOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final LookupJoinRunner runner; @@ -56,7 +55,7 @@ public void open() throws Exception { NativeAllocator.initializeFor(this); allocator = NativeAllocator.SHARED; FunctionUtils.setFunctionRuntimeContext(runner, getRuntimeContext()); - FunctionUtils.openFunction(runner, DefaultOpenContext.INSTANCE); + FlinkLookupCompat.open(runner); } @Override @@ -90,7 +89,7 @@ public void close() {} for (int i = 0; i < rowCount; i++) { RowData probe = reader.read(i); runner.prepareCollector(probe, gather); - if (runner.preFilter(FilterCondition.Context.INVALID_CONTEXT, probe)) { + if (FlinkLookupCompat.preFilter(runner, probe)) { runner.doFetch(probe); } runner.padNullForLeftJoin(probe, gather); diff --git a/src/main/java/tech/streamfusion/operator/NativeMemoryBudget.java b/src/main/java/tech/streamfusion/operator/NativeMemoryBudget.java index 3a474c615..0faf8d976 100644 --- a/src/main/java/tech/streamfusion/operator/NativeMemoryBudget.java +++ b/src/main/java/tech/streamfusion/operator/NativeMemoryBudget.java @@ -21,7 +21,8 @@ public static NativeMemoryBudget registerFor(AbstractStreamOperator operator) StreamingRuntimeContext context = operator.getRuntimeContext(); TaskOffHeapMemory.initialize(context.getTaskManagerRuntimeInfo().getConfiguration()); long ownerId = - TaskOffHeapMemory.registerOwner("operator", context.getTaskInfo().getTaskNameWithSubtasks()); + TaskOffHeapMemory.registerOwner( + "operator", tech.streamfusion.compat.RuntimeCompat.taskName(context)); TaskOffHeapMemory.registerMetrics(operator.getMetricGroup()); return new NativeMemoryBudget(ownerId); } diff --git a/src/main/java/tech/streamfusion/operator/NativeWindowOperatorCore.java b/src/main/java/tech/streamfusion/operator/NativeWindowOperatorCore.java index 810f4c94c..012aa4f4f 100644 --- a/src/main/java/tech/streamfusion/operator/NativeWindowOperatorCore.java +++ b/src/main/java/tech/streamfusion/operator/NativeWindowOperatorCore.java @@ -448,6 +448,7 @@ private void updateColumnarInternal( int[] keyColumns, int[] keyTypes) { boolean proctime = timeColumn < 0; + TimestampAccessor srcTs = proctime ? null : new TimestampAccessor(in.getVector(timeColumn)); int rows = in.getRowCount(); BigIntVector ts = new BigIntVector("ts", allocator); FieldVector[] values = new FieldVector[valueColumns.length]; @@ -468,7 +469,6 @@ private void updateColumnarInternal( keys[j] = newKeyVector("key" + j, keyTypes[j]); vectors.add(keys[j]); } - TimestampAccessor srcTs = proctime ? null : new TimestampAccessor(in.getVector(timeColumn)); TinyIntVector sourceKinds = (TinyIntVector) in.getVector(RowDataArrowConverter.ROW_KIND_COLUMN); TinyIntVector changes = sourceKinds == null diff --git a/src/main/java/tech/streamfusion/operator/NativeWindowTableFunctionOperator.java b/src/main/java/tech/streamfusion/operator/NativeWindowTableFunctionOperator.java index 214cfecb2..4fa6b9cd6 100644 --- a/src/main/java/tech/streamfusion/operator/NativeWindowTableFunctionOperator.java +++ b/src/main/java/tech/streamfusion/operator/NativeWindowTableFunctionOperator.java @@ -7,26 +7,27 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.flink.metrics.Counter; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import tech.streamfusion.Native; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Stateless windowing table function, columnar in and out: the Arrow-batch analog of Flink's {@link - * org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnsliceWindowAggProcessor}-fed {@code - * WindowTableFunctionOperator}. Each incoming batch is assigned to its window(s) natively and emitted - * with {@code window_start}/{@code window_end}/{@code window_time} appended (rows fanned out, one copy - * per window, for hopping/cumulative). It does no event-time buffering — the downstream window join - * or aggregate does — so it forwards watermarks unchanged (the default {@link AbstractStreamOperator} - * behavior); carrying Arrow lets it chain with the native window operators without transposing. + * org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnsliceWindowAggProcessor}-fed + * {@code WindowTableFunctionOperator}. Each incoming batch is assigned to its window(s) natively + * and emitted with {@code window_start}/{@code window_end}/{@code window_time} appended (rows + * fanned out, one copy per window, for hopping/cumulative). It does no event-time buffering — the + * downstream window join or aggregate does — so it forwards watermarks unchanged (the default + * {@link AbstractStreamOperator} behavior); carrying Arrow lets it chain with the native window + * operators without transposing. * *

An event-time TVF assigns each row by its rowtime column. A **proctime** TVF assigns every row * in a batch to the window(s) covering the operator's current processing-time clock (Flink's * processing-time assigner uses the clock, not a row value); the downstream window join/rank closes * those windows on a processing-time timer. The TVF itself stays stateless either way. */ -public class NativeWindowTableFunctionOperator extends AbstractStreamOperator +public class NativeWindowTableFunctionOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final int timeColumn; diff --git a/src/main/java/tech/streamfusion/operator/NullableBytesTypeInformation.java b/src/main/java/tech/streamfusion/operator/NullableBytesTypeInformation.java index 8c6ffe0c3..3b35d8d00 100644 --- a/src/main/java/tech/streamfusion/operator/NullableBytesTypeInformation.java +++ b/src/main/java/tech/streamfusion/operator/NullableBytesTypeInformation.java @@ -1,10 +1,9 @@ package tech.streamfusion.operator; -import org.apache.flink.api.common.serialization.SerializerConfig; -import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer; import org.apache.flink.api.java.typeutils.runtime.NullableSerializer; +import tech.streamfusion.compat.FixedSerializerTypeInformation; /** * The stream element type for the raw-message edge between Flink's byte source and the native @@ -13,7 +12,7 @@ * it, raw decodes it to a null field — so the edge must carry null through the chain's serializer * copy, which Flink's plain {@code byte[]} serializer cannot. */ -public final class NullableBytesTypeInformation extends TypeInformation { +public final class NullableBytesTypeInformation extends FixedSerializerTypeInformation { public static final NullableBytesTypeInformation INSTANCE = new NullableBytesTypeInformation(); @@ -50,7 +49,7 @@ public boolean isKeyType() { } @Override - public TypeSerializer createSerializer(SerializerConfig config) { + public TypeSerializer createSerializer() { return NullableSerializer.wrap(BytePrimitiveArraySerializer.INSTANCE, false); } diff --git a/src/main/java/tech/streamfusion/operator/OrderedKeyGroupReassembler.java b/src/main/java/tech/streamfusion/operator/OrderedKeyGroupReassembler.java index fe6756e96..f60ff7550 100644 --- a/src/main/java/tech/streamfusion/operator/OrderedKeyGroupReassembler.java +++ b/src/main/java/tech/streamfusion/operator/OrderedKeyGroupReassembler.java @@ -21,10 +21,10 @@ import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; import org.apache.flink.streaming.api.operators.BoundedOneInput; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Reconstructs destination-local parent batches after the unaligned-safe shuffle emits one record @@ -32,14 +32,15 @@ * independently processable because some siblings may already live in downstream operator state; * the new producer attempt gets a fresh epoch and resumes ordered parent reassembly. */ -public final class OrderedKeyGroupReassembler extends AbstractStreamOperator +public final class OrderedKeyGroupReassembler extends FlinkStreamOperator implements OneInputStreamOperator, BoundedOneInput { private static final ListStateDescriptor STATE = new ListStateDescriptor<>( "streamfusion-ordered-key-group-fragments", BytePrimitiveArraySerializer.INSTANCE); private static final ListStateDescriptor WATERMARK_STATE = - new ListStateDescriptor<>("streamfusion-ordered-key-group-watermark", LongSerializer.INSTANCE); + new ListStateDescriptor<>( + "streamfusion-ordered-key-group-watermark", LongSerializer.INSTANCE); private static final ListStateDescriptor EPOCH_STATE = new ListStateDescriptor<>( "streamfusion-ordered-key-group-epochs", BytePrimitiveArraySerializer.INSTANCE); @@ -157,8 +158,10 @@ private void add(ArrowBatch fragment, boolean emitWhenComplete) throws Exception private boolean owns(int keyGroup) { return KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup( - maxParallelism, getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks(), keyGroup) - == getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); + maxParallelism, + tech.streamfusion.compat.RuntimeCompat.parallelism(getRuntimeContext()), + keyGroup) + == tech.streamfusion.compat.RuntimeCompat.subtask(getRuntimeContext()); } private int expectedFragments(int[] keyGroups) { diff --git a/src/main/java/tech/streamfusion/operator/PartitionedArrowBatchTypeInformation.java b/src/main/java/tech/streamfusion/operator/PartitionedArrowBatchTypeInformation.java index f74663f4b..19cb77369 100644 --- a/src/main/java/tech/streamfusion/operator/PartitionedArrowBatchTypeInformation.java +++ b/src/main/java/tech/streamfusion/operator/PartitionedArrowBatchTypeInformation.java @@ -1,15 +1,14 @@ package tech.streamfusion.operator; -import org.apache.flink.api.common.serialization.SerializerConfig; -import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; +import tech.streamfusion.compat.FixedSerializerTypeInformation; /** * The stream element type on the edge from the sink's partition splitter to its file writer: one * bucket-routed {@link PartitionedArrowBatch} per record. */ public final class PartitionedArrowBatchTypeInformation - extends TypeInformation { + extends FixedSerializerTypeInformation { public static final PartitionedArrowBatchTypeInformation INSTANCE = new PartitionedArrowBatchTypeInformation(); @@ -45,7 +44,7 @@ public boolean isKeyType() { } @Override - public TypeSerializer createSerializer(SerializerConfig config) { + public TypeSerializer createSerializer() { return new PartitionedArrowBatchSerializer(); } diff --git a/src/main/java/tech/streamfusion/operator/PrunedRowData.java b/src/main/java/tech/streamfusion/operator/PrunedRowData.java index 9b5be7cf3..407aeb16e 100644 --- a/src/main/java/tech/streamfusion/operator/PrunedRowData.java +++ b/src/main/java/tech/streamfusion/operator/PrunedRowData.java @@ -13,18 +13,19 @@ import org.apache.flink.types.RowKind; /** - * A reusable, zero-copy view that presents a wide source {@link RowData} as a narrower pruned schema, - * recursively — top-level columns and, within {@code ROW} columns, nested sub-fields. Each field is - * matched to the source by name (names are unique within a row type), so {@link #getRow} returns a - * child {@code PrunedRowData} over the source struct rather than the full struct. + * A reusable, zero-copy view that presents a wide source {@link RowData} as a narrower pruned + * schema, recursively — top-level columns and, within {@code ROW} columns, nested sub-fields. Each + * field is matched to the source by name (names are unique within a row type), so {@link #getRow} + * returns a child {@code PrunedRowData} over the source struct rather than the full struct. * - *

Used by the entry transpose for nested projection pushdown: the converter, driven by the pruned - * schema, builds and fills only the Arrow columns the native region actually reads — the unread fields - * of a wide source row (e.g. a Nexmark {@code bid.channel}/{@code bid.url}) never touch Arrow. Reusable - * like Flink's {@link org.apache.flink.table.data.utils.ProjectedRowData} (which is top-level only): - * {@link #replaceRow} repoints it, and the converter reads each row inline before the next. + *

Used by the entry transpose for nested projection pushdown: the converter, driven by the + * pruned schema, builds and fills only the Arrow columns the native region actually reads — the + * unread fields of a wide source row (e.g. a Nexmark {@code bid.channel}/{@code bid.url}) never + * touch Arrow. Reusable like Flink's {@link org.apache.flink.table.data.utils.ProjectedRowData} + * (which is top-level only): {@link #replaceRow} repoints it, and the converter reads each row + * inline before the next. */ -public final class PrunedRowData implements RowData { +public final class PrunedRowData extends tech.streamfusion.compat.ProjectedRowDataCompat { private final int[] indexMapping; private final PrunedRowData[] children; // non-null only for nested ROW fields @@ -151,8 +152,13 @@ public byte[] getBinary(int pos) { } @Override - public org.apache.flink.types.variant.Variant getVariant(int pos) { - return row.getVariant(indexMapping[pos]); + protected RowData sourceRow() { + return row; + } + + @Override + protected int sourceIndex(int pos) { + return indexMapping[pos]; } @Override diff --git a/src/main/java/tech/streamfusion/operator/RowDataToArrowOperator.java b/src/main/java/tech/streamfusion/operator/RowDataToArrowOperator.java index fc385c927..3ca180e24 100644 --- a/src/main/java/tech/streamfusion/operator/RowDataToArrowOperator.java +++ b/src/main/java/tech/streamfusion/operator/RowDataToArrowOperator.java @@ -1,12 +1,11 @@ package tech.streamfusion.operator; -import tech.streamfusion.planner.NativeConfig; import java.util.AbstractList; import java.util.ArrayList; import java.util.List; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.metrics.Counter; import org.apache.flink.streaming.api.operators.BoundedOneInput; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.watermark.Watermark; @@ -14,7 +13,8 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.runtime.typeutils.RowDataSerializer; import org.apache.flink.table.types.logical.RowType; -import org.apache.flink.metrics.Counter; +import tech.streamfusion.compat.FlinkStreamOperator; +import tech.streamfusion.planner.NativeConfig; /** * Transpose entering a columnar region: buffers rows and emits them as {@link ArrowBatch}es. Sits @@ -22,9 +22,10 @@ * once at the boundary rather than inside every native operator. * *

Ownership of an emitted batch passes to the downstream operator, which closes it once read (in - * a chained task the downstream consumes it inline). Watermarks pass through after the buffer flushes. + * a chained task the downstream consumes it inline). Watermarks pass through after the buffer + * flushes. */ -public class RowDataToArrowOperator extends AbstractStreamOperator +public class RowDataToArrowOperator extends FlinkStreamOperator implements OneInputStreamOperator, BoundedOneInput { private final RowType rowType; diff --git a/src/main/java/tech/streamfusion/operator/SplitByKeyGroupOperator.java b/src/main/java/tech/streamfusion/operator/SplitByKeyGroupOperator.java index 9f1c73d57..4dc60f6c1 100644 --- a/src/main/java/tech/streamfusion/operator/SplitByKeyGroupOperator.java +++ b/src/main/java/tech/streamfusion/operator/SplitByKeyGroupOperator.java @@ -1,6 +1,5 @@ package tech.streamfusion.operator; -import tech.streamfusion.Native; import java.util.UUID; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; @@ -9,16 +8,17 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.flink.metrics.Counter; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.Native; +import tech.streamfusion.compat.FlinkStreamOperator; /** * Splits each incoming Arrow batch for a keyed shuffle. Aligned-only jobs emit one sub-batch per * destination channel. Unaligned-enabled jobs emit one independently recoverable fragment per key * group with the metadata {@link OrderedKeyGroupReassembler} needs to restore parent order. */ -public class SplitByKeyGroupOperator extends AbstractStreamOperator +public class SplitByKeyGroupOperator extends FlinkStreamOperator implements OneInputStreamOperator { private final int[] keyColumns; diff --git a/src/main/java/tech/streamfusion/paimon/NativePaimonBucketAssigner.java b/src/main/java/tech/streamfusion/paimon/NativePaimonBucketAssigner.java index c2f11b9d2..9b418b426 100644 --- a/src/main/java/tech/streamfusion/paimon/NativePaimonBucketAssigner.java +++ b/src/main/java/tech/streamfusion/paimon/NativePaimonBucketAssigner.java @@ -6,7 +6,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.flink.runtime.state.StateInitializationContext; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.paimon.data.BinaryRow; @@ -15,12 +14,13 @@ import org.apache.paimon.index.HashBucketAssigner; import org.apache.paimon.table.FileStoreTable; import tech.streamfusion.Native; +import tech.streamfusion.compat.FlinkStreamOperator; import tech.streamfusion.operator.BucketedArrowBatch; import tech.streamfusion.operator.ColumnarRecordMetrics; import tech.streamfusion.operator.NativeAllocator; /** Paimon's released dynamic bucket index applied to native key hashes without unpacking rows. */ -public final class NativePaimonBucketAssigner extends AbstractStreamOperator +public final class NativePaimonBucketAssigner extends FlinkStreamOperator implements OneInputStreamOperator { private final FileStoreTable table; private final String initialCommitUser; diff --git a/src/main/java/tech/streamfusion/paimon/NativePaimonDynamicPartitionOperator.java b/src/main/java/tech/streamfusion/paimon/NativePaimonDynamicPartitionOperator.java index 48ed67cda..da15e258b 100644 --- a/src/main/java/tech/streamfusion/paimon/NativePaimonDynamicPartitionOperator.java +++ b/src/main/java/tech/streamfusion/paimon/NativePaimonDynamicPartitionOperator.java @@ -5,25 +5,17 @@ import org.apache.arrow.c.Data; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.runtime.event.WatermarkEvent; import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.runtime.operators.coordination.OperatorCoordinator; import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory; import org.apache.flink.streaming.api.operators.CoordinatedOperatorFactory; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; -import org.apache.flink.streaming.api.operators.Output; import org.apache.flink.streaming.api.operators.StreamOperator; import org.apache.flink.streaming.api.operators.StreamOperatorParameters; -import org.apache.flink.streaming.api.watermark.Watermark; -import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; -import org.apache.flink.streaming.runtime.streamrecord.RecordAttributes; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; -import org.apache.flink.util.OutputTag; import org.apache.paimon.data.InternalRow; import org.apache.paimon.flink.sink.partition.DataStatisticsOperator; import org.apache.paimon.flink.sink.partition.DataStatisticsOperatorFactory; @@ -31,13 +23,14 @@ import org.apache.paimon.flink.sink.partition.StatisticsOrRecordChannelComputer; import org.apache.paimon.schema.TableSchema; import tech.streamfusion.Native; +import tech.streamfusion.compat.FlinkStreamOperator; import tech.streamfusion.operator.BucketedArrowBatch; import tech.streamfusion.operator.ColumnarRecordMetrics; import tech.streamfusion.operator.NativeAllocator; /** Paimon's adaptive partition shuffle over Arrow payloads and partition-only metadata. */ public final class NativePaimonDynamicPartitionOperator - extends AbstractStreamOperator + extends FlinkStreamOperator implements OneInputStreamOperator { private final int partitionArity; @@ -57,13 +50,8 @@ public final class NativePaimonDynamicPartitionOperator this.statistics = new DataStatisticsOperatorFactory(partitionSchema) .createStreamOperator( - new StreamOperatorParameters<>( - parameters.getContainingTask(), - parameters.getStreamConfig(), - new StatisticsOutput(), - parameters::getProcessingTimeService, - parameters.getOperatorEventDispatcher(), - parameters.getMailboxExecutor())); + tech.streamfusion.compat.RuntimeCompat.withOutput( + parameters, new StatisticsOutput())); } @Override @@ -136,46 +124,18 @@ public void processElement(StreamRecord element) throws Exce } } - private final class StatisticsOutput implements Output> { + private final class StatisticsOutput + extends tech.streamfusion.compat.FlinkOutput { + StatisticsOutput() { + super(NativePaimonDynamicPartitionOperator.this.output); + } + @Override public void collect(StreamRecord record) { // Statistics events replace the stock assignment map. Their arbitrary output channel is // discarded, as it is by Paimon's downstream Strip Statistics operator. selectedChannel = channels.channel(record.getValue()); } - - @Override - public void emitWatermark(Watermark watermark) { - output.emitWatermark(watermark); - } - - @Override - public void emitWatermark(WatermarkEvent watermark) { - output.emitWatermark(watermark); - } - - @Override - public void emitWatermarkStatus(WatermarkStatus status) { - output.emitWatermarkStatus(status); - } - - @Override - public void collect(OutputTag tag, StreamRecord record) { - output.collect(tag, record); - } - - @Override - public void emitLatencyMarker(LatencyMarker marker) { - output.emitLatencyMarker(marker); - } - - @Override - public void emitRecordAttributes(RecordAttributes attributes) { - output.emitRecordAttributes(attributes); - } - - @Override - public void close() {} } public static final class Factory extends AbstractStreamOperatorFactory diff --git a/src/main/java/tech/streamfusion/paimon/NativePaimonLocalMergeOperator.java b/src/main/java/tech/streamfusion/paimon/NativePaimonLocalMergeOperator.java index c9425920f..f5f4234cc 100644 --- a/src/main/java/tech/streamfusion/paimon/NativePaimonLocalMergeOperator.java +++ b/src/main/java/tech/streamfusion/paimon/NativePaimonLocalMergeOperator.java @@ -23,7 +23,6 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.BoundedOneInput; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.watermark.Watermark; @@ -52,6 +51,7 @@ import org.apache.paimon.utils.UserDefinedSeqComparator; import tech.streamfusion.arrow.ArrowConversion; import tech.streamfusion.arrow.ArrowWriter; +import tech.streamfusion.compat.FlinkStreamOperator; import tech.streamfusion.operator.ArrowBatch; import tech.streamfusion.operator.KeyedUpsertBuffer; import tech.streamfusion.operator.NativeAllocator; @@ -60,7 +60,7 @@ /** * Merges updates before shuffle, retaining Java grouping where the intermediate rows are visible. */ -public final class NativePaimonLocalMergeOperator extends AbstractStreamOperator +public final class NativePaimonLocalMergeOperator extends FlinkStreamOperator implements OneInputStreamOperator, BoundedOneInput { private final FileStoreTable table; private final boolean nativeMerge; diff --git a/src/main/java/tech/streamfusion/paimon/NativePaimonSplitReader.java b/src/main/java/tech/streamfusion/paimon/NativePaimonSplitReader.java index 18d4051f1..ad82d112b 100644 --- a/src/main/java/tech/streamfusion/paimon/NativePaimonSplitReader.java +++ b/src/main/java/tech/streamfusion/paimon/NativePaimonSplitReader.java @@ -264,7 +264,7 @@ private VectorSchemaRoot javaBatch() throws IOException { rowBatch.releaseBatch(); rowBatch = null; } else { - batch.add(copy.copy(new FlinkRowData(row))); + batch.add(tech.streamfusion.compat.RuntimeCompat.copyRow(copy, new FlinkRowData(row))); } } return batch.isEmpty() diff --git a/src/main/java/tech/streamfusion/planner/ChangelogNormalizeMatcher.java b/src/main/java/tech/streamfusion/planner/ChangelogNormalizeMatcher.java index 59e014ce0..49a2edbd3 100644 --- a/src/main/java/tech/streamfusion/planner/ChangelogNormalizeMatcher.java +++ b/src/main/java/tech/streamfusion/planner/ChangelogNormalizeMatcher.java @@ -1,10 +1,10 @@ package tech.streamfusion.planner; -import tech.streamfusion.operator.RowDataArrowConverter; import org.apache.calcite.rel.RelNode; import org.apache.flink.table.planner.calcite.FlinkTypeFactory$; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize; import org.apache.flink.table.planner.plan.utils.ChangelogPlanUtils; +import tech.streamfusion.operator.RowDataArrowConverter; /** * Recognizes the changelog normalization the native operator reproduces: a {@link @@ -20,10 +20,10 @@ final class ChangelogNormalizeMatcher { private ChangelogNormalizeMatcher() {} static boolean matches(StreamPhysicalChangelogNormalize node) { - if (node.filterCondition() != null) { + if (tech.streamfusion.compat.FlinkCompat.normalizeHasFilter(node)) { return false; // a pushed filter condition is not yet reproduced } - if (node.sourceReused() || node.commonFilter().length > 0) { + if (tech.streamfusion.compat.FlinkCompat.normalizeSharesSource(node)) { return false; // the source-reuse rewrite changes the operator's contract } return RowDataArrowConverter.supports( @@ -39,10 +39,10 @@ static boolean generateUpdateBefore(StreamPhysicalChangelogNormalize node) { } static String unsupportedReason(StreamPhysicalChangelogNormalize node) { - if (node.filterCondition() != null) { + if (tech.streamfusion.compat.FlinkCompat.normalizeHasFilter(node)) { return "changelog normalize: a pushed filter condition is not supported"; } - if (node.sourceReused() || node.commonFilter().length > 0) { + if (tech.streamfusion.compat.FlinkCompat.normalizeSharesSource(node)) { return "changelog normalize: the source-reuse variant is not supported"; } return "changelog normalize: needs a row type the Arrow conversion supports"; diff --git a/src/main/java/tech/streamfusion/planner/ColumnarKeyGroupPartitioner.java b/src/main/java/tech/streamfusion/planner/ColumnarKeyGroupPartitioner.java index b3d65a5f4..01d6a71b9 100644 --- a/src/main/java/tech/streamfusion/planner/ColumnarKeyGroupPartitioner.java +++ b/src/main/java/tech/streamfusion/planner/ColumnarKeyGroupPartitioner.java @@ -1,19 +1,20 @@ package tech.streamfusion.planner; -import tech.streamfusion.operator.ArrowBatch; import org.apache.flink.runtime.io.network.api.writer.SubtaskStateMapper; import org.apache.flink.runtime.plugable.SerializationDelegate; import org.apache.flink.runtime.state.KeyGroupRangeAssignment; import org.apache.flink.streaming.runtime.partitioner.ConfigurableStreamPartitioner; import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import tech.streamfusion.operator.ArrowBatch; /** * Routes an Arrow shuffle record using its key-group tag. Destination-batched records force the * edge aligned; recovery-mode records contain exactly one key group and support Flink's ordinary * unaligned {@link SubtaskStateMapper#RANGE} filtering after rescaling. */ -public class ColumnarKeyGroupPartitioner extends StreamPartitioner +public class ColumnarKeyGroupPartitioner + extends tech.streamfusion.compat.ColumnarPartitionerCompat implements ConfigurableStreamPartitioner { private static final long serialVersionUID = 1L; @@ -49,7 +50,10 @@ public int selectChannel(SerializationDelegate> record) @Override public StreamPartitioner copy() { - return new ColumnarKeyGroupPartitioner(maxParallelism, recoverable); + ColumnarKeyGroupPartitioner copy = new ColumnarKeyGroupPartitioner(maxParallelism, recoverable); + // Flink 1.18's recovery filter copies after setup and does not configure the copy again. + copy.setup(numberOfChannels); + return copy; } @Override diff --git a/src/main/java/tech/streamfusion/planner/FileSinkMatcher.java b/src/main/java/tech/streamfusion/planner/FileSinkMatcher.java index 5a82ed3f0..b5db59376 100644 --- a/src/main/java/tech/streamfusion/planner/FileSinkMatcher.java +++ b/src/main/java/tech/streamfusion/planner/FileSinkMatcher.java @@ -110,7 +110,7 @@ static Planned plan(StreamPhysicalSink sink, Format format) { Map catalogOptions = table.getOptions(); boolean changelog = format.changelog(catalogOptions); Map options = format.writerOptions(catalogOptions); - for (SinkAbilitySpec spec : sink.abilitySpecs()) { + for (SinkAbilitySpec spec : tech.streamfusion.compat.FlinkCompat.sinkAbilities(sink)) { if (spec instanceof OverwriteSpec) { // Falling back reproduces the host's own error: streaming INSERT OVERWRITE is rejected. return Planned.fallback("INSERT OVERWRITE is not supported in streaming mode"); diff --git a/src/main/java/tech/streamfusion/planner/FlinkExpressionFunction.java b/src/main/java/tech/streamfusion/planner/FlinkExpressionFunction.java index 90d054037..6c906150f 100644 --- a/src/main/java/tech/streamfusion/planner/FlinkExpressionFunction.java +++ b/src/main/java/tech/streamfusion/planner/FlinkExpressionFunction.java @@ -119,7 +119,7 @@ private FlinkExpressionFunction( this.rowResultType = body.rowType(); var context = body.context(); functions = List.copyOf(context.functionInstances.values()); - String className = "FlinkExpressionEvaluator" + context.getNameCounter().getAndIncrement(); + String className = tech.streamfusion.compat.FlinkCompat.expressionClassName(context); String code = "public final class " + className diff --git a/src/main/java/tech/streamfusion/planner/GlobalGroupAggregateMatcher.java b/src/main/java/tech/streamfusion/planner/GlobalGroupAggregateMatcher.java index 774fe0590..a98cb5d47 100644 --- a/src/main/java/tech/streamfusion/planner/GlobalGroupAggregateMatcher.java +++ b/src/main/java/tech/streamfusion/planner/GlobalGroupAggregateMatcher.java @@ -1,6 +1,5 @@ package tech.streamfusion.planner; -import tech.streamfusion.operator.RowDataArrowConverter; import java.util.ArrayList; import java.util.List; import org.apache.calcite.rel.RelNode; @@ -8,9 +7,10 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.flink.table.planner.calcite.FlinkTypeFactory$; -import org.apache.flink.table.planner.hint.StateTtlHint; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGlobalGroupAggregate; import org.apache.flink.table.planner.plan.utils.ChangelogPlanUtils; +import tech.streamfusion.compat.FlinkCompat; +import tech.streamfusion.operator.RowDataArrowConverter; /** * Recognizes the global half of a two-phase non-windowed {@code GROUP BY}: it merges the local @@ -389,7 +389,7 @@ static RelNode substitute(StreamPhysicalGlobalGroupAggregate agg, PlanContext ct int[] keyColumns = GlobalGroupAggregateMatcher.keyColumns(agg); // TTL lives on the stateful global half only (the local is a transient per-bundle buffer); // a STATE_TTL hint on the aggregate overrides the job-wide retention, as single-phase. - Long stateTtlHint = StateTtlHint.getStateTtlFromHintOnSingleRel(agg.hints()); + Long stateTtlHint = FlinkCompat.singleStateTtl(agg); return new StreamPhysicalNativeColumnarGroupAggregate( agg.getCluster(), agg.getTraitSet(), diff --git a/src/main/java/tech/streamfusion/planner/GroupAggregateMatcher.java b/src/main/java/tech/streamfusion/planner/GroupAggregateMatcher.java index f2a957acb..039315d4c 100644 --- a/src/main/java/tech/streamfusion/planner/GroupAggregateMatcher.java +++ b/src/main/java/tech/streamfusion/planner/GroupAggregateMatcher.java @@ -5,10 +5,10 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.flink.table.planner.calcite.FlinkTypeFactory$; -import org.apache.flink.table.planner.hint.StateTtlHint; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGroupAggregate; import org.apache.flink.table.planner.plan.utils.ChangelogPlanUtils; import scala.collection.Seq; +import tech.streamfusion.compat.FlinkCompat; import tech.streamfusion.operator.RowDataArrowConverter; /** @@ -56,7 +56,7 @@ static String unsupportedReason(StreamPhysicalGroupAggregate agg) { } if (kind >= 12 && kind <= 16) { if (kind >= 15) { - Long hint = StateTtlHint.getStateTtlFromHintOnSingleRel(agg.hints()); + Long hint = FlinkCompat.singleStateTtl(agg); long retention = hint == null ? org.apache.flink.table.planner.utils.ShortcutUtils.unwrapTableConfig(agg) @@ -306,7 +306,7 @@ static RelNode substitute(StreamPhysicalGroupAggregate agg, PlanContext ctx) { int[] keyColumns = GroupAggregateMatcher.keyColumns(agg); // A STATE_TTL hint overrides the job-wide retention for this aggregate alone (Flink's // StateMetadata precedence); null means no hint, resolved at translate time. - Long stateTtlHint = StateTtlHint.getStateTtlFromHintOnSingleRel(agg.hints()); + Long stateTtlHint = FlinkCompat.singleStateTtl(agg); // The aggregate is columnar (Arrow in/out). Keep the keyed shuffle columnar where the input // sits on a columnar producer (a native exchange splits the batch by the grouping keys); // otherwise the transition pass inserts a transpose at the host exchange boundary. Same key diff --git a/src/main/java/tech/streamfusion/planner/GroupWindowAggregateMatcher.java b/src/main/java/tech/streamfusion/planner/GroupWindowAggregateMatcher.java index 0f29984b0..3cda2870e 100644 --- a/src/main/java/tech/streamfusion/planner/GroupWindowAggregateMatcher.java +++ b/src/main/java/tech/streamfusion/planner/GroupWindowAggregateMatcher.java @@ -53,6 +53,9 @@ static String unsupportedReason(StreamPhysicalGroupWindowAggregate agg) { return "legacy group-window: the time attribute must be TIMESTAMP or TIMESTAMP_LTZ"; } boolean proctime = LogicalTypeChecks.isProctimeAttribute(timeType); + if (!proctime && timeColumn(agg) < 0) { + return "legacy group-window: host input time attribute is unavailable"; + } if (session && proctime) { return "legacy group-window: processing-time SESSION is not native"; } @@ -182,7 +185,7 @@ private static Duration gap(StreamPhysicalGroupWindowAggregate agg) { } private static int timeColumn(StreamPhysicalGroupWindowAggregate agg) { - return agg.window().timeAttribute().getFieldIndex(); + return tech.streamfusion.compat.FlinkCompat.groupWindowTimeColumn(agg); } private static boolean isProctime(StreamPhysicalGroupWindowAggregate agg) { diff --git a/src/main/java/tech/streamfusion/planner/HostCastFunction.java b/src/main/java/tech/streamfusion/planner/HostCastFunction.java index 4ba858224..647052796 100644 --- a/src/main/java/tech/streamfusion/planner/HostCastFunction.java +++ b/src/main/java/tech/streamfusion/planner/HostCastFunction.java @@ -1,15 +1,11 @@ package tech.streamfusion.planner; import java.math.BigDecimal; -import java.time.ZoneId; -import org.apache.flink.configuration.Configuration; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.StringData; -import org.apache.flink.table.functions.ScalarFunction; -import org.apache.flink.table.functions.FunctionContext; import org.apache.flink.table.data.utils.CastExecutor; -import org.apache.flink.table.planner.codegen.CodeGeneratorContext; -import org.apache.flink.table.planner.functions.casting.CastRule; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.ScalarFunction; import org.apache.flink.table.planner.functions.casting.CastRuleProvider; import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.LogicalType; @@ -72,16 +68,12 @@ private void initializeExecutor(ClassLoader classLoader) { if (classLoader == null) { classLoader = ClassLoader.getSystemClassLoader(); } - // The admitted casts (number↔string, string length) never consult the zone; UTC is a placeholder. + // The admitted casts (number↔string, string length) never consult the zone; UTC is a + // placeholder. executor = (CastExecutor) CastRuleProvider.create( - CastRule.Context.create( - false, - false, - ZoneId.of("UTC"), - classLoader, - new CodeGeneratorContext(new Configuration(), classLoader)), + tech.streamfusion.compat.FlinkCompat.castContext(classLoader), inputType, targetType); if (executor == null) { diff --git a/src/main/java/tech/streamfusion/planner/JsonPathSpec.java b/src/main/java/tech/streamfusion/planner/JsonPathSpec.java index b06efb0db..19d94fee2 100644 --- a/src/main/java/tech/streamfusion/planner/JsonPathSpec.java +++ b/src/main/java/tech/streamfusion/planner/JsonPathSpec.java @@ -2,8 +2,8 @@ import java.util.Locale; import java.util.regex.Pattern; -import org.apache.flink.shaded.com.jayway.jsonpath.internal.Utils; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.io.JsonStringEncoder; +import tech.streamfusion.compat.FlinkCompat; /** Admission grammar shared with the native SQL/JSON path reader. */ final class JsonPathSpec { @@ -92,7 +92,7 @@ static String normalize(String path) { boolean single = step.group("single") != null; String name = step.group(single ? "single" : "quoted"); if (name.indexOf('\\') >= 0) { - name = Utils.unescape(name); + name = FlinkCompat.unescapeJsonPath(name); if (name.codePoints().anyMatch(c -> c >= 0xd800 && c <= 0xdfff)) return null; // The native wire grammar uses JSON escaping, independent of the SQL path's quote style. normalized diff --git a/src/main/java/tech/streamfusion/planner/KafkaSinkMatcher.java b/src/main/java/tech/streamfusion/planner/KafkaSinkMatcher.java index 79f153caa..e84b6ad27 100644 --- a/src/main/java/tech/streamfusion/planner/KafkaSinkMatcher.java +++ b/src/main/java/tech/streamfusion/planner/KafkaSinkMatcher.java @@ -1,9 +1,5 @@ package tech.streamfusion.planner; -import tech.streamfusion.format.EncodeFormat; -import tech.streamfusion.format.FormatCodes; -import tech.streamfusion.kafka.JdkFloatSpelling; -import tech.streamfusion.kafka.NativeKafka; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -23,6 +19,10 @@ import org.apache.flink.table.types.logical.LogicalTypeRoot; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.TimeType; +import tech.streamfusion.format.EncodeFormat; +import tech.streamfusion.format.FormatCodes; +import tech.streamfusion.kafka.JdkFloatSpelling; +import tech.streamfusion.kafka.NativeKafka; /** Conservative match boundary for native JSON serialization into Flink's Kafka sink. */ final class KafkaSinkMatcher { @@ -93,8 +93,8 @@ static Planned plan(StreamPhysicalSink sink) { return Planned.fallback( "an upsert-materialized sink (SinkUpsertMaterializer) is not natively reproduced"); } - if (sink.abilitySpecs().length != 0) { - SinkAbilitySpec spec = sink.abilitySpecs()[0]; + if (!tech.streamfusion.compat.FlinkCompat.sinkAbilities(sink).isEmpty()) { + SinkAbilitySpec spec = tech.streamfusion.compat.FlinkCompat.sinkAbilities(sink).get(0); return Planned.fallback("sink ability " + spec.getClass().getSimpleName()); } KafkaSinkTranslator.Result translated = KafkaSinkTranslator.translate(options(sink)); diff --git a/src/main/java/tech/streamfusion/planner/KafkaSinkTranslator.java b/src/main/java/tech/streamfusion/planner/KafkaSinkTranslator.java index d93248f13..6ca160104 100644 --- a/src/main/java/tech/streamfusion/planner/KafkaSinkTranslator.java +++ b/src/main/java/tech/streamfusion/planner/KafkaSinkTranslator.java @@ -5,7 +5,7 @@ import java.util.Map; import java.util.Properties; import org.apache.flink.connector.base.DeliveryGuarantee; -import org.apache.flink.connector.kafka.sink.TransactionNamingStrategy; +import tech.streamfusion.kafka.compat.KafkaSinkCompat; /** Conservative table-option boundary for the native-serialization/stock-KafkaSink hybrid. */ final class KafkaSinkTranslator { @@ -39,7 +39,7 @@ static final class Planned { final Properties producerProperties; final DeliveryGuarantee deliveryGuarantee; final String transactionalIdPrefix; - final TransactionNamingStrategy transactionNamingStrategy; + final String transactionNamingStrategy; final Integer parallelism; final String valueFormat; final String keyFormat; @@ -52,7 +52,7 @@ private Planned( Properties producerProperties, DeliveryGuarantee deliveryGuarantee, String transactionalIdPrefix, - TransactionNamingStrategy transactionNamingStrategy, + String transactionNamingStrategy, Integer parallelism, String valueFormat, String keyFormat, @@ -104,7 +104,7 @@ static Result translate(Map options) { } DeliveryGuarantee guarantee; - TransactionNamingStrategy naming; + String naming; try { guarantee = DeliveryGuarantee.valueOf( @@ -112,12 +112,7 @@ static Result translate(Map options) { .getOrDefault("sink.delivery-guarantee", "at-least-once") .replace('-', '_') .toUpperCase(Locale.ROOT)); - String namingOption = options.get("sink.transaction-naming-strategy"); - naming = - namingOption == null || "default".equalsIgnoreCase(namingOption) - ? TransactionNamingStrategy.DEFAULT - : TransactionNamingStrategy.valueOf( - namingOption.replace('-', '_').toUpperCase(Locale.ROOT)); + naming = KafkaSinkCompat.transactionNaming(options.get("sink.transaction-naming-strategy")); } catch (IllegalArgumentException invalid) { return Result.fallback("invalid Kafka sink delivery or transaction option"); } diff --git a/src/main/java/tech/streamfusion/planner/LookupJoinMatcher.java b/src/main/java/tech/streamfusion/planner/LookupJoinMatcher.java index c9dfce158..7c3d50a07 100644 --- a/src/main/java/tech/streamfusion/planner/LookupJoinMatcher.java +++ b/src/main/java/tech/streamfusion/planner/LookupJoinMatcher.java @@ -1,14 +1,13 @@ package tech.streamfusion.planner; -import java.util.HashMap; -import java.util.Map; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.JoinRelType; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalLookupJoin; import org.apache.flink.table.planner.plan.schema.TableSourceTable; -import org.apache.flink.table.planner.plan.utils.FunctionCallUtil; import org.apache.flink.types.RowKind; +import tech.streamfusion.compat.FlinkLookupCompat; +import tech.streamfusion.compat.LookupKeys; /** * Recognizes the processing-time lookup joins the native operator runs: {@code probe JOIN dim FOR @@ -32,7 +31,8 @@ static boolean matches(StreamPhysicalLookupJoin join) { } static String unsupportedReason(StreamPhysicalLookupJoin join) { - if (join.asyncOptions().isDefined() && join.asyncOptions().get().keyOrdered) { + var asyncOptions = FlinkLookupCompat.asyncOptions(join); + if (asyncOptions != null && asyncOptions.keyOrdered()) { return "lookup join: key-ordered asynchronous lookup requires Flink's keyed scheduling"; } if (!join.inputChangelogMode().containsOnly(RowKind.INSERT)) { @@ -47,22 +47,11 @@ static String unsupportedReason(StreamPhysicalLookupJoin join) { if (!(unwrapTable(join.temporalTable()) instanceof TableSourceTable)) { return "lookup join: temporal table is not a (non-legacy) table source"; } - for (FunctionCallUtil.FunctionParam param : lookupKeys(join).values()) { - if (!(param instanceof FunctionCallUtil.FieldRef) - && !(param instanceof FunctionCallUtil.Constant)) { - return "lookup join: unsupported lookup key shape " + param.getClass().getSimpleName(); - } - } - return null; + return FlinkLookupCompat.unsupportedKeyShape(lookupKeys(join)); } - /** The dimension key → probe field/constant map the generated fetcher builds its key row from. */ - static Map lookupKeys(StreamPhysicalLookupJoin join) { - Map keys = new HashMap<>(); - scala.collection.JavaConverters.mapAsJavaMapConverter(join.allLookupKeys()) - .asJava() - .forEach((index, param) -> keys.put((Integer) index, param)); - return keys; + static LookupKeys lookupKeys(StreamPhysicalLookupJoin join) { + return FlinkLookupCompat.lookupKeys(join); } static boolean isLeftOuterJoin(StreamPhysicalLookupJoin join) { @@ -89,12 +78,12 @@ static RelNode substitute(StreamPhysicalLookupJoin join, PlanContext ctx) { LookupJoinMatcher.temporalTable(join), LookupJoinMatcher.lookupKeys(join), join.calcOnTemporalTable().isDefined() ? join.calcOnTemporalTable().get() : null, - join.finalPreFilterCondition().isDefined() ? join.finalPreFilterCondition().get() : null, - join.finalRemainingCondition().isDefined() ? join.finalRemainingCondition().get() : null, + FlinkLookupCompat.preFilter(join), + FlinkLookupCompat.remainingCondition(join), LookupJoinMatcher.isLeftOuterJoin(join), - join.asyncOptions().isDefined() ? join.asyncOptions().get() : null, + FlinkLookupCompat.asyncOptions(join), join.retryOptions().isDefined() ? join.retryOptions().get() : null, - join.preferCustomShuffle(), + FlinkLookupCompat.preferCustomShuffle(join), join.inputChangelogMode()); } } diff --git a/src/main/java/tech/streamfusion/planner/NativeDeduplicateExecNode.java b/src/main/java/tech/streamfusion/planner/NativeDeduplicateExecNode.java index 4e794e325..9acd8f2fb 100644 --- a/src/main/java/tech/streamfusion/planner/NativeDeduplicateExecNode.java +++ b/src/main/java/tech/streamfusion/planner/NativeDeduplicateExecNode.java @@ -1,9 +1,5 @@ package tech.streamfusion.planner; -import tech.streamfusion.operator.ArrowBatch; -import tech.streamfusion.operator.ArrowBatchTypeInformation; -import tech.streamfusion.operator.NativeColumnarDeduplicateOperator; -import tech.streamfusion.operator.NativeColumnarKeepLastDeduplicateOperator; import java.util.Collections; import org.apache.flink.api.dag.Transformation; import org.apache.flink.configuration.ReadableConfig; @@ -19,6 +15,10 @@ import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecNode; import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil; import org.apache.flink.table.types.logical.RowType; +import tech.streamfusion.operator.ArrowBatch; +import tech.streamfusion.operator.ArrowBatchTypeInformation; +import tech.streamfusion.operator.NativeColumnarDeduplicateOperator; +import tech.streamfusion.operator.NativeColumnarKeepLastDeduplicateOperator; /** * Wraps the native columnar keep-first deduplicator into the plan; it consumes and produces Arrow @@ -75,7 +75,11 @@ protected Transformation translateToPlanInternal( // row once a watermark completes it). Under mini-batch Flink always plans a rowtime dedup as // its bundled retracting function — keep-first merely flips the comparator — so that shape // routes to the eager operator too. - boolean eager = proctime || keepLast || miniBatchEnabled; + boolean eager = + proctime + || keepLast + || miniBatchEnabled + || tech.streamfusion.compat.FlinkCompat.EAGER_ROWTIME_KEEP_FIRST; int maxParallelism = FlinkKeyGroupUtils.maxParallelism(planner.getExecEnv(), input.getParallelism()); // Proctime keep-first stays eager (its bundled function emits the same insert-only rows); diff --git a/src/main/java/tech/streamfusion/planner/NativeFileSinkExecNode.java b/src/main/java/tech/streamfusion/planner/NativeFileSinkExecNode.java index e50f74037..c923aea3d 100644 --- a/src/main/java/tech/streamfusion/planner/NativeFileSinkExecNode.java +++ b/src/main/java/tech/streamfusion/planner/NativeFileSinkExecNode.java @@ -2,7 +2,6 @@ import java.util.Collections; import java.util.Optional; -import java.util.UUID; import org.apache.flink.api.dag.Transformation; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.ReadableConfig; @@ -14,8 +13,6 @@ import org.apache.flink.core.fs.Path; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; -import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig; -import org.apache.flink.streaming.api.functions.sink.filesystem.legacy.StreamingFileSink; import org.apache.flink.streaming.api.operators.SimpleOperatorFactory; import org.apache.flink.streaming.api.transformations.OneInputTransformation; import org.apache.flink.table.planner.delegation.PlannerBase; @@ -29,10 +26,8 @@ import tech.streamfusion.operator.ArrowBatch; import tech.streamfusion.operator.FilePartitionSplitOperator; import tech.streamfusion.operator.NativeFileBulkWriterFactory; -import tech.streamfusion.operator.NativeFileRollingPolicy; import tech.streamfusion.operator.PartitionedArrowBatch; import tech.streamfusion.operator.PartitionedArrowBatchTypeInformation; -import tech.streamfusion.operator.PartitionedBatchBucketAssigner; /** * Builds the native columnar sink's operator chain, mirroring the host's own streaming filesystem @@ -98,35 +93,15 @@ protected Transformation translateToPlanInternal( planned.encoderValues, planned.changelog); Path location = new Path(planned.path); - StreamingFileSink.DefaultBulkFormatBuilder buckets = - StreamingFileSink.forBulkFormat(location, writerFactory) - .withBucketAssigner(new PartitionedBatchBucketAssigner()) - .withRollingPolicy( - new NativeFileRollingPolicy( - options - .get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_FILE_SIZE) - .getBytes(), - options - .get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_ROLLOVER_INTERVAL) - .toMillis(), - options - .get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_INACTIVITY_INTERVAL) - .toMillis())) - // The host's exact naming: a fresh UUID per sink keeps restarted or parallel writers - // from colliding on part-file names within a bucket. - .withOutputFileConfig( - OutputFileConfig.builder().withPartPrefix("part-" + UUID.randomUUID()).build()); - DataStream stream = new DataStream<>(planner.getExecEnv(), split); DataStream writer = - StreamingSink.writer( - name -> Optional.empty(), + tech.streamfusion.compat.FileSinkCompat.writer( + options, + location, + writerFactory, stream, - options.get(FileSystemConnectorOptions.SINK_ROLLING_POLICY_CHECK_INTERVAL).toMillis(), - buckets, parallelism, planned.partitionKeys, - options, parallelismConfigured); DataStreamSink end = StreamingSink.sink( diff --git a/src/main/java/tech/streamfusion/planner/NativeKafkaSinkExecNode.java b/src/main/java/tech/streamfusion/planner/NativeKafkaSinkExecNode.java index 57b249b5f..c63c5307b 100644 --- a/src/main/java/tech/streamfusion/planner/NativeKafkaSinkExecNode.java +++ b/src/main/java/tech/streamfusion/planner/NativeKafkaSinkExecNode.java @@ -1,10 +1,5 @@ package tech.streamfusion.planner; -import tech.streamfusion.format.LogicalTypeDescriptors; -import tech.streamfusion.kafka.NativeKafkaSerializationOperator; -import tech.streamfusion.kafka.PreSerializedKafkaRecord; -import tech.streamfusion.kafka.PreSerializedKafkaRecordSchema; -import tech.streamfusion.operator.ArrowBatch; import java.util.Collections; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.dag.Transformation; @@ -23,6 +18,11 @@ import org.apache.flink.table.planner.plan.nodes.exec.SingleTransformationTranslator; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecNode; import org.apache.flink.table.types.logical.RowType; +import tech.streamfusion.format.LogicalTypeDescriptors; +import tech.streamfusion.kafka.NativeKafkaSerializationOperator; +import tech.streamfusion.kafka.PreSerializedKafkaRecord; +import tech.streamfusion.kafka.PreSerializedKafkaRecordSchema; +import tech.streamfusion.operator.ArrowBatch; /** Builds native batch serialization followed by Flink's unmodified Kafka sink. */ public final class NativeKafkaSinkExecNode extends ExecNodeBase @@ -75,8 +75,9 @@ protected Transformation translateToPlanInternal( KafkaSink.builder() .setKafkaProducerConfig(planned.sink.producerProperties) .setRecordSerializer(new PreSerializedKafkaRecordSchema(planned.sink.topic)) - .setDeliveryGuarantee(planned.sink.deliveryGuarantee) - .setTransactionNamingStrategy(planned.sink.transactionNamingStrategy); + .setDeliveryGuarantee(planned.sink.deliveryGuarantee); + tech.streamfusion.kafka.compat.KafkaSinkCompat.setTransactionNaming( + builder, planned.sink.transactionNamingStrategy); if (planned.sink.transactionalIdPrefix != null) { builder.setTransactionalIdPrefix(planned.sink.transactionalIdPrefix); } diff --git a/src/main/java/tech/streamfusion/planner/NativeLookupJoinExecNode.java b/src/main/java/tech/streamfusion/planner/NativeLookupJoinExecNode.java index abf8e58b2..d7768ef74 100644 --- a/src/main/java/tech/streamfusion/planner/NativeLookupJoinExecNode.java +++ b/src/main/java/tech/streamfusion/planner/NativeLookupJoinExecNode.java @@ -1,9 +1,7 @@ package tech.streamfusion.planner; -import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; import javax.annotation.Nullable; import org.apache.calcite.plan.RelOptTable; @@ -13,7 +11,6 @@ import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.api.dag.Transformation; import org.apache.flink.configuration.ReadableConfig; -import org.apache.flink.streaming.api.functions.async.AsyncFunction; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.table.catalog.DataTypeFactory; import org.apache.flink.table.connector.ChangelogMode; @@ -25,8 +22,6 @@ import org.apache.flink.table.functions.UserDefinedFunction; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.codegen.CodeGeneratorContext; -import org.apache.flink.table.planner.codegen.FilterCodeGenerator; -import org.apache.flink.table.planner.codegen.FunctionCallCodeGenerator; import org.apache.flink.table.planner.codegen.LookupJoinCodeGenerator; import org.apache.flink.table.planner.delegation.PlannerBase; import org.apache.flink.table.planner.plan.nodes.exec.ExecNodeBase; @@ -36,25 +31,24 @@ import org.apache.flink.table.planner.plan.nodes.exec.SingleTransformationTranslator; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecNode; import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil; -import org.apache.flink.table.planner.plan.utils.FunctionCallUtil; import org.apache.flink.table.planner.plan.utils.LookupJoinUtil; import org.apache.flink.table.planner.utils.JavaScalaConversionUtil; import org.apache.flink.table.planner.utils.ShortcutUtils; import org.apache.flink.table.runtime.collector.ListenableCollector; import org.apache.flink.table.runtime.collector.TableFunctionResultFuture; import org.apache.flink.table.runtime.generated.GeneratedCollector; -import org.apache.flink.table.runtime.generated.GeneratedFilterCondition; import org.apache.flink.table.runtime.generated.GeneratedFunction; import org.apache.flink.table.runtime.generated.GeneratedResultFuture; import org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinRunner; -import org.apache.flink.table.runtime.operators.join.lookup.AsyncLookupJoinWithCalcRunner; import org.apache.flink.table.runtime.operators.join.lookup.LookupJoinRunner; -import org.apache.flink.table.runtime.operators.join.lookup.LookupJoinWithCalcRunner; import org.apache.flink.table.runtime.operators.join.lookup.ResultRetryStrategy; import org.apache.flink.table.runtime.typeutils.InternalSerializers; import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.types.RowKind; +import tech.streamfusion.compat.FlinkLookupCompat; +import tech.streamfusion.compat.LookupAsyncOptions; +import tech.streamfusion.compat.LookupKeys; import tech.streamfusion.operator.ArrowBatch; import tech.streamfusion.operator.ArrowBatchTypeInformation; import tech.streamfusion.operator.ArrowToRowDataOperator; @@ -81,13 +75,13 @@ public class NativeLookupJoinExecNode extends ExecNodeBase private final RelOptTable temporalTable; private final RowType probeType; - private final Map lookupKeys; + private final LookupKeys lookupKeys; private final @Nullable List projectionOnTemporalTable; private final @Nullable RexNode filterOnTemporalTable; private final @Nullable RexNode preFilterCondition; private final @Nullable RexNode remainingJoinCondition; private final boolean leftOuterJoin; - private final @Nullable FunctionCallUtil.AsyncOptions asyncOptions; + private final @Nullable LookupAsyncOptions asyncOptions; private final @Nullable LookupJoinUtil.RetryLookupOptions retryOptions; private final boolean preferCustomShuffle; private final ChangelogMode inputChangelogMode; @@ -99,13 +93,13 @@ public NativeLookupJoinExecNode( String description, RelOptTable temporalTable, RowType probeType, - Map lookupKeys, + LookupKeys lookupKeys, @Nullable List projectionOnTemporalTable, @Nullable RexNode filterOnTemporalTable, @Nullable RexNode preFilterCondition, @Nullable RexNode remainingJoinCondition, boolean leftOuterJoin, - @Nullable FunctionCallUtil.AsyncOptions asyncOptions, + @Nullable LookupAsyncOptions asyncOptions, @Nullable LookupJoinUtil.RetryLookupOptions retryOptions, boolean preferCustomShuffle, ChangelogMode inputChangelogMode) { @@ -144,21 +138,12 @@ protected Transformation translateToPlanInternal( RowType resultRowType = (RowType) getOutputType(); String tableName = String.join(".", temporalTable.getQualifiedName()); - List orderedKeys = new ArrayList<>(lookupKeys.size()); - for (int key : LookupJoinUtil.getOrderedLookupKeys(lookupKeys.keySet())) { - orderedKeys.add(lookupKeys.get(key)); - } boolean async = asyncOptions != null; ResultRetryStrategy retryStrategy = retryOptions == null ? ResultRetryStrategy.NO_RETRY_STRATEGY : retryOptions.toRetryStrategy(); UserDefinedFunction lookupFunction = - LookupJoinUtil.getLookupFunction( - temporalTable, - lookupKeys.keySet(), - classLoader, - async, - retryStrategy, - preferCustomShuffle); + FlinkLookupCompat.lookupFunction( + temporalTable, lookupKeys, classLoader, async, retryStrategy, preferCustomShuffle); if (preferCustomShuffle) { // The connector owns this partitioning contract and receives the projected lookup-key @@ -173,7 +158,7 @@ protected Transformation translateToPlanInternal( input.getParallelism(), false); rows = - LookupJoinUtil.tryApplyCustomShufflePartitioner( + FlinkLookupCompat.customShuffle( planner, temporalTable, probeType, @@ -206,9 +191,6 @@ protected Transformation translateToPlanInternal( projectionOutputRelDataType != null ? (RowType) FlinkTypeFactory.toLogicalType(projectionOutputRelDataType) : tableSourceRowType; - GeneratedFilterCondition generatedPreFilter = - FilterCodeGenerator.generateFilterCondition( - config, classLoader, preFilterCondition, probeType); GeneratedFunction> generatedCalc = projectionOnTemporalTable != null ? LookupJoinCodeGenerator.generateCalcMapFunction( @@ -222,18 +204,17 @@ protected Transformation translateToPlanInternal( OneInputStreamOperator operator; if (async) { - FunctionCallCodeGenerator.GeneratedTableFunctionWithDataType> - generatedFetcher = - LookupJoinCodeGenerator.generateAsyncLookupFunction( - config, - classLoader, - dataTypeFactory, - probeType, - tableSourceRowType, - resultRowType, - orderedKeys, - (AsyncTableFunction) lookupFunction, - tableName); + var generatedFetcher = + FlinkLookupCompat.asyncFetcher( + config, + classLoader, + dataTypeFactory, + probeType, + tableSourceRowType, + resultRowType, + lookupKeys, + (AsyncTableFunction) lookupFunction, + tableName); GeneratedResultFuture> generatedResultFuture = LookupJoinCodeGenerator.generateTableAsyncCollector( config, @@ -247,41 +228,35 @@ protected Transformation translateToPlanInternal( (DataStructureConverter) DataStructureConverters.getConverter(generatedFetcher.dataType()); AsyncLookupJoinRunner runner = - generatedCalc != null - ? new AsyncLookupJoinWithCalcRunner( - generatedFetcher.tableFunc(), - fetcherConverter, - generatedCalc, - generatedResultFuture, - generatedPreFilter, - InternalSerializers.create(rightRowType), - leftOuterJoin, - asyncOptions.asyncBufferCapacity) - : new AsyncLookupJoinRunner( - generatedFetcher.tableFunc(), - fetcherConverter, - generatedResultFuture, - generatedPreFilter, - InternalSerializers.create(rightRowType), - leftOuterJoin, - asyncOptions.asyncBufferCapacity); + FlinkLookupCompat.asyncRunner( + generatedFetcher.tableFunc(), + fetcherConverter, + generatedCalc, + generatedResultFuture, + config, + classLoader, + preFilterCondition, + probeType, + InternalSerializers.create(rightRowType), + leftOuterJoin, + asyncOptions.asyncBufferCapacity()); operator = new NativeAsyncLookupJoinOperator( runner, probeType, resultRowType, - asyncOptions.asyncBufferCapacity, - asyncOptions.asyncTimeout); + asyncOptions.asyncBufferCapacity(), + asyncOptions.asyncTimeout()); } else { GeneratedFunction> generatedFetcher = - LookupJoinCodeGenerator.generateSyncLookupFunction( + FlinkLookupCompat.syncFetcher( config, classLoader, dataTypeFactory, probeType, tableSourceRowType, resultRowType, - orderedKeys, + lookupKeys, (TableFunction) lookupFunction, tableName, planner.getExecEnv().getConfig().isObjectReuseEnabled()); @@ -295,20 +270,16 @@ protected Transformation translateToPlanInternal( JavaScalaConversionUtil.toScala(Optional.empty()), true); LookupJoinRunner runner = - generatedCalc != null - ? new LookupJoinWithCalcRunner( - generatedFetcher, - generatedCalc, - generatedCollector, - generatedPreFilter, - leftOuterJoin, - rightRowType.getFieldCount()) - : new LookupJoinRunner( - generatedFetcher, - generatedCollector, - generatedPreFilter, - leftOuterJoin, - rightRowType.getFieldCount()); + FlinkLookupCompat.syncRunner( + generatedFetcher, + generatedCalc, + generatedCollector, + config, + classLoader, + preFilterCondition, + probeType, + leftOuterJoin, + rightRowType.getFieldCount()); operator = new NativeLookupJoinOperator(runner, probeType, resultRowType); } diff --git a/src/main/java/tech/streamfusion/planner/NativePaimonSourceExecNode.java b/src/main/java/tech/streamfusion/planner/NativePaimonSourceExecNode.java index 5f2363460..d9b7fdb2d 100644 --- a/src/main/java/tech/streamfusion/planner/NativePaimonSourceExecNode.java +++ b/src/main/java/tech/streamfusion/planner/NativePaimonSourceExecNode.java @@ -74,10 +74,12 @@ protected Transformation translateToPlanInternal( boolean infer = Boolean.parseBoolean( env.getConfiguration() - .toMap() - .getOrDefault( - "paimon.scan.infer-parallelism", - String.valueOf(options.get(FlinkConnectorOptions.INFER_SCAN_PARALLELISM)))); + .getOptional( + org.apache.flink.configuration.ConfigOptions.key( + "paimon.scan.infer-parallelism") + .stringType() + .noDefaultValue()) + .orElse(String.valueOf(options.get(FlinkConnectorOptions.INFER_SCAN_PARALLELISM)))); if (parallelism == null && env.getParallelism() == -1 && infer diff --git a/src/main/java/tech/streamfusion/planner/NativePlanner.java b/src/main/java/tech/streamfusion/planner/NativePlanner.java index 93ff861a5..dc4694536 100644 --- a/src/main/java/tech/streamfusion/planner/NativePlanner.java +++ b/src/main/java/tech/streamfusion/planner/NativePlanner.java @@ -71,7 +71,13 @@ public static PhysicalPlanScan install(TableConfig config) { * first optimizes. */ public static PhysicalPlanScan install(TableEnvironment tableEnv) { - return install(tableEnv.getConfig()); + PhysicalPlanScan scan = install(tableEnv.getConfig()); + if (tableEnv instanceof org.apache.flink.table.api.internal.TableEnvironmentImpl internal + && internal.getPlanner() + instanceof org.apache.flink.table.planner.delegation.PlannerBase planner) { + scan.executionEnvironment(planner.getExecEnv()); + } + return scan; } /** diff --git a/src/main/java/tech/streamfusion/planner/NativeStreamPlanner.java b/src/main/java/tech/streamfusion/planner/NativeStreamPlanner.java index c875d35cc..2a2329184 100644 --- a/src/main/java/tech/streamfusion/planner/NativeStreamPlanner.java +++ b/src/main/java/tech/streamfusion/planner/NativeStreamPlanner.java @@ -24,6 +24,7 @@ final class NativeStreamPlanner extends StreamPlanner { context.getCatalogManager(), context.getClassLoader()); scan = NativePlanner.install(context.getTableConfig()); + scan.executionEnvironment(getExecEnv()); scan.deferToCompletePlan(); } diff --git a/src/main/java/tech/streamfusion/planner/PaimonSinkMatcher.java b/src/main/java/tech/streamfusion/planner/PaimonSinkMatcher.java index 719c1a539..aefcf20f2 100644 --- a/src/main/java/tech/streamfusion/planner/PaimonSinkMatcher.java +++ b/src/main/java/tech/streamfusion/planner/PaimonSinkMatcher.java @@ -78,7 +78,7 @@ static boolean appliesTo(StreamPhysicalSink sink) { } static Planned plan(StreamPhysicalSink sink) { - for (SinkAbilitySpec ability : sink.abilitySpecs()) { + for (SinkAbilitySpec ability : tech.streamfusion.compat.FlinkCompat.sinkAbilities(sink)) { if (ability instanceof OverwriteSpec) { return Planned.fallback("INSERT OVERWRITE is not supported"); } @@ -252,6 +252,7 @@ private static FileStoreTable resolveTable(StreamPhysicalSink sink) { ResolvedCatalogBaseTable resolvedTable = resolved.getResolvedTable(); CatalogBaseTable origin = resolvedTable.getOrigin(); Map options = new HashMap<>(resolvedTable.getOptions()); + options.putAll(org.apache.flink.table.planner.hint.FlinkHints.getHintedOptions(sink.hints())); options.putAll(PaimonDynamicOptions.forTable(sink, resolved.getIdentifier())); FileStoreTable table; if (origin instanceof DataCatalogTable) { diff --git a/src/main/java/tech/streamfusion/planner/ParquetSinkTranslator.java b/src/main/java/tech/streamfusion/planner/ParquetSinkTranslator.java index a51d015bb..4684ab914 100644 --- a/src/main/java/tech/streamfusion/planner/ParquetSinkTranslator.java +++ b/src/main/java/tech/streamfusion/planner/ParquetSinkTranslator.java @@ -228,6 +228,7 @@ private static boolean containsTimestamp(LogicalType type) { private static Result encoderConfig( Map options, HadoopConfigLookup hadoopConfig) { Map config = new LinkedHashMap<>(); + config.put("schema.shape", tech.streamfusion.compat.FileSinkCompat.parquetSchemaShape()); for (String booleanOption : List.of("parquet.write.int64.timestamp", "parquet.utc-timezone")) { diff --git a/src/main/java/tech/streamfusion/planner/PhysicalPlanScan.java b/src/main/java/tech/streamfusion/planner/PhysicalPlanScan.java index 8d569fb4c..68cd3eaae 100644 --- a/src/main/java/tech/streamfusion/planner/PhysicalPlanScan.java +++ b/src/main/java/tech/streamfusion/planner/PhysicalPlanScan.java @@ -16,7 +16,6 @@ import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalCalc; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalChangelogNormalize; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalCorrelate; -import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalDeltaJoin; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalExpand; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGlobalGroupAggregate; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGlobalWindowAggregate; @@ -48,6 +47,7 @@ import org.apache.flink.table.planner.utils.ShortcutUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import tech.streamfusion.compat.FlinkCompat; /** * Rewrites the host engine's optimized physical plan, replacing supported operators with native @@ -70,6 +70,18 @@ public final class PhysicalPlanScan implements FlinkOptimizeProgram fallbackReasons = new ArrayList<>(); private int substitutions; private boolean completePlan; + private org.apache.flink.streaming.api.environment.StreamExecutionEnvironment + executionEnvironment; + private String keyedStateUnsupportedReason; + + void executionEnvironment( + org.apache.flink.streaming.api.environment.StreamExecutionEnvironment environment) { + executionEnvironment = environment; + } + + String keyedStateUnsupportedReason() { + return keyedStateUnsupportedReason; + } private static final List EXTENSIONS = loadExtensions(); private static final List> REGISTRY = buildRegistry(); @@ -104,6 +116,9 @@ private List optimizeConfigured(List roots, boolean finalOutpu operatorTypes.clear(); fallbackReasons.clear(); substitutions = 0; + keyedStateUnsupportedReason = + tech.streamfusion.compat.FlinkStateBackendCompat.unsupportedNativeStateReason( + executionEnvironment, ShortcutUtils.unwrapTableConfig(roots.get(0))); roots.forEach(this::record); // Master switch: with native acceleration off, substitute nothing — the query runs on the host. if (!NativeConfig.nativeEnabled()) { @@ -115,7 +130,8 @@ private List optimizeConfigured(List roots, boolean finalOutpu "delta join: table.optimizer.delta-join.strategy is FORCE but this optimizer block" + " contains a regular join and no delta join"); LOG.info( - "StreamFusion declined the optimizer block so Flink can enforce its FORCE delta-join strategy"); + "StreamFusion declined the optimizer block so Flink can enforce its FORCE delta-join" + + " strategy"); return roots; } Set repeatedSources = @@ -208,19 +224,23 @@ private static List> buildRegistry() { StreamPhysicalGroupAggregate.class, "groupAggregate", GroupAggregateMatcher::substitute) + .keyedState() .matching(GroupAggregateMatcher::matches) .reason(GroupAggregateMatcher::unsupportedReason) .changelogSafe()); - // The global half of a two-phase non-windowed GROUP BY. It merges the local half's partials into + // The global half of a two-phase non-windowed GROUP BY. It merges the local half's partials + // into // the final per-key result and emits a changelog exactly like the single-phase GROUP BY above — // so it reuses the same native group-aggregate operator, fed positional partial columns (COUNT - // merges as a SUM over its partial counts). Exempt from the insert-only guard for the same reason. + // merges as a SUM over its partial counts). Exempt from the insert-only guard for the same + // reason. entries.add( Substitution.of( StreamPhysicalGlobalGroupAggregate.class, "groupAggregate", GlobalGroupAggregateMatcher::substitute) + .keyedState() .matching(GlobalGroupAggregateMatcher::matches) .reason(GlobalGroupAggregateMatcher::unsupportedReason) .changelogSafe()); @@ -239,8 +259,8 @@ private static List> buildRegistry() { // A regular (non-windowed) join emits a changelog and consumes one on either side, so it is // exempt from the insert-only guard (like the GROUP BY above). entries.add( - Substitution.of( - StreamPhysicalJoin.class, "updatingJoin", RegularJoinMatcher::substitute) + Substitution.of(StreamPhysicalJoin.class, "updatingJoin", RegularJoinMatcher::substitute) + .keyedState() .matching(RegularJoinMatcher::matches) .reason(RegularJoinMatcher::unsupportedReason) .changelogSafe()); @@ -252,6 +272,7 @@ private static List> buildRegistry() { // StreamPhysicalRank, but a rowtime-ordered rank is deduplication, which TopNMatcher declines. entries.add( Substitution.of(StreamPhysicalRank.class, "deduplicate", DeduplicateMatcher::substitute) + .keyedState() .matching( rank -> DeduplicateMatcher.matches(rank) @@ -262,21 +283,25 @@ private static List> buildRegistry() { entries.add( Substitution.of(StreamPhysicalRank.class, "topN", FirstNMatcher::substitute) + .keyedState() .matching(FirstNMatcher::matches) .changelogSafe()); // A streaming Top-N emits a changelog (it deletes a row when one is displaced), so it is exempt // from the insert-only guard. An insert-only input uses the append-only ranker; a changelog - // input uses the retracting ranker (Flink's RetractableTopNFunction), which keeps the full buffer + // input uses the retracting ranker (Flink's RetractableTopNFunction), which keeps the full + // buffer // so a deleted top-N row can be replaced by promoting rank N+1. entries.add( Substitution.of(StreamPhysicalRank.class, "topN", TopNMatcher::substitute) + .keyedState() .matching(TopNMatcher::matches) .reason(TopNMatcher::unsupportedReason) .changelogSafe()); // A global FETCH/LIMIT — ORDER BY … LIMIT n (StreamPhysicalSortLimit) or plain LIMIT n - // (StreamPhysicalLimit). Both lower to a global (no-partition) ROW_NUMBER rank, so they reuse the + // (StreamPhysicalLimit). Both lower to a global (no-partition) ROW_NUMBER rank, so they reuse + // the // native columnar Top-N operator with an empty partition key: the sort-limit carries the order // keys and emits a changelog as the top set changes; the plain limit has no sort keys, so the // ranker keeps the first n rows by arrival. Updating inputs use Flink's selected replacement or @@ -284,9 +309,13 @@ private static List> buildRegistry() { // a sort-limit emits a changelog, so it would otherwise slip past the insert-only guard // unreported, leaving a non-accelerating query unable to explain itself (ticket 29). entries.add( - Substitution.of(StreamPhysicalSortLimit.class, LimitMatcher::substitute).changelogSafe()); + Substitution.of(StreamPhysicalSortLimit.class, LimitMatcher::substitute) + .keyedState() + .changelogSafe()); entries.add( - Substitution.of(StreamPhysicalLimit.class, LimitMatcher::substitute).changelogSafe()); + Substitution.of(StreamPhysicalLimit.class, LimitMatcher::substitute) + .keyedState() + .changelogSafe()); // A Calc transforms each row independently — a per-row projection plus an optional deterministic // filter — and the native operator carries the `$row_kind$` tag through unchanged, so it is @@ -303,8 +332,10 @@ private static List> buildRegistry() { .reason(CalcMatcher::unsupportedReason) .changelogSafe()); - // Changelog normalization (upsert / duplicate-bearing source → regular changelog): keep the last - // row per unique key, emitting INSERT/UPDATE_BEFORE/UPDATE_AFTER/DELETE. Both consumes and emits a + // Changelog normalization (upsert / duplicate-bearing source → regular changelog): keep the + // last + // row per unique key, emitting INSERT/UPDATE_BEFORE/UPDATE_AFTER/DELETE. Both consumes and + // emits a // changelog, so (like the GROUP BY) it is exempt from the insert-only guard. The keyed // shuffle (by the unique key) stays columnar where the input sits on a columnar producer. entries.add( @@ -312,6 +343,7 @@ private static List> buildRegistry() { StreamPhysicalChangelogNormalize.class, "changelogNormalize", ChangelogNormalizeMatcher::substitute) + .keyedState() .matching(ChangelogNormalizeMatcher::matches) .reason(ChangelogNormalizeMatcher::unsupportedReason) .changelogSafe()); @@ -365,14 +397,15 @@ private static List> buildRegistry() { || wm.getInputs().get(0).getInputs().isEmpty()) && WatermarkAssignerMatcher.matches(wm))); - // Event-time sort (ORDER BY rowtime): buffer rows, release them in rowtime order as the watermark - // advances. Insert-only. Its single (gather) exchange becomes a native columnar exchange with no + // Event-time sort (ORDER BY rowtime): buffer rows, release them in rowtime order as the + // watermark + // advances. Insert-only. Its single (gather) exchange becomes a native columnar exchange with + // no // key (an empty key list, like the non-partitioned OVER), so the whole thing stays columnar. entries.add( Substitution.of( - StreamPhysicalTemporalSort.class, - "temporalSort", - TemporalSortMatcher::substitute) + StreamPhysicalTemporalSort.class, "temporalSort", TemporalSortMatcher::substitute) + .keyedState() .matching(TemporalSortMatcher::matches) .reason(TemporalSortMatcher::unsupportedReason)); @@ -390,11 +423,12 @@ private static List> buildRegistry() { .reason(WindowTableFunctionMatcher::unsupportedReason)); // Window Top-N over a windowing-TVF input: per window and partition key, keep the top-N rows by - // the order key and emit them when a watermark closes the window. Append-only; the keyed shuffle + // the order key and emit them when a watermark closes the window. Append-only; the keyed + // shuffle // (or single gather when there is no partition key) stays columnar via columnarInput. entries.add( - Substitution.of( - StreamPhysicalWindowRank.class, "windowRank", WindowRankMatcher::substitute) + Substitution.of(StreamPhysicalWindowRank.class, "windowRank", WindowRankMatcher::substitute) + .keyedState() .matching(WindowRankMatcher::matches) .reason(WindowRankMatcher::unsupportedReason)); @@ -405,6 +439,7 @@ private static List> buildRegistry() { StreamPhysicalWindowDeduplicate.class, "windowRank", WindowDeduplicateMatcher::substitute) + .keyedState() .matching(WindowDeduplicateMatcher::matches) .reason(WindowDeduplicateMatcher::unsupportedReason)); @@ -413,6 +448,7 @@ private static List> buildRegistry() { StreamPhysicalWindowAggregate.class, "windowAggregate", WindowAggregateMatcher::substitute) + .keyedState() .matching( agg -> WindowAggregateMatcher.matches( @@ -430,6 +466,7 @@ private static List> buildRegistry() { StreamPhysicalWindowAggregate.class, "windowAggregate", WindowAggregateMatcher::substituteSession) + .keyedState() .matching( agg -> WindowAggregateMatcher.matchesSession( @@ -446,6 +483,7 @@ private static List> buildRegistry() { StreamPhysicalGroupWindowAggregate.class, "windowAggregate", GroupWindowAggregateMatcher::substitute) + .keyedState() .matching(GroupWindowAggregateMatcher::matches) .reason(GroupWindowAggregateMatcher::unsupportedReason)); @@ -466,6 +504,7 @@ private static List> buildRegistry() { StreamPhysicalLocalWindowAggregate.class, "localWindowAggregate", WindowAggregateMatcher::substituteLocal) + .keyedState() .matching(agg -> WindowAggregateMatcher.localWindowVariant(agg) != null) .reason(agg -> WindowAggregateMatcher.unsupportedReason(agg, agg.windowing())) .changelogSafe()); @@ -476,29 +515,28 @@ private static List> buildRegistry() { // planners route the same physical shape. entries.add( Substitution.of(StreamPhysicalOverAggregate.class, "over", OverAggregateMatcher::substitute) + .keyedState() .matching(OverAggregateMatcher::matches) .reason(OverAggregateMatcher::unsupportedReason) .changelogSafe()); entries.add( Substitution.of( - StreamPhysicalIntervalJoin.class, - "intervalJoin", - IntervalJoinMatcher::substitute) + StreamPhysicalIntervalJoin.class, "intervalJoin", IntervalJoinMatcher::substitute) + .keyedState() .matching(IntervalJoinMatcher::matches) .reason(IntervalJoinMatcher::unsupportedReason)); entries.add( - Substitution.of( - StreamPhysicalWindowJoin.class, "windowJoin", WindowJoinMatcher::substitute) + Substitution.of(StreamPhysicalWindowJoin.class, "windowJoin", WindowJoinMatcher::substitute) + .keyedState() .matching(WindowJoinMatcher::matches) .reason(WindowJoinMatcher::unsupportedReason)); entries.add( Substitution.of( - StreamPhysicalTemporalJoin.class, - "temporalJoin", - TemporalJoinMatcher::substitute) + StreamPhysicalTemporalJoin.class, "temporalJoin", TemporalJoinMatcher::substitute) + .keyedState() .matching(TemporalJoinMatcher::matches) .reason(TemporalJoinMatcher::unsupportedReason)); @@ -513,14 +551,17 @@ private static List> buildRegistry() { StreamPhysicalGlobalWindowAggregate.class, "globalWindowAggregate", GlobalWindowAggregateMatcher::substitute) + .keyedState() .matching(GlobalWindowAggregateMatcher::matches) .reason(GlobalWindowAggregateMatcher::unsupportedReason)); + FlinkPlannerCompat.addSubstitutions(entries); EXTENSIONS.forEach(extension -> extension.addSubstitutions(entries)); return List.copyOf(entries); } private RelNode rewrite(RelNode node, PlanContext ctx) { + node = FlinkPlannerCompat.prepareForRewrite(node); List inputs = new ArrayList<>(node.getInputs().size()); boolean changed = false; for (RelNode input : node.getInputs()) { @@ -766,9 +807,7 @@ private void record(RelNode node) { // Unlike Flink's statement-wide validation, this guard is conservative per optimizer block. static boolean deltaJoinForceWouldReject(RelNode root) { - if (ShortcutUtils.unwrapTableConfig(root) - .get(OptimizerConfigOptions.TABLE_OPTIMIZER_DELTA_JOIN_STRATEGY) - != OptimizerConfigOptions.DeltaJoinStrategy.FORCE) { + if (!FlinkCompat.forceDeltaJoin(root)) { return false; } class JoinFinder extends RelVisitor { @@ -780,7 +819,7 @@ public void visit(RelNode node, int ordinal, RelNode parent) { if (deltaJoinExists) { return; } - if (node instanceof StreamPhysicalDeltaJoin) { + if (FlinkCompat.isDeltaJoin(node)) { deltaJoinExists = true; return; } diff --git a/src/main/java/tech/streamfusion/planner/PlanContext.java b/src/main/java/tech/streamfusion/planner/PlanContext.java index 6fce120e0..c13c9144b 100644 --- a/src/main/java/tech/streamfusion/planner/PlanContext.java +++ b/src/main/java/tech/streamfusion/planner/PlanContext.java @@ -21,6 +21,10 @@ final class PlanContext { this.repeatedSources = repeatedSources; } + String keyedStateUnsupportedReason() { + return scan.keyedStateUnsupportedReason(); + } + /** Counts one host node replaced by a native one. */ void substituted() { scan.countSubstitution(); diff --git a/src/main/java/tech/streamfusion/planner/RegularJoinMatcher.java b/src/main/java/tech/streamfusion/planner/RegularJoinMatcher.java index 0745a8a19..233078c9b 100644 --- a/src/main/java/tech/streamfusion/planner/RegularJoinMatcher.java +++ b/src/main/java/tech/streamfusion/planner/RegularJoinMatcher.java @@ -1,21 +1,21 @@ package tech.streamfusion.planner; -import tech.streamfusion.operator.RowDataArrowConverter; import java.util.Map; import java.util.Optional; -import org.apache.calcite.rel.core.Exchange; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Exchange; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.flink.table.planner.calcite.FlinkTypeFactory$; -import org.apache.flink.table.planner.hint.StateTtlHint; import org.apache.flink.table.planner.plan.nodes.exec.spec.JoinSpec; import org.apache.flink.table.planner.plan.nodes.physical.common.CommonPhysicalJoin; -import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalJoin; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGroupAggregateBase; +import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalJoin; import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalRel; import org.apache.flink.table.planner.plan.utils.ChangelogPlanUtils; import org.apache.flink.table.runtime.operators.join.FlinkJoinType; +import tech.streamfusion.compat.FlinkCompat; +import tech.streamfusion.operator.RowDataArrowConverter; /** * Recognizes the regular (non-windowed) equi-joins the native updating join implements: @@ -189,7 +189,7 @@ static RelNode substitute(StreamPhysicalJoin join, PlanContext ctx) { // A STATE_TTL hint sets each side's retention independently (0 = left, 1 = right — // Flink's FlinkHints.LEFT_INPUT convention), overriding the job-wide retention for that // side alone; -1 means no hint, resolved at translate time. - Map hintTtls = StateTtlHint.getStateTtlFromHintOnBiRel(join.getHints()); + Map hintTtls = FlinkCompat.joinStateTtl(join.getHints()); // Columnar (Arrow in/out); keep each side's keyed shuffle columnar where it sits on a // columnar producer, else the transition pass transposes at the boundary. return new StreamPhysicalNativeColumnarUpdatingJoin( diff --git a/src/main/java/tech/streamfusion/planner/RexExpression.java b/src/main/java/tech/streamfusion/planner/RexExpression.java index 52855abe0..616b9a0dc 100644 --- a/src/main/java/tech/streamfusion/planner/RexExpression.java +++ b/src/main/java/tech/streamfusion/planner/RexExpression.java @@ -161,7 +161,7 @@ final class RexExpression { // A bare predicate without a relational context conservatively declines host-exact numeric casts. private Boolean legacyCastBehaviour; private boolean watermarkAvailable; - private org.apache.flink.configuration.Configuration temporalConfig = + private org.apache.flink.configuration.ReadableConfig temporalConfig = new org.apache.flink.configuration.Configuration(); // Root of the projection currently being encoded; null for conditions and bare predicates. private RexNode projectionRoot; @@ -354,9 +354,11 @@ static boolean isRowCalc(int[] kinds) { private void configure(org.apache.flink.table.api.TableConfig tableConfig) { sessionZoneId = tableConfig.getLocalTimeZone().getId(); - temporalConfig = - org.apache.flink.configuration.Configuration.fromMap(tableConfig.toMap()); - temporalConfig.setString("table.local-time-zone", sessionZoneId); + var expressionConfig = org.apache.flink.table.api.TableConfig.getDefault(); + expressionConfig.setRootConfiguration(tableConfig.getRootConfiguration()); + expressionConfig.addConfiguration(tableConfig.getConfiguration()); + expressionConfig.setLocalTimeZone(tableConfig.getLocalTimeZone()); + temporalConfig = expressionConfig; legacyCastBehaviour = tableConfig .get( @@ -1031,6 +1033,13 @@ && hasImplicitStringNumericCast(call)) { if ("JSON_UNQUOTE".equals(functionName)) { return emitCharacterFunction(call, 123, 1, 1); } + if (("JSON_STRING".equals(functionName) || "JSON_OBJECT".equals(functionName)) + && !tech.streamfusion.compat.JsonRuntimeCompat.PRESERVES_DECIMAL_SCALE + && call.getOperands().stream() + .anyMatch(operand -> operand.getType().getSqlTypeName() == SqlTypeName.DECIMAL)) { + return reject( + "JSON decimal rendering requires the selected Flink line's generated evaluator"); + } if ("JSON_STRING".equals(functionName)) { if (call.getOperands().size() != 1 || !isJsonScalarValue(call.getOperands().get(0))) { return reject("JSON_STRING requires a character, boolean, integer, or decimal scalar"); diff --git a/src/main/java/tech/streamfusion/planner/ScanWatermarkSpec.java b/src/main/java/tech/streamfusion/planner/ScanWatermarkSpec.java index 6a385a1de..56c054f9a 100644 --- a/src/main/java/tech/streamfusion/planner/ScanWatermarkSpec.java +++ b/src/main/java/tech/streamfusion/planner/ScanWatermarkSpec.java @@ -81,8 +81,11 @@ static ScanWatermarkSpec of(StreamPhysicalTableSourceScan scan) { // computed rowtime); it must be one of the supported terms and agree with the watermark // expression's column. Integer rowtimeFromExpr = null; - if (pushed.getRowtimeExpr().isPresent()) { - Integer index = rowtimeTerm(stripReinterpret(pushed.getRowtimeExpr().get())); + if (tech.streamfusion.compat.FlinkCompat.watermarkRowtime(pushed).isPresent()) { + Integer index = + rowtimeTerm( + stripReinterpret( + tech.streamfusion.compat.FlinkCompat.watermarkRowtime(pushed).get())); if (index == null) { return UNSUPPORTED; } diff --git a/src/main/java/tech/streamfusion/planner/StreamPhysicalNativeLookupJoin.java b/src/main/java/tech/streamfusion/planner/StreamPhysicalNativeLookupJoin.java index 128f69380..d7c10a6dc 100644 --- a/src/main/java/tech/streamfusion/planner/StreamPhysicalNativeLookupJoin.java +++ b/src/main/java/tech/streamfusion/planner/StreamPhysicalNativeLookupJoin.java @@ -1,7 +1,6 @@ package tech.streamfusion.planner; import java.util.List; -import java.util.Map; import javax.annotation.Nullable; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptTable; @@ -10,14 +9,15 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgram; +import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.planner.calcite.FlinkTypeFactory$; import org.apache.flink.table.planner.plan.nodes.exec.ExecNode; import org.apache.flink.table.planner.plan.nodes.exec.InputProperty; -import org.apache.flink.table.planner.plan.utils.FlinkRexUtil; -import org.apache.flink.table.planner.plan.utils.FunctionCallUtil; import org.apache.flink.table.planner.plan.utils.LookupJoinUtil; import org.apache.flink.table.planner.utils.ShortcutUtils; -import org.apache.flink.table.connector.ChangelogMode; +import tech.streamfusion.compat.FlinkLookupCompat; +import tech.streamfusion.compat.LookupAsyncOptions; +import tech.streamfusion.compat.LookupKeys; /** * Physical node standing in for a processing-time lookup join the native operator runs. Columnar on @@ -31,12 +31,12 @@ public class StreamPhysicalNativeLookupJoin extends StreamPhysicalNativeSingleRe implements ColumnarInput, ColumnarOutput { private final RelOptTable temporalTable; - private final Map lookupKeys; + private final LookupKeys lookupKeys; private final @Nullable RexProgram calcOnTemporalTable; private final @Nullable RexNode preFilterCondition; private final @Nullable RexNode remainingJoinCondition; private final boolean leftOuterJoin; - private final @Nullable FunctionCallUtil.AsyncOptions asyncOptions; + private final @Nullable LookupAsyncOptions asyncOptions; private final @Nullable LookupJoinUtil.RetryLookupOptions retryOptions; private final boolean preferCustomShuffle; private final ChangelogMode inputChangelogMode; @@ -47,12 +47,12 @@ public StreamPhysicalNativeLookupJoin( RelNode input, RelDataType outputRowType, RelOptTable temporalTable, - Map lookupKeys, + LookupKeys lookupKeys, @Nullable RexProgram calcOnTemporalTable, @Nullable RexNode preFilterCondition, @Nullable RexNode remainingJoinCondition, boolean leftOuterJoin, - @Nullable FunctionCallUtil.AsyncOptions asyncOptions, + @Nullable LookupAsyncOptions asyncOptions, @Nullable LookupJoinUtil.RetryLookupOptions retryOptions, boolean preferCustomShuffle, ChangelogMode inputChangelogMode) { @@ -100,10 +100,9 @@ public ExecNode translateToExecNode() { List projectionOnTemporalTable = null; RexNode filterOnTemporalTable = null; if (calcOnTemporalTable != null) { - scala.Tuple2, scala.Option> expanded = - FlinkRexUtil.expandRexProgram(calcOnTemporalTable); - projectionOnTemporalTable = expanded._1(); - filterOnTemporalTable = expanded._2().isDefined() ? expanded._2().get() : null; + var expanded = FlinkLookupCompat.expandCalc(calcOnTemporalTable); + projectionOnTemporalTable = expanded.projection(); + filterOnTemporalTable = expanded.filter(); } return new NativeLookupJoinExecNode( ShortcutUtils.unwrapTableConfig(this), diff --git a/src/main/java/tech/streamfusion/planner/Substitution.java b/src/main/java/tech/streamfusion/planner/Substitution.java index 594a37232..494a88dcd 100644 --- a/src/main/java/tech/streamfusion/planner/Substitution.java +++ b/src/main/java/tech/streamfusion/planner/Substitution.java @@ -37,6 +37,7 @@ enum OnDecline { private Predicate owns = node -> true; private Function reason; private boolean changelogSafe; + private boolean keyedState; private OnDecline onDecline = OnDecline.STOP; private Substitution(Class shape, String operatorKey, Planner planner) { @@ -71,6 +72,11 @@ Substitution changelogSafe() { return this; } + Substitution keyedState() { + this.keyedState = true; + return this; + } + /** Lets later entries see a node this one declined, for a shape two entries share unevenly. */ Substitution yieldingOnDecline() { this.onDecline = OnDecline.YIELD; @@ -113,6 +119,10 @@ RelNode apply(RelNode node, PlanContext ctx) { ctx.decline(disabledReason(operatorKey)); return node; } + if (keyedState && ctx.keyedStateUnsupportedReason() != null) { + ctx.decline(ctx.keyedStateUnsupportedReason()); + return node; + } RelNode planned = planner.plan(typed, ctx); if (planned != null) { ctx.substituted(); diff --git a/src/main/java/tech/streamfusion/planner/WindowAggregateMatcher.java b/src/main/java/tech/streamfusion/planner/WindowAggregateMatcher.java index 7fc65c837..d6542f45b 100644 --- a/src/main/java/tech/streamfusion/planner/WindowAggregateMatcher.java +++ b/src/main/java/tech/streamfusion/planner/WindowAggregateMatcher.java @@ -14,7 +14,6 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.flink.table.planner.plan.logical.CumulativeWindowSpec; import org.apache.flink.table.planner.plan.logical.HoppingWindowSpec; -import org.apache.flink.table.planner.plan.logical.SessionWindowSpec; import org.apache.flink.table.planner.plan.logical.TimeAttributeWindowingStrategy; import org.apache.flink.table.planner.plan.logical.TumblingWindowSpec; import org.apache.flink.table.planner.plan.logical.WindowAttachedWindowingStrategy; @@ -25,6 +24,7 @@ import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWindowAggregate; import org.apache.flink.table.planner.plan.utils.ChangelogPlanUtils; import org.apache.flink.table.types.logical.LogicalTypeRoot; +import tech.streamfusion.compat.FlinkCompat; /** Shared admission and encoding for single-phase and local window aggregation. */ final class WindowAggregateMatcher { @@ -235,7 +235,7 @@ static boolean matchesSession( RelDataType inputType) { if (hasFilters(aggCalls) || !insertOnlyInput(node) - || !(windowing.getWindow() instanceof SessionWindowSpec)) { + || !(FlinkCompat.isSessionWindow(windowing.getWindow()))) { return false; } if (!WindowZoneGate.admits(node, windowing)) { @@ -466,7 +466,7 @@ static long windowSlide(WindowingStrategy windowing) { } static long gapMillis(WindowingStrategy windowing) { - return ((SessionWindowSpec) windowing.getWindow()).getGap().toMillis(); + return FlinkCompat.sessionGapMillis(windowing.getWindow()); } static int timeColumn(WindowingStrategy windowing) { diff --git a/src/main/java/tech/streamfusion/planner/WindowDeduplicateMatcher.java b/src/main/java/tech/streamfusion/planner/WindowDeduplicateMatcher.java index 347f42efc..3c0a59649 100644 --- a/src/main/java/tech/streamfusion/planner/WindowDeduplicateMatcher.java +++ b/src/main/java/tech/streamfusion/planner/WindowDeduplicateMatcher.java @@ -24,13 +24,15 @@ final class WindowDeduplicateMatcher { private WindowDeduplicateMatcher() {} static boolean matches(StreamPhysicalWindowDeduplicate dedup) { - if (!(dedup.getWindowingStrategy() instanceof WindowAttachedWindowingStrategy)) { + if (!(tech.streamfusion.compat.FlinkCompat.windowDedupWindowing(dedup) + instanceof WindowAttachedWindowingStrategy)) { return false; // the window must be attached as columns (the windowing-TVF output) } if (keepLastRow(dedup) == null) { return false; // cannot determine keep-first/keep-last → fall back } - if (!WindowZoneGate.admits(dedup, dedup.getWindowingStrategy())) { + if (!WindowZoneGate.admits( + dedup, tech.streamfusion.compat.FlinkCompat.windowDedupWindowing(dedup))) { return false; } return RowDataArrowConverter.supports( @@ -38,7 +40,8 @@ static boolean matches(StreamPhysicalWindowDeduplicate dedup) { } private static WindowAttachedWindowingStrategy windowing(StreamPhysicalWindowDeduplicate dedup) { - return (WindowAttachedWindowingStrategy) dedup.getWindowingStrategy(); + return (WindowAttachedWindowingStrategy) + tech.streamfusion.compat.FlinkCompat.windowDedupWindowing(dedup); } static int windowStartColumn(StreamPhysicalWindowDeduplicate dedup) { @@ -94,9 +97,14 @@ static Boolean keepLastRow(StreamPhysicalWindowDeduplicate dedup) { } static String unsupportedReason(StreamPhysicalWindowDeduplicate dedup) { + if (tech.streamfusion.compat.FlinkCompat.windowDedupWindowing(dedup) == null) { + return "window deduplication: host window metadata is unavailable"; + } String zoneReason = - dedup.getWindowingStrategy() instanceof WindowAttachedWindowingStrategy - ? WindowZoneGate.unsupportedReason(dedup, dedup.getWindowingStrategy()) + tech.streamfusion.compat.FlinkCompat.windowDedupWindowing(dedup) + instanceof WindowAttachedWindowingStrategy + ? WindowZoneGate.unsupportedReason( + dedup, tech.streamfusion.compat.FlinkCompat.windowDedupWindowing(dedup)) : null; if (zoneReason != null) { return "window deduplication: " + zoneReason; @@ -125,7 +133,8 @@ static RelNode substitute(StreamPhysicalWindowDeduplicate dedup, PlanContext ctx WindowDeduplicateMatcher.windowMillis(dedup), WindowDeduplicateMatcher.slideMillis(dedup), WindowDeduplicateMatcher.cumulative(dedup), - WindowZoneGate.boundaryOffsetMillis(dedup, dedup.getWindowingStrategy()), + WindowZoneGate.boundaryOffsetMillis( + dedup, tech.streamfusion.compat.FlinkCompat.windowDedupWindowing(dedup)), Boolean.TRUE.equals(keepLastRow(dedup))); } } diff --git a/src/main/java/tech/streamfusion/planner/WindowZoneGate.java b/src/main/java/tech/streamfusion/planner/WindowZoneGate.java index 27f74f917..e3c4cb7ef 100644 --- a/src/main/java/tech/streamfusion/planner/WindowZoneGate.java +++ b/src/main/java/tech/streamfusion/planner/WindowZoneGate.java @@ -4,12 +4,12 @@ import java.time.zone.ZoneRules; import org.apache.calcite.rel.RelNode; import org.apache.flink.table.planner.plan.logical.CumulativeWindowSpec; -import org.apache.flink.table.planner.plan.logical.SessionWindowSpec; import org.apache.flink.table.planner.plan.logical.WindowSpec; import org.apache.flink.table.planner.plan.logical.WindowingStrategy; import org.apache.flink.table.planner.utils.ShortcutUtils; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeRoot; +import tech.streamfusion.compat.FlinkCompat; /** * Admission gate for windows over a TIMESTAMP_LTZ time attribute (event-time or proctime). Flink @@ -31,7 +31,7 @@ static String unsupportedReason(RelNode node, WindowingStrategy windowing) { } ZoneRules rules = ShortcutUtils.unwrapTableConfig(node).getLocalTimeZone().getRules(); WindowSpec spec = windowing.getWindow(); - if (spec instanceof SessionWindowSpec) { + if (FlinkCompat.isSessionWindow(spec)) { return rules.isFixedOffset() ? null : "TIMESTAMP_LTZ session windows require a fixed offset for the full timestamp range"; diff --git a/src/main/java/tech/streamfusion/state/CanonicalNativeState.java b/src/main/java/tech/streamfusion/state/CanonicalNativeState.java index ae84ceab7..ac71b64eb 100644 --- a/src/main/java/tech/streamfusion/state/CanonicalNativeState.java +++ b/src/main/java/tech/streamfusion/state/CanonicalNativeState.java @@ -216,7 +216,7 @@ private static ValueState stateForKeyGroup( ValueStateDescriptor descriptor) throws Exception { CheckpointableKeyedStateBackend raw = backend; - raw.setCurrentKeyAndKeyGroup(keyGroup, keyGroup); + tech.streamfusion.compat.StateCompat.setCurrentKeyAndGroup(raw, keyGroup, keyGroup); return (ValueState) raw.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descriptor); @@ -251,8 +251,8 @@ private static void restoreKey( clearCurrentKey(backend); return; } - ((CheckpointableKeyedStateBackend) backend) - .setCurrentKeyAndKeyGroup(previous.key, previous.keyGroup); + tech.streamfusion.compat.StateCompat.setCurrentKeyAndGroup( + (CheckpointableKeyedStateBackend) backend, previous.key, previous.keyGroup); } private static void clearCurrentKey(CheckpointableKeyedStateBackend backend) { diff --git a/src/main/java/tech/streamfusion/state/FlinkRocksDBOptions.java b/src/main/java/tech/streamfusion/state/FlinkRocksDBOptions.java index dba54a1f6..a8a42acf3 100644 --- a/src/main/java/tech/streamfusion/state/FlinkRocksDBOptions.java +++ b/src/main/java/tech/streamfusion/state/FlinkRocksDBOptions.java @@ -1,20 +1,15 @@ package tech.streamfusion.state; -import static org.apache.flink.state.rocksdb.RocksDBConfigurableOptions.*; -import static org.apache.flink.state.rocksdb.RocksDBOptions.OPTIONS_FACTORY; -import static org.apache.flink.state.rocksdb.RocksDBOptions.PREDEFINED_OPTIONS; +import static tech.streamfusion.compat.RocksDBOptionsCompat.*; +import java.io.Serializable; +import java.util.Locale; +import java.util.stream.Collectors; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.MemorySize; import org.apache.flink.configuration.ReadableConfig; -import org.rocksdb.CompactionStyle; import org.rocksdb.CompressionType; -import java.io.Serializable; -import java.util.List; -import java.util.Locale; -import java.util.stream.Collectors; - /** * Flink's resolved public RocksDB options, serialized once at backend construction and handed to * Rust when a native state store opens. Explicit configuration wins over the predefined profile, @@ -71,7 +66,7 @@ public static FlinkRocksDBOptions from(ReadableConfig config) { profileValue(config, BLOCK_CACHE_SIZE, profile, MemorySize.parse("256mb")).getBytes(); String compression = - config.get(COMPRESSION_PER_LEVEL).stream() + compressionPerLevel(config).stream() .map(CompressionType::name) .map(FlinkRocksDBOptions::quote) .collect(Collectors.joining(",")); @@ -96,12 +91,8 @@ public static FlinkRocksDBOptions from(ReadableConfig config) { + field("maxWriteBufferNumber", buffers) + field("minWriteBufferNumberToMerge", mergeBuffers) + field("writeBatchSize", config.get(WRITE_BATCH_SIZE).getBytes()) - + field( - "compactionFilterQueryTimeAfterNumEntries", - config.get(COMPACT_FILTER_QUERY_TIME_AFTER_NUM_ENTRIES)) - + field( - "periodicCompactionSeconds", - config.get(COMPACT_FILTER_PERIODIC_COMPACTION_TIME).getSeconds()) + + field("compactionFilterQueryTimeAfterNumEntries", compactionQueryEntries(config)) + + field("periodicCompactionSeconds", periodicCompactionSeconds(config)) + field("blockSize", blockSize) + field("metadataBlockSize", config.get(METADATA_BLOCK_SIZE).getBytes()) + field("blockCacheSize", cacheSize) diff --git a/src/main/java/tech/streamfusion/state/RocksDBNativeKeyedStateBackend.java b/src/main/java/tech/streamfusion/state/RocksDBNativeKeyedStateBackend.java index 0f4c8c376..f813a6e29 100644 --- a/src/main/java/tech/streamfusion/state/RocksDBNativeKeyedStateBackend.java +++ b/src/main/java/tech/streamfusion/state/RocksDBNativeKeyedStateBackend.java @@ -1,13 +1,21 @@ package tech.streamfusion.state; +import static org.apache.flink.runtime.state.SnapshotExecutionType.ASYNCHRONOUS; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.RunnableFuture; +import java.util.stream.Stream; +import javax.annotation.Nonnull; import org.apache.flink.api.common.state.CheckpointListener; import org.apache.flink.api.common.state.State; import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.fs.CloseableRegistry; -import org.apache.flink.runtime.memory.OpaqueMemoryResource; import org.apache.flink.runtime.checkpoint.CheckpointOptions; +import org.apache.flink.runtime.memory.OpaqueMemoryResource; import org.apache.flink.runtime.state.AbstractKeyedStateBackend; import org.apache.flink.runtime.state.CheckpointStreamFactory; import org.apache.flink.runtime.state.CheckpointableKeyedStateBackend; @@ -27,16 +35,6 @@ import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.function.SupplierWithException; -import javax.annotation.Nonnull; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.RunnableFuture; -import java.util.stream.Stream; - -import static org.apache.flink.runtime.state.SnapshotExecutionType.ASYNCHRONOUS; - /** * The keyed state backend given to every keyed operator when the RocksDB state backend is selected. * JVM-side keyed state (descriptors, timer queues) delegates untouched to the wrapped backend; a @@ -71,6 +69,9 @@ public final class RocksDBNativeKeyedStateBackend private SupplierWithException, Exception> delegateSupplier; private CheckpointableKeyedStateBackend delegate; + private SupplierWithException, Exception> + canonicalProjectionFactory; + private boolean canonicalProjection; private K bufferedKey; private int bufferedKeyGroup; @@ -108,7 +109,8 @@ private CheckpointableKeyedStateBackend delegate() throws Exception { delegate = delegateSupplier.get(); delegateSupplier = null; if (bufferedKey != null) { - delegate.setCurrentKeyAndKeyGroup(bufferedKey, bufferedKeyGroup); + tech.streamfusion.compat.StateCompat.setCurrentKeyAndGroup( + delegate, bufferedKey, bufferedKeyGroup); } } return delegate; @@ -126,6 +128,42 @@ void materializeDelegate() throws Exception { delegate(); } + void setCanonicalProjectionFactory( + SupplierWithException, Exception> factory) { + canonicalProjectionFactory = factory; + } + + private CheckpointableKeyedStateBackend canonicalDelegate() throws Exception { + if (canonicalProjectionFactory == null) return delegate(); + if (delegateStateUsed) { + throw new IllegalStateException( + "canonical native state cannot replace an active JVM state backend"); + } + var projection = canonicalProjectionFactory.get(); + try { + if (delegate != null) { + try { + delegate.close(); + } finally { + delegate.dispose(); + } + } + } catch (Exception failure) { + try { + projection.close(); + } catch (Exception cleanup) { + failure.addSuppressed(cleanup); + } + projection.dispose(); + throw failure; + } + delegate = projection; + delegateSupplier = null; + canonicalProjectionFactory = null; + canonicalProjection = true; + return delegate; + } + // ---- The native operator's surface ----------------------------------------------------------- /** The local directory the operator's RocksDB table lives in (created by the native side). */ @@ -173,7 +211,8 @@ public CanonicalRestore restoreCanonicalState(String operatorId) throws Exceptio if (delegate == null) { return new CanonicalRestore(List.of(), Long.MIN_VALUE); } - CanonicalNativeState.Restore restored = CanonicalNativeState.readAndClear(delegate, operatorId); + CanonicalNativeState.Restore restored = + CanonicalNativeState.readAndClear(canonicalDelegate(), operatorId); return new CanonicalRestore(restored.partitions, restored.timerDeadline); } @@ -221,15 +260,26 @@ public void notifyCheckpointAborted(long checkpointId) throws Exception { @Nonnull @Override public SavepointResources savepoint() throws Exception { - if (snapshotStrategy.hasNativeState()) { - RocksDBNativeState nativeState = snapshotStrategy.nativeState(); - CanonicalNativeState.write( - delegate(), - nativeState.canonicalPartitions(), - nativeState.canonicalOperatorId(), - nativeState.canonicalTimerDeadline()); + if (!snapshotStrategy.hasNativeState()) return delegate().savepoint(); + RocksDBNativeState nativeState = snapshotStrategy.nativeState(); + var projection = canonicalDelegate(); + CanonicalNativeState.write( + projection, + nativeState.canonicalPartitions(), + nativeState.canonicalOperatorId(), + nativeState.canonicalTimerDeadline()); + var resources = projection.savepoint(); + if (canonicalProjection) { + // The heap snapshot owns its stable view. Release the live staging entries immediately. + try { + CanonicalNativeState.write( + projection, new byte[0][], nativeState.canonicalOperatorId(), Long.MIN_VALUE); + } catch (Exception failure) { + resources.getSnapshotResources().release(); + throw failure; + } } - return delegate().savepoint(); + return resources; } public static final class CanonicalRestore { @@ -342,14 +392,13 @@ void clearCurrentKey() { + delegate.getClass().getName()); } - @Override public void setCurrentKeyAndKeyGroup(K newKey, int newKeyGroupIndex) { if (delegate == null) { bufferedKey = newKey; bufferedKeyGroup = newKeyGroupIndex; return; } - delegate.setCurrentKeyAndKeyGroup(newKey, newKeyGroupIndex); + tech.streamfusion.compat.StateCompat.setCurrentKeyAndGroup(delegate, newKey, newKeyGroupIndex); } // ---- Delegate-materializing state access ------------------------------------------------------ @@ -370,9 +419,8 @@ public Stream getKeys(String state, N namespace) { return delegateUnchecked().getKeys(state, namespace); } - @Override public Stream getKeys(List states, N namespace) { - return delegateUnchecked().getKeys(states, namespace); + return tech.streamfusion.compat.StateCompat.keys(delegateUnchecked(), states, namespace); } @Override @@ -432,8 +480,7 @@ public boolean isSafeToReuseKVState() { return delegateUnchecked().isSafeToReuseKVState(); } - @Override public String getBackendTypeIdentifier() { - return delegateUnchecked().getBackendTypeIdentifier(); + return tech.streamfusion.compat.StateCompat.backendType(delegateUnchecked()); } } diff --git a/src/main/java/tech/streamfusion/state/RocksDBNativeSnapshotStrategy.java b/src/main/java/tech/streamfusion/state/RocksDBNativeSnapshotStrategy.java index a4ad65cde..b45a75760 100644 --- a/src/main/java/tech/streamfusion/state/RocksDBNativeSnapshotStrategy.java +++ b/src/main/java/tech/streamfusion/state/RocksDBNativeSnapshotStrategy.java @@ -171,7 +171,8 @@ private RocksDBSnapshotResources prepareResources(long checkpointId, File linkDi if (System.getenv("SF_STATE_PROFILE") != null) { System.err.printf( "SFPROF rocksdb barrier chk=%d sync_ms=%d%n", - checkpointId, (System.nanoTime() - profileStart) / 1_000_000); + checkpointId, + (System.nanoTime() - profileStart) / 1_000_000); } String snapshotToken = manifest[0]; List dataFiles = new ArrayList<>(); @@ -205,7 +206,7 @@ private RocksDBSnapshotResources prepareResources(long checkpointId, File linkDi StreamStateHandle confirmed = mayReuse ? confirmedBase.get(rel) : null; if (confirmed != null && currentStreamFactory != null - && currentStreamFactory.couldReuseStateHandle(confirmed)) { + && tech.streamfusion.compat.StateCompat.couldReuse(currentStreamFactory, confirmed)) { reusable.put(rel, confirmed); continue; } @@ -274,8 +275,7 @@ public SnapshotResultSupplier asyncSnapshot( StreamStateHandle confirmed = reuseBase.get(relPath); if (confirmed != null) { StreamStateHandle placeholder = - new PlaceholderStreamStateHandle( - confirmed.getStreamStateHandleID(), confirmed.getStateSize(), false); + tech.streamfusion.compat.StateCompat.placeholder(confirmed); sharedState.add(HandleAndLocalPath.of(placeholder, relPath)); reused.add(confirmed); } else { @@ -301,7 +301,7 @@ public SnapshotResultSupplier asyncSnapshot( checkpointedSize += metaHandle.getStateSize(); // Reuse registration can fail. Keep ownership until the factory accepts the snapshot. - streamFactory.reusePreviousStateHandle(reused); + tech.streamfusion.compat.StateCompat.reused(streamFactory, reused); if (sharing != SnapshotType.SharingFilesStrategy.NO_SHARING) { synchronized (uploadedFiles) { uploadedFiles.put(checkpointId, Collections.unmodifiableList(sharedState)); diff --git a/src/main/java/tech/streamfusion/state/RocksDBNativeStateBackend.java b/src/main/java/tech/streamfusion/state/RocksDBNativeStateBackend.java index 7bc793d65..328133a29 100644 --- a/src/main/java/tech/streamfusion/state/RocksDBNativeStateBackend.java +++ b/src/main/java/tech/streamfusion/state/RocksDBNativeStateBackend.java @@ -1,41 +1,37 @@ package tech.streamfusion.state; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; import org.apache.flink.configuration.CheckpointingOptions; import org.apache.flink.configuration.MemorySize; import org.apache.flink.configuration.ReadableConfig; import org.apache.flink.core.execution.SavepointFormatType; -import org.apache.flink.runtime.memory.OpaqueMemoryResource; import org.apache.flink.runtime.memory.MemoryManager; +import org.apache.flink.runtime.memory.OpaqueMemoryResource; import org.apache.flink.runtime.memory.SharedResources; import org.apache.flink.runtime.state.CheckpointableKeyedStateBackend; import org.apache.flink.runtime.state.IncrementalKeyedStateHandle.HandleAndLocalPath; import org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle; -import org.apache.flink.runtime.state.KeyedStateBackendParametersImpl; import org.apache.flink.runtime.state.KeyedStateHandle; -import org.apache.flink.runtime.state.OperatorStateBackend; -import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.state.StreamStateHandle; -import org.apache.flink.state.rocksdb.EmbeddedRocksDBStateBackend; -import org.apache.flink.state.rocksdb.RocksDBOptions; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import tech.streamfusion.compat.FlinkStateBackendCompat; +import tech.streamfusion.compat.KeyedBackendContext; +import tech.streamfusion.compat.RocksDBOptionsCompat; /** * Flink state backend that gives native StreamFusion operators a Rust-owned RocksDB instance and * delegates ordinary JVM keyed state, timers, and operator state to Flink's RocksDB backend. */ -public final class RocksDBNativeStateBackend implements StateBackend { +public final class RocksDBNativeStateBackend extends FlinkStateBackendCompat { private static final long serialVersionUID = 1L; - private final EmbeddedRocksDBStateBackend delegate; private final FlinkRocksDBOptions nativeOptions; private final boolean incrementalCheckpoints; private final String localDirectories; @@ -44,28 +40,29 @@ public final class RocksDBNativeStateBackend implements StateBackend { private final double writeBufferRatio; RocksDBNativeStateBackend(ReadableConfig config, ClassLoader classLoader) { - this.delegate = new EmbeddedRocksDBStateBackend().configure(config, classLoader); + super(config, classLoader); this.nativeOptions = FlinkRocksDBOptions.from(config); this.incrementalCheckpoints = config.get(CheckpointingOptions.INCREMENTAL_CHECKPOINTS); - this.localDirectories = config.get(RocksDBOptions.LOCAL_DIRECTORIES); - this.useManagedMemory = config.get(RocksDBOptions.USE_MANAGED_MEMORY); + this.localDirectories = config.get(RocksDBOptionsCompat.LOCAL_DIRECTORIES); + this.useManagedMemory = config.get(RocksDBOptionsCompat.USE_MANAGED_MEMORY); this.fixedPerSlotBytes = - config.getOptional(RocksDBOptions.FIX_PER_SLOT_MEMORY_SIZE) + config + .getOptional(RocksDBOptionsCompat.FIX_PER_SLOT_MEMORY_SIZE) .map(MemorySize::getBytes) .orElse(0L); - this.writeBufferRatio = config.get(RocksDBOptions.WRITE_BUFFER_RATIO); + this.writeBufferRatio = config.get(RocksDBOptionsCompat.WRITE_BUFFER_RATIO); } /** * Leases the slot's native RocksDB memory pool, resolved with Flink's own precedence: * fixed-per-slot wins, else the slot's managed-memory share (the default), else fixed-per-TM at * TM scope; nothing configured (or a zero budget) leaves stores on their per-instance options. - * The C++ pool lives in StreamFusion's RocksDB library, so it is leased under its own resource - * id alongside the delegate backend's pool rather than shared with it. + * The C++ pool lives in StreamFusion's RocksDB library, so it is leased under its own resource id + * alongside the delegate backend's pool rather than shared with it. */ private OpaqueMemoryResource leaseSharedResources( - KeyedStateBackendParameters parameters) throws Exception { - MemoryManager memoryManager = parameters.getEnv().getMemoryManager(); + KeyedBackendContext parameters) throws Exception { + MemoryManager memoryManager = parameters.environment().getMemoryManager(); double ratio = writeBufferRatio; if (fixedPerSlotBytes > 0) { return memoryManager.getExternalSharedMemoryResource( @@ -73,8 +70,8 @@ private OpaqueMemoryResource leaseSharedResources( size -> new NativeRocksSharedResources(size, ratio), fixedPerSlotBytes); } - if (useManagedMemory && parameters.getManagedMemoryFraction() > 0) { - long budget = memoryManager.computeMemorySize(parameters.getManagedMemoryFraction()); + if (useManagedMemory && parameters.managedMemoryFraction() > 0) { + long budget = memoryManager.computeMemorySize(parameters.managedMemoryFraction()); if (budget > 0) { return memoryManager.getExternalSharedMemoryResource( "streamfusion-rocksdb-slot-memory", @@ -84,14 +81,14 @@ private OpaqueMemoryResource leaseSharedResources( } long fixedPerTm = parameters - .getEnv() + .environment() .getTaskManagerInfo() .getConfiguration() - .getOptional(RocksDBOptions.FIX_PER_TM_MEMORY_SIZE) + .getOptional(RocksDBOptionsCompat.FIX_PER_TM_MEMORY_SIZE) .map(MemorySize::getBytes) .orElse(0L); if (fixedPerTm > 0) { - SharedResources sharedResources = parameters.getEnv().getSharedResources(); + SharedResources sharedResources = parameters.environment().getSharedResources(); Object leaseHolder = new Object(); SharedResources.ResourceAndSize resource = sharedResources.getOrAllocateSharedResource( @@ -110,22 +107,21 @@ private OpaqueMemoryResource leaseSharedResources( } @Override - public CheckpointableKeyedStateBackend createKeyedStateBackend( - KeyedStateBackendParameters parameters) throws Exception { + protected CheckpointableKeyedStateBackend createNativeKeyedBackend( + KeyedBackendContext parameters) throws Exception { List nativeHandles = new ArrayList<>(); List delegateHandles = new ArrayList<>(); - for (KeyedStateHandle handle : parameters.getStateHandles()) { + for (KeyedStateHandle handle : parameters.stateHandles()) { if (handle instanceof IncrementalRemoteKeyedStateHandle && RocksDBNativeSnapshotStrategy.isNativeMeta( - ((IncrementalRemoteKeyedStateHandle) handle).getMetaDataStateHandle())) { + tech.streamfusion.compat.StateCompat.metaHandle( + (IncrementalRemoteKeyedStateHandle) handle))) { nativeHandles.add((IncrementalRemoteKeyedStateHandle) handle); } else { delegateHandles.add(handle); } } - KeyedStateBackendParametersImpl delegateParameters = - new KeyedStateBackendParametersImpl<>(parameters).setStateHandles(delegateHandles); File workingDirectory = workingDirectory(parameters); List sources = new ArrayList<>(); for (int i = 0; i < nativeHandles.size(); i++) { @@ -134,13 +130,13 @@ public CheckpointableKeyedStateBackend createKeyedStateBackend( boolean alignedRestore = nativeHandles.size() == 1 - && nativeHandles.get(0).getKeyGroupRange().equals(parameters.getKeyGroupRange()); + && nativeHandles.get(0).getKeyGroupRange().equals(parameters.keyGroupRange()); UUID backendUID = alignedRestore ? nativeHandles.get(0).getBackendIdentifier() : UUID.randomUUID(); RocksDBNativeSnapshotStrategy strategy = new RocksDBNativeSnapshotStrategy( backendUID, - parameters.getKeyGroupRange(), + parameters.keyGroupRange(), new File(workingDirectory, "checkpoints"), incrementalCheckpoints); if (alignedRestore && incrementalCheckpoints) { @@ -149,15 +145,19 @@ public CheckpointableKeyedStateBackend createKeyedStateBackend( } RocksDBNativeKeyedStateBackend backend = new RocksDBNativeKeyedStateBackend<>( - () -> delegate.createKeyedStateBackend(delegateParameters), - parameters.getKeyGroupRange(), - parameters.getKeySerializer(), - parameters.getNumberOfKeyGroups(), + () -> parameters.delegateFactory().apply(delegateHandles), + parameters.keyGroupRange(), + parameters.keySerializer(), + parameters.numberOfKeyGroups(), strategy, workingDirectory, sources, nativeOptions.json(), leaseSharedResources(parameters)); + if (parameters.canonicalProjectionFactory() != null) { + backend.setCanonicalProjectionFactory( + () -> parameters.canonicalProjectionFactory().apply(delegateHandles)); + } if (!delegateHandles.isEmpty()) { try { backend.materializeDelegate(); @@ -169,13 +169,14 @@ public CheckpointableKeyedStateBackend createKeyedStateBackend( return backend; } - private File workingDirectory(KeyedStateBackendParameters parameters) { - String operator = parameters.getOperatorIdentifier().replaceAll("[^A-Za-z0-9_-]", "_"); - int subtask = parameters.getEnv().getTaskInfo().getIndexOfThisSubtask(); - int attempt = parameters.getEnv().getTaskInfo().getAttemptNumber(); - File base = parameters.getEnv().getTaskManagerInfo().getTmpWorkingDirectory(); + private File workingDirectory(KeyedBackendContext parameters) { + String operator = parameters.operatorIdentifier().replaceAll("[^A-Za-z0-9_-]", "_"); + int subtask = parameters.environment().getTaskInfo().getIndexOfThisSubtask(); + int attempt = parameters.environment().getTaskInfo().getAttemptNumber(); + File base = parameters.environment().getTaskManagerInfo().getTmpWorkingDirectory(); if (localDirectories != null && !localDirectories.isBlank()) { - String[] configured = localDirectories.split(",|" + java.util.regex.Pattern.quote(File.pathSeparator)); + String[] configured = + localDirectories.split(",|" + java.util.regex.Pattern.quote(File.pathSeparator)); String selected = configured[Math.floorMod(subtask, configured.length)].trim(); if (!selected.isEmpty()) { base = new File(selected); @@ -184,7 +185,7 @@ private File workingDirectory(KeyedStateBackendParameters parameters) { return new File( base, "streamfusion-rocksdb/" - + parameters.getJobID() + + parameters.jobId() + "/" + operator + "_" @@ -196,7 +197,8 @@ private File workingDirectory(KeyedStateBackendParameters parameters) { private static RocksDBRestoredSource materialize( IncrementalRemoteKeyedStateHandle handle, File directory) throws IOException { String token = - RocksDBNativeSnapshotStrategy.readMetaDocument(handle.getMetaDataStateHandle()); + RocksDBNativeSnapshotStrategy.readMetaDocument( + tech.streamfusion.compat.StateCompat.metaHandle(handle)); List files = new ArrayList<>(handle.getSharedState()); files.addAll(handle.getPrivateState()); for (HandleAndLocalPath file : files) { @@ -219,17 +221,6 @@ private static RocksDBRestoredSource materialize( handle.getKeyGroupRange().getEndKeyGroup()); } - @Override - public OperatorStateBackend createOperatorStateBackend(OperatorStateBackendParameters parameters) - throws Exception { - return delegate.createOperatorStateBackend(parameters); - } - - @Override - public boolean useManagedMemory() { - return delegate.useManagedMemory(); - } - @Override public boolean supportsNoClaimRestoreMode() { return true; diff --git a/src/test/java-flink1.18/tech/streamfusion/Flink118NexmarkPerimeterTest.java b/src/test/java-flink1.18/tech/streamfusion/Flink118NexmarkPerimeterTest.java new file mode 100644 index 000000000..fdd2eebfb --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/Flink118NexmarkPerimeterTest.java @@ -0,0 +1,46 @@ +package tech.streamfusion; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Set; +import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.NativePlanner; + +/** Verifies the existing benchmark plans without changing its queries or source fixtures. */ +class Flink118NexmarkPerimeterTest { + @Test + void generatorComparisonsRetainBothTransposes() throws Exception { + Set selected = Set.of("q0", "q1", "q2", "q3", "q5", "q7", "q8", "q19", "q20", "q21"); + var field = NexmarkMatrixBenchmark.class.getDeclaredField("ALL_QUERIES"); + field.setAccessible(true); + int verified = 0; + for (Object query : (Object[]) field.get(null)) { + String label = (String) value(query, "label"); + if (!selected.contains(label)) continue; + var environment = NexmarkBenchmark.environment(2_000_000); + environment.createTemporarySystemFunction( + "count_char", NexmarkMatrixBenchmark.CountChar.class); + String[] setup = (String[]) value(query, "setup"); + if (setup != null) { + for (String statement : setup) environment.executeSql(statement); + } + environment.executeSql( + ((String) value(query, "sinkDdl")) + .replace("%TS%", "TIMESTAMP(3)") + .replace("%WTS%", "TIMESTAMP(3)")); + var scan = NativePlanner.install(environment); + String plan = environment.explainSql((String) value(query, "insertSql")); + assertTrue(scan.substitutions() > 0, label + ": " + scan.explainSummary()); + assertTrue(plan.contains("RowDataToArrow"), label + ": " + plan); + assertTrue(plan.contains("ArrowToRowData"), label + ": " + plan); + verified++; + } + org.junit.jupiter.api.Assertions.assertEquals(selected.size(), verified); + } + + private static Object value(Object query, String name) throws Exception { + var field = query.getClass().getDeclaredField(name); + field.setAccessible(true); + return field.get(query); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/FlinkStateBackendAdmissionTest.java b/src/test/java-flink1.18/tech/streamfusion/FlinkStateBackendAdmissionTest.java new file mode 100644 index 000000000..2e8b00c9d --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/FlinkStateBackendAdmissionTest.java @@ -0,0 +1,73 @@ +package tech.streamfusion; + +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.StateBackendOptions; +import org.apache.flink.contrib.streaming.state.EmbeddedRocksDBStateBackend; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Test; +import tech.streamfusion.compat.FlinkTestSources; + +class FlinkStateBackendAdmissionTest { + private static final String GROUP = "SELECT k, SUM(v) FROM t GROUP BY k"; + private static final String REASON = + "state backend: Flink 1.18 native keyed state requires heap state or the StreamFusion RocksDB" + + " backend"; + + @Test + void explicitStockRocksBackendDeclinesKeyedSqlBeforeExecution() throws Exception { + NativeParity.assertFallbackReasonContains(() -> environment(true, false), GROUP, REASON); + } + + @Test + void configuredStockRocksBackendDeclinesKeyedSqlBeforeExecution() throws Exception { + NativeParity.assertFallbackReasonContains(() -> environment(false, true), GROUP, REASON); + } + + @Test + void stockRocksBackendStillAdmitsStatelessSql() throws Exception { + NativeParity.assertParity(() -> environment(true, false), "SELECT v + 1 FROM t"); + } + + @Test + void heapBackendStillAdmitsKeyedSql() throws Exception { + NativeParity.assertChangelogParity(() -> environment(false, false), GROUP); + } + + @Test + void changelogHeapBackendDeclinesKeyedSqlBeforeExecution() throws Exception { + NativeParity.assertFallbackReasonContains( + () -> environment(false, false, true), + GROUP, + "state backend: Flink 1.18 changelog state is not verified for native keyed state"); + } + + @Test + void changelogBackendStillAdmitsStatelessSql() throws Exception { + NativeParity.assertParity(() -> environment(false, false, true), "SELECT v + 1 FROM t"); + } + + private static TableEnvironment environment(boolean explicitRocks, boolean configuredRocks) { + return environment(explicitRocks, configuredRocks, false); + } + + private static TableEnvironment environment( + boolean explicitRocks, boolean configuredRocks, boolean changelog) { + Configuration configuration = new Configuration(); + if (configuredRocks) configuration.set(StateBackendOptions.STATE_BACKEND, "rocksdb"); + StreamExecutionEnvironment env = + StreamExecutionEnvironment.getExecutionEnvironment(configuration); + env.setParallelism(1); + env.enableChangelogStateBackend(changelog); + if (explicitRocks) env.setStateBackend(new EmbeddedRocksDBStateBackend()); + StreamTableEnvironment table = StreamTableEnvironment.create(env); + table.createTemporaryView( + "t", + FlinkTestSources.fromData(env, Row.of(1L, 10L), Row.of(1L, 5L), Row.of(2L, 7L)) + .returns(Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG))); + return table; + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/CheckpointFileSink.java b/src/test/java-flink1.18/tech/streamfusion/compat/CheckpointFileSink.java new file mode 100644 index 000000000..b5919ffcb --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/CheckpointFileSink.java @@ -0,0 +1,21 @@ +package tech.streamfusion.compat; + +import java.nio.file.Path; +import org.apache.flink.api.common.serialization.SimpleStringEncoder; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy; + +/** Uses the line's filesystem SQL writer generation for exchange checkpoint verification. */ +public final class CheckpointFileSink { + private CheckpointFileSink() {} + + public static void attach(DataStream rows, Path output, String uid) { + var sink = + org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink.forRowFormat( + new org.apache.flink.core.fs.Path(output.toUri()), + new SimpleStringEncoder("UTF-8")) + .withRollingPolicy(OnCheckpointRollingPolicy.build()) + .build(); + rows.addSink(sink).uid(uid); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/CheckpointTestCapabilities.java b/src/test/java-flink1.18/tech/streamfusion/compat/CheckpointTestCapabilities.java new file mode 100644 index 000000000..3337f5a66 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/CheckpointTestCapabilities.java @@ -0,0 +1,13 @@ +package tech.streamfusion.compat; + +public final class CheckpointTestCapabilities { + private CheckpointTestCapabilities() {} + + public static final boolean PER_EDGE_ALIGNMENT = false; + + public static void configure( + org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env, + boolean recoverable) { + if (PER_EDGE_ALIGNMENT || recoverable) env.getCheckpointConfig().enableUnalignedCheckpoints(); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/CheckpointTestConfig.java b/src/test/java-flink1.18/tech/streamfusion/compat/CheckpointTestConfig.java new file mode 100644 index 000000000..0b4532ca4 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/CheckpointTestConfig.java @@ -0,0 +1,19 @@ +package tech.streamfusion.compat; + +import org.apache.flink.configuration.Configuration; + +public final class CheckpointTestConfig { + private CheckpointTestConfig() {} + + public static void retainOnCancellation(Configuration config) { + config.set( + org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions + .EXTERNALIZED_CHECKPOINT, + org.apache.flink.streaming.api.environment.CheckpointConfig.ExternalizedCheckpointCleanup + .RETAIN_ON_CANCELLATION); + } + + public static void restore(Configuration config, String path) { + config.set(org.apache.flink.runtime.jobgraph.SavepointConfigOptions.SAVEPOINT_PATH, path); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/FileWriterTestFactory.java b/src/test/java-flink1.18/tech/streamfusion/compat/FileWriterTestFactory.java new file mode 100644 index 000000000..6fb347b0f --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/FileWriterTestFactory.java @@ -0,0 +1,31 @@ +package tech.streamfusion.compat; + +import java.util.List; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.file.table.stream.StreamingFileWriter; +import org.apache.flink.core.fs.Path; +import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig; +import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink; +import tech.streamfusion.operator.*; + +public final class FileWriterTestFactory { + private FileWriterTestFactory() {} + + public static StreamingFileWriter writer( + java.nio.file.Path directory, + NativeFileBulkWriterFactory factory, + List partitionKeys) { + StreamingFileSink.BucketsBuilder< + PartitionedArrowBatch, + String, + ? extends StreamingFileSink.BucketsBuilder> + buckets = + StreamingFileSink.forBulkFormat(new Path(directory.toUri()), factory) + .withBucketAssigner(new PartitionedBatchBucketAssigner()) + .withRollingPolicy( + new NativeFileRollingPolicy(128 << 20, Long.MAX_VALUE, Long.MAX_VALUE)) + .withOutputFileConfig( + OutputFileConfig.builder().withPartPrefix("part-test").build()); + return new StreamingFileWriter<>(1000, buckets, partitionKeys, new Configuration()); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/FlinkTestCapabilities.java b/src/test/java-flink1.18/tech/streamfusion/compat/FlinkTestCapabilities.java new file mode 100644 index 000000000..fc7e68f46 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/FlinkTestCapabilities.java @@ -0,0 +1,85 @@ +package tech.streamfusion.compat; + +import org.junit.jupiter.api.Assumptions; + +/** Explicit N/A checks for syntax or host operations absent from the selected released line. */ +public final class FlinkTestCapabilities { + private FlinkTestCapabilities() {} + + public static final boolean CHECKPOINT_REUSE_NOTIFICATION = false; + + private static final java.util.Set ABSENT_SQL_FUNCTIONS = + java.util.Set.of( + "BTRIM", + "ENDS_WITH", + "ENDSWITH", + "SPLIT", + "URL_ENCODE", + "ELT", + "JSON", + "JSON_QUOTE", + "JSON_UNQUOTE", + "STARTSWITH", + "TRANSLATE3", + "UNHEX", + "URL_DECODE"); + public static final boolean RETRACTING_WINDOW_TVF = false; + public static final boolean SESSION_TABLE_FUNCTION = false; + public static final boolean DYNAMIC_JSON_QUERY = false; + + public static final boolean VARIABLE_LENGTH_ENCODE = false; + public static final boolean CORRECTED_AVRO_TIMESTAMPS = false; + public static final boolean AVRO_ENCODING_OPTION = false; + public static final boolean JSON_PARSER_OPTION = false; + public static final boolean UNNEST_ORDINALITY = false; + public static final boolean SINK_LENGTH_ERROR = false; + public static final boolean TIMESTAMP_LTZ_TEXT_OVERLOADS = false; + public static final boolean TIMESTAMP_LTZ_DEFAULT_PRECISION = false; + public static final boolean SMALL_INTEGER_ARRAY_LOOKUP = false; + public static final boolean SUB_HOUR_TIMESTAMP_ROUNDING = false; + public static final boolean NEGATIVE_TIMESTAMP_TO_TIME = false; + public static final boolean RANK_OVER_AGGREGATE = false; + + public static org.apache.flink.table.types.DataType ifNullStringType() { + return org.apache.flink.table.api.DataTypes.CHAR(7).notNull(); + } + + public static final String JSON_FRACTION_CLASS = "java.lang.Double"; + + public static void requireJsonFunctions(String expression) { + for (String name : java.util.List.of("JSON_QUOTE", "JSON_UNQUOTE")) { + if (expression.contains(name + "(")) requireSqlFunction(name); + } + } + + public static void requireSqlFunction(String name) { + Assumptions.assumeFalse( + ABSENT_SQL_FUNCTIONS.contains(name), + name + " is absent from Flink 1.18's released SQL function catalog"); + } + + public static void requireSessionTableFunction() { + Assumptions.assumeTrue( + SESSION_TABLE_FUNCTION, + "The SESSION table function is absent from Flink 1.18; grouped session windows are" + + " separate"); + } + + public static final boolean KEY_ORDERED_ASYNC_LOOKUP = false; + public static final boolean STATE_TTL_HINTS = false; + public static final boolean TEMPORAL_FIRST_LAST = false; + public static final boolean KAFKA_TRANSACTION_NAMING = false; + + public static void requireStateTtlHint() { + Assumptions.assumeTrue( + STATE_TTL_HINTS, + "STATE_TTL hints are absent from Flink 1.18; global retention is tested separately"); + } + + public static void requireFirstLastType(String type) { + boolean temporal = type.equals("DATE") || type.startsWith("TIMESTAMP"); + Assumptions.assumeTrue( + !temporal || TEMPORAL_FIRST_LAST, + "Flink 1.18 cannot plan FIRST_VALUE/LAST_VALUE over " + type); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/FlinkTestSources.java b/src/test/java-flink1.18/tech/streamfusion/compat/FlinkTestSources.java new file mode 100644 index 000000000..d5b91eaa2 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/FlinkTestSources.java @@ -0,0 +1,35 @@ +package tech.streamfusion.compat; + +import java.util.Collection; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.DataStreamSource; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +/** Uses the selected host's collection-source API with the same elements and type information. */ +public final class FlinkTestSources { + private FlinkTestSources() {} + + public static DataStreamSource fromData(StreamExecutionEnvironment env, T... data) { + return env.fromElements(data); + } + + public static DataStreamSource fromData( + StreamExecutionEnvironment env, TypeInformation type, T... data) { + return env.fromCollection(java.util.Arrays.asList(data), type); + } + + public static DataStreamSource fromData( + StreamExecutionEnvironment env, Collection data, TypeInformation type) { + return env.fromCollection(data, type); + } + + public static DataStreamSource fromData( + StreamExecutionEnvironment env, Class type, T... data) { + return env.fromCollection(java.util.Arrays.asList(data), TypeInformation.of(type)); + } + + public static DataStreamSource fromData( + StreamExecutionEnvironment env, Collection data) { + return env.fromCollection(data); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/JsonTestSchemas.java b/src/test/java-flink1.18/tech/streamfusion/compat/JsonTestSchemas.java new file mode 100644 index 000000000..7952d4bf7 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/JsonTestSchemas.java @@ -0,0 +1,76 @@ +package tech.streamfusion.compat; + +import org.apache.flink.formats.common.TimestampFormat; +import org.apache.flink.formats.json.*; +import org.apache.flink.table.types.logical.RowType; + +public final class JsonTestSchemas { + private JsonTestSchemas() {} + + public static final boolean[] IGNORE_NULL_FIELD_MODES = new boolean[] {false}; + + public static org.apache.flink.formats.json.JsonRowDataSerializationSchema json( + RowType type, + TimestampFormat timestamps, + JsonFormatOptions.MapNullKeyMode mode, + String literal, + boolean plainDecimal, + boolean ignoreNulls) { + org.junit.jupiter.api.Assumptions.assumeFalse( + ignoreNulls, "JSON encode.ignore-null-fields is absent from Flink 1.18"); + return new org.apache.flink.formats.json.JsonRowDataSerializationSchema( + type, timestamps, mode, literal, plainDecimal); + } + + public static org.apache.flink.formats.json.debezium.DebeziumJsonSerializationSchema debezium( + RowType type, + TimestampFormat timestamps, + JsonFormatOptions.MapNullKeyMode mode, + String literal, + boolean plainDecimal, + boolean ignoreNulls) { + org.junit.jupiter.api.Assumptions.assumeFalse( + ignoreNulls, "JSON encode.ignore-null-fields is absent from Flink 1.18"); + return new org.apache.flink.formats.json.debezium.DebeziumJsonSerializationSchema( + type, timestamps, mode, literal, plainDecimal); + } + + public static org.apache.flink.formats.json.canal.CanalJsonSerializationSchema canal( + RowType type, + TimestampFormat timestamps, + JsonFormatOptions.MapNullKeyMode mode, + String literal, + boolean plainDecimal, + boolean ignoreNulls) { + org.junit.jupiter.api.Assumptions.assumeFalse( + ignoreNulls, "JSON encode.ignore-null-fields is absent from Flink 1.18"); + return new org.apache.flink.formats.json.canal.CanalJsonSerializationSchema( + type, timestamps, mode, literal, plainDecimal); + } + + public static org.apache.flink.formats.json.maxwell.MaxwellJsonSerializationSchema maxwell( + RowType type, + TimestampFormat timestamps, + JsonFormatOptions.MapNullKeyMode mode, + String literal, + boolean plainDecimal, + boolean ignoreNulls) { + org.junit.jupiter.api.Assumptions.assumeFalse( + ignoreNulls, "JSON encode.ignore-null-fields is absent from Flink 1.18"); + return new org.apache.flink.formats.json.maxwell.MaxwellJsonSerializationSchema( + type, timestamps, mode, literal, plainDecimal); + } + + public static org.apache.flink.formats.json.ogg.OggJsonSerializationSchema ogg( + RowType type, + TimestampFormat timestamps, + JsonFormatOptions.MapNullKeyMode mode, + String literal, + boolean plainDecimal, + boolean ignoreNulls) { + org.junit.jupiter.api.Assumptions.assumeFalse( + ignoreNulls, "JSON encode.ignore-null-fields is absent from Flink 1.18"); + return new org.apache.flink.formats.json.ogg.OggJsonSerializationSchema( + type, timestamps, mode, literal, plainDecimal); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/RichAsyncFunction.java b/src/test/java-flink1.18/tech/streamfusion/compat/RichAsyncFunction.java new file mode 100644 index 000000000..5b275f38e --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/RichAsyncFunction.java @@ -0,0 +1,11 @@ +package tech.streamfusion.compat; + +public abstract class RichAsyncFunction + extends org.apache.flink.streaming.api.functions.async.RichAsyncFunction { + @Override + public final void open(org.apache.flink.configuration.Configuration context) throws Exception { + initialize(); + } + + protected abstract void initialize() throws Exception; +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/RichMapFunction.java b/src/test/java-flink1.18/tech/streamfusion/compat/RichMapFunction.java new file mode 100644 index 000000000..54d06f3af --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/RichMapFunction.java @@ -0,0 +1,11 @@ +package tech.streamfusion.compat; + +public abstract class RichMapFunction + extends org.apache.flink.api.common.functions.RichMapFunction { + @Override + public final void open(org.apache.flink.configuration.Configuration parameters) throws Exception { + initialize(); + } + + protected abstract void initialize() throws Exception; +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/RichSinkFunction.java b/src/test/java-flink1.18/tech/streamfusion/compat/RichSinkFunction.java new file mode 100644 index 000000000..5fe0db210 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/RichSinkFunction.java @@ -0,0 +1,4 @@ +package tech.streamfusion.compat; + +public abstract class RichSinkFunction + extends org.apache.flink.streaming.api.functions.sink.RichSinkFunction {} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/SinkFunctionProvider.java b/src/test/java-flink1.18/tech/streamfusion/compat/SinkFunctionProvider.java new file mode 100644 index 000000000..9fefc48c4 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/SinkFunctionProvider.java @@ -0,0 +1,12 @@ +package tech.streamfusion.compat; + +public final class SinkFunctionProvider { + private SinkFunctionProvider() {} + + public static org.apache.flink.table.connector.sink.DynamicTableSink.SinkRuntimeProvider of( + org.apache.flink.streaming.api.functions.sink.SinkFunction< + org.apache.flink.table.data.RowData> + function) { + return org.apache.flink.table.connector.sink.SinkFunctionProvider.of(function); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/SourceFunction.java b/src/test/java-flink1.18/tech/streamfusion/compat/SourceFunction.java new file mode 100644 index 000000000..7ad0413d4 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/SourceFunction.java @@ -0,0 +1,4 @@ +package tech.streamfusion.compat; + +public interface SourceFunction + extends org.apache.flink.streaming.api.functions.source.SourceFunction {} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/StreamTestSinks.java b/src/test/java-flink1.18/tech/streamfusion/compat/StreamTestSinks.java new file mode 100644 index 000000000..2480fc37e --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/StreamTestSinks.java @@ -0,0 +1,9 @@ +package tech.streamfusion.compat; + +public final class StreamTestSinks { + private StreamTestSinks() {} + + public static void discard(org.apache.flink.streaming.api.datastream.DataStream stream) { + stream.addSink(new org.apache.flink.streaming.api.functions.sink.DiscardingSink<>()); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/TestOutput.java b/src/test/java-flink1.18/tech/streamfusion/compat/TestOutput.java new file mode 100644 index 000000000..697acf718 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/TestOutput.java @@ -0,0 +1,3 @@ +package tech.streamfusion.compat; + +public abstract class TestOutput implements org.apache.flink.streaming.api.operators.Output {} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/WatermarkTestOutputs.java b/src/test/java-flink1.18/tech/streamfusion/compat/WatermarkTestOutputs.java new file mode 100644 index 000000000..3798d28a8 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/WatermarkTestOutputs.java @@ -0,0 +1,10 @@ +package tech.streamfusion.compat; + +public final class WatermarkTestOutputs { + private WatermarkTestOutputs() {} + + public static void register( + org.apache.flink.api.common.eventtime.WatermarkOutputMultiplexer multiplexer, String id) { + multiplexer.registerNewOutput(id, watermark -> {}); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/compat/WindowTestAssigner.java b/src/test/java-flink1.18/tech/streamfusion/compat/WindowTestAssigner.java new file mode 100644 index 000000000..dac8b5a9c --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/compat/WindowTestAssigner.java @@ -0,0 +1,21 @@ +package tech.streamfusion.compat; + +import java.time.Duration; +import java.util.Collection; +import org.apache.flink.table.runtime.operators.window.TimeWindow; +import org.apache.flink.table.runtime.operators.window.assigners.*; + +public final class WindowTestAssigner { + private WindowTestAssigner() {} + + public static Collection assign(int kind, long size, long timestamp) + throws java.io.IOException { + WindowAssigner assigner = + kind == 0 + ? TumblingWindowAssigner.of(Duration.ofMillis(size)) + : kind == 1 + ? SlidingWindowAssigner.of(Duration.ofMillis(size), Duration.ofMillis(1000)) + : CumulativeWindowAssigner.of(Duration.ofMillis(size), Duration.ofMillis(1000)); + return assigner.assignWindows(null, timestamp); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/format/avro/compat/AvroTestSchemas.java b/src/test/java-flink1.18/tech/streamfusion/format/avro/compat/AvroTestSchemas.java new file mode 100644 index 000000000..c0e357ea3 --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/format/avro/compat/AvroTestSchemas.java @@ -0,0 +1,27 @@ +package tech.streamfusion.format.avro.compat; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.formats.avro.*; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; + +public final class AvroTestSchemas { + private AvroTestSchemas() {} + + public static final boolean CORRECTED_TIMESTAMP_MAPPING = false; + + public static AvroRowDataDeserializationSchema decoder( + RowType type, TypeInformation info, boolean legacy) { + org.junit.jupiter.api.Assumptions.assumeTrue( + legacy || CORRECTED_TIMESTAMP_MAPPING, + "Corrected Avro timestamp mapping is absent from Flink 1.18"); + return new AvroRowDataDeserializationSchema(type, info); + } + + public static AvroRowDataSerializationSchema encoder(RowType type, boolean legacy) { + org.junit.jupiter.api.Assumptions.assumeTrue( + legacy || CORRECTED_TIMESTAMP_MAPPING, + "Corrected Avro timestamp mapping is absent from Flink 1.18"); + return new AvroRowDataSerializationSchema(type); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/kafka/KafkaLineageAssertions.java b/src/test/java-flink1.18/tech/streamfusion/kafka/KafkaLineageAssertions.java new file mode 100644 index 000000000..cf89dff2a --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/kafka/KafkaLineageAssertions.java @@ -0,0 +1,9 @@ +package tech.streamfusion.kafka; + +final class KafkaLineageAssertions { + private KafkaLineageAssertions() {} + + static void assertTopic(PreSerializedKafkaRecordSchema schema, String topic) { + // Dataset lineage is absent from the released Kafka 3.2.0-1.18 connector. + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/operator/NativeJsonRuntimeTest.java b/src/test/java-flink1.18/tech/streamfusion/operator/NativeJsonRuntimeTest.java new file mode 100644 index 000000000..259d9e53d --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/operator/NativeJsonRuntimeTest.java @@ -0,0 +1,23 @@ +package tech.streamfusion.operator; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonFactory; +import org.junit.jupiter.api.Test; + +class NativeJsonRuntimeTest { + @Test + void olderJacksonDeclinesUnverifiedBufferEmulation() { + assertFalse(NativeJsonRuntime.available()); + assertFalse(NativeJsonRuntime.probe(JsonFactory::new, JsonFactory::new)); + } + + @Test + void missingRuntimeFailsClosed() { + assertFalse( + NativeJsonRuntime.probe( + () -> { + throw new NoClassDefFoundError("shaded Jackson"); + }, + JsonFactory::new)); + } +} diff --git a/src/test/java-flink1.18/tech/streamfusion/state/FlinkRocksDBOptionsTest.java b/src/test/java-flink1.18/tech/streamfusion/state/FlinkRocksDBOptionsTest.java new file mode 100644 index 000000000..06b79302c --- /dev/null +++ b/src/test/java-flink1.18/tech/streamfusion/state/FlinkRocksDBOptionsTest.java @@ -0,0 +1,56 @@ +package tech.streamfusion.state; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.MemorySize; +import org.apache.flink.contrib.streaming.state.RocksDBConfigurableOptions; +import org.apache.flink.contrib.streaming.state.RocksDBOptions; +import org.junit.jupiter.api.Test; + +class FlinkRocksDBOptionsTest { + + @Test + void resolvesFlinkDefaultsAndExplicitOverrides() { + Configuration configuration = new Configuration(); + configuration.set( + RocksDBConfigurableOptions.WRITE_BUFFER_SIZE, MemorySize.parse("17mb")); + configuration.set( + RocksDBConfigurableOptions.WRITE_BATCH_SIZE, MemorySize.parse("3mb")); + configuration.set(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE, true); + configuration.set(RocksDBConfigurableOptions.COMPACTION_STYLE, org.rocksdb.CompactionStyle.UNIVERSAL); + + String json = FlinkRocksDBOptions.from(configuration).json(); + + assertTrue(json.contains("\"writeBufferSize\":17825792")); + assertTrue(json.contains("\"writeBatchSize\":3145728")); + assertTrue(json.contains("\"useDynamicLevelSize\":true")); + assertTrue(json.contains("\"compactionStyle\":\"UNIVERSAL\"")); + assertTrue(json.contains("\"periodicCompactionSeconds\":0")); + assertTrue(json.contains("\"compactionFilterQueryTimeAfterNumEntries\":1000")); + assertTrue(json.contains("\"compressionPerLevel\":[]")); + } + + @Test + void appliesPredefinedProfileBeforeExplicitOverrides() { + Configuration configuration = new Configuration(); + configuration.set(RocksDBOptions.PREDEFINED_OPTIONS, "SPINNING_DISK_OPTIMIZED_HIGH_MEM"); + configuration.set(RocksDBConfigurableOptions.MAX_BACKGROUND_THREADS, 7); + + String json = FlinkRocksDBOptions.from(configuration).json(); + + assertTrue(json.contains("\"maxBackgroundThreads\":7")); + assertTrue(json.contains("\"blockCacheSize\":268435456")); + assertTrue(json.contains("\"blockSize\":131072")); + assertTrue(json.contains("\"maxSizeLevelBase\":1073741824")); + } + + @Test + void rejectsJavaOptionsFactoryInsteadOfSilentlyDiverging() { + Configuration configuration = new Configuration(); + configuration.set(RocksDBOptions.OPTIONS_FACTORY, "example.OptionsFactory"); + + assertThrows(IllegalArgumentException.class, () -> FlinkRocksDBOptions.from(configuration)); + } +} diff --git a/src/test/java/tech/streamfusion/NativeJsonBufferHistoryTest.java b/src/test/java-flink2.2/tech/streamfusion/NativeJsonBufferHistoryTest.java similarity index 100% rename from src/test/java/tech/streamfusion/NativeJsonBufferHistoryTest.java rename to src/test/java-flink2.2/tech/streamfusion/NativeJsonBufferHistoryTest.java diff --git a/src/test/java/tech/streamfusion/NativeJsonDoubleTest.java b/src/test/java-flink2.2/tech/streamfusion/NativeJsonDoubleTest.java similarity index 100% rename from src/test/java/tech/streamfusion/NativeJsonDoubleTest.java rename to src/test/java-flink2.2/tech/streamfusion/NativeJsonDoubleTest.java diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/CheckpointFileSink.java b/src/test/java-flink2.2/tech/streamfusion/compat/CheckpointFileSink.java new file mode 100644 index 000000000..6b27065c2 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/CheckpointFileSink.java @@ -0,0 +1,21 @@ +package tech.streamfusion.compat; + +import java.nio.file.Path; +import org.apache.flink.api.common.serialization.SimpleStringEncoder; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy; + +/** Uses the line's filesystem SQL writer generation for exchange checkpoint verification. */ +public final class CheckpointFileSink { + private CheckpointFileSink() {} + + public static void attach(DataStream rows, Path output, String uid) { + var sink = + org.apache.flink.connector.file.sink.FileSink.forRowFormat( + new org.apache.flink.core.fs.Path(output.toUri()), + new SimpleStringEncoder("UTF-8")) + .withRollingPolicy(OnCheckpointRollingPolicy.build()) + .build(); + rows.sinkTo(sink).uid(uid); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/CheckpointTestCapabilities.java b/src/test/java-flink2.2/tech/streamfusion/compat/CheckpointTestCapabilities.java new file mode 100644 index 000000000..2f0cbaea1 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/CheckpointTestCapabilities.java @@ -0,0 +1,13 @@ +package tech.streamfusion.compat; + +public final class CheckpointTestCapabilities { + private CheckpointTestCapabilities() {} + + public static final boolean PER_EDGE_ALIGNMENT = true; + + public static void configure( + org.apache.flink.streaming.api.environment.StreamExecutionEnvironment env, + boolean recoverable) { + if (PER_EDGE_ALIGNMENT || recoverable) env.getCheckpointConfig().enableUnalignedCheckpoints(); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/CheckpointTestConfig.java b/src/test/java-flink2.2/tech/streamfusion/compat/CheckpointTestConfig.java new file mode 100644 index 000000000..47565146e --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/CheckpointTestConfig.java @@ -0,0 +1,17 @@ +package tech.streamfusion.compat; + +import org.apache.flink.configuration.Configuration; + +public final class CheckpointTestConfig { + private CheckpointTestConfig() {} + + public static void retainOnCancellation(Configuration config) { + config.set( + org.apache.flink.configuration.CheckpointingOptions.EXTERNALIZED_CHECKPOINT_RETENTION, + org.apache.flink.configuration.ExternalizedCheckpointRetention.RETAIN_ON_CANCELLATION); + } + + public static void restore(Configuration config, String path) { + config.set(org.apache.flink.configuration.StateRecoveryOptions.SAVEPOINT_PATH, path); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/FileWriterTestFactory.java b/src/test/java-flink2.2/tech/streamfusion/compat/FileWriterTestFactory.java new file mode 100644 index 000000000..f3a3f8279 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/FileWriterTestFactory.java @@ -0,0 +1,31 @@ +package tech.streamfusion.compat; + +import java.util.List; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.file.table.stream.StreamingFileWriter; +import org.apache.flink.core.fs.Path; +import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig; +import org.apache.flink.streaming.api.functions.sink.filesystem.legacy.StreamingFileSink; +import tech.streamfusion.operator.*; + +public final class FileWriterTestFactory { + private FileWriterTestFactory() {} + + public static StreamingFileWriter writer( + java.nio.file.Path directory, + NativeFileBulkWriterFactory factory, + List partitionKeys) { + StreamingFileSink.BucketsBuilder< + PartitionedArrowBatch, + String, + ? extends StreamingFileSink.BucketsBuilder> + buckets = + StreamingFileSink.forBulkFormat(new Path(directory.toUri()), factory) + .withBucketAssigner(new PartitionedBatchBucketAssigner()) + .withRollingPolicy( + new NativeFileRollingPolicy(128 << 20, Long.MAX_VALUE, Long.MAX_VALUE)) + .withOutputFileConfig( + OutputFileConfig.builder().withPartPrefix("part-test").build()); + return new StreamingFileWriter<>(1000, buckets, partitionKeys, new Configuration()); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/FlinkTestCapabilities.java b/src/test/java-flink2.2/tech/streamfusion/compat/FlinkTestCapabilities.java new file mode 100644 index 000000000..f652817a7 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/FlinkTestCapabilities.java @@ -0,0 +1,71 @@ +package tech.streamfusion.compat; + +import org.junit.jupiter.api.Assumptions; + +/** Explicit N/A checks for syntax or host operations absent from the selected released line. */ +public final class FlinkTestCapabilities { + private FlinkTestCapabilities() {} + + public static final boolean CHECKPOINT_REUSE_NOTIFICATION = true; + + private static final java.util.Set ABSENT_SQL_FUNCTIONS = java.util.Set.of(); + public static final boolean RETRACTING_WINDOW_TVF = true; + public static final boolean SESSION_TABLE_FUNCTION = true; + public static final boolean DYNAMIC_JSON_QUERY = true; + + public static final boolean VARIABLE_LENGTH_ENCODE = true; + public static final boolean CORRECTED_AVRO_TIMESTAMPS = true; + public static final boolean AVRO_ENCODING_OPTION = true; + public static final boolean JSON_PARSER_OPTION = true; + public static final boolean UNNEST_ORDINALITY = true; + public static final boolean SINK_LENGTH_ERROR = true; + public static final boolean TIMESTAMP_LTZ_TEXT_OVERLOADS = true; + public static final boolean TIMESTAMP_LTZ_DEFAULT_PRECISION = true; + public static final boolean SMALL_INTEGER_ARRAY_LOOKUP = true; + public static final boolean SUB_HOUR_TIMESTAMP_ROUNDING = true; + public static final boolean NEGATIVE_TIMESTAMP_TO_TIME = true; + public static final boolean RANK_OVER_AGGREGATE = true; + + public static org.apache.flink.table.types.DataType ifNullStringType() { + return org.apache.flink.table.api.DataTypes.STRING().notNull(); + } + + public static final String JSON_FRACTION_CLASS = "java.math.BigDecimal"; + + public static void requireJsonFunctions(String expression) { + for (String name : java.util.List.of("JSON_QUOTE", "JSON_UNQUOTE")) { + if (expression.contains(name + "(")) requireSqlFunction(name); + } + } + + public static void requireSqlFunction(String name) { + Assumptions.assumeFalse( + ABSENT_SQL_FUNCTIONS.contains(name), + name + " is absent from Flink 1.18's released SQL function catalog"); + } + + public static void requireSessionTableFunction() { + Assumptions.assumeTrue( + SESSION_TABLE_FUNCTION, + "The SESSION table function is absent from Flink 1.18; grouped session windows are" + + " separate"); + } + + public static final boolean KEY_ORDERED_ASYNC_LOOKUP = true; + public static final boolean STATE_TTL_HINTS = true; + public static final boolean TEMPORAL_FIRST_LAST = true; + public static final boolean KAFKA_TRANSACTION_NAMING = true; + + public static void requireStateTtlHint() { + Assumptions.assumeTrue( + STATE_TTL_HINTS, + "STATE_TTL hints are absent from Flink 1.18; global retention is tested separately"); + } + + public static void requireFirstLastType(String type) { + boolean temporal = type.equals("DATE") || type.startsWith("TIMESTAMP"); + Assumptions.assumeTrue( + !temporal || TEMPORAL_FIRST_LAST, + "Flink 1.18 cannot plan FIRST_VALUE/LAST_VALUE over " + type); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/FlinkTestSources.java b/src/test/java-flink2.2/tech/streamfusion/compat/FlinkTestSources.java new file mode 100644 index 000000000..de5aea946 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/FlinkTestSources.java @@ -0,0 +1,35 @@ +package tech.streamfusion.compat; + +import java.util.Collection; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.DataStreamSource; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +/** Uses the selected host's collection-source API with the same elements and type information. */ +public final class FlinkTestSources { + private FlinkTestSources() {} + + public static DataStreamSource fromData(StreamExecutionEnvironment env, T... data) { + return env.fromData(data); + } + + public static DataStreamSource fromData( + StreamExecutionEnvironment env, TypeInformation type, T... data) { + return env.fromData(type, data); + } + + public static DataStreamSource fromData( + StreamExecutionEnvironment env, Collection data, TypeInformation type) { + return env.fromData(data, type); + } + + public static DataStreamSource fromData( + StreamExecutionEnvironment env, Class type, T... data) { + return env.fromData(type, data); + } + + public static DataStreamSource fromData( + StreamExecutionEnvironment env, Collection data) { + return env.fromData(data); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/JsonTestSchemas.java b/src/test/java-flink2.2/tech/streamfusion/compat/JsonTestSchemas.java new file mode 100644 index 000000000..d6428c44e --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/JsonTestSchemas.java @@ -0,0 +1,66 @@ +package tech.streamfusion.compat; + +import org.apache.flink.formats.common.TimestampFormat; +import org.apache.flink.formats.json.*; +import org.apache.flink.table.types.logical.RowType; + +public final class JsonTestSchemas { + private JsonTestSchemas() {} + + public static final boolean[] IGNORE_NULL_FIELD_MODES = new boolean[] {false, true}; + + public static org.apache.flink.formats.json.JsonRowDataSerializationSchema json( + RowType type, + TimestampFormat timestamps, + JsonFormatOptions.MapNullKeyMode mode, + String literal, + boolean plainDecimal, + boolean ignoreNulls) { + return new org.apache.flink.formats.json.JsonRowDataSerializationSchema( + type, timestamps, mode, literal, plainDecimal, ignoreNulls); + } + + public static org.apache.flink.formats.json.debezium.DebeziumJsonSerializationSchema debezium( + RowType type, + TimestampFormat timestamps, + JsonFormatOptions.MapNullKeyMode mode, + String literal, + boolean plainDecimal, + boolean ignoreNulls) { + return new org.apache.flink.formats.json.debezium.DebeziumJsonSerializationSchema( + type, timestamps, mode, literal, plainDecimal, ignoreNulls); + } + + public static org.apache.flink.formats.json.canal.CanalJsonSerializationSchema canal( + RowType type, + TimestampFormat timestamps, + JsonFormatOptions.MapNullKeyMode mode, + String literal, + boolean plainDecimal, + boolean ignoreNulls) { + return new org.apache.flink.formats.json.canal.CanalJsonSerializationSchema( + type, timestamps, mode, literal, plainDecimal, ignoreNulls); + } + + public static org.apache.flink.formats.json.maxwell.MaxwellJsonSerializationSchema maxwell( + RowType type, + TimestampFormat timestamps, + JsonFormatOptions.MapNullKeyMode mode, + String literal, + boolean plainDecimal, + boolean ignoreNulls) { + return new org.apache.flink.formats.json.maxwell.MaxwellJsonSerializationSchema( + type, timestamps, mode, literal, plainDecimal, ignoreNulls); + } + + public static org.apache.flink.formats.json.ogg.OggJsonSerializationSchema ogg( + RowType type, + TimestampFormat timestamps, + JsonFormatOptions.MapNullKeyMode mode, + String literal, + boolean plainDecimal, + boolean ignoreNulls) { + return new org.apache.flink.formats.json.ogg.OggJsonSerializationSchema( + type, timestamps, mode, literal, plainDecimal, ignoreNulls); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/RichAsyncFunction.java b/src/test/java-flink2.2/tech/streamfusion/compat/RichAsyncFunction.java new file mode 100644 index 000000000..df919f7e6 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/RichAsyncFunction.java @@ -0,0 +1,12 @@ +package tech.streamfusion.compat; + +public abstract class RichAsyncFunction + extends org.apache.flink.streaming.api.functions.async.RichAsyncFunction { + @Override + public final void open(org.apache.flink.api.common.functions.OpenContext context) + throws Exception { + initialize(); + } + + protected abstract void initialize() throws Exception; +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/RichMapFunction.java b/src/test/java-flink2.2/tech/streamfusion/compat/RichMapFunction.java new file mode 100644 index 000000000..5bbd9dbff --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/RichMapFunction.java @@ -0,0 +1,12 @@ +package tech.streamfusion.compat; + +public abstract class RichMapFunction + extends org.apache.flink.api.common.functions.RichMapFunction { + @Override + public final void open(org.apache.flink.api.common.functions.OpenContext context) + throws Exception { + initialize(); + } + + protected abstract void initialize() throws Exception; +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/RichSinkFunction.java b/src/test/java-flink2.2/tech/streamfusion/compat/RichSinkFunction.java new file mode 100644 index 000000000..76d127093 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/RichSinkFunction.java @@ -0,0 +1,4 @@ +package tech.streamfusion.compat; + +public abstract class RichSinkFunction + extends org.apache.flink.streaming.api.functions.sink.legacy.RichSinkFunction {} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/SinkFunctionProvider.java b/src/test/java-flink2.2/tech/streamfusion/compat/SinkFunctionProvider.java new file mode 100644 index 000000000..5ab7bb8db --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/SinkFunctionProvider.java @@ -0,0 +1,12 @@ +package tech.streamfusion.compat; + +public final class SinkFunctionProvider { + private SinkFunctionProvider() {} + + public static org.apache.flink.table.connector.sink.DynamicTableSink.SinkRuntimeProvider of( + org.apache.flink.streaming.api.functions.sink.legacy.SinkFunction< + org.apache.flink.table.data.RowData> + function) { + return org.apache.flink.table.connector.sink.legacy.SinkFunctionProvider.of(function); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/SourceFunction.java b/src/test/java-flink2.2/tech/streamfusion/compat/SourceFunction.java new file mode 100644 index 000000000..8d3bec758 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/SourceFunction.java @@ -0,0 +1,4 @@ +package tech.streamfusion.compat; + +public interface SourceFunction + extends org.apache.flink.streaming.api.functions.source.legacy.SourceFunction {} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/StreamTestSinks.java b/src/test/java-flink2.2/tech/streamfusion/compat/StreamTestSinks.java new file mode 100644 index 000000000..dc915ea05 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/StreamTestSinks.java @@ -0,0 +1,9 @@ +package tech.streamfusion.compat; + +public final class StreamTestSinks { + private StreamTestSinks() {} + + public static void discard(org.apache.flink.streaming.api.datastream.DataStream stream) { + stream.sinkTo(new org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink<>()); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/TestOutput.java b/src/test/java-flink2.2/tech/streamfusion/compat/TestOutput.java new file mode 100644 index 000000000..e0976f3d6 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/TestOutput.java @@ -0,0 +1,10 @@ +package tech.streamfusion.compat; + +public abstract class TestOutput implements org.apache.flink.streaming.api.operators.Output { + @Override + public void emitWatermark(org.apache.flink.runtime.event.WatermarkEvent event) {} + + @Override + public void emitRecordAttributes( + org.apache.flink.streaming.runtime.streamrecord.RecordAttributes attributes) {} +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/WatermarkTestOutputs.java b/src/test/java-flink2.2/tech/streamfusion/compat/WatermarkTestOutputs.java new file mode 100644 index 000000000..8e865d70f --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/WatermarkTestOutputs.java @@ -0,0 +1,10 @@ +package tech.streamfusion.compat; + +public final class WatermarkTestOutputs { + private WatermarkTestOutputs() {} + + public static void register( + org.apache.flink.api.common.eventtime.WatermarkOutputMultiplexer multiplexer, String id) { + multiplexer.registerNewOutput(id); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/compat/WindowTestAssigner.java b/src/test/java-flink2.2/tech/streamfusion/compat/WindowTestAssigner.java new file mode 100644 index 000000000..cd9574745 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/compat/WindowTestAssigner.java @@ -0,0 +1,21 @@ +package tech.streamfusion.compat; + +import java.time.Duration; +import java.util.Collection; +import org.apache.flink.table.runtime.operators.window.TimeWindow; +import org.apache.flink.table.runtime.operators.window.groupwindow.assigners.*; + +public final class WindowTestAssigner { + private WindowTestAssigner() {} + + public static Collection assign(int kind, long size, long timestamp) + throws java.io.IOException { + GroupWindowAssigner assigner = + kind == 0 + ? TumblingWindowAssigner.of(Duration.ofMillis(size)) + : kind == 1 + ? SlidingWindowAssigner.of(Duration.ofMillis(size), Duration.ofMillis(1000)) + : CumulativeWindowAssigner.of(Duration.ofMillis(size), Duration.ofMillis(1000)); + return assigner.assignWindows(null, timestamp); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/format/avro/compat/AvroTestSchemas.java b/src/test/java-flink2.2/tech/streamfusion/format/avro/compat/AvroTestSchemas.java new file mode 100644 index 000000000..873426dd2 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/format/avro/compat/AvroTestSchemas.java @@ -0,0 +1,28 @@ +package tech.streamfusion.format.avro.compat; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.formats.avro.*; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; + +public final class AvroTestSchemas { + private AvroTestSchemas() {} + + public static final boolean CORRECTED_TIMESTAMP_MAPPING = true; + + public static AvroRowDataDeserializationSchema decoder( + RowType type, TypeInformation info, boolean legacy) { + org.junit.jupiter.api.Assumptions.assumeTrue( + legacy || CORRECTED_TIMESTAMP_MAPPING, + "Corrected Avro timestamp mapping is absent from Flink 1.18"); + return new AvroRowDataDeserializationSchema( + type, info, AvroFormatOptions.AvroEncoding.BINARY, legacy); + } + + public static AvroRowDataSerializationSchema encoder(RowType type, boolean legacy) { + org.junit.jupiter.api.Assumptions.assumeTrue( + legacy || CORRECTED_TIMESTAMP_MAPPING, + "Corrected Avro timestamp mapping is absent from Flink 1.18"); + return new AvroRowDataSerializationSchema(type, AvroFormatOptions.AvroEncoding.BINARY, legacy); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/kafka/KafkaLineageAssertions.java b/src/test/java-flink2.2/tech/streamfusion/kafka/KafkaLineageAssertions.java new file mode 100644 index 000000000..ecca9ddb1 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/kafka/KafkaLineageAssertions.java @@ -0,0 +1,11 @@ +package tech.streamfusion.kafka; + +final class KafkaLineageAssertions { + private KafkaLineageAssertions() {} + + static void assertTopic(PreSerializedKafkaRecordSchema schema, String topic) { + org.junit.jupiter.api.Assertions.assertEquals( + java.util.List.of(topic), + schema.getKafkaDatasetFacet().orElseThrow().getTopicIdentifier().getTopics()); + } +} diff --git a/src/test/java-flink2.2/tech/streamfusion/operator/ColumnarExchangeAlignmentFlagTest.java b/src/test/java-flink2.2/tech/streamfusion/operator/ColumnarExchangeAlignmentFlagTest.java new file mode 100644 index 000000000..38c488f00 --- /dev/null +++ b/src/test/java-flink2.2/tech/streamfusion/operator/ColumnarExchangeAlignmentFlagTest.java @@ -0,0 +1,59 @@ +package tech.streamfusion.operator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.flink.runtime.io.network.api.writer.SubtaskStateMapper; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.graph.StreamEdge; +import org.apache.flink.streaming.api.transformations.PartitionTransformation; +import org.apache.flink.streaming.api.transformations.StreamExchangeMode; +import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.ColumnarKeyGroupPartitioner; + +class ColumnarExchangeAlignmentFlagTest { + @Test + void partitionerUsesKeyGroupRangeRecoveryAndRequiresAlignedChannelState() { + ColumnarKeyGroupPartitioner partitioner = new ColumnarKeyGroupPartitioner(128); + assertEquals(SubtaskStateMapper.RANGE, partitioner.getDownstreamSubtaskStateMapper()); + assertFalse( + partitioner.isSupportsUnalignedCheckpoint(), + "a channel batch can span key groups that separate after rescaling"); + } + + @Test + void recoverablePartitionerSupportsUnalignedRangeChannelState() { + ColumnarKeyGroupPartitioner partitioner = new ColumnarKeyGroupPartitioner(128, true); + assertEquals(SubtaskStateMapper.RANGE, partitioner.getDownstreamSubtaskStateMapper()); + assertTrue(partitioner.isSupportsUnalignedCheckpoint()); + assertTrue(partitioner.copy().isSupportsUnalignedCheckpoint()); + } + + @Test + void streamGraphForcesAlignedColumnarExchange() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.enableCheckpointing(10); + env.getCheckpointConfig().enableUnalignedCheckpoints(); + DataStream input = + tech.streamfusion.compat.FlinkTestSources.fromData(env, 1) + .map(ignored -> (ArrowBatch) null) + .returns(ArrowBatchTypeInformation.INSTANCE); + PartitionTransformation partition = + new PartitionTransformation<>( + input.getTransformation(), + new ColumnarKeyGroupPartitioner(128), + StreamExchangeMode.PIPELINED); + tech.streamfusion.compat.StreamTestSinks.discard(new DataStream<>(env, partition)); + + List columnarEdges = + env.getStreamGraph().getStreamNodes().stream() + .flatMap(node -> node.getOutEdges().stream()) + .filter(edge -> edge.getPartitioner() instanceof ColumnarKeyGroupPartitioner) + .toList(); + assertEquals(1, columnarEdges.size()); + assertFalse(columnarEdges.get(0).supportsUnalignedCheckpoints()); + } +} diff --git a/src/test/java/tech/streamfusion/operator/NativeJsonRuntimeTest.java b/src/test/java-flink2.2/tech/streamfusion/operator/NativeJsonRuntimeTest.java similarity index 100% rename from src/test/java/tech/streamfusion/operator/NativeJsonRuntimeTest.java rename to src/test/java-flink2.2/tech/streamfusion/operator/NativeJsonRuntimeTest.java diff --git a/src/test/java/tech/streamfusion/planner/DeltaJoinForceGateTest.java b/src/test/java-flink2.2/tech/streamfusion/planner/DeltaJoinForceGateTest.java similarity index 89% rename from src/test/java/tech/streamfusion/planner/DeltaJoinForceGateTest.java rename to src/test/java-flink2.2/tech/streamfusion/planner/DeltaJoinForceGateTest.java index 727744c8f..92cd7363e 100644 --- a/src/test/java/tech/streamfusion/planner/DeltaJoinForceGateTest.java +++ b/src/test/java-flink2.2/tech/streamfusion/planner/DeltaJoinForceGateTest.java @@ -46,23 +46,31 @@ void forceWithoutADeltaJoinLeavesThePlanForFlinkToReject() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream a = - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 20L)); DataStream b = - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED(new String[] {"k", "w"}, Types.LONG, Types.LONG), Row.of(1L, 100L), Row.of(2L, 200L)); tEnv.createTemporaryView( "A", a, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); tEnv.createTemporaryView( "B", b, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("w", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("w", DataTypes.BIGINT()) + .build()); PhysicalPlanScan scan = NativePlanner.install(tEnv); tEnv.getConfig() .set( @@ -90,14 +98,18 @@ void forceStillAcceleratesAPlanWithoutAJoin() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream a = - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 20L)); tEnv.createTemporaryView( "A", a, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); PhysicalPlanScan scan = NativePlanner.install(tEnv); tEnv.getConfig() .set( @@ -115,23 +127,31 @@ void theDefaultStrategyAcceleratesTheSameJoin() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream a = - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 20L)); DataStream b = - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED(new String[] {"k", "w"}, Types.LONG, Types.LONG), Row.of(1L, 100L), Row.of(2L, 200L)); tEnv.createTemporaryView( "A", a, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); tEnv.createTemporaryView( "B", b, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("w", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("w", DataTypes.BIGINT()) + .build()); PhysicalPlanScan scan = NativePlanner.install(tEnv); tEnv.explainSql(JOIN_QUERY); @@ -148,23 +168,31 @@ void noneStrategyAcceleratesTheSameJoin() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream a = - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 20L)); DataStream b = - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED(new String[] {"k", "w"}, Types.LONG, Types.LONG), Row.of(1L, 100L), Row.of(2L, 200L)); tEnv.createTemporaryView( "A", a, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); tEnv.createTemporaryView( "B", b, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("w", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("w", DataTypes.BIGINT()) + .build()); PhysicalPlanScan scan = NativePlanner.install(tEnv); tEnv.getConfig() .set( diff --git a/src/test/java/tech/streamfusion/state/FlinkRocksDBOptionsTest.java b/src/test/java-flink2.2/tech/streamfusion/state/FlinkRocksDBOptionsTest.java similarity index 100% rename from src/test/java/tech/streamfusion/state/FlinkRocksDBOptionsTest.java rename to src/test/java-flink2.2/tech/streamfusion/state/FlinkRocksDBOptionsTest.java diff --git a/src/test/java/tech/streamfusion/ArrayNaNKeyParityTest.java b/src/test/java/tech/streamfusion/ArrayNaNKeyParityTest.java index 5d37233ea..927876bdc 100644 --- a/src/test/java/tech/streamfusion/ArrayNaNKeyParityTest.java +++ b/src/test/java/tech/streamfusion/ArrayNaNKeyParityTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -42,20 +44,32 @@ private static TableEnvironment input(boolean canonical, int batch) { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); StreamTableEnvironment table = StreamTableEnvironment.create(env); - table.getConfig().set("table.optimizer.agg-phase-strategy", batch == 0 ? "ONE_PHASE" : "TWO_PHASE"); + table + .getConfig() + .set("table.optimizer.agg-phase-strategy", batch == 0 ? "ONE_PHASE" : "TWO_PHASE"); if (batch > 0) { table.getConfig().set("table.exec.mini-batch.enabled", "true"); table.getConfig().set("table.exec.mini-batch.allow-latency", "100000 d"); table.getConfig().set("table.exec.mini-batch.size", Integer.toString(batch)); } // Construct the NaNs after source deserialization, which may otherwise canonicalize their bits. - table.createTemporaryView("n", env.fromData(1, 2).map(id -> { - double d = Double.longBitsToDouble(0x7ff8000000000000L + (canonical ? 0 : id)); - float f = Float.intBitsToFloat(0x7fc00000 + (canonical ? 0 : id)); - return Row.of(id, d, f, new Double[] {d}, new Float[] {f}); - }).returns(Types.ROW_NAMED(new String[] {"id", "d", "f", "da", "fa"}, - Types.INT, Types.DOUBLE, Types.FLOAT, - Types.OBJECT_ARRAY(Types.DOUBLE), Types.OBJECT_ARRAY(Types.FLOAT)))); + table.createTemporaryView( + "n", + fromData(env, 1, 2) + .map( + id -> { + double d = Double.longBitsToDouble(0x7ff8000000000000L + (canonical ? 0 : id)); + float f = Float.intBitsToFloat(0x7fc00000 + (canonical ? 0 : id)); + return Row.of(id, d, f, new Double[] {d}, new Float[] {f}); + }) + .returns( + Types.ROW_NAMED( + new String[] {"id", "d", "f", "da", "fa"}, + Types.INT, + Types.DOUBLE, + Types.FLOAT, + Types.OBJECT_ARRAY(Types.DOUBLE), + Types.OBJECT_ARRAY(Types.FLOAT)))); return table; } } diff --git a/src/test/java/tech/streamfusion/ChangelogParquetTableFactory.java b/src/test/java/tech/streamfusion/ChangelogParquetTableFactory.java index 4ac638af5..b845f788b 100644 --- a/src/test/java/tech/streamfusion/ChangelogParquetTableFactory.java +++ b/src/test/java/tech/streamfusion/ChangelogParquetTableFactory.java @@ -4,8 +4,6 @@ import java.util.Collections; import java.util.List; import java.util.Set; -import org.apache.flink.api.common.functions.OpenContext; -import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.connector.file.sink.FileSink; @@ -24,6 +22,7 @@ import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.VarCharType; import org.apache.hadoop.conf.Configuration; +import tech.streamfusion.compat.RichMapFunction; /** Benchmark sink that persists every change as a Parquet row with its {@code RowKind}. */ public final class ChangelogParquetTableFactory implements DynamicTableSinkFactory { @@ -82,21 +81,22 @@ public SinkRuntimeProvider getSinkRuntimeProvider(Context context) { ParquetRowDataBuilder.createWriterFactory( outputType, parquetConfiguration(), true)) .build(); - return (DataStreamSinkProvider) - (providerContext, input) -> { - SingleOutputStreamOperator changes = - input - .map(new AppendRowKind(inputType)) - .returns(InternalTypeInfo.of(outputType)) - .name("Append Parquet changelog row kind"); - providerContext - .generateUid("changelog-parquet-row-kind") - .ifPresent(changes::uid); - DataStreamSink files = - changes.sinkTo(sink).name("Changelog Parquet files"); - providerContext.generateUid("changelog-parquet-files").ifPresent(files::uid); - return files; - }; + return new DataStreamSinkProvider() { + @Override + public DataStreamSink consumeDataStream( + org.apache.flink.table.connector.ProviderContext providerContext, + org.apache.flink.streaming.api.datastream.DataStream input) { + SingleOutputStreamOperator changes = + input + .map(new AppendRowKind(inputType)) + .returns(InternalTypeInfo.of(outputType)) + .name("Append Parquet changelog row kind"); + providerContext.generateUid("changelog-parquet-row-kind").ifPresent(changes::uid); + DataStreamSink files = changes.sinkTo(sink).name("Changelog Parquet files"); + providerContext.generateUid("changelog-parquet-files").ifPresent(files::uid); + return files; + } + }; } @Override @@ -132,7 +132,7 @@ private AppendRowKind(RowType inputType) { } @Override - public void open(OpenContext openContext) { + protected void initialize() { getters = new RowData.FieldGetter[inputType.getFieldCount()]; for (int i = 0; i < getters.length; i++) { getters[i] = RowData.createFieldGetter(inputType.getTypeAt(i), i); diff --git a/src/test/java/tech/streamfusion/ChangelogParquetTableFactoryTest.java b/src/test/java/tech/streamfusion/ChangelogParquetTableFactoryTest.java index b69af0d28..f5a3f4d67 100644 --- a/src/test/java/tech/streamfusion/ChangelogParquetTableFactoryTest.java +++ b/src/test/java/tech/streamfusion/ChangelogParquetTableFactoryTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.nio.file.Files; import java.nio.file.Path; @@ -38,7 +39,8 @@ private static List writeInsertOnlyAndRead() throws Exception { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.createTemporaryView( "insert_rows", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 20L)), @@ -79,7 +81,8 @@ private static List writeAndRead(boolean nativeSink) throws Exception { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.createTemporaryView( "input_rows", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(1L, 20L), diff --git a/src/test/java/tech/streamfusion/CountingBlackholeTableFactory.java b/src/test/java/tech/streamfusion/CountingBlackholeTableFactory.java index 159252e0f..20b3ea19e 100644 --- a/src/test/java/tech/streamfusion/CountingBlackholeTableFactory.java +++ b/src/test/java/tech/streamfusion/CountingBlackholeTableFactory.java @@ -5,12 +5,12 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; import org.apache.flink.configuration.ConfigOption; -import org.apache.flink.streaming.api.functions.sink.legacy.RichSinkFunction; import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.connector.sink.DynamicTableSink; -import org.apache.flink.table.connector.sink.legacy.SinkFunctionProvider; import org.apache.flink.table.data.RowData; import org.apache.flink.table.factories.DynamicTableSinkFactory; +import tech.streamfusion.compat.RichSinkFunction; +import tech.streamfusion.compat.SinkFunctionProvider; /** * A blackhole that counts ({@code 'connector' = 'counting-blackhole'}): swallows {@link RowData} diff --git a/src/test/java/tech/streamfusion/DecimalJsonTestInputs.java b/src/test/java/tech/streamfusion/DecimalJsonTestInputs.java index 142e638c8..bc7e2485f 100644 --- a/src/test/java/tech/streamfusion/DecimalJsonTestInputs.java +++ b/src/test/java/tech/streamfusion/DecimalJsonTestInputs.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.math.BigDecimal; import java.util.ArrayList; import org.apache.flink.api.common.typeinfo.Types; @@ -34,7 +36,8 @@ static TableEnvironment decimals(int precision, int scale) { var tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "decimals", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "n"}, Types.INT, Types.BIG_DEC), rows.toArray(Row[]::new)), Schema.newBuilder() diff --git a/src/test/java/tech/streamfusion/DecimalSumRecoveryParityTest.java b/src/test/java/tech/streamfusion/DecimalSumRecoveryParityTest.java index 25f689bce..f524f003e 100644 --- a/src/test/java/tech/streamfusion/DecimalSumRecoveryParityTest.java +++ b/src/test/java/tech/streamfusion/DecimalSumRecoveryParityTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.math.BigDecimal; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -28,12 +30,21 @@ private static TableEnvironment decimals() { var table = StreamTableEnvironment.create(env); table.getConfig().set("table.optimizer.agg-phase-strategy", "ONE_PHASE"); BigDecimal max = new BigDecimal("99999999999999999999999999999999999.999"); - table.createTemporaryView("n", env.fromData( - Types.ROW_NAMED(new String[] {"g", "d", "keep_value"}, Types.INT, Types.BIG_DEC, Types.BOOLEAN), - Row.of(1, max, true), Row.of(1, max, true), Row.of(1, null, true), - Row.of(1, BigDecimal.ONE, true)), - Schema.newBuilder().column("g", DataTypes.INT()).column("d", DataTypes.DECIMAL(38, 3)) - .column("keep_value", DataTypes.BOOLEAN()).build()); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED( + new String[] {"g", "d", "keep_value"}, Types.INT, Types.BIG_DEC, Types.BOOLEAN), + Row.of(1, max, true), + Row.of(1, max, true), + Row.of(1, null, true), + Row.of(1, BigDecimal.ONE, true)), + Schema.newBuilder() + .column("g", DataTypes.INT()) + .column("d", DataTypes.DECIMAL(38, 3)) + .column("keep_value", DataTypes.BOOLEAN()) + .build()); return table; } } diff --git a/src/test/java/tech/streamfusion/FlinkAsyncLookupJoinSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkAsyncLookupJoinSqlHarnessTest.java index b2d55b0e5..adb73543d 100644 --- a/src/test/java/tech/streamfusion/FlinkAsyncLookupJoinSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkAsyncLookupJoinSqlHarnessTest.java @@ -17,6 +17,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import tech.streamfusion.compat.FlinkTestCapabilities; +import tech.streamfusion.compat.FlinkTestSources; /** * Nexmark q13's shape against an async lookup connector: the planner picks the async path, and @@ -107,6 +109,9 @@ void asynchronousLookupMissRetriesUseFlinksRetryWrapper() { @ParameterizedTest @ValueSource(booleans = {false, true}) void updatingProbesRetainFlinksChangelogAndKeyedScheduling(boolean keyOrdered) throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + !keyOrdered || FlinkTestCapabilities.KEY_ORDERED_ASYNC_LOOKUP, + "Flink 1.18 has no key-ordered async lookup mode"); NativeParity.assertFallbackReasonContains( () -> { var env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -117,7 +122,8 @@ void updatingProbesRetainFlinksChangelogAndKeyedScheduling(boolean keyOrdered) t .set("table.exec.async-lookup.key-ordered-enabled", Boolean.toString(keyOrdered)); table.getConfig().set("table.exec.async-lookup.output-mode", "ALLOW_UNORDERED"); var changes = - env.fromData( + FlinkTestSources.fromData( + env, Types.ROW_NAMED(new String[] {"auction", "price"}, Types.LONG, Types.LONG), Row.ofKind(RowKind.INSERT, 1L, 100L), Row.ofKind(RowKind.DELETE, 1L, 100L), @@ -157,7 +163,8 @@ private static Supplier environment(String mode) { tEnv.getConfig().set("table.exec.async-lookup.timeout", "30 ms"); } DataStream bid = - env.fromData( + FlinkTestSources.fromData( + env, Types.ROW_NAMED(new String[] {"auction", "price"}, Types.LONG, Types.LONG), Row.of(1L, 100L), Row.of(2L, 200L), diff --git a/src/test/java/tech/streamfusion/FlinkBtrimSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkBtrimSqlHarnessTest.java index 7aa886045..1302ef65f 100644 --- a/src/test/java/tech/streamfusion/FlinkBtrimSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkBtrimSqlHarnessTest.java @@ -3,6 +3,11 @@ import org.junit.jupiter.api.Test; class FlinkBtrimSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("BTRIM"); + } + @Test void btrimUsesCharacterSetsAndPreservesNonSpaceWhitespace() throws Exception { parity( diff --git a/src/test/java/tech/streamfusion/FlinkCalcSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkCalcSqlHarnessTest.java index 6822ac5b5..2459ec0b2 100644 --- a/src/test/java/tech/streamfusion/FlinkCalcSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkCalcSqlHarnessTest.java @@ -1,9 +1,9 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.time.Instant; -import tech.streamfusion.planner.NativePlanner; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -13,6 +13,7 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.NativePlanner; /** General Calc (computed/constant projections, with and without a filter) matches the host. */ class FlinkCalcSqlHarnessTest { @@ -38,7 +39,8 @@ void floatLiteralArithmeticMatchesHost() throws Exception { @Test void floatColumnArithmeticMatchesHost() throws Exception { - NativeParity.assertParity(FlinkCalcSqlHarnessTest::environment, "SELECT f4 + f4, f4 - f4 FROM f"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::environment, "SELECT f4 + f4, f4 - f4 FROM f"); } @Test @@ -99,7 +101,8 @@ void caseInFilterMatchesHost() throws Exception { @Test void wideningCastMatchesHost() throws Exception { - NativeParity.assertParity(FlinkCalcSqlHarnessTest::environment, "SELECT CAST(v AS BIGINT) FROM f"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::environment, "SELECT CAST(v AS BIGINT) FROM f"); } @Test @@ -111,7 +114,8 @@ void wideningTimestampPrecisionCastMatchesHost() throws Exception { @Test void castToDoubleMatchesHost() throws Exception { - NativeParity.assertParity(FlinkCalcSqlHarnessTest::environment, "SELECT CAST(v AS DOUBLE) FROM f"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::environment, "SELECT CAST(v AS DOUBLE) FROM f"); } @Test @@ -123,10 +127,14 @@ void caseWithMixedWidthBranchesMatchesHost() throws Exception { @Test void narrowingIntCastMatchesHost() throws Exception { - // BIGINT → INT narrows: Flink's primitive Java cast truncates to the low 32 bits (two's-complement - // wraparound), which the native wrapping kernel reproduces. The environment includes values past - // the INT range (2^31 and 2^32+1), so this pins the overflow behavior, not just the in-range case. - NativeParity.assertParity(FlinkCalcSqlHarnessTest::castEnvironment, "SELECT CAST(big AS INT) FROM c"); + // BIGINT → INT narrows: Flink's primitive Java cast truncates to the low 32 bits + // (two's-complement + // wraparound), which the native wrapping kernel reproduces. The environment includes values + // past + // the INT range (2^31 and 2^32+1), so this pins the overflow behavior, not just the in-range + // case. + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::castEnvironment, "SELECT CAST(big AS INT) FROM c"); } @Test @@ -139,9 +147,11 @@ void narrowingIntCastToTinyintMatchesHost() throws Exception { @Test void floatToIntCastMatchesHost() throws Exception { // DOUBLE → INT: Flink's primitive Java cast rounds toward zero and saturates to the INT range - // (NaN → 0, ±overflow → Integer.MIN/MAX), which Rust's `as` reproduces. The environment includes + // (NaN → 0, ±overflow → Integer.MIN/MAX), which Rust's `as` reproduces. The environment + // includes // NaN, ±infinity and ±1e20 to pin the saturation edges. - NativeParity.assertParity(FlinkCalcSqlHarnessTest::castEnvironment, "SELECT CAST(dbl AS INT) FROM c"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::castEnvironment, "SELECT CAST(dbl AS INT) FROM c"); } @Test @@ -154,18 +164,21 @@ void unsupportedProjectionFunctionFallsBack() throws Exception { @Test void isNullFilterMatchesHost() throws Exception { - NativeParity.assertParity(FlinkCalcSqlHarnessTest::nullableEnvironment, "SELECT k FROM g WHERE s IS NULL"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::nullableEnvironment, "SELECT k FROM g WHERE s IS NULL"); } @Test void isNotNullFilterMatchesHost() throws Exception { - NativeParity.assertParity(FlinkCalcSqlHarnessTest::nullableEnvironment, "SELECT k FROM g WHERE v IS NOT NULL"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::nullableEnvironment, "SELECT k FROM g WHERE v IS NOT NULL"); } @Test void coalesceMatchesHost() throws Exception { // COALESCE lowers to CASE in sql-to-rel, so it rides the admitted CASE path (numeric branches). - NativeParity.assertParity(FlinkCalcSqlHarnessTest::nullableEnvironment, "SELECT COALESCE(v, 0) FROM g"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::nullableEnvironment, "SELECT COALESCE(v, 0) FROM g"); } @Test @@ -186,7 +199,8 @@ void charToVarcharCastMatchesHost() throws Exception { @Test void nullifMatchesHost() throws Exception { // NULLIF lowers to CASE WHEN a = b THEN NULL ELSE a, exercising a NULL literal in a branch. - NativeParity.assertParity(FlinkCalcSqlHarnessTest::nullableEnvironment, "SELECT NULLIF(v, 30) FROM g"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::nullableEnvironment, "SELECT NULLIF(v, 30) FROM g"); } @Test @@ -210,7 +224,8 @@ void moduloMatchesHost() throws Exception { @Test void divisionInFilterMatchesHost() throws Exception { - NativeParity.assertParity(FlinkCalcSqlHarnessTest::environment, "SELECT k FROM f WHERE v / 3 > 5"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::environment, "SELECT k FROM f WHERE v / 3 > 5"); } @Test @@ -266,7 +281,8 @@ void concatMatchesHost() throws Exception { @Test void trimMatchesHost() throws Exception { // Default whitespace both-sides trim maps to DataFusion btrim; spaced values exercise it. - NativeParity.assertParity(FlinkCalcSqlHarnessTest::spacedStringEnvironment, "SELECT TRIM(s) FROM ss"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::spacedStringEnvironment, "SELECT TRIM(s) FROM ss"); } @Test @@ -284,13 +300,15 @@ void substringFromMatchesHost() throws Exception { @Test void substringFromForMatchesHost() throws Exception { NativeParity.assertParity( - FlinkCalcSqlHarnessTest::spacedStringEnvironment, "SELECT SUBSTRING(s FROM 2 FOR 3) FROM ss"); + FlinkCalcSqlHarnessTest::spacedStringEnvironment, + "SELECT SUBSTRING(s FROM 2 FOR 3) FROM ss"); } @Test void substringStartBelowOneMatchesHost() throws Exception { NativeParity.assertParity( - FlinkCalcSqlHarnessTest::spacedStringEnvironment, "SELECT SUBSTRING(s FROM 0 FOR 3) FROM ss"); + FlinkCalcSqlHarnessTest::spacedStringEnvironment, + "SELECT SUBSTRING(s FROM 0 FOR 3) FROM ss"); } @Test @@ -301,7 +319,8 @@ void substringRuntimePositionMatchesHost() throws Exception { @Test void likeFilterMatchesHost() throws Exception { - NativeParity.assertParity(FlinkCalcSqlHarnessTest::environment, "SELECT k FROM f WHERE s LIKE '%a%'"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::environment, "SELECT k FROM f WHERE s LIKE '%a%'"); } @Test @@ -322,19 +341,22 @@ void reverseMatchesHost() throws Exception { @Test void ltrimRtrimMatchHost() throws Exception { - NativeParity.assertParity(FlinkCalcSqlHarnessTest::spacedStringEnvironment, "SELECT LTRIM(s), RTRIM(s) FROM ss"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::spacedStringEnvironment, "SELECT LTRIM(s), RTRIM(s) FROM ss"); } @Test void positionMatchesHost() throws Exception { - NativeParity.assertParity(FlinkCalcSqlHarnessTest::environment, "SELECT POSITION('c' IN s) FROM f"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::environment, "SELECT POSITION('c' IN s) FROM f"); } @Test void absFloatMatchesHost() throws Exception { // ABS over a double expression (the E-notation literal forces DOUBLE; goes negative for some // rows). Integer ABS stays on host (overflow edge). - NativeParity.assertParity(FlinkCalcSqlHarnessTest::environment, "SELECT ABS(v - 25.5E0) FROM f"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::environment, "SELECT ABS(v - 25.5E0) FROM f"); } @Test @@ -352,7 +374,8 @@ void floorCeilMatchHost() throws Exception { @Test void signFloatMatchesHost() throws Exception { - NativeParity.assertParity(FlinkCalcSqlHarnessTest::environment, "SELECT SIGN(v - 25.5E0) FROM f"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::environment, "SELECT SIGN(v - 25.5E0) FROM f"); } @Test @@ -371,7 +394,8 @@ void roundRoutesWhenIncompatibleAllowed() throws Exception { System.setProperty("streamfusion.expression.ROUND.allowIncompatible", "true"); try { // Sampled values happen to agree; the flag is the user accepting the input-dependent risk. - NativeParity.assertParity(FlinkCalcSqlHarnessTest::doubleEnvironment, "SELECT ROUND(d, 2) FROM dd"); + NativeParity.assertParity( + FlinkCalcSqlHarnessTest::doubleEnvironment, "SELECT ROUND(d, 2) FROM dd"); } finally { System.clearProperty("streamfusion.expression.ROUND.allowIncompatible"); } @@ -382,7 +406,8 @@ void transcendentalRoutesWhenIncompatibleAllowed() throws Exception { // TAN diverges at the last ULP, so only assert it routes (not value parity) under the flag. System.setProperty("streamfusion.expression.TAN.allowIncompatible", "true"); try { - NativeParity.assertRoutes(FlinkCalcSqlHarnessTest::doubleEnvironment, "SELECT TAN(d) FROM dd"); + NativeParity.assertRoutes( + FlinkCalcSqlHarnessTest::doubleEnvironment, "SELECT TAN(d) FROM dd"); } finally { System.clearProperty("streamfusion.expression.TAN.allowIncompatible"); } @@ -393,7 +418,8 @@ void masterSwitchDisablesNative() throws Exception { // With native acceleration off, a normally-accelerated filter runs entirely on the host. System.setProperty("streamfusion.native.enabled", "false"); try { - NativeParity.assertFallback(FlinkCalcSqlHarnessTest::environment, "SELECT k FROM f WHERE v > 15"); + NativeParity.assertFallback( + FlinkCalcSqlHarnessTest::environment, "SELECT k FROM f WHERE v > 15"); } finally { System.clearProperty("streamfusion.native.enabled"); } @@ -405,7 +431,9 @@ void perOperatorFlagKeepsFilterOnHost() throws Exception { System.setProperty("streamfusion.operator.filter.enabled", "false"); try { NativeParity.assertFallbackReasonContains( - FlinkCalcSqlHarnessTest::environment, "SELECT k FROM f WHERE v > 15", "filter: disabled by config"); + FlinkCalcSqlHarnessTest::environment, + "SELECT k FROM f WHERE v > 15", + "filter: disabled by config"); } finally { System.clearProperty("streamfusion.operator.filter.enabled"); } @@ -416,7 +444,8 @@ void masterFlagEnablesIncompatible() throws Exception { // The blanket flag enables any incompatible function (here SIN) without naming it. System.setProperty("streamfusion.expression.allowIncompatible", "true"); try { - NativeParity.assertRoutes(FlinkCalcSqlHarnessTest::doubleEnvironment, "SELECT SIN(d) FROM dd"); + NativeParity.assertRoutes( + FlinkCalcSqlHarnessTest::doubleEnvironment, "SELECT SIN(d) FROM dd"); } finally { System.clearProperty("streamfusion.expression.allowIncompatible"); } @@ -460,7 +489,8 @@ void leftRightMatchHost() throws Exception { @Test void lpadRpadMatchHost() throws Exception { NativeParity.assertParity( - FlinkCalcSqlHarnessTest::spacedStringEnvironment, "SELECT LPAD(s, 8, '*'), RPAD(s, 8, '*') FROM ss"); + FlinkCalcSqlHarnessTest::spacedStringEnvironment, + "SELECT LPAD(s, 8, '*'), RPAD(s, 8, '*') FROM ss"); } @Test @@ -495,7 +525,8 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "v", "s", "f4"}, Types.LONG, @@ -523,7 +554,8 @@ private static TableEnvironment timestampEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"ts"}, Types.INSTANT), Row.of(Instant.parse("2026-08-21T12:34:56.789Z")), Row.of(Instant.parse("1969-12-31T23:59:59.001Z"))); @@ -539,7 +571,8 @@ private static TableEnvironment doubleEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"d"}, Types.DOUBLE), Row.of(0.5), Row.of(2.5), @@ -556,7 +589,8 @@ private static TableEnvironment spacedStringEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"s"}, Types.STRING), Row.of(" pad "), Row.of("x"), @@ -572,9 +606,9 @@ private static TableEnvironment narrowIntEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( - Types.ROW_NAMED( - new String[] {"a", "b", "c"}, Types.BYTE, Types.BYTE, Types.SHORT), + fromData( + env, + Types.ROW_NAMED(new String[] {"a", "b", "c"}, Types.BYTE, Types.BYTE, Types.SHORT), Row.of((byte) 100, (byte) 100, (short) 300), Row.of((byte) 1, (byte) 2, (short) 3)); tEnv.createTemporaryView( @@ -593,7 +627,8 @@ private static TableEnvironment castEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"big", "dbl"}, Types.LONG, Types.DOUBLE), Row.of(300L, 3.9), Row.of(2147483648L, -3.9), @@ -617,7 +652,8 @@ private static TableEnvironment nullableEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "s"}, Types.LONG, Types.INT, Types.STRING), Row.of(1L, 10, "a"), Row.of(2L, 30, null), diff --git a/src/test/java/tech/streamfusion/FlinkChangelogNormalizeSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkChangelogNormalizeSqlHarnessTest.java index 27ddf0049..5bffbdff4 100644 --- a/src/test/java/tech/streamfusion/FlinkChangelogNormalizeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkChangelogNormalizeSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.DataTypes; @@ -76,7 +78,8 @@ private static TableEnvironment environment() { // An upsert stream keyed by f0: insert, update (same key), insert another key, then a delete, // and a no-op update (same value) to exercise the unchanged-row suppression. DataStream source = - env.fromData( + fromData( + env, Row.ofKind(RowKind.INSERT, 1L, 10L), Row.ofKind(RowKind.UPDATE_AFTER, 1L, 20L), Row.ofKind(RowKind.INSERT, 2L, 5L), diff --git a/src/test/java/tech/streamfusion/FlinkChangingTopNSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkChangingTopNSqlHarnessTest.java index 9edc12f0a..863c4f1eb 100644 --- a/src/test/java/tech/streamfusion/FlinkChangingTopNSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkChangingTopNSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.ArrayList; import java.util.List; @@ -157,7 +158,8 @@ private static TableEnvironment environment( table.getConfig().set("table.exec.mini-batch.size", "37"); } var source = - env.fromData( + fromData( + env, rows, Types.ROW_NAMED( new String[] {"k", "score", "bound"}, Types.LONG, Types.LONG, Types.LONG)); diff --git a/src/test/java/tech/streamfusion/FlinkCoalesceEvaluationSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkCoalesceEvaluationSqlHarnessTest.java index 61905b676..d408f0217 100644 --- a/src/test/java/tech/streamfusion/FlinkCoalesceEvaluationSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkCoalesceEvaluationSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.stream.Stream; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -67,7 +69,7 @@ static TableEnvironment environment(int count) { Row[] rows = new Row[count]; for (int i = 0; i < count; i++) rows[i] = Row.of(i + 1); table.createTemporaryView( - "src", env.fromData(Types.ROW_NAMED(new String[] {"id"}, Types.INT), rows)); + "src", fromData(env, Types.ROW_NAMED(new String[] {"id"}, Types.INT), rows)); table.createTemporarySystemFunction("tick", new Tick()); table.createTemporarySystemFunction("text_tick", new TextTick()); table.createTemporarySystemFunction("null_tick", new NullTick()); diff --git a/src/test/java/tech/streamfusion/FlinkColumnarWindowSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkColumnarWindowSqlHarnessTest.java index a14903ae4..aa5478ebc 100644 --- a/src/test/java/tech/streamfusion/FlinkColumnarWindowSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkColumnarWindowSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.nio.file.Files; import java.nio.file.Path; @@ -104,6 +105,7 @@ void rowTimeMiniBatchWindowMatchesHost() throws Exception { @Test void keyedSessionOverColumnarSourceMatchesHost() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); Path input = Files.createTempDirectory("csession-in"); writeInput(input); // After the source-edge transpose: watermark assigner → columnar keyed exchange → columnar @@ -190,6 +192,7 @@ void proctimeCumulateWindowRoutesToNative() throws Exception { @Test void proctimeSessionWindowRoutesToNative() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); // A proctime SESSION window: the gap is timed on the clock and a session closes on a // processing-time timer at the last element's `now + gap`. Non-deterministic boundaries (see the // CLAUDE.md note) — assert it routes and runs; NativeColumnarSessionWindowAggregateOperatorTest @@ -292,7 +295,8 @@ private static TableEnvironment proctimeEnvironment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.getConfig().setLocalTimeZone(ZoneId.of("UTC")); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 20L), diff --git a/src/test/java/tech/streamfusion/FlinkComplexTypeSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkComplexTypeSqlHarnessTest.java index d4923c4fc..4e9115d41 100644 --- a/src/test/java/tech/streamfusion/FlinkComplexTypeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkComplexTypeSqlHarnessTest.java @@ -2,9 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; -import tech.streamfusion.planner.NativePlanner; -import tech.streamfusion.planner.PhysicalPlanScan; import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Comparator; @@ -22,6 +21,8 @@ import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.NativePlanner; +import tech.streamfusion.planner.PhysicalPlanScan; /** * A filter/projection that passes a complex (ARRAY/MAP/ROW) column through matches the host. The @@ -197,7 +198,8 @@ private static TableEnvironment arrayEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "arr"}, Types.LONG, Types.OBJECT_ARRAY(Types.LONG)), Row.of(1L, new Long[] {10L, 20L}), Row.of(2L, new Long[] {30L}), @@ -218,7 +220,8 @@ private static TableEnvironment repeatedArrayEnvironment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); // Two rows share the array [10,20] so grouping/joining by the array key is actually exercised. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "arr"}, Types.LONG, Types.OBJECT_ARRAY(Types.LONG)), Row.of(1L, new Long[] {10L, 20L}), Row.of(2L, new Long[] {10L, 20L}), @@ -238,7 +241,8 @@ private static TableEnvironment mapEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "m"}, Types.LONG, Types.MAP(Types.STRING, Types.LONG)), Row.of(1L, Map.of("a", 1L)), @@ -259,7 +263,8 @@ private static TableEnvironment rowEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "r"}, Types.LONG, diff --git a/src/test/java/tech/streamfusion/FlinkCountDistinctSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkCountDistinctSqlHarnessTest.java index 7d2a506ee..da69a2151 100644 --- a/src/test/java/tech/streamfusion/FlinkCountDistinctSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkCountDistinctSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -87,9 +89,11 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.getConfig().set("table.optimizer.agg-phase-strategy", "ONE_PHASE"); - // Repeats per key so distinct < total: k=1 has v {10,20} (10 twice), s {a,b}; k=2 has v {5}, s {c}. + // Repeats per key so distinct < total: k=1 has v {10,20} (10 twice), s {a,b}; k=2 has v {5}, s + // {c}. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "v", "s", "d", "dcm", "millis"}, Types.LONG, diff --git a/src/test/java/tech/streamfusion/FlinkCurrentTimeSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkCurrentTimeSqlHarnessTest.java index 0a343d037..6c6d2d4fd 100644 --- a/src/test/java/tech/streamfusion/FlinkCurrentTimeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkCurrentTimeSqlHarnessTest.java @@ -1,6 +1,7 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.time.Instant; import java.time.ZoneId; @@ -69,7 +70,8 @@ private static TableEnvironment watermarks() { tables.getConfig().setLocalTimeZone(ZoneId.of("America/New_York")); var type = Types.ROW_NAMED(new String[] {"id", "rt"}, Types.INT, Types.INSTANT); var source = - env.fromData( + fromData( + env, type, Row.of(0, Instant.ofEpochMilli(1000)), Row.of(1, Instant.ofEpochMilli(3000)), diff --git a/src/test/java/tech/streamfusion/FlinkDateFormatSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkDateFormatSqlHarnessTest.java index 976092836..17a61e010 100644 --- a/src/test/java/tech/streamfusion/FlinkDateFormatSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkDateFormatSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.LocalDateTime; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -45,7 +47,8 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "ts"}, Types.LONG, Types.LOCAL_DATE_TIME), Row.of(1L, LocalDateTime.of(2024, 3, 5, 8, 30, 15)), Row.of(2L, LocalDateTime.of(2024, 12, 31, 23, 59, 59)), diff --git a/src/test/java/tech/streamfusion/FlinkDecimalExprSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkDecimalExprSqlHarnessTest.java index f0b6eb22d..8677ead1f 100644 --- a/src/test/java/tech/streamfusion/FlinkDecimalExprSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkDecimalExprSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.math.BigDecimal; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -87,9 +89,9 @@ private static TableEnvironment decimalPriceEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( - Types.ROW_NAMED( - new String[] {"auction", "price"}, Types.LONG, Types.BIG_DEC), + fromData( + env, + Types.ROW_NAMED(new String[] {"auction", "price"}, Types.LONG, Types.BIG_DEC), Row.of(1L, new BigDecimal("100.000")), Row.of(2L, new BigDecimal("999.999")), Row.of(3L, new BigDecimal("0.001")), @@ -109,7 +111,8 @@ private static TableEnvironment bigintPriceEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"auction", "price"}, Types.LONG, Types.LONG), Row.of(1L, 100L), Row.of(2L, 999L), diff --git a/src/test/java/tech/streamfusion/FlinkDecimalFloatingSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkDecimalFloatingSqlHarnessTest.java index 97430975c..d99e2b8c9 100644 --- a/src/test/java/tech/streamfusion/FlinkDecimalFloatingSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkDecimalFloatingSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.math.BigDecimal; import java.math.BigInteger; import java.util.Random; @@ -70,7 +72,8 @@ private static TableEnvironment decimals(int precision, int scale, boolean notNu } table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "d", "a", "idx"}, Types.INT, diff --git a/src/test/java/tech/streamfusion/FlinkDecimalOverflowSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkDecimalOverflowSqlHarnessTest.java index 1205be0d3..27f02db2e 100644 --- a/src/test/java/tech/streamfusion/FlinkDecimalOverflowSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkDecimalOverflowSqlHarnessTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.math.BigDecimal; import java.time.Instant; @@ -179,7 +180,7 @@ private static TableEnvironment decimalRows( } table.createTemporaryView( "t", - env.fromData(Types.ROW_NAMED(new String[] {"id", "a"}, Types.LONG, Types.BIG_DEC), rows), + fromData(env, Types.ROW_NAMED(new String[] {"id", "a"}, Types.LONG, Types.BIG_DEC), rows), Schema.newBuilder() .column("id", DataTypes.BIGINT()) .column( diff --git a/src/test/java/tech/streamfusion/FlinkDecimalRemainderSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkDecimalRemainderSqlHarnessTest.java index 4668ace1f..11e228d46 100644 --- a/src/test/java/tech/streamfusion/FlinkDecimalRemainderSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkDecimalRemainderSqlHarnessTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.math.BigDecimal; import java.util.function.Supplier; @@ -153,7 +154,8 @@ private static TableEnvironment environment(String left, String right) { StreamTableEnvironment table = StreamTableEnvironment.create(env); table.createTemporaryView( "t", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"a", "b"}, Types.BIG_DEC, Types.BIG_DEC), Row.of(new BigDecimal(left), new BigDecimal(right))), Schema.newBuilder() @@ -176,7 +178,8 @@ private static TableEnvironment guardedRows(Row... rows) { StreamTableEnvironment table = StreamTableEnvironment.create(env); table.createTemporaryView( "t", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "guard_value", "a", "b"}, Types.INT, diff --git a/src/test/java/tech/streamfusion/FlinkDecimalTruncateSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkDecimalTruncateSqlHarnessTest.java index 7cf6a38a5..1f843ea58 100644 --- a/src/test/java/tech/streamfusion/FlinkDecimalTruncateSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkDecimalTruncateSqlHarnessTest.java @@ -1,6 +1,7 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertEquals; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.math.BigDecimal; import java.util.List; @@ -112,7 +113,8 @@ void identityScaleRetainsCompactParserRoundingCarry() throws Exception { var table = StreamTableEnvironment.create(env); table.createTemporaryView( "t", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"s"}, Types.STRING), Row.of("999.995"), Row.of("-999.995"), @@ -139,7 +141,7 @@ private static TableEnvironment decimals(int precision, int scale, int count, St } table.createTemporaryView( "t", - env.fromData(Types.ROW_NAMED(new String[] {"id", "a"}, Types.INT, Types.BIG_DEC), rows), + fromData(env, Types.ROW_NAMED(new String[] {"id", "a"}, Types.INT, Types.BIG_DEC), rows), Schema.newBuilder() .column("id", DataTypes.INT().notNull()) .column("a", DataTypes.DECIMAL(precision, scale)) diff --git a/src/test/java/tech/streamfusion/FlinkDecodeSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkDecodeSqlHarnessTest.java index c8097398f..4c77ab36e 100644 --- a/src/test/java/tech/streamfusion/FlinkDecodeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkDecodeSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -74,9 +76,9 @@ private static TableEnvironment utf16Bytes() { StreamTableEnvironment tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "inputs", - env.fromData( - Types.ROW_NAMED( - new String[] {"id", "b"}, Types.INT, Types.PRIMITIVE_ARRAY(Types.BYTE)), + fromData( + env, + Types.ROW_NAMED(new String[] {"id", "b"}, Types.INT, Types.PRIMITIVE_ARRAY(Types.BYTE)), rows.toArray(Row[]::new)), Schema.newBuilder().column("id", DataTypes.INT()).column("b", DataTypes.BYTES()).build()); return tables; diff --git a/src/test/java/tech/streamfusion/FlinkDeduplicateSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkDeduplicateSqlHarnessTest.java index 2126e0077..37bad49a3 100644 --- a/src/test/java/tech/streamfusion/FlinkDeduplicateSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkDeduplicateSqlHarnessTest.java @@ -2,9 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; -import tech.streamfusion.planner.NativePlanner; -import tech.streamfusion.planner.PhysicalPlanScan; import java.time.Duration; import java.util.List; import org.apache.flink.api.common.eventtime.WatermarkStrategy; @@ -17,6 +16,8 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.NativePlanner; +import tech.streamfusion.planner.PhysicalPlanScan; /** * Row-time deduplication: per key the native operator keeps either the minimum-rowtime row @@ -26,12 +27,12 @@ class FlinkDeduplicateSqlHarnessTest { private static final String KEEP_FIRST = - "SELECT k, v, rt FROM (" - + "SELECT *, ROW_NUMBER() OVER (PARTITION BY k ORDER BY rt ASC) AS rn FROM src) WHERE rn = 1"; + "SELECT k, v, rt FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY k ORDER BY rt ASC) AS rn" + + " FROM src) WHERE rn = 1"; private static final String KEEP_LAST = - "SELECT k, v, rt FROM (" - + "SELECT *, ROW_NUMBER() OVER (PARTITION BY k ORDER BY rt DESC) AS rn FROM src) WHERE rn = 1"; + "SELECT k, v, rt FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY k ORDER BY rt DESC) AS rn" + + " FROM src) WHERE rn = 1"; @Test void keepFirstDeduplicationMatchesHost() throws Exception { @@ -49,12 +50,12 @@ void keepLastDeduplicationMatchesHost() throws Exception { // Proctime dedup orders by arrival (no rowtime). Only k,v are projected (the PROCTIME() column is // wall-clock, hence non-deterministic) so the comparison is deterministic at parallelism 1. private static final String KEEP_FIRST_PROCTIME = - "SELECT k, v FROM (" - + "SELECT *, ROW_NUMBER() OVER (PARTITION BY k ORDER BY pt ASC) AS rn FROM src) WHERE rn = 1"; + "SELECT k, v FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY k ORDER BY pt ASC) AS rn FROM" + + " src) WHERE rn = 1"; private static final String KEEP_LAST_PROCTIME = - "SELECT k, v FROM (" - + "SELECT *, ROW_NUMBER() OVER (PARTITION BY k ORDER BY pt DESC) AS rn FROM src) WHERE rn = 1"; + "SELECT k, v FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY k ORDER BY pt DESC) AS rn FROM" + + " src) WHERE rn = 1"; @Test void keepFirstProctimeDeduplicationMatchesHost() throws Exception { @@ -83,7 +84,8 @@ private static TableEnvironment proctimeEnvironment() { // (Flink's heap-state aliasing) — so it emits an identical -U/+U pair even with TTL off. // With TTL on the suppression is disabled and both duplicates emit. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 30L), Row.of(2L, 50L), @@ -345,7 +347,8 @@ private static TableEnvironment miniBatchKeepFirstEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "ts"}, Types.LONG, Types.LONG, Types.LONG), Row.of(1L, 10L, 300L), Row.of(2L, 7L, 500L), @@ -381,7 +384,8 @@ private static TableEnvironment miniBatchEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "ts"}, Types.LONG, Types.LONG, Types.LONG), Row.of(1L, 10L, 100L), Row.of(2L, 7L, 500L), @@ -416,7 +420,8 @@ private static TableEnvironment environment() { // Multiple rows per key, out of order, so "first by rowtime" is not "first to arrive": key 1's // minimum-rowtime row is (v=20, rt=0); key 2's is (v=40, rt=1000). DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "ts"}, Types.LONG, Types.LONG, Types.LONG), Row.of(1L, 30L, 2000L), Row.of(2L, 50L, 1500L), diff --git a/src/test/java/tech/streamfusion/FlinkDynamicCollectionSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkDynamicCollectionSqlHarnessTest.java index e5a3ad353..e69bc8fa4 100644 --- a/src/test/java/tech/streamfusion/FlinkDynamicCollectionSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkDynamicCollectionSqlHarnessTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.lang.reflect.Array; import java.math.BigDecimal; @@ -84,6 +85,10 @@ void runtimeMapKeysMatchFlink(String kind) throws Exception { "VARBINARY" }) void runtimeArrayValuesKeepTheirTypes(String kind) throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.SMALL_INTEGER_ARRAY_LOOKUP + || !(kind.equals("TINYINT") || kind.equals("SMALLINT")), + "Flink 1.18 generates an invalid int-to-byte/short conditional for nullable array lookup"); assertNativeParity(() -> typedArrays(kind), "SELECT id, arr[idx] FROM src"); } @@ -255,7 +260,8 @@ private static TableEnvironment collections() { Map.of())); table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] { "id", "idx", "lookup_key", "nums", "texts", "nested", "mappings", "grouped" @@ -281,7 +287,8 @@ private static TableEnvironment mixedKeyTypes(String kind) { if (kind.equals("INT")) { table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "m", "lookup_key"}, Types.INT, @@ -292,7 +299,8 @@ private static TableEnvironment mixedKeyTypes(String kind) { } else { table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "m", "lookup_key"}, Types.INT, @@ -391,7 +399,8 @@ private static TableEnvironment typedArrays(String kind) { rows.add(Row.of(rows.size(), null, 1)); table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "arr", "idx"}, Types.INT, @@ -417,7 +426,8 @@ private static TableEnvironment nullStringKeys(int order) { map.put("a", 7); table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "m", "lookup_key"}, Types.INT, @@ -449,7 +459,8 @@ private static TableEnvironment typedMaps(String kind, boolean nullStoredKey) { nullable.put(spec.present(), null); table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "m", "lookup_key"}, Types.INT, diff --git a/src/test/java/tech/streamfusion/FlinkEltSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkEltSqlHarnessTest.java index 900b966bd..66d5d16f8 100644 --- a/src/test/java/tech/streamfusion/FlinkEltSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkEltSqlHarnessTest.java @@ -3,6 +3,11 @@ import org.junit.jupiter.api.Test; class FlinkEltSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("ELT"); + } + @Test void eltOnlyPropagatesSelectedNullAndBoundsChecksIndex() throws Exception { parity( diff --git a/src/test/java/tech/streamfusion/FlinkEncodeSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkEncodeSqlHarnessTest.java index 9871e13de..ca106d9e2 100644 --- a/src/test/java/tech/streamfusion/FlinkEncodeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkEncodeSqlHarnessTest.java @@ -5,7 +5,7 @@ class FlinkEncodeSqlHarnessTest { @Test void encodesUnicodeNullsAndUnmappableCharacters() throws Exception { - NativeParity.assertParity( + assertEncodingParity( StringFunctionTestInputs::encodings, "SELECT id, ENCODE(s, 'UTF-8'), ENCODE(s, 'ASCII'), ENCODE(s, 'latin1') FROM encodings"); } @@ -18,10 +18,18 @@ void unverifiedFormsFallBackBeforeExecution() throws Exception { @Test void utf16EncodesBomEndianAliasesAndSupplementaryCharacters() throws Exception { - NativeParity.assertParity( - () -> TextTimeFunctionTestInputs.textRows( - null, "", "a\u0000b", "\ufeff", "\ufffe", "\u007f\u0080", - "\u4e2d\ud83d\ude00", "\ud7ff\ue000\uffff", "\ud83d\ude00".repeat(2048)), + assertEncodingParity( + () -> + TextTimeFunctionTestInputs.textRows( + null, + "", + "a\u0000b", + "\ufeff", + "\ufffe", + "\u007f\u0080", + "\u4e2d\ud83d\ude00", + "\ud7ff\ue000\uffff", + "\ud83d\ude00".repeat(2048)), "SELECT id, ENCODE(s, 'UTF-16'), ENCODE(s, 'UnicodeBigUnmarked')," + " ENCODE(s, 'UnicodeLittleUnmarked'), ENCODE(s, 'Unicode') FROM inputs"); } @@ -33,4 +41,15 @@ void dynamicCharsetFallsBack() throws Exception { "SELECT id, ENCODE(s, CASE WHEN n > 0 THEN 'UTF-8' ELSE 'ASCII' END) FROM inputs", "literal charset"); } + + private static void assertEncodingParity( + java.util.function.Supplier environment, + String sql) + throws Exception { + if (tech.streamfusion.compat.FlinkTestCapabilities.VARIABLE_LENGTH_ENCODE) { + NativeParity.assertParity(environment, sql); + } else { + NativeParity.assertFallbackReasonContains(environment, sql, "plan declares BINARY(1)"); + } + } } diff --git a/src/test/java/tech/streamfusion/FlinkEndsWithSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkEndsWithSqlHarnessTest.java index cede37b34..88825d072 100644 --- a/src/test/java/tech/streamfusion/FlinkEndsWithSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkEndsWithSqlHarnessTest.java @@ -3,6 +3,11 @@ import org.junit.jupiter.api.Test; class FlinkEndsWithSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("ENDSWITH"); + } + @Test void suffixArgumentsAndWildcards() throws Exception { parity( diff --git a/src/test/java/tech/streamfusion/FlinkExactDecimalFunctionsSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkExactDecimalFunctionsSqlHarnessTest.java index 6bb9a7b71..154a97525 100644 --- a/src/test/java/tech/streamfusion/FlinkExactDecimalFunctionsSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkExactDecimalFunctionsSqlHarnessTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.math.BigDecimal; import java.util.List; @@ -229,7 +230,7 @@ private static TableEnvironment decimals(int precision, int scale, String... val .toArray(Row[]::new); table.createTemporaryView( "t", - env.fromData(Types.ROW_NAMED(new String[] {"id", "a"}, Types.INT, Types.BIG_DEC), rows), + fromData(env, Types.ROW_NAMED(new String[] {"id", "a"}, Types.INT, Types.BIG_DEC), rows), Schema.newBuilder() .column("id", DataTypes.INT().notNull()) .column("a", DataTypes.DECIMAL(precision, scale)) diff --git a/src/test/java/tech/streamfusion/FlinkExpandSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkExpandSqlHarnessTest.java index a1269a536..57d14b89d 100644 --- a/src/test/java/tech/streamfusion/FlinkExpandSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkExpandSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -72,7 +74,8 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "s"}, Types.LONG, Types.LONG, Types.STRING), Row.of(1L, 5L, "a"), Row.of(2L, 3L, "b"), diff --git a/src/test/java/tech/streamfusion/FlinkExtractSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkExtractSqlHarnessTest.java index d3571bbb3..07c8ecf95 100644 --- a/src/test/java/tech/streamfusion/FlinkExtractSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkExtractSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.LocalDateTime; import java.util.function.Supplier; import org.apache.flink.api.common.typeinfo.Types; @@ -42,7 +44,8 @@ private static Supplier environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "ts"}, Types.LONG, Types.LOCAL_DATE_TIME), Row.of(1L, LocalDateTime.of(2023, 3, 15, 8, 30, 45)), Row.of(2L, LocalDateTime.of(2024, 12, 31, 23, 59, 59)), diff --git a/src/test/java/tech/streamfusion/FlinkFailureParitySqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkFailureParitySqlHarnessTest.java index 49044bf94..f36f36c9e 100644 --- a/src/test/java/tech/streamfusion/FlinkFailureParitySqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkFailureParitySqlHarnessTest.java @@ -1,15 +1,14 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static tech.streamfusion.NativeFailureParity.Phase.*; import static tech.streamfusion.NativeFailureParity.Route.*; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.concurrent.atomic.AtomicInteger; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.legacy.SourceFunction; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.functions.FunctionContext; @@ -17,8 +16,9 @@ import org.apache.flink.types.Row; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import tech.streamfusion.compat.SourceFunction; class FlinkFailureParitySqlHarnessTest { @Test @@ -50,11 +50,13 @@ void malformedDecimalEvaluatesBothEngines() { } @ParameterizedTest - @ValueSource(strings = { - "SELECT CASE WHEN v = '1.2.3' THEN CAST(7 AS DECIMAL(10,2)) ELSE CAST(v AS DECIMAL(10,2)) END FROM src", - "SELECT JSON_VALUE(v, '$.v' RETURNING INTEGER NULL ON ERROR) FROM src", - "SELECT JSON_VALUE(v, '$.v' RETURNING INTEGER DEFAULT 7 ON ERROR) FROM src" - }) + @ValueSource( + strings = { + "SELECT CASE WHEN v = '1.2.3' THEN CAST(7 AS DECIMAL(10,2)) ELSE CAST(v AS DECIMAL(10,2))" + + " END FROM src", + "SELECT JSON_VALUE(v, '$.v' RETURNING INTEGER NULL ON ERROR) FROM src", + "SELECT JSON_VALUE(v, '$.v' RETURNING INTEGER DEFAULT 7 ON ERROR) FROM src" + }) void errorPolicyAndShortCircuitControlsSucceed(String sql) { NativeFailureParity.run(() -> environment("1.2.3"), sql).assertSuccess(NATIVE); } @@ -81,14 +83,20 @@ void aSuccessFailureMismatchCannotPass(boolean hostFails) { @Test void sourceFailureWithoutOperatorPhaseEvidenceRecordsCollectionBoundary() { - NativeFailureParity.run(() -> { - var env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.setParallelism(1); - var table = StreamTableEnvironment.create(env); - table.createTemporaryView("src", env.addSource(new FailSource()).returns(Types.ROW_NAMED(new String[] {"v"}, Types.STRING))); - return table; - }, "SELECT CAST(v AS DECIMAL(10,2)) FROM src") - .assertFailure(IllegalStateException.class, "intentional source failure", COLLECTION, NATIVE); + NativeFailureParity.run( + () -> { + var env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + var table = StreamTableEnvironment.create(env); + table.createTemporaryView( + "src", + env.addSource(new FailSource()) + .returns(Types.ROW_NAMED(new String[] {"v"}, Types.STRING))); + return table; + }, + "SELECT CAST(v AS DECIMAL(10,2)) FROM src") + .assertFailure( + IllegalStateException.class, "intentional source failure", COLLECTION, NATIVE); } @Test @@ -101,7 +109,8 @@ void planningFailureIsNotCountedAsNativeCoverage() { @Test void openFailureRemainsInitializationOnBothEngines() { NativeFailureParity.run(() -> environment("1"), "SELECT FAIL_OPEN(v) FROM src") - .assertFailure(IllegalStateException.class, "intentional open failure", INITIALIZATION, NATIVE); + .assertFailure( + IllegalStateException.class, "intentional open failure", INITIALIZATION, NATIVE); } @ParameterizedTest @@ -117,6 +126,8 @@ void openFailureRemainsInitializationOnBothEngines() { void jsonReturningConversionFailureMatchesHost(String type, String token, String source) { var comparison = NativeFailureParity.run(() -> environment("{\"v\":" + token + "}"), "SELECT JSON_VALUE(v, '$.v' RETURNING " + type + " NULL ON ERROR) FROM src"); + if (source.equals("java.math.BigDecimal")) + source = tech.streamfusion.compat.FlinkTestCapabilities.JSON_FRACTION_CLASS; comparison.assertFailure(ClassCastException.class, "class " + source + " cannot be cast", ROW_EVALUATION, NATIVE); assertEquals(comparison.host().rootCause().getMessage(), comparison.nativeRun().rootCause().getMessage()); @@ -124,10 +135,16 @@ void jsonReturningConversionFailureMatchesHost(String type, String token, String @Test void jsonConversionFailureInFilterMatchesHost() { - var comparison = NativeFailureParity.run(() -> environment("{\"v\":1}"), - "SELECT v FROM src WHERE JSON_VALUE(v, '$.v' RETURNING BOOLEAN DEFAULT FALSE ON EMPTY DEFAULT FALSE ON ERROR)"); - comparison.assertFailure(ClassCastException.class, "java.lang.Integer cannot be cast to class java.lang.Boolean", - ROW_EVALUATION, NATIVE); + var comparison = + NativeFailureParity.run( + () -> environment("{\"v\":1}"), + "SELECT v FROM src WHERE JSON_VALUE(v, '$.v' RETURNING BOOLEAN DEFAULT FALSE ON EMPTY" + + " DEFAULT FALSE ON ERROR)"); + comparison.assertFailure( + ClassCastException.class, + "java.lang.Integer cannot be cast to class java.lang.Boolean", + ROW_EVALUATION, + NATIVE); } @Test @@ -135,10 +152,14 @@ void jsonConversionFailureAfterSeveralBatchesMatchesHost() { String[] documents = new String[5003]; java.util.Arrays.fill(documents, "{\"v\":true}"); documents[5002] = "{\"v\":1}"; - NativeFailureParity.run(() -> environment(documents), - "SELECT JSON_VALUE(v, '$.v' RETURNING BOOLEAN NULL ON ERROR) FROM src") - .assertFailure(ClassCastException.class, "java.lang.Integer cannot be cast to class java.lang.Boolean", - ROW_EVALUATION, NATIVE); + NativeFailureParity.run( + () -> environment(documents), + "SELECT JSON_VALUE(v, '$.v' RETURNING BOOLEAN NULL ON ERROR) FROM src") + .assertFailure( + ClassCastException.class, + "java.lang.Integer cannot be cast to class java.lang.Boolean", + ROW_EVALUATION, + NATIVE); } private static TableEnvironment environment(String... values) { @@ -146,8 +167,8 @@ private static TableEnvironment environment(String... values) { env.setParallelism(1); var table = StreamTableEnvironment.create(env); Row[] rows = java.util.Arrays.stream(values).map(Row::of).toArray(Row[]::new); - table.createTemporaryView("src", env.fromData( - Types.ROW_NAMED(new String[] {"v"}, Types.STRING), rows)); + table.createTemporaryView( + "src", fromData(env, Types.ROW_NAMED(new String[] {"v"}, Types.STRING), rows)); table.createTemporarySystemFunction("FAIL_OPEN", new FailOpen()); return table; } diff --git a/src/test/java/tech/streamfusion/FlinkFieldAccessSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkFieldAccessSqlHarnessTest.java index b86d42c7a..2be88eb86 100644 --- a/src/test/java/tech/streamfusion/FlinkFieldAccessSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkFieldAccessSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -85,7 +87,8 @@ private static TableEnvironment environment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"event_type", "bid"}, Types.INT, diff --git a/src/test/java/tech/streamfusion/FlinkFilterSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkFilterSqlHarnessTest.java index 20b68184d..0edee3c69 100644 --- a/src/test/java/tech/streamfusion/FlinkFilterSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkFilterSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.LocalDateTime; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -126,7 +128,8 @@ private static TableEnvironment nullableEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.INT), Row.of(1L, 10), Row.of(2L, null), @@ -143,7 +146,8 @@ private static TableEnvironment overflowEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData(Types.ROW_NAMED(new String[] {"v"}, Types.INT), Row.of(30), Row.of(2_000_000_000)); + fromData( + env, Types.ROW_NAMED(new String[] {"v"}, Types.INT), Row.of(30), Row.of(2_000_000_000)); tEnv.createTemporaryView( "o", source, Schema.newBuilder().column("v", DataTypes.INT()).build()); return tEnv; @@ -154,9 +158,9 @@ private static TableEnvironment timestampEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( - Types.ROW_NAMED( - new String[] {"v", "t"}, Types.INT, Types.LOCAL_DATE_TIME), + fromData( + env, + Types.ROW_NAMED(new String[] {"v", "t"}, Types.INT, Types.LOCAL_DATE_TIME), Row.of(10, LocalDateTime.of(2020, 1, 1, 0, 0, 0)), Row.of(30, LocalDateTime.of(2020, 1, 2, 12, 30, 0)), Row.of(20, LocalDateTime.of(2020, 1, 3, 6, 15, 45))); @@ -175,9 +179,9 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( - Types.ROW_NAMED( - new String[] {"k", "v", "s"}, Types.LONG, Types.INT, Types.STRING), + fromData( + env, + Types.ROW_NAMED(new String[] {"k", "v", "s"}, Types.LONG, Types.INT, Types.STRING), Row.of(1L, 10, "a"), Row.of(2L, 30, "b"), Row.of(3L, 20, "c"), diff --git a/src/test/java/tech/streamfusion/FlinkFirstNSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkFirstNSqlHarnessTest.java index 16731aa79..a9ddef321 100644 --- a/src/test/java/tech/streamfusion/FlinkFirstNSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkFirstNSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.ArrayList; import java.util.List; @@ -82,7 +83,8 @@ private static TableEnvironment environment(boolean miniBatch) { } table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 30L), Row.of(null, 90L), diff --git a/src/test/java/tech/streamfusion/FlinkFloatingCastSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkFloatingCastSqlHarnessTest.java index b7288548f..b0047a671 100644 --- a/src/test/java/tech/streamfusion/FlinkFloatingCastSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkFloatingCastSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -31,8 +33,9 @@ private static TableEnvironment input() { Double value = values[i]; rows[i] = Row.of(value, value == null ? null : value.floatValue()); } - table.createTemporaryView("n", env.fromData( - Types.ROW_NAMED(new String[] {"d", "f"}, Types.DOUBLE, Types.FLOAT), rows)); + table.createTemporaryView( + "n", + fromData(env, Types.ROW_NAMED(new String[] {"d", "f"}, Types.DOUBLE, Types.FLOAT), rows)); return table; } } diff --git a/src/test/java/tech/streamfusion/FlinkFloatingComparisonSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkFloatingComparisonSqlHarnessTest.java index 98087ca1a..710e54b1b 100644 --- a/src/test/java/tech/streamfusion/FlinkFloatingComparisonSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkFloatingComparisonSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -38,9 +40,18 @@ private static TableEnvironment input() { b == null ? null : b.floatValue()); } } - table.createTemporaryView("n", env.fromData(Types.ROW_NAMED( - new String[] {"id", "d", "e", "f", "g"}, - Types.INT, Types.DOUBLE, Types.DOUBLE, Types.FLOAT, Types.FLOAT), rows)); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED( + new String[] {"id", "d", "e", "f", "g"}, + Types.INT, + Types.DOUBLE, + Types.DOUBLE, + Types.FLOAT, + Types.FLOAT), + rows)); return table; } } diff --git a/src/test/java/tech/streamfusion/FlinkFloatingWindowReproTest.java b/src/test/java/tech/streamfusion/FlinkFloatingWindowReproTest.java index 031fe734d..965bd85eb 100644 --- a/src/test/java/tech/streamfusion/FlinkFloatingWindowReproTest.java +++ b/src/test/java/tech/streamfusion/FlinkFloatingWindowReproTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.LocalDateTime; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -33,6 +35,7 @@ void hopNanFirst() throws Exception { @Test void sessionNanFirst() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); check("SESSION", Double.NaN, 2.0); } @@ -72,7 +75,8 @@ void filteredExtremaInitializeFromTheFirstSelectedValue(String phase, String fun var ts = LocalDateTime.of(2026, 1, 1, 0, 0); table.createTemporaryView( "n", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"ts", "d", "f", "keep"}, Types.LOCAL_DATE_TIME, @@ -119,13 +123,20 @@ private static TableEnvironment input(double first, double second) { table.getConfig().setLocalTimeZone(java.time.ZoneId.of("UTC")); table.getConfig().set("table.optimizer.agg-phase-strategy", "ONE_PHASE"); LocalDateTime ts = LocalDateTime.of(2026, 1, 1, 0, 0); - table.createTemporaryView("n", env.fromData( - Types.ROW_NAMED(new String[] {"ts", "d", "f"}, - Types.LOCAL_DATE_TIME, Types.DOUBLE, Types.FLOAT), - Row.of(ts, first, (float) first), Row.of(ts, second, (float) second)), - Schema.newBuilder().column("ts", DataTypes.TIMESTAMP(3)) - .column("d", DataTypes.DOUBLE()).column("f", DataTypes.FLOAT()) - .watermark("ts", "ts - INTERVAL '1' SECOND").build()); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED( + new String[] {"ts", "d", "f"}, Types.LOCAL_DATE_TIME, Types.DOUBLE, Types.FLOAT), + Row.of(ts, first, (float) first), + Row.of(ts, second, (float) second)), + Schema.newBuilder() + .column("ts", DataTypes.TIMESTAMP(3)) + .column("d", DataTypes.DOUBLE()) + .column("f", DataTypes.FLOAT()) + .watermark("ts", "ts - INTERVAL '1' SECOND") + .build()); return table; } } diff --git a/src/test/java/tech/streamfusion/FlinkGroupAggregateSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkGroupAggregateSqlHarnessTest.java index c31801ccb..11eeea903 100644 --- a/src/test/java/tech/streamfusion/FlinkGroupAggregateSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkGroupAggregateSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -182,6 +184,7 @@ void stateTtlEmitsUnsuppressedUpdatesAndMatchesHost() throws Exception { @Test void stateTtlHintOverridesJobRetention() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireStateTtlHint(); // STATE_TTL('kv' = '1h') with the job retention at 0: the hint alone must switch the operator // into TTL emission (unsuppressed -U/+U pairs), matching Flink's hint-over-config precedence. NativeParity.assertKindedParity( @@ -206,7 +209,8 @@ private static TableEnvironment minimalEnvironment() { // The 0-value row leaves key 7's sum unchanged: suppressed with TTL off, an identical -U/+U // pair with TTL on — the emission difference the TTL tests above pin. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "value"}, Types.LONG, Types.LONG), Row.of(7L, 1L), Row.of(7L, 2L), @@ -247,7 +251,8 @@ private static TableEnvironment environment(int parallelism) { tEnv.getConfig().set("table.optimizer.agg-phase-strategy", "ONE_PHASE"); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "s", "value", "qty", "price", "vs", "vt", "vf"}, Types.LONG, diff --git a/src/test/java/tech/streamfusion/FlinkGroupedValueSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkGroupedValueSqlHarnessTest.java index 5418351e6..502fdf354 100644 --- a/src/test/java/tech/streamfusion/FlinkGroupedValueSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkGroupedValueSqlHarnessTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static tech.streamfusion.NativeFailureParity.Phase.ROW_EVALUATION; import static tech.streamfusion.NativeFailureParity.Route.NATIVE; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.time.ZoneOffset; import java.util.ArrayList; @@ -33,6 +34,7 @@ class FlinkGroupedValueSqlHarnessTest { "TIMESTAMP_LTZ(3)" }) void firstAndLastPreserveArrivalOrderAndIgnoreNulls(String type) throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireFirstLastType(type); compare( () -> environment(type, false), "SELECT k, FIRST_VALUE(v), LAST_VALUE(v), COUNT(*) FROM src GROUP BY k"); @@ -50,6 +52,7 @@ void firstAndLastPreserveArrivalOrderAndIgnoreNulls(String type) throws Exceptio "TIMESTAMP_LTZ(3)" }) void retractionsRemoveTheOldestMatchingOccurrence(String type) throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireFirstLastType(type); compare( () -> environment(type, true), "SELECT k, FIRST_VALUE(v), LAST_VALUE(v), COUNT(*) FROM src GROUP BY k"); @@ -108,6 +111,7 @@ void unverifiedValueTypesRetainExplicitFallback(String type) throws Exception { @ParameterizedTest @ValueSource(booleans = {false, true}) void retractingFirstLastWithIndependentMapTtlFallBack(boolean hint) throws Exception { + if (hint) tech.streamfusion.compat.FlinkTestCapabilities.requireStateTtlHint(); Supplier source = () -> { var table = environment("BIGINT", true); @@ -124,6 +128,7 @@ void retractingFirstLastWithIndependentMapTtlFallBack(boolean hint) throws Excep @org.junit.jupiter.api.Test void zeroRetentionHintAdmitsRetractionsDespiteGlobalTtl() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireStateTtlHint(); compare( () -> { var table = environment("BIGINT", true); @@ -172,7 +177,8 @@ private static TableEnvironment environment(String type, boolean retract) { Row.of(9, 1, c))); } var source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "k", "raw_value"}, Types.INT, Types.INT, Types.STRING), rows.toArray(Row[]::new)); diff --git a/src/test/java/tech/streamfusion/FlinkGuardedDivisionSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkGuardedDivisionSqlHarnessTest.java index aacf2295d..c3419dcbf 100644 --- a/src/test/java/tech/streamfusion/FlinkGuardedDivisionSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkGuardedDivisionSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -54,9 +56,17 @@ private static TableEnvironment input() { var env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); var table = StreamTableEnvironment.create(env); - table.createTemporaryView("n", env.fromData(Types.ROW_NAMED( - new String[] {"i"}, Types.INT), Row.of(0), Row.of(2), Row.of(-3), - Row.of(Integer.MIN_VALUE), Row.of(Integer.MAX_VALUE), Row.of((Object) null))); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED(new String[] {"i"}, Types.INT), + Row.of(0), + Row.of(2), + Row.of(-3), + Row.of(Integer.MIN_VALUE), + Row.of(Integer.MAX_VALUE), + Row.of((Object) null))); return table; } } diff --git a/src/test/java/tech/streamfusion/FlinkHashCodeSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkHashCodeSqlHarnessTest.java index b71cf10db..cc1ba047b 100644 --- a/src/test/java/tech/streamfusion/FlinkHashCodeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkHashCodeSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.math.BigDecimal; import java.time.LocalDateTime; import org.apache.flink.api.common.typeinfo.Types; @@ -75,7 +77,8 @@ private static TableEnvironment environment() { var beforeEpoch = LocalDateTime.of(1969, 12, 31, 23, 59, 59, 999999999); table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "s", "i", "n", "b", "d", "t"}, Types.INT, diff --git a/src/test/java/tech/streamfusion/FlinkHostChangelogIntoNativeTest.java b/src/test/java/tech/streamfusion/FlinkHostChangelogIntoNativeTest.java index 846c3e2a2..4ed544705 100644 --- a/src/test/java/tech/streamfusion/FlinkHostChangelogIntoNativeTest.java +++ b/src/test/java/tech/streamfusion/FlinkHostChangelogIntoNativeTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.math.BigDecimal; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -36,7 +38,8 @@ private static TableEnvironment environment() { // Repeated g so the inner SUM updates (emitting -U/+U); the outer COUNT(*) must retract the old // total's count each time. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"g", "d"}, Types.LONG, Types.BIG_DEC), Row.of(1L, new BigDecimal("1.00")), Row.of(1L, new BigDecimal("2.00")), diff --git a/src/test/java/tech/streamfusion/FlinkHotItemsSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkHotItemsSqlHarnessTest.java index 5a555b8b1..6b864ea28 100644 --- a/src/test/java/tech/streamfusion/FlinkHotItemsSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkHotItemsSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Duration; import java.time.ZoneId; import org.apache.flink.api.common.eventtime.WatermarkStrategy; @@ -23,21 +25,16 @@ class FlinkHotItemsSqlHarnessTest { private static final String HOT_ITEMS = - "SELECT AuctionBids.auction, AuctionBids.num FROM (" - + " SELECT auction, count(*) AS num, window_start AS starttime, window_end AS endtime" - + " FROM TABLE(HOP(TABLE bid, DESCRIPTOR(rt), INTERVAL '2' SECOND, INTERVAL '10' SECOND))" - + " GROUP BY auction, window_start, window_end" - + ") AS AuctionBids " - + "JOIN (" - + " SELECT max(CountBids.num) AS maxn, CountBids.starttime, CountBids.endtime FROM (" - + " SELECT auction, count(*) AS num, window_start AS starttime, window_end AS endtime" - + " FROM TABLE(HOP(TABLE bid, DESCRIPTOR(rt), INTERVAL '2' SECOND, INTERVAL '10' SECOND))" - + " GROUP BY auction, window_start, window_end" - + " ) AS CountBids GROUP BY CountBids.starttime, CountBids.endtime" - + ") AS MaxBids " - + "ON AuctionBids.starttime = MaxBids.starttime" - + " AND AuctionBids.endtime = MaxBids.endtime" - + " AND AuctionBids.num >= MaxBids.maxn"; + "SELECT AuctionBids.auction, AuctionBids.num FROM ( SELECT auction, count(*) AS num," + + " window_start AS starttime, window_end AS endtime FROM TABLE(HOP(TABLE bid," + + " DESCRIPTOR(rt), INTERVAL '2' SECOND, INTERVAL '10' SECOND)) GROUP BY auction," + + " window_start, window_end) AS AuctionBids JOIN ( SELECT max(CountBids.num) AS maxn," + + " CountBids.starttime, CountBids.endtime FROM ( SELECT auction, count(*) AS num," + + " window_start AS starttime, window_end AS endtime FROM TABLE(HOP(TABLE bid," + + " DESCRIPTOR(rt), INTERVAL '2' SECOND, INTERVAL '10' SECOND)) GROUP BY auction," + + " window_start, window_end ) AS CountBids GROUP BY CountBids.starttime," + + " CountBids.endtime) AS MaxBids ON AuctionBids.starttime = MaxBids.starttime AND" + + " AuctionBids.endtime = MaxBids.endtime AND AuctionBids.num >= MaxBids.maxn"; @Test void hotItemsMatchesHost() throws Exception { @@ -54,9 +51,11 @@ private static TableEnvironment bidEnvironment() { } private static DataStream bids(StreamExecutionEnvironment env) { - // Bids for auctions 1/2/3 spread across time so several overlapping 10s/2s windows form; auction 1 + // Bids for auctions 1/2/3 spread across time so several overlapping 10s/2s windows form; + // auction 1 // is the hot item in the early windows, auction 2 later — the per-window MAX picks the leaders. - return env.fromData( + return fromData( + env, Types.ROW_NAMED(new String[] {"auction", "ts"}, Types.LONG, Types.LONG), Row.of(1L, 1000L), Row.of(1L, 1500L), diff --git a/src/test/java/tech/streamfusion/FlinkIfNullSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkIfNullSqlHarnessTest.java index 928bc7e5f..633f5173b 100644 --- a/src/test/java/tech/streamfusion/FlinkIfNullSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkIfNullSqlHarnessTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.math.BigDecimal; import java.util.List; @@ -46,7 +47,7 @@ void resultSchemaRetainsFlinksNullabilityAndDecimalScale() throws Exception { + "IFNULL(d, CAST(0 AS DECIMAL(12,3))), IFNULL(i, n) FROM src"; var types = environment().sqlQuery(sql).getResolvedSchema().getColumnDataTypes(); assertEquals(DataTypes.INT().notNull(), types.get(0)); - assertEquals(DataTypes.STRING().notNull(), types.get(1)); + assertEquals(tech.streamfusion.compat.FlinkTestCapabilities.ifNullStringType(), types.get(1)); assertEquals(DataTypes.DECIMAL(12, 3).notNull(), types.get(2)); assertEquals(DataTypes.BIGINT(), types.get(3)); NativeParity.assertParity(FlinkIfNullSqlHarnessTest::environment, sql); @@ -112,7 +113,8 @@ private static TableEnvironment nullableIds() { StreamTableEnvironment table = StreamTableEnvironment.create(env); table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id"}, Types.INT), Row.of(2), Row.of(0), @@ -182,10 +184,17 @@ private static TableEnvironment environment(Row... rows) { StreamTableEnvironment table = StreamTableEnvironment.create(env); table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "i", "n", "s", "d", "z", "b"}, - Types.INT, Types.INT, Types.LONG, Types.STRING, Types.BIG_DEC, Types.INT, Types.BOOLEAN), + Types.INT, + Types.INT, + Types.LONG, + Types.STRING, + Types.BIG_DEC, + Types.INT, + Types.BOOLEAN), rows), Schema.newBuilder() .column("id", DataTypes.INT().notNull()) diff --git a/src/test/java/tech/streamfusion/FlinkIfSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkIfSqlHarnessTest.java index 6b90311ea..d1fd3e387 100644 --- a/src/test/java/tech/streamfusion/FlinkIfSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkIfSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.math.BigDecimal; import java.time.LocalDateTime; @@ -44,6 +45,10 @@ class FlinkIfSqlHarnessTest { "IF(flag, IF(i > 0, s, other), IF(i IS NULL, 'missing', s))" }) void resolvedValuesAndTypesMatchFlink(String expression) throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.NEGATIVE_TIMESTAMP_TO_TIME + || !expression.contains("AS TIME)"), + "Flink 1.18 external TIME conversion rejects a negative pre-epoch fraction"); NativeParity.assertParity( FlinkIfSqlHarnessTest::environment, "SELECT id, " + expression + " FROM src"); } @@ -151,7 +156,8 @@ private static TableEnvironment environment() { var timestamp = LocalDateTime.of(1969, 12, 31, 23, 59, 59, 999999999); table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "flag", "i", "n", "d", "s", "other", "t", "bytes"}, Types.INT, diff --git a/src/test/java/tech/streamfusion/FlinkInstrSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkInstrSqlHarnessTest.java index 312173cb1..2c7f9dd1b 100644 --- a/src/test/java/tech/streamfusion/FlinkInstrSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkInstrSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.ArrayList; import java.util.List; @@ -167,7 +168,8 @@ private static TableEnvironment environment(List rows) { StreamTableEnvironment table = StreamTableEnvironment.create(env); table.createTemporaryView( "searches", - env.fromData( + fromData( + env, rows, Types.ROW_NAMED( new String[] {"id", "s", "needle", "start_pos", "occurrence"}, diff --git a/src/test/java/tech/streamfusion/FlinkIntegerDivisionSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkIntegerDivisionSqlHarnessTest.java index 74217c2fc..121ce4768 100644 --- a/src/test/java/tech/streamfusion/FlinkIntegerDivisionSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkIntegerDivisionSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -23,12 +25,18 @@ private static TableEnvironment input() { var env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); var table = StreamTableEnvironment.create(env); - table.createTemporaryView("n", env.fromData(Types.ROW_NAMED( - new String[] {"i", "l", "j", "k"}, Types.INT, Types.LONG, Types.INT, Types.LONG), - Row.of(Integer.MIN_VALUE, Long.MIN_VALUE, -1, -1L), - Row.of(Integer.MAX_VALUE, Long.MAX_VALUE, -1, -1L), - Row.of(-7, -7L, 2, 2L), Row.of(7, 7L, -2, -2L), - Row.of(null, null, 0, 0L), Row.of(1, 1L, null, null))); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED( + new String[] {"i", "l", "j", "k"}, Types.INT, Types.LONG, Types.INT, Types.LONG), + Row.of(Integer.MIN_VALUE, Long.MIN_VALUE, -1, -1L), + Row.of(Integer.MAX_VALUE, Long.MAX_VALUE, -1, -1L), + Row.of(-7, -7L, 2, 2L), + Row.of(7, 7L, -2, -2L), + Row.of(null, null, 0, 0L), + Row.of(1, 1L, null, null))); return table; } } diff --git a/src/test/java/tech/streamfusion/FlinkIntegerStringCastSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkIntegerStringCastSqlHarnessTest.java index 9bf5c65eb..8be88fb3d 100644 --- a/src/test/java/tech/streamfusion/FlinkIntegerStringCastSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkIntegerStringCastSqlHarnessTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.List; import java.util.stream.IntStream; @@ -35,11 +36,41 @@ void validValuesMatchAcrossBatchesAndWidths(String type) throws Exception { @ParameterizedTest @ValueSource(strings = {"TINYINT", "SMALLINT", "INT", "BIGINT"}) void tryCastAndLegacyHandleBoundariesMalformedTextAndNulls(String type) throws Exception { - String[] values = {"-128", "127", "128", "-129", "-32768", "32767", "32768", "-32769", - "-2147483648.9", "2147483647.9", "2147483648", "-2147483649", - "-9223372036854775808.9", "9223372036854775807.9", "9223372036854775808", "-9223372036854775809", - "", " ", "+", "-", "1e2", "1.2.3", "1.2x", "\t1", "1\n", "\u00a01", "1\u2003", - "\u0661", "\uff11", "1\u0000", " 42 ", "1 2", null}; + String[] values = { + "-128", + "127", + "128", + "-129", + "-32768", + "32767", + "32768", + "-32769", + "-2147483648.9", + "2147483647.9", + "2147483648", + "-2147483649", + "-9223372036854775808.9", + "9223372036854775807.9", + "9223372036854775808", + "-9223372036854775809", + "", + " ", + "+", + "-", + "1e2", + "1.2.3", + "1.2x", + "\t1", + "1\n", + "\u00a01", + "1\u2003", + "\u0661", + "\uff11", + "1\u0000", + " 42 ", + "1 2", + null + }; NativeParity.assertParity(() -> input(false, DataTypes.STRING(), values), "SELECT id, TRY_CAST(s AS " + type + ") FROM src"); NativeParity.assertParity(() -> input(true, DataTypes.STRING(), values), @@ -68,7 +99,8 @@ void malformedOrdinaryCastFailsInBothEngines(String value) { @Test void nonNullableInputAndCaseRetainHostBehavior() throws Exception { - NativeParity.assertParity(() -> input(false, DataTypes.VARCHAR(12).notNull(), "1", "-2147483648"), + NativeParity.assertParity( + () -> input(false, DataTypes.VARCHAR(12).notNull(), "1", "-2147483648"), "SELECT id, CAST(s AS INT), CAST(CAST(s AS INT) AS STRING) FROM src"); NativeParity.assertParity(() -> input(false, DataTypes.STRING(), "skip", "42", null), "SELECT id, CASE WHEN s = 'skip' THEN 0 ELSE CAST(s AS INT) END FROM src"); @@ -78,7 +110,8 @@ void nonNullableInputAndCaseRetainHostBehavior() throws Exception { @Test void fallibleConjunctionsKeepHostShortCircuiting() throws Exception { - for (String predicate : List.of("s = 'skip' OR CAST(s AS INT) > 0", "s <> 'skip' AND CAST(s AS INT) > 0")) { + for (String predicate : + List.of("s = 'skip' OR CAST(s AS INT) > 0", "s <> 'skip' AND CAST(s AS INT) > 0")) { NativeParity.assertFallbackReasonContains( () -> input(false, DataTypes.STRING(), "skip", "42", "-1"), "SELECT id, " + predicate + " FROM src", "short-circuit"); @@ -103,7 +136,10 @@ void legacyFailureOnNonNullableInputRetainsSinkEnforcement() throws Exception { for (String enforcement : List.of("ERROR", "DROP")) { for (boolean nativeRun : new boolean[] {false, true}) { TableEnvironment table = input(true, DataTypes.STRING().notNull(), "invalid"); - table.getConfig().getConfiguration().setString("table.exec.sink.not-null-enforcer", enforcement); + table + .getConfig() + .getConfiguration() + .setString("table.exec.sink.not-null-enforcer", enforcement); var scan = nativeRun ? NativePlanner.install(table) : null; if (enforcement.equals("ERROR")) { assertThrows(Exception.class, () -> collect(table, "SELECT CAST(s AS INT) FROM src")); @@ -127,12 +163,20 @@ private static TableEnvironment numbers(boolean legacy) { var env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); var table = StreamTableEnvironment.create(env); - table.getConfig().getConfiguration().setString("table.exec.legacy-cast-behaviour", legacy ? "ENABLED" : "DISABLED"); - table.createTemporaryView("nums", env.fromData( - Types.ROW_NAMED(new String[] {"b", "s", "i", "l"}, Types.BYTE, Types.SHORT, Types.INT, Types.LONG), - Row.of(Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE), - Row.of(Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE), - Row.of((byte) 0, (short) 0, 0, 0L), Row.of(null, null, null, null))); + table + .getConfig() + .getConfiguration() + .setString("table.exec.legacy-cast-behaviour", legacy ? "ENABLED" : "DISABLED"); + table.createTemporaryView( + "nums", + fromData( + env, + Types.ROW_NAMED( + new String[] {"b", "s", "i", "l"}, Types.BYTE, Types.SHORT, Types.INT, Types.LONG), + Row.of(Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE), + Row.of(Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE), + Row.of((byte) 0, (short) 0, 0, 0L), + Row.of(null, null, null, null))); return table; } @@ -140,10 +184,14 @@ private static TableEnvironment input(boolean legacy, DataType type, String... v var env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); var table = StreamTableEnvironment.create(env); - table.getConfig().getConfiguration().setString("table.exec.legacy-cast-behaviour", legacy ? "ENABLED" : "DISABLED"); + table + .getConfig() + .getConfiguration() + .setString("table.exec.legacy-cast-behaviour", legacy ? "ENABLED" : "DISABLED"); Row[] rows = IntStream.range(0, values.length).mapToObj(i -> Row.of(i, values[i])).toArray(Row[]::new); - table.createTemporaryView("src", env.fromData( - Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING), rows), + table.createTemporaryView( + "src", + fromData(env, Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING), rows), Schema.newBuilder().column("id", DataTypes.INT().notNull()).column("s", type).build()); return table; } diff --git a/src/test/java/tech/streamfusion/FlinkIntervalExprSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkIntervalExprSqlHarnessTest.java index e6ec5499c..46e64de6b 100644 --- a/src/test/java/tech/streamfusion/FlinkIntervalExprSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkIntervalExprSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.LocalDateTime; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -20,7 +22,8 @@ void dayTimeIntervalLiteralMatchesHost() throws Exception { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k"}, Types.LONG), Row.of(1L), Row.of(2L), @@ -42,7 +45,8 @@ void subSecondIntervalLiteralMatchesHost() throws Exception { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k"}, Types.LONG), Row.of(1L), Row.of(2L), @@ -65,7 +69,8 @@ void intervalLiteralChosenPerRowMatchesHost() throws Exception { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k"}, Types.LONG), Row.of(1L), Row.of(2L), @@ -88,7 +93,8 @@ void intervalLiteralGroupKeyMatchesHost() throws Exception { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k"}, Types.LONG), Row.of(1L), Row.of(2L), @@ -113,7 +119,8 @@ void nullableIntervalLiteralGroupKeyMatchesHost() throws Exception { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k"}, Types.LONG), Row.of(1L), Row.of(2L), @@ -139,7 +146,8 @@ void intervalComparisonAfterTopNMatchesHost() throws Exception { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k"}, Types.LONG), Row.of(1L), Row.of(2L), @@ -163,19 +171,35 @@ void intervalOuterJoinResidualMatchesHost() throws Exception { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream left = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), - Row.of(1L, 1L), Row.of(2L, 3L), Row.of(3L, 1L), Row.of(4L, null)); + Row.of(1L, 1L), + Row.of(2L, 3L), + Row.of(3L, 1L), + Row.of(4L, null)); DataStream right = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), - Row.of(1L, 1L), Row.of(2L, 3L), Row.of(4L, null), Row.of(5L, 3L)); + Row.of(1L, 1L), + Row.of(2L, 3L), + Row.of(4L, null), + Row.of(5L, 3L)); tEnv.createTemporaryView( - "a", left, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + "a", + left, + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); tEnv.createTemporaryView( - "b", right, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + "b", + right, + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); return tEnv; }, "SELECT l.k, l.d, r.k, r.d FROM " @@ -195,19 +219,35 @@ void intervalOuterJoinColumnResidualMatchesHost() throws Exception { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream left = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), - Row.of(1L, 1L), Row.of(2L, 3L), Row.of(3L, 1L), Row.of(4L, null)); + Row.of(1L, 1L), + Row.of(2L, 3L), + Row.of(3L, 1L), + Row.of(4L, null)); DataStream right = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), - Row.of(1L, 1L), Row.of(2L, 3L), Row.of(4L, null), Row.of(5L, 3L)); + Row.of(1L, 1L), + Row.of(2L, 3L), + Row.of(4L, null), + Row.of(5L, 3L)); tEnv.createTemporaryView( - "a", left, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + "a", + left, + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); tEnv.createTemporaryView( - "b", right, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + "b", + right, + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); return tEnv; }, "SELECT l.k, l.d, r.k, r.d FROM " @@ -228,7 +268,8 @@ void nanosecondTimestampDifferenceMatchesHost() throws Exception { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "a", "b"}, Types.LONG, @@ -274,7 +315,8 @@ void yearMonthIntervalChosenPerRowMatchesHost() throws Exception { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k"}, Types.LONG), Row.of(1L), Row.of(2L), diff --git a/src/test/java/tech/streamfusion/FlinkIntervalJoinSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkIntervalJoinSqlHarnessTest.java index db901bd97..507c79302 100644 --- a/src/test/java/tech/streamfusion/FlinkIntervalJoinSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkIntervalJoinSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; @@ -62,7 +64,8 @@ private static TableEnvironment fractionalEnvironment(int precision) { java.time.LocalDateTime.of( 1970, 1, 1, 0, 0, side.equals("A") ? 1 : 2, side.equals("A") ? 1 : 999999); DataStream input = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "v", "rt"}, Types.LONG, Types.LONG, Types.LOCAL_DATE_TIME), Row.of(1L, side.equals("A") ? 10L : 100L, time), @@ -182,7 +185,8 @@ private static TableEnvironment proctimeEnvironment() { } private static DataStream proctimeStream(StreamExecutionEnvironment env) { - return env.fromData( + return fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 30L), @@ -200,7 +204,8 @@ private static TableEnvironment dataStreamEnvironment() { } private static DataStream stream(StreamExecutionEnvironment env) { - return env.fromData( + return fromData( + env, Types.ROW_NAMED( new String[] {"k", "v", "ts", "d"}, Types.LONG, diff --git a/src/test/java/tech/streamfusion/FlinkIsTrueSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkIsTrueSqlHarnessTest.java index 54fa4e83c..c297b7b59 100644 --- a/src/test/java/tech/streamfusion/FlinkIsTrueSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkIsTrueSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -31,7 +33,8 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.INT), Row.of(1L, 9), Row.of(2L, 1), diff --git a/src/test/java/tech/streamfusion/FlinkJsonJvmSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkJsonJvmSqlHarnessTest.java index cb90fcd0d..3d78f9642 100644 --- a/src/test/java/tech/streamfusion/FlinkJsonJvmSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkJsonJvmSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.ArrayList; import java.util.List; @@ -34,6 +35,17 @@ class FlinkJsonJvmSqlHarnessTest { + " ERROR)" }) void dynamicPathsUseReleasedFlinkForSelectionAndErrors(String expression) throws Exception { + if (expression.startsWith("JSON_QUERY") + && !tech.streamfusion.compat.FlinkTestCapabilities.DYNAMIC_JSON_QUERY) { + NativeFailureParity.run( + FlinkJsonJvmSqlHarnessTest::dynamicPaths, "SELECT id, " + expression + " FROM inputs") + .assertFailure( + org.apache.flink.table.planner.codegen.CodeGenException.class, + "Unsupported call: JSON_QUERY", + NativeFailureParity.Phase.PLANNING, + NativeFailureParity.Route.FALLBACK); + return; + } NativeParity.assertParity( FlinkJsonJvmSqlHarnessTest::dynamicPaths, "SELECT id, " + expression + " FROM inputs"); } @@ -66,7 +78,8 @@ void filteredChangelogRowsKeepTheirKindsAndSkipInvalidProjections() throws Excep env.setParallelism(1); var table = StreamTableEnvironment.create(env); var source = - env.fromData( + fromData( + env, List.of( Row.ofKind(RowKind.INSERT, 1, "{\"n\":1}"), Row.ofKind(RowKind.INSERT, 0, "invalid"), @@ -100,7 +113,7 @@ void generatedJsonResultsComposeWithNativeAggregation() throws Exception { void planIdentifiesTheJvmEvaluatorInsideNativeCalc() { String plan = tech.streamfusion.planner.NativePlanner.explain( - dynamicPaths(), "SELECT JSON_QUERY(s, p) FROM inputs"); + dynamicPaths(), "SELECT JSON_QUERY(s, '$.a[0:2]') FROM inputs"); org.junit.jupiter.api.Assertions.assertTrue(plan.contains("NativeCalc"), plan); org.junit.jupiter.api.Assertions.assertTrue(plan.contains("jsonEvaluation=[JVM]"), plan); } @@ -117,10 +130,18 @@ void planIdentifiesTheJvmEvaluatorInsideNativeCalc() { "JSON_OBJECT('a' VALUE s)" }) void verifiedFastPathsKeepTheirNativeEvaluator(String expression) throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireJsonFunctions(expression); String sql = "SELECT " + expression + " FROM inputs"; String plan = tech.streamfusion.planner.NativePlanner.explain(dynamicPaths(), sql); org.junit.jupiter.api.Assertions.assertTrue(plan.contains("NativeCalc"), plan); - org.junit.jupiter.api.Assertions.assertFalse(plan.contains("jsonEvaluation=[JVM]"), plan); + boolean parser = + expression.startsWith("JSON_VALUE") + || expression.startsWith("JSON_EXISTS") + || expression.contains("IS JSON"); + assertEquals( + parser && !tech.streamfusion.operator.NativeJsonRuntime.available(), + plan.contains("jsonEvaluation=[JVM]"), + plan); NativeParity.assertParity(FlinkJsonJvmSqlHarnessTest::dynamicPaths, sql); } @@ -128,7 +149,11 @@ void verifiedFastPathsKeepTheirNativeEvaluator(String expression) throws Excepti void simpleJsonPredicateKeepsTheNativeFilter() throws Exception { String sql = "SELECT id FROM inputs WHERE JSON_EXISTS(s, '$.a')"; String plan = tech.streamfusion.planner.NativePlanner.explain(dynamicPaths(), sql); - org.junit.jupiter.api.Assertions.assertTrue(plan.contains("NativeFilter"), plan); + if (tech.streamfusion.operator.NativeJsonRuntime.available()) { + assertTrue(plan.contains("NativeFilter"), plan); + } else { + assertTrue(plan.contains("NativeCalc") && plan.contains("jsonEvaluation=[JVM]"), plan); + } NativeParity.assertParity(FlinkJsonJvmSqlHarnessTest::dynamicPaths, sql); } @@ -141,7 +166,8 @@ void arrayInputUsesTheGeneratedBatchBridge() throws Exception { var table = StreamTableEnvironment.create(env); table.createTemporaryView( "inputs", - env.fromData( + fromData( + env, List.of(Row.of((Object) new String[] {"value", null})), Types.ROW_NAMED(new String[] {"a"}, Types.OBJECT_ARRAY(Types.STRING)))); return table; @@ -264,7 +290,8 @@ private static TableEnvironment dynamicPaths() { } table.createTemporaryView( "inputs", - env.fromData( + fromData( + env, rows, Types.ROW_NAMED(new String[] {"id", "s", "p"}, Types.INT, Types.STRING, Types.STRING))); return table; diff --git a/src/test/java/tech/streamfusion/FlinkJsonObjectSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkJsonObjectSqlHarnessTest.java index ec25ce55b..5d586bd45 100644 --- a/src/test/java/tech/streamfusion/FlinkJsonObjectSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkJsonObjectSqlHarnessTest.java @@ -74,6 +74,7 @@ void decimalValuesShareExactScalarFormattingAndNullPolicies() throws Exception { @Test void containersAndNestedRawJsonUseHostSemantics() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("JSON"); for (String expression : new String[] { "CAST(n AS DOUBLE)", diff --git a/src/test/java/tech/streamfusion/FlinkJsonQuoteSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkJsonQuoteSqlHarnessTest.java index 3e61606cf..f63a1d42e 100644 --- a/src/test/java/tech/streamfusion/FlinkJsonQuoteSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkJsonQuoteSqlHarnessTest.java @@ -3,6 +3,11 @@ import org.junit.jupiter.api.Test; class FlinkJsonQuoteSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("JSON_QUOTE"); + } + @Test void functionAlsoRunsInsideTheNativePredicate() throws Exception { NativeParity.assertParity( diff --git a/src/test/java/tech/streamfusion/FlinkJsonReturningHostContractTest.java b/src/test/java/tech/streamfusion/FlinkJsonReturningHostContractTest.java index f7821db9c..328bc4a86 100644 --- a/src/test/java/tech/streamfusion/FlinkJsonReturningHostContractTest.java +++ b/src/test/java/tech/streamfusion/FlinkJsonReturningHostContractTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -14,8 +15,12 @@ class FlinkJsonReturningHostContractTest { @ParameterizedTest - @CsvSource({"BOOLEAN,java.lang.Boolean", "DOUBLE,java.math.BigDecimal"}) - void integerJsonTokenFailsHostReturningConversion(String type, String targetClass) { + @org.junit.jupiter.params.provider.ValueSource(strings = {"BOOLEAN", "DOUBLE"}) + void integerJsonTokenFailsHostReturningConversion(String type) { + String targetClass = + type.equals("DOUBLE") + ? tech.streamfusion.compat.FlinkTestCapabilities.JSON_FRACTION_CLASS + : "java.lang.Boolean"; TableEnvironment table = runtimeInput("{\"v\":1}"); String expression = "JSON_VALUE(doc, '$.v' RETURNING " + type + " NULL ON ERROR)"; String runtimeSql = "SELECT " + expression + " FROM inputs"; @@ -58,8 +63,9 @@ private static TableEnvironment runtimeInput(String document) { var env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); var table = StreamTableEnvironment.create(env); - table.createTemporaryView("inputs", env.fromData( - Types.ROW_NAMED(new String[] {"doc"}, Types.STRING), Row.of(document))); + table.createTemporaryView( + "inputs", + fromData(env, Types.ROW_NAMED(new String[] {"doc"}, Types.STRING), Row.of(document))); return table; } } diff --git a/src/test/java/tech/streamfusion/FlinkJsonStringIdentitySqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkJsonStringIdentitySqlHarnessTest.java index 5a7e683ab..fe3edfc39 100644 --- a/src/test/java/tech/streamfusion/FlinkJsonStringIdentitySqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkJsonStringIdentitySqlHarnessTest.java @@ -16,6 +16,8 @@ class FlinkJsonStringIdentitySqlHarnessTest { @ParameterizedTest @ValueSource(strings = {"JSON_VALUE(s, '$')", "JSON_UNQUOTE(s)"}) void consumersKeepSurrogatesDistinctFromQuestionMarks(String value) throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("JSON_QUOTE"); + tech.streamfusion.compat.FlinkTestCapabilities.requireJsonFunctions(value); NativeParity.assertParity( () -> environment(5003), "SELECT id, " @@ -43,6 +45,7 @@ void consumersKeepSurrogatesDistinctFromQuestionMarks(String value) throws Excep @ParameterizedTest @ValueSource(strings = {"JSON_VALUE(s, '$')", "JSON_UNQUOTE(s)"}) void filtersKeepOnlyTheActualQuestionMark(String value) throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireJsonFunctions(value); String sql = "SELECT id FROM inputs WHERE " + value + " = '?'"; for (boolean nativeEnabled : new boolean[] {false, true}) { TableEnvironment tables = environment(4); @@ -68,6 +71,7 @@ void filtersKeepOnlyTheActualQuestionMark(String value) throws Exception { "JSON_VALUE(s, '$') IS NULL" }) void nestedConsumersRemainInOneGeneratedExpression(String expression) throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireJsonFunctions(expression); NativeParity.assertParity(() -> environment(5003), "SELECT id, " + expression + " FROM inputs"); } @@ -84,6 +88,7 @@ void nestedConsumersRemainInOneGeneratedExpression(String expression) throws Exc + "GROUP BY LOWER(JSON_VALUE(s, '$'))" }) void stringResultsCrossingOperatorsFallBackExplicitly(String sql) throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireJsonFunctions(sql); NativeParity.assertFallbackReasonContains( () -> environment(32), sql, "JSON string identity requires a final projection"); } @@ -98,6 +103,7 @@ void scalarResultsCanStillFeedNativeAggregation() throws Exception { @Test void fusedUnquoteConsumerPreservesTheHostException() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("JSON_UNQUOTE"); NativeFailureParity.run( () -> TextTimeFunctionTestInputs.textRows("\"a\" \\u1\""), "SELECT JSON_UNQUOTE(s) = '?' FROM inputs") @@ -133,6 +139,7 @@ void fusedConsumersUseFlinksSelectorAndPolicyRules(String expression) throws Exc @ParameterizedTest @ValueSource(strings = {"\\uD800", "\\uDC00", "\\uD83D\\uDE00"}) void constantFoldedJsonResultsKeepTheirIdentity(String escaped) throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("JSON_UNQUOTE"); NativeParity.assertParity( () -> TextTimeFunctionTestInputs.textRows("?", "😀", "", null), "SELECT id, JSON_VALUE('\"" diff --git a/src/test/java/tech/streamfusion/FlinkJsonUnquoteSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkJsonUnquoteSqlHarnessTest.java index f5afd9021..e39d1b841 100644 --- a/src/test/java/tech/streamfusion/FlinkJsonUnquoteSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkJsonUnquoteSqlHarnessTest.java @@ -3,6 +3,11 @@ import org.junit.jupiter.api.Test; class FlinkJsonUnquoteSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("JSON_UNQUOTE"); + } + @Test void jacksonPrefixValidationKeepsTheTrailingTextBehavior() throws Exception { NativeParity.assertParity( diff --git a/src/test/java/tech/streamfusion/FlinkLimitSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkLimitSqlHarnessTest.java index 7527f5468..48b410242 100644 --- a/src/test/java/tech/streamfusion/FlinkLimitSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkLimitSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -92,7 +94,8 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 5L), Row.of(1L, 3L), @@ -103,7 +106,10 @@ private static TableEnvironment environment() { tEnv.createTemporaryView( "src", source, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); return tEnv; } } diff --git a/src/test/java/tech/streamfusion/FlinkLookupJoinSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkLookupJoinSqlHarnessTest.java index 2635fe568..427c83ab0 100644 --- a/src/test/java/tech/streamfusion/FlinkLookupJoinSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkLookupJoinSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.function.Supplier; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -99,7 +101,8 @@ private static Supplier environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream bid = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"auction", "price"}, Types.LONG, Types.LONG), Row.of(1L, 100L), Row.of(2L, 200L), diff --git a/src/test/java/tech/streamfusion/FlinkLtzDateTimeSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkLtzDateTimeSqlHarnessTest.java index 6aaf44e32..262ef2c2c 100644 --- a/src/test/java/tech/streamfusion/FlinkLtzDateTimeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkLtzDateTimeSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Instant; import java.time.ZoneId; import org.apache.flink.api.common.typeinfo.Types; @@ -30,8 +32,8 @@ class FlinkLtzDateTimeSqlHarnessTest { void dateFormatLtzDefaultMatchesHost() throws Exception { NativeParity.assertParity( FlinkLtzDateTimeSqlHarnessTest::environment, - "SELECT id, DATE_FORMAT(rt, 'yyyy-MM-dd') AS d, DATE_FORMAT(rt, 'yyyy-MM-dd HH:mm:ss') AS dt" - + " FROM t"); + "SELECT id, DATE_FORMAT(rt, 'yyyy-MM-dd') AS d, DATE_FORMAT(rt, 'yyyy-MM-dd HH:mm:ss') AS" + + " dt FROM t"); } @Test @@ -86,11 +88,14 @@ private static TableEnvironment environment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.getConfig().setLocalTimeZone(ZoneId.of("America/New_York")); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "rt"}, Types.LONG, Types.INSTANT), Row.of(1L, Instant.parse("2024-07-01T12:00:00Z")), // DST: NY = 08:00, UTC−4 Row.of(2L, Instant.parse("2024-01-01T12:00:00Z")), // standard: NY = 07:00, UTC−5 - Row.of(3L, Instant.parse("2024-07-01T02:30:00Z")), // NY = prior day 22:30 — date shifts back + Row.of( + 3L, + Instant.parse("2024-07-01T02:30:00Z")), // NY = prior day 22:30 — date shifts back Row.of(4L, (Instant) null)); tEnv.createTemporaryView( "t", diff --git a/src/test/java/tech/streamfusion/FlinkLtzWindowBoundarySqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkLtzWindowBoundarySqlHarnessTest.java index a41f9bb09..375678fac 100644 --- a/src/test/java/tech/streamfusion/FlinkLtzWindowBoundarySqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkLtzWindowBoundarySqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.time.Duration; import java.time.Instant; @@ -176,7 +177,8 @@ private static TableEnvironment environment(String zone, boolean includeNull) { if (includeNull) rows.add(Row.of(9, 1L, null)); table.createTemporaryView( "src", - env.fromData( + fromData( + env, rows, Types.ROW_NAMED( new String[] {"id", "k", "ts"}, Types.INT, Types.LONG, Types.INSTANT)) diff --git a/src/test/java/tech/streamfusion/FlinkMemoryAccountingTest.java b/src/test/java/tech/streamfusion/FlinkMemoryAccountingTest.java index 0691a9c9d..7ce61e1b0 100644 --- a/src/test/java/tech/streamfusion/FlinkMemoryAccountingTest.java +++ b/src/test/java/tech/streamfusion/FlinkMemoryAccountingTest.java @@ -3,11 +3,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import tech.streamfusion.planner.NativePlanner; -import tech.streamfusion.planner.PhysicalPlanScan; -import java.time.ZoneId; import java.nio.file.Files; import java.nio.file.Path; +import java.time.ZoneId; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.DeploymentOptions; import org.apache.flink.configuration.MemorySize; @@ -18,6 +16,8 @@ import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.ExceptionUtils; import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.NativePlanner; +import tech.streamfusion.planner.PhysicalPlanScan; /** * End-to-end task-off-heap accounting: native state draws from the TaskManager-wide StreamFusion @@ -98,10 +98,14 @@ private static void writeHighCardinalityInput(Path directory) throws Exception { } private static void drain(StreamTableEnvironment tEnv, String sql) throws Exception { - try (CloseableIterator iterator = tEnv.executeSql(sql).collect()) { + var result = tEnv.executeSql(sql); + try (CloseableIterator iterator = result.collect()) { while (iterator.hasNext()) { iterator.next(); } + } catch (Exception failure) { + throw NativeFailureParity.terminalFailure( + result.getJobClient().orElseThrow().getJobExecutionResult(), failure); } } } diff --git a/src/test/java/tech/streamfusion/FlinkMultiwayJoinSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkMultiwayJoinSqlHarnessTest.java index 8c8ba75ed..7f26e9f93 100644 --- a/src/test/java/tech/streamfusion/FlinkMultiwayJoinSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkMultiwayJoinSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.LocalDateTime; import java.util.function.Supplier; import org.apache.flink.api.common.typeinfo.Types; @@ -39,7 +41,8 @@ private static Supplier environment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream bid = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"bidder", "price", "channel"}, Types.LONG, @@ -50,12 +53,14 @@ private static Supplier environment() { Row.of(2L, 200L, "baidu"), Row.of(3L, 300L, "facebook")); DataStream person = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "name"}, Types.LONG, Types.STRING), Row.of(1L, "alice"), Row.of(2L, "bob")); DataStream auction = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"seller", "itemName", "dateTime"}, Types.LONG, diff --git a/src/test/java/tech/streamfusion/FlinkNestedJsonJvmSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkNestedJsonJvmSqlHarnessTest.java index c6cfbbd23..971397432 100644 --- a/src/test/java/tech/streamfusion/FlinkNestedJsonJvmSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkNestedJsonJvmSqlHarnessTest.java @@ -1,6 +1,7 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.ArrayList; import java.util.List; @@ -49,7 +50,7 @@ class FlinkNestedJsonJvmSqlHarnessTest { "JSON_QUERY(s, p), ARRAY[id, CAST(NULL AS INT), id + 1]" }) void nestedInputsAndOutputsMatchTheGeneratedHostCalc(String projection) throws Exception { - String sql = "SELECT id, " + projection + " FROM inputs"; + String sql = compatibleQuery("SELECT id, " + projection + " FROM inputs", false); String plan = NativePlanner.explain(environment(false), sql); assertTrue(plan.contains("NativeCalc") && plan.contains("jsonEvaluation=[JVM]"), plan); NativeParity.assertParity(() -> environment(false), sql); @@ -64,7 +65,8 @@ void decimalAndNanoTimestampArraysRetainTheirDeclaredValues() throws Exception { var table = StreamTableEnvironment.create(env); table.createTemporaryView( "inputs", - env.fromData( + fromData( + env, List.of( Row.of( new java.math.BigDecimal[] {new java.math.BigDecimal("12.345"), null}, @@ -90,7 +92,8 @@ void decimalAndNanoTimestampArraysRetainTheirDeclaredValues() throws Exception { @Test void nestedResultsKeepRawChangelogKindsAfterFiltering() throws Exception { NativeParity.assertKindedParity( - () -> environment(true), "SELECT id, a, r, JSON_QUERY(s, p) FROM inputs WHERE id <> 2"); + () -> environment(true), + compatibleQuery("SELECT id, a, r, JSON_QUERY(s, p) FROM inputs WHERE id <> 2", true)); } @ParameterizedTest @@ -103,7 +106,8 @@ void mapBoundariesRemainExplicitFallback(String projection) throws Exception { var table = StreamTableEnvironment.create(env); table.createTemporaryView( "inputs", - env.fromData( + fromData( + env, List.of(Row.of(Map.of("a", 1), "{}")), Types.ROW_NAMED( new String[] {"m", "s"}, Types.MAP(Types.STRING, Types.INT), Types.STRING))); @@ -113,6 +117,20 @@ void mapBoundariesRemainExplicitFallback(String projection) throws Exception { "row-fused UDF"); } + private static String compatibleQuery(String sql, boolean changelog) { + if (tech.streamfusion.compat.FlinkTestCapabilities.DYNAMIC_JSON_QUERY + || !sql.contains("JSON_QUERY(s, p)")) { + return sql; + } + NativeFailureParity.run(() -> environment(changelog), sql) + .assertFailure( + org.apache.flink.table.planner.codegen.CodeGenException.class, + "Unsupported call: JSON_QUERY", + NativeFailureParity.Phase.PLANNING, + NativeFailureParity.Route.FALLBACK); + return sql.replace("JSON_QUERY(s, p)", "JSON_QUERY(s, '$.items')"); + } + private static TableEnvironment environment(boolean changelog) { var env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); @@ -157,9 +175,9 @@ private static TableEnvironment environment(boolean changelog) { Row deleted = Row.copy(rows.get(3)); deleted.setKind(RowKind.DELETE); rows.add(deleted); - table.createTemporaryView("inputs", table.fromChangelogStream(env.fromData(rows, INPUT))); + table.createTemporaryView("inputs", table.fromChangelogStream(fromData(env, rows, INPUT))); } else { - table.createTemporaryView("inputs", env.fromData(rows, INPUT)); + table.createTemporaryView("inputs", fromData(env, rows, INPUT)); } return table; } diff --git a/src/test/java/tech/streamfusion/FlinkNullSafeSemiAntiJoinSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkNullSafeSemiAntiJoinSqlHarnessTest.java index 5497d5756..58b343fd3 100644 --- a/src/test/java/tech/streamfusion/FlinkNullSafeSemiAntiJoinSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkNullSafeSemiAntiJoinSqlHarnessTest.java @@ -1,6 +1,7 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.math.BigDecimal; import java.time.LocalDateTime; @@ -42,9 +43,12 @@ void mixedCompositeKeysPreserveOrdinaryNullFiltering(boolean anti) throws Except @ParameterizedTest @ValueSource(booleans = {false, true}) void changelogSourcesPreserveDuplicateMatchCountsAndDeletes(boolean anti) throws Exception { - assertNative("SELECT a.s, a.i, a.v FROM A a WHERE " + (anti ? "NOT " : "") - + "EXISTS (SELECT 1 FROM B b WHERE a.s IS NOT DISTINCT FROM b.s " - + "AND a.d IS NOT DISTINCT FROM b.d AND a.ts IS NOT DISTINCT FROM b.ts AND a.v < b.v)", true); + assertNative( + "SELECT a.s, a.i, a.v FROM A a WHERE " + + (anti ? "NOT " : "") + + "EXISTS (SELECT 1 FROM B b WHERE a.s IS NOT DISTINCT FROM b.s " + + "AND a.d IS NOT DISTINCT FROM b.d AND a.ts IS NOT DISTINCT FROM b.ts AND a.v < b.v)", + true); } @ParameterizedTest @@ -71,14 +75,29 @@ private static TableEnvironment environment(boolean changes) { var schema = Schema.newBuilder().column("s", DataTypes.STRING()).column("i", DataTypes.INT()) .column("l", DataTypes.BIGINT()).column("d", DataTypes.DECIMAL(12, 2)) .column("ts", DataTypes.TIMESTAMP(9)).column("v", DataTypes.INT()).build(); - Row[] left = {row(null, null, 1), row(null, 1, 2), row("a", 1, 3), row("a", 1, 3), row("missing", 3, 4)}; - Row[] right = changes - ? new Row[] {row(null, null, 8), row(null, null, 8), row("a", 1, 9), - change(RowKind.DELETE, null, null, 8), change(RowKind.UPDATE_BEFORE, "a", 1, 9), - change(RowKind.UPDATE_AFTER, "a", 1, 1), row("right", 4, 9)} - : new Row[] {row(null, null, 8), row(null, 1, 8), row("a", 1, 9), row("a", 1, 9), row("right", 4, 9)}; - table.createTemporaryView("A", table.fromChangelogStream(env.fromData(type, left), schema)); - table.createTemporaryView("B", table.fromChangelogStream(env.fromData(type, right), schema)); + Row[] left = { + row(null, null, 1), row(null, 1, 2), row("a", 1, 3), row("a", 1, 3), row("missing", 3, 4) + }; + Row[] right = + changes + ? new Row[] { + row(null, null, 8), + row(null, null, 8), + row("a", 1, 9), + change(RowKind.DELETE, null, null, 8), + change(RowKind.UPDATE_BEFORE, "a", 1, 9), + change(RowKind.UPDATE_AFTER, "a", 1, 1), + row("right", 4, 9) + } + : new Row[] { + row(null, null, 8), + row(null, 1, 8), + row("a", 1, 9), + row("a", 1, 9), + row("right", 4, 9) + }; + table.createTemporaryView("A", table.fromChangelogStream(fromData(env, type, left), schema)); + table.createTemporaryView("B", table.fromChangelogStream(fromData(env, type, right), schema)); return table; } diff --git a/src/test/java/tech/streamfusion/FlinkOverAggregateSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkOverAggregateSqlHarnessTest.java index 5d7444c72..ac78af7e8 100644 --- a/src/test/java/tech/streamfusion/FlinkOverAggregateSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkOverAggregateSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -53,6 +55,9 @@ void rowNumberMatchesHost() throws Exception { @Test void rankAndDenseRankMatchHost() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.RANK_OVER_AGGREGATE, + "Flink 1.18 fails to construct its OVER rank aggregate: null order-key array"); // RANK and DENSE_RANK over (PARTITION BY k ORDER BY rt). Tied rowtimes would share a rank; // tie semantics are covered by the native test, here we verify routing + host parity. NativeParity.assertParity( @@ -103,11 +108,13 @@ void boundedRowsFrameMatchesHost() throws Exception { @Test void boundedRowsFrameUnpartitionedMatchesHost() throws Exception { - // ROWS BETWEEN 2 PRECEDING over the whole stream: with globally distinct rowtimes the frame is the + // ROWS BETWEEN 2 PRECEDING over the whole stream: with globally distinct rowtimes the frame is + // the // current row and the two before it in rowtime order. NativeParity.assertParity( FlinkOverAggregateSqlHarnessTest::boundedEnvironment, - "SELECT v, SUM(v) OVER (ORDER BY rt ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS s FROM src"); + "SELECT v, SUM(v) OVER (ORDER BY rt ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS s FROM" + + " src"); } @Test @@ -185,9 +192,9 @@ private static TableEnvironment boundedEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( - Types.ROW_NAMED( - new String[] {"k", "v", "ts"}, Types.LONG, Types.LONG, Types.LONG), + fromData( + env, + Types.ROW_NAMED(new String[] {"k", "v", "ts"}, Types.LONG, Types.LONG, Types.LONG), Row.of(1L, 10L, 1000L), Row.of(2L, 20L, 1500L), Row.of(1L, 30L, 2000L), @@ -218,7 +225,8 @@ private static TableEnvironment narrowEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "si", "ti", "fl", "ts"}, Types.LONG, @@ -274,7 +282,8 @@ private static TableEnvironment proctimeEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 100L), @@ -325,7 +334,8 @@ private static TableEnvironment environment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); // Out-of-order within the bound so the running totals exercise rowtime ordering and ties. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "v", "ts", "b"}, Types.LONG, diff --git a/src/test/java/tech/streamfusion/FlinkOverLateAdmissionReproTest.java b/src/test/java/tech/streamfusion/FlinkOverLateAdmissionReproTest.java index 7fc673da2..96c079501 100644 --- a/src/test/java/tech/streamfusion/FlinkOverLateAdmissionReproTest.java +++ b/src/test/java/tech/streamfusion/FlinkOverLateAdmissionReproTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.eventtime.Watermark; import org.apache.flink.api.common.eventtime.WatermarkGenerator; import org.apache.flink.api.common.eventtime.WatermarkOutput; @@ -89,18 +91,26 @@ private static TableEnvironment input(Row[] rows) { env.setParallelism(1); StreamTableEnvironment table = StreamTableEnvironment.create(env); table.getConfig().setLocalTimeZone(java.time.ZoneOffset.UTC); - var stream = env.fromData( - Types.ROW_NAMED(new String[] {"g", "millis", "v"}, Types.INT, Types.LONG, Types.LONG), rows) - .assignTimestampsAndWatermarks(WatermarkStrategy.forGenerator(context -> - new WatermarkGenerator() { - @Override - public void onEvent(Row event, long timestamp, WatermarkOutput output) { - output.emitWatermark(new Watermark(timestamp)); - } + var stream = + fromData( + env, + Types.ROW_NAMED( + new String[] {"g", "millis", "v"}, Types.INT, Types.LONG, Types.LONG), + rows) + .assignTimestampsAndWatermarks( + WatermarkStrategy.forGenerator( + context -> + new WatermarkGenerator() { + @Override + public void onEvent( + Row event, long timestamp, WatermarkOutput output) { + output.emitWatermark(new Watermark(timestamp)); + } - @Override - public void onPeriodicEmit(WatermarkOutput output) {} - }).withTimestampAssigner((event, previous) -> (Long) event.getField(1))); + @Override + public void onPeriodicEmit(WatermarkOutput output) {} + }) + .withTimestampAssigner((event, previous) -> (Long) event.getField(1))); table.createTemporaryView("n", stream, Schema.newBuilder().column("g", DataTypes.INT()).column("millis", DataTypes.BIGINT()) .column("v", DataTypes.BIGINT()) diff --git a/src/test/java/tech/streamfusion/FlinkOverlaySqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkOverlaySqlHarnessTest.java index a8d939c43..c7e467b08 100644 --- a/src/test/java/tech/streamfusion/FlinkOverlaySqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkOverlaySqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.ArrayList; import java.util.List; import org.apache.flink.api.common.typeinfo.Types; @@ -47,7 +49,8 @@ void overlayMatchesFlinkAtEverySurrogateBoundary() throws Exception { StreamTableEnvironment tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "overlays", - env.fromData( + fromData( + env, rows, Types.ROW_NAMED( new String[] {"id", "s", "t", "p", "n"}, diff --git a/src/test/java/tech/streamfusion/FlinkParquetSinkFallbackTest.java b/src/test/java/tech/streamfusion/FlinkParquetSinkFallbackTest.java index 5e30759b0..582a555df 100644 --- a/src/test/java/tech/streamfusion/FlinkParquetSinkFallbackTest.java +++ b/src/test/java/tech/streamfusion/FlinkParquetSinkFallbackTest.java @@ -3,11 +3,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import tech.streamfusion.planner.NativePlanner; -import tech.streamfusion.planner.PhysicalPlanScan; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.NativePlanner; +import tech.streamfusion.planner.PhysicalPlanScan; /** * Every filesystem sink configuration the native writer cannot honor 1:1 declines with a recorded @@ -91,6 +91,9 @@ void enforcedCharacterLengthFallsBackToFlinksEnforcer() { @Test void enforcedBinaryLengthFallsBackToFlinksEnforcer() { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.SINK_LENGTH_ERROR, + "Flink 1.18 only provides IGNORE and TRIM_PAD sink length policies"); assertConstraintFallsBack( "fixed BINARY(3), limited VARBINARY(3)", "fixed BINARY(3), limited VARBINARY(3)", diff --git a/src/test/java/tech/streamfusion/FlinkParquetSinkS3IntegrationTest.java b/src/test/java/tech/streamfusion/FlinkParquetSinkS3IntegrationTest.java index e7f9907d1..4cf918056 100644 --- a/src/test/java/tech/streamfusion/FlinkParquetSinkS3IntegrationTest.java +++ b/src/test/java/tech/streamfusion/FlinkParquetSinkS3IntegrationTest.java @@ -2,15 +2,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; -import tech.streamfusion.planner.NativePlanner; -import tech.streamfusion.planner.PhysicalPlanScan; import java.util.ArrayList; import java.util.List; +import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; -import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.DataTypes; @@ -21,6 +20,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.testcontainers.containers.MinIOContainer; +import tech.streamfusion.planner.NativePlanner; +import tech.streamfusion.planner.PhysicalPlanScan; /** * The native Parquet sink writing to an object store end to end: a partitioned INSERT lands on @@ -78,7 +79,8 @@ private static PhysicalPlanScan writeInsert(String location) throws Exception { + "'sink.partition-commit.policy.kind' = 'success-file')"); // Fixed rows through a DataStream view keep the plan a bare source→sink (no Calc to gate on). DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"dt", "v"}, Types.STRING, Types.INT), Row.of("a", 1), Row.of("a", 2), diff --git a/src/test/java/tech/streamfusion/FlinkParquetSinkSmokeTest.java b/src/test/java/tech/streamfusion/FlinkParquetSinkSmokeTest.java index ffbdbea39..db423445e 100644 --- a/src/test/java/tech/streamfusion/FlinkParquetSinkSmokeTest.java +++ b/src/test/java/tech/streamfusion/FlinkParquetSinkSmokeTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.nio.file.Files; import java.nio.file.Path; @@ -37,7 +38,8 @@ void hostWritesParquetFiles() throws Exception { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.INT), Row.of(1L, 10), Row.of(2L, 20), @@ -74,20 +76,16 @@ void nativeWritesNestedStructListAndMap() throws Exception { String columns = "id BIGINT, details ROW>, tags MAP"; DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "details", "tags"}, Types.LONG, Types.ROW_NAMED( - new String[] {"name", "scores"}, - Types.STRING, - Types.OBJECT_ARRAY(Types.INT)), + new String[] {"name", "scores"}, Types.STRING, Types.OBJECT_ARRAY(Types.INT)), Types.MAP(Types.STRING, Types.LONG)), Row.of(1L, Row.of("first", new Integer[] {1, 2}), Map.of("x", 10L)), - Row.of( - 2L, - Row.of(null, new Integer[] {3, null}), - Collections.singletonMap("y", null))); + Row.of(2L, Row.of(null, new Integer[] {3, null}), Collections.singletonMap("y", null))); tEnv.createTemporaryView( "nested_source", source, diff --git a/src/test/java/tech/streamfusion/FlinkParquetSinkSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkParquetSinkSqlHarnessTest.java index 700912b73..e438f9494 100644 --- a/src/test/java/tech/streamfusion/FlinkParquetSinkSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkParquetSinkSqlHarnessTest.java @@ -2,9 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; -import tech.streamfusion.planner.NativePlanner; -import tech.streamfusion.planner.PhysicalPlanScan; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -19,6 +18,8 @@ import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.NativePlanner; +import tech.streamfusion.planner.PhysicalPlanScan; /** The native Parquet sink writes the same data the host's filesystem+parquet sink does. */ class FlinkParquetSinkSqlHarnessTest { @@ -41,7 +42,8 @@ private static void writeInsert(Path directory, boolean useNative) throws Except StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.INT), Row.of(1L, 10), Row.of(2L, 20), diff --git a/src/test/java/tech/streamfusion/FlinkPartitionedParquetSinkSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkPartitionedParquetSinkSqlHarnessTest.java index 91617e4cf..b812d178a 100644 --- a/src/test/java/tech/streamfusion/FlinkPartitionedParquetSinkSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkPartitionedParquetSinkSqlHarnessTest.java @@ -2,9 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; -import tech.streamfusion.planner.NativePlanner; -import tech.streamfusion.planner.PhysicalPlanScan; import java.io.File; import java.math.BigDecimal; import java.nio.file.Files; @@ -28,6 +27,8 @@ import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.util.HadoopInputFile; import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.NativePlanner; +import tech.streamfusion.planner.PhysicalPlanScan; /** * A PARTITIONED BY table through the native sink matches the host end to end: the same rows read @@ -80,7 +81,8 @@ private static void writeInsert(Path directory, boolean useNative) throws Except // test). The 1969-12-31T23:59:59.998500 timestamp lands between milliseconds: written at // millis precision it must floor to .998, not truncate toward zero to .999. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] { "source_partition", @@ -205,7 +207,8 @@ private static TreeSet successFiles(Path directory) { /** The Parquet message type of one committed part file — identical schemas prove type parity. */ private static String footerSchema(Path directory) throws Exception { File partition = new File(directory.toFile(), "dt=a"); - File[] parts = partition.listFiles((dir, name) -> !name.startsWith(".") && !name.startsWith("_")); + File[] parts = + partition.listFiles((dir, name) -> !name.startsWith(".") && !name.startsWith("_")); assertTrue(parts != null && parts.length > 0, "no committed part file under dt=a"); try (ParquetFileReader reader = ParquetFileReader.open( diff --git a/src/test/java/tech/streamfusion/FlinkPlainTimestampTvfSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkPlainTimestampTvfSqlHarnessTest.java index 95b7acce8..5c7821ef6 100644 --- a/src/test/java/tech/streamfusion/FlinkPlainTimestampTvfSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkPlainTimestampTvfSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.time.Duration; import java.time.Instant; @@ -167,7 +168,8 @@ private static TableEnvironment environment(String zone, boolean ltz, boolean in } table.createTemporaryView( "src", - env.fromData( + fromData( + env, rows, Types.ROW_NAMED( new String[] {"id", "k", "ts"}, diff --git a/src/test/java/tech/streamfusion/FlinkPowerSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkPowerSqlHarnessTest.java index 3a3ef124a..2d4f65829 100644 --- a/src/test/java/tech/streamfusion/FlinkPowerSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkPowerSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -55,7 +57,8 @@ private static TableEnvironment input() { } table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "x", "y"}, Types.INT, Types.DOUBLE, Types.DOUBLE), rows)); return table; diff --git a/src/test/java/tech/streamfusion/FlinkRandomSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkRandomSqlHarnessTest.java index 3edcb4817..ad955b508 100644 --- a/src/test/java/tech/streamfusion/FlinkRandomSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkRandomSqlHarnessTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.HashSet; import java.util.Set; @@ -40,12 +41,16 @@ void floatingNegationPreservesSignedZeroAndSpecialValues() throws Exception { var env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); var table = StreamTableEnvironment.create(env); - table.createTemporaryView("numbers", env.fromData( - Types.ROW_NAMED(new String[] {"d", "f"}, Types.DOUBLE, Types.FLOAT), - Row.of(0.0, 0.0f), Row.of(-0.0, -0.0f), - Row.of(Double.NaN, Float.NaN), - Row.of(Double.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY), - Row.of(null, null))); + table.createTemporaryView( + "numbers", + fromData( + env, + Types.ROW_NAMED(new String[] {"d", "f"}, Types.DOUBLE, Types.FLOAT), + Row.of(0.0, 0.0f), + Row.of(-0.0, -0.0f), + Row.of(Double.NaN, Float.NaN), + Row.of(Double.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY), + Row.of(null, null))); return table; }, "SELECT -d, -f FROM numbers"); @@ -92,11 +97,15 @@ void invalidBoundsFailOnlyWhenEvaluated(int bound) { for (boolean nativeRun : new boolean[] {false, true}) { TableEnvironment table = input(Row.of(1, 42, bound)); var scan = nativeRun ? NativePlanner.install(table) : null; - Exception failure = assertThrows(Exception.class, () -> { - try (var rows = table.executeSql("SELECT RAND_INTEGER(seed, bound) FROM inputs").collect()) { - while (rows.hasNext()) rows.next(); - } - }); + Exception failure = + assertThrows( + Exception.class, + () -> { + try (var rows = + table.executeSql("SELECT RAND_INTEGER(seed, bound) FROM inputs").collect()) { + while (rows.hasNext()) rows.next(); + } + }); StringBuilder messages = new StringBuilder(); for (Throwable cause = failure; cause != null; cause = cause.getCause()) { messages.append(cause.getMessage()); @@ -138,8 +147,12 @@ private static TableEnvironment input(Row... rows) { var env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); var table = StreamTableEnvironment.create(env); - table.createTemporaryView("inputs", env.fromData( - Types.ROW_NAMED(new String[] {"id", "seed", "bound"}, Types.INT, Types.INT, Types.INT), rows)); + table.createTemporaryView( + "inputs", + fromData( + env, + Types.ROW_NAMED(new String[] {"id", "seed", "bound"}, Types.INT, Types.INT, Types.INT), + rows)); return table; } } diff --git a/src/test/java/tech/streamfusion/FlinkRegexpExtractSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkRegexpExtractSqlHarnessTest.java index e9bc61e9f..9b56e3761 100644 --- a/src/test/java/tech/streamfusion/FlinkRegexpExtractSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkRegexpExtractSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.function.Supplier; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -50,7 +52,8 @@ private static Supplier environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream bid = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"channel", "url"}, Types.STRING, Types.STRING), Row.of("Apple", "https://site/x?channel_id=apple_1&foo=1"), Row.of("Google", "https://site/y?a=1&channel_id=goog_2"), diff --git a/src/test/java/tech/streamfusion/FlinkRegularJoinSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkRegularJoinSqlHarnessTest.java index 5487aee93..c0a31570f 100644 --- a/src/test/java/tech/streamfusion/FlinkRegularJoinSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkRegularJoinSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -186,6 +188,7 @@ void stateTtlMatchesHost() throws Exception { @Test void perSideStateTtlHintRoutesAndMatchesHost() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireStateTtlHint(); // A per-side STATE_TTL hint (KV syntax keyed by table alias, 0 = left / 1 = right internally) // with the job retention at 0: the hint alone must switch each side's retention on the native // operator, matching Flink's hint-over-config precedence. @@ -214,13 +217,15 @@ private static TableEnvironment environment() { tEnv.getConfig().set("table.optimizer.agg-phase-strategy", "ONE_PHASE"); DataStream a = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(1L, 20L), Row.of(2L, 30L)); DataStream b = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "w"}, Types.LONG, Types.LONG), Row.of(1L, 100L), Row.of(2L, 200L), @@ -229,11 +234,17 @@ private static TableEnvironment environment() { tEnv.createTemporaryView( "A", a, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); tEnv.createTemporaryView( "B", b, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("w", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("w", DataTypes.BIGINT()) + .build()); return tEnv; } diff --git a/src/test/java/tech/streamfusion/FlinkRetractingTopNSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkRetractingTopNSqlHarnessTest.java index ade00bd71..07b13357b 100644 --- a/src/test/java/tech/streamfusion/FlinkRetractingTopNSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkRetractingTopNSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -19,16 +21,14 @@ class FlinkRetractingTopNSqlHarnessTest { private static final String TOP_N = - "SELECT g, k, total FROM (" - + " SELECT g, k, total, ROW_NUMBER() OVER (PARTITION BY g ORDER BY total DESC, k DESC) AS rn" - + " FROM (SELECT g, k, SUM(v) AS total FROM src GROUP BY g, k)" - + ") WHERE rn <= 2"; + "SELECT g, k, total FROM ( SELECT g, k, total, ROW_NUMBER() OVER (PARTITION BY g ORDER BY" + + " total DESC, k DESC) AS rn FROM (SELECT g, k, SUM(v) AS total FROM src GROUP BY g," + + " k)) WHERE rn <= 2"; private static final String TOP_N_WITH_RANK = - "SELECT g, k, total, rn FROM (" - + " SELECT g, k, total, ROW_NUMBER() OVER (PARTITION BY g ORDER BY total DESC, k DESC) AS rn" - + " FROM (SELECT g, k, SUM(v) AS total FROM src GROUP BY g, k)" - + ") WHERE rn <= 2"; + "SELECT g, k, total, rn FROM ( SELECT g, k, total, ROW_NUMBER() OVER (PARTITION BY g ORDER" + + " BY total DESC, k DESC) AS rn FROM (SELECT g, k, SUM(v) AS total FROM src GROUP BY g," + + " k)) WHERE rn <= 2"; @Test void retractingTopNMatchesHost() throws Exception { @@ -75,9 +75,9 @@ private static TableEnvironment environment() { // Two groups; within each, several keys whose running totals cross as v accumulates, so the // per-group top-2 by total changes over the stream (exercising retraction + promotion). DataStream source = - env.fromData( - Types.ROW_NAMED( - new String[] {"g", "k", "v"}, Types.LONG, Types.LONG, Types.LONG), + fromData( + env, + Types.ROW_NAMED(new String[] {"g", "k", "v"}, Types.LONG, Types.LONG, Types.LONG), Row.of(1L, 10L, 5L), Row.of(1L, 20L, 3L), Row.of(1L, 30L, 1L), diff --git a/src/test/java/tech/streamfusion/FlinkRetractingWindowSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkRetractingWindowSqlHarnessTest.java index 16e27c54f..421759c52 100644 --- a/src/test/java/tech/streamfusion/FlinkRetractingWindowSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkRetractingWindowSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.math.BigDecimal; import java.math.RoundingMode; @@ -34,7 +35,6 @@ import org.apache.flink.runtime.testutils.MiniClusterResource; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; -import org.apache.flink.streaming.api.functions.source.legacy.SourceFunction; import org.apache.flink.streaming.util.TestStreamEnvironment; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Schema; @@ -46,9 +46,17 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import tech.streamfusion.compat.SourceFunction; import tech.streamfusion.planner.NativePlanner; class FlinkRetractingWindowSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireHostRetractingWindow() { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.RETRACTING_WINDOW_TVF, + "Flink 1.18 window table aggregates cannot consume update/delete changes"); + } + private static InMemoryReporter reporter; private static MiniClusterResource cluster; private static final Map RECOVERY = new ConcurrentHashMap<>(); @@ -158,7 +166,8 @@ void lateRetractionsChangeOnlyUnfiredWindows( table.getConfig().setLocalTimeZone(ZoneOffset.UTC); table.getConfig().set("table.optimizer.agg-phase-strategy", phase); var source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "millis", "v", "wm"}, Types.INT, @@ -837,7 +846,8 @@ void floatingAggregatesPreserveNonfiniteValuesAndSignedZero(String phase, String table.getConfig().setLocalTimeZone(ZoneOffset.UTC); table.getConfig().set("table.optimizer.agg-phase-strategy", phase); var source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "millis", "f", "v"}, Types.INT, @@ -977,7 +987,8 @@ void decimalSumAndAveragePreserveDifferentOverflowRules( table.getConfig().setLocalTimeZone(ZoneOffset.UTC); table.getConfig().set("table.optimizer.agg-phase-strategy", phase); var source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "millis", "v"}, Types.INT, Types.LONG, Types.BIG_DEC), decimalRow(RowKind.INSERT, 1, max), @@ -1184,7 +1195,8 @@ private static TableEnvironment environment(String phase) { table.getConfig().setLocalTimeZone(ZoneOffset.UTC); table.getConfig().set("table.optimizer.agg-phase-strategy", phase); var source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "millis", "v"}, Types.INT, Types.LONG, Types.LONG), Row.of(1, 1000L, 10L), diff --git a/src/test/java/tech/streamfusion/FlinkRocksDBNativeStateBackendSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkRocksDBNativeStateBackendSqlHarnessTest.java index bd19badb1..95925478b 100644 --- a/src/test/java/tech/streamfusion/FlinkRocksDBNativeStateBackendSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkRocksDBNativeStateBackendSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; @@ -40,7 +42,8 @@ void groupBySumOnRocksDBBackendMatchesHost() throws Exception { Path input = Files.createTempDirectory("rocksdb-sum-in"); writeInput(input); NativeParity.assertChangelogParity( - () -> rocksdbEnvironment(input), "SELECT k, SUM(v) AS total, COUNT(*) AS c FROM t GROUP BY k"); + () -> rocksdbEnvironment(input), + "SELECT k, SUM(v) AS total, COUNT(*) AS c FROM t GROUP BY k"); } @Test @@ -285,6 +288,7 @@ void hoppingWindowAggregateOnRocksDBBackendMatchesHost() throws Exception { @Test void sessionAggregateOnRocksDBBackendMatchesHost() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); // Event-time session aggregate: sessions extend and merge across 50 ms barriers — an // extension rewrites the same start, a merge tombstones the consumed start — and a watermark // firing merges the decoded sessions with a committed range scan. @@ -370,7 +374,8 @@ private static TableEnvironment rocksdbTemporalEnvironment() { .watermark("rt", "SOURCE_WATERMARK()") .build(); DataStream orders = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"currency", "amount", "ts"}, Types.STRING, @@ -382,12 +387,10 @@ private static TableEnvironment rocksdbTemporalEnvironment() { Row.of("USD", 3L, 450L)) .assignTimestampsAndWatermarks(watermarks); DataStream rates = - env.fromData( + fromData( + env, Types.ROW_NAMED( - new String[] {"currency", "rate", "ts"}, - Types.STRING, - Types.LONG, - Types.LONG), + new String[] {"currency", "rate", "ts"}, Types.STRING, Types.LONG, Types.LONG), Row.of("USD", 10L, 100L), Row.of("EUR", 99L, 100L), Row.of("USD", 20L, 300L)) @@ -511,7 +514,8 @@ private static TableEnvironment rocksdbRowtimeEnvironment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.getConfig().setLocalTimeZone(ZoneId.of("UTC")); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "ts"}, Types.LONG, Types.LONG, Types.LONG), Row.of(1L, 30L, 2000L), Row.of(2L, 50L, 1500L), @@ -551,7 +555,8 @@ private static TableEnvironment rocksdbBoundedOverEnvironment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.getConfig().setLocalTimeZone(ZoneId.of("UTC")); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "ts"}, Types.LONG, Types.LONG, Types.LONG), Row.of(1L, 30L, 2001L), Row.of(2L, 50L, 1501L), @@ -585,7 +590,8 @@ private static TableEnvironment rocksdbProctimeEnvironment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.getConfig().setLocalTimeZone(ZoneId.of("UTC")); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 100L), diff --git a/src/test/java/tech/streamfusion/FlinkSharedBinaryUdfSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkSharedBinaryUdfSqlHarnessTest.java index ef52e653f..61e7b0d30 100644 --- a/src/test/java/tech/streamfusion/FlinkSharedBinaryUdfSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkSharedBinaryUdfSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.math.BigDecimal; import java.util.ArrayList; @@ -180,7 +181,7 @@ private static TableEnvironment environment(int count, boolean changelog) { for (int i = 0; i < count; i++) rows.add(Row.of(i, decimals[i % decimals.length])); } var source = - env.fromData(rows, Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING)); + fromData(env, rows, Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING)); var schema = Schema.newBuilder().column("id", DataTypes.INT()).column("s", DataTypes.STRING()).build(); table.createTemporaryView( diff --git a/src/test/java/tech/streamfusion/FlinkSignSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkSignSqlHarnessTest.java index e361bb4c6..0d258779b 100644 --- a/src/test/java/tech/streamfusion/FlinkSignSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkSignSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -26,8 +28,9 @@ private static TableEnvironment input() { Double value = values[i]; rows[i] = Row.of(value, value == null ? null : value.floatValue()); } - table.createTemporaryView("n", env.fromData(Types.ROW_NAMED( - new String[] {"d", "f"}, Types.DOUBLE, Types.FLOAT), rows)); + table.createTemporaryView( + "n", + fromData(env, Types.ROW_NAMED(new String[] {"d", "f"}, Types.DOUBLE, Types.FLOAT), rows)); return table; } } diff --git a/src/test/java/tech/streamfusion/FlinkSplitIndexSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkSplitIndexSqlHarnessTest.java index 05ca796da..917bedd95 100644 --- a/src/test/java/tech/streamfusion/FlinkSplitIndexSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkSplitIndexSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -46,7 +48,8 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"auction", "url"}, Types.LONG, Types.STRING), Row.of(1L, "https://www.nexmark.com/path/to/item.html"), Row.of(2L, "http://x.com/a"), diff --git a/src/test/java/tech/streamfusion/FlinkSplitSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkSplitSqlHarnessTest.java index bea7b230a..3e7bdb8f9 100644 --- a/src/test/java/tech/streamfusion/FlinkSplitSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkSplitSqlHarnessTest.java @@ -3,6 +3,11 @@ import org.junit.jupiter.api.Test; class FlinkSplitSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("SPLIT"); + } + @Test void splitsLiteralSeparatorsAndPreservesEmptyTokens() throws Exception { NativeParity.assertParity( diff --git a/src/test/java/tech/streamfusion/FlinkStartsWithSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkStartsWithSqlHarnessTest.java index bd6c98c1f..bb78e9a36 100644 --- a/src/test/java/tech/streamfusion/FlinkStartsWithSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkStartsWithSqlHarnessTest.java @@ -3,6 +3,11 @@ import org.junit.jupiter.api.Test; class FlinkStartsWithSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("STARTSWITH"); + } + @Test void prefixArgumentsAndWildcards() throws Exception { parity( diff --git a/src/test/java/tech/streamfusion/FlinkStringBooleanCastSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkStringBooleanCastSqlHarnessTest.java index eaf3f57ce..5ec6da768 100644 --- a/src/test/java/tech/streamfusion/FlinkStringBooleanCastSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkStringBooleanCastSqlHarnessTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.List; import java.util.stream.IntStream; @@ -63,7 +64,8 @@ void malformedInputFailsLikeHost(String value) { assertThrows( Exception.class, () -> { - try (var rows = table.executeSql("SELECT CAST(s AS BOOLEAN) FROM src").collect()) { + try (var rows = + table.executeSql("SELECT CAST(s AS BOOLEAN) FROM src").collect()) { while (rows.hasNext()) { rows.next(); } @@ -91,7 +93,9 @@ void malformedInputFailsLikeHost(String value) { @Test void legacyModeReturnsNullForMalformedInput() throws Exception { NativeParity.assertParity( - () -> environment(DataTypes.STRING(), true, "true", "false", "", " true", "2", "\u662f", null), + () -> + environment( + DataTypes.STRING(), true, "true", "false", "", " true", "2", "\u662f", null), "SELECT id, CAST(s AS BOOLEAN) FROM src"); NativeParity.assertParity( () -> environment(DataTypes.STRING().notNull(), true, "TRUE", "no"), @@ -143,7 +147,8 @@ void fallibleBooleanConjunctionsRetainHostShortCircuiting() throws Exception { "SELECT id, s = 'skip' OR CAST(s AS BOOLEAN) FROM src", "SELECT id, s <> 'skip' AND CAST(s AS BOOLEAN) FROM src")) { TableEnvironment host = environment(DataTypes.STRING(), false, "skip", "true", "false"); - TableEnvironment nativeTable = environment(DataTypes.STRING(), false, "skip", "true", "false"); + TableEnvironment nativeTable = + environment(DataTypes.STRING(), false, "skip", "true", "false"); PhysicalPlanScan scan = NativePlanner.install(nativeTable); assertEquals(collect(host, sql), collect(nativeTable, sql)); assertEquals(0, scan.substitutions()); @@ -179,8 +184,7 @@ private static TableEnvironment environment(DataType type, boolean legacy, Strin IntStream.range(0, values.length).mapToObj(i -> Row.of(i, values[i])).toArray(Row[]::new); table.createTemporaryView( "src", - env.fromData( - Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING), rows), + fromData(env, Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING), rows), Schema.newBuilder().column("id", DataTypes.INT().notNull()).column("s", type).build()); return table; } diff --git a/src/test/java/tech/streamfusion/FlinkStringCaseSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkStringCaseSqlHarnessTest.java index 964fa49da..655421983 100644 --- a/src/test/java/tech/streamfusion/FlinkStringCaseSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkStringCaseSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.function.Supplier; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -32,7 +34,8 @@ private static Supplier environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream t = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"s"}, Types.STRING), Row.of("Apple"), Row.of("GOOGLE"), @@ -40,8 +43,7 @@ private static Supplier environment() { Row.of("Grüße"), Row.of("MAÑANA"), Row.of("straße")); - tEnv.createTemporaryView( - "t", t, Schema.newBuilder().column("s", DataTypes.STRING()).build()); + tEnv.createTemporaryView("t", t, Schema.newBuilder().column("s", DataTypes.STRING()).build()); return tEnv; }; } diff --git a/src/test/java/tech/streamfusion/FlinkStringCastSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkStringCastSqlHarnessTest.java index 912a3a8bb..316d80ae8 100644 --- a/src/test/java/tech/streamfusion/FlinkStringCastSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkStringCastSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.math.BigDecimal; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -30,9 +32,11 @@ void nonNullableCharToDoubleInitializesAndMatchesHost() throws Exception { var table = StreamTableEnvironment.create(env); table.createTemporaryView( "inputs", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"s"}, Types.STRING), - Row.of("1234.567"), Row.of("9876.543")), + Row.of("1234.567"), + Row.of("9876.543")), Schema.newBuilder().column("s", DataTypes.CHAR(8).notNull()).build()); return table; }, @@ -80,7 +84,8 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"l", "i", "sm", "ty", "d", "f", "dm", "sl", "si", "sd", "sdec"}, Types.LONG, diff --git a/src/test/java/tech/streamfusion/FlinkStringHashSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkStringHashSqlHarnessTest.java index 944a3bac6..0f19cdcee 100644 --- a/src/test/java/tech/streamfusion/FlinkStringHashSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkStringHashSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -88,10 +89,15 @@ private static TableEnvironment environment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.createTemporaryView( "strings", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "s", "other", "sep", "bits"}, - Types.INT, Types.STRING, Types.STRING, Types.STRING, Types.INT), + Types.INT, + Types.STRING, + Types.STRING, + Types.STRING, + Types.INT), Row.of(0, "abc", "", ":", 224), Row.of(1, "", "abc", "", 256), Row.of(2, null, "abc", ":", 384), diff --git a/src/test/java/tech/streamfusion/FlinkTemporalFunctionsSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTemporalFunctionsSqlHarnessTest.java index 82ceb278d..a5c7aa0ac 100644 --- a/src/test/java/tech/streamfusion/FlinkTemporalFunctionsSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTemporalFunctionsSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; @@ -20,15 +22,14 @@ class FlinkTemporalFunctionsSqlHarnessTest { @Test void parsesTimestampsAndFormatsDynamicPatterns() throws Exception { parity( - "TO_TIMESTAMP(s), TO_TIMESTAMP(s, p), TO_TIMESTAMP_LTZ(s), TO_TIMESTAMP_LTZ(s, p)," - + " TO_TIMESTAMP_LTZ(s, p, z), DATE_FORMAT(ts, p), DATE_FORMAT(ltz, p), DATE_FORMAT(ts," - + " 'EEEE MMMM dd yyyy'), DATE_FORMAT(ts, 'yyyy-MM-dd HH:mm:ss')"); + "TO_TIMESTAMP(s), TO_TIMESTAMP(s, p), DATE_FORMAT(ts, p), DATE_FORMAT(ltz, p)," + + " DATE_FORMAT(ts, 'EEEE MMMM dd yyyy'), DATE_FORMAT(ts, 'yyyy-MM-dd HH:mm:ss')"); } @Test void epochConversionsComposeWithTimestampExpressions() throws Exception { parity( - "TO_TIMESTAMP_LTZ(epoch), TO_TIMESTAMP_LTZ(epoch, 3), " + "TO_TIMESTAMP_LTZ(epoch, 3), " + "TO_TIMESTAMP_LTZ(n, 0), TO_TIMESTAMP_LTZ(CAST(epoch AS DECIMAL(18, 3)), 3), " + "EXTRACT(SECOND FROM TO_TIMESTAMP_LTZ(epoch, 3)), " + "UNIX_TIMESTAMP(s), UNIX_TIMESTAMP(s, p), FROM_UNIXTIME(n), FROM_UNIXTIME(n, p)"); @@ -42,6 +43,10 @@ void convertsTimeZonesAndParsesFormattedDates() throws Exception { @ParameterizedTest @ValueSource(strings = {"YEAR", "MONTH", "DAY", "HOUR", "MINUTE", "SECOND", "MILLISECOND"}) void temporalRoundingPreservesTypesAndNestedExpressions(String unit) throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.SUB_HOUR_TIMESTAMP_ROUNDING + || java.util.Set.of("YEAR", "MONTH", "DAY", "HOUR").contains(unit), + "Flink 1.18 emits an invalid TimestampData floor/ceil call below HOUR precision"); parity( "FLOOR(ts TO " + unit @@ -87,7 +92,7 @@ void addsIntervalsAndCalculatesDifferences() throws Exception { @Test void castsTemporalValuesAndKeepsSubMillisecondPrecision() throws Exception { parity( - "CAST(ts AS TIMESTAMP(3)), CAST(ts AS DATE), CAST(ts AS TIME), " + "CAST(ts AS TIMESTAMP(3)), CAST(ts AS DATE), " + "CAST(ts AS STRING), CAST(d AS STRING), CAST(tm AS STRING), CAST(ltz AS STRING), " + "CAST(ts AS TIMESTAMP_LTZ(9)), CAST(ltz AS TIMESTAMP(9)), CAST(s AS TIMESTAMP(3))"); } @@ -97,10 +102,9 @@ void temporalCastsRespectLegacyBehaviorFromTheEnvironmentConfiguration() throws NativeParity.assertParity( () -> { var tables = environment(); - var root = - org.apache.flink.configuration.Configuration.fromMap( - tables.getConfig().getRootConfiguration().toMap()); - root.setString("table.exec.legacy-cast-behaviour", "ENABLED"); + var root = org.apache.flink.table.api.TableConfig.getDefault(); + root.setRootConfiguration(tables.getConfig().getRootConfiguration()); + root.set("table.exec.legacy-cast-behaviour", "ENABLED"); tables.getConfig().setRootConfiguration(root); return tables; }, @@ -148,7 +152,8 @@ void parsedRowtimeCanFeedAWindow() throws Exception { var tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "parsed", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"s"}, Types.STRING), Row.of("2024-01-01 00:00:00"), Row.of("2024-01-01 00:00:01"), @@ -168,24 +173,61 @@ void parsedRowtimeCanFeedAWindow() throws Exception { @Test void invalidTextAndPatternsRetainFlinksNullAndSentinelResults() throws Exception { NativeParity.assertParity( - () -> { + FlinkTemporalFunctionsSqlHarnessTest::invalidDates, + "SELECT TO_DATE(s, p), TO_TIMESTAMP(s, p), " + + "UNIX_TIMESTAMP(s, p), CONVERT_TZ(s, z, 'UTC') FROM invalid_dates"); + } + + @Test + void invalidLocalTimeZoneTimestampTextKeepsHostResults() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.TIMESTAMP_LTZ_TEXT_OVERLOADS, + "Flink 1.18 TO_TIMESTAMP_LTZ has no string overloads"); + NativeParity.assertParity( + FlinkTemporalFunctionsSqlHarnessTest::invalidDates, + "SELECT TO_TIMESTAMP_LTZ(s, p, z) FROM invalid_dates"); + } + + private static TableEnvironment invalidDates() { var env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); var tables = StreamTableEnvironment.create(env); - tables.createTemporaryView( - "invalid_dates", - env.fromData( - Types.ROW_NAMED( - new String[] {"s", "p", "z"}, Types.STRING, Types.STRING, Types.STRING), - Row.of("not a date", "yyyy-MM-dd HH:mm:ss", "UTC"), - Row.of("2024-02-30 12:00:00", "yyyy-MM-dd HH:mm:ss", "unknown/timezone"), - Row.of("2024-01-01 12:00:00", "yyyy/MM/dd", "UTC"), - Row.of("", "", ""), - Row.of(null, null, null))); + tables.createTemporaryView( + "invalid_dates", + fromData( + env, + Types.ROW_NAMED(new String[] {"s", "p", "z"}, Types.STRING, Types.STRING, Types.STRING), + Row.of("not a date", "yyyy-MM-dd HH:mm:ss", "UTC"), + Row.of("2024-02-30 12:00:00", "yyyy-MM-dd HH:mm:ss", "unknown/timezone"), + Row.of("2024-01-01 12:00:00", "yyyy/MM/dd", "UTC"), + Row.of("", "", ""), + Row.of(null, null, null))); return tables; - }, - "SELECT TO_DATE(s, p), TO_TIMESTAMP(s, p), TO_TIMESTAMP_LTZ(s, p, z), " - + "UNIX_TIMESTAMP(s, p), CONVERT_TZ(s, z, 'UTC') FROM invalid_dates"); + } + + @Test + void parsesLocalTimeZoneTimestampsFromText() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.TIMESTAMP_LTZ_TEXT_OVERLOADS, + "Flink 1.18 TO_TIMESTAMP_LTZ only accepts numeric epochs with explicit precision"); + parity("TO_TIMESTAMP_LTZ(s), TO_TIMESTAMP_LTZ(s, p), TO_TIMESTAMP_LTZ(s, p, z)"); + } + + @Test + void epochConversionsUseDefaultPrecision() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.TIMESTAMP_LTZ_DEFAULT_PRECISION, + "Flink 1.18 TO_TIMESTAMP_LTZ requires the precision argument"); + parity("TO_TIMESTAMP_LTZ(epoch)"); + } + + @Test + void castsPreEpochTimestampToTime() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.NEGATIVE_TIMESTAMP_TO_TIME, + "Flink 1.18 emits a negative millisecond fraction which its external TIME converter" + + " rejects"); + parity("CAST(ts AS TIME)"); } @Test @@ -270,7 +312,8 @@ static TableEnvironment environment() { (Instant) null); tables.createTemporaryView( "temporal_inputs", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "s", "p", "z", "n", "epoch", "d", "tm", "ts", "ltz"}, Types.INT, diff --git a/src/test/java/tech/streamfusion/FlinkTemporalJoinSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTemporalJoinSqlHarnessTest.java index c1157d8e4..91240ebe1 100644 --- a/src/test/java/tech/streamfusion/FlinkTemporalJoinSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTemporalJoinSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Duration; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.typeinfo.Types; @@ -86,7 +88,8 @@ private static TableEnvironment environment() { // Orders: USD@150, EUR@250, USD@450, GBP@260 (GBP has no rate). private static DataStream orders(StreamExecutionEnvironment env) { - return env.fromData( + return fromData( + env, Types.ROW_NAMED( new String[] {"currency", "amount", "ts"}, Types.STRING, Types.LONG, Types.LONG), Row.of("USD", 1L, 150L), @@ -98,7 +101,8 @@ private static DataStream orders(StreamExecutionEnvironment env) { // Rates versions: USD 10@100 then 20@300; EUR 99@100. private static DataStream rates(StreamExecutionEnvironment env) { - return env.fromData( + return fromData( + env, Types.ROW_NAMED( new String[] {"currency", "rate", "ts"}, Types.STRING, Types.LONG, Types.LONG), Row.of("USD", 10L, 100L), diff --git a/src/test/java/tech/streamfusion/FlinkTemporalSortSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTemporalSortSqlHarnessTest.java index 08eea61e6..82c5398b1 100644 --- a/src/test/java/tech/streamfusion/FlinkTemporalSortSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTemporalSortSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Duration; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.typeinfo.Types; @@ -51,7 +53,8 @@ public void onPeriodicEmit( org.apache.flink.api.common.eventtime.WatermarkOutput output) {} }; var source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "ts"}, Types.LONG, Types.LONG, Types.LONG), Row.of(1L, 1L, 100L), Row.of(1L, 3L, 300L), @@ -105,7 +108,8 @@ private static TableEnvironment environment(int parallelism) { // Out-of-order timestamps so the sort actually reorders; the 2s bounded-out-of-orderness keeps // every row open until end-of-input MAX, so all rows are released together (no mid-stream cut). DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "ts"}, Types.LONG, Types.LONG, Types.LONG), Row.of(1L, 30L, 2000L), Row.of(2L, 10L, 500L), diff --git a/src/test/java/tech/streamfusion/FlinkTemporalTimestampRangeSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTemporalTimestampRangeSqlHarnessTest.java index d08a937c2..3b55bd9ef 100644 --- a/src/test/java/tech/streamfusion/FlinkTemporalTimestampRangeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTemporalTimestampRangeSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -38,7 +40,8 @@ private static TableEnvironment environment() { var tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "dates", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"s"}, Types.STRING), Row.of("0001-01-01 00:00:00.123456"), Row.of("1582-10-15 23:59:59.999999"), diff --git a/src/test/java/tech/streamfusion/FlinkTimestampExtractSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTimestampExtractSqlHarnessTest.java index cfd41cd35..1fdcffc9b 100644 --- a/src/test/java/tech/streamfusion/FlinkTimestampExtractSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTimestampExtractSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; @@ -82,7 +84,8 @@ private static TableEnvironment environment(int precision) { withNull[rows.length] = Row.of("null", (LocalDateTime) null, (Instant) null); tables.createTemporaryView( "timestamps", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "ts", "ltz"}, Types.STRING, diff --git a/src/test/java/tech/streamfusion/FlinkTimestampExtremaSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTimestampExtremaSqlHarnessTest.java index 8f3bc5889..a8c580add 100644 --- a/src/test/java/tech/streamfusion/FlinkTimestampExtremaSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTimestampExtremaSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -82,7 +84,8 @@ private static TableEnvironment retractingEnvironment() { table.createTemporaryView( "src", table.fromChangelogStream( - env.fromData( + fromData( + env, rows, Types.ROW_NAMED( new String[] {"k", "ts", "ltz"}, @@ -129,7 +132,8 @@ private static TableEnvironment environment(String zone, int precision) { } table.createTemporaryView( "src", - env.fromData( + fromData( + env, rows, Types.ROW_NAMED( new String[] {"k", "ts", "ltz"}, Types.INT, Types.LOCAL_DATE_TIME, Types.INSTANT)), diff --git a/src/test/java/tech/streamfusion/FlinkTimestampIntervalSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTimestampIntervalSqlHarnessTest.java index abae1acd8..a8dce16fc 100644 --- a/src/test/java/tech/streamfusion/FlinkTimestampIntervalSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTimestampIntervalSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.LocalDateTime; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -41,13 +43,15 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream a = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "tsa"}, Types.LONG, Types.LOCAL_DATE_TIME), Row.of(1L, LocalDateTime.of(2024, 6, 1, 12, 0, 5)), Row.of(1L, LocalDateTime.of(2024, 6, 1, 12, 0, 30)), Row.of(2L, LocalDateTime.of(2024, 6, 1, 12, 0, 8))); DataStream b = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "tsb"}, Types.LONG, Types.LOCAL_DATE_TIME), Row.of(1L, LocalDateTime.of(2024, 6, 1, 12, 0, 12)), Row.of(2L, LocalDateTime.of(2024, 6, 1, 12, 0, 9))); diff --git a/src/test/java/tech/streamfusion/FlinkTimestampWindowSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTimestampWindowSqlHarnessTest.java index 95dbc9750..91e9cd0f2 100644 --- a/src/test/java/tech/streamfusion/FlinkTimestampWindowSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTimestampWindowSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Duration; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -228,18 +230,16 @@ private static TableEnvironment retractingEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "value", "ts"}, Types.LONG, Types.LONG, Types.LOCAL_DATE_TIME), - Row.ofKind( - RowKind.INSERT, 1L, 5L, LocalDateTime.of(2024, 6, 1, 12, 0, 1)), - Row.ofKind( - RowKind.UPDATE_BEFORE, 1L, 5L, LocalDateTime.of(2024, 6, 1, 12, 0, 1)), - Row.ofKind( - RowKind.UPDATE_AFTER, 1L, 7L, LocalDateTime.of(2024, 6, 1, 12, 0, 1))) + Row.ofKind(RowKind.INSERT, 1L, 5L, LocalDateTime.of(2024, 6, 1, 12, 0, 1)), + Row.ofKind(RowKind.UPDATE_BEFORE, 1L, 5L, LocalDateTime.of(2024, 6, 1, 12, 0, 1)), + Row.ofKind(RowKind.UPDATE_AFTER, 1L, 7L, LocalDateTime.of(2024, 6, 1, 12, 0, 1))) .assignTimestampsAndWatermarks( WatermarkStrategy.forBoundedOutOfOrderness(Duration.ZERO) .withTimestampAssigner((row, timestamp) -> timestampMillis(row))); @@ -265,7 +265,8 @@ private static TableEnvironment build(String phaseStrategy) { // ts is a plain TIMESTAMP(3) rowtime attribute (not local-time-zone); the source carries the // watermarks (SOURCE_WATERMARK), so no interior watermark-assigner breaks the columnar island. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "value", "ts"}, Types.LONG, diff --git a/src/test/java/tech/streamfusion/FlinkTopNSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTopNSqlHarnessTest.java index bdf45d621..9e1fb23da 100644 --- a/src/test/java/tech/streamfusion/FlinkTopNSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTopNSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -48,8 +50,8 @@ void topNWithRankNumberMatchesHost() throws Exception { void topNWithRankNumberDescendingMatchesHost() throws Exception { NativeParity.assertParity( FlinkTopNSqlHarnessTest::environment, - "SELECT k, v, rn FROM (SELECT k, v, ROW_NUMBER() OVER (PARTITION BY k ORDER BY v DESC) AS rn " - + "FROM src) WHERE rn <= 2"); + "SELECT k, v, rn FROM (SELECT k, v, ROW_NUMBER() OVER (PARTITION BY k ORDER BY v DESC) AS" + + " rn FROM src) WHERE rn <= 2"); } @Test @@ -107,7 +109,8 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 5L), Row.of(1L, 3L), @@ -118,7 +121,10 @@ private static TableEnvironment environment() { tEnv.createTemporaryView( "src", source, - Schema.newBuilder().column("k", DataTypes.BIGINT()).column("v", DataTypes.BIGINT()).build()); + Schema.newBuilder() + .column("k", DataTypes.BIGINT()) + .column("v", DataTypes.BIGINT()) + .build()); return tEnv; } } diff --git a/src/test/java/tech/streamfusion/FlinkTranslateSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTranslateSqlHarnessTest.java index d5d1f7da3..042ade746 100644 --- a/src/test/java/tech/streamfusion/FlinkTranslateSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTranslateSqlHarnessTest.java @@ -3,6 +3,11 @@ import org.junit.jupiter.api.Test; class FlinkTranslateSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("TRANSLATE3"); + } + @Test void translateUsesCodepointsFirstDuplicateAndSpecialNullRules() throws Exception { parity( diff --git a/src/test/java/tech/streamfusion/FlinkTryDecimalSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTryDecimalSqlHarnessTest.java index 98c3d43b6..9ac090574 100644 --- a/src/test/java/tech/streamfusion/FlinkTryDecimalSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTryDecimalSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.math.BigDecimal; import java.util.stream.Stream; import org.apache.flink.api.common.typeinfo.Types; @@ -112,7 +114,7 @@ private static TableEnvironment strings(boolean legacy) { for (int i = 0; i < rows.length; i++) rows[i] = Row.of(i, values[i % values.length]); table.createTemporaryView( "src", - env.fromData(Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING), rows)); + fromData(env, Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING), rows)); return table; } @@ -122,7 +124,8 @@ private static TableEnvironment decimals() { var table = StreamTableEnvironment.create(env); table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "d"}, Types.INT, Types.BIG_DEC), Row.of(1, new BigDecimal("999.995")), Row.of(2, new BigDecimal("-1.255")), diff --git a/src/test/java/tech/streamfusion/FlinkTwoPhaseGroupAggregateSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTwoPhaseGroupAggregateSqlHarnessTest.java index ea81b87ef..490ff595d 100644 --- a/src/test/java/tech/streamfusion/FlinkTwoPhaseGroupAggregateSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTwoPhaseGroupAggregateSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -278,7 +280,8 @@ void averageUpdateWithZeroNetCountMatchesHost(int bundleSize) throws Exception { tEnv.createTemporaryView( "t", tEnv.fromChangelogStream( - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 5L), @@ -332,6 +335,7 @@ void stateTtlEmitsUnsuppressedUpdatesAndMatchesHost(boolean enabled) throws Exce @Test void stateTtlHintRoutesTheGlobalHalfAndMatchesHost() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireStateTtlHint(); // A STATE_TTL hint with the job retention at 0 must switch the global merge into TTL emission, // mirroring Flink's hint-over-config precedence on the two-phase plan. NativeParity.assertKindedParity( @@ -354,7 +358,8 @@ private static TableEnvironment ttlEnvironment() { tEnv.createTemporaryView( "t", tEnv.fromDataStream( - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(1L, 20L), @@ -369,7 +374,7 @@ private static TableEnvironment ttlEnvironment() { private static List> ttlChangelog(boolean enabled) { // Four local partials become two global bundles with the same MIN. Only TTL emits the no-op. - return enabled + return enabled && tech.streamfusion.compat.FlinkCompat.GLOBAL_TTL_EMITS_UNCHANGED ? List.of(List.of("+I", 1L, 10L), List.of("-U", 1L, 10L), List.of("+U", 1L, 10L)) : List.of(List.of("+I", 1L, 10L)); } @@ -423,19 +428,14 @@ private static void writeInput(Path directory) throws Exception { // over key 2 divides negatively at some prefixes of the changelog). The u/us columns repeat // values within and across keys so distinct aggregates see real multiplicities. tEnv.executeSql( - "INSERT INTO in_write VALUES" - + " (1, 10, 1.5, 7, CAST(100 AS SMALLINT), CAST(3 AS TINYINT), CAST(1.25 AS FLOAT)," - + " 12.34, 10, 'a')," - + " (1, 20, 2.5, 3, CAST(-7 AS SMALLINT), CAST(-2 AS TINYINT), CAST(2.5 AS FLOAT)," - + " -0.07, 10, 'b')," - + " (2, 5, 0.5, 9, CAST(250 AS SMALLINT), CAST(9 AS TINYINT), CAST(-0.75 AS FLOAT)," - + " 99999999.99, 30, 'x')," - + " (1, 30, 3.5, 1, CAST(42 AS SMALLINT), CAST(5 AS TINYINT), CAST(4.5 AS FLOAT)," - + " 3.00, 20, 'a')," - + " (2, 15, 1.0, 4, CAST(-11 AS SMALLINT), CAST(-4 AS TINYINT), CAST(5.125 AS FLOAT)," - + " -42.42, 30, 'x')," - + " (2, -7, -1.25, -8, CAST(-3 AS SMALLINT), CAST(-7 AS TINYINT), CAST(0.5 AS FLOAT)," - + " 0.01, 30, 'y')") + "INSERT INTO in_write VALUES (1, 10, 1.5, 7, CAST(100 AS SMALLINT), CAST(3 AS TINYINT)," + + " CAST(1.25 AS FLOAT), 12.34, 10, 'a'), (1, 20, 2.5, 3, CAST(-7 AS SMALLINT)," + + " CAST(-2 AS TINYINT), CAST(2.5 AS FLOAT), -0.07, 10, 'b'), (2, 5, 0.5, 9," + + " CAST(250 AS SMALLINT), CAST(9 AS TINYINT), CAST(-0.75 AS FLOAT), 99999999.99," + + " 30, 'x'), (1, 30, 3.5, 1, CAST(42 AS SMALLINT), CAST(5 AS TINYINT), CAST(4.5 AS" + + " FLOAT), 3.00, 20, 'a'), (2, 15, 1.0, 4, CAST(-11 AS SMALLINT), CAST(-4 AS" + + " TINYINT), CAST(5.125 AS FLOAT), -42.42, 30, 'x'), (2, -7, -1.25, -8, CAST(-3 AS" + + " SMALLINT), CAST(-7 AS TINYINT), CAST(0.5 AS FLOAT), 0.01, 30, 'y')") .await(); } diff --git a/src/test/java/tech/streamfusion/FlinkTypedNullSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkTypedNullSqlHarnessTest.java index 0ff11e695..d74f8d1b1 100644 --- a/src/test/java/tech/streamfusion/FlinkTypedNullSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkTypedNullSqlHarnessTest.java @@ -1,6 +1,7 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -87,7 +88,7 @@ private static TableEnvironment inputs(int count) { for (int i = 0; i < count; i++) rows[i] = Row.of(i); tables.createTemporaryView( "inputs", - env.fromData(Types.ROW_NAMED(new String[] {"id"}, Types.INT), rows), + fromData(env, Types.ROW_NAMED(new String[] {"id"}, Types.INT), rows), Schema.newBuilder().column("id", DataTypes.INT()).build()); return tables; } @@ -98,7 +99,8 @@ private static TableEnvironment collections() { StreamTableEnvironment tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "inputs", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "arr", "m", "am"}, Types.INT, diff --git a/src/test/java/tech/streamfusion/FlinkUdfExactTypesSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUdfExactTypesSqlHarnessTest.java index f91f62a57..e7be546c0 100644 --- a/src/test/java/tech/streamfusion/FlinkUdfExactTypesSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUdfExactTypesSqlHarnessTest.java @@ -1,6 +1,7 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertEquals; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.math.BigDecimal; import java.util.ArrayList; @@ -183,7 +184,8 @@ static TableEnvironment environment(int count) { } table.createTemporaryView( "src", - env.fromData( + fromData( + env, rows, Types.ROW_NAMED( new String[] {"id", "d", "b", "s"}, diff --git a/src/test/java/tech/streamfusion/FlinkUdfIdentitySqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUdfIdentitySqlHarnessTest.java index d97fdd916..14a208395 100644 --- a/src/test/java/tech/streamfusion/FlinkUdfIdentitySqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUdfIdentitySqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -11,7 +13,16 @@ class FlinkUdfIdentitySqlHarnessTest { @ParameterizedTest - @ValueSource(strings = {"REV", "UPPER", "LOWER", "CHAR_LENGTH", "JSON_UNQUOTE", "TO_DATE", "FROM_UNIXTIME"}) + @ValueSource( + strings = { + "REV", + "UPPER", + "LOWER", + "CHAR_LENGTH", + "JSON_UNQUOTE", + "TO_DATE", + "FROM_UNIXTIME" + }) void registeredFunctionTakesPrecedenceOverBuiltinName(String name) throws Exception { NativeParity.assertParity( () -> environment(name), "SELECT id, " + name + "(s) FROM inputs"); @@ -32,7 +43,8 @@ private static TableEnvironment environment(String name) { table.createTemporarySystemFunction(name, Reverse.class); table.createTemporaryView( "inputs", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING), Row.of(1, "Alice"), Row.of(2, "Bob"), diff --git a/src/test/java/tech/streamfusion/FlinkUdfLifecycleSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUdfLifecycleSqlHarnessTest.java index 871765032..36a355968 100644 --- a/src/test/java/tech/streamfusion/FlinkUdfLifecycleSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUdfLifecycleSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -81,7 +83,7 @@ private static TableEnvironment environment() { } table.createTemporaryView( "inputs", - env.fromData(Types.ROW_NAMED(new String[] {"a", "b"}, Types.INT, Types.INT), rows)); + fromData(env, Types.ROW_NAMED(new String[] {"a", "b"}, Types.INT, Types.INT), rows)); return table; } diff --git a/src/test/java/tech/streamfusion/FlinkUdfSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUdfSqlHarnessTest.java index e387b1e88..4da9980ee 100644 --- a/src/test/java/tech/streamfusion/FlinkUdfSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUdfSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.function.Supplier; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; @@ -47,7 +49,8 @@ void udfWithOtherProjectionsMatchesHost() throws Exception { // The UDF alongside a native scalar/arithmetic projection — the whole Calc stays native. NativeParity.assertParity( environment(), - "SELECT k, UPPER_LEN(s), count_char(s, 'c') AS c FROM t".replace("UPPER_LEN(s)", "CHAR_LENGTH(s)")); + "SELECT k, UPPER_LEN(s), count_char(s, 'c') AS c FROM t" + .replace("UPPER_LEN(s)", "CHAR_LENGTH(s)")); } private static Supplier environment() { @@ -57,7 +60,8 @@ private static Supplier environment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.createTemporarySystemFunction("count_char", CountChar.class); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "s"}, Types.LONG, Types.STRING), Row.of(1L, "abccc"), Row.of(2L, "cccc"), diff --git a/src/test/java/tech/streamfusion/FlinkUnhexSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUnhexSqlHarnessTest.java index 80787da6a..1f9a7577a 100644 --- a/src/test/java/tech/streamfusion/FlinkUnhexSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUnhexSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; @@ -7,6 +9,11 @@ import org.junit.jupiter.api.Test; class FlinkUnhexSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("UNHEX"); + } + @Test void oddLengthsAndInvalidInputNulls() throws Exception { parity("SELECT id, UNHEX(hex_text) FROM encodings"); @@ -28,7 +35,8 @@ void unhexRejectsInvalidTailsAfterDecodingValidPrefixes() throws Exception { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.createTemporaryView( "unhex_inputs", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"s"}, Types.STRING), Row.of("aF0012GG"), Row.of("Af"), diff --git a/src/test/java/tech/streamfusion/FlinkUnionSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUnionSqlHarnessTest.java index f36f74f80..acb2f9c3a 100644 --- a/src/test/java/tech/streamfusion/FlinkUnionSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUnionSqlHarnessTest.java @@ -1,7 +1,8 @@ package tech.streamfusion; -import tech.streamfusion.planner.NativePlanner; -import tech.streamfusion.planner.PhysicalPlanScan; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -11,7 +12,8 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; +import tech.streamfusion.planner.NativePlanner; +import tech.streamfusion.planner.PhysicalPlanScan; /** * UNION ALL matches the host. A union is a pure stream merge — the native node carries no operator @@ -96,7 +98,8 @@ private static TableEnvironment environment() { private static void register( StreamTableEnvironment tEnv, StreamExecutionEnvironment env, String name) { DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "s"}, Types.LONG, Types.INT, Types.STRING), Row.of(1L, 10, "a"), Row.of(2L, 30, "b"), diff --git a/src/test/java/tech/streamfusion/FlinkUnnestSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUnnestSqlHarnessTest.java index 699eac20e..f3f9d92bc 100644 --- a/src/test/java/tech/streamfusion/FlinkUnnestSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUnnestSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -79,6 +81,9 @@ void leftUnnestMatchesHost() throws Exception { @Test void leftUnnestWithOrdinalityMatchesHost() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.UNNEST_ORDINALITY, + "Flink 1.18 cannot plan UNNEST WITH ORDINALITY"); // A LEFT null-pad row carries a null ordinal too. NativeParity.assertParity( FlinkUnnestSqlHarnessTest::environment, @@ -87,6 +92,9 @@ void leftUnnestWithOrdinalityMatchesHost() throws Exception { @Test void unnestWithOrdinalityMatchesHost() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.UNNEST_ORDINALITY, + "Flink 1.18 cannot plan UNNEST WITH ORDINALITY"); // WITH ORDINALITY appends a 1-based position column (the element's index in its array). NativeParity.assertParity( FlinkUnnestSqlHarnessTest::environment, @@ -95,6 +103,9 @@ void unnestWithOrdinalityMatchesHost() throws Exception { @Test void unnestMapWithOrdinalityMatchesHost() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.UNNEST_ORDINALITY, + "Flink 1.18 cannot plan UNNEST WITH ORDINALITY"); NativeParity.assertParity( FlinkUnnestSqlHarnessTest::mapEnvironment, "SELECT k, mk, mv, o FROM t CROSS JOIN UNNEST(m) WITH ORDINALITY AS u(mk, mv, o)"); @@ -134,7 +145,8 @@ private static TableEnvironment environment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "vs"}, Types.LONG, Types.OBJECT_ARRAY(Types.LONG)), Row.of(1L, new Long[] {10L, 20L}), Row.of(2L, new Long[] {30L}), @@ -156,7 +168,8 @@ private static TableEnvironment mapEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "m"}, Types.LONG, Types.MAP(Types.STRING, Types.LONG)), Row.of(1L, java.util.Map.of("a", 10L, "b", 20L)), @@ -177,7 +190,8 @@ private static TableEnvironment multisetEnvironment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); // A MULTISET is carried as a MAP. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "ms"}, Types.LONG, Types.MAP(Types.STRING, Types.INT)), Row.of(1L, java.util.Map.of("a", 2, "b", 1)), @@ -197,7 +211,8 @@ private static TableEnvironment rowArrayEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "rs"}, Types.LONG, @@ -226,9 +241,9 @@ private static TableEnvironment stringArrayEnvironment() { env.setParallelism(1); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); DataStream source = - env.fromData( - Types.ROW_NAMED( - new String[] {"k", "ss"}, Types.LONG, Types.OBJECT_ARRAY(Types.STRING)), + fromData( + env, + Types.ROW_NAMED(new String[] {"k", "ss"}, Types.LONG, Types.OBJECT_ARRAY(Types.STRING)), Row.of(1L, new String[] {"a", "b"}), Row.of(2L, new String[] {"c"})); tEnv.createTemporaryView( diff --git a/src/test/java/tech/streamfusion/FlinkUpdateFastTopNSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUpdateFastTopNSqlHarnessTest.java index ad8d9f0b8..9ea38f838 100644 --- a/src/test/java/tech/streamfusion/FlinkUpdateFastTopNSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUpdateFastTopNSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -95,7 +97,8 @@ private static TableEnvironment environment() { // Two groups; counts tie (k=10 and k=30 both reach 2 in group 1) and cross (k=30 passes k=10), // so the top-2 re-sorts, a full top-N rejects a low count, and an eviction promotes. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"g", "k"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(1L, 20L), diff --git a/src/test/java/tech/streamfusion/FlinkUpdateFastVariableTopNSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUpdateFastVariableTopNSqlHarnessTest.java index 647f1ef70..e7f7ca140 100644 --- a/src/test/java/tech/streamfusion/FlinkUpdateFastVariableTopNSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUpdateFastVariableTopNSqlHarnessTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.ArrayList; import java.util.List; @@ -218,7 +219,8 @@ private static TableEnvironment environment(StreamExecutionEnvironment env, bool } } var source = - env.fromData( + fromData( + env, rows, Types.ROW_NAMED(new String[] {"k", "id", "v"}, Types.LONG, Types.LONG, Types.LONG)); table.createTemporaryView( diff --git a/src/test/java/tech/streamfusion/FlinkUpdatingLimitSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUpdatingLimitSqlHarnessTest.java index dac2b0f02..6cce46676 100644 --- a/src/test/java/tech/streamfusion/FlinkUpdatingLimitSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUpdatingLimitSqlHarnessTest.java @@ -1,6 +1,7 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.time.Duration; import java.util.function.Supplier; @@ -24,9 +25,12 @@ class FlinkUpdatingLimitSqlHarnessTest { @ParameterizedTest @ValueSource(ints = {1, 2, 100}) void monotonicCountsUseUpdateFastRank(int limit) throws Exception { - assertNative(() -> environment(false, false), - "SELECT k, COUNT(*) AS n FROM src GROUP BY k ORDER BY n DESC, k ASC NULLS FIRST LIMIT " + limit, - true, "UpdateFastStrategy"); + assertNative( + () -> environment(false, false), + "SELECT k, COUNT(*) AS n FROM src GROUP BY k ORDER BY n DESC, k ASC NULLS FIRST LIMIT " + + limit, + true, + "UpdateFastStrategy"); } @ParameterizedTest @@ -48,9 +52,12 @@ void unorderedUpdatingLimitMatchesHost(String range) throws Exception { @ParameterizedTest @ValueSource(booleans = {false, true}) void miniBatchPreservesMaterializedSelection(boolean changes) throws Exception { - assertNative(() -> environment(changes, true), - "SELECT k, SUM(v) AS total FROM src GROUP BY k ORDER BY total DESC NULLS LAST, k ASC LIMIT 2", - false, null); + assertNative( + () -> environment(changes, true), + "SELECT k, SUM(v) AS total FROM src GROUP BY k ORDER BY total DESC NULLS LAST, k ASC LIMIT" + + " 2", + false, + null); } @ParameterizedTest @@ -66,7 +73,8 @@ void updateFastOffsetMatchesHost(int offset) throws Exception { @CsvSource({"false,1", "true,1", "false,2", "true,2", "false,100", "true,100"}) void retractingOffsetPreservesHostStoredRowKinds(boolean ordered, int offset) throws Exception { String order = ordered ? "ORDER BY total DESC NULLS LAST, k ASC NULLS FIRST " : ""; - String sql = "SELECT k, SUM(v) AS total FROM src GROUP BY k " + order + "LIMIT 2 OFFSET " + offset; + String sql = + "SELECT k, SUM(v) AS total FROM src GROUP BY k " + order + "LIMIT 2 OFFSET " + offset; assertNative(() -> environment(true, false), sql, true, null); NativeParity.assertChangelogParity(() -> environment(true, false), sql); } @@ -83,22 +91,31 @@ void miniBatchAggregatesPreserveHostChangelogOrder(boolean ordered) throws Excep @ParameterizedTest @ValueSource(booleans = {false, true}) void nullableDuplicatesWithOffsetMatchHost(boolean miniBatch) throws Exception { - Supplier input = () -> { - var env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.setParallelism(1); - var table = StreamTableEnvironment.create(env); - if (miniBatch) { - table.getConfig().set("table.exec.mini-batch.enabled", "true"); - table.getConfig().set("table.exec.mini-batch.allow-latency", MINI_BATCH_LATENCY); - table.getConfig().set("table.exec.mini-batch.size", "4"); - } - table.createTemporaryView("src", table.fromChangelogStream(env.fromData( - Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), - Row.of(1L, 20L), Row.of(1L, 20L), Row.of(2L, 10L), Row.of(null, null), - Row.ofKind(RowKind.DELETE, 1L, 20L), Row.of(3L, 5L), - Row.ofKind(RowKind.DELETE, 2L, 10L)))); - return table; - }; + Supplier input = + () -> { + var env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + var table = StreamTableEnvironment.create(env); + if (miniBatch) { + table.getConfig().set("table.exec.mini-batch.enabled", "true"); + table.getConfig().set("table.exec.mini-batch.allow-latency", MINI_BATCH_LATENCY); + table.getConfig().set("table.exec.mini-batch.size", "4"); + } + table.createTemporaryView( + "src", + table.fromChangelogStream( + fromData( + env, + Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), + Row.of(1L, 20L), + Row.of(1L, 20L), + Row.of(2L, 10L), + Row.of(null, null), + Row.ofKind(RowKind.DELETE, 1L, 20L), + Row.of(3L, 5L), + Row.ofKind(RowKind.DELETE, 2L, 10L)))); + return table; + }; String sql = "SELECT k, v FROM src ORDER BY v DESC NULLS LAST, k ASC LIMIT 2 OFFSET 1"; assertTrue(NativePlanner.explain(input.get(), sql).contains("NativeColumnarTopN")); NativeParity.assertOrderedKindedParity(input, sql); @@ -149,9 +166,9 @@ private static TableEnvironment environment(boolean changes, boolean miniBatch) updates[rows.length + 3] = Row.ofKind(RowKind.UPDATE_BEFORE, 2L, 9L); updates[rows.length + 4] = Row.ofKind(RowKind.UPDATE_AFTER, 2L, -9L); updates[rows.length + 5] = Row.of(4L, 20L); - table.createTemporaryView("src", table.fromChangelogStream(env.fromData(type, updates))); + table.createTemporaryView("src", table.fromChangelogStream(fromData(env, type, updates))); } else { - table.createTemporaryView("src", env.fromData(type, rows)); + table.createTemporaryView("src", fromData(env, type, rows)); } return table; } diff --git a/src/test/java/tech/streamfusion/FlinkUrlDecodeSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUrlDecodeSqlHarnessTest.java index e0fc7fd8d..f17d86b5a 100644 --- a/src/test/java/tech/streamfusion/FlinkUrlDecodeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUrlDecodeSqlHarnessTest.java @@ -3,6 +3,11 @@ import org.junit.jupiter.api.Test; class FlinkUrlDecodeSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("URL_DECODE"); + } + @Test void formDecodingAndMalformedUtf8() throws Exception { parity("SELECT id, URL_DECODE(s), URL_DECODE(COALESCE(s, '')) FROM texts"); diff --git a/src/test/java/tech/streamfusion/FlinkUrlEncodeSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkUrlEncodeSqlHarnessTest.java index 24fd8d9eb..9495ad019 100644 --- a/src/test/java/tech/streamfusion/FlinkUrlEncodeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkUrlEncodeSqlHarnessTest.java @@ -3,6 +3,11 @@ import org.junit.jupiter.api.Test; class FlinkUrlEncodeSqlHarnessTest { + @org.junit.jupiter.api.BeforeEach + void requireReleasedHostFunction() { + tech.streamfusion.compat.FlinkTestCapabilities.requireSqlFunction("URL_ENCODE"); + } + @Test void formEncodingAndUnicode() throws Exception { parity("SELECT id, URL_ENCODE(s), URL_ENCODE(COALESCE(s, '')) FROM texts"); diff --git a/src/test/java/tech/streamfusion/FlinkVariableTopNSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkVariableTopNSqlHarnessTest.java index 3b887f887..2a87d804c 100644 --- a/src/test/java/tech/streamfusion/FlinkVariableTopNSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkVariableTopNSqlHarnessTest.java @@ -1,6 +1,7 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.util.ArrayList; import java.util.List; @@ -290,7 +291,8 @@ private static TableEnvironment retractingEnvironment(boolean miniBatch) { for (int i = 0; i < 210; i++) rows.add(Row.ofKind(RowKind.INSERT, 200L, 1000L + i, 100L + i)); for (int i = 0; i < 20; i++) rows.add(Row.ofKind(RowKind.DELETE, 200L, 1000L + i, 100L + i)); var source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "id"}, Types.LONG, Types.LONG, Types.LONG), rows.toArray(Row[]::new)); table.createTemporaryView( @@ -343,7 +345,8 @@ private static TableEnvironment environment(boolean nullable, boolean miniBatch) } if (nullable) rows.add(Row.of(null, 1L, id)); var source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "id"}, Types.LONG, Types.LONG, Types.LONG), rows.toArray(Row[]::new)); table.createTemporaryView( diff --git a/src/test/java/tech/streamfusion/FlinkWatermarkIntervalSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkWatermarkIntervalSqlHarnessTest.java index 9ca73ebd9..6369083a8 100644 --- a/src/test/java/tech/streamfusion/FlinkWatermarkIntervalSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkWatermarkIntervalSqlHarnessTest.java @@ -1,6 +1,7 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.time.LocalDateTime; import java.time.ZoneId; @@ -128,7 +129,8 @@ private static TableEnvironment environment( // Both rows survive a month of lateness; interpreting one month as 1 ms drops January. table.createTemporaryView( "t", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"rt"}, localZoned ? Types.INSTANT : Types.LOCAL_DATE_TIME), Arrays.stream(values) diff --git a/src/test/java/tech/streamfusion/FlinkWindowCountDistinctSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkWindowCountDistinctSqlHarnessTest.java index 6417e7ddb..0b9551e37 100644 --- a/src/test/java/tech/streamfusion/FlinkWindowCountDistinctSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkWindowCountDistinctSqlHarnessTest.java @@ -194,6 +194,7 @@ void legacyWindowsRetainDistinctSets(String shape) throws Exception { @Test void sessionWindowsMergeDistinctValues() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); String sql = "SELECT k, window_start, window_end, COUNT(DISTINCT s), COUNT(DISTINCT t) " + "FROM TABLE(SESSION(TABLE src PARTITION BY k, DESCRIPTOR(rt), INTERVAL '2' SECOND)) " diff --git a/src/test/java/tech/streamfusion/FlinkWindowJoinSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkWindowJoinSqlHarnessTest.java index 7993a2a26..0b6758862 100644 --- a/src/test/java/tech/streamfusion/FlinkWindowJoinSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkWindowJoinSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; @@ -56,7 +58,8 @@ private static TableEnvironment negativeFractionEnvironment() { java.time.Instant time = java.time.Instant.ofEpochSecond(-1, side.equals("A") ? 999999999 : 999000000); DataStream input = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "v", "rt"}, Types.LONG, Types.LONG, Types.INSTANT), Row.of(1L, side.equals("A") ? 10L : 100L, time)) @@ -166,7 +169,8 @@ private static TableEnvironment proctimeEnvironment() { } private static DataStream proctimeStream(StreamExecutionEnvironment env) { - return env.fromData( + return fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(2L, 30L), @@ -184,7 +188,8 @@ private static TableEnvironment dataStreamEnvironment() { } private static DataStream stream(StreamExecutionEnvironment env) { - return env.fromData( + return fromData( + env, Types.ROW_NAMED( new String[] {"k", "v", "ts", "g"}, Types.LONG, diff --git a/src/test/java/tech/streamfusion/FlinkWindowLateSliceSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkWindowLateSliceSqlHarnessTest.java index 3032a9f4b..7e5201548 100644 --- a/src/test/java/tech/streamfusion/FlinkWindowLateSliceSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkWindowLateSliceSqlHarnessTest.java @@ -1,6 +1,7 @@ package tech.streamfusion; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -78,7 +79,8 @@ private static TableEnvironment environment(String phase) { table.getConfig().setLocalTimeZone(ZoneOffset.UTC); table.getConfig().set("table.optimizer.agg-phase-strategy", phase); var source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"k", "millis", "v", "wm"}, Types.INT, diff --git a/src/test/java/tech/streamfusion/FlinkWindowRankSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkWindowRankSqlHarnessTest.java index 94d23a23a..1bb737b91 100644 --- a/src/test/java/tech/streamfusion/FlinkWindowRankSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkWindowRankSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Duration; import java.time.ZoneId; import org.apache.flink.api.common.eventtime.WatermarkStrategy; @@ -88,7 +90,8 @@ private static TableEnvironment proctimeEnvironment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.getConfig().setLocalTimeZone(ZoneId.of("UTC")); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 10L), Row.of(1L, 30L), @@ -110,10 +113,12 @@ private static TableEnvironment environment() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.getConfig().setLocalTimeZone(ZoneId.of("UTC")); - // Two 1s windows; distinct v per window so the rank order is unambiguous, and repeated k within a + // Two 1s windows; distinct v per window so the rank order is unambiguous, and repeated k within + // a // window so deduplication has something to collapse. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v", "ts"}, Types.LONG, Types.LONG, Types.LONG), Row.of(1L, 10L, 100L), Row.of(1L, 30L, 800L), diff --git a/src/test/java/tech/streamfusion/FlinkWindowSqlHarnessTest.java b/src/test/java/tech/streamfusion/FlinkWindowSqlHarnessTest.java index d93d72e04..cbac6bc2d 100644 --- a/src/test/java/tech/streamfusion/FlinkWindowSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/FlinkWindowSqlHarnessTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Duration; import java.time.ZoneId; import org.apache.flink.api.common.eventtime.WatermarkStrategy; @@ -101,11 +103,12 @@ void twoKeyTumblingMatchesHost() throws Exception { @Test void twoKeySessionMatchesHost() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentWithSource, - "SELECT k, g, window_start, window_end, SUM(`value`) AS s " - + "FROM TABLE(SESSION(TABLE src PARTITION BY (k, g), DESCRIPTOR(rt), INTERVAL '1' SECOND)) " - + "GROUP BY k, g, window_start, window_end"); + "SELECT k, g, window_start, window_end, SUM(`value`) AS s FROM TABLE(SESSION(TABLE src" + + " PARTITION BY (k, g), DESCRIPTOR(rt), INTERVAL '1' SECOND)) GROUP BY k, g," + + " window_start, window_end"); } @Test @@ -142,9 +145,9 @@ void hoppingSumMatchesHost() throws Exception { // One-phase HOP: a row falls in two overlapping 2s windows sliding every 1s. NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentWithSource, - "SELECT window_start, window_end, SUM(`value`) AS s " - + "FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) " - + "GROUP BY window_start, window_end"); + "SELECT window_start, window_end, SUM(`value`) AS s FROM TABLE(HOP(TABLE src," + + " DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) GROUP BY window_start," + + " window_end"); } @Test @@ -248,6 +251,7 @@ void tvfLtzTumbleWithDstSessionZoneFallsBack() throws Exception { @Test void tvfLtzSessionWithDstSessionZoneFallsBack() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); NativeParity.assertFallbackReasonContains( FlinkWindowSqlHarnessTest::environmentWithLosAngelesZone, "SELECT k, SUM(`value`) AS s, window_start, window_end " @@ -270,9 +274,9 @@ void twoPhaseLtzHopWithMisalignedSessionZoneFallsBack() throws Exception { void keyedHoppingMultiAggregateMatchesHost() throws Exception { NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentWithSource, - "SELECT k, window_start, window_end, SUM(`value`) AS s, COUNT(`value`) AS c " - + "FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) " - + "GROUP BY k, window_start, window_end"); + "SELECT k, window_start, window_end, SUM(`value`) AS s, COUNT(`value`) AS c FROM" + + " TABLE(HOP(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND))" + + " GROUP BY k, window_start, window_end"); } @Test @@ -297,6 +301,7 @@ void doubleValueAggregateMatchesHost() throws Exception { @Test void sessionSumMatchesHost() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); // Session windows: consecutive rows within the gap form one window; a larger gap splits them. NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentWithSource, @@ -307,6 +312,7 @@ void sessionSumMatchesHost() throws Exception { @Test void keyedSessionMultiAggregateMatchesHost() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); // Per-key sessions: each key's gaps are independent, partitioned by the session TVF. NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentWithSource, @@ -317,6 +323,7 @@ void keyedSessionMultiAggregateMatchesHost() throws Exception { @Test void perOperatorFlagKeepsSessionOnHost() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); // A session is a window aggregate, so it answers to the windowAggregate kill switch like the // fixed-bin windows — it reached the native operator ungated until the registry gave every // substitution its own gate. @@ -335,6 +342,7 @@ void perOperatorFlagKeepsSessionOnHost() throws Exception { @Test void sessionMergeMatchesHost() throws Exception { + tech.streamfusion.compat.FlinkTestCapabilities.requireSessionTableFunction(); // An out-of-order element lands between two open sessions and bridges them into one; the native // merge of the two windows' accumulators must match the host's merging assigner. NativeParity.assertParity( @@ -349,39 +357,41 @@ void cumulativeSumMatchesHost() throws Exception { // Cumulative windows: nested windows sharing a bucket start, ends growing by the step. NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentWithSource, - "SELECT window_start, window_end, SUM(`value`) AS s, COUNT(`value`) AS c " - + "FROM TABLE(CUMULATE(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) " - + "GROUP BY window_start, window_end"); + "SELECT window_start, window_end, SUM(`value`) AS s, COUNT(`value`) AS c FROM" + + " TABLE(CUMULATE(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2'" + + " SECOND)) GROUP BY window_start, window_end"); } @Test void keyedCumulativeMultiAggregateMatchesHost() throws Exception { NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentWithSource, - "SELECT k, window_start, window_end, SUM(`value`) AS s, MAX(`value`) AS m " - + "FROM TABLE(CUMULATE(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) " - + "GROUP BY k, window_start, window_end"); + "SELECT k, window_start, window_end, SUM(`value`) AS s, MAX(`value`) AS m FROM" + + " TABLE(CUMULATE(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2'" + + " SECOND)) GROUP BY k, window_start, window_end"); } @Test void twoPhaseCumulativeMatchesHost() throws Exception { - // Two-phase (the default plan): a native local pre-aggregates per 1s slice, the host shuffles by - // key, and a native global re-buckets each slice into the nested cumulative windows up to the 3s + // Two-phase (the default plan): a native local pre-aggregates per 1s slice, the host shuffles + // by + // key, and a native global re-buckets each slice into the nested cumulative windows up to the + // 3s // max size. Both halves substitute (unlike one-phase, which is a single window operator). NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentTwoPhase, - "SELECT k, window_start, window_end, SUM(`value`) AS s, COUNT(`value`) AS c " - + "FROM TABLE(CUMULATE(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '3' SECOND)) " - + "GROUP BY k, window_start, window_end"); + "SELECT k, window_start, window_end, SUM(`value`) AS s, COUNT(`value`) AS c FROM" + + " TABLE(CUMULATE(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '3'" + + " SECOND)) GROUP BY k, window_start, window_end"); } @Test void multiAggregateMatchesHost() throws Exception { NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentWithSource, - "SELECT window_start, window_end, SUM(`value`) AS s, COUNT(`value`) AS c, MAX(`value`) AS m " - + "FROM TABLE(TUMBLE(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND)) " - + "GROUP BY window_start, window_end"); + "SELECT window_start, window_end, SUM(`value`) AS s, COUNT(`value`) AS c, MAX(`value`) AS m" + + " FROM TABLE(TUMBLE(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND)) GROUP BY" + + " window_start, window_end"); } @Test @@ -390,18 +400,18 @@ void twoPhaseHoppingSumMatchesHost() throws Exception { // shuffle, and a global that combines each window's slices must agree with the host. NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentTwoPhase, - "SELECT window_start, window_end, SUM(`value`) AS s " - + "FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) " - + "GROUP BY window_start, window_end"); + "SELECT window_start, window_end, SUM(`value`) AS s FROM TABLE(HOP(TABLE src," + + " DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) GROUP BY window_start," + + " window_end"); } @Test void twoPhaseKeyedHoppingMultiAggregateMatchesHost() throws Exception { NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentTwoPhase, - "SELECT k, window_start, window_end, SUM(`value`) AS s, COUNT(`value`) AS c " - + "FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) " - + "GROUP BY k, window_start, window_end"); + "SELECT k, window_start, window_end, SUM(`value`) AS s, COUNT(`value`) AS c FROM" + + " TABLE(HOP(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND))" + + " GROUP BY k, window_start, window_end"); } @Test @@ -500,18 +510,17 @@ void twoPhaseHoppingCountStarMatchesHost() throws Exception { // counts fanned into each window. NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentTwoPhase, - "SELECT window_start, window_end, COUNT(*) AS n " - + "FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) " - + "GROUP BY window_start, window_end"); + "SELECT window_start, window_end, COUNT(*) AS n FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt)," + + " INTERVAL '1' SECOND, INTERVAL '2' SECOND)) GROUP BY window_start, window_end"); } @Test void twoPhaseHoppingCountStarWithSumMatchesHost() throws Exception { NativeParity.assertParity( FlinkWindowSqlHarnessTest::environmentTwoPhase, - "SELECT window_start, window_end, COUNT(*) AS n, SUM(`value`) AS s " - + "FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) " - + "GROUP BY window_start, window_end"); + "SELECT window_start, window_end, COUNT(*) AS n, SUM(`value`) AS s FROM TABLE(HOP(TABLE" + + " src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) GROUP BY" + + " window_start, window_end"); } @Test @@ -711,9 +720,8 @@ void twoPhaseHoppingNarrowSumWrapsLikeHost() throws Exception { // host's merge expression (a SMALLINT plus) does. NativeParity.assertParity( FlinkWindowSqlHarnessTest::narrowTwoPhaseOverflowEnvironment, - "SELECT window_start, window_end, SUM(sm) AS s " - + "FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) " - + "GROUP BY window_start, window_end"); + "SELECT window_start, window_end, SUM(sm) AS s FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt)," + + " INTERVAL '1' SECOND, INTERVAL '2' SECOND)) GROUP BY window_start, window_end"); } @Test @@ -738,9 +746,8 @@ void twoPhaseDecimalSumOverflowedPartialSkippedLikeHost() throws Exception { // [0s,2s) is 5 (NULL partial skipped), [−1s,1s) is NULL, [1s,3s) is 5. NativeParity.assertParity( FlinkWindowSqlHarnessTest::decimalTwoPhaseOverflowEnvironment, - "SELECT window_start, window_end, SUM(pr) AS s " - + "FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt), INTERVAL '1' SECOND, INTERVAL '2' SECOND)) " - + "GROUP BY window_start, window_end"); + "SELECT window_start, window_end, SUM(pr) AS s FROM TABLE(HOP(TABLE src, DESCRIPTOR(rt)," + + " INTERVAL '1' SECOND, INTERVAL '2' SECOND)) GROUP BY window_start, window_end"); } private static void assertNarrowSumAvgMatchHost(String column) throws Exception { @@ -841,7 +848,8 @@ private static TableEnvironment environmentForSessionMerge() { // Out-of-order: the row at ts=700 arrives after the rows at 0 and 1500, which would otherwise // be separate sessions (gap 1s), and its [700, 1700) window bridges them into [0, 2500). DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"value", "ts"}, Types.LONG, Types.LONG), Row.of(1L, 0L), Row.of(2L, 1500L), @@ -879,7 +887,8 @@ private static TableEnvironment floatPrecisionEnvironment() { // Eight 0.1f's in one 1s window: their float running sum is not 0.8f (rounding accumulates), // so the result distinguishes a float accumulation (the host's) from a double one. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"fl", "ts"}, Types.FLOAT, Types.LONG), Row.of(0.1f, 0L), Row.of(0.1f, 100L), @@ -916,7 +925,8 @@ private static TableEnvironment narrowOverflowEnvironment() { // Two values per 1s window that overflow the narrow width when summed. DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"tn", "sm", "ts"}, Types.BYTE, Types.SHORT, Types.LONG), Row.of((byte) 100, (short) 30000, 0L), @@ -949,7 +959,8 @@ private static TableEnvironment buildEnvironment(boolean onePhase) { } DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] { "k", "value", "ts", "amount", "qty", "g", "s", "sm", "tn", "fl", "b", "d", @@ -969,11 +980,81 @@ private static TableEnvironment buildEnvironment(boolean onePhase) { Types.LOCAL_DATE, Types.BIG_DEC, Types.LOCAL_DATE_TIME), - Row.of(7L, 1L, 0L, 1.5, 10, 100L, "a", (short) 10, (byte) 1, 1.5f, true, DAY_ONE, dec("1.10"), T0), - Row.of(7L, 2L, 500L, 2.5, 20, 100L, "a", (short) 20, (byte) 2, 2.5f, true, DAY_ONE, dec("2.20"), T0), - Row.of(9L, 3L, 600L, 3.0, 30, 200L, "b", (short) 30, (byte) 3, 3.5f, false, DAY_TWO, dec("3.30"), T1), - Row.of(7L, 4L, 1500L, 4.5, 40, 100L, "a", (short) 40, (byte) 4, 4.5f, true, DAY_ONE, dec("4.40"), T0), - Row.of(9L, 5L, 2500L, 5.5, 50, 200L, "b", (short) 50, (byte) 5, 5.5f, false, DAY_TWO, dec("5.50"), T1)) + Row.of( + 7L, + 1L, + 0L, + 1.5, + 10, + 100L, + "a", + (short) 10, + (byte) 1, + 1.5f, + true, + DAY_ONE, + dec("1.10"), + T0), + Row.of( + 7L, + 2L, + 500L, + 2.5, + 20, + 100L, + "a", + (short) 20, + (byte) 2, + 2.5f, + true, + DAY_ONE, + dec("2.20"), + T0), + Row.of( + 9L, + 3L, + 600L, + 3.0, + 30, + 200L, + "b", + (short) 30, + (byte) 3, + 3.5f, + false, + DAY_TWO, + dec("3.30"), + T1), + Row.of( + 7L, + 4L, + 1500L, + 4.5, + 40, + 100L, + "a", + (short) 40, + (byte) 4, + 4.5f, + true, + DAY_ONE, + dec("4.40"), + T0), + Row.of( + 9L, + 5L, + 2500L, + 5.5, + 50, + 200L, + "b", + (short) 50, + (byte) 5, + 5.5f, + false, + DAY_TWO, + dec("5.50"), + T1)) .assignTimestampsAndWatermarks( WatermarkStrategy.forMonotonousTimestamps() .withTimestampAssigner((row, ts) -> (Long) row.getField(2))); @@ -1012,7 +1093,8 @@ private static TableEnvironment narrowTwoPhaseOverflowEnvironment() { // One in-range value per 1s slice; a 2s hopping window merges two slice partials and the merge // itself overflows the SMALLINT width (30000 + 30000 → -5536). DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"sm", "ts"}, Types.SHORT, Types.LONG), Row.of((short) 30000, 0L), Row.of((short) 30000, 1500L)) @@ -1054,7 +1136,8 @@ private static TableEnvironment decimalOverflowEnvironment( java.math.BigDecimal big = new java.math.BigDecimal("99000000000000000000000000000000000000"); DataStream source = - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"pr", "ts"}, Types.BIG_DEC, Types.LONG), Row.of(big, 0L), Row.of(big, secondTs), diff --git a/src/test/java/tech/streamfusion/KindCapturingUpsertTableFactory.java b/src/test/java/tech/streamfusion/KindCapturingUpsertTableFactory.java index 98a2fec39..0c9a48d32 100644 --- a/src/test/java/tech/streamfusion/KindCapturingUpsertTableFactory.java +++ b/src/test/java/tech/streamfusion/KindCapturingUpsertTableFactory.java @@ -5,12 +5,12 @@ import java.util.List; import java.util.Set; import org.apache.flink.configuration.ConfigOption; -import org.apache.flink.streaming.api.functions.sink.legacy.RichSinkFunction; import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.connector.sink.DynamicTableSink; -import org.apache.flink.table.connector.sink.legacy.SinkFunctionProvider; import org.apache.flink.table.data.RowData; import org.apache.flink.table.factories.DynamicTableSinkFactory; +import tech.streamfusion.compat.RichSinkFunction; +import tech.streamfusion.compat.SinkFunctionProvider; /** * An upsert sink over two BIGINT columns that records each arriving change as {@code "kind:k,v"} diff --git a/src/test/java/tech/streamfusion/MapNullKeyLookupParityTest.java b/src/test/java/tech/streamfusion/MapNullKeyLookupParityTest.java index 8e21e05c3..e2bc1f748 100644 --- a/src/test/java/tech/streamfusion/MapNullKeyLookupParityTest.java +++ b/src/test/java/tech/streamfusion/MapNullKeyLookupParityTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.LinkedHashMap; import java.util.Map; import org.apache.flink.api.common.typeinfo.Types; @@ -12,7 +14,8 @@ class MapNullKeyLookupParityTest { @Test void lookupMapWithNullKey() throws Exception { - NativeParity.assertParity(() -> maps(true), "SELECT id, m['a'], m['missing'], m['nullable'] FROM n"); + NativeParity.assertParity( + () -> maps(true), "SELECT id, m['a'], m['missing'], m['nullable'] FROM n"); } private static TableEnvironment maps(boolean withNullKey) { @@ -23,9 +26,15 @@ private static TableEnvironment maps(boolean withNullKey) { if (withNullKey) map.put(null, 7); map.put("a", 1); map.put("nullable", null); - table.createTemporaryView("n", env.fromData( - Types.ROW_NAMED(new String[] {"id", "m"}, Types.INT, Types.MAP(Types.STRING, Types.INT)), - Row.of(1, map), Row.of(2, Map.of()), Row.of(3, null))); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED( + new String[] {"id", "m"}, Types.INT, Types.MAP(Types.STRING, Types.INT)), + Row.of(1, map), + Row.of(2, Map.of()), + Row.of(3, null))); return table; } } diff --git a/src/test/java/tech/streamfusion/MapNullKeyUnnestParityTest.java b/src/test/java/tech/streamfusion/MapNullKeyUnnestParityTest.java index d3c105865..e82ce8d3f 100644 --- a/src/test/java/tech/streamfusion/MapNullKeyUnnestParityTest.java +++ b/src/test/java/tech/streamfusion/MapNullKeyUnnestParityTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.LinkedHashMap; import java.util.Map; import org.apache.flink.api.common.typeinfo.Types; @@ -12,8 +14,13 @@ class MapNullKeyUnnestParityTest { @Test void ordinalityAndNullKeyFilter() throws Exception { - NativeParity.assertParity(() -> maps(true), - "SELECT id, u.k, u.v, u.pos FROM n CROSS JOIN UNNEST(m) WITH ORDINALITY AS u(k, v, pos) WHERE u.k IS NULL"); + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.UNNEST_ORDINALITY, + "Flink 1.18 cannot plan UNNEST WITH ORDINALITY"); + NativeParity.assertParity( + () -> maps(true), + "SELECT id, u.k, u.v, u.pos FROM n CROSS JOIN UNNEST(m) WITH ORDINALITY AS u(k, v, pos)" + + " WHERE u.k IS NULL"); } @Test @@ -35,9 +42,15 @@ private static TableEnvironment maps(boolean withNullKey) { if (withNullKey) map.put(null, 7); map.put("a", 1); map.put("nullable", null); - table.createTemporaryView("n", env.fromData( - Types.ROW_NAMED(new String[] {"id", "m"}, Types.INT, Types.MAP(Types.STRING, Types.INT)), - Row.of(1, map), Row.of(2, Map.of()), Row.of(3, null))); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED( + new String[] {"id", "m"}, Types.INT, Types.MAP(Types.STRING, Types.INT)), + Row.of(1, map), + Row.of(2, Map.of()), + Row.of(3, null))); return table; } } diff --git a/src/test/java/tech/streamfusion/NaNPayloadDistinctParityTest.java b/src/test/java/tech/streamfusion/NaNPayloadDistinctParityTest.java index 43acbcaf9..e01f7684d 100644 --- a/src/test/java/tech/streamfusion/NaNPayloadDistinctParityTest.java +++ b/src/test/java/tech/streamfusion/NaNPayloadDistinctParityTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -36,20 +38,32 @@ private static TableEnvironment input(boolean canonical, int batch) { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); StreamTableEnvironment table = StreamTableEnvironment.create(env); - table.getConfig().set("table.optimizer.agg-phase-strategy", batch == 0 ? "ONE_PHASE" : "TWO_PHASE"); + table + .getConfig() + .set("table.optimizer.agg-phase-strategy", batch == 0 ? "ONE_PHASE" : "TWO_PHASE"); if (batch > 0) { table.getConfig().set("table.exec.mini-batch.enabled", "true"); table.getConfig().set("table.exec.mini-batch.allow-latency", "100000 d"); table.getConfig().set("table.exec.mini-batch.size", Integer.toString(batch)); } // Construct the NaNs after source deserialization, which may otherwise canonicalize their bits. - table.createTemporaryView("n", env.fromData(1, 2).map(id -> { - double d = Double.longBitsToDouble(0x7ff8000000000000L + (canonical ? 0 : id)); - float f = Float.intBitsToFloat(0x7fc00000 + (canonical ? 0 : id)); - return Row.of(id, d, f, new Double[] {d}, new Float[] {f}); - }).returns(Types.ROW_NAMED(new String[] {"id", "d", "f", "da", "fa"}, - Types.INT, Types.DOUBLE, Types.FLOAT, - Types.OBJECT_ARRAY(Types.DOUBLE), Types.OBJECT_ARRAY(Types.FLOAT)))); + table.createTemporaryView( + "n", + fromData(env, 1, 2) + .map( + id -> { + double d = Double.longBitsToDouble(0x7ff8000000000000L + (canonical ? 0 : id)); + float f = Float.intBitsToFloat(0x7fc00000 + (canonical ? 0 : id)); + return Row.of(id, d, f, new Double[] {d}, new Float[] {f}); + }) + .returns( + Types.ROW_NAMED( + new String[] {"id", "d", "f", "da", "fa"}, + Types.INT, + Types.DOUBLE, + Types.FLOAT, + Types.OBJECT_ARRAY(Types.DOUBLE), + Types.OBJECT_ARRAY(Types.FLOAT)))); return table; } } diff --git a/src/test/java/tech/streamfusion/NativeAvroDecodeSqlHarnessTest.java b/src/test/java/tech/streamfusion/NativeAvroDecodeSqlHarnessTest.java index e629dae9c..649ec8355 100644 --- a/src/test/java/tech/streamfusion/NativeAvroDecodeSqlHarnessTest.java +++ b/src/test/java/tech/streamfusion/NativeAvroDecodeSqlHarnessTest.java @@ -61,7 +61,8 @@ void avroMessagesDecodeNativelyWithFlinkParity() throws Exception { .getLogicalType(); produce(brokers, "avro-flat", flat, NativeAvroDecodeSqlHarnessTest::flatRecord); NativeParity.assertParity( - environment(brokers, "avro-flat", "id BIGINT, name STRING, score DOUBLE"), "SELECT * FROM t"); + environment(brokers, "avro-flat", "id BIGINT, name STRING, score DOUBLE"), + "SELECT * FROM t"); RowType complex = (RowType) @@ -73,14 +74,16 @@ void avroMessagesDecodeNativelyWithFlinkParity() throws Exception { DataTypes.FIELD("a", DataTypes.BIGINT()), DataTypes.FIELD("b", DataTypes.STRING()))), DataTypes.FIELD("nums", DataTypes.ARRAY(DataTypes.BIGINT())), - DataTypes.FIELD("tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT()))) + DataTypes.FIELD( + "tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT()))) .getLogicalType(); produce(brokers, "avro-complex", complex, NativeAvroDecodeSqlHarnessTest::complexRecord); NativeParity.assertParity( environment( brokers, "avro-complex", - "id BIGINT, nested ROW, nums ARRAY, tags MAP"), + "id BIGINT, nested ROW, nums ARRAY, tags MAP"), "SELECT id, nested.a, nested.b, nums[1], nums[2], tags['a'], tags['b'] FROM t"); } } @@ -146,9 +149,22 @@ void temporalAndDecimalColumnsDecodeNativelyThroughTheFullPlan() throws Exceptio produce(brokers, "avro-temporal", temporal, NativeAvroDecodeSqlHarnessTest::temporalRecord); NativeParity.assertParity( environment( - brokers, "avro-temporal", "id BIGINT, price DECIMAL(10, 2), dt DATE, ts TIMESTAMP(3)"), + brokers, + "avro-temporal", + "id BIGINT, price DECIMAL(10, 2), dt DATE, ts TIMESTAMP(3)"), "SELECT * FROM t"); + } + } + @Test + void correctedTimestampMappingDecodesNativelyThroughTheFullPlan() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.format.avro.compat.AvroTestSchemas.CORRECTED_TIMESTAMP_MAPPING, + "Corrected Avro timestamp mapping is absent from Flink 1.18"); + try (KafkaContainer kafka = + new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:7.6.1"))) { + kafka.start(); + String brokers = kafka.getBootstrapServers(); // The corrected timestamp mapping: TIMESTAMP_LTZ is only representable with // avro.timestamp_mapping.legacy = false, and the writer schema derives differently. RowType corrected = @@ -161,7 +177,7 @@ void temporalAndDecimalColumnsDecodeNativelyThroughTheFullPlan() throws Exceptio produce( brokers, "avro-corrected", - AvroSchemaConverter.convertToSchema(corrected.copy(false), false), + tech.streamfusion.format.avro.compat.AvroCompat.schema(corrected.copy(false), false), NativeAvroDecodeSqlHarnessTest::correctedRecord); NativeParity.assertParity( environment( @@ -192,7 +208,8 @@ private static void wideRecord(GenericRecord record, int i, Schema schema) { int eventType = i % 2 == 0 ? 0 : 2; record.put("event_type", eventType); if (eventType == 0) { - GenericRecord person = new GenericData.Record(recordBranch(schema.getField("person").schema())); + GenericRecord person = + new GenericData.Record(recordBranch(schema.getField("person").schema())); person.put("id", (long) i); person.put("name", "n-" + i); person.put("email", "e-" + i); @@ -299,8 +316,8 @@ private static Supplier environment( + brokers + "', 'properties.group.id' = '" + topic - + "', 'scan.startup.mode' = 'earliest-offset', 'scan.bounded.mode' = 'latest-offset', " - + "'format' = 'avro'" + + "', 'scan.startup.mode' = 'earliest-offset', 'scan.bounded.mode' = 'latest-offset'," + + " 'format' = 'avro'" + extraOptions + ")"); return tEnv; diff --git a/src/test/java/tech/streamfusion/NexmarkFormatDecodeBenchmark.java b/src/test/java/tech/streamfusion/NexmarkFormatDecodeBenchmark.java index f0a11d31a..2846b1495 100644 --- a/src/test/java/tech/streamfusion/NexmarkFormatDecodeBenchmark.java +++ b/src/test/java/tech/streamfusion/NexmarkFormatDecodeBenchmark.java @@ -31,7 +31,6 @@ import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.EncoderFactory; import org.apache.flink.api.common.serialization.DeserializationSchema; -import org.apache.flink.formats.avro.AvroFormatOptions.AvroEncoding; import org.apache.flink.formats.avro.AvroRowDataDeserializationSchema; import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; import org.apache.flink.formats.common.TimestampFormat; @@ -225,8 +224,8 @@ private static JavaDecoder javaDecoder(String format, RowType rowType) throws Ex }; case "avro": AvroRowDataDeserializationSchema avro = - new AvroRowDataDeserializationSchema( - physicalType, typeInfo, AvroEncoding.BINARY, true); + tech.streamfusion.format.avro.compat.AvroTestSchemas.decoder( + physicalType, typeInfo, true); avro.open(initializationContext()); return rowDecoder(avro); case "protobuf": diff --git a/src/test/java/tech/streamfusion/NexmarkMatrixBenchmark.java b/src/test/java/tech/streamfusion/NexmarkMatrixBenchmark.java index c1061cbb4..ef807f3f0 100644 --- a/src/test/java/tech/streamfusion/NexmarkMatrixBenchmark.java +++ b/src/test/java/tech/streamfusion/NexmarkMatrixBenchmark.java @@ -14,7 +14,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BooleanSupplier; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.sink.legacy.RichSinkFunction; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; diff --git a/src/test/java/tech/streamfusion/PortableSqlFixtures.java b/src/test/java/tech/streamfusion/PortableSqlFixtures.java index b32413bff..0f5a85ab1 100644 --- a/src/test/java/tech/streamfusion/PortableSqlFixtures.java +++ b/src/test/java/tech/streamfusion/PortableSqlFixtures.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.ZoneId; import java.util.Map; import org.apache.flink.api.common.RuntimeExecutionMode; @@ -55,7 +57,8 @@ static TableEnvironment environment(RuntimeExecutionMode mode, Map, AutoCloseable { diff --git a/src/test/java/tech/streamfusion/ScalarFunctionBenchmark.java b/src/test/java/tech/streamfusion/ScalarFunctionBenchmark.java index eebf2728d..26d9aa42b 100644 --- a/src/test/java/tech/streamfusion/ScalarFunctionBenchmark.java +++ b/src/test/java/tech/streamfusion/ScalarFunctionBenchmark.java @@ -237,6 +237,127 @@ String ddl() { "JSON_EXISTS(s, '$.order-id.123.a\tb')", "BOOLEAN"), new Query("JSON_QUERY_SLICE", "tt_json_negative", "JSON_QUERY(s, '$.a[0:2]')"), + new Query( + "JSON_EXISTS_SLICE", + "tt_json_negative", + "JSON_EXISTS(s, '$.a[0:2]')", + "BOOLEAN"), + new Query( + "JSON_VALUE_WILDCARD", + "tt_json_negative", + "JSON_VALUE(s, 'lax $.a[*]' DEFAULT 'empty' ON EMPTY)", + "STRING"), + new Query( + "JSON_EXISTS_WILDCARD", + "tt_json_negative", + "JSON_EXISTS(s, '$.a[*]')", + "BOOLEAN"), + new Query( + "JSON_VALUE_INDEX_UNION", + "tt_json_negative", + "JSON_VALUE(s, 'lax $.a[0,16,-1]' DEFAULT 'empty' ON EMPTY)", + "STRING"), + new Query( + "JSON_EXISTS_INDEX_UNION", + "tt_json_negative", + "JSON_EXISTS(s, '$.a[0,16,-1]')", + "BOOLEAN"), + new Query( + "JSON_VALUE_INDEX_WHITESPACE", + "tt_json_negative", + "JSON_VALUE(s, '$.a[31\t\n ]')", + "STRING"), + new Query( + "JSON_EXISTS_INDEX_WHITESPACE", + "tt_json_negative", + "JSON_EXISTS(s, '$.a[31\t\n ]')", + "BOOLEAN"), + new Query( + "JSON_VALUE_POSITIVE_CONTROL", + "tt_json_negative", + "JSON_VALUE(s, '$.a[31]')", + "STRING"), + new Query( + "JSON_EXISTS_NEGATIVE", + "tt_json_negative", + "JSON_EXISTS(s, '$.a[-1]')", + "BOOLEAN"), + new Query( + "JSON_VALUE_IDENTITY", + "tt_json_surrogate", + "JSON_VALUE(s, '$') = '?'", + "BOOLEAN"), + new Query( + "JSON_UNQUOTE_IDENTITY", + "tt_json_surrogate", + "JSON_UNQUOTE(s) = '?'", + "BOOLEAN"), + new Query( + "JSON_VALUE_EMPTY_MEMBER", + "tt_json_empty_member", + "JSON_VALUE(s, 'lax $['''']')", + "STRING"), + new Query( + "JSON_EXISTS_EMPTY_MEMBER", + "tt_json_empty_member", + "JSON_EXISTS(s, 'lax $[\"\"]')", + "BOOLEAN"), + new Query( + "JSON_VALUE_SPACED_PATH", + "tt_json", + "JSON_VALUE(s, 'lax $[ ''user'' ][ ''name'' ]')", + "STRING"), + new Query( + "JSON_VALUE_NONSTANDARD_ESCAPE", + "tt_json", + "JSON_VALUE(s, 'lax $[\"u\\ser\"][\"na\\me\"]')", + "STRING"), + new Query( + "JSON_EXISTS_NONSTANDARD_ESCAPE", + "tt_json", + "JSON_EXISTS(s, 'lax $[\"u\\ser\"][\"na\\me\"]')", + "BOOLEAN"), + new Query( + "JSON_EXISTS_SPACED_PATH", + "tt_json", + "JSON_EXISTS(s, 'lax $[ ''user'' ][ ''name'' ]')", + "BOOLEAN"), + new Query( + "JSON_VALUE_ACTIVE", + "tt_json", + "JSON_VALUE(s, 'lax $.user.active')", + "STRING"), + new Query( + "JSON_VALUE_ERROR", + "tt_json", + "JSON_VALUE(s, 'lax $.user.active' ERROR ON EMPTY ERROR ON ERROR)", + "STRING"), + new Query( + "JSON_VALUE_ESCAPED_PATH", + "tt_json_escaped", + "JSON_VALUE(s, '$[\"a\\\\b\"][\"a\\nb\"]')", + "STRING"), + new Query( + "JSON_EXISTS_ESCAPED_PATH", + "tt_json_escaped", + "JSON_EXISTS(s, '$[\"a\\\\b\"][\"a\\nb\"]')", + "BOOLEAN"), + new Query( + "JSON_VALUE_NEGATIVE", + "tt_json_negative", + "JSON_VALUE(s, '$.a[-1]')", + "STRING"), + new Query( + "JSON_VALUE_DOT_MEMBER", + "tt_json_dot_member", + "JSON_VALUE(s, '$.order-id.123.a\tb')", + "STRING"), + new Query( + "JSON_EXISTS_DOT_MEMBER", + "tt_json_dot_member", + "JSON_EXISTS(s, '$.order-id.123.a\tb')", + "BOOLEAN"), + new Query("JSON_QUERY_SLICE", "tt_json_negative", "JSON_QUERY(s, '$.a[0:2]')"), new Query( "JSON_EXISTS_SLICE", "tt_json_negative", diff --git a/src/test/java/tech/streamfusion/StatefulMapPayloadParityTest.java b/src/test/java/tech/streamfusion/StatefulMapPayloadParityTest.java index 8d302aea9..5713830d1 100644 --- a/src/test/java/tech/streamfusion/StatefulMapPayloadParityTest.java +++ b/src/test/java/tech/streamfusion/StatefulMapPayloadParityTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.LinkedHashMap; import java.util.Map; import org.apache.flink.api.common.typeinfo.Types; @@ -51,9 +53,13 @@ private static TableEnvironment maps(boolean withNullKey) { Map map = new LinkedHashMap<>(); if (withNullKey) map.put(null, 7); map.put("a", 1); - table.createTemporaryView("n", env.fromData( - Types.ROW_NAMED(new String[] {"id", "m"}, Types.INT, Types.MAP(Types.STRING, Types.INT)), - Row.of(1, map))); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED( + new String[] {"id", "m"}, Types.INT, Types.MAP(Types.STRING, Types.INT)), + Row.of(1, map))); return table; } } diff --git a/src/test/java/tech/streamfusion/StringFunctionTestInputs.java b/src/test/java/tech/streamfusion/StringFunctionTestInputs.java index 12db672c3..6ad5dbfc1 100644 --- a/src/test/java/tech/streamfusion/StringFunctionTestInputs.java +++ b/src/test/java/tech/streamfusion/StringFunctionTestInputs.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -20,7 +22,8 @@ static TableEnvironment search() { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.createTemporaryView( "searches", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "s", "needle", "start_pos", "binary_value"}, Types.INT, @@ -132,7 +135,8 @@ static TableEnvironment encodings() { } tEnv.createTemporaryView( "encodings", - env.fromData( + fromData( + env, rows, Types.ROW_NAMED( new String[] {"id", "s", "hex_text", "n", "i", "sh", "t"}, @@ -276,7 +280,8 @@ static TableEnvironment text() { } tables.createTemporaryView( "texts", - env.fromData( + fromData( + env, rows, Types.ROW_NAMED( new String[] {"id", "s", "f", "t", "n", "i", "p", "b", "d", "u"}, @@ -356,7 +361,7 @@ static TableEnvironment urlBoundaries() { StreamTableEnvironment tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "urls", - env.fromData(rows, Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING)), + fromData(env, rows, Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING)), Schema.newBuilder().column("id", DataTypes.INT()).column("s", DataTypes.STRING()).build()); return tables; } diff --git a/src/test/java/tech/streamfusion/TextTimeFunctionTestInputs.java b/src/test/java/tech/streamfusion/TextTimeFunctionTestInputs.java index 371c9c64a..a7fc41fb7 100644 --- a/src/test/java/tech/streamfusion/TextTimeFunctionTestInputs.java +++ b/src/test/java/tech/streamfusion/TextTimeFunctionTestInputs.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.util.ArrayList; import java.util.List; import org.apache.flink.api.common.typeinfo.Types; @@ -49,7 +51,8 @@ static TableEnvironment strings() { StreamTableEnvironment tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "inputs", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING), rows.toArray(Row[]::new)), Schema.newBuilder().column("id", DataTypes.INT()).column("s", DataTypes.STRING()).build()); @@ -75,7 +78,8 @@ static TableEnvironment parameters() { StreamTableEnvironment tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "inputs", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "s", "n", "p"}, Types.INT, @@ -132,7 +136,8 @@ static TableEnvironment textRows(String... text) { StreamTableEnvironment tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "inputs", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "s"}, Types.INT, Types.STRING), rows.toArray(Row[]::new)), Schema.newBuilder().column("id", DataTypes.INT()).column("s", DataTypes.STRING()).build()); @@ -169,7 +174,8 @@ static TableEnvironment calendar(int precision) { StreamTableEnvironment tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "inputs", - env.fromData( + fromData( + env, Types.ROW_NAMED( new String[] {"id", "d", "ts"}, Types.INT, Types.LOCAL_DATE, Types.LOCAL_DATE_TIME), rows.toArray(Row[]::new)), @@ -195,7 +201,8 @@ static TableEnvironment bytes() { StreamTableEnvironment tables = StreamTableEnvironment.create(env); tables.createTemporaryView( "inputs", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"id", "b"}, Types.INT, Types.PRIMITIVE_ARRAY(Types.BYTE)), rows.toArray(Row[]::new)), Schema.newBuilder().column("id", DataTypes.INT()).column("b", DataTypes.BYTES()).build()); diff --git a/src/test/java/tech/streamfusion/TimestampRangeParityTest.java b/src/test/java/tech/streamfusion/TimestampRangeParityTest.java index b6f934263..c62472ed9 100644 --- a/src/test/java/tech/streamfusion/TimestampRangeParityTest.java +++ b/src/test/java/tech/streamfusion/TimestampRangeParityTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import java.time.Instant; import java.time.LocalDateTime; import org.apache.flink.api.common.typeinfo.Types; @@ -15,8 +17,14 @@ class TimestampRangeParityTest { @ParameterizedTest - @ValueSource(strings = {"0001-01-01T00:00:00.123456789", "1582-10-15T23:59:59.999999999", - "1969-12-31T23:59:59.999999999", "2262-04-12T00:00:00.123456789", "9999-01-01T00:00:00.999999999"}) + @ValueSource( + strings = { + "0001-01-01T00:00:00.123456789", + "1582-10-15T23:59:59.999999999", + "1969-12-31T23:59:59.999999999", + "2262-04-12T00:00:00.123456789", + "9999-01-01T00:00:00.999999999" + }) void fullRangePassThrough(String value) throws Exception { NativeParity.assertParity(() -> timestamps(value), "SELECT id + 1, ts, ltz FROM n"); @@ -71,11 +79,19 @@ private static TableEnvironment timestamps(String value) { env.setParallelism(1); StreamTableEnvironment table = StreamTableEnvironment.create(env); table.getConfig().setLocalTimeZone(java.time.ZoneId.of("UTC")); - table.createTemporaryView("n", env.fromData( - Types.ROW_NAMED(new String[] {"id", "ts", "ltz"}, Types.INT, Types.LOCAL_DATE_TIME, Types.INSTANT), - Row.of(1, LocalDateTime.parse(value), Instant.parse(value + "Z"))), - Schema.newBuilder().column("id", DataTypes.INT()).column("ts", DataTypes.TIMESTAMP(3)) - .column("ltz", DataTypes.TIMESTAMP_LTZ(3)).watermark("ts", "ts - INTERVAL '1' SECOND").build()); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED( + new String[] {"id", "ts", "ltz"}, Types.INT, Types.LOCAL_DATE_TIME, Types.INSTANT), + Row.of(1, LocalDateTime.parse(value), Instant.parse(value + "Z"))), + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("ts", DataTypes.TIMESTAMP(3)) + .column("ltz", DataTypes.TIMESTAMP_LTZ(3)) + .watermark("ts", "ts - INTERVAL '1' SECOND") + .build()); return table; } } diff --git a/src/test/java/tech/streamfusion/TopNSignedZeroParityTest.java b/src/test/java/tech/streamfusion/TopNSignedZeroParityTest.java index abf4188de..79fa6abf5 100644 --- a/src/test/java/tech/streamfusion/TopNSignedZeroParityTest.java +++ b/src/test/java/tech/streamfusion/TopNSignedZeroParityTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -63,9 +64,13 @@ private static TableEnvironment zeros() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); StreamTableEnvironment table = StreamTableEnvironment.create(env); - table.createTemporaryView("n", env.fromData( - Types.ROW_NAMED(new String[] {"id", "d", "f"}, Types.INT, Types.DOUBLE, Types.FLOAT), - Row.of(1, 0.0d, 0.0f), Row.of(2, -0.0d, -0.0f))); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED(new String[] {"id", "d", "f"}, Types.INT, Types.DOUBLE, Types.FLOAT), + Row.of(1, 0.0d, 0.0f), + Row.of(2, -0.0d, -0.0f))); return table; } } diff --git a/src/test/java/tech/streamfusion/UnicodeOrderingParityReproTest.java b/src/test/java/tech/streamfusion/UnicodeOrderingParityReproTest.java index 68ce1d972..6894e166c 100644 --- a/src/test/java/tech/streamfusion/UnicodeOrderingParityReproTest.java +++ b/src/test/java/tech/streamfusion/UnicodeOrderingParityReproTest.java @@ -1,5 +1,7 @@ package tech.streamfusion; +import static tech.streamfusion.compat.FlinkTestSources.fromData; + import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.TableEnvironment; @@ -58,8 +60,10 @@ void serializedInputControl() throws Exception { @Test void topNControl() throws Exception { - NativeParity.assertChangelogParity(UnicodeOrderingParityReproTest::unicode, - "SELECT id, rn FROM (SELECT id, ROW_NUMBER() OVER (ORDER BY s, id) rn FROM n) WHERE rn <= 2"); + NativeParity.assertChangelogParity( + UnicodeOrderingParityReproTest::unicode, + "SELECT id, rn FROM (SELECT id, ROW_NUMBER() OVER (ORDER BY s, id) rn FROM n) WHERE rn <=" + + " 2"); } private static void assertOrderingFallback( @@ -76,9 +80,13 @@ private static TableEnvironment strings(String first, String second, boolean ser env.setParallelism(1); if (serialized) env.disableOperatorChaining(); StreamTableEnvironment table = StreamTableEnvironment.create(env); - table.createTemporaryView("n", env.fromData(Types.ROW_NAMED( - new String[] {"id", "s", "t"}, Types.INT, Types.STRING, Types.STRING), - Row.of(1, first, second), Row.of(2, second, first))); + table.createTemporaryView( + "n", + fromData( + env, + Types.ROW_NAMED(new String[] {"id", "s", "t"}, Types.INT, Types.STRING, Types.STRING), + Row.of(1, first, second), + Row.of(2, second, first))); return table; } } diff --git a/src/test/java/tech/streamfusion/format/AvroDecodeGateTest.java b/src/test/java/tech/streamfusion/format/AvroDecodeGateTest.java index 407458d1e..4249d61c2 100644 --- a/src/test/java/tech/streamfusion/format/AvroDecodeGateTest.java +++ b/src/test/java/tech/streamfusion/format/AvroDecodeGateTest.java @@ -3,9 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import tech.streamfusion.format.avro.AvroFormatProvider; -import tech.streamfusion.format.avroconfluent.AvroConfluentFormatProvider; -import tech.streamfusion.format.avroconfluent.DebeziumAvroConfluentFormatProvider; import java.util.Map; import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; @@ -26,6 +23,9 @@ import org.apache.flink.table.types.logical.VarBinaryType; import org.apache.flink.table.types.logical.VarCharType; import org.junit.jupiter.api.Test; +import tech.streamfusion.format.avro.AvroFormatProvider; +import tech.streamfusion.format.avroconfluent.AvroConfluentFormatProvider; +import tech.streamfusion.format.avroconfluent.DebeziumAvroConfluentFormatProvider; /** * The Avro providers must decline — not crash job submission for — every table whose type or @@ -165,6 +165,10 @@ void declinesUnreproducedOptions() { void correctedTimestampMappingFollowsFlinksOwnAcceptance() { Map nonLegacy = Map.of("format", "avro", "avro.timestamp_mapping.legacy", "false"); + if (!tech.streamfusion.compat.FlinkTestCapabilities.CORRECTED_AVRO_TIMESTAMPS) { + assertFalse(bareAvro(SUPPORTED, nonLegacy, false)); + return; + } assertTrue(bareAvro(SUPPORTED, nonLegacy, false)); // The corrected mapping unlocks TIMESTAMP_LTZ and micros-precision timestamps at the top level. LogicalType[] corrected = { diff --git a/src/test/java/tech/streamfusion/kafka/KafkaDecodeRoutingTest.java b/src/test/java/tech/streamfusion/kafka/KafkaDecodeRoutingTest.java index 05b9748aa..aa0e2c917 100644 --- a/src/test/java/tech/streamfusion/kafka/KafkaDecodeRoutingTest.java +++ b/src/test/java/tech/streamfusion/kafka/KafkaDecodeRoutingTest.java @@ -3,12 +3,12 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import tech.streamfusion.planner.NativePlanner; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.NativePlanner; /** * Plan-time gating for the native Kafka decode paths (no broker needed): a table whose schema or @@ -90,6 +90,9 @@ void nonStringMapKeyFailsPlanningOnFlinkItself() { @Test void treeDeserializerOptionKeepsTheScanOnFlink() { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.JSON_PARSER_OPTION, + "Flink 1.18 omits json.decode.json-parser.enabled from its factory option whitelist"); // decode.json-parser.enabled = false switches Flink to its tree deserializer, whose coercions // differ from the parser path the native decode mirrors. StreamTableEnvironment tEnv = env(); @@ -113,7 +116,8 @@ void debeziumAvroConfluentRoutes() { table("id BIGINT, name STRING", "debezium-avro-confluent") .replace( "'debezium-avro-confluent')", - "'debezium-avro-confluent', 'debezium-avro-confluent.url' = 'http://localhost:8081')")); + "'debezium-avro-confluent', 'debezium-avro-confluent.url' =" + + " 'http://localhost:8081')")); String plan = NativePlanner.explain(tEnv, "SELECT id, name FROM t"); assertTrue( plan.contains("NativeKafkaDecode"), @@ -129,7 +133,8 @@ void debeziumAvroConfluentUserInfoAuthRoutes() { table("id BIGINT, name STRING", "debezium-avro-confluent") .replace( "'debezium-avro-confluent')", - "'debezium-avro-confluent', 'debezium-avro-confluent.url' = 'http://localhost:8081'," + "'debezium-avro-confluent', 'debezium-avro-confluent.url' =" + + " 'http://localhost:8081'," + " 'debezium-avro-confluent.basic-auth.credentials-source' = 'USER_INFO'," + " 'debezium-avro-confluent.basic-auth.user-info' = 'user:pw')")); String plan = NativePlanner.explain(tEnv, "SELECT id, name FROM t"); @@ -148,7 +153,8 @@ void debeziumAvroConfluentUntranslatedRegistryAuthKeepsTheScanOnFlink() { table("id BIGINT, name STRING", "debezium-avro-confluent") .replace( "'debezium-avro-confluent')", - "'debezium-avro-confluent', 'debezium-avro-confluent.url' = 'http://localhost:8081'," + "'debezium-avro-confluent', 'debezium-avro-confluent.url' =" + + " 'http://localhost:8081'," + " 'debezium-avro-confluent.basic-auth.credentials-source' = 'URL')")); assertStaysOnFlink(tEnv, "SELECT id, name FROM t"); } diff --git a/src/test/java/tech/streamfusion/kafka/NativeKafkaAvroEncoderTest.java b/src/test/java/tech/streamfusion/kafka/NativeKafkaAvroEncoderTest.java index 59fd8ff2a..b9611791a 100644 --- a/src/test/java/tech/streamfusion/kafka/NativeKafkaAvroEncoderTest.java +++ b/src/test/java/tech/streamfusion/kafka/NativeKafkaAvroEncoderTest.java @@ -7,9 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import tech.streamfusion.format.EncodeFormat; -import tech.streamfusion.format.LogicalTypeDescriptors; -import tech.streamfusion.operator.RowDataArrowConverter; import java.math.BigDecimal; import java.util.LinkedHashMap; import java.util.List; @@ -21,7 +18,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.formats.avro.AvroFormatOptions.AvroEncoding; import org.apache.flink.formats.avro.AvroRowDataSerializationSchema; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.GenericArrayData; @@ -52,6 +48,9 @@ import org.apache.flink.table.types.logical.VarCharType; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import tech.streamfusion.format.EncodeFormat; +import tech.streamfusion.format.LogicalTypeDescriptors; +import tech.streamfusion.operator.RowDataArrowConverter; /** * Byte-level referee against Flink's own Avro serializer: the native encode must produce the exact @@ -291,7 +290,7 @@ private static byte[][] encode( private static AvroRowDataSerializationSchema referee(RowType rowType, boolean legacy) throws Exception { AvroRowDataSerializationSchema referee = - new AvroRowDataSerializationSchema(rowType, AvroEncoding.BINARY, legacy); + tech.streamfusion.format.avro.compat.AvroTestSchemas.encoder(rowType, legacy); referee.open(null); return referee; } diff --git a/src/test/java/tech/streamfusion/kafka/NativeKafkaCdcEncoderTest.java b/src/test/java/tech/streamfusion/kafka/NativeKafkaCdcEncoderTest.java index e2267441e..a00cc7027 100644 --- a/src/test/java/tech/streamfusion/kafka/NativeKafkaCdcEncoderTest.java +++ b/src/test/java/tech/streamfusion/kafka/NativeKafkaCdcEncoderTest.java @@ -5,9 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import tech.streamfusion.format.EncodeFormat; -import tech.streamfusion.format.LogicalTypeDescriptors; -import tech.streamfusion.operator.RowDataArrowConverter; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -24,10 +21,6 @@ import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.formats.common.TimestampFormat; import org.apache.flink.formats.json.JsonFormatOptions; -import org.apache.flink.formats.json.canal.CanalJsonSerializationSchema; -import org.apache.flink.formats.json.debezium.DebeziumJsonSerializationSchema; -import org.apache.flink.formats.json.maxwell.MaxwellJsonSerializationSchema; -import org.apache.flink.formats.json.ogg.OggJsonSerializationSchema; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; import org.apache.flink.table.data.DecimalData; @@ -47,6 +40,9 @@ import org.apache.flink.util.UserCodeClassLoader; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import tech.streamfusion.format.EncodeFormat; +import tech.streamfusion.format.LogicalTypeDescriptors; +import tech.streamfusion.operator.RowDataArrowConverter; /** * Byte-level referee for the four CDC JSON envelope encoders against Flink's own serialization @@ -122,7 +118,7 @@ private static SerializationSchema debezium( String literal, boolean plainDecimal, boolean ignoreNulls) { - return new DebeziumJsonSerializationSchema( + return tech.streamfusion.compat.JsonTestSchemas.debezium( rowType, timestampFormat, mode, literal, plainDecimal, ignoreNulls); } @@ -133,7 +129,7 @@ private static SerializationSchema canal( String literal, boolean plainDecimal, boolean ignoreNulls) { - return new CanalJsonSerializationSchema( + return tech.streamfusion.compat.JsonTestSchemas.canal( rowType, timestampFormat, mode, literal, plainDecimal, ignoreNulls); } @@ -144,7 +140,7 @@ private static SerializationSchema maxwell( String literal, boolean plainDecimal, boolean ignoreNulls) { - return new MaxwellJsonSerializationSchema( + return tech.streamfusion.compat.JsonTestSchemas.maxwell( rowType, timestampFormat, mode, literal, plainDecimal, ignoreNulls); } @@ -155,7 +151,7 @@ private static SerializationSchema ogg( String literal, boolean plainDecimal, boolean ignoreNulls) { - return new OggJsonSerializationSchema( + return tech.streamfusion.compat.JsonTestSchemas.ogg( rowType, timestampFormat, mode, literal, plainDecimal, ignoreNulls); } @@ -184,7 +180,8 @@ private static void assertMatchesFlink(String identifier, FlinkCdcSchema flinkSc throws Exception { for (TimestampFormat timestampFormat : new TimestampFormat[] {TimestampFormat.SQL, TimestampFormat.ISO_8601}) { - for (boolean ignoreNullFields : new boolean[] {false, true}) { + for (boolean ignoreNullFields : + tech.streamfusion.compat.JsonTestSchemas.IGNORE_NULL_FIELD_MODES) { assertMatchesFlink(identifier, flinkSchema, timestampFormat, ignoreNullFields); } } diff --git a/src/test/java/tech/streamfusion/kafka/NativeKafkaJsonEncoderTest.java b/src/test/java/tech/streamfusion/kafka/NativeKafkaJsonEncoderTest.java index 679af2dc0..962b235c1 100644 --- a/src/test/java/tech/streamfusion/kafka/NativeKafkaJsonEncoderTest.java +++ b/src/test/java/tech/streamfusion/kafka/NativeKafkaJsonEncoderTest.java @@ -6,9 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import tech.streamfusion.format.EncodeFormat; -import tech.streamfusion.format.LogicalTypeDescriptors; -import tech.streamfusion.operator.RowDataArrowConverter; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.time.LocalDate; @@ -60,6 +57,9 @@ import org.apache.flink.util.UserCodeClassLoader; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import tech.streamfusion.format.EncodeFormat; +import tech.streamfusion.format.LogicalTypeDescriptors; +import tech.streamfusion.operator.RowDataArrowConverter; @Tag("streamfusion-kafka") class NativeKafkaJsonEncoderTest { @@ -76,7 +76,8 @@ class NativeKafkaJsonEncoderTest { @Test void matchesFlinkForWholeBatchesWithAndWithoutNullFields() throws Exception { - GenericRowData first = GenericRowData.of(1, StringData.fromString("quote: \" and 雪"), true, 25L); + GenericRowData first = + GenericRowData.of(1, StringData.fromString("quote: \" and 雪"), true, 25L); GenericRowData nulls = GenericRowData.of(2, null, false, null); List rows = List.of(first, nulls); @@ -538,7 +539,7 @@ void failsLikeFlinkOnNullMapKeys() throws Exception { List rows = List.of(GenericRowData.of(new GenericMapData(data))); JsonRowDataSerializationSchema flink = - new JsonRowDataSerializationSchema( + tech.streamfusion.compat.JsonTestSchemas.json( rowType, TimestampFormat.SQL, JsonFormatOptions.MapNullKeyMode.FAIL, @@ -616,7 +617,7 @@ void honorsKeyFormatOptionsIndependentlyOfValueOptions() throws Exception { List rows = List.of(insert, delete); JsonRowDataSerializationSchema flinkKey = - new JsonRowDataSerializationSchema( + tech.streamfusion.compat.JsonTestSchemas.json( keyType, TimestampFormat.ISO_8601, JsonFormatOptions.MapNullKeyMode.LITERAL, @@ -625,7 +626,7 @@ void honorsKeyFormatOptionsIndependentlyOfValueOptions() throws Exception { false); flinkKey.open(initializationContext()); JsonRowDataSerializationSchema flinkValue = - new JsonRowDataSerializationSchema( + tech.streamfusion.compat.JsonTestSchemas.json( rowType, TimestampFormat.SQL, JsonFormatOptions.MapNullKeyMode.LITERAL, @@ -685,7 +686,7 @@ void serializesInsertOnlyUpsertBatchesWithoutARowKindColumn() throws Exception { GenericRowData.of(2L, null)); JsonRowDataSerializationSchema flinkKey = - new JsonRowDataSerializationSchema( + tech.streamfusion.compat.JsonTestSchemas.json( RowType.of(new LogicalType[] {new BigIntType()}, new String[] {"id"}), TimestampFormat.SQL, JsonFormatOptions.MapNullKeyMode.FAIL, @@ -694,8 +695,12 @@ void serializesInsertOnlyUpsertBatchesWithoutARowKindColumn() throws Exception { false); flinkKey.open(initializationContext()); JsonRowDataSerializationSchema flinkValue = - new JsonRowDataSerializationSchema( - rowType, TimestampFormat.SQL, JsonFormatOptions.MapNullKeyMode.FAIL, "null", false, + tech.streamfusion.compat.JsonTestSchemas.json( + rowType, + TimestampFormat.SQL, + JsonFormatOptions.MapNullKeyMode.FAIL, + "null", + false, false); flinkValue.open(initializationContext()); @@ -771,7 +776,7 @@ private static void assertMatchesFlink( String mapNullKeyLiteral) throws Exception { JsonRowDataSerializationSchema flink = - new JsonRowDataSerializationSchema( + tech.streamfusion.compat.JsonTestSchemas.json( rowType, timestampFormat, mapNullKeyMode, diff --git a/src/test/java/tech/streamfusion/kafka/NativeKafkaSerializationOperatorTest.java b/src/test/java/tech/streamfusion/kafka/NativeKafkaSerializationOperatorTest.java index e89f77bf0..f523139db 100644 --- a/src/test/java/tech/streamfusion/kafka/NativeKafkaSerializationOperatorTest.java +++ b/src/test/java/tech/streamfusion/kafka/NativeKafkaSerializationOperatorTest.java @@ -5,10 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertNull; import com.sun.net.httpserver.HttpServer; -import tech.streamfusion.format.EncodeFormat; -import tech.streamfusion.operator.ArrowBatch; -import tech.streamfusion.operator.ArrowBatchSerializer; -import tech.streamfusion.operator.RowDataArrowConverter; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -31,6 +27,10 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import tech.streamfusion.format.EncodeFormat; +import tech.streamfusion.operator.ArrowBatch; +import tech.streamfusion.operator.ArrowBatchSerializer; +import tech.streamfusion.operator.RowDataArrowConverter; @Tag("streamfusion-kafka") class NativeKafkaSerializationOperatorTest { @@ -139,9 +139,7 @@ void stockKafkaSchemaPublishesTheBytesUnchanged() { assertArrayEquals(key, record.key()); assertNull(record.timestamp()); assertArrayEquals(value, record.value()); - assertEquals( - List.of("output"), - schema.getKafkaDatasetFacet().orElseThrow().getTopicIdentifier().getTopics()); + KafkaLineageAssertions.assertTopic(schema, "output"); } @Test diff --git a/src/test/java/tech/streamfusion/kafka/NativeKafkaSinkIntegrationTest.java b/src/test/java/tech/streamfusion/kafka/NativeKafkaSinkIntegrationTest.java index 9a105a2af..ad010d2db 100644 --- a/src/test/java/tech/streamfusion/kafka/NativeKafkaSinkIntegrationTest.java +++ b/src/test/java/tech/streamfusion/kafka/NativeKafkaSinkIntegrationTest.java @@ -3,9 +3,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; -import tech.streamfusion.planner.NativePlanner; -import tech.streamfusion.planner.PhysicalPlanScan; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -26,9 +25,7 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.configuration.CheckpointingOptions; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.ExternalizedCheckpointRetention; import org.apache.flink.configuration.RestartStrategyOptions; -import org.apache.flink.configuration.StateRecoveryOptions; import org.apache.flink.runtime.state.FunctionInitializationContext; import org.apache.flink.runtime.state.FunctionSnapshotContext; import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; @@ -39,9 +36,9 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; +import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TransactionListing; import org.apache.kafka.clients.admin.TransactionState; @@ -59,6 +56,8 @@ import org.junit.jupiter.api.io.TempDir; import org.testcontainers.containers.KafkaContainer; import org.testcontainers.utility.DockerImageName; +import tech.streamfusion.planner.NativePlanner; +import tech.streamfusion.planner.PhysicalPlanScan; /** Broker-level proof that native values remain governed by Flink's exactly-once Kafka writer. */ @Tag("streamfusion-kafka") @@ -299,7 +298,8 @@ void publishesWithNonTransactionalDeliveryGuarantees() throws Exception { } table.createTemporaryView( "src", - environment.fromData( + fromData( + environment, Types.ROW_NAMED(new String[] {"id", "name"}, Types.LONG, Types.STRING), input.toArray(Row[]::new)), Schema.newBuilder() @@ -345,8 +345,7 @@ void publishesUpdatingResultsAsExactlyOnceUpserts() throws Exception { } table.createTemporaryView( "src", - environment.fromData( - Types.ROW_NAMED(new String[] {"id"}, Types.LONG), input), + fromData(environment, Types.ROW_NAMED(new String[] {"id"}, Types.LONG), input), Schema.newBuilder().column("id", DataTypes.BIGINT()).build()); table.executeSql( "CREATE TABLE output (id BIGINT, total BIGINT, PRIMARY KEY (id) NOT ENFORCED) WITH (" @@ -402,7 +401,8 @@ void publishesEveryNativeCsvValueInCommittedTransactions() throws Exception { } table.createTemporaryView( "src", - environment.fromData( + fromData( + environment, Types.ROW_NAMED(new String[] {"id", "name"}, Types.LONG, Types.STRING), input.toArray(Row[]::new)), Schema.newBuilder() @@ -452,7 +452,8 @@ void publishesEveryNativeJsonValueInCommittedTransactions() throws Exception { input.add(Row.of(id, "row-" + id)); } DataStream source = - environment.fromData( + fromData( + environment, Types.ROW_NAMED(new String[] {"id", "name"}, Types.LONG, Types.STRING), input.toArray(Row[]::new)); table.createTemporaryView( @@ -509,15 +510,12 @@ void abortsReplayedTransactionsAcrossFailover() throws Exception { input.add(Row.of(id, "row-" + id)); } DataStream source = - environment - .fromData( - Types.ROW_NAMED( - new String[] {"id", "name"}, Types.LONG, Types.STRING), + fromData( + environment, + Types.ROW_NAMED(new String[] {"id", "name"}, Types.LONG, Types.STRING), input.toArray(Row[]::new)) .map(new CheckpointFailingMap()) - .returns( - Types.ROW_NAMED( - new String[] {"id", "name"}, Types.LONG, Types.STRING)); + .returns(Types.ROW_NAMED(new String[] {"id", "name"}, Types.LONG, Types.STRING)); table.createTemporaryView( "src", source, @@ -567,13 +565,12 @@ void commitsExactlyOnceWhenFailoverStrikesTheCommitPhase() throws Exception { environment.configure(configuration); StreamTableEnvironment table = StreamTableEnvironment.create(environment); DataStream source = - environment - .fromData( + fromData( + environment, Types.ROW_NAMED(new String[] {"id", "name"}, Types.LONG, Types.STRING), rowRange(rows).toArray(Row[]::new)) .map(new CommitPhaseFailingMap()) - .returns( - Types.ROW_NAMED(new String[] {"id", "name"}, Types.LONG, Types.STRING)); + .returns(Types.ROW_NAMED(new String[] {"id", "name"}, Types.LONG, Types.STRING)); table.createTemporaryView( "src", source, @@ -649,11 +646,10 @@ private static void runDownscaleJob( configuration.set(RestartStrategyOptions.RESTART_STRATEGY, "disable"); configuration.set( CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpoints.toUri().toString()); - configuration.set( - CheckpointingOptions.EXTERNALIZED_CHECKPOINT_RETENTION, - ExternalizedCheckpointRetention.RETAIN_ON_CANCELLATION); + tech.streamfusion.compat.CheckpointTestConfig.retainOnCancellation(configuration); if (restoreFrom != null) { - configuration.set(StateRecoveryOptions.SAVEPOINT_PATH, restoreFrom.toUri().toString()); + tech.streamfusion.compat.CheckpointTestConfig.restore( + configuration, restoreFrom.toUri().toString()); } StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment(configuration); @@ -661,8 +657,8 @@ private static void runDownscaleJob( environment.enableCheckpointing(50); StreamTableEnvironment table = StreamTableEnvironment.create(environment); DataStream source = - environment - .fromData( + fromData( + environment, Types.ROW_NAMED(new String[] {"id", "name"}, Types.LONG, Types.STRING), rowRange(rows).toArray(Row[]::new)) .uid("downscale-source") diff --git a/src/test/java/tech/streamfusion/kafka/NativeKafkaSinkSqlPlanTest.java b/src/test/java/tech/streamfusion/kafka/NativeKafkaSinkSqlPlanTest.java index 1ef8cfbc9..20843fb55 100644 --- a/src/test/java/tech/streamfusion/kafka/NativeKafkaSinkSqlPlanTest.java +++ b/src/test/java/tech/streamfusion/kafka/NativeKafkaSinkSqlPlanTest.java @@ -5,8 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import tech.streamfusion.planner.NativePlanner; -import tech.streamfusion.planner.PhysicalPlanScan; import java.util.List; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.ExplainDetail; @@ -14,6 +12,8 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.NativePlanner; +import tech.streamfusion.planner.PhysicalPlanScan; @Tag("streamfusion-kafka") class NativeKafkaSinkSqlPlanTest { @@ -786,7 +786,7 @@ void upsertAvroSinkRoutesNativelyWithItsAggregate() { */ @Test void keepsUnreproducedAvroShapesOnFlink() { - assertAvroFallback("(id BIGINT)", "'format' = 'avro', 'avro.encoding' = 'json'"); + assertAvroFallback("(id BIGINT, tod TIME(0))", "'format' = 'avro'"); assertAvroFallback( "(id BIGINT)", @@ -795,6 +795,14 @@ void keepsUnreproducedAvroShapesOnFlink() { + "'avro-confluent.basic-auth.credentials-source' = 'SASL_INHERIT'"); } + @Test + void jsonAvroEncodingStaysOnFlink() { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.AVRO_ENCODING_OPTION, + "Flink 1.18 Avro has no encoding option"); + assertAvroFallback("(id BIGINT)", "'format' = 'avro', 'avro.encoding' = 'json'"); + } + private static void assertAvroFallback(String columns, String formatOptions) { StreamTableEnvironment table = environment(); table.executeSql( diff --git a/src/test/java/tech/streamfusion/operator/AlignedColumnarExchangeRecoveryTest.java b/src/test/java/tech/streamfusion/operator/AlignedColumnarExchangeRecoveryTest.java index 06c3f1acf..1d3c5614c 100644 --- a/src/test/java/tech/streamfusion/operator/AlignedColumnarExchangeRecoveryTest.java +++ b/src/test/java/tech/streamfusion/operator/AlignedColumnarExchangeRecoveryTest.java @@ -3,7 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import tech.streamfusion.planner.ColumnarKeyGroupPartitioner; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; @@ -12,18 +11,13 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; import org.apache.flink.api.common.functions.RichMapFunction; -import org.apache.flink.api.common.serialization.SimpleStringEncoder; import org.apache.flink.api.common.state.CheckpointListener; import org.apache.flink.configuration.CheckpointingOptions; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.ExternalizedCheckpointRetention; import org.apache.flink.configuration.RestartStrategyOptions; -import org.apache.flink.configuration.StateRecoveryOptions; -import org.apache.flink.connector.file.sink.FileSink; import org.apache.flink.runtime.checkpoint.metadata.CheckpointMetadata; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy; import org.apache.flink.streaming.api.transformations.PartitionTransformation; import org.apache.flink.streaming.api.transformations.StreamExchangeMode; import org.apache.flink.table.data.GenericRowData; @@ -34,8 +28,11 @@ import org.apache.flink.table.types.logical.RowType; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import tech.streamfusion.planner.ColumnarKeyGroupPartitioner; class AlignedColumnarExchangeRecoveryTest { + private static final java.util.Map + OPERATOR_IDS = new java.util.HashMap<>(); private static final int ROWS = 12_000; private static final int MAX_PARALLELISM = 257; @@ -61,8 +58,8 @@ void restoresAfterAlignedExchangeCheckpointAndRescale( metadata.getOperatorStates().stream() .filter( operator -> - operator.getOperatorName().orElse("").contains("key-group-split") - || operator.getOperatorName().orElse("").contains("arrow-to-row")) + operator.getOperatorID().equals(OPERATOR_IDS.get("key-group-split")) + || operator.getOperatorID().equals(OPERATOR_IDS.get("arrow-to-row"))) .flatMap(operator -> operator.getStates().stream()) .noneMatch( state -> @@ -125,8 +122,10 @@ private static void assertUnalignedRestore( metadata.getOperatorStates().stream() .filter( operator -> - operator.getOperatorName().orElse("").contains("key-group-split") - || operator.getOperatorName().orElse("").contains("key-group-reassemble")) + operator.getOperatorID().equals(OPERATOR_IDS.get("key-group-split")) + || operator + .getOperatorID() + .equals(OPERATOR_IDS.get("key-group-reassemble"))) .flatMap(operator -> operator.getStates().stream()) .anyMatch( state -> @@ -149,18 +148,17 @@ private static void runJob( configuration.set(RestartStrategyOptions.RESTART_STRATEGY, "disable"); configuration.set( CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpoints.toUri().toString()); - configuration.set( - CheckpointingOptions.EXTERNALIZED_CHECKPOINT_RETENTION, - ExternalizedCheckpointRetention.RETAIN_ON_CANCELLATION); + tech.streamfusion.compat.CheckpointTestConfig.retainOnCancellation(configuration); if (restoreFrom != null) { - configuration.set(StateRecoveryOptions.SAVEPOINT_PATH, restoreFrom.toUri().toString()); + tech.streamfusion.compat.CheckpointTestConfig.restore( + configuration, restoreFrom.toUri().toString()); } StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(configuration); env.setParallelism(parallelism); env.setMaxParallelism(MAX_PARALLELISM); env.enableCheckpointing(50); - env.getCheckpointConfig().enableUnalignedCheckpoints(); + tech.streamfusion.compat.CheckpointTestCapabilities.configure(env, recoverable); DataStream rows = env.fromSequence(0, ROWS - 1) @@ -215,18 +213,25 @@ private static void runJob( .uid("aligned-arrow-to-row") .setMaxParallelism(MAX_PARALLELISM); - FileSink sink = - FileSink.forRowFormat( - new org.apache.flink.core.fs.Path(output.toUri()), - new SimpleStringEncoder("UTF-8")) - .withRollingPolicy(OnCheckpointRollingPolicy.build()) - .build(); - restoredRows - .map(new SlowCheckpointFailingMap()) - .uid("aligned-failing-map") - .sinkTo(sink) - .uid("aligned-file-sink"); - env.execute("aligned-columnar-exchange-recovery"); + tech.streamfusion.compat.CheckpointFileSink.attach( + restoredRows.map(new SlowCheckpointFailingMap()).uid("aligned-failing-map"), + output, + "aligned-file-sink"); + var graph = env.getStreamGraph(); + var hashes = + new org.apache.flink.streaming.api.graph.StreamGraphHasherV2() + .traverseStreamGraphAndGenerateHashes(graph); + OPERATOR_IDS.clear(); + for (var node : graph.getStreamNodes()) { + for (String name : new String[] {"key-group-split", "key-group-reassemble", "arrow-to-row"}) { + if (node.getOperatorName().equals(name)) + OPERATOR_IDS.put( + name, new org.apache.flink.runtime.jobgraph.OperatorID(hashes.get(node.getId()))); + } + } + assertTrue(OPERATOR_IDS.containsKey("key-group-split")); + assertTrue(OPERATOR_IDS.containsKey(recoverable ? "key-group-reassemble" : "arrow-to-row")); + env.execute(graph); } private static void assertExactlyOnceOutput(Path output, String message) throws Exception { diff --git a/src/test/java/tech/streamfusion/operator/AvroDecodeParityTest.java b/src/test/java/tech/streamfusion/operator/AvroDecodeParityTest.java index 2b510586e..8e61ebaf3 100644 --- a/src/test/java/tech/streamfusion/operator/AvroDecodeParityTest.java +++ b/src/test/java/tech/streamfusion/operator/AvroDecodeParityTest.java @@ -1,6 +1,5 @@ package tech.streamfusion.operator; -import tech.streamfusion.format.avro.AvroFormatProvider; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.Arrays; @@ -15,7 +14,6 @@ import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.EncoderFactory; import org.apache.flink.api.common.serialization.DeserializationSchema; -import org.apache.flink.formats.avro.AvroFormatOptions.AvroEncoding; import org.apache.flink.formats.avro.AvroRowDataDeserializationSchema; import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; import org.apache.flink.metrics.MetricGroup; @@ -45,6 +43,7 @@ import org.apache.flink.util.UserCodeClassLoader; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import tech.streamfusion.format.avro.AvroFormatProvider; /** * Pins the native bare-Avro decode to Flink's own {@link AvroRowDataDeserializationSchema}, @@ -204,7 +203,10 @@ void correctedTimestampMappingMatchesFlink() throws Exception { }, new String[] {"ts3", "ts6", "ltz3", "ltz6"}) .copy(false); - Schema writer = AvroSchemaConverter.convertToSchema(corrected, false); + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.format.avro.compat.AvroTestSchemas.CORRECTED_TIMESTAMP_MAPPING, + "Corrected Avro timestamp mapping is absent from Flink 1.18"); + Schema writer = tech.streamfusion.format.avro.compat.AvroCompat.schema(corrected, false); Map options = Map.of("format", "avro", "avro.timestamp_mapping.legacy", "false"); Consumer filler = @@ -290,8 +292,8 @@ private static List> flinkDecode( DecodeParityHarness harness, RowType rowType, byte[] message, boolean legacyTimestampMapping) throws Exception { AvroRowDataDeserializationSchema schema = - new AvroRowDataDeserializationSchema( - rowType, InternalTypeInfo.of(rowType), AvroEncoding.BINARY, legacyTimestampMapping); + tech.streamfusion.format.avro.compat.AvroTestSchemas.decoder( + rowType, InternalTypeInfo.of(rowType), legacyTimestampMapping); schema.open( new DeserializationSchema.InitializationContext() { @Override diff --git a/src/test/java/tech/streamfusion/operator/ColumnarRecordMetricsTest.java b/src/test/java/tech/streamfusion/operator/ColumnarRecordMetricsTest.java index b2c669c30..3cac68f29 100644 --- a/src/test/java/tech/streamfusion/operator/ColumnarRecordMetricsTest.java +++ b/src/test/java/tech/streamfusion/operator/ColumnarRecordMetricsTest.java @@ -8,12 +8,9 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.flink.metrics.groups.OperatorMetricGroup; -import org.apache.flink.runtime.event.WatermarkEvent; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; -import org.apache.flink.streaming.api.operators.Output; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; -import org.apache.flink.streaming.runtime.streamrecord.RecordAttributes; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; import org.apache.flink.table.data.GenericRowData; @@ -49,7 +46,8 @@ private static ArrowBatch batchOf(BufferAllocator allocator, int rows) { } /** Stands in for the runtime's own counting output, which charges exactly one per collect. */ - private static final class CountingCollector implements Output> { + private static final class CountingCollector + extends tech.streamfusion.compat.TestOutput> { private final OperatorMetricGroup metrics; private final List collected = new ArrayList<>(); @@ -75,12 +73,6 @@ public void emitWatermarkStatus(WatermarkStatus status) {} @Override public void emitLatencyMarker(LatencyMarker marker) {} - @Override - public void emitWatermark(WatermarkEvent watermark) {} - - @Override - public void emitRecordAttributes(RecordAttributes attributes) {} - @Override public void close() {} } @@ -94,7 +86,9 @@ void emittingABatchChargesItsRowsRatherThanOneRecord() { ColumnarRecordMetrics.emit(output, metrics, batch); assertEquals( - 4096, metrics.getIOMetricGroup().getNumRecordsOutCounter().getCount(), "rows, not batches"); + 4096, + metrics.getIOMetricGroup().getNumRecordsOutCounter().getCount(), + "rows, not batches"); assertEquals(1, output.collected.size(), "the batch is emitted exactly once"); try (VectorSchemaRoot root = output.collected.get(0).root()) { assertEquals(4096, root.getRowCount(), "the consumer still gets to take the batch"); diff --git a/src/test/java/tech/streamfusion/operator/FlinkFirstBoundTopNOperatorTest.java b/src/test/java/tech/streamfusion/operator/FlinkFirstBoundTopNOperatorTest.java index ba87504b4..c73b065f4 100644 --- a/src/test/java/tech/streamfusion/operator/FlinkFirstBoundTopNOperatorTest.java +++ b/src/test/java/tech/streamfusion/operator/FlinkFirstBoundTopNOperatorTest.java @@ -384,7 +384,6 @@ public InternalTypeInfo getProducedType() { return InternalTypeInfo.ofFields(new BigIntType()); } - @Override public RowDataKeySelector copy() { return new LongKey(index); } diff --git a/src/test/java/tech/streamfusion/operator/FlinkUpdateFastFirstBoundOperatorTest.java b/src/test/java/tech/streamfusion/operator/FlinkUpdateFastFirstBoundOperatorTest.java index 9390c2cfb..24de967c3 100644 --- a/src/test/java/tech/streamfusion/operator/FlinkUpdateFastFirstBoundOperatorTest.java +++ b/src/test/java/tech/streamfusion/operator/FlinkUpdateFastFirstBoundOperatorTest.java @@ -413,7 +413,6 @@ public InternalTypeInfo getProducedType() { return InternalTypeInfo.ofFields(new BigIntType()); } - @Override public RowDataKeySelector copy() { return new LongKey(index); } diff --git a/src/test/java/tech/streamfusion/operator/NativeAsyncLookupJoinOperatorTest.java b/src/test/java/tech/streamfusion/operator/NativeAsyncLookupJoinOperatorTest.java index 4cfc264e4..1fc4314a9 100644 --- a/src/test/java/tech/streamfusion/operator/NativeAsyncLookupJoinOperatorTest.java +++ b/src/test/java/tech/streamfusion/operator/NativeAsyncLookupJoinOperatorTest.java @@ -21,9 +21,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.api.common.functions.OpenContext; import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.apache.flink.table.data.GenericRowData; @@ -32,6 +30,7 @@ import org.apache.flink.table.types.logical.RowType; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import tech.streamfusion.compat.RichAsyncFunction; @Timeout(10) class NativeAsyncLookupJoinOperatorTest { @@ -250,7 +249,7 @@ private static final class ControlledLookup extends RichAsyncFunction writer( partitionColumns, new String[0], new String[0]); - StreamingFileSink.BucketsBuilder< - PartitionedArrowBatch, - String, - ? extends StreamingFileSink.BucketsBuilder> - buckets = - StreamingFileSink.forBulkFormat(new Path(directory.toUri()), factory) - .withBucketAssigner(new PartitionedBatchBucketAssigner()) - .withRollingPolicy( - new NativeFileRollingPolicy(128 << 20, Long.MAX_VALUE, Long.MAX_VALUE)) - .withOutputFileConfig( - OutputFileConfig.builder().withPartPrefix("part-test").build()); - return new StreamingFileWriter<>(1000, buckets, partitionKeys, new Configuration()); + return tech.streamfusion.compat.FileWriterTestFactory.writer(directory, factory, partitionKeys); } /** Committed (visible) files under the directory, recursively; in-progress files are hidden. */ diff --git a/src/test/java/tech/streamfusion/operator/NativeGroupedValueStateTest.java b/src/test/java/tech/streamfusion/operator/NativeGroupedValueStateTest.java index 45f5b95e7..04e88eacb 100644 --- a/src/test/java/tech/streamfusion/operator/NativeGroupedValueStateTest.java +++ b/src/test/java/tech/streamfusion/operator/NativeGroupedValueStateTest.java @@ -15,7 +15,6 @@ import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness; -import org.apache.flink.table.api.TableRuntimeException; import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.BigIntType; @@ -150,7 +149,8 @@ void singleValueRestoresNullCardinalityAndCanRetractBeforeReplacement(boolean ro List.of(Arrays.asList(RowKind.DELETE, 1L, null), Arrays.asList(RowKind.INSERT, 1L, 7L)), drain(restored, output)); assertThrows( - TableRuntimeException.class, () -> push(restored, allocator, row(RowKind.INSERT, null))); + tech.streamfusion.compat.TableErrors.exceptionType(), + () -> push(restored, allocator, row(RowKind.INSERT, null))); } try (var allocator = new RootAllocator(); var restored = harness(rocks, 14, -1, 0)) { @@ -158,7 +158,8 @@ void singleValueRestoresNullCardinalityAndCanRetractBeforeReplacement(boolean ro restored.initializeState(snapshot); restored.open(); assertThrows( - TableRuntimeException.class, () -> push(restored, allocator, row(RowKind.INSERT, null))); + tech.streamfusion.compat.TableErrors.exceptionType(), + () -> push(restored, allocator, row(RowKind.INSERT, null))); } } diff --git a/src/test/java/tech/streamfusion/operator/NativeSourceWatermarksTest.java b/src/test/java/tech/streamfusion/operator/NativeSourceWatermarksTest.java index 5e2f423de..fe77dbb8a 100644 --- a/src/test/java/tech/streamfusion/operator/NativeSourceWatermarksTest.java +++ b/src/test/java/tech/streamfusion/operator/NativeSourceWatermarksTest.java @@ -180,8 +180,8 @@ void sourceCallbackUsesCalendarCandidateAfterDownstreamClosesTheArrowRoot() { void splitMinimumAndIdlenessStillUseFlinksCoordination() { CapturingOutput output = new CapturingOutput(); var multiplexer = new WatermarkOutputMultiplexer(output); - multiplexer.registerNewOutput("a"); - multiplexer.registerNewOutput("b"); + tech.streamfusion.compat.WatermarkTestOutputs.register(multiplexer, "a"); + tech.streamfusion.compat.WatermarkTestOutputs.register(multiplexer, "b"); var first = NativeSourceWatermarks.strategy(0).createWatermarkGenerator(null); var second = NativeSourceWatermarks.strategy(0).createWatermarkGenerator(null); try (BufferAllocator allocator = new RootAllocator()) { diff --git a/src/test/java/tech/streamfusion/operator/NativeWindowTableFunctionOperatorTest.java b/src/test/java/tech/streamfusion/operator/NativeWindowTableFunctionOperatorTest.java index c7aacc04f..b6bc0b454 100644 --- a/src/test/java/tech/streamfusion/operator/NativeWindowTableFunctionOperatorTest.java +++ b/src/test/java/tech/streamfusion/operator/NativeWindowTableFunctionOperatorTest.java @@ -3,7 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import java.time.Duration; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -24,10 +23,6 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.TimestampData; import org.apache.flink.table.runtime.operators.window.TimeWindow; -import org.apache.flink.table.runtime.operators.window.groupwindow.assigners.CumulativeWindowAssigner; -import org.apache.flink.table.runtime.operators.window.groupwindow.assigners.GroupWindowAssigner; -import org.apache.flink.table.runtime.operators.window.groupwindow.assigners.SlidingWindowAssigner; -import org.apache.flink.table.runtime.operators.window.groupwindow.assigners.TumblingWindowAssigner; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -103,12 +98,6 @@ void cumulativeFansEachRowIntoEveryNestedWindow() throws Exception { void timestampLayoutsMatchFlinkWindowAssignment(Field field) throws Exception { for (int kind = 0; kind < 3; kind++) { long size = kind == 0 ? 1000 : 2000; - GroupWindowAssigner flink = - kind == 0 - ? TumblingWindowAssigner.of(Duration.ofMillis(size)) - : kind == 1 - ? SlidingWindowAssigner.of(Duration.ofMillis(size), Duration.ofMillis(1000)) - : CumulativeWindowAssigner.of(Duration.ofMillis(size), Duration.ofMillis(1000)); try (BufferAllocator allocator = new RootAllocator(); OneInputStreamOperatorTestHarness harness = new OneInputStreamOperatorTestHarness<>( @@ -142,7 +131,9 @@ void timestampLayoutsMatchFlinkWindowAssignment(Field field) throws Exception { List> expected = new ArrayList<>(); TimestampAccessor timestamps = new TimestampAccessor(time); for (int i = 0; i < values.length - 1; i++) { - for (TimeWindow window : flink.assignWindows(null, timestamps.getMillis(i))) { + for (TimeWindow window : + tech.streamfusion.compat.WindowTestAssigner.assign( + kind, size, timestamps.getMillis(i))) { expected.add( List.of( timestamps.getMillis(i), diff --git a/src/test/java/tech/streamfusion/operator/SplitByKeyGroupOperatorTest.java b/src/test/java/tech/streamfusion/operator/SplitByKeyGroupOperatorTest.java index 27e4e4801..5113d7ab2 100644 --- a/src/test/java/tech/streamfusion/operator/SplitByKeyGroupOperatorTest.java +++ b/src/test/java/tech/streamfusion/operator/SplitByKeyGroupOperatorTest.java @@ -1,10 +1,9 @@ package tech.streamfusion.operator; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static tech.streamfusion.compat.FlinkTestSources.fromData; -import tech.streamfusion.planner.ColumnarKeyGroupPartitioner; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -13,34 +12,33 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.flink.runtime.io.network.api.writer.SubtaskStateMapper; import org.apache.flink.runtime.plugable.SerializationDelegate; import org.apache.flink.runtime.state.KeyGroupRangeAssignment; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink; import org.apache.flink.streaming.api.graph.StreamEdge; +import org.apache.flink.streaming.api.transformations.PartitionTransformation; +import org.apache.flink.streaming.api.transformations.StreamExchangeMode; import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; -import org.apache.flink.streaming.api.transformations.PartitionTransformation; -import org.apache.flink.streaming.api.transformations.StreamExchangeMode; -import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.GenericArrayData; import org.apache.flink.table.data.GenericMapData; +import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; import org.apache.flink.table.data.TimestampData; import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LogicalType; -import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.MapType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.TimestampType; import org.apache.flink.table.types.logical.VarCharType; import org.junit.jupiter.api.Test; +import tech.streamfusion.planner.ColumnarKeyGroupPartitioner; class SplitByKeyGroupOperatorTest { @@ -182,6 +180,11 @@ void partitionerReroutesAKeyGroupAtRestoredParallelism() { KeyGroupRangeAssignment.computeOperatorIndexForKeyGroup(maxParallelism, 4, keyGroup), p.selectChannel(delegate)); + assertEquals( + p.selectChannel(delegate), + partitioner.copy().selectChannel(delegate), + "Flink 1.18's recovery filter uses the configured copy without another setup call"); + // Recovery configures the copied partitioner for the restored topology and filters/reroutes // this whole record without inspecting its Arrow rows. ColumnarKeyGroupPartitioner restored = @@ -196,55 +199,13 @@ void partitionerReroutesAKeyGroupAtRestoredParallelism() { assertEquals(0, p.selectChannel(delegate), "an unrouted batch goes to channel 0"); } - @Test - void partitionerUsesKeyGroupRangeRecoveryAndRequiresAlignedChannelState() { - ColumnarKeyGroupPartitioner partitioner = new ColumnarKeyGroupPartitioner(128); - assertEquals(SubtaskStateMapper.RANGE, partitioner.getDownstreamSubtaskStateMapper()); - assertFalse( - partitioner.isSupportsUnalignedCheckpoint(), - "a channel batch can span key groups that separate after rescaling"); - } - - @Test - void recoverablePartitionerSupportsUnalignedRangeChannelState() { - ColumnarKeyGroupPartitioner partitioner = new ColumnarKeyGroupPartitioner(128, true); - assertEquals(SubtaskStateMapper.RANGE, partitioner.getDownstreamSubtaskStateMapper()); - assertTrue(partitioner.isSupportsUnalignedCheckpoint()); - assertTrue(partitioner.copy().isSupportsUnalignedCheckpoint()); - } - - @Test - void streamGraphForcesAlignedColumnarExchange() { - StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.enableCheckpointing(10); - env.getCheckpointConfig().enableUnalignedCheckpoints(); - DataStream input = - env.fromData(1) - .map(ignored -> (ArrowBatch) null) - .returns(ArrowBatchTypeInformation.INSTANCE); - PartitionTransformation partition = - new PartitionTransformation<>( - input.getTransformation(), - new ColumnarKeyGroupPartitioner(128), - StreamExchangeMode.PIPELINED); - new DataStream<>(env, partition).sinkTo(new DiscardingSink<>()); - - List columnarEdges = - env.getStreamGraph().getStreamNodes().stream() - .flatMap(node -> node.getOutEdges().stream()) - .filter(edge -> edge.getPartitioner() instanceof ColumnarKeyGroupPartitioner) - .toList(); - assertEquals(1, columnarEdges.size()); - assertFalse(columnarEdges.get(0).supportsUnalignedCheckpoints()); - } - @Test void streamGraphAllowsRecoverableColumnarExchangeChannelState() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.enableCheckpointing(10); env.getCheckpointConfig().enableUnalignedCheckpoints(); DataStream input = - env.fromData(1) + fromData(env, 1) .map(ignored -> (ArrowBatch) null) .returns(ArrowBatchTypeInformation.INSTANCE); PartitionTransformation partition = @@ -252,7 +213,7 @@ void streamGraphAllowsRecoverableColumnarExchangeChannelState() { input.getTransformation(), new ColumnarKeyGroupPartitioner(128, true), StreamExchangeMode.PIPELINED); - new DataStream<>(env, partition).sinkTo(new DiscardingSink<>()); + tech.streamfusion.compat.StreamTestSinks.discard(new DataStream<>(env, partition)); StreamEdge edge = env.getStreamGraph().getStreamNodes().stream() .flatMap(node -> node.getOutEdges().stream()) diff --git a/src/test/java/tech/streamfusion/planner/FlinkKeyGroupUtilsTest.java b/src/test/java/tech/streamfusion/planner/FlinkKeyGroupUtilsTest.java index edb26ff4d..f336db737 100644 --- a/src/test/java/tech/streamfusion/planner/FlinkKeyGroupUtilsTest.java +++ b/src/test/java/tech/streamfusion/planner/FlinkKeyGroupUtilsTest.java @@ -2,8 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static tech.streamfusion.compat.FlinkTestSources.fromData; -import tech.streamfusion.operator.NativeColumnarGroupAggregateOperator; import java.util.List; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.runtime.state.KeyGroupRangeAssignment; @@ -13,6 +13,7 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import org.junit.jupiter.api.Test; +import tech.streamfusion.operator.NativeColumnarGroupAggregateOperator; class FlinkKeyGroupUtilsTest { @@ -42,7 +43,8 @@ void nativeSqlPlanKeepsConfiguredMaxParallelism() { StreamTableEnvironment table = StreamTableEnvironment.create(env); table.createTemporaryView( "src", - env.fromData( + fromData( + env, Types.ROW_NAMED(new String[] {"k", "v"}, Types.LONG, Types.LONG), Row.of(1L, 1L), Row.of(1L, 2L), @@ -54,8 +56,9 @@ void nativeSqlPlanKeepsConfiguredMaxParallelism() { NativePlanner.install(table); table.toChangelogStream(table.sqlQuery("SELECT k, SUM(v) FROM src GROUP BY k")); + var graph = env.getStreamGraph(); List nativeKeyedNodes = - env.getStreamGraph().getStreamNodes().stream() + graph.getStreamNodes().stream() .filter(node -> node.getOperatorFactory() != null) .filter( node -> @@ -64,8 +67,21 @@ void nativeSqlPlanKeepsConfiguredMaxParallelism() { .equals(NativeColumnarGroupAggregateOperator.class)) .toList(); assertFalse(nativeKeyedNodes.isEmpty(), "group aggregate was not planned natively"); + var hashes = + new org.apache.flink.streaming.api.graph.StreamGraphHasherV2() + .traverseStreamGraphAndGenerateHashes(graph); + var job = graph.getJobGraph(); for (var node : nativeKeyedNodes) { - assertEquals(257, node.getMaxParallelism(), node.getOperatorName()); + var id = new org.apache.flink.runtime.jobgraph.OperatorID(hashes.get(node.getId())); + var vertex = + java.util.stream.StreamSupport.stream(job.getVertices().spliterator(), false) + .filter( + candidate -> + candidate.getOperatorIDs().stream() + .anyMatch(pair -> pair.getGeneratedOperatorID().equals(id))) + .findFirst() + .orElseThrow(); + assertEquals(257, vertex.getMaxParallelism(), node.getOperatorName()); } } } diff --git a/src/test/java/tech/streamfusion/planner/JsonPathSpecTest.java b/src/test/java/tech/streamfusion/planner/JsonPathSpecTest.java index 8b72eb789..a11f168a6 100644 --- a/src/test/java/tech/streamfusion/planner/JsonPathSpecTest.java +++ b/src/test/java/tech/streamfusion/planner/JsonPathSpecTest.java @@ -224,6 +224,10 @@ public Charset getDefaultCharset() { rex.makeCall( output, function, List.of(rex.makeInputRef(text, 0), rex.makeLiteral(path))); var encoded = RexExpression.encodeProjections(List.of(literalPath), List.of("v")); + if (!tech.streamfusion.operator.NativeJsonRuntime.available()) { + assertNull(encoded); + continue; + } assertNotNull(encoded); var binding = encoded.udfBinding(); long[] constants = encoded.longs(); diff --git a/src/test/java/tech/streamfusion/planner/KafkaSinkTranslatorTest.java b/src/test/java/tech/streamfusion/planner/KafkaSinkTranslatorTest.java index 02b4f3017..b381a6ea1 100644 --- a/src/test/java/tech/streamfusion/planner/KafkaSinkTranslatorTest.java +++ b/src/test/java/tech/streamfusion/planner/KafkaSinkTranslatorTest.java @@ -6,14 +6,13 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import tech.streamfusion.format.EncodeFormat; import java.util.Map; import org.apache.flink.connector.base.DeliveryGuarantee; -import org.apache.flink.connector.kafka.sink.TransactionNamingStrategy; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.RowType; import org.junit.jupiter.api.Test; +import tech.streamfusion.format.EncodeFormat; class KafkaSinkTranslatorTest { @@ -86,7 +85,9 @@ void resolvesAvroFormatInstancesThroughTheirProviders() { // Legacy mapping (the default) cannot derive TIMESTAMP_LTZ; the corrected mapping can. RowType ltz = RowType.of(false, new LocalZonedTimestampType(3)); assertNull(EncodeFormat.of("avro", Map.of(), ltz)); - assertNotNull(EncodeFormat.of("avro", Map.of("timestamp_mapping.legacy", "false"), ltz)); + assertEquals( + tech.streamfusion.compat.FlinkTestCapabilities.CORRECTED_AVRO_TIMESTAMPS, + EncodeFormat.of("avro", Map.of("timestamp_mapping.legacy", "false"), ltz) != null); Map confluent = Map.of("url", "http://registry:8081", "schema-registry.subject", "t-value"); @@ -172,6 +173,9 @@ void requiresAStableTransactionalPrefixForExactlyOnce() { @Test void leavesPoolingTransactionNamingToFlinksKafkaSink() { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.KAFKA_TRANSACTION_NAMING, + "Transaction naming strategies are absent from the 1.18 Kafka connector"); KafkaSinkTranslator.Result result = KafkaSinkTranslator.translate( Map.of( @@ -182,7 +186,7 @@ void leavesPoolingTransactionNamingToFlinksKafkaSink() { "sink.transactional-id-prefix", "orders", "sink.transaction-naming-strategy", "pooling")); assertNull(result.fallbackReason); - assertEquals(TransactionNamingStrategy.POOLING, result.planned().transactionNamingStrategy); + assertEquals("POOLING", result.planned().transactionNamingStrategy); } @Test diff --git a/src/test/java/tech/streamfusion/planner/KeyedDecodeOperatorTest.java b/src/test/java/tech/streamfusion/planner/KeyedDecodeOperatorTest.java index 8d4aeda2e..3d9883995 100644 --- a/src/test/java/tech/streamfusion/planner/KeyedDecodeOperatorTest.java +++ b/src/test/java/tech/streamfusion/planner/KeyedDecodeOperatorTest.java @@ -4,12 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import tech.streamfusion.format.NativeFormatContext; -import tech.streamfusion.format.json.JsonFormatProvider; -import tech.streamfusion.operator.ArrowBatch; -import tech.streamfusion.operator.ArrowBatchSerializer; -import tech.streamfusion.operator.NativeBytesDecodeOperator; -import tech.streamfusion.operator.RowDataArrowConverter; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -25,6 +19,12 @@ import org.apache.flink.table.types.logical.VarCharType; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import tech.streamfusion.format.NativeFormatContext; +import tech.streamfusion.format.json.JsonFormatProvider; +import tech.streamfusion.operator.ArrowBatch; +import tech.streamfusion.operator.ArrowBatchSerializer; +import tech.streamfusion.operator.NativeBytesDecodeOperator; +import tech.streamfusion.operator.RowDataArrowConverter; /** * The keyed composition through the real operator and native JSON decoder (so it runs in the @@ -65,7 +65,7 @@ void framesDecodeThroughTheKeyedOperator() throws Exception { List frames = List.of( frame(7L, "{\"id\": 1, \"name\": \"a\"}"), - // A top-level array fans out into two rows sharing record 1's key. + // Current Flink fans out this array; 1.18 rejects it and drops the whole record. frame(8L, "[{\"id\": 2, \"name\": \"b\"}, {\"id\": 3, \"name\": \"c\"}]"), // A null Kafka key keeps the record with a NULL key column. NativeBytesDecodeOperator.frame( @@ -102,11 +102,13 @@ void framesDecodeThroughTheKeyedOperator() throws Exception { } assertEquals( - List.of( - List.of("1", "a", "7"), - List.of("2", "b", "8"), - List.of("3", "c", "8"), - List.of("4", "d", "null")), + tech.streamfusion.compat.JsonRuntimeCompat.ACCEPTS_ARRAY_ROOTS + ? List.of( + List.of("1", "a", "7"), + List.of("2", "b", "8"), + List.of("3", "c", "8"), + List.of("4", "d", "null")) + : List.of(List.of("1", "a", "7"), List.of("4", "d", "null")), rows); } diff --git a/src/test/java/tech/streamfusion/state/RocksDBNativeSnapshotStrategyTest.java b/src/test/java/tech/streamfusion/state/RocksDBNativeSnapshotStrategyTest.java index adb1027b5..fac6eab18 100644 --- a/src/test/java/tech/streamfusion/state/RocksDBNativeSnapshotStrategyTest.java +++ b/src/test/java/tech/streamfusion/state/RocksDBNativeSnapshotStrategyTest.java @@ -40,7 +40,9 @@ void failedNativeSnapshotDeletesItsPartialDirectory() throws Exception { failNativeSnapshot = false; IncrementalRemoteKeyedStateHandle next = snapshot(strategy, storage, 2); assertEquals( - "token", RocksDBNativeSnapshotStrategy.readMetaDocument(next.getMetaDataStateHandle())); + "token", + RocksDBNativeSnapshotStrategy.readMetaDocument( + tech.streamfusion.compat.StateCompat.metaHandle(next))); next.discardState(); assertEquals(0, storage.files()); } @@ -85,6 +87,9 @@ void abortedCheckpointCannotBecomeAReuseBase() throws Exception { @Test void failedReuseNotificationCleansNewFilesWithoutDiscardingConfirmedState() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + tech.streamfusion.compat.FlinkTestCapabilities.CHECKPOINT_REUSE_NOTIFICATION, + "Flink 1.18 has no checkpoint-storage reuse callback"); RocksDBNativeSnapshotStrategy strategy = strategy(); TrackingFactory confirmedStorage = new TrackingFactory(directory.resolve("confirmed")); IncrementalRemoteKeyedStateHandle confirmed = snapshot(strategy, confirmedStorage, 1); @@ -251,7 +256,6 @@ public List duplicate( throw new UnsupportedOperationException(); } - @Override public void reusePreviousStateHandle(Collection handles) { if (failReuse) throw new IllegalStateException("reuse notification failed"); } diff --git a/streamfusion-avro-confluent-registry/pom.xml b/streamfusion-avro-confluent-registry/pom.xml index 379aa68d4..710662a6a 100644 --- a/streamfusion-avro-confluent-registry/pom.xml +++ b/streamfusion-avro-confluent-registry/pom.xml @@ -2,17 +2,22 @@ 4.0.0 tech.streamfusionStreamFusion${revision} - streamfusion-avro-confluent-registry + streamfusion-avro-confluent-registry${streamfusion.artifact.suffix} StreamFusion Avro Confluent Registry -p streamfusion-avro true - tech.streamfusionstreamfusion-core${project.version}provided - tech.streamfusionstreamfusion-avro${project.version}provided + tech.streamfusionstreamfusion-core${streamfusion.artifact.suffix}${project.version}provided + tech.streamfusionstreamfusion-avro${streamfusion.artifact.suffix}${project.version}provided ${project.basedir}/../src/main/java${project.basedir}/src/main/resources + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-avro/pom.xml b/streamfusion-avro/pom.xml index 97eb562b0..13a1a6c85 100644 --- a/streamfusion-avro/pom.xml +++ b/streamfusion-avro/pom.xml @@ -2,15 +2,20 @@ 4.0.0 tech.streamfusionStreamFusion${revision} - streamfusion-avro + streamfusion-avro${streamfusion.artifact.suffix} StreamFusion Avro -p streamfusion-avro streamfusion_avro tech/streamfusion/native/avro - tech.streamfusionstreamfusion-core${project.version}provided + tech.streamfusionstreamfusion-core${streamfusion.artifact.suffix}${project.version}provided ${project.basedir}/../src/main/java${project.basedir}/src/main/resources + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-core/pom.xml b/streamfusion-core/pom.xml index adb5c4164..badfec109 100644 --- a/streamfusion-core/pom.xml +++ b/streamfusion-core/pom.xml @@ -10,12 +10,17 @@ ${revision} - streamfusion-core + streamfusion-core${streamfusion.artifact.suffix} StreamFusion Core ${project.basedir}/../src/main/java + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-csv/pom.xml b/streamfusion-csv/pom.xml index 39f78acf3..6601b6d67 100644 --- a/streamfusion-csv/pom.xml +++ b/streamfusion-csv/pom.xml @@ -2,15 +2,20 @@ 4.0.0 tech.streamfusionStreamFusion${revision} - streamfusion-csv + streamfusion-csv${streamfusion.artifact.suffix} StreamFusion CSV -p streamfusion-csv streamfusion_csv tech/streamfusion/native/csv - tech.streamfusionstreamfusion-core${project.version}provided + tech.streamfusionstreamfusion-core${streamfusion.artifact.suffix}${project.version}provided ${project.basedir}/../src/main/java${project.basedir}/src/main/resources + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-delta/pom.xml b/streamfusion-delta/pom.xml index 056d4b68f..4077ce994 100644 --- a/streamfusion-delta/pom.xml +++ b/streamfusion-delta/pom.xml @@ -8,7 +8,7 @@ StreamFusion ${revision} - streamfusion-delta + streamfusion-delta${streamfusion.artifact.suffix} StreamFusion Delta -p streamfusion -p streamfusion-parquet @@ -17,32 +17,32 @@ tech.streamfusion - streamfusion-runtime + streamfusion-runtime${streamfusion.artifact.suffix} ${project.version} test-jar test tech.streamfusion - streamfusion-kafka + streamfusion-kafka${streamfusion.artifact.suffix} ${project.version} test tech.streamfusion - streamfusion-json + streamfusion-json${streamfusion.artifact.suffix} ${project.version} test tech.streamfusion - streamfusion-core + streamfusion-core${streamfusion.artifact.suffix} ${project.version} provided tech.streamfusion - streamfusion-parquet + streamfusion-parquet${streamfusion.artifact.suffix} ${project.version} provided @@ -85,6 +85,11 @@ ${project.basedir}/src/main/resources + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-image-it/pom.xml b/streamfusion-image-it/pom.xml index 5d305b927..e18504480 100644 --- a/streamfusion-image-it/pom.xml +++ b/streamfusion-image-it/pom.xml @@ -10,7 +10,11 @@ ${revision} - streamfusion-image-it + streamfusion-image-it${streamfusion.artifact.suffix} + + + true + @@ -28,6 +32,19 @@ + + org.codehaus.mojo + build-helper-maven-plugin + + flink-compatibility-sources + none + + + + org.codehaus.mojo + flatten-maven-plugin + ossrh + @@ -48,6 +65,8 @@ ${project.build.directory}/${project.build.finalName}.jar ${project.basedir}/.. ${project.version} + ${flink.line} + ${streamfusion.artifact.suffix} diff --git a/streamfusion-image-it/src/test/java/tech/streamfusion/imageit/FlinkImageIT.java b/streamfusion-image-it/src/test/java/tech/streamfusion/imageit/FlinkImageIT.java index ffa885240..9f85ee323 100644 --- a/streamfusion-image-it/src/test/java/tech/streamfusion/imageit/FlinkImageIT.java +++ b/streamfusion-image-it/src/test/java/tech/streamfusion/imageit/FlinkImageIT.java @@ -10,12 +10,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.util.jar.JarFile; import org.junit.jupiter.api.Test; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.utility.MountableFile; import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; /** Runs a normal Session-cluster submission against the locally built Flink image. */ class FlinkImageIT { @@ -27,13 +28,26 @@ class FlinkImageIT { + "jobmanager.memory.process.size: 1024m\n" + "taskmanager.memory.process.size: 2048m\n" + "taskmanager.memory.task.off-heap.size: 256m\n" - + "state.backend.type: tech.streamfusion.state.RocksDBNativeStateBackendFactory\n" + + ("1.18".equals(System.getProperty("streamfusion.flink.line")) + ? "state.backend: " + : "state.backend.type: ") + + "tech.streamfusion.state.RocksDBNativeStateBackendFactory\n" + "taskmanager.numberOfTaskSlots: 1"; @Test void baseImageRunsAUserSqlJobThroughTheNativePlannerAndRuntime() throws Exception { Path jobJar = Path.of(requiredProperty("streamfusion.image.job.jar")); assertTrue(Files.isRegularFile(jobJar), "missing user job JAR: " + jobJar); + try (JarFile jar = new JarFile(jobJar.toFile())) { + assertTrue( + jar.stream() + .noneMatch( + entry -> + entry.getName().startsWith("tech/streamfusion/native/") + || (entry.getName().endsWith(".class") + && !entry.getName().startsWith("tech/streamfusion/imageit/"))), + "Image validation requires a thin user job without embedded engine classes or libraries"); + } DockerImageName image = DockerImageName.parse(requiredProperty("streamfusion.image.name")); try (Network network = Network.newNetwork(); @@ -100,7 +114,8 @@ private static void awaitRegisteredTaskManager(GenericContainer jobManager) t try { HttpResponse response = client.send( - HttpRequest.newBuilder(taskManagers).GET().build(), HttpResponse.BodyHandlers.ofString()); + HttpRequest.newBuilder(taskManagers).GET().build(), + HttpResponse.BodyHandlers.ofString()); lastResponse = response.body(); if (response.statusCode() == 200 && !lastResponse.contains("\"taskmanagers\":[]")) { return; diff --git a/streamfusion-image-it/src/test/java/tech/streamfusion/imageit/NativeExtensionJarIT.java b/streamfusion-image-it/src/test/java/tech/streamfusion/imageit/NativeExtensionJarIT.java index 874b2971a..c2a83f63b 100644 --- a/streamfusion-image-it/src/test/java/tech/streamfusion/imageit/NativeExtensionJarIT.java +++ b/streamfusion-image-it/src/test/java/tech/streamfusion/imageit/NativeExtensionJarIT.java @@ -34,7 +34,8 @@ void extensionJarsLoadTheirNativeLibraries() throws Exception { "parquet")) { Process process = extensionProcess(extension); assertTrue( - process.waitFor(LOAD_TIMEOUT.toSeconds(), TimeUnit.SECONDS), extension + " probe timed out"); + process.waitFor(LOAD_TIMEOUT.toSeconds(), TimeUnit.SECONDS), + extension + " probe timed out"); String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); assertEquals(0, process.exitValue(), () -> extension + " probe failed:\n" + output); } @@ -50,13 +51,16 @@ private static Process extensionProcess(String extension) throws IOException { ExtensionProbe.class.getName(), requiredProperty("streamfusion.project.dir"), requiredProperty("streamfusion.version"), - extension) + extension, + System.getProperty("streamfusion.artifact.suffix", "")) .redirectErrorStream(true); process.environment().put("GLIBC_TUNABLES", "glibc.rtld.optional_static_tls=131072"); return process.start(); } - /** Runs in a fresh JVM so each extension gets its own isolated core/native-library class loader. */ + /** + * Runs in a fresh JVM so each extension gets its own isolated core/native-library class loader. + */ public static final class ExtensionProbe { private ExtensionProbe() {} @@ -65,16 +69,18 @@ public static void main(String[] args) throws Exception { Path projectDirectory = Path.of(args[0]); String version = args[1]; String extension = args[2]; + System.setProperty("streamfusion.artifact.suffix", args[3]); Path core = artifact(projectDirectory, "streamfusion-core", version); Path extensionJar = artifact(projectDirectory, "streamfusion-" + extension, version); - URL[] classpath = extensionClasspath(projectDirectory, version, extension, core, extensionJar); + URL[] classpath = + extensionClasspath(projectDirectory, version, extension, core, extensionJar); try (URLClassLoader loader = - new URLClassLoader( - classpath, ClassLoader.getPlatformClassLoader())) { + new URLClassLoader(classpath, ClassLoader.getPlatformClassLoader())) { Class facade = Class.forName(facadeClass(extension), true, loader); Object loaded = facade.getMethod("isLoaded").invoke(null); if (!Boolean.TRUE.equals(loaded)) { - throw new IllegalStateException("Native " + extension + " extension did not report loaded"); + throw new IllegalStateException( + "Native " + extension + " extension did not report loaded"); } String format = formatIdentifier(extension); if (format != null && !providesFormat(loader, format)) { @@ -106,7 +112,12 @@ private static Path artifact(Path projectDirectory, String module, String versio projectDirectory .resolve(module) .resolve("target") - .resolve(module + "-" + version + ".jar"); + .resolve( + module + + System.getProperty("streamfusion.artifact.suffix", "") + + "-" + + version + + ".jar"); if (!Files.isRegularFile(artifact)) { throw new IllegalStateException("Missing packaged extension artifact: " + artifact); } diff --git a/streamfusion-json/pom.xml b/streamfusion-json/pom.xml index 08ef7efd3..b6f5743db 100644 --- a/streamfusion-json/pom.xml +++ b/streamfusion-json/pom.xml @@ -4,14 +4,14 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 tech.streamfusionStreamFusion${revision} - streamfusion-json + streamfusion-json${streamfusion.artifact.suffix} StreamFusion JSON -p streamfusion-json streamfusion_json tech/streamfusion/native/json - tech.streamfusionstreamfusion-core${project.version}provided + tech.streamfusionstreamfusion-core${streamfusion.artifact.suffix}${project.version}provided ${project.basedir}/../src/main/java @@ -26,6 +26,11 @@ + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-kafka/pom.xml b/streamfusion-kafka/pom.xml index 76310ecf7..0ead3b546 100644 --- a/streamfusion-kafka/pom.xml +++ b/streamfusion-kafka/pom.xml @@ -10,7 +10,7 @@ ${revision} - streamfusion-kafka + streamfusion-kafka${streamfusion.artifact.suffix} StreamFusion Kafka @@ -22,7 +22,7 @@ tech.streamfusion - streamfusion-core + streamfusion-core${streamfusion.artifact.suffix} ${project.version} provided @@ -44,6 +44,11 @@ + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-loader/pom.xml b/streamfusion-loader/pom.xml index fe6333339..2482dfbae 100644 --- a/streamfusion-loader/pom.xml +++ b/streamfusion-loader/pom.xml @@ -5,7 +5,7 @@ 4.0.0 tech.streamfusion - streamfusion-loader + streamfusion-loader${streamfusion.artifact.suffix} ${revision} StreamFusion Loader @@ -49,6 +49,9 @@ 2.2.0 2.2.1 2.2 + + [2.2.0,2.3.0) + 5.0.0-2.2 5.10.2 @@ -63,7 +66,7 @@ are separate lib-directory JARs, matching Flink's connector packaging model. --> tech.streamfusion - streamfusion-core + streamfusion-core${streamfusion.artifact.suffix} ${project.version} runtime @@ -100,7 +103,7 @@ org.apache.flink flink-connector-kafka - 5.0.0-2.2 + ${flink.connector.kafka.version} test @@ -121,6 +124,12 @@ ${flink.runtime.version} test + + org.apache.flink + flink-statebackend-rocksdb + ${flink.runtime.version} + test + org.junit.jupiter junit-jupiter @@ -137,6 +146,25 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + 3.6.3 + + target-flink-line + enforce + + true + StreamFusion dependencies must target Flink ${flink.line}; select matching connector and format releases. + org.apache.flink:* + + org.apache.flink:*:${flink.dependency.range} + org.apache.flink:flink-shaded-* + org.apache.flink:flink-connector-kafka:[${flink.connector.kafka.version}] + + + + org.apache.maven.plugins maven-jar-plugin @@ -163,7 +191,7 @@ tech.streamfusion - streamfusion-core + streamfusion-core${streamfusion.artifact.suffix} ${project.version} runtime streamfusion-planner.jar @@ -189,6 +217,31 @@ + + flink-1.18 + + 1.18.1 + 1.18.1 + 1.18 + -flink1.18 + [1.18.0,1.19.0) + 3.2.0-1.18 + + + org.codehaus.mojo + flatten-maven-plugin + 1.7.3 + + ossrh + true + + + flatten-release-pom + process-resources + flatten + + + @@ -198,9 +251,9 @@ org.codehaus.mojo flatten-maven-plugin - 1.6.0 + 1.7.3 - resolveCiFriendliesOnly + ossrh true diff --git a/streamfusion-loader/src/main/java/org/apache/flink/table/planner/loader/PlannerModule.java b/streamfusion-loader/src/main/java/org/apache/flink/table/planner/loader/PlannerModule.java index ae545339c..973621f4f 100644 --- a/streamfusion-loader/src/main/java/org/apache/flink/table/planner/loader/PlannerModule.java +++ b/streamfusion-loader/src/main/java/org/apache/flink/table/planner/loader/PlannerModule.java @@ -47,18 +47,17 @@ import org.apache.flink.util.IOUtils; /** - * StreamFusion's Flink 2.2 planner loader shim. + * StreamFusion's planner loader shim for its declared Flink line. * *

The class intentionally has Flink's loader name so it is selected before Flink's bundled - * loader when installed first in {@code $FLINK_HOME/lib}. It adds a StreamFusion planner payload - * to the isolated component classloader and directly instantiates that payload's factory. All - * other planner behavior continues through Flink's normal implementation. + * loader when installed first in {@code $FLINK_HOME/lib}. It adds a StreamFusion planner payload to + * the isolated component classloader and directly instantiates that payload's factory. All other + * planner behavior continues through Flink's normal implementation. */ @Internal public class PlannerModule { static final String FLINK_TABLE_PLANNER_FAT_JAR = "flink-table-planner.jar"; - private static final Set SUPPORTED_FLINK_VERSIONS = Set.of("2.2.0", "2.2.1"); private static final String STREAMFUSION_PLANNER_JAR = "streamfusion-planner.jar"; private static final Set STREAMFUSION_PLANNER_EXTENSIONS = Set.of( @@ -166,15 +165,23 @@ public static PlannerModule getInstance() { return PlannerComponentsHolder.INSTANCE; } - private static void verifyFlinkVersion() { + private static void verifyFlinkVersion() throws IOException { + String line = FlinkPayloadIdentity.loaderLine(); + Set supportedVersions = + switch (line) { + case "2.2" -> Set.of("2.2.0", "2.2.1"); + case "1.18" -> Set.of("1.18.1"); + default -> throw new TableException("Unverified StreamFusion target Flink line: " + line); + }; Package flinkApiPackage = PlannerFactory.class.getPackage(); String version = flinkApiPackage == null ? null : flinkApiPackage.getImplementationVersion(); - if (version == null || !SUPPORTED_FLINK_VERSIONS.contains(version)) { + if (version == null || !supportedVersions.contains(version)) { throw new TableException( String.format( "StreamFusion's planner loader supports exactly Flink %s, but found %s." + " Refusing to cross an unverified planner ABI boundary.", - SUPPORTED_FLINK_VERSIONS, version == null ? "an unversioned Flink API" : "Flink " + version)); + supportedVersions, + version == null ? "an unversioned Flink API" : "Flink " + version)); } } @@ -185,7 +192,8 @@ private static URL extractResource( throw new TableException("Could not find planner resource '" + resource + "'."); } - Path output = Files.createFile(temporaryDirectory.resolve(resource + "_" + UUID.randomUUID() + ".jar")); + Path output = + Files.createFile(temporaryDirectory.resolve(resource + "_" + UUID.randomUUID() + ".jar")); try (InputStream resourceStream = input) { IOUtils.copyBytes(resourceStream, Files.newOutputStream(output)); } @@ -221,7 +229,13 @@ private static List extensionJars(String flinkLine) throws IOException { || attributes.getValue(FlinkPayloadIdentity.LINE_ATTRIBUTE) != null || path.getFileName().toString().matches("(?:[0-9]+-)?streamfusion-.*\\.jar")) { FlinkPayloadIdentity.verify(url, attributes, flinkLine); - if (module != null && STREAMFUSION_PLANNER_EXTENSIONS.contains(module)) extensions.add(url); + String suffix = "-flink" + flinkLine; + String logicalModule = + module != null && module.endsWith(suffix) + ? module.substring(0, module.length() - suffix.length()) + : module; + if (logicalModule != null && STREAMFUSION_PLANNER_EXTENSIONS.contains(logicalModule)) + extensions.add(url); } } return extensions; diff --git a/streamfusion-loader/src/test/java/tech/streamfusion/loader/StreamFusionPlannerLoaderTest.java b/streamfusion-loader/src/test/java/tech/streamfusion/loader/StreamFusionPlannerLoaderTest.java index d4106a845..65425d31f 100644 --- a/streamfusion-loader/src/test/java/tech/streamfusion/loader/StreamFusionPlannerLoaderTest.java +++ b/streamfusion-loader/src/test/java/tech/streamfusion/loader/StreamFusionPlannerLoaderTest.java @@ -10,6 +10,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Properties; import java.util.jar.Attributes; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; @@ -33,8 +34,13 @@ void acceptsARenamedMatchingExtensionAndRejectsAnUnmarkedLegacyExtension() throw Path renamed = directory.resolve("renamed.jar"); Manifest manifest = new Manifest(); manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); - manifest.getMainAttributes().putValue("StreamFusion-Module", "streamfusion-json"); - manifest.getMainAttributes().putValue("StreamFusion-Flink-Line", "2.2"); + String line = loaderLine(); + manifest + .getMainAttributes() + .putValue( + "StreamFusion-Module", + "streamfusion-json" + (line.equals("2.2") ? "" : "-flink" + line)); + manifest.getMainAttributes().putValue("StreamFusion-Flink-Line", line); try (var ignored = new JarOutputStream(Files.newOutputStream(renamed), manifest)) {} Path legacy = directory.resolve("01-streamfusion-paimon.jar"); try (var ignored = new JarOutputStream(Files.newOutputStream(legacy))) {} @@ -65,7 +71,9 @@ void rejectsMixedInstalledPayloadsBeforeCreatingThePlanner(String module) throws Manifest manifest = new Manifest(); manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); manifest.getMainAttributes().putValue("StreamFusion-Module", module); - manifest.getMainAttributes().putValue("StreamFusion-Flink-Line", "1.18"); + String line = loaderLine(); + String otherLine = line.equals("2.2") ? "1.18" : "2.2"; + manifest.getMainAttributes().putValue("StreamFusion-Flink-Line", otherLine); try (var ignored = new JarOutputStream(Files.newOutputStream(jar), manifest)) {} String original = System.getProperty("java.class.path"); var constructor = PlannerModule.class.getDeclaredConstructor(); @@ -75,8 +83,8 @@ void rejectsMixedInstalledPayloadsBeforeCreatingThePlanner(String module) throws System.setProperty("java.class.path", original + java.io.File.pathSeparator + jar); var failure = assertThrows(InvocationTargetException.class, constructor::newInstance); - assertTrue(failure.getCause().getMessage().contains("loader targets Flink 2.2")); - assertTrue(failure.getCause().getMessage().contains("targets Flink 1.18")); + assertTrue(failure.getCause().getMessage().contains("loader targets Flink " + line)); + assertTrue(failure.getCause().getMessage().contains("targets Flink " + otherLine)); } finally { System.setProperty("java.class.path", original); } @@ -98,7 +106,8 @@ void installsTheNativePlannerStageWithoutApplicationCode() throws Exception { PlannerModule.class.getResource("/streamfusion-planner.jar"), "the loader artifact must embed the StreamFusion runtime payload"); - TableEnvironment tableEnvironment = TableEnvironment.create(EnvironmentSettings.inStreamingMode()); + TableEnvironment tableEnvironment = + TableEnvironment.create(EnvironmentSettings.inStreamingMode()); String sql = "SELECT c0 * 2 AS doubled FROM (VALUES (3), (4), (5)) AS t(c0)"; String explain = tableEnvironment.explainSql(sql); @@ -106,6 +115,42 @@ void installsTheNativePlannerStageWithoutApplicationCode() throws Exception { assertEquals(List.of(6, 8, 10), collectInts(tableEnvironment.executeSql(sql))); } + @Test + void installsNativeStatefulPlannerWithConfiguredHostBackend() throws Exception { + org.apache.flink.configuration.Configuration configuration = + new org.apache.flink.configuration.Configuration(); + configuration.set( + org.apache.flink.configuration.StateBackendOptions.STATE_BACKEND, + "tech.streamfusion.state.RocksDBNativeStateBackendFactory"); + TableEnvironment table = + TableEnvironment.create( + EnvironmentSettings.newInstance() + .inStreamingMode() + .withConfiguration(configuration) + .build()); + String sql = "SELECT c0, SUM(c1) FROM (VALUES (1, 2), (1, 3), (2, 4)) AS t(c0, c1) GROUP BY c0"; + Class planner = + Class.forName( + "tech.streamfusion.planner.NativePlanner", + true, + PlannerModule.getInstance().getSubmoduleClassLoader()); + String explain = + (String) + planner + .getMethod("explain", TableEnvironment.class, String.class) + .invoke(null, table, sql); + assertTrue(explain.contains("NativeColumnarGroupAggregate"), explain); + } + + private static String loaderLine() throws Exception { + Properties properties = new Properties(); + try (var input = PlannerModule.class.getResourceAsStream("streamfusion-loader.properties")) { + assertNotNull(input); + properties.load(input); + } + return properties.getProperty("flink.line"); + } + private static List collectInts(TableResult result) throws Exception { List values = new ArrayList<>(); try (CloseableIterator rows = result.collect()) { diff --git a/streamfusion-orc/pom.xml b/streamfusion-orc/pom.xml index b659f9234..1f3cf83b1 100644 --- a/streamfusion-orc/pom.xml +++ b/streamfusion-orc/pom.xml @@ -10,7 +10,7 @@ ${revision} - streamfusion-orc + streamfusion-orc${streamfusion.artifact.suffix} StreamFusion Orc -p streamfusion-orc @@ -21,7 +21,7 @@ tech.streamfusion - streamfusion-core + streamfusion-core${streamfusion.artifact.suffix} ${project.version} provided @@ -47,6 +47,11 @@ + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-paimon/pom.xml b/streamfusion-paimon/pom.xml index ffdbacd95..e3bb002ea 100644 --- a/streamfusion-paimon/pom.xml +++ b/streamfusion-paimon/pom.xml @@ -8,7 +8,7 @@ StreamFusion ${revision} - streamfusion-paimon + streamfusion-paimon${streamfusion.artifact.suffix} StreamFusion Paimon -p streamfusion -p streamfusion-parquet -p streamfusion-orc -p streamfusion-paimon @@ -18,42 +18,42 @@ tech.streamfusion - streamfusion-runtime + streamfusion-runtime${streamfusion.artifact.suffix} ${project.version} test-jar test tech.streamfusion - streamfusion-kafka + streamfusion-kafka${streamfusion.artifact.suffix} ${project.version} test tech.streamfusion - streamfusion-json + streamfusion-json${streamfusion.artifact.suffix} ${project.version} test tech.streamfusion - streamfusion-core + streamfusion-core${streamfusion.artifact.suffix} ${project.version} provided tech.streamfusion - streamfusion-parquet + streamfusion-parquet${streamfusion.artifact.suffix} ${project.version} provided - tech.streamfusionstreamfusion-orc + tech.streamfusionstreamfusion-orc${streamfusion.artifact.suffix} ${project.version}provided org.apache.paimon - paimon-flink-2.2 + paimon-flink-${flink.line} ${paimon.version} provided @@ -108,6 +108,11 @@ + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.apache.maven.plugins maven-surefire-plugin diff --git a/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/PaimonDynamicPartitionOperatorTest.java b/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/PaimonDynamicPartitionOperatorTest.java index bc2e75333..05380d88f 100644 --- a/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/PaimonDynamicPartitionOperatorTest.java +++ b/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/PaimonDynamicPartitionOperatorTest.java @@ -269,11 +269,9 @@ private static final class RecordingFactory extends AbstractStreamOpera public > T createStreamOperator( StreamOperatorParameters parameters) { return delegate.createStreamOperator( - new StreamOperatorParameters<>( - parameters.getContainingTask(), - parameters.getStreamConfig(), + tech.streamfusion.compat.RuntimeCompat.withOutput( + parameters, parameters.getOutput(), - parameters::getProcessingTimeService, new OperatorEventDispatcher() { @Override public void registerEventHandler(OperatorID id, OperatorEventHandler registered) { @@ -284,8 +282,7 @@ public void registerEventHandler(OperatorID id, OperatorEventHandler registered) public OperatorEventGateway getOperatorEventGateway(OperatorID id) { return events::add; } - }, - parameters.getMailboxExecutor())); + })); } @Override diff --git a/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/PaimonSinkParityTest.java b/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/PaimonSinkParityTest.java index 291bddad7..a75d9b422 100644 --- a/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/PaimonSinkParityTest.java +++ b/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/PaimonSinkParityTest.java @@ -302,7 +302,8 @@ private static FileStoreTable insertConstraintFixture( + name + " (id BIGINT NOT NULL, fixed CHAR(3), limited VARCHAR(3)) WITH ('bucket' = '-1')"); DataStream stream = - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED( new String[] {"id", "fixed", "limited"}, Types.LONG, Types.STRING, Types.STRING), Row.of(null, "x", "abcdef"), @@ -338,7 +339,9 @@ private static FileStoreTable insertAliasedFixture( + name + " (id BIGINT NOT NULL, label STRING, nested ROW, pt STRING)" + " PARTITIONED BY (pt) WITH ('bucket' = '2', 'bucket-key' = 'id')"); - DataStream stream = env.fromData(fixtureTypeInformation(), fixtureRows()); + DataStream stream = + tech.streamfusion.compat.FlinkTestSources.fromData( + env, fixtureTypeInformation(), fixtureRows()); tableEnv.createTemporaryView( "fixture_source", tableEnv.fromDataStream(stream, fixtureSchema())); PhysicalPlanScan scan = nativeSink ? NativePlanner.install(tableEnv) : null; @@ -584,7 +587,7 @@ private static final class FailAfterCheckpoint extends RichMapFunction 0) { + if (tech.streamfusion.compat.RuntimeCompat.attempt(getRuntimeContext()) > 0) { COORDINATED_JOB_RESTARTED.set(true); } LockSupport.parkNanos(2_000_000); @@ -884,11 +887,16 @@ static void compactPostponeTables(java.nio.file.Path warehouse) throws Exception StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setRuntimeMode(org.apache.flink.api.common.RuntimeExecutionMode.BATCH); StreamTableEnvironment compactor = catalogEnvironment(env, warehouse); + compactor.getConfig().set( + org.apache.flink.configuration.ExecutionOptions.RUNTIME_MODE, + org.apache.flink.api.common.RuntimeExecutionMode.BATCH); compactor .getConfig() .set(org.apache.flink.table.api.config.TableConfigOptions.TABLE_DML_SYNC, true); - compactor.executeSql("CALL sys.compact(`table` => 'default.pk_native')").await(); - compactor.executeSql("CALL sys.compact(`table` => 'default.pk_stock')").await(); + // Flink 1.18 cannot infer Paimon's newer named/optional-argument annotations. + String optionalArguments = ", ''".repeat(6) + ", 'full'"; + compactor.executeSql("CALL sys.compact('default.pk_native'" + optionalArguments + ")").await(); + compactor.executeSql("CALL sys.compact('default.pk_stock'" + optionalArguments + ")").await(); } @Test @@ -1134,7 +1142,9 @@ private static FileStoreTable insertOnlyUpsertFixture( + name + " (id BIGINT NOT NULL, name STRING, price DECIMAL(10, 2), pt STRING NOT NULL," + " PRIMARY KEY (id, pt) NOT ENFORCED) PARTITIONED BY (pt) WITH ('bucket' = '2')"); - DataStream stream = env.fromData(fixtureTypeInformation(), fixtureRows()); + DataStream stream = + tech.streamfusion.compat.FlinkTestSources.fromData( + env, fixtureTypeInformation(), fixtureRows()); tableEnv.createTemporaryView( "fixture_source", tableEnv.fromDataStream(stream, fixtureSchema())); PhysicalPlanScan scan = nativeSink ? NativePlanner.install(tableEnv) : null; @@ -1209,7 +1219,8 @@ private static void registerChangelog( v[11])); } DataStream stream = - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED( new String[] { "id", "cat", "name", "price", "ts", "dt", "flag", "dbl", "bin", "small", "pt" @@ -1278,7 +1289,8 @@ void primaryKeyMergeOptionsRunThroughTheNativeSqlSink(String options) throws Exc tableEnv.createTemporaryView( "merge_source", tableEnv.fromDataStream( - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED( new String[] {"id", "v", "op"}, Types.LONG, Types.INT, Types.STRING), Row.of(1L, 9, "+I"), @@ -1323,7 +1335,8 @@ void collectionAggregatesRunThroughTheNativeSqlSink(boolean distinct) throws Exc tableEnv.createTemporaryView( "collect_source", tableEnv.fromDataStream( - env.fromData( + tech.streamfusion.compat.FlinkTestSources.fromData( + env, Types.ROW_NAMED( new String[] {"id", "v"}, Types.LONG, Types.OBJECT_ARRAY(Types.INT)), Row.of(1L, new Integer[] {1, 2, 2}), @@ -1455,6 +1468,69 @@ void dynamicTableOptionsFromTheJobConfigurationShapeAdmission() throws Exception assertDeclined(scan, "write-buffer-for-append"); } + @Test + void statementHintOptionsShapeSinkAdmission() throws Exception { + java.nio.file.Path warehouse = Files.createTempDirectory("paimon-sink-hints"); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + StreamTableEnvironment tableEnv = catalogEnvironment(env, warehouse); + tableEnv.executeSql("CREATE TABLE hinted (id BIGINT, v INT) WITH ('bucket' = '-1')"); + tableEnv.executeSql( + "CREATE TEMPORARY TABLE src (id BIGINT, v INT) WITH ('connector' = 'datagen')"); + PhysicalPlanScan scan = NativePlanner.install(tableEnv); + + tableEnv.explainSql( + "INSERT INTO hinted /*+ OPTIONS('write-buffer-for-append'='true') */ SELECT * FROM src"); + + assertDeclined(scan, "write-buffer-for-append"); + } + + @Test + @Timeout(60) + void statementBranchHintKeepsWritesOffTheMainBranch() throws Exception { + java.nio.file.Path warehouse = Files.createTempDirectory("paimon-branch-hints"); + List> mainRows = new ArrayList<>(); + List> branchRows = new ArrayList<>(); + for (boolean nativeWriter : new boolean[] {false, true}) { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + StreamTableEnvironment tableEnv = catalogEnvironment(env, warehouse); + String name = nativeWriter ? "branch_native" : "branch_stock"; + tableEnv.executeSql( + "CREATE TABLE " + + name + + " (k INT PRIMARY KEY NOT ENFORCED, v STRING) WITH ('bucket'='2')"); + PhysicalPlanScan scan = nativeWriter ? NativePlanner.install(tableEnv) : null; + tableEnv.executeSql("INSERT INTO " + name + " VALUES (1, 'main')").await(); + if (scan != null) assertAccelerated(scan); + tableEnv.executeSql("CALL sys.create_tag('default." + name + "', 'tag1', 1, '5 d')").await(); + tableEnv.executeSql("CALL sys.create_branch('default." + name + "', 'work', 'tag1')").await(); + String branchInsert = + "INSERT INTO " + + name + + " /*+ OPTIONS('branch'='work') */ VALUES (2, 'branch'), (3, 'branch')"; + if (nativeWriter) { + String plan = tableEnv.explainSql(branchInsert); + assertTrue(plan.contains("NativePaimonSink"), plan); + } + tableEnv.executeSql(branchInsert).await(); + if (scan != null) assertAccelerated(scan); + FileStoreTable main = openTable(warehouse, name); + FileStoreTable branch = main.switchToBranch("work"); + mainRows.add(PaimonTestTables.readRows(main, main.rowType())); + branchRows.add(PaimonTestTables.readRows(branch, branch.rowType())); + assertEquals(1, mainRows.get(mainRows.size() - 1).size()); + assertEquals(3, branchRows.get(branchRows.size() - 1).size()); + assertEquals(1L, main.snapshotManager().latestSnapshotId()); + if (nativeWriter) { + NativeAppendSinkWriteTest.assertNativeFiles(main); + NativeAppendSinkWriteTest.assertNativeFiles(branch); + } + } + assertEquals(mainRows.get(0), mainRows.get(1)); + assertEquals(branchRows.get(0), branchRows.get(1)); + } + private static String selectFor(String schema) { if (schema.contains("ARRAY")) { return schema.contains("pt STRING") ? "ARRAY[v] AS v, pt" : "ARRAY[v] AS v"; @@ -1489,7 +1565,9 @@ private static FileStoreTable insertFixture( + " WITH (" + options + ")"); - DataStream stream = env.fromData(fixtureTypeInformation(), fixtureRows()); + DataStream stream = + tech.streamfusion.compat.FlinkTestSources.fromData( + env, fixtureTypeInformation(), fixtureRows()); Table source = tableEnv.fromDataStream(stream, fixtureSchema()); tableEnv.createTemporaryView("fixture_source", source); PhysicalPlanScan scan = nativeSink ? NativePlanner.install(tableEnv) : null; diff --git a/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/SharingCaptureFactory.java b/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/SharingCaptureFactory.java index 9de9901c8..af1c17814 100644 --- a/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/SharingCaptureFactory.java +++ b/streamfusion-paimon/src/test/java/tech/streamfusion/paimon/SharingCaptureFactory.java @@ -5,12 +5,12 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.flink.configuration.ConfigOption; -import org.apache.flink.streaming.api.functions.sink.legacy.RichSinkFunction; import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.connector.sink.DynamicTableSink; -import org.apache.flink.table.connector.sink.legacy.SinkFunctionProvider; import org.apache.flink.table.data.RowData; import org.apache.flink.table.factories.DynamicTableSinkFactory; +import tech.streamfusion.compat.RichSinkFunction; +import tech.streamfusion.compat.SinkFunctionProvider; /** Captures every branch's row kinds and values after the ordinary Arrow-to-row sink boundary. */ public class SharingCaptureFactory implements DynamicTableSinkFactory { diff --git a/streamfusion-parquet/pom.xml b/streamfusion-parquet/pom.xml index 525439cb7..8bf5a58ec 100644 --- a/streamfusion-parquet/pom.xml +++ b/streamfusion-parquet/pom.xml @@ -10,7 +10,7 @@ ${revision} - streamfusion-parquet + streamfusion-parquet${streamfusion.artifact.suffix} StreamFusion Parquet -p streamfusion-parquet @@ -21,7 +21,7 @@ tech.streamfusion - streamfusion-core + streamfusion-core${streamfusion.artifact.suffix} ${project.version} provided @@ -43,6 +43,11 @@ + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-protobuf/pom.xml b/streamfusion-protobuf/pom.xml index 655eff367..e4da729b3 100644 --- a/streamfusion-protobuf/pom.xml +++ b/streamfusion-protobuf/pom.xml @@ -2,15 +2,20 @@ 4.0.0 tech.streamfusionStreamFusion${revision} - streamfusion-protobuf + streamfusion-protobuf${streamfusion.artifact.suffix} StreamFusion Protobuf -p streamfusion-protobuf streamfusion_protobuf tech/streamfusion/native/protobuf - tech.streamfusionstreamfusion-core${project.version}provided + tech.streamfusionstreamfusion-core${streamfusion.artifact.suffix}${project.version}provided ${project.basedir}/../src/main/java${project.basedir}/src/main/resources + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-raw/pom.xml b/streamfusion-raw/pom.xml index 85b581f33..ef2a66d0b 100644 --- a/streamfusion-raw/pom.xml +++ b/streamfusion-raw/pom.xml @@ -2,15 +2,20 @@ 4.0.0 tech.streamfusionStreamFusion${revision} - streamfusion-raw + streamfusion-raw${streamfusion.artifact.suffix} StreamFusion Raw -p streamfusion-raw streamfusion_raw tech/streamfusion/native/raw - tech.streamfusionstreamfusion-core${project.version}provided + tech.streamfusionstreamfusion-core${streamfusion.artifact.suffix}${project.version}provided ${project.basedir}/../src/main/java${project.basedir}/src/main/resources + + org.codehaus.mojo + flatten-maven-plugin + ossrh + org.codehaus.mojo exec-maven-plugin diff --git a/streamfusion-runtime/pom.xml b/streamfusion-runtime/pom.xml index 1a5e27c0d..cc2707a34 100644 --- a/streamfusion-runtime/pom.xml +++ b/streamfusion-runtime/pom.xml @@ -10,7 +10,7 @@ ${revision} - streamfusion-runtime + streamfusion-runtime${streamfusion.artifact.suffix} StreamFusion Runtime @@ -49,6 +49,25 @@ + + org.codehaus.mojo + flatten-maven-plugin + ossrh + + + org.codehaus.mojo + build-helper-maven-plugin + + + flink-compatibility-test-sources + generate-test-sources + add-test-source + + ${project.basedir}/../src/test/java-flink${flink.line} + + + + org.apache.maven.plugins maven-compiler-plugin