Add an isolated Flink 1.18 build with host compatibility adapters

.github/workflows/ci.yml

@@ -40,7 +40,17 @@ jobs:
         run: cargo fmt --all -- --check
 
   java-tests:
-    name: Java tests (debug native build)
+    name: Java tests (debug native build)${{ matrix.suffix }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - line: '2.2'
+            profile: ''
+            suffix: ''
+          - line: '1.18'
+            profile: '-Pflink-1.18'
+            suffix: ' (Flink 1.18)'
     runs-on: ubuntu-24.04
     timeout-minutes: 150
     steps:
@@ -64,8 +74,15 @@ jobs:
         run: |
           mvn -B -ntp -f dev/flink-suite/agent/pom.xml package
           python3 -m unittest discover -s dev/flink-suite -p 'test_*.py'
-          python3 -m unittest discover -s bin -p 'test_check_native_tls.py'
+          python3 -m unittest discover -s bin -p 'test_check_flink_suite_classpath.py'
           python3 -m unittest discover -s bin -p 'test_check_native_glibc.py'
+          python3 -m unittest discover -s bin -p 'test_check_native_tls.py'
+
+      - name: Test Flink 1.18 native and fallback contract selection
+        if: matrix.line == '1.18'
+        run: |
+          mvn -B -ntp -f dev/flink-suite/agent/pom.xml -Dstreamfusion.flink-suite.flink-line=1.18 test
+          mvn -B -ntp -f dev/flink-suite/agent/pom.xml -Dstreamfusion.flink-suite.flink-line=1.18 -Dstreamfusion.flink-suite.native-rocksdb=true test
 
       - uses: dtolnay/rust-toolchain@1.94.0
 
@@ -77,13 +94,13 @@ jobs:
       # full JUnit suite. The Kafka decode harnesses use Testcontainers, which the
       # hosted runners' preinstalled Docker daemon serves directly.
       - name: Run Java test suite
-        run: mvn -B -ntp test
+        run: mvn -B -ntp ${{ matrix.profile }} test
 
       - name: Upload portable SQL audit outcomes
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: portable-sql-audit
+          name: portable-sql-audit-${{ matrix.line }}
           path: streamfusion-runtime/target/sql-audit/*.json
           if-no-files-found: error
 
@@ -109,6 +126,14 @@ jobs:
             profile: paimon
             module: streamfusion-paimon
             format: orc
+          - name: Paimon Parquet (Flink 1.18)
+            profile: paimon,flink-1.18
+            module: streamfusion-paimon
+            format: parquet
+          - name: Paimon ORC (Flink 1.18)
+            profile: paimon,flink-1.18
+            module: streamfusion-paimon
+            format: orc
     steps:
       - uses: actions/checkout@v6
 
@@ -134,12 +159,22 @@ jobs:
           SF_PAIMON_FILE_FORMAT: ${{ matrix.format || 'parquet' }}
         run: >
           mvn -B -ntp -P${{ matrix.profile }}
-          -pl :${{ matrix.module }} -am test -Dsf.runtime.tests.skip=true
+          -pl ${{ matrix.module }} -am test -Dsf.runtime.tests.skip=true
 
   image-integration:
-    name: Flink image integration (optimized native build)
+    name: Flink image integration (optimized native build)${{ matrix.suffix }}
     runs-on: ubuntu-22.04
     timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - line: '2.2'
+            profile: image-it
+            suffix: ''
+          - line: '1.18'
+            profile: image-it,flink-1.18
+            suffix: ' (Flink 1.18)'
     steps:
       - uses: actions/checkout@v6
 
@@ -166,26 +201,16 @@ jobs:
       # Host-only is still a production Linux/x86_64 release (including mimalloc); cross-platform
       # manifest assembly is covered by the release builder, but does not belong on this smoke path.
       - name: Build optimized core image
+        env:
+          FLINK_LINE: ${{ matrix.line }}
         run: |
-          bin/build-release.sh --host-only
-          version="$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version)"
-          jar tf "streamfusion-kafka/target/streamfusion-kafka-${version}.jar" | grep -q 'native/kafka/linux/x86_64/libstreamfusion_kafka.so'
-          jar tf "streamfusion-json/target/streamfusion-json-${version}.jar" | grep -q 'native/json/linux/x86_64/libstreamfusion_json.so'
-          jar tf "streamfusion-csv/target/streamfusion-csv-${version}.jar" | grep -q 'native/csv/linux/x86_64/libstreamfusion_csv.so'
-          jar tf "streamfusion-raw/target/streamfusion-raw-${version}.jar" | grep -q 'native/raw/linux/x86_64/libstreamfusion_raw.so'
-          jar tf "streamfusion-avro/target/streamfusion-avro-${version}.jar" | grep -q 'native/avro/linux/x86_64/libstreamfusion_avro.so'
-          if jar tf "streamfusion-avro-confluent-registry/target/streamfusion-avro-confluent-registry-${version}.jar" | grep -q 'libstreamfusion_avro'; then
-            exit 1
-          fi
-          jar tf "streamfusion-protobuf/target/streamfusion-protobuf-${version}.jar" | grep -q 'native/protobuf/linux/x86_64/libstreamfusion_protobuf.so'
-          jar tf "streamfusion-parquet/target/streamfusion-parquet-${version}.jar" | grep -q 'native/parquet/linux/x86_64/libstreamfusion_parquet.so'
-          jar tf "streamfusion-orc/target/streamfusion-orc-${version}.jar" | grep -q 'native/orc/linux/x86_64/libstreamfusion_orc.so'
-          bin/check-artifacts.sh --host-only
-          bin/build-flink-image.sh --tag streamfusion-flink:image-it --load --skip-release-build
+          bin/build-release.sh --host-only --flink-line "$FLINK_LINE"
+          bin/check-artifacts.sh --host-only --flink-line "$FLINK_LINE"
+          bin/build-flink-image.sh --flink-line "$FLINK_LINE" --tag streamfusion-flink:image-it --load --skip-release-build
 
       - name: Run Flink Session-cluster image smoke test
         run: >
-          mvn -B -ntp -Pimage-it -pl :streamfusion-image-it verify
+          mvn -B -ntp -P${{ matrix.profile }} -pl streamfusion-image-it verify
           -Dstreamfusion.image.name=streamfusion-flink:image-it
 
   module-tests:
@@ -223,6 +248,42 @@ jobs:
           - name: Parquet
             tag: streamfusion-parquet
             package: streamfusion-parquet
+          - name: Kafka (Flink 1.18)
+            tag: streamfusion-kafka
+            package: streamfusion-kafka
+            profile: '-Pflink-1.18'
+          - name: JSON (Flink 1.18)
+            tag: streamfusion-json
+            package: streamfusion-json
+            profile: '-Pflink-1.18'
+          - name: CSV (Flink 1.18)
+            tag: streamfusion-csv
+            package: streamfusion-csv
+            profile: '-Pflink-1.18'
+          - name: RAW (Flink 1.18)
+            tag: streamfusion-raw
+            package: streamfusion-raw
+            profile: '-Pflink-1.18'
+          - name: Avro (Flink 1.18)
+            tag: streamfusion-avro
+            package: streamfusion-avro
+            profile: '-Pflink-1.18'
+          - name: Avro Confluent Registry (Flink 1.18)
+            tag: streamfusion-avro-confluent
+            package: streamfusion-avro
+            profile: '-Pflink-1.18'
+          - name: Protobuf (Flink 1.18)
+            tag: streamfusion-protobuf
+            package: streamfusion-protobuf
+            profile: '-Pflink-1.18'
+          - name: ORC (Flink 1.18)
+            tag: streamfusion-orc
+            package: streamfusion-orc
+            profile: '-Pflink-1.18'
+          - name: Parquet (Flink 1.18)
+            tag: streamfusion-parquet
+            package: streamfusion-parquet
+            profile: '-Pflink-1.18'
     steps:
       - uses: actions/checkout@v6
 
@@ -249,13 +310,50 @@ jobs:
 
       - name: Run tagged Java module suite
         run: >
-          mvn -B -ntp -pl :streamfusion-runtime test
+          mvn -B -ntp ${{ matrix.profile }} -pl streamfusion-runtime test
           -Dgroups="${{ matrix.tag }}"
 
+  flink118-artifacts:
+    name: Flink 1.18 qualified artifacts and loader
+    runs-on: ubuntu-22.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v6
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup
+          df -h /
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: "17"
+          cache: maven
+      - uses: dtolnay/rust-toolchain@1.94.0
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: ubuntu-22.04-glibc-2.35
+          workspaces: native
+      - name: Build and validate qualified optimized payloads
+        run: |
+          bin/build-release.sh --host-only --flink-line 1.18
+          bin/check-artifacts.sh --host-only --flink-line 1.18
+          bin/package-release.sh --flink-line 1.18
+      - name: Execute the packaged loader and mixed-line identity tests
+        run: >
+          mvn -B -ntp -Pflink-1.18,universal -pl streamfusion-loader -am test
+          -Dtest=FlinkPayloadIdentityTest,StreamFusionPlannerLoaderTest
+          -Dsurefire.failIfNoSpecifiedTests=false
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: flink1.18-optimized-linux-payloads
+          path: target/release/streamfusion-flink1.18-*-bin.tar.gz*
+          if-no-files-found: error
+
   all-tests:
     name: All CI tests
     if: ${{ always() }}
-    needs: [native-tests, java-tests, lake-module-tests, image-integration, module-tests]
+    needs: [native-tests, java-tests, lake-module-tests, image-integration, module-tests, flink118-artifacts]
     runs-on: ubuntu-24.04
     timeout-minutes: 5
     steps:

.github/workflows/flink-suite.yml

@@ -17,13 +17,24 @@ concurrency:
 
 jobs:
   upstream-suite:
-    name: ${{ matrix.suite }} suite
+    name: ${{ matrix.suite }} suite${{ matrix.suffix }}
     runs-on: ubuntu-24.04
     timeout-minutes: 180
     strategy:
       fail-fast: false
       matrix:
         suite: [runtime, formats, parquet, orc, kafka, paimon, delta, state]
+        line: ['2.2', '1.18']
+        exclude:
+          - suite: delta
+            line: '1.18'
+        include:
+          - line: '2.2'
+            version: '2.2.1'
+            suffix: ''
+          - line: '1.18'
+            version: '1.18.1'
+            suffix: ' (Flink 1.18)'
     steps:
       - uses: actions/checkout@v6
 
@@ -42,21 +53,23 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
         with:
-          workspaces: native -> ../.flink-suite/streamfusion-source/native/target
+          workspaces: native -> ../.flink-suite/${{ matrix.line }}/streamfusion-source/native/target
 
       - name: Run unchanged upstream suite with StreamFusion
+        env:
+          FLINK_VERSION: ${{ matrix.version }}
         run: |
           set -o pipefail
           bin/flink-suite.sh "${{ matrix.suite }}" 2>&1 | tee upstream-suite.log
 
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: upstream-flink-${{ matrix.suite }}-${{ github.sha }}
+          name: upstream-flink-${{ matrix.line }}-${{ matrix.suite }}-${{ github.sha }}
           path: |
             upstream-suite.log
-            .flink-suite/native-execution/
-            .flink-suite/diagnostics/
+            .flink-suite/**/native-execution/
+            .flink-suite/**/diagnostics/
             .flink-suite/**/target/surefire-reports/
           include-hidden-files: true
           retention-days: 7

bin/build-flink-image.sh

@@ -12,15 +12,17 @@ Builds a job-neutral StreamFusion Flink base image.
   --push                  Build linux/amd64 and linux/arm64, then push a manifest list.
   --load                  Build one platform and load it into the local Docker daemon.
   --platform <platform>   Platform for --load (default: Docker server platform).
-  --flink-image <image>   Flink base image (default: flink:2.2.1-scala_2.12-java17).
+  --flink-line <line>     Payload line: 2.2 (default) or 1.18.
+  --flink-image <image>   Override the official image selected for the payload line.
   --skip-release-build    Reuse the already-built StreamFusion JARs.
 EOF
   exit 64
 }
 
 script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 repo_root=$(cd "$script_dir/.." && pwd)
-flink_image=flink:2.2.1-scala_2.12-java17
+flink_image=
+flink_line=2.2
 image_tag=
 mode=
 platform=
@@ -43,6 +45,11 @@ while [ "$#" -gt 0 ]; do
       platform=$2
       shift 2
       ;;
+    --flink-line)
+      [ "$#" -ge 2 ] || usage
+      flink_line=$2
+      shift 2
+      ;;
     --flink-image)
       [ "$#" -ge 2 ] || usage
       flink_image=$2
@@ -60,19 +67,27 @@ done
 
 [ -n "$image_tag" ] && [ -n "$mode" ] || usage
 
+artifact_suffix=
+case "$flink_line" in
+  2.2) default_flink_image=flink:2.2.1-scala_2.12-java17 ;;
+  1.18) artifact_suffix=-flink1.18; default_flink_image=flink:1.18.1-scala_2.12-java17 ;;
+  *) echo "unsupported Flink line: $flink_line" >&2; exit 64 ;;
+esac
+flink_image=${flink_image:-$default_flink_image}
+
 command -v docker >/dev/null 2>&1 || {
   echo "Docker with buildx is required." >&2
   exit 69
 }
 docker buildx version >/dev/null
 
 if [ "$skip_release_build" = false ]; then
-  "$repo_root/bin/build-release.sh" --linux-only
+  "$repo_root/bin/build-release.sh" --linux-only --flink-line "$flink_line"
 fi
 
 artifact_version=$(cd "$repo_root" && mvn -q -DforceStdout help:evaluate -Dexpression=project.version)
-loader_jar=$repo_root/streamfusion-loader/target/streamfusion-loader-$artifact_version.jar
-core_jar=$repo_root/streamfusion-core/target/streamfusion-core-$artifact_version-runtime.jar
+loader_jar=$repo_root/streamfusion-loader/target/streamfusion-loader$artifact_suffix-$artifact_version.jar
+core_jar=$repo_root/streamfusion-core/target/streamfusion-core$artifact_suffix-$artifact_version-runtime.jar
 [ -f "$loader_jar" ] && [ -f "$core_jar" ] || {
   echo "StreamFusion release JARs are missing; run bin/build-release.sh first." >&2
   exit 66
@@ -104,6 +119,8 @@ docker buildx build \
   --platform "$platforms" \
   --build-arg "FLINK_IMAGE=$flink_image" \
   --build-arg "STREAMFUSION_VERSION=$artifact_version" \
+  --build-arg "STREAMFUSION_ARTIFACT_SUFFIX=$artifact_suffix" \
+  --build-arg "FLINK_LINE=$flink_line" \
   --tag "$image_tag" \
   --file "$repo_root/docker/flink-base.Dockerfile" \
   "$output" \

bin/build-release.sh

@@ -2,23 +2,33 @@
 
 set -eu
 
-if [ "$#" -gt 1 ] || { [ "$#" -eq 1 ] && [ "$1" != "--host-only" ] && [ "$1" != "--linux-only" ]; }; then
-  echo "usage: $0 [--host-only | --linux-only]" >&2
+host_only=false
+linux_only=false
+flink_line=2.2
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --host-only) host_only=true; shift ;;
+    --linux-only) linux_only=true; shift ;;
+    --flink-line)
+      if [ "$#" -lt 2 ]; then echo "--flink-line requires 2.2 or 1.18" >&2; exit 64; fi
+      flink_line=$2; shift 2 ;;
+    *) echo "usage: $0 [--host-only | --linux-only] [--flink-line 2.2|1.18]" >&2; exit 64 ;;
+  esac
+done
+if [ "$host_only" = true ] && [ "$linux_only" = true ]; then
+  echo "--host-only and --linux-only are mutually exclusive" >&2
   exit 64
 fi
+case "$flink_line" in
+  2.2) maven_profiles=dist,universal,release,delta,paimon ;;
+  1.18) maven_profiles=dist,universal,release,paimon,flink-1.18 ;;
+  *) echo "unsupported Flink line: $flink_line" >&2; exit 64 ;;
+esac
 
 script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
 repo_root=$(cd "$script_dir/.." && pwd)
 native_dir=$repo_root/native
 stage_dir=$native_dir/target/universal
-host_only=false
-linux_only=false
-
-if [ "$#" -eq 1 ] && [ "$1" = "--host-only" ]; then
-  host_only=true
-elif [ "$#" -eq 1 ]; then
-  linux_only=true
-fi
 
 host_platform() {
   case "$(uname -s)" in
@@ -203,4 +213,4 @@ fi
 # platform build. A release always starts from empty Java output directories. The release profile
 # builds the same source and javadoc attachments as the publish workflow, unsigned, so attachment
 # failures surface here instead of on the release runner.
-(cd "$repo_root" && mvn clean package -Pdist,universal,release,delta,paimon -Dgpg.skip=true -DskipTests)
+(cd "$repo_root" && mvn clean package "-P$maven_profiles" -Dgpg.skip=true -DskipTests)