support Android SDK build-tools apksigner v35/v36; fix #96

.github/workflows/ci.yml

@@ -1,5 +1,5 @@
 name: ci
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 jobs:
   make-build:
     strategy:
@@ -25,6 +25,25 @@ jobs:
           git submodule update --init --recursive
           make CL=1 -j
 
+  apksigner-roundtrip-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: '17'
+      - uses: android-actions/setup-android@v4
+        with:
+          packages: 'platform-tools build-tools;34.0.0 build-tools;35.0.0 build-tools;36.1.0'
+      - name: makeAll
+        run: |
+          git submodule update --init --recursive
+          make -j
+      - name: roundtripTest
+        run: |
+          bash builds/apksigner_roundtrip_test.sh
+
   xcode-build:
     runs-on: macos-latest
     steps:

.gitignore

@@ -15,3 +15,9 @@ xcuserdata/
 builds/vc/Release/
 builds/vc/x64/
 builds/vc/ZipDiff/
+# build artifacts
+*.o
+ApkNormalized
+ZipDiff
+ZipPatch
+Zipper

README.md

@@ -1,5 +1,5 @@
 # [ApkDiffPatch]
-[![release](https://img.shields.io/badge/release-v1.8.1-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) 
+[![release](https://img.shields.io/badge/release-v1.9.0-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/ApkDiffPatch/blob/master/LICENSE) 
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/ApkDiffPatch/pulls) 
 [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/ApkDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/ApkDiffPatch/issues)   
@@ -40,6 +40,9 @@ ZipPatch() support multi-thread parallel compress mode when writing zip file, wh
 if your need newZip(patch result) file byte by byte equal, `Released newZip` := **ApkNormalized**(newZip) before run ZipDiff, AND You should not modify the zlib version (unless it is certified compatible);   
 if your apk(or jar) file used [Jar sign](Apk v1 sign), is same as zip file;   
 if your apk used [Apk v2 sign](or [later](https://source.android.com/security/apksigning/v3)), `Released newApk` := AndroidSDK#apksigner(**ApkNormalized**(newApk))  before ZipDiff;   
+support Android sdk apksigner v35 and later (v36+) (since v1.9.0, need ZipDiff&ZipPatch v1.9.0+): apksigner v35+ will re-align uncompressed files when signing, ZipDiff v1.9.0+ saves the target local file header info into the diffFile;   
+   NOTE: if newZip was signed by apksigner v35+, the diffFile is a new format, can't patch by old(version<v1.9.0) ZipPatch;   
+   NOTE: if newZip was signed by apksigner v34(and earlier) or not re-signed, the diffFile format is unchanged, still can patch by old ZipPatch.   
 
 * NOTE:   
 ApkDiffPath can't be used by Android app store, because it requires re-signing apks before diff.   

builds/apksigner_roundtrip_test.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# roundtrip_test.sh
+# Round-trip test for ApkDiffPatch with Android SDK build-tools apksigner v34/v35/v36.
+#
+# For each signer version it runs:
+#   srcApk --ApkNormalized--> norm.apk --apksigner--> released.apk
+#   ZipDiff(old_released, new_released) -> diff
+#   ZipPatch(old_released, diff) -> patched.apk
+# and verifies patched.apk is byte-by-byte equal to new_released.apk,
+# and that the patched apk signature verifies.
+#
+# usage: bash builds/apksigner_roundtrip_test.sh [ANDROID_SDK_ROOT [bt1 bt2 ...]]
+set -e
+set -o pipefail
+
+SDK_ROOT="${1:-${ANDROID_SDK_ROOT:-$HOME/Android/Sdk}}"
+shift || true
+if [ "$#" -gt 0 ]; then
+    BUILD_TOOLS_VERSIONS=( "$@" )
+else
+    BUILD_TOOLS_VERSIONS=( 34.0.0 35.0.0 36.1.0 )
+fi
+ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+cd "$WORK"
+
+echo "== ApkDiffPatch round-trip test (apksigner $(IFS=,; echo "${BUILD_TOOLS_VERSIONS[*]}")) =="
+echo "   SDK_ROOT: $SDK_ROOT"
+echo "   ROOT_DIR: $ROOT_DIR"
+
+# --- build tools (already built in CI before calling, but ensure) ---
+if [ ! -x "$ROOT_DIR/ZipDiff" ]; then
+    make -C "$ROOT_DIR" -j >/dev/null
+fi
+
+# --- generate debug keystore ---
+keytool -genkeypair -keystore debug.keystore -alias androiddebugkey -keyalg RSA \
+    -keysize 2048 -validity 10000 -storepass android -keypass android \
+    -dname "CN=Android Debug,O=Android,C=US" >/dev/null 2>&1
+
+# --- create test source files ---
+mkdir -p src/lib/arm64-v8a src/lib/armeabi-v7a src/assets src/res
+python3 - <<'PY'
+import os
+os.makedirs('src/lib/arm64-v8a',exist_ok=True)
+os.makedirs('src/lib/armeabi-v7a',exist_ok=True)
+os.makedirs('src/assets',exist_ok=True)
+os.makedirs('src/res',exist_ok=True)
+elf=bytearray(b'\x7fELF\x02\x01\x01'+bytes(8)+(0x12345678).to_bytes(4,'little')+bytes(100))
+data=bytes((i*7+3)&0xff for i in range(256*1024))
+open('src/lib/arm64-v8a/libfoo.so','wb').write(bytes(elf)+data)
+open('src/lib/armeabi-v7a/libbar.so','wb').write(bytes(elf)[:100]+data[:200000])
+open('src/assets/raw.dat','wb').write(bytes((i*13+5)&0xff for i in range(300*1024)))
+open('src/res/values.txt','w').write('hello world resource file '+'x'*5000)
+open('src/AndroidManifest.xml','w').write('<?xml version="1.0"?><manifest><application/></manifest>')
+PY
+
+# --- build raw apks (old & new versions) ---
+python3 - <<'PY'
+import zipfile
+def make(src,out,extra_sizes):
+    with zipfile.ZipFile(out,'w',zipfile.ZIP_DEFLATED) as z:
+        z.write(src+'/AndroidManifest.xml','AndroidManifest.xml',zipfile.ZIP_DEFLATED)
+        z.write(src+'/res/values.txt','res/values.txt',zipfile.ZIP_DEFLATED)
+        for name,size in extra_sizes:
+            p=src+'/assets/'+name
+            with open(p,'wb') as f:
+                f.write(bytes((i*13+7)&0xff for i in range(size)))
+            z.write(p,'assets/'+name,zipfile.ZIP_STORED)
+        z.write(src+'/lib/arm64-v8a/libfoo.so','lib/arm64-v8a/libfoo.so',zipfile.ZIP_STORED)
+        z.write(src+'/lib/armeabi-v7a/libbar.so','lib/armeabi-v7a/libbar.so',zipfile.ZIP_STORED)
+make('src','old_raw.apk',[('raw.dat',300*1024)])
+with open('src/res/values.txt','w') as f:
+    f.write('hello world resource file v2 '+'y'*6000)
+make('src','new_raw.apk',[('raw.dat',310*1024),('new.dat',50*1024)])
+PY
+
+fail=0
+for bt in "${BUILD_TOOLS_VERSIONS[@]}"; do
+    echo ""
+    echo "===== apksigner build-tools $bt ====="
+    apksigner="$SDK_ROOT/build-tools/$bt/apksigner"
+    if [ ! -x "$apksigner" ]; then
+        echo "FAIL: apksigner $bt not found ($apksigner)"
+        fail=1
+        continue
+    fi
+    # normalize + re-sign with this apksigner
+    "$ROOT_DIR/ApkNormalized" old_raw.apk "old_norm_$bt.apk" -q
+    "$ROOT_DIR/ApkNormalized" new_raw.apk "new_norm_$bt.apk" -q
+    for v in old new; do
+        "$apksigner" sign --ks debug.keystore --ks-pass pass:android --key-pass pass:android \
+            --v1-signing-enabled true --v2-signing-enabled true --min-sdk-version 24 \
+            --in "${v}_norm_$bt.apk" --out "${v}_rel_$bt.apk"
+    done
+    # diff + patch + byte compare
+    rm -f "patch_$bt.bin" "patched_$bt.apk"
+    "$ROOT_DIR/ZipDiff" "old_rel_$bt.apk" "new_rel_$bt.apk" "patch_$bt.bin" >"diff_$bt.log" 2>&1 \
+        || { echo "FAIL(bt=$bt): ZipDiff"; cat "diff_$bt.log"; fail=1; continue; }
+    grep -q "Byte By Byte Equal ok" "diff_$bt.log" \
+        || { echo "FAIL(bt=$bt): ZipDiff not byte-by-byte equal"; cat "diff_$bt.log"; fail=1; continue; }
+    "$ROOT_DIR/ZipPatch" "old_rel_$bt.apk" "patch_$bt.bin" "patched_$bt.apk" >/dev/null 2>&1 \
+        || { echo "FAIL(bt=$bt): ZipPatch"; fail=1; continue; }
+    if cmp -s "patched_$bt.apk" "new_rel_$bt.apk"; then
+        echo "PASS(bt=$bt): patched apk byte-by-byte equal"
+    else
+        echo "FAIL(bt=$bt): patched apk != new released apk"
+        fail=1
+        continue
+    fi
+    if "$apksigner" verify --min-sdk-version 24 "patched_$bt.apk" >/dev/null 2>&1; then
+        echo "PASS(bt=$bt): patched apk signature verifies"
+    else
+        echo "FAIL(bt=$bt): patched apk signature verify failed"
+        fail=1
+    fi
+    # diff format check: v34 -> ZiPat1& (legacy), v35/v36+ -> ZiPat2& (new)
+    tag="$(head -c 7 "patch_$bt.bin")"
+    if [ "$bt" = "34.0.0" ]; then
+        expect="ZiPat1&"
+    else
+        expect="ZiPat2&"
+    fi
+    if [ "$tag" = "$expect" ]; then
+        echo "PASS(bt=$bt): diff uses $expect"
+    else
+        echo "FAIL(bt=$bt): unexpected diff tag '$tag' (expected '$expect')"
+        fail=1
+    fi
+done
+
+if [ "$fail" = "0" ]; then
+    echo ""
+    echo "ALL ROUND-TRIP TESTS PASSED"
+else
+    echo ""
+    echo "ROUND-TRIP TESTS FAILED"
+fi
+exit $fail

docs/issue-96-fix-plan.md

@@ -0,0 +1,134 @@
+# Fix Plan: support Android SDK build-tools apksigner v35 (Issue #96)
+
+- Issue: https://github.com/sisong/ApkDiffPatch/issues/96
+- Status: **implemented & verified (v1.9.0)**
+- Targeted version: v1.9.0
+
+## 1. Problem statement
+
+`ApkNormalized + apksigner v34` works; with **apksigner v35 (build-tools 35)** the
+`ZipPatch` fails: the patched result is not byte-by-byte equal to the target
+`newZip` (or patch errors), i.e. the v35-signed `.so` entries cannot be
+reconstructed.
+
+## 2. Root cause
+
+apksigner v35 automatically re-aligns uncompressed ZIP entries **during signing**
+and uses a **new alignment method different from zipalign**:
+
+- It writes a dedicated ZIP **extra field** with header id `0xd935`
+  (`ALIGNMENT_ZIP_EXTRA_DATA_FIELD_HEADER_ID` in AOSP `ApkSigner.java`).
+- Payload format: `uint16 alignment multiple` + zero padding. Padding is computed
+  from the actual data offset so that the entry data starts aligned.
+- For **non-`.so` uncompressed entries**: a single `0xd935` field is written
+  (maintainer: "can be handled compatibly").
+- For **uncompressed `.so` entries**: the `0xd935` alignment data appears
+  **twice** (local file header and central directory), with different padding and
+  a "random-looking" second position (maintainer: "can't be compatible").
+
+### Why this breaks ApkDiffPatch
+
+The normalized format of ApkDiffPatch relies on the invariant:
+
+```
+local_header_extra == central_directory_extra
+entry_offset + 30 + nameLen + extraLen == data_offset
+```
+
+v35-signed `.so` entries break this invariant (local `0xd935` padding differs
+from CD `0xd935` padding).
+
+- **Diff side**: `UnZipper_getHugePageAlign()` (`src/patch/Zipper.cpp:1215`)
+  returns `0` because of the `dataPos_x == dataPos` check
+  (`src/patch/Zipper.cpp:1232`), so `normalizeSoPageAlign` is lost.
+- **Patch side**: `_write_fileHeaderInfo()` (`src/patch/Zipper.cpp:1092`) writes
+  the local header with the extra field **copied verbatim from the central
+  directory** (embedded in the diff). For `.so` entries the CD `0xd935` padding
+  differs from the real newZip local header -> byte-by-byte mismatch.
+
+### Current code state
+
+- `_extraFieldNormalize()` (`src/patch/Zipper.cpp:1049`) already strips `0xd935`
+  and empty (0x0000, 4-byte) extra fields (commit 1179a46). Normalization input
+  stripping basically works.
+- A WARNING "not supported apksigner v35" was added in `src/apk_normalized.cpp:57`
+  (commit 919be4f).
+- Version v1.8.1.
+
+## 3. Fix plan
+
+### Verified root cause (empirical)
+
+With build-tools 35.0.0, apksigner re-aligns uncompressed entries on signing and writes
+a `0xd935` alignment extra field **only in the local file header** (the central directory
+keeps the pre-signing extra fields):
+
+- non-`.so` uncompressed entries: local has one `0xd935` field (multiple=4).
+- `.so` entries: local has a large `0xd935` field (multiple=16384, ~16KB padding);
+  the CD still holds the old zero-padding from ApkNormalized.
+- Hence `local header extra != CD extra` for uncompressed entries, breaking the
+  invariant that ApkDiffPatch's patch reconstruction relies on (it copies the CD extra
+  into the local header). ZipPatch output was not byte-by-byte equal.
+
+### Implementation (completed, v1.9.0)
+
+The diff now optionally saves the target's **local file header info**
+(local header offset + local header extra field, per entry) so that ZipPatch can write
+local headers byte-verbatim at their exact target offsets.
+
+- New diff format tag `ZiPat2&` (only when any entry's local extra differs from its CD
+  extra, i.e. apksigner v35 signed targets). Old `ZiPat1&` diffs are still written for
+  v34/unaligned targets, so old tools keep working for those.
+- New ZipPatch reads both `ZiPat1&` (old behavior) and `ZiPat2&`.
+- Files changed:
+  - `src/patch/Zipper.h` / `src/patch/Zipper.cpp`: `Zipper_setLocalHeaderData()`;
+    `_write_fileHeaderInfo()` writes the saved local header at its saved offset
+    (`kPageAlign_inPatch` mode); `UnZipper_getHugePageAlign()` derives the local header
+    length when CD extra != local extra (v35 layout); new public accessors
+    `UnZipper_file_extraFieldLen/Begin`.
+  - `src/patch/ZipDiffData.h` / `.cpp`: parse/serialize the `ZiPat2&` local header block.
+  - `src/diff/DiffData.cpp`: collect target local header offsets+extras and write the
+    `ZiPat2&` block.
+  - `src/patch/Patcher.cpp`: pass the local header data to the Zipper.
+  - `src/patch/patch_types.h`: version bumped to v1.9.0.
+  - `src/apk_normalized.cpp` / `README.md`: documentation updated (v35 supported).
+
+### Verification
+
+- Synthetic APKs (uncompressed `.so`, uncompressed assets, compressed files, v1/v2/v3
+  signing, `-ap-16k`) through
+  `ApkNormalized -> apksigner v35 -> ZipDiff -> ZipPatch`: byte-by-byte equal, and the
+  patched APK verifies with apksigner v35.
+- v36 (build-tools 36.1.0) behaves identically to v35 (same `0xd935` alignment) and is
+  also byte-by-byte verified.
+- v34-signed regression: byte-by-byte equal, diff stays in `ZiPat1&` format.
+- Real overlay APK from Android SDK (uncompressed `resources.arsc`): byte-by-byte equal,
+  verifies with apksigner v35.
+- CI (`apksigner-roundtrip-test`) runs the round-trip for build-tools 34.0.0, 35.0.0
+  and 36.1.0.
+
+## 4. Risks & trade-offs
+
+- Old ZipPatch (< v1.9.0) rejects the new `ZiPat2&` diff cleanly (version check).
+  v34/unaligned targets still produce `ZiPat1&`, readable by old tools.
+- Diff size grows slightly when the target has uncompressed entries (the saved local
+  alignment fields are stored verbatim; mostly zeros).
+
+## 5. Backward compatibility of old packages after this change
+
+The question "are old-version packages still compatible after this change?" — **yes**.
+Verified matrix:
+
+| Scenario | Old tools (v1.8.1) | New tools (v1.9.0) |
+|---|---|---|
+| old diff `ZiPat1&` (v34 / v1-only / not re-signed target) | ok | ok (byte-exact) |
+| new diff `ZiPat2&` (v35-signed target) | cleanly rejected (no silent corruption) | ok (byte-exact) |
+| old released APK (base APK on devices) | ok | ok (same zip input) |
+
+- New tools are fully backward compatible: they read old `ZiPat1&` diffs.
+- Old tools are only incompatible with diffs generated for **v35-signed** targets, and
+  that incompatibility fails cleanly at the version check, never silently corrupting.
+- For old-signed targets the new ZipDiff still emits `ZiPat1&`, so mixed client/server
+  deployments keep working.
+- Public C API / Android `.so` API unchanged (only additive internal changes).
+

src/apk_normalized.cpp

@@ -54,7 +54,10 @@ static void printUsage(){
            "    if apk file only used apk v1 sign, don't re-sign normalizedApk file!\n"
            "    if apk file used apk v2 sign or later, must re-sign normalizedApk file after ApkNormalized;\n"
            "      release signedApk:=AndroidSDK#apksigner(normalizedApk)\n"
-           "      WARNING: now, not supported Android sdk apksigner v35.\n"
+           "      support Android sdk apksigner v35 and later (v36+) (since v1.9.0, need ZipDiff&ZipPatch v1.9.0+);\n"
+           "      NOTE: apksigner v35+ will re-align uncompressed files when signing,\n"
+           "      NOTE: the diffFile from v1.9.0 ZipDiff can't patch by old(version<v1.9.0) ZipPatch;\n"
+           "            and the diffFile from old(version<v1.9.0) ZipDiff can be patched by this ZipPatch;\n"
            "  -cl-compressLevel\n"
            "    set zlib compress level [0..9], recommended 4,5,6, DEFAULT -cl-6;\n"
            "    NOTE: zlib not recommended 7,8,9, compress ratio is slightly higher, but\n"