diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b29b16f..2b88108 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ name: ci -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: make-build: strategy: @@ -25,6 +25,25 @@ jobs: git submodule update --init --recursive make CL=1 -j + apksigner-roundtrip-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: '17' + - uses: android-actions/setup-android@v4 + with: + packages: 'platform-tools build-tools;34.0.0 build-tools;35.0.0 build-tools;36.1.0' + - name: makeAll + run: | + git submodule update --init --recursive + make -j + - name: roundtripTest + run: | + bash builds/apksigner_roundtrip_test.sh + xcode-build: runs-on: macos-latest steps: diff --git a/.gitignore b/.gitignore index e3cc881..80ae6f3 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,9 @@ xcuserdata/ builds/vc/Release/ builds/vc/x64/ builds/vc/ZipDiff/ +# build artifacts +*.o +ApkNormalized +ZipDiff +ZipPatch +Zipper diff --git a/README.md b/README.md index 9eadb46..9658653 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # [ApkDiffPatch] -[![release](https://img.shields.io/badge/release-v1.8.1-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) +[![release](https://img.shields.io/badge/release-v1.9.0-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/ApkDiffPatch/blob/master/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/ApkDiffPatch/pulls) [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/ApkDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/ApkDiffPatch/issues) @@ -40,6 +40,9 @@ ZipPatch() support multi-thread parallel compress mode when writing zip file, wh if your need newZip(patch result) file byte by byte equal, `Released newZip` := **ApkNormalized**(newZip) before run ZipDiff, AND You should not modify the zlib version (unless it is certified compatible); if your apk(or jar) file used [Jar sign](Apk v1 sign), is same as zip file; if your apk used [Apk v2 sign](or [later](https://source.android.com/security/apksigning/v3)), `Released newApk` := AndroidSDK#apksigner(**ApkNormalized**(newApk)) before ZipDiff; +support Android sdk apksigner v35 and later (v36+) (since v1.9.0, need ZipDiff&ZipPatch v1.9.0+): apksigner v35+ will re-align uncompressed files when signing, ZipDiff v1.9.0+ saves the target local file header info into the diffFile; + NOTE: if newZip was signed by apksigner v35+, the diffFile is a new format, can't patch by old(version norm.apk --apksigner--> released.apk +# ZipDiff(old_released, new_released) -> diff +# ZipPatch(old_released, diff) -> patched.apk +# and verifies patched.apk is byte-by-byte equal to new_released.apk, +# and that the patched apk signature verifies. +# +# usage: bash builds/apksigner_roundtrip_test.sh [ANDROID_SDK_ROOT [bt1 bt2 ...]] +set -e +set -o pipefail + +SDK_ROOT="${1:-${ANDROID_SDK_ROOT:-$HOME/Android/Sdk}}" +shift || true +if [ "$#" -gt 0 ]; then + BUILD_TOOLS_VERSIONS=( "$@" ) +else + BUILD_TOOLS_VERSIONS=( 34.0.0 35.0.0 36.1.0 ) +fi +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +cd "$WORK" + +echo "== ApkDiffPatch round-trip test (apksigner $(IFS=,; echo "${BUILD_TOOLS_VERSIONS[*]}")) ==" +echo " SDK_ROOT: $SDK_ROOT" +echo " ROOT_DIR: $ROOT_DIR" + +# --- build tools (already built in CI before calling, but ensure) --- +if [ ! -x "$ROOT_DIR/ZipDiff" ]; then + make -C "$ROOT_DIR" -j >/dev/null +fi + +# --- generate debug keystore --- +keytool -genkeypair -keystore debug.keystore -alias androiddebugkey -keyalg RSA \ + -keysize 2048 -validity 10000 -storepass android -keypass android \ + -dname "CN=Android Debug,O=Android,C=US" >/dev/null 2>&1 + +# --- create test source files --- +mkdir -p src/lib/arm64-v8a src/lib/armeabi-v7a src/assets src/res +python3 - <<'PY' +import os +os.makedirs('src/lib/arm64-v8a',exist_ok=True) +os.makedirs('src/lib/armeabi-v7a',exist_ok=True) +os.makedirs('src/assets',exist_ok=True) +os.makedirs('src/res',exist_ok=True) +elf=bytearray(b'\x7fELF\x02\x01\x01'+bytes(8)+(0x12345678).to_bytes(4,'little')+bytes(100)) +data=bytes((i*7+3)&0xff for i in range(256*1024)) +open('src/lib/arm64-v8a/libfoo.so','wb').write(bytes(elf)+data) +open('src/lib/armeabi-v7a/libbar.so','wb').write(bytes(elf)[:100]+data[:200000]) +open('src/assets/raw.dat','wb').write(bytes((i*13+5)&0xff for i in range(300*1024))) +open('src/res/values.txt','w').write('hello world resource file '+'x'*5000) +open('src/AndroidManifest.xml','w').write('') +PY + +# --- build raw apks (old & new versions) --- +python3 - <<'PY' +import zipfile +def make(src,out,extra_sizes): + with zipfile.ZipFile(out,'w',zipfile.ZIP_DEFLATED) as z: + z.write(src+'/AndroidManifest.xml','AndroidManifest.xml',zipfile.ZIP_DEFLATED) + z.write(src+'/res/values.txt','res/values.txt',zipfile.ZIP_DEFLATED) + for name,size in extra_sizes: + p=src+'/assets/'+name + with open(p,'wb') as f: + f.write(bytes((i*13+7)&0xff for i in range(size))) + z.write(p,'assets/'+name,zipfile.ZIP_STORED) + z.write(src+'/lib/arm64-v8a/libfoo.so','lib/arm64-v8a/libfoo.so',zipfile.ZIP_STORED) + z.write(src+'/lib/armeabi-v7a/libbar.so','lib/armeabi-v7a/libbar.so',zipfile.ZIP_STORED) +make('src','old_raw.apk',[('raw.dat',300*1024)]) +with open('src/res/values.txt','w') as f: + f.write('hello world resource file v2 '+'y'*6000) +make('src','new_raw.apk',[('raw.dat',310*1024),('new.dat',50*1024)]) +PY + +fail=0 +for bt in "${BUILD_TOOLS_VERSIONS[@]}"; do + echo "" + echo "===== apksigner build-tools $bt =====" + apksigner="$SDK_ROOT/build-tools/$bt/apksigner" + if [ ! -x "$apksigner" ]; then + echo "FAIL: apksigner $bt not found ($apksigner)" + fail=1 + continue + fi + # normalize + re-sign with this apksigner + "$ROOT_DIR/ApkNormalized" old_raw.apk "old_norm_$bt.apk" -q + "$ROOT_DIR/ApkNormalized" new_raw.apk "new_norm_$bt.apk" -q + for v in old new; do + "$apksigner" sign --ks debug.keystore --ks-pass pass:android --key-pass pass:android \ + --v1-signing-enabled true --v2-signing-enabled true --min-sdk-version 24 \ + --in "${v}_norm_$bt.apk" --out "${v}_rel_$bt.apk" + done + # diff + patch + byte compare + rm -f "patch_$bt.bin" "patched_$bt.apk" + "$ROOT_DIR/ZipDiff" "old_rel_$bt.apk" "new_rel_$bt.apk" "patch_$bt.bin" >"diff_$bt.log" 2>&1 \ + || { echo "FAIL(bt=$bt): ZipDiff"; cat "diff_$bt.log"; fail=1; continue; } + grep -q "Byte By Byte Equal ok" "diff_$bt.log" \ + || { echo "FAIL(bt=$bt): ZipDiff not byte-by-byte equal"; cat "diff_$bt.log"; fail=1; continue; } + "$ROOT_DIR/ZipPatch" "old_rel_$bt.apk" "patch_$bt.bin" "patched_$bt.apk" >/dev/null 2>&1 \ + || { echo "FAIL(bt=$bt): ZipPatch"; fail=1; continue; } + if cmp -s "patched_$bt.apk" "new_rel_$bt.apk"; then + echo "PASS(bt=$bt): patched apk byte-by-byte equal" + else + echo "FAIL(bt=$bt): patched apk != new released apk" + fail=1 + continue + fi + if "$apksigner" verify --min-sdk-version 24 "patched_$bt.apk" >/dev/null 2>&1; then + echo "PASS(bt=$bt): patched apk signature verifies" + else + echo "FAIL(bt=$bt): patched apk signature verify failed" + fail=1 + fi + # diff format check: v34 -> ZiPat1& (legacy), v35/v36+ -> ZiPat2& (new) + tag="$(head -c 7 "patch_$bt.bin")" + if [ "$bt" = "34.0.0" ]; then + expect="ZiPat1&" + else + expect="ZiPat2&" + fi + if [ "$tag" = "$expect" ]; then + echo "PASS(bt=$bt): diff uses $expect" + else + echo "FAIL(bt=$bt): unexpected diff tag '$tag' (expected '$expect')" + fail=1 + fi +done + +if [ "$fail" = "0" ]; then + echo "" + echo "ALL ROUND-TRIP TESTS PASSED" +else + echo "" + echo "ROUND-TRIP TESTS FAILED" +fi +exit $fail diff --git a/docs/issue-96-fix-plan.md b/docs/issue-96-fix-plan.md new file mode 100644 index 0000000..0af7f38 --- /dev/null +++ b/docs/issue-96-fix-plan.md @@ -0,0 +1,134 @@ +# Fix Plan: support Android SDK build-tools apksigner v35 (Issue #96) + +- Issue: https://github.com/sisong/ApkDiffPatch/issues/96 +- Status: **implemented & verified (v1.9.0)** +- Targeted version: v1.9.0 + +## 1. Problem statement + +`ApkNormalized + apksigner v34` works; with **apksigner v35 (build-tools 35)** the +`ZipPatch` fails: the patched result is not byte-by-byte equal to the target +`newZip` (or patch errors), i.e. the v35-signed `.so` entries cannot be +reconstructed. + +## 2. Root cause + +apksigner v35 automatically re-aligns uncompressed ZIP entries **during signing** +and uses a **new alignment method different from zipalign**: + +- It writes a dedicated ZIP **extra field** with header id `0xd935` + (`ALIGNMENT_ZIP_EXTRA_DATA_FIELD_HEADER_ID` in AOSP `ApkSigner.java`). +- Payload format: `uint16 alignment multiple` + zero padding. Padding is computed + from the actual data offset so that the entry data starts aligned. +- For **non-`.so` uncompressed entries**: a single `0xd935` field is written + (maintainer: "can be handled compatibly"). +- For **uncompressed `.so` entries**: the `0xd935` alignment data appears + **twice** (local file header and central directory), with different padding and + a "random-looking" second position (maintainer: "can't be compatible"). + +### Why this breaks ApkDiffPatch + +The normalized format of ApkDiffPatch relies on the invariant: + +``` +local_header_extra == central_directory_extra +entry_offset + 30 + nameLen + extraLen == data_offset +``` + +v35-signed `.so` entries break this invariant (local `0xd935` padding differs +from CD `0xd935` padding). + +- **Diff side**: `UnZipper_getHugePageAlign()` (`src/patch/Zipper.cpp:1215`) + returns `0` because of the `dataPos_x == dataPos` check + (`src/patch/Zipper.cpp:1232`), so `normalizeSoPageAlign` is lost. +- **Patch side**: `_write_fileHeaderInfo()` (`src/patch/Zipper.cpp:1092`) writes + the local header with the extra field **copied verbatim from the central + directory** (embedded in the diff). For `.so` entries the CD `0xd935` padding + differs from the real newZip local header -> byte-by-byte mismatch. + +### Current code state + +- `_extraFieldNormalize()` (`src/patch/Zipper.cpp:1049`) already strips `0xd935` + and empty (0x0000, 4-byte) extra fields (commit 1179a46). Normalization input + stripping basically works. +- A WARNING "not supported apksigner v35" was added in `src/apk_normalized.cpp:57` + (commit 919be4f). +- Version v1.8.1. + +## 3. Fix plan + +### Verified root cause (empirical) + +With build-tools 35.0.0, apksigner re-aligns uncompressed entries on signing and writes +a `0xd935` alignment extra field **only in the local file header** (the central directory +keeps the pre-signing extra fields): + +- non-`.so` uncompressed entries: local has one `0xd935` field (multiple=4). +- `.so` entries: local has a large `0xd935` field (multiple=16384, ~16KB padding); + the CD still holds the old zero-padding from ApkNormalized. +- Hence `local header extra != CD extra` for uncompressed entries, breaking the + invariant that ApkDiffPatch's patch reconstruction relies on (it copies the CD extra + into the local header). ZipPatch output was not byte-by-byte equal. + +### Implementation (completed, v1.9.0) + +The diff now optionally saves the target's **local file header info** +(local header offset + local header extra field, per entry) so that ZipPatch can write +local headers byte-verbatim at their exact target offsets. + +- New diff format tag `ZiPat2&` (only when any entry's local extra differs from its CD + extra, i.e. apksigner v35 signed targets). Old `ZiPat1&` diffs are still written for + v34/unaligned targets, so old tools keep working for those. +- New ZipPatch reads both `ZiPat1&` (old behavior) and `ZiPat2&`. +- Files changed: + - `src/patch/Zipper.h` / `src/patch/Zipper.cpp`: `Zipper_setLocalHeaderData()`; + `_write_fileHeaderInfo()` writes the saved local header at its saved offset + (`kPageAlign_inPatch` mode); `UnZipper_getHugePageAlign()` derives the local header + length when CD extra != local extra (v35 layout); new public accessors + `UnZipper_file_extraFieldLen/Begin`. + - `src/patch/ZipDiffData.h` / `.cpp`: parse/serialize the `ZiPat2&` local header block. + - `src/diff/DiffData.cpp`: collect target local header offsets+extras and write the + `ZiPat2&` block. + - `src/patch/Patcher.cpp`: pass the local header data to the Zipper. + - `src/patch/patch_types.h`: version bumped to v1.9.0. + - `src/apk_normalized.cpp` / `README.md`: documentation updated (v35 supported). + +### Verification + +- Synthetic APKs (uncompressed `.so`, uncompressed assets, compressed files, v1/v2/v3 + signing, `-ap-16k`) through + `ApkNormalized -> apksigner v35 -> ZipDiff -> ZipPatch`: byte-by-byte equal, and the + patched APK verifies with apksigner v35. +- v36 (build-tools 36.1.0) behaves identically to v35 (same `0xd935` alignment) and is + also byte-by-byte verified. +- v34-signed regression: byte-by-byte equal, diff stays in `ZiPat1&` format. +- Real overlay APK from Android SDK (uncompressed `resources.arsc`): byte-by-byte equal, + verifies with apksigner v35. +- CI (`apksigner-roundtrip-test`) runs the round-trip for build-tools 34.0.0, 35.0.0 + and 36.1.0. + +## 4. Risks & trade-offs + +- Old ZipPatch (< v1.9.0) rejects the new `ZiPat2&` diff cleanly (version check). + v34/unaligned targets still produce `ZiPat1&`, readable by old tools. +- Diff size grows slightly when the target has uncompressed entries (the saved local + alignment fields are stored verbatim; mostly zeros). + +## 5. Backward compatibility of old packages after this change + +The question "are old-version packages still compatible after this change?" — **yes**. +Verified matrix: + +| Scenario | Old tools (v1.8.1) | New tools (v1.9.0) | +|---|---|---| +| old diff `ZiPat1&` (v34 / v1-only / not re-signed target) | ok | ok (byte-exact) | +| new diff `ZiPat2&` (v35-signed target) | cleanly rejected (no silent corruption) | ok (byte-exact) | +| old released APK (base APK on devices) | ok | ok (same zip input) | + +- New tools are fully backward compatible: they read old `ZiPat1&` diffs. +- Old tools are only incompatible with diffs generated for **v35-signed** targets, and + that incompatibility fails cleanly at the version check, never silently corrupting. +- For old-signed targets the new ZipDiff still emits `ZiPat1&`, so mixed client/server + deployments keep working. +- Public C API / Android `.so` API unchanged (only additive internal changes). + diff --git a/src/apk_normalized.cpp b/src/apk_normalized.cpp index 1fb21da..11b8f07 100644 --- a/src/apk_normalized.cpp +++ b/src/apk_normalized.cpp @@ -54,7 +54,10 @@ static void printUsage(){ " if apk file only used apk v1 sign, don't re-sign normalizedApk file!\n" " if apk file used apk v2 sign or later, must re-sign normalizedApk file after ApkNormalized;\n" " release signedApk:=AndroidSDK#apksigner(normalizedApk)\n" - " WARNING: now, not supported Android sdk apksigner v35.\n" + " support Android sdk apksigner v35 and later (v36+) (since v1.9.0, need ZipDiff&ZipPatch v1.9.0+);\n" + " NOTE: apksigner v35+ will re-align uncompressed files when signing,\n" + " NOTE: the diffFile from v1.9.0 ZipDiff can't patch by old(version& out_data,const uint32_t* list,size_t static bool _serializeZipDiffData(std::vector& out_data,const ZipDiffData* data, const std::vector& hdiffzData, - const hdiff_TCompress* compressPlugin,const UnZipper* newZip){ + const hdiff_TCompress* compressPlugin,const UnZipper* newZip, + bool isHaveLocalHeaderData, + const std::vector& localHeaderOffsets, + const std::vector& localHeaderExtraLens, + const std::vector& localHeaderExtraPos, + const std::vector& localHeaderExtras){ std::vector headData; {//head data uint32_t backPairNew=~(uint32_t)0; @@ -419,8 +424,13 @@ static bool _serializeZipDiffData(std::vector& out_data,const ZipDiffData } {//type version - static const char* kVersionType="ZiPat1&"; - pushBack(out_data,(const TByte*)kVersionType,(const TByte*)kVersionType+strlen(kVersionType)); + if (isHaveLocalHeaderData){ + //new diff format, saved target local file header info (support apksigner v35) + pushBack(out_data,(const TByte*)kLocalHeaderTag,(const TByte*)kLocalHeaderTag+kLocalHeaderTagLen); + }else{ + static const char* kVersionType="ZiPat1&"; + pushBack(out_data,(const TByte*)kVersionType,(const TByte*)kVersionType+strlen(kVersionType)); + } } {//compressType const char* compressType=compressPlugin->compressType(); @@ -465,6 +475,19 @@ static bool _serializeZipDiffData(std::vector& out_data,const ZipDiffData headCode.clear(); pushBack(out_data,hdiffzData); + {//LocalHeaderData, saved target local file header info (support apksigner v35) + if (isHaveLocalHeaderData){ + for (size_t i=0;inewZipFileCount;++i){ + uint32_t offset=localHeaderOffsets[i]; + pushBack(out_data,(const TByte*)&offset,(const TByte*)&offset+4); + uint16_t extraLen=localHeaderExtraLens[i]; + pushBack(out_data,(const TByte*)&extraLen,(const TByte*)&extraLen+2); + const TByte* extraData=localHeaderExtras.data()+localHeaderExtraPos[i]; + pushBack(out_data,extraData,extraData+extraLen); + } + } + } + {//ExtraEdit pushBack(out_data,newZip->_cache_fvce,newZip->_centralDirectory); uint32_t extraSize=(uint32_t)(newZip->_centralDirectory-newZip->_cache_fvce); @@ -510,5 +533,42 @@ bool serializeZipDiffData(std::vector& out_data, UnZipper* newZip,UnZippe data.oldRefCount=oldRefList.size(); data.oldCrc=OldStream_getOldCrc(oldZip,oldRefList.data(),oldRefList.size()); data.normalizeSoPageAlign=UnZipper_getHugePageAlign(newZip,newZipAlignSize); - return _serializeZipDiffData(out_data,&data,hdiffzData,compressPlugin,newZip); + + //collect target local file header info (saved local header offsets & extra fields) + bool isHaveLocalHeaderData=false; + std::vector localHeaderOffsets; + std::vector localHeaderExtraLens; + std::vector localHeaderExtraPos; + std::vector localHeaderExtras; + if (UnZipper_isHaveApkV2Sign(newZip)){ + int fileCount=(int)data.newZipFileCount; + localHeaderOffsets.resize(fileCount); + localHeaderExtraLens.resize(fileCount); + localHeaderExtraPos.resize(fileCount); + localHeaderExtras.clear(); + size_t total=0; + for (int i=0;i0xffff)) return false; + localHeaderOffsets[i]=(uint32_t)entryPos; + localHeaderExtraLens[i]=(uint16_t)localExtraLen; + localHeaderExtraPos[i]=(uint32_t)total; + if ((int)localExtraLen!=cdExtraLen) isHaveLocalHeaderData=true; + localHeaderExtras.resize(total+(size_t)localExtraLen); + if (localExtraLen>0){ + check(UnZipper_fileData_read(newZip,entryPos+30+nameLen, + localHeaderExtras.data()+total, + localHeaderExtras.data()+total+localExtraLen)); + } + total+=(size_t)localExtraLen; + } + } + return _serializeZipDiffData(out_data,&data,hdiffzData,compressPlugin,newZip, + isHaveLocalHeaderData,localHeaderOffsets,localHeaderExtraLens, + localHeaderExtraPos,localHeaderExtras); } diff --git a/src/patch/Patcher.cpp b/src/patch/Patcher.cpp index 483fff8..a5e01cd 100644 --- a/src/patch/Patcher.cpp +++ b/src/patch/Patcher.cpp @@ -156,6 +156,11 @@ TPatchResult VirtualZipPatchWithStream(const hpatch_TStreamInput* oldZipStream,c check(Zipper_openStream(&out_newZip,outNewZipStream,(int)zipDiffData.newZipFileCount,(int)zipDiffData.newZipAlignSize, (int)zipDiffData.newCompressLevel,(int)zipDiffData.newCompressMemLevel, zipDiffData.normalizeSoPageAlign,zipDiffData.pageAlignCompatible,kPageAlign_inPatch),PATCH_OPENWRITE_ERROR); + if (zipDiffData._isHaveLocalHeaderData){ + //saved target local file header info, write local headers at their saved offset with saved extra field + Zipper_setLocalHeaderData(&out_newZip,zipDiffData._newLocalHeaderOffsets,zipDiffData._newLocalHeaderExtraLens, + zipDiffData._newLocalHeaderExtraPos,zipDiffData._newLocalHeaderExtras); + } check(NewStream_open(&newStream,&out_newZip,&oldZip, (size_t)diffInfo.newDataSize, zipDiffData.newZipIsDataNormalized!=0, zipDiffData.newZipCESize,zipDiffData.extraEdit, diff --git a/src/patch/ZipDiffData.cpp b/src/patch/ZipDiffData.cpp index 591f1bc..c098b66 100644 --- a/src/patch/ZipDiffData.cpp +++ b/src/patch/ZipDiffData.cpp @@ -34,6 +34,10 @@ void ZipDiffData_init(ZipDiffData* self){ } void ZipDiffData_close(ZipDiffData* self){ if (self->_buf) { free(self->_buf); self->_buf=0; } + if (self->_newLocalHeaderOffsets) { free(self->_newLocalHeaderOffsets); self->_newLocalHeaderOffsets=0; } + if (self->_newLocalHeaderExtraLens) { free(self->_newLocalHeaderExtraLens); self->_newLocalHeaderExtraLens=0; } + if (self->_newLocalHeaderExtraPos) { free(self->_newLocalHeaderExtraPos); self->_newLocalHeaderExtraPos=0; } + if (self->_newLocalHeaderExtras) { free(self->_newLocalHeaderExtras); self->_newLocalHeaderExtras=0; } } #define check(value) { \ @@ -49,12 +53,18 @@ void ZipDiffData_close(ZipDiffData* self){ check(v==(TUInt)v); \ *(result)=(TUInt)v; } +inline static uint16_t _readUInt16(const TByte* buf){ + return (uint16_t)(buf[0]|((uint16_t)buf[1]<<8)); +} + static bool _openZipDiffData(const hpatch_TStreamInput* diffData,hpatch_TDecompress* decompressPlugin, - size_t* out_headInfoPos=0){ - const char* kVersionType="ZiPat1&"; + size_t* out_headInfoPos=0,bool* out_isHaveLocalHeaderData=0){ + const char* kVersionType0="ZiPat1&"; + const char* kVersionType1="ZiPat2&"; //new diff format, saved target local file header info const size_t kVersionTypeLen=7; - assert(kVersionTypeLen==strlen(kVersionType)); + assert(kVersionTypeLen==strlen(kVersionType0)); + assert(kVersionTypeLen==strlen(kVersionType1)); TByte buf[kVersionTypeLen + hpatch_kMaxPluginTypeLength+1+1]; int readLen=sizeof(buf)-1; @@ -63,7 +73,10 @@ static bool _openZipDiffData(const hpatch_TStreamInput* diffData,hpatch_TDecompr readLen=(int)diffData->streamSize; check(diffData->read(diffData,0,buf,buf+readLen)); //check type+version - check(0==strncmp((const char*)buf,kVersionType,kVersionTypeLen)); + check((0==strncmp((const char*)buf,kVersionType0,kVersionTypeLen)) + ||(0==strncmp((const char*)buf,kVersionType1,kVersionTypeLen))); + if (out_isHaveLocalHeaderData) + *out_isHaveLocalHeaderData=(0==strncmp((const char*)buf,kVersionType1,kVersionTypeLen)); {//read compressType check(decompressPlugin!=0); const char* compressType=(const char*)buf+kVersionTypeLen; @@ -99,7 +112,7 @@ bool ZipDiffData_openRead(ZipDiffData* self,const hpatch_TStreamInput* diffData, size_t hdiffzSize=0; { size_t headInoPos=0; - check(_openZipDiffData(diffData,decompressPlugin,&headInoPos)); + check(_openZipDiffData(diffData,decompressPlugin,&headInoPos,&self->_isHaveLocalHeaderData)); //read head info TByte buf[hpatch_kMaxPackedUIntBytes*(16+3)]; int readLen=sizeof(buf); @@ -211,5 +224,45 @@ bool ZipDiffData_openRead(ZipDiffData* self,const hpatch_TStreamInput* diffData, TStreamInputClip_init(&self->_extraEdit,diffData,extraEditPos,extraEditPos+extraEditSize); self->extraEdit=&self->_extraEdit.base; } + if (self->_isHaveLocalHeaderData){ + //saved target local file header info, located between hdiffzData and ExtraEdit + const size_t localDataPos=self->_hdiffzData.clipBeginPos+hdiffzSize; + const size_t extraEditPos=(size_t)self->_extraEdit.clipBeginPos; + check(localDataPos<=extraEditPos); + const size_t localDataSize=extraEditPos-localDataPos; + TByte* block=(TByte*)malloc(localDataSize); + check(block!=0); + check(diffData->read(diffData,localDataPos,block,block+localDataSize)); + self->_newLocalHeaderOffsets=(uint32_t*)malloc(sizeof(uint32_t)*self->newZipFileCount); + self->_newLocalHeaderExtraLens=(uint16_t*)malloc(sizeof(uint16_t)*self->newZipFileCount); + self->_newLocalHeaderExtraPos=(uint32_t*)malloc(sizeof(uint32_t)*self->newZipFileCount); + check((self->_newLocalHeaderOffsets!=0)&&(self->_newLocalHeaderExtraLens!=0)&&(self->_newLocalHeaderExtraPos!=0)); + const TByte* cur=block; + const TByte* const blockEnd=block+localDataSize; + size_t totalExtraSize=0; + for (size_t i=0;inewZipFileCount;++i){ + check(cur+6<=blockEnd); + uint32_t offset=readUInt32(cur); + uint16_t extraLen=_readUInt16(cur+4); + cur+=6; + check(cur+extraLen<=blockEnd); + self->_newLocalHeaderOffsets[i]=offset; + self->_newLocalHeaderExtraLens[i]=extraLen; + self->_newLocalHeaderExtraPos[i]=(uint32_t)totalExtraSize; + totalExtraSize+=extraLen; + cur+=extraLen; + } + self->_newLocalHeaderExtras=(TByte*)malloc(totalExtraSize); + check(self->_newLocalHeaderExtras!=0); + cur=block; + for (size_t i=0;inewZipFileCount;++i){ + uint16_t extraLen=_readUInt16(cur+4); + cur+=6; + memcpy(self->_newLocalHeaderExtras+self->_newLocalHeaderExtraPos[i],cur,extraLen); + cur+=extraLen; + } + check(cur==blockEnd); + free(block); + } return true; } diff --git a/src/patch/ZipDiffData.h b/src/patch/ZipDiffData.h index fdd4117..8eb4c93 100644 --- a/src/patch/ZipDiffData.h +++ b/src/patch/ZipDiffData.h @@ -64,6 +64,12 @@ typedef struct ZipDiffData{ TByte* _buf; TStreamInputClip _hdiffzData; TStreamInputClip _extraEdit; + //new diff format (support apksigner v35): saved target local file header info + bool _isHaveLocalHeaderData; + uint32_t* _newLocalHeaderOffsets; //[newZipFileCount] + uint16_t* _newLocalHeaderExtraLens; //[newZipFileCount] + uint32_t* _newLocalHeaderExtraPos; //[newZipFileCount] + TByte* _newLocalHeaderExtras; //[sum of extra lens] } ZipDiffData; void ZipDiffData_init(ZipDiffData* self); @@ -74,8 +80,11 @@ void ZipDiffData_close(ZipDiffData* self); static const TByte kPackedEmptyPrefix=(1<<7)|0; static const char* kExtraEdit = "ZiPat1&Extra"; -#define kExtraEditLen 12 // ==strlen(kExtraEdit) -//zip diff out:[head+hdiffzData+ ExtraData +SizeOf(ExtraData)4Byte+kExtraEdit] +#define kExtraEditLen 12 //==strlen(kExtraEdit) +//new diff format tag, saved target local file header info (support apksigner v35) +static const char* kLocalHeaderTag = "ZiPat2&"; +#define kLocalHeaderTagLen 7 //==strlen(kLocalHeaderTag) +//zip diff out:[head+hdiffzData+ (LocalHeaderData) +ExtraData +SizeOf(ExtraData)4Byte+kExtraEdit] #ifdef __cplusplus } diff --git a/src/patch/Zipper.cpp b/src/patch/Zipper.cpp index 0dacebe..b6252c3 100644 --- a/src/patch/Zipper.cpp +++ b/src/patch/Zipper.cpp @@ -229,6 +229,15 @@ const char* UnZipper_file_nameBegin(const UnZipper* self,int fileIndex){ return (char*)headBuf+kMinFileHeaderSize; } +int UnZipper_file_extraFieldLen(const UnZipper* self,int fileIndex){ + const TByte* headBuf=fileHeaderBuf(self,fileIndex); + return readUInt16(headBuf+30); +} +const unsigned char* UnZipper_file_extraFieldBegin(const UnZipper* self,int fileIndex){ + const TByte* headBuf=fileHeaderBuf(self,fileIndex); + return headBuf+kMinFileHeaderSize+UnZipper_file_nameLen(self,fileIndex); +} + bool UnZipper_isHaveApkV1_or_jarSign(const UnZipper* self){ int fCount=UnZipper_fileCount(self); for (int i=fCount-1; i>=0; --i) { @@ -965,6 +974,14 @@ void Zipper_by_multi_thread(Zipper* self,int threadNum){ #endif } +void Zipper_setLocalHeaderData(Zipper* self,const uint32_t* offsets,const uint16_t* extraLens, + const uint32_t* extraPos,const TByte* extras){ + self->_localHeaderOffsets=(uint32_t*)offsets; + self->_localHeaderExtraLens=(uint16_t*)extraLens; + self->_localHeaderExtraPos=(uint32_t*)extraPos; + self->_localHeaderExtras=(TByte*)extras; +} + bool Zipper_openFile(Zipper* self,const char* zipFileName,int fileEntryMaxCount, int ZipAlignSize,int compressLevel,int compressMemLevel, size_t normalizeSoPageAlign,bool pageAlignCompatible,TPageAlignState pageAlignState){ @@ -1110,8 +1127,18 @@ static bool _write_fileHeaderInfo(Zipper* self,int fileIndex,UnZipper* srcZip,in extraFieldBufLen=_extraFieldNormalize(self->_normalizeExtraFieldBuf,extraFieldBuf,extraFieldLen); extraFieldBuf=self->_normalizeExtraFieldBuf; } + const bool isUseLocalHeaderData=(self->_localHeaderExtraLens!=0)&&(!isFullInfo); + if (isUseLocalHeaderData){ + //for new diff format: write the target local file header at its saved offset + const ZipFilePos_t targetPos=self->_localHeaderOffsets[fileIndex]; + check(targetPos>=self->_curFilePos); + if (targetPos>self->_curFilePos) + check(_writeAlignSkip(self,(size_t)(targetPos-self->_curFilePos))); + extraFieldBufLen=self->_localHeaderExtraLens[fileIndex]; + extraFieldBuf=self->_localHeaderExtras+self->_localHeaderExtraPos[fileIndex]; + } uint16_t extraFieldLen_for_align=extraFieldBufLen; - const bool isNeedAlign=(!isFullInfo)&&(!isCompressed); //dir or 0 size file need align too, same AndroidSDK#zipalign + const bool isNeedAlign=(!isFullInfo)&&(!isCompressed)&&(!isUseLocalHeaderData); //dir or 0 size file need align too, same AndroidSDK#zipalign size_t curAlignSize=self->_ZipAlignSize; bool curPageAlignCompatible=(self->_pageAlignCompatible)&&(self->_pageAlignState==kPageAlign_inNormalize); bool isOldPatchSo=false; @@ -1209,6 +1236,8 @@ static bool _write_fileHeaderInfo(Zipper* self,int fileIndex,UnZipper* srcZip,in check(_write(self,fileNameBuf+fileNameLen+extraFieldLen,fileCommentLen));//[+文件注释]; if (isNeedAlign) assert_align(self->_curFilePos,curAlignSize);//对齐检查; + else if (isUseLocalHeaderData) + assert(self->_curFilePos==(self->_localHeaderOffsets[fileIndex]+30+fileNameLen+extraFieldBufLen));//检查数据起始位置; return true; } @@ -1227,9 +1256,13 @@ size_t UnZipper_getHugePageAlign(const UnZipper* self,size_t baseZipAlignSize){ // if (!_file_getIsPageAlignSoFile(self,i)) isOld4k=false; const ZipFilePos_t entryPos=UnZipper_fileEntry_offset_unsafe(self,i); - const int headInfoLen=30+UnZipper_file_nameLen(self,i)+_file_extraFieldLen(self,i); + int headInfoLen=30+UnZipper_file_nameLen(self,i)+_file_extraFieldLen(self,i); const ZipFilePos_t dataPos_x=entryPos+headInfoLen; - if (dataPos_x!=dataPos) return 0; // not normalized + if (dataPos_x!=dataPos){ + //maybe CD extra field length != local header extra field length (signed by apksigner v35) + if (dataPos<=entryPos+30+UnZipper_file_nameLen(self,i)) return 0; // not normalized + headInfoLen=(int)(dataPos-entryPos); + } if (i==0) continue; const ZipFilePos_t lastEndPos=UnZipper_fileData_offset(self,i-1)+ diff --git a/src/patch/Zipper.h b/src/patch/Zipper.h index 3da8cd8..952eb89 100644 --- a/src/patch/Zipper.h +++ b/src/patch/Zipper.h @@ -72,6 +72,8 @@ bool UnZipper_close(UnZipper* self); int UnZipper_fileCount(const UnZipper* self); int UnZipper_file_nameLen(const UnZipper* self,int fileIndex); const char* UnZipper_file_nameBegin(const UnZipper* self,int fileIndex); +int UnZipper_file_extraFieldLen(const UnZipper* self,int fileIndex); //central directory extra field length +const unsigned char* UnZipper_file_extraFieldBegin(const UnZipper* self,int fileIndex); //central directory extra field data bool UnZipper_file_isCompressed(const UnZipper* self,int fileIndex); ZipFilePos_t UnZipper_file_uncompressedSize(const UnZipper* self,int fileIndex); ZipFilePos_t UnZipper_file_compressedSize(const UnZipper* self,int fileIndex); @@ -194,6 +196,12 @@ typedef struct Zipper{ bool _isUpdateIsCompress; bool _newIsCompress; + //saved target local file header info, only for patch (new diff format, support apksigner v35) + uint32_t* _localHeaderOffsets; //per entry, target local header offset + uint16_t* _localHeaderExtraLens;//per entry, target local header extra field length + uint32_t* _localHeaderExtraPos; //per entry, target local header extra field data position in _localHeaderExtras + TByte* _localHeaderExtras; //concatenated target local header extra field data + //thread int _threadNum; TZipThreadWorks* _threadWorks; @@ -205,6 +213,8 @@ bool Zipper_openFile(Zipper* self,const char* zipFileName,int fileEntryMaxCount, bool Zipper_openStream(Zipper* self,const hpatch_TStreamOutput* zipStream,int fileEntryMaxCount, int ZipAlignSize,int compressLevel,int compressMemLevel, size_t normalizeSoPageAlign,bool pageAlignCompatible,TPageAlignState pageAlignState); +void Zipper_setLocalHeaderData(Zipper* self,const uint32_t* offsets,const uint16_t* extraLens, + const uint32_t* extraPos,const TByte* extras); bool Zipper_close(Zipper* self); bool Zipper_file_append_copy(Zipper* self,UnZipper* srcZip,int srcFileIndex, bool isAlwaysReCompress=false); diff --git a/src/patch/patch_types.h b/src/patch/patch_types.h index 28f74e1..ba73a33 100644 --- a/src/patch/patch_types.h +++ b/src/patch/patch_types.h @@ -32,8 +32,8 @@ #include //uint32_t uint16_t #define APKDIFFPATCH_VERSION_MAJOR 1 -#define APKDIFFPATCH_VERSION_MINOR 8 -#define APKDIFFPATCH_VERSION_RELEASE 1 +#define APKDIFFPATCH_VERSION_MINOR 9 +#define APKDIFFPATCH_VERSION_RELEASE 0 #define _APKDIFFPATCH_VERSION APKDIFFPATCH_VERSION_MAJOR.APKDIFFPATCH_VERSION_MINOR.APKDIFFPATCH_VERSION_RELEASE #define _APKDIFFPATCH_QUOTE(str) #str