From efe63f4ad55621583009e84ba7b65ce85e193d2c Mon Sep 17 00:00:00 2001 From: youngslin <2305731910@qq.com> Date: Fri, 13 Nov 2026 08:04:34 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(maaend):=20=E7=94=9F=E6=88=90=E5=AE=98?= =?UTF-8?q?=E6=96=B9=E5=8C=85=E5=A2=9E=E9=87=8F=E5=B9=B6=E5=8F=91=E5=B8=83?= =?UTF-8?q?=E8=87=B3=E4=B8=8B=E8=BD=BD=E7=AB=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/maaend-delta.yml | 72 +++++++++++++ .gitignore | 2 + README.md | 34 +++++++ src/copilot_data/maaend_delta.py | 78 +++++++++++++++ src/copilot_data/publish.py | 2 +- src/copilot_data/zip_delta.py | 156 +++++++++++++++++++++++++++++ tests/test_maaend_delta.py | 55 ++++++++++ tests/test_snapshots.py | 1 + 8 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/maaend-delta.yml create mode 100644 src/copilot_data/maaend_delta.py create mode 100644 src/copilot_data/zip_delta.py create mode 100644 tests/test_maaend_delta.py diff --git a/.github/workflows/maaend-delta.yml b/.github/workflows/maaend-delta.yml new file mode 100644 index 0000000..1b19bdf --- /dev/null +++ b/.github/workflows/maaend-delta.yml @@ -0,0 +1,72 @@ +name: 生成 MaaEnd 增量包并上传下载站 + +on: + workflow_dispatch: + inputs: + base_tag: + description: 起始官方版本（如 v2.31.0） + required: true + type: string + target_tag: + description: 目标官方版本（如 v3.0.0） + required: true + type: string + arch: + description: 安装架构 + type: choice + default: x64 + options: [x64, arm64] + channel: + description: 下载站发布通道 + type: choice + default: stable + options: [stable, beta] + +permissions: + contents: read + +concurrency: + group: maaend-delta-${{ inputs.arch }}-${{ inputs.channel }} + cancel-in-progress: false + +jobs: + publish: + if: github.repository == 'AUTO-MAS-Project/AUTO-MAS-Copilot-Data' && github.ref == 'refs/heads/dev' + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: 下载官方包并生成、校验增量 + env: + BASE_TAG: ${{ inputs.base_tag }} + TARGET_TAG: ${{ inputs.target_tag }} + ARCH: ${{ inputs.arch == 'x64' && 'x86_64' || 'aarch64' }} + GITHUB_TOKEN: ${{ github.token }} + PYTHONPATH: src + run: >- + python -m copilot_data.maaend_delta --base-tag "$BASE_TAG" + --target-tag "$TARGET_TAG" --arch "$ARCH" --output .delta-output + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: maaend-${{ inputs.arch }}-${{ inputs.channel }}-delta + path: .delta-output/maaend.delta.zip + if-no-files-found: error + retention-days: 7 + - name: 安装发布工具依赖 + run: python -m pip install httpx==0.28.1 + - name: 上传独立增量文件并回读校验 + env: + AUTOMAS_TOKEN: ${{ secrets.AUTOMAS_MAAEND_DELTA_TOKEN }} + AUTOMAS_FILE_ID: ${{ vars[format('MAAEND_DELTA_{0}_{1}_FILE_ID', inputs.arch, inputs.channel)] }} + AUTOMAS_PUBLIC_URL: https://data.auto-mas.top/api/v1/files/auto-mas/Client/maaend-${{ inputs.arch }}-${{ inputs.channel }}-delta/download + PYTHONPATH: src + run: >- + python -c "import asyncio; from pathlib import Path; + from copilot_data.publish import publish; + asyncio.run(publish(Path('.delta-output/maaend.delta.zip')))" diff --git a/.gitignore b/.gitignore index 94eefeb..89dc891 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .venv/ +.dev/ +.delta-output/ __pycache__/ .ruff_cache/ .pytest_cache/ diff --git a/README.md b/README.md index bc0cb7d..955fa41 100644 --- a/README.md +++ b/README.md @@ -108,3 +108,37 @@ uv run --no-sync python -m copilot_data build ``` 许可证：AGPL-3.0-or-later。作业内容及其权利归原作者所有。 + +## MaaEnd 增量包 + +配合 AUTO-MAS 主程序 PR #1949，独立工作流 **生成 MaaEnd 增量包并上传下载站** +在本仓库的 `dev` 手动运行。输入起始和目标官方标签、架构（`x64` / `arm64`）以及通道 +（`stable` / `beta`）。它下载两版官方 Windows ZIP，分别核对 GitHub Release 公布的 +大小及 SHA256，复用旧 ZIP 中未变的压缩字节，生成增量并在发布前重建验证。 +不运行 MaaEnd，不解释其资源、任务或上游私有增量清单。 + +增量产物先保留为 7 天的 Actions artifact，再复用本仓库 `publish.py` 上传与公开回读校验。 +生成失败、增量不小于全量、超过当前发布工具的 32 MiB 上限、凭据缺失或未发布时， +工作流报错，不替换已发布文件；客户端遇到基线或目标不匹配就回退官方全量包。 +重新执行同样输入可重试生成与上传。 + +在下载站 AUTO-MAS → Client 创建独立文件，键为 +`maaend-{arch}-{channel}-delta`。例如 x64 稳定版公开地址必须是 +`https://data.auto-mas.top/api/v1/files/auto-mas/Client/maaend-x64-stable-delta/download`。 +不要使用作业或养成快照的文件 ID 和密钥。 + +| 类型 | 名称 | 含义 | +| --- | --- | --- | +| Secret | `AUTOMAS_MAAEND_DELTA_TOKEN` | 仅授权 MaaEnd 增量文件上传与自动发布的 Access Key | +| Variable | `MAAEND_DELTA_X64_STABLE_FILE_ID` | x64 稳定版文件数字 ID | +| Variable | `MAAEND_DELTA_ARM64_STABLE_FILE_ID` | arm64 稳定版文件数字 ID | +| Variable | `MAAEND_DELTA_X64_BETA_FILE_ID` | x64 公测版文件数字 ID | +| Variable | `MAAEND_DELTA_ARM64_BETA_FILE_ID` | arm64 公测版文件数字 ID | + +每个目标只发布最近一次指定的版本对。选择优先增量后，客户端保留上次经 MAS 成功安装、官方 SHA256 +校验通过的原始 ZIP；首次无缓存或落后于已发布版本对时使用全量，并建立新缓存。 +客户端从 GitHub 独立读取目标整包大小与 SHA256，站点清单不能改写校验依据。 +完整 ZIP 重建校验后仍交给 MAS 现有的可恢复安装器，包含原有用户文件保护和安装回执。 + +字节协议为 v1，`zip_delta.py` 与主程序同名模块共同维护；修改协议时必须同时验证 +生成端与客户端，不能只用生成端自己的重建结果证明客户端兼容。 diff --git a/src/copilot_data/maaend_delta.py b/src/copilot_data/maaend_delta.py new file mode 100644 index 0000000..25a74a1 --- /dev/null +++ b/src/copilot_data/maaend_delta.py @@ -0,0 +1,78 @@ +"""下载官方两版 ZIP、生成字节增量并独立重建验证；不运行 MaaEnd。""" + +import argparse +import hashlib +import json +import os +import re +import threading +from pathlib import Path +from urllib.parse import quote +from urllib.request import Request, urlopen + + +def download_release(tag: str, arch: str, destination: Path) -> tuple[int, str]: + if not re.fullmatch(r"v?\d+\.\d+\.\d+(?:-(?:beta|rc)\.\d+)?", tag): + raise ValueError("只接受正式版或 beta 官方版本标签") + headers = {"Accept": "application/vnd.github+json", "User-Agent": "AUTO-MAS-delta"} + if token := os.environ.get("GITHUB_TOKEN"): + headers["Authorization"] = f"Bearer {token}" + url = f"https://api.github.com/repos/MaaEnd/MaaEnd/releases/tags/{quote(tag, safe='')}" + with urlopen(Request(url, headers=headers), timeout=30) as response: + release = json.load(response) + asset = next( + item + for item in release["assets"] + if item["name"] == f"MaaEnd-win-{arch}-{tag}.zip" + ) + size, digest = asset["size"], str(asset.get("digest", "")) + if not 0 < size < 1024**3 or not re.fullmatch(r"sha256:[a-f0-9]{64}", digest): + raise ValueError("官方发行包缺少有效大小或 SHA256") + if not asset["browser_download_url"].startswith( + "https://github.com/MaaEnd/MaaEnd/releases/download/" + ): + raise ValueError("发行包下载地址无效") + checksum, copied = hashlib.sha256(), 0 + with urlopen(asset["browser_download_url"], timeout=120) as response: + with destination.open("xb") as output: + while chunk := response.read(1024 * 1024): + copied += len(chunk) + if copied > size: + raise ValueError("发行包下载超出官方大小") + checksum.update(chunk) + output.write(chunk) + if copied != size or checksum.hexdigest() != digest[7:]: + raise ValueError("发行包未通过官方 SHA256 校验") + return size, digest[7:] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-tag", required=True) + parser.add_argument("--target-tag", required=True) + parser.add_argument("--arch", choices=["x86_64", "aarch64"], required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.base_tag == args.target_tag: + raise ValueError("起始和目标版本不能相同") + from .zip_delta import apply_patch, create_patch + + args.output.mkdir(parents=True, exist_ok=True) + baseline, target, rebuilt = ( + args.output / name for name in ("base.zip", "target.zip", "rebuilt.zip") + ) + download_release(args.base_tag, args.arch, baseline) + size, digest = download_release(args.target_tag, args.arch, target) + patch = args.output / "maaend.delta.zip" + create_patch(baseline, target, patch) + apply_patch(baseline, patch, rebuilt, size, digest, threading.Event()) + print( + f"{args.base_tag} -> {args.target_tag} ({args.arch}): " + f"增量 {patch.stat().st_size} / 全量 {size} 字节，重建 SHA256 校验通过" + ) + for path in (baseline, target, rebuilt): + path.unlink() + + +if __name__ == "__main__": + main() diff --git a/src/copilot_data/publish.py b/src/copilot_data/publish.py index 5e83048..c145bbc 100644 --- a/src/copilot_data/publish.py +++ b/src/copilot_data/publish.py @@ -87,7 +87,7 @@ def upload( "--config", "-", "--form", - f"file=@{source};type=application/gzip", + f"file=@{source};type={'application/zip' if filename.endswith('.zip') else 'application/gzip'}", "--form-string", f"change_note=GitHub Actions run {os.environ.get('GITHUB_RUN_ID', 'manual')}", "--cookie", diff --git a/src/copilot_data/zip_delta.py b/src/copilot_data/zip_delta.py new file mode 100644 index 0000000..41b1256 --- /dev/null +++ b/src/copilot_data/zip_delta.py @@ -0,0 +1,156 @@ +"""从旧 ZIP 复用压缩数据，重建远端 ZIP 的原始字节；调用方必须校验整包摘要。""" + +import hashlib +import json +import struct +import threading +import zipfile +from pathlib import Path +from typing import BinaryIO + +MAX_PATCH_BYTES = 32 * 1024**2 + + +def file_digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def create_patch(baseline: Path, target: Path, patch: Path) -> None: + """生成字节重建包，不包含上游私有增量清单或安装语义。""" + operations = [] + with baseline.open("rb") as old, target.open("rb") as new: + with zipfile.ZipFile(old) as old_zip, zipfile.ZipFile(new) as new_zip: + previous = {item.filename: item for item in old_zip.infolist()} + cursor = 0 + literals = [] + for item in sorted(new_zip.infolist(), key=lambda item: item.header_offset): + candidate = previous.get(item.filename) + start = _data_offset(new, item) + if start < cursor: + raise ValueError("ZIP 区间冲突") + if start > cursor: + literals.append((cursor, start - cursor)) + reusable = False + if ( + candidate is not None + and candidate.compress_size == item.compress_size + ): + old_start = _data_offset(old, candidate) + old.seek(old_start) + new.seek(start) + remaining = item.compress_size + reusable = True + while remaining: + length = min(remaining, 1024 * 1024) + if old.read(length) != new.read(length): + reusable = False + break + remaining -= length + if reusable and item.compress_size: + for offset, length in literals: + operations.append(["new", offset, length]) + literals.clear() + operations.append(["old", old_start, item.compress_size]) + elif item.compress_size: + literals.append((start, item.compress_size)) + cursor = start + item.compress_size + literals.append((cursor, target.stat().st_size - cursor)) + operations.extend( + ["new", offset, length] for offset, length in literals if length + ) + manifest = { + "version": 1, + "base_sha256": file_digest(baseline), + "target_sha256": file_digest(target), + "target_size": target.stat().st_size, + "operations": [], + } + with zipfile.ZipFile(patch, "w", compression=zipfile.ZIP_DEFLATED) as output: + with output.open("payload", "w", force_zip64=True) as payload: + position = 0 + for kind, offset, length in operations: + if kind == "new": + new.seek(offset) + remaining = length + while remaining: + chunk = new.read(min(remaining, 1024 * 1024)) + if not chunk: + raise ValueError("ZIP 数据不完整") + payload.write(chunk) + remaining -= len(chunk) + offset = position + position += length + manifest["operations"].append([kind, offset, length]) + info = zipfile.ZipInfo("manifest.json") + info.compress_type = zipfile.ZIP_DEFLATED + output.writestr(info, json.dumps(manifest)) + if patch.stat().st_size >= min(MAX_PATCH_BYTES, target.stat().st_size): + patch.unlink() + raise ValueError("增量包未节省流量或超过发布限制，使用全量") + + +def apply_patch( + baseline: Path, + patch: Path, + destination: Path, + size: int, + digest: str, + cancelled: threading.Event, +) -> None: + """重建归档，独立官方大小与 SHA256 校验通过才允许调用方安装。""" + if patch.stat().st_size > MAX_PATCH_BYTES: + raise ValueError("增量包过大") + with zipfile.ZipFile(patch) as archive: + info = archive.getinfo("manifest.json") + if info.file_size > 16 * 1024**2: + raise ValueError("增量清单过大") + metadata = json.loads(archive.read(info)) + if ( + metadata.get("version") != 1 + or metadata.get("target_size") != size + or metadata.get("target_sha256") != digest + or metadata.get("base_sha256") != file_digest(baseline) + ): + raise ValueError("增量包与官方版本或本地基线不一致") + operations = metadata["operations"] + if len(operations) > 300_000 or archive.getinfo("payload").file_size > size: + raise ValueError("增量包展开超限") + with baseline.open("rb") as old, archive.open("payload") as payload: + with destination.open("xb") as output: + written = 0 + for kind, offset, length in operations: + if ( + kind not in {"old", "new"} + or not isinstance(offset, int) + or not isinstance(length, int) + or offset < 0 + or length < 0 + or written + length > size + ): + raise ValueError("增量重建区间无效") + source = old if kind == "old" else payload + if kind == "new" and offset != payload.tell(): + raise ValueError("增量数据顺序无效") + source.seek(offset) + remaining = length + while remaining: + if cancelled.is_set(): + raise InterruptedError("增量重建已取消") + chunk = source.read(min(remaining, 1024 * 1024)) + if not chunk: + raise ValueError("增量数据不完整") + output.write(chunk) + remaining -= len(chunk) + written += length + if destination.stat().st_size != size or file_digest(destination) != digest: + raise ValueError("重建包未通过官方 SHA256 校验") + + +def _data_offset(stream: BinaryIO, item: zipfile.ZipInfo) -> int: + stream.seek(item.header_offset) + header = stream.read(30) + if len(header) != 30 or header[:4] != b"PK\x03\x04": + raise ValueError("ZIP 文件头无效") + name_size, extra_size = struct.unpack_from(" Date: Fri, 13 Nov 2026 21:31:39 +0800 Subject: [PATCH 2/2] =?UTF-8?q?docs(maaend):=20=E7=BB=9F=E4=B8=80=E5=A2=9E?= =?UTF-8?q?=E9=87=8F=E6=9B=B4=E6=96=B0=E9=80=89=E9=A1=B9=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 955fa41..dd17264 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ uv run --no-sync python -m copilot_data build | Variable | `MAAEND_DELTA_X64_BETA_FILE_ID` | x64 公测版文件数字 ID | | Variable | `MAAEND_DELTA_ARM64_BETA_FILE_ID` | arm64 公测版文件数字 ID | -每个目标只发布最近一次指定的版本对。选择优先增量后，客户端保留上次经 MAS 成功安装、官方 SHA256 +每个目标只发布最近一次指定的版本对。选择增量更新后，客户端保留上次经 MAS 成功安装、官方 SHA256 校验通过的原始 ZIP；首次无缓存或落后于已发布版本对时使用全量，并建立新缓存。 客户端从 GitHub 独立读取目标整包大小与 SHA256，站点清单不能改写校验依据。 完整 ZIP 重建校验后仍交给 MAS 现有的可恢复安装器，包含原有用户文件保护和安装回执。