diff --git a/.github/actions/setup-flutter/action.yml b/.github/actions/setup-flutter/action.yml index 7e9da9dc..e3fa43d4 100644 --- a/.github/actions/setup-flutter/action.yml +++ b/.github/actions/setup-flutter/action.yml @@ -10,7 +10,7 @@ runs: using: composite steps: - name: Install Flutter - uses: subosito/flutter-action@v2 + uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 with: channel: stable flutter-version: ${{ inputs.flutter-version }} diff --git a/.github/actions/setup-linux-dependencies/action.yml b/.github/actions/setup-linux-dependencies/action.yml index 52ca85d0..260f55e3 100644 --- a/.github/actions/setup-linux-dependencies/action.yml +++ b/.github/actions/setup-linux-dependencies/action.yml @@ -10,10 +10,6 @@ inputs: description: Install GTK and Xvfb for desktop integration tests. required: false default: 'false' - coverage: - description: Install lcov and bc for coverage gates. - required: false - default: 'false' arm64: description: Install the AArch64 cross compiler used by release artifacts. required: false @@ -26,7 +22,6 @@ runs: shell: bash env: DESKTOP: ${{ inputs.desktop }} - COVERAGE: ${{ inputs.coverage }} ANDROID: ${{ inputs.android }} ARM64: ${{ inputs.arm64 }} run: | @@ -86,9 +81,6 @@ runs: if [[ "$DESKTOP" == 'true' ]]; then packages+=(libgtk-3-dev xvfb) fi - if [[ "$COVERAGE" == 'true' ]]; then - packages+=(bc lcov) - fi if [[ "$ARM64" == 'true' ]]; then packages+=(gcc-aarch64-linux-gnu g++-aarch64-linux-gnu) fi diff --git a/.github/scripts/update_versions.dart b/.github/scripts/update_versions.dart index 0c3343ba..f2e1abf8 100644 --- a/.github/scripts/update_versions.dart +++ b/.github/scripts/update_versions.dart @@ -4,9 +4,8 @@ /// Example: dart run .github/scripts/update_versions.dart 1.3.30 --release /// /// The optional --release flag also pins the Apple SwiftPM manifests and -/// updates the CocoaPods podspec versions. Development checkouts may keep -/// using the develop branch; published archives must contain an immutable -/// dependency declaration. +/// updates the CocoaPods podspec versions. Apple dependencies remain pinned +/// to the repository tag carrying the same release. import 'dart:io'; @@ -42,6 +41,8 @@ void main(List args) { print('Updated $path'); } + _updateCliVersion(repoRoot, version); + if (release) { _updateReleaseMetadata(repoRoot, version); } @@ -63,12 +64,29 @@ void main(List args) { } } + _verifyCliVersion(repoRoot, version); + if (release) { _verifyReleaseMetadata(repoRoot, version); } print('All versions updated and verified.'); } +void _updateCliVersion(String root, String version) { + final cliRunner = File('$root/zstandard_cli/lib/src/cli_runner.dart'); + if (!cliRunner.existsSync()) { + print('Error: CLI runner not found: ${cliRunner.path}'); + exit(1); + } + cliRunner.writeAsStringSync( + cliRunner.readAsStringSync().replaceFirst( + RegExp(r"const String zstandardCliVersion = '[^']+';"), + "const String zstandardCliVersion = '$version';", + ), + ); + print('Updated CLI --version to $version'); +} + void _updateReleaseMetadata(String root, String version) { final manifests = [ '$root/zstandard_ios/ios/zstandard_ios/Package.swift', @@ -82,7 +100,7 @@ void _updateReleaseMetadata(String root, String version) { } var content = file.readAsStringSync(); content = content.replaceFirstMapped( - RegExp(r'branch:\s*"develop"'), + RegExp(r'(?:branch:\s*"develop"|exact:\s*"\d+\.\d+\.\d+")'), (_) => 'exact: "$version"', ); file.writeAsStringSync(content); @@ -109,6 +127,15 @@ void _updateReleaseMetadata(String root, String version) { } } +void _verifyCliVersion(String root, String version) { + final cliRunner = File('$root/zstandard_cli/lib/src/cli_runner.dart') + .readAsStringSync(); + if (!cliRunner.contains("const String zstandardCliVersion = '$version';")) { + print('Error: CLI --version is not $version'); + exit(1); + } +} + void _verifyReleaseMetadata(String root, String version) { final manifests = [ '$root/zstandard_ios/ios/zstandard_ios/Package.swift', @@ -154,12 +181,12 @@ String _updateVersionInContent( String version, ) { content = content.replaceFirst( - RegExp(r'^version:\s*[\d.]+\s*$', multiLine: true), - 'version: $version\n', + RegExp(r'^version:[ \t]*[\d.]+[ \t]*$', multiLine: true), + 'version: $version', ); for (final dep in spec.deps) { content = content.replaceFirstMapped( - RegExp(r'(\s*' + dep + r':\s*)\^?[\d.]+'), + RegExp(r'^([ \t]*' + dep + r':[ \t]*)\^?[\d.]+[ \t]*$', multiLine: true), (m) => '${m[1]}^$version', ); } diff --git a/.github/workflows/copilot_commit.yml b/.github/workflows/copilot_commit.yml deleted file mode 100644 index 2b5387aa..00000000 --- a/.github/workflows/copilot_commit.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Copilot - Commit - -on: - push: - branches: - - '**' - - '!master' - - '!develop' - -jobs: - copilot-commits: - name: Copilot - Commit - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - uses: vypdev/copilot@v2 - with: - debug: ${{ vars.DEBUG }} - project-ids: ${{ vars.PROJECT_IDS }} - opencode-model: ${{ vars.OPENCODE_MODEL }} - token: ${{ secrets.PAT }} diff --git a/.github/workflows/copilot_issue.yml b/.github/workflows/copilot_issue.yml index 4c1b9b41..79c3f71b 100644 --- a/.github/workflows/copilot_issue.yml +++ b/.github/workflows/copilot_issue.yml @@ -4,15 +4,19 @@ on: issues: types: [opened, reopened, edited, labeled, unlabeled, assigned, unassigned] +permissions: + contents: read + jobs: copilot-issues: name: Copilot - Issue + if: contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.issue.author_association) runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - - uses: vypdev/copilot@v2 + - uses: vypdev/copilot@274db4058088d0b2e93c807d27e24c160162d54a with: ai-ignore-files: build/* debug: ${{ vars.DEBUG }} diff --git a/.github/workflows/copilot_issue_comment.yml b/.github/workflows/copilot_issue_comment.yml index 0c23c361..2abb5a79 100644 --- a/.github/workflows/copilot_issue_comment.yml +++ b/.github/workflows/copilot_issue_comment.yml @@ -7,14 +7,15 @@ on: jobs: copilot-issues: name: Copilot - Issue Comment + if: contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) runs-on: ubuntu-latest permissions: - contents: write + contents: read steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - - uses: vypdev/copilot@v2 + - uses: vypdev/copilot@274db4058088d0b2e93c807d27e24c160162d54a with: ai-ignore-files: build/* debug: ${{ vars.DEBUG }} diff --git a/.github/workflows/copilot_pull_request.yml b/.github/workflows/copilot_pull_request.yml index ccf1288d..95869db7 100644 --- a/.github/workflows/copilot_pull_request.yml +++ b/.github/workflows/copilot_pull_request.yml @@ -1,18 +1,21 @@ name: Copilot - Pull Request on: - pull_request: + pull_request_target: types: [opened, reopened, edited, labeled, unlabeled, closed, assigned, unassigned, synchronize] jobs: copilot-pull-requests: name: Copilot - Pull Request + if: contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association) runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - - uses: vypdev/copilot@v2 + - uses: vypdev/copilot@274db4058088d0b2e93c807d27e24c160162d54a with: ai-ignore-files: build/* debug: ${{ vars.DEBUG }} diff --git a/.github/workflows/copilot_pull_request_comment.yml b/.github/workflows/copilot_pull_request_comment.yml index a521f9e7..6c76873f 100644 --- a/.github/workflows/copilot_pull_request_comment.yml +++ b/.github/workflows/copilot_pull_request_comment.yml @@ -7,14 +7,15 @@ on: jobs: copilot-pull-requests: name: Copilot - Pull Request Comment + if: contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) runs-on: ubuntu-latest permissions: - contents: write + contents: read steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - - uses: vypdev/copilot@v2 + - uses: vypdev/copilot@274db4058088d0b2e93c807d27e24c160162d54a with: ai-ignore-files: build/* debug: ${{ vars.DEBUG }} @@ -22,4 +23,3 @@ jobs: project-ids: ${{ vars.PROJECT_IDS }} token: ${{ secrets.PAT }} bugbot-fix-verify-commands: ${{ vars.BUGBOT_AUTOFIX_VERIFY_COMMANDS }} - \ No newline at end of file diff --git a/.github/workflows/hotfix_workflow.yml b/.github/workflows/hotfix_workflow.yml deleted file mode 100644 index e631b923..00000000 --- a/.github/workflows/hotfix_workflow.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: Task - Hotfix - -on: - workflow_dispatch: - inputs: - version: - description: 'Hotfix version' - required: true - default: '1.0.0' - title: - description: 'Title' - required: true - default: 'New Version' - changelog: - description: 'Changelog' - required: true - default: '- Several improvements' - issue: - description: 'Launcher issue' - required: true - default: '-1' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - prepare-version-files: - name: Prepare files for hotfix - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Validate inputs - env: - VERSION: ${{ github.event.inputs.version }} - ISSUE: ${{ github.event.inputs.issue }} - TITLE: ${{ github.event.inputs.title }} - CHANGELOG: ${{ github.event.inputs.changelog }} - run: | - err=0 - if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "::error::Version must be in semver format (e.g. 1.0.0)." - err=1 - fi - if ! [[ "$ISSUE" =~ ^-?[0-9]+$ ]]; then - echo "::error::Issue must be a number (e.g. 123 or -1)." - err=1 - fi - if [[ ${#TITLE} -gt 1000 ]]; then - echo "::error::Title must be at most 1000 characters." - err=1 - fi - if [[ ${#CHANGELOG} -gt 50000 ]]; then - echo "::error::Changelog must be at most 50000 characters." - err=1 - fi - [[ $err -eq 0 ]] || exit 1 - - # Example: generic step to perform the version update or compilation (uncomment and adjust as needed) - # - name: Generic step - # uses: whatever/action@v2 - - - name: Commit updated package.json and dist directory - uses: EndBug/add-and-commit@v9 - with: - add: './build/ ./package.json' - committer_name: GitHub Actions - committer_email: actions@github.com - default_author: user_info - message: 'gh-action: updated compiled files and bumped version to ${{ github.event.inputs.version }} (hotfix)' - - tag: - name: Publish version - runs-on: ubuntu-latest - needs: [ prepare-version-files ] - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Copilot - Create Tag - uses: vypdev/copilot@v2 - if: ${{ success() }} - with: - debug: ${{ vars.DEBUG }} - single-action: 'create_tag' - single-action-issue: '${{ github.event.inputs.issue }}' - single-action-version: '${{ github.event.inputs.version }}' - token: ${{ secrets.PAT }} - - - name: Copilot - Create Release - uses: vypdev/copilot@v2 - if: ${{ success() }} - with: - debug: ${{ vars.DEBUG }} - single-action: 'create_release' - single-action-issue: '${{ github.event.inputs.issue }}' - single-action-version: '${{ github.event.inputs.version }}' - single-action-title: '${{ github.event.inputs.title }}' - single-action-changelog: '${{ github.event.inputs.changelog }}' - token: ${{ secrets.PAT }} - - # Example: generic step to perform the deployment (uncomment and adjust as needed) - # - name: Generic step - # uses: whatever/action@v2 - - - name: Copilot - Deploy success notification - uses: vypdev/copilot@v2 - if: ${{ success() }} - with: - debug: ${{ vars.DEBUG }} - single-action: 'deployed_action' - single-action-issue: '${{ github.event.inputs.issue }}' - opencode-model: ${{ vars.OPENCODE_MODEL }} - token: ${{ secrets.PAT }} diff --git a/.github/workflows/integrate_release.yml b/.github/workflows/integrate_release.yml new file mode 100644 index 00000000..d3805640 --- /dev/null +++ b/.github/workflows/integrate_release.yml @@ -0,0 +1,126 @@ +name: Integrate Published Release + +on: + workflow_dispatch: + inputs: + version: + description: 'Published stable version without the v prefix' + required: true + type: string + +concurrency: + group: integrate-release-${{ inputs.version }} + cancel-in-progress: false + +permissions: + contents: read + pull-requests: write + +jobs: + open_integration_pull_requests: + name: Open reviewed integration pull requests + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + fetch-depth: 0 + + - name: Validate the completed release + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Version must be stable SemVer without a v prefix: $VERSION" >&2 + exit 1 + fi + git fetch --no-tags origin \ + "+refs/heads/release/$VERSION:refs/remotes/origin/release/$VERSION" \ + "+refs/heads/master:refs/remotes/origin/master" \ + "+refs/heads/develop:refs/remotes/origin/develop" + git fetch --no-tags origin \ + "+refs/tags/v$VERSION:refs/tags/v$VERSION" + + release_commit="$(git rev-parse "origin/release/$VERSION")" + tag_commit="$(git rev-list -n 1 "refs/tags/v$VERSION")" + if [[ "$release_commit" != "$tag_commit" ]]; then + echo "v$VERSION is not the tip of release/$VERSION." >&2 + exit 1 + fi + + if [[ "$(gh api \ + "repos/$GITHUB_REPOSITORY/releases/tags/v$VERSION" \ + --jq '(.draft == false) and (.prerelease == false)')" != true ]]; then + echo "GitHub Release v$VERSION is missing, draft, or prerelease." >&2 + exit 1 + fi + + for package in \ + zstandard_platform_interface zstandard_native zstandard_android \ + zstandard_ios zstandard_linux zstandard_macos zstandard_web \ + zstandard_windows zstandard_cli zstandard; do + if ! curl --fail --silent --show-error --retry 3 \ + "https://pub.dev/api/packages/$package" \ + | jq --exit-status --arg version "$VERSION" \ + '.versions | any(.version == $version)' >/dev/null; then + echo "$package $VERSION is not visible on pub.dev." >&2 + exit 1 + fi + done + + - name: Open or reuse integration pull requests + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + owner="$GITHUB_REPOSITORY_OWNER" + head="release/$VERSION" + + for base in master develop; do + if git merge-base --is-ancestor \ + "origin/release/$VERSION" "origin/$base"; then + echo "release/$VERSION is already integrated into $base." + continue + fi + + existing="$(gh pr list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --base "$base" \ + --head "$owner:$head" \ + --json number \ + --jq '.[0].number // empty')" + if [[ -n "$existing" ]]; then + echo "Reusing pull request #$existing for $base." + continue + fi + + gh pr create \ + --repo "$GITHUB_REPOSITORY" \ + --base "$base" \ + --head "$head" \ + --title "Integrate release v$VERSION into $base" \ + --body "Release v$VERSION has been published and independently verified. Merge this pull request through the normal review and required-check policy. This workflow never merges or bypasses branch protection. Keep release/$VERSION until both integration pull requests are merged." + done + + - name: Summarize integration state + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + { + echo "## Release v$VERSION integration" + echo + echo 'Published artifacts were verified before opening pull requests.' + echo + gh pr list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --head "$GITHUB_REPOSITORY_OWNER:release/$VERSION" \ + --json number,baseRefName,url \ + --template '{{range .}}- [#{{.number}} into {{.baseRefName}}]({{.url}}){{"\n"}}{{end}}' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pr_check_android.yml b/.github/workflows/pr_check_android.yml index 57831e2d..634ab72b 100644 --- a/.github/workflows/pr_check_android.yml +++ b/.github/workflows/pr_check_android.yml @@ -1,13 +1,10 @@ name: PR Check Android on: - pull_request: - branches: - - '**' - - '!master' push: branches: - develop + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -28,12 +25,15 @@ jobs: name: Analyze Android runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_android + - name: Get dependencies working-directory: zstandard_android run: flutter pub get @@ -72,9 +72,9 @@ jobs: max-parallel: 1 matrix: example: [example, example_legacy] - source: [workspace, pub-cache] + source: [workspace, package-config] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-linux-dependencies with: @@ -84,9 +84,20 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Remove zstandard_native to force resolution from pub cache - if: matrix.source == 'pub-cache' - run: mv zstandard_native "$RUNNER_TEMP/zstandard_native" + - name: Stage native package outside the repository + if: matrix.source == 'package-config' + run: | + mkdir -p "$RUNNER_TEMP/zstandard-stage" + mv zstandard_native "$RUNNER_TEMP/zstandard-stage/zstandard_native" + + - name: Resolve the selected native source exactly + run: | + if [[ "${{ matrix.source }}" == package-config ]]; then + dart scripts/create_local_overrides.dart zstandard_android \ + "$RUNNER_TEMP/zstandard-stage/zstandard_native" + else + dart scripts/create_local_overrides.dart zstandard_android + fi - name: Get dependencies working-directory: zstandard_android/${{ matrix.example }} @@ -98,7 +109,7 @@ jobs: flutter clean # The plugin CMake cache is kept outside Flutter's build directory # on persistent self-hosted runners. Remove it so workspace and - # pub-cache matrix entries cannot reuse a previous native source. + # package-config matrix entries cannot reuse a previous native source. rm -rf android/.cxx flutter pub get @@ -109,13 +120,13 @@ jobs: flutter pub get - name: Set up JDK 17 (required by Android SDK tools and Gradle) - uses: actions/setup-java@v5 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 with: distribution: 'temurin' java-version: '17' - name: Set up Android SDK tools - uses: android-actions/setup-android@v4 + uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 - name: Verify Android SDK prerequisites run: bash scripts/check_android_ci_prerequisites.sh @@ -176,7 +187,7 @@ jobs: - name: Upload Android emulator diagnostics if: always() - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 with: name: android-emulator-logs-${{ matrix.example }}-${{ matrix.source }}-${{ github.run_id }} path: | @@ -185,17 +196,17 @@ jobs: if-no-files-found: ignore - name: Restore workspace native package - if: always() && matrix.source == 'pub-cache' + if: always() && matrix.source == 'package-config' run: | - if [[ -d "$RUNNER_TEMP/zstandard_native" ]]; then - mv "$RUNNER_TEMP/zstandard_native" zstandard_native + if [[ -d "$RUNNER_TEMP/zstandard-stage/zstandard_native" ]]; then + mv "$RUNNER_TEMP/zstandard-stage/zstandard_native" zstandard_native fi check_android_publish_dry_run: name: Dry Run Publish Android runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: @@ -206,6 +217,9 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_android + - name: Get dependencies working-directory: zstandard_android run: flutter pub get diff --git a/.github/workflows/pr_check_cli.yml b/.github/workflows/pr_check_cli.yml index af41b730..9282075c 100644 --- a/.github/workflows/pr_check_cli.yml +++ b/.github/workflows/pr_check_cli.yml @@ -1,13 +1,10 @@ name: PR Check CLI on: - pull_request: - branches: - - '**' - - '!master' push: branches: - develop + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,12 +20,15 @@ jobs: name: Analyze CLI runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_cli + - name: Get dependencies working-directory: zstandard_cli run: dart pub get @@ -42,16 +42,17 @@ jobs: runs-on: [self-hosted, Linux] timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-linux-dependencies - with: - coverage: 'true' - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_cli + - name: Get dependencies working-directory: zstandard_cli run: dart pub get @@ -62,25 +63,23 @@ jobs: - name: Format coverage to lcov working-directory: zstandard_cli - run: dart run coverage:format_coverage --lcov -i coverage -o coverage/lcov.info --packages=.dart_tool/package_config.json + run: >- + dart run coverage:format_coverage --lcov -i coverage + -o coverage/lcov.info --packages=.dart_tool/package_config.json + --report-on=lib - # Coverage threshold: lib/ only, excluding generated bindings - - name: Check coverage threshold (79% of lib/) + # Coverage threshold: this package's lib/ only. + - name: Check coverage threshold (100% of lib/) working-directory: zstandard_cli - run: | - lcov --extract coverage/lcov.info '*zstandard_cli/lib/*' -o coverage/lcov_lib.info - lcov --remove coverage/lcov_lib.info '*bindings_generated*' -o coverage/lcov_lib.info --ignore-errors unused - COVERAGE=$(lcov --summary coverage/lcov_lib.info 2>&1 | grep "lines" | awk '{print $2}' | sed 's/%//') - echo "Coverage (lib/ excluding generated): ${COVERAGE}%" - if [ -z "$COVERAGE" ]; then echo "Could not parse coverage"; exit 1; fi - if (( $(echo "$COVERAGE < 79" | bc -l) )); then echo "ERROR: Coverage ${COVERAGE}% is below 79%"; exit 1; fi + run: dart ../scripts/check_lcov_coverage.dart coverage/lcov.info 100 - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f with: token: ${{ secrets.CODECOV_TOKEN }} files: zstandard_cli/coverage/lcov.info flags: cli + disable_search: true fail_ci_if_error: true - name: Run benchmarks (informational) @@ -103,12 +102,15 @@ jobs: name: Dry Run Publish CLI runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_cli + - name: Get dependencies working-directory: zstandard_cli run: dart pub get diff --git a/.github/workflows/pr_check_ios.yml b/.github/workflows/pr_check_ios.yml index 13be3594..86d319e1 100644 --- a/.github/workflows/pr_check_ios.yml +++ b/.github/workflows/pr_check_ios.yml @@ -1,13 +1,10 @@ name: PR Check iOS on: - pull_request: - branches: - - '**' - - '!master' push: branches: - develop + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,7 +20,7 @@ jobs: runs-on: [self-hosted, macOS, ARM64] timeout-minutes: 45 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: @@ -34,6 +31,9 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_ios + - name: Get dependencies working-directory: zstandard_ios run: flutter pub get @@ -60,9 +60,9 @@ jobs: max-parallel: 1 matrix: dependency_manager: [swiftpm, cocoapods] - source: [workspace, pub-cache] + source: [workspace, package-config] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: @@ -85,10 +85,24 @@ jobs: if: matrix.dependency_manager == 'swiftpm' && matrix.source == 'workspace' run: echo "ZSTANDARD_NATIVE_PACKAGE_PATH=$GITHUB_WORKSPACE" >> "$GITHUB_ENV" - - name: Hide workspace native package to force Pub cache resolution - if: matrix.source == 'pub-cache' + - name: Stage native SwiftPM and Dart packages outside the repository + if: matrix.source == 'package-config' + run: | + mkdir -p "$RUNNER_TEMP/zstandard-stage" + mv zstandard_native "$RUNNER_TEMP/zstandard-stage/zstandard_native" + cp Package.swift "$RUNNER_TEMP/zstandard-stage/Package.swift" + if [[ "${{ matrix.dependency_manager }}" == swiftpm ]]; then + echo "ZSTANDARD_NATIVE_PACKAGE_PATH=$RUNNER_TEMP/zstandard-stage" >> "$GITHUB_ENV" + fi + + - name: Resolve the selected native source exactly run: | - mv zstandard_native "$RUNNER_TEMP/zstandard_native" + if [[ "${{ matrix.source }}" == package-config ]]; then + dart scripts/create_local_overrides.dart zstandard_ios \ + "$RUNNER_TEMP/zstandard-stage/zstandard_native" + else + dart scripts/create_local_overrides.dart zstandard_ios + fi - name: Clean plugin and example run: | @@ -117,22 +131,6 @@ jobs: working-directory: zstandard_ios/example run: flutter build ios --simulator --debug --no-codesign - - name: Copy generated Flutter framework for SwiftPM integration tests - if: matrix.dependency_manager == 'swiftpm' && matrix.source == 'pub-cache' - run: | - framework_package="$GITHUB_WORKSPACE/zstandard_ios/example/ios/Flutter/ephemeral/Packages/.packages/FlutterFramework" - framework_copy="$GITHUB_WORKSPACE/zstandard_ios/ios/FlutterFramework" - test -f "$framework_package/Package.swift" - if [[ -e "$framework_copy" || -L "$framework_copy" ]]; then - echo "Removing stale FlutterFramework path: $framework_copy" - if [[ -L "$framework_copy" ]]; then - rm "$framework_copy" - else - rm -R "$framework_copy" - fi - fi - cp -R "$framework_package" "$framework_copy" - - name: Boot iOS simulator headlessly run: | chmod +x scripts/manage_ios_simulator.sh @@ -162,7 +160,7 @@ jobs: - name: Upload iOS simulator diagnostics if: failure() - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 with: name: ios-simulator-diagnostics-${{ matrix.dependency_manager }}-${{ matrix.source }}-${{ github.run_id }} path: ${{ runner.temp }}/ios-simulator-diagnostics.txt @@ -172,31 +170,23 @@ jobs: if: always() run: ./scripts/manage_ios_simulator.sh stop - - name: Remove temporary SwiftPM framework copy - if: always() && matrix.dependency_manager == 'swiftpm' && matrix.source == 'pub-cache' - run: | - framework_copy="$GITHUB_WORKSPACE/zstandard_ios/ios/FlutterFramework" - if [[ -L "$framework_copy" ]]; then - rm "$framework_copy" - elif [[ -d "$framework_copy" ]]; then - rm -R "$framework_copy" - elif [[ -e "$framework_copy" ]]; then - rm "$framework_copy" - fi - - name: Restore workspace native package - if: always() && matrix.source == 'pub-cache' + if: always() && matrix.source == 'package-config' run: | - if [[ -d "$RUNNER_TEMP/zstandard_native" ]]; then - mv "$RUNNER_TEMP/zstandard_native" zstandard_native + if [[ -d "$RUNNER_TEMP/zstandard-stage/zstandard_native" ]]; then + mv "$RUNNER_TEMP/zstandard-stage/zstandard_native" zstandard_native fi + - name: Restore Flutter dependency-manager default + if: always() + run: flutter config --no-enable-swift-package-manager + check_ios_publish_dry_run: name: Dry Run Publish iOS runs-on: [self-hosted, macOS, ARM64] timeout-minutes: 45 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: @@ -207,6 +197,9 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_ios + - name: Get dependencies working-directory: zstandard_ios run: flutter pub get diff --git a/.github/workflows/pr_check_linux.yml b/.github/workflows/pr_check_linux.yml index afddfe01..bf172533 100644 --- a/.github/workflows/pr_check_linux.yml +++ b/.github/workflows/pr_check_linux.yml @@ -1,13 +1,10 @@ name: PR Check Linux on: - pull_request: - branches: - - '**' - - '!master' push: branches: - develop + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,12 +20,15 @@ jobs: name: Analyze Linux runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_linux + - name: Get dependencies working-directory: zstandard_linux run: flutter pub get @@ -42,7 +42,7 @@ jobs: runs-on: [self-hosted, Linux] timeout-minutes: 45 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-linux-dependencies with: @@ -52,6 +52,9 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_linux + - name: Get dependencies working-directory: zstandard_linux run: flutter pub get @@ -80,16 +83,27 @@ jobs: cmake --build "$cmake_build_dir" --target zstandard_linux_test --parallel ctest --test-dir "$cmake_build_dir" --output-on-failure + - name: Run Linux package tests against the built plugin + working-directory: zstandard_linux + run: | + set -euo pipefail + plugin_library="$(find example/build/linux -type f \ + -name libzstandard_linux_plugin.so -print -quit)" + test -n "$plugin_library" + plugin_directory="$(cd "$(dirname "$plugin_library")" && pwd)" + export LD_LIBRARY_PATH="$plugin_directory${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + flutter test --coverage + - name: Run integration tests working-directory: zstandard_linux/example run: xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' flutter test integration_test/ -d linux - check_linux_test_from_pub_cache: - name: Test Linux (from pub cache) + check_linux_test_from_package_config: + name: Test Linux (exact package-config source) runs-on: [self-hosted, Linux] timeout-minutes: 45 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-linux-dependencies with: @@ -99,26 +113,30 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Remove zstandard_native to force resolution from pub cache - run: rm -rf zstandard_native + - name: Stage zstandard_native outside the repository + run: | + mkdir -p "$RUNNER_TEMP/zstandard-stage" + mv zstandard_native "$RUNNER_TEMP/zstandard-stage/zstandard_native" + dart scripts/create_local_overrides.dart zstandard_linux \ + "$RUNNER_TEMP/zstandard-stage/zstandard_native" - - name: Clean and get dependencies (force pub cache) + - name: Clean and get dependencies from package_config working-directory: zstandard_linux run: | flutter clean flutter pub get - - name: Get dependencies in example (force pub cache) + - name: Get dependencies in example from package_config working-directory: zstandard_linux/example run: | flutter clean flutter pub get - - name: Build Linux example from pub cache + - name: Build Linux example from package_config working-directory: zstandard_linux/example run: flutter build linux --debug - - name: Build and run Linux native tests from pub cache + - name: Build and run Linux native tests from package_config working-directory: zstandard_linux/example run: | set -euo pipefail @@ -130,15 +148,33 @@ jobs: cmake --build "$cmake_build_dir" --target zstandard_linux_test --parallel ctest --test-dir "$cmake_build_dir" --output-on-failure - - name: Run integration tests (C sources from pub cache) + - name: Run Linux package tests against the package-config plugin + working-directory: zstandard_linux + run: | + set -euo pipefail + plugin_library="$(find example/build/linux -type f \ + -name libzstandard_linux_plugin.so -print -quit)" + test -n "$plugin_library" + plugin_directory="$(cd "$(dirname "$plugin_library")" && pwd)" + export LD_LIBRARY_PATH="$plugin_directory${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + flutter test --coverage + + - name: Run integration tests (C source from package_config) working-directory: zstandard_linux/example run: xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' flutter test integration_test/ -d linux + - name: Restore workspace native package + if: always() + run: | + if [[ -d "$RUNNER_TEMP/zstandard-stage/zstandard_native" ]]; then + mv "$RUNNER_TEMP/zstandard-stage/zstandard_native" zstandard_native + fi + check_linux_publish_dry_run: name: Dry Run Publish Linux runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: @@ -149,6 +185,9 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_linux + - name: Get dependencies working-directory: zstandard_linux run: flutter pub get diff --git a/.github/workflows/pr_check_macos.yml b/.github/workflows/pr_check_macos.yml index 9c7e7f5d..c92c972a 100644 --- a/.github/workflows/pr_check_macos.yml +++ b/.github/workflows/pr_check_macos.yml @@ -1,13 +1,10 @@ name: PR Check macOS on: - pull_request: - branches: - - '**' - - '!master' push: branches: - develop + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,7 +20,7 @@ jobs: runs-on: [self-hosted, macOS, ARM64] timeout-minutes: 45 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: @@ -34,6 +31,9 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_macos + - name: Get dependencies working-directory: zstandard_macos run: flutter pub get @@ -60,9 +60,9 @@ jobs: max-parallel: 1 matrix: dependency_manager: [swiftpm, cocoapods] - source: [workspace, pub-cache] + source: [workspace, package-config] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: @@ -85,10 +85,24 @@ jobs: if: matrix.dependency_manager == 'swiftpm' && matrix.source == 'workspace' run: echo "ZSTANDARD_NATIVE_PACKAGE_PATH=$GITHUB_WORKSPACE" >> "$GITHUB_ENV" - - name: Hide workspace native package to force Pub cache resolution - if: matrix.source == 'pub-cache' + - name: Stage native SwiftPM and Dart packages outside the repository + if: matrix.source == 'package-config' + run: | + mkdir -p "$RUNNER_TEMP/zstandard-stage" + mv zstandard_native "$RUNNER_TEMP/zstandard-stage/zstandard_native" + cp Package.swift "$RUNNER_TEMP/zstandard-stage/Package.swift" + if [[ "${{ matrix.dependency_manager }}" == swiftpm ]]; then + echo "ZSTANDARD_NATIVE_PACKAGE_PATH=$RUNNER_TEMP/zstandard-stage" >> "$GITHUB_ENV" + fi + + - name: Resolve the selected native source exactly run: | - mv zstandard_native "$RUNNER_TEMP/zstandard_native" + if [[ "${{ matrix.source }}" == package-config ]]; then + dart scripts/create_local_overrides.dart zstandard_macos \ + "$RUNNER_TEMP/zstandard-stage/zstandard_native" + else + dart scripts/create_local_overrides.dart zstandard_macos + fi - name: Clean plugin and example run: | @@ -117,22 +131,6 @@ jobs: working-directory: zstandard_macos/example run: flutter build macos --debug - - name: Copy generated Flutter framework for SwiftPM integration tests - if: matrix.dependency_manager == 'swiftpm' && matrix.source == 'pub-cache' - run: | - framework_package="$GITHUB_WORKSPACE/zstandard_macos/example/macos/Flutter/ephemeral/Packages/.packages/FlutterFramework" - framework_copy="$GITHUB_WORKSPACE/zstandard_macos/macos/FlutterFramework" - test -f "$framework_package/Package.swift" - if [[ -e "$framework_copy" || -L "$framework_copy" ]]; then - echo "Removing stale FlutterFramework path: $framework_copy" - if [[ -L "$framework_copy" ]]; then - rm "$framework_copy" - else - rm -R "$framework_copy" - fi - fi - cp -R "$framework_package" "$framework_copy" - - name: Launch macOS example and run integration tests working-directory: zstandard_macos/example run: | @@ -140,31 +138,23 @@ jobs: flutter test integration_test/macos_integration_test.dart \ -d macos --reporter expanded - - name: Remove temporary SwiftPM framework copy - if: always() && matrix.dependency_manager == 'swiftpm' && matrix.source == 'pub-cache' - run: | - framework_copy="$GITHUB_WORKSPACE/zstandard_macos/macos/FlutterFramework" - if [[ -L "$framework_copy" ]]; then - rm "$framework_copy" - elif [[ -d "$framework_copy" ]]; then - rm -R "$framework_copy" - elif [[ -e "$framework_copy" ]]; then - rm "$framework_copy" - fi - - name: Restore workspace native package - if: always() && matrix.source == 'pub-cache' + if: always() && matrix.source == 'package-config' run: | - if [[ -d "$RUNNER_TEMP/zstandard_native" ]]; then - mv "$RUNNER_TEMP/zstandard_native" zstandard_native + if [[ -d "$RUNNER_TEMP/zstandard-stage/zstandard_native" ]]; then + mv "$RUNNER_TEMP/zstandard-stage/zstandard_native" zstandard_native fi + - name: Restore Flutter dependency-manager default + if: always() + run: flutter config --no-enable-swift-package-manager + check_macos_publish_dry_run: name: Dry Run Publish macOS runs-on: [self-hosted, macOS, ARM64] timeout-minutes: 45 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: @@ -175,6 +165,9 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_macos + - name: Get dependencies working-directory: zstandard_macos run: flutter pub get diff --git a/.github/workflows/pr_check_platform_interface.yml b/.github/workflows/pr_check_platform_interface.yml index f4712e62..e73e5c75 100644 --- a/.github/workflows/pr_check_platform_interface.yml +++ b/.github/workflows/pr_check_platform_interface.yml @@ -1,13 +1,10 @@ name: PR Check Platform Interface on: - pull_request: - branches: - - '**' - - '!master' push: branches: - develop + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,7 +20,7 @@ jobs: name: Analyze Platform Interface runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-flutter with: @@ -41,11 +38,9 @@ jobs: name: Test Platform Interface runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-linux-dependencies - with: - coverage: 'true' - uses: ./.github/actions/setup-flutter with: @@ -62,26 +57,22 @@ jobs: # Coverage threshold: lib/ only (same pattern as CLI) - name: Check coverage threshold (85% of lib/) working-directory: zstandard_platform_interface - run: | - lcov --extract coverage/lcov.info 'lib/*' -o coverage/lcov_lib.info - COVERAGE=$(lcov --summary coverage/lcov_lib.info 2>&1 | grep "lines" | awk '{print $2}' | sed 's/%//') - echo "Coverage (lib/ only): ${COVERAGE}%" - if [ -z "$COVERAGE" ]; then echo "Could not parse coverage"; exit 1; fi - if (( $(echo "$COVERAGE < 85" | bc -l) )); then echo "ERROR: Coverage ${COVERAGE}% is below 85%"; exit 1; fi + run: dart ../scripts/check_lcov_coverage.dart coverage/lcov.info 85 - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f with: token: ${{ secrets.CODECOV_TOKEN }} files: zstandard_platform_interface/coverage/lcov.info flags: platform_interface + disable_search: true fail_ci_if_error: true check_platform_interface_publish_dry_run: name: Dry Run Publish Platform Interface runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: diff --git a/.github/workflows/pr_check_web.yml b/.github/workflows/pr_check_web.yml index 7819532a..26297105 100644 --- a/.github/workflows/pr_check_web.yml +++ b/.github/workflows/pr_check_web.yml @@ -1,13 +1,10 @@ name: PR Check Web on: - pull_request: - branches: - - '**' - - '!master' push: branches: - develop + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,12 +20,15 @@ jobs: name: Analyze Web runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_web + - name: Get dependencies working-directory: zstandard_web run: flutter pub get @@ -42,7 +42,7 @@ jobs: runs-on: [self-hosted, Linux] timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-linux-dependencies with: @@ -52,6 +52,9 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_web + - name: Get dependencies working-directory: zstandard_web run: flutter pub get @@ -62,7 +65,7 @@ jobs: - name: Set up Chrome and matching ChromeDriver id: chrome - uses: browser-actions/setup-chrome@v2 + uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 with: chrome-version: stable install-chromedriver: true @@ -92,8 +95,14 @@ jobs: git diff --check git diff --exit-code -- \ zstandard_web/blob/zstd.js \ + zstandard_web/blob/zstd_core.js \ + zstandard_web/blob/zstd_worker.js \ zstandard_web/example/web/zstd.js \ - zstandard/example/web/zstd.js + zstandard_web/example/web/zstd_core.js \ + zstandard_web/example/web/zstd_worker.js \ + zstandard/example/web/zstd.js \ + zstandard/example/web/zstd_core.js \ + zstandard/example/web/zstd_worker.js # Emscripten/LLVM may choose a different valid WASM layout on # different host architectures, even with the same SDK revision. @@ -110,16 +119,22 @@ jobs: - name: Upload regenerated artifacts when verification fails if: failure() - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 with: name: web-regenerated-artifacts-${{ github.run_id }} if-no-files-found: error path: | zstandard_web/blob/zstd.js + zstandard_web/blob/zstd_core.js + zstandard_web/blob/zstd_worker.js zstandard_web/blob/zstd.wasm zstandard_web/example/web/zstd.js + zstandard_web/example/web/zstd_core.js + zstandard_web/example/web/zstd_worker.js zstandard_web/example/web/zstd.wasm zstandard/example/web/zstd.js + zstandard/example/web/zstd_core.js + zstandard/example/web/zstd_worker.js zstandard/example/web/zstd.wasm - name: Build Web example @@ -163,7 +178,7 @@ jobs: --web-browser-flag=--disable-dev-shm-usage \ --web-browser-flag=--disable-gpu \ -d web-server \ - --web-port=8080 + --web-port=0 DRIVE_EXIT=$? cat "$DRIVER_LOG" exit "$DRIVE_EXIT" @@ -173,7 +188,7 @@ jobs: name: Dry Run Publish Web runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: @@ -184,6 +199,9 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard_web + - name: Get dependencies working-directory: zstandard_web run: flutter pub get diff --git a/.github/workflows/pr_check_windows.yml b/.github/workflows/pr_check_windows.yml index 4e02a3ad..ec75a30a 100644 --- a/.github/workflows/pr_check_windows.yml +++ b/.github/workflows/pr_check_windows.yml @@ -1,13 +1,10 @@ name: PR Check Windows on: - pull_request: - branches: - - '**' - - '!master' push: branches: - develop + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -24,12 +21,16 @@ jobs: runs-on: [self-hosted, Windows, X64] timeout-minutes: 45 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-preinstalled-flutter-windows with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + shell: cmd + run: call dart scripts\create_local_overrides.dart zstandard_windows + - name: Get dependencies working-directory: zstandard_windows shell: cmd @@ -45,12 +46,16 @@ jobs: runs-on: [self-hosted, Windows, X64] timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-preinstalled-flutter-windows with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + shell: cmd + run: call dart scripts\create_local_overrides.dart zstandard_windows + - name: Get dependencies working-directory: zstandard_windows shell: cmd @@ -111,43 +116,48 @@ jobs: shell: cmd run: call flutter test integration_test/windows_integration_test.dart -d windows - check_windows_test_from_pub_cache: - name: Test Windows (from pub cache) + check_windows_test_from_package_config: + name: Test Windows (exact package-config source) runs-on: [self-hosted, Windows, X64] timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-preinstalled-flutter-windows with: flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Remove zstandard_native to force resolution from pub cache + - name: Stage zstandard_native outside the repository shell: cmd run: | @echo off - if exist zstandard_native rmdir /s /q zstandard_native - - - name: Clean and get dependencies (force pub cache) + set "STAGE=%RUNNER_TEMP%\zstandard-stage" + if not exist "%STAGE%" mkdir "%STAGE%" + move zstandard_native "%STAGE%\zstandard_native" + if errorlevel 1 exit /b %ERRORLEVEL% + call dart scripts\create_local_overrides.dart zstandard_windows "%STAGE%\zstandard_native" + if errorlevel 1 exit /b %ERRORLEVEL% + + - name: Clean and get dependencies from package_config working-directory: zstandard_windows shell: cmd run: | call flutter clean call flutter pub get - - name: Get dependencies in example (force pub cache) + - name: Get dependencies in example from package_config working-directory: zstandard_windows/example shell: cmd run: | call flutter clean call flutter pub get - - name: Build Windows example from pub cache + - name: Build Windows example from package_config working-directory: zstandard_windows/example shell: cmd run: call flutter build windows --debug - - name: Run Windows native C++ tests from pub cache + - name: Run Windows native C++ tests from package_config working-directory: zstandard_windows/example shell: cmd run: | @@ -166,7 +176,7 @@ jobs: "%TEST_EXE%" if errorlevel 1 exit /b 1 - - name: Run Windows package tests against the pub-cache DLL + - name: Run Windows package tests against the package-config DLL shell: cmd run: | @echo off @@ -182,17 +192,24 @@ jobs: popd exit /b %EXIT_CODE% - - name: Launch Windows example and run integration tests from pub cache + - name: Launch Windows example and run package-config integration tests working-directory: zstandard_windows/example shell: cmd run: call flutter test integration_test/windows_integration_test.dart -d windows + - name: Restore workspace native package + if: always() + shell: cmd + run: | + @echo off + if exist "%RUNNER_TEMP%\zstandard-stage\zstandard_native" move "%RUNNER_TEMP%\zstandard-stage\zstandard_native" zstandard_native + check_windows_federated_example: name: Test federated Windows example runs-on: [self-hosted, Windows, X64] timeout-minutes: 60 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-preinstalled-flutter-windows with: @@ -220,12 +237,16 @@ jobs: runs-on: [self-hosted, Windows, X64] timeout-minutes: 45 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-preinstalled-flutter-windows with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + shell: cmd + run: call dart scripts\create_local_overrides.dart zstandard_windows + - name: Get dependencies working-directory: zstandard_windows shell: cmd diff --git a/.github/workflows/pr_check_zstandard.yml b/.github/workflows/pr_check_zstandard.yml index b8d565a5..7197f9fe 100644 --- a/.github/workflows/pr_check_zstandard.yml +++ b/.github/workflows/pr_check_zstandard.yml @@ -1,13 +1,10 @@ name: PR Check Zstandard on: - pull_request: - branches: - - '**' - - '!master' push: branches: - develop + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,12 +20,15 @@ jobs: name: Analyze Zstandard runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard + - name: Verify publication metadata run: dart scripts/check_pub_metadata.dart @@ -45,16 +45,17 @@ jobs: runs-on: [self-hosted, Linux] timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/setup-linux-dependencies - with: - coverage: 'true' - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard + - name: Get dependencies working-directory: zstandard run: flutter pub get @@ -66,32 +67,22 @@ jobs: # Coverage threshold: lib/ only (same pattern as CLI and platform_interface) - name: Check coverage threshold (76% of lib/) working-directory: zstandard - run: | - lcov --extract coverage/lcov.info 'lib/*' -o coverage/lcov_lib.info - COVERAGE=$(lcov --summary coverage/lcov_lib.info 2>&1 | grep "lines" | awk '{print $2}' | sed 's/%//') - echo "Coverage (lib/ only): ${COVERAGE}%" - if [ -z "$COVERAGE" ]; then - echo "Could not parse coverage" - exit 1 - fi - if (( $(echo "$COVERAGE < 76" | bc -l) )); then - echo "ERROR: Coverage ${COVERAGE}% is below 76% threshold" - exit 1 - fi + run: dart ../scripts/check_lcov_coverage.dart coverage/lcov.info 76 - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f with: token: ${{ secrets.CODECOV_TOKEN }} files: zstandard/coverage/lcov.info flags: zstandard + disable_search: true fail_ci_if_error: true check_zstandard_publish_dry_run: name: Dry Run Publish Zstandard runs-on: [self-hosted, Linux] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - uses: ./.github/actions/preserve-flutter-project-files with: @@ -102,6 +93,9 @@ jobs: with: flutter-version: ${{ env.FLUTTER_VERSION }} + - name: Resolve workspace dependencies + run: dart scripts/create_local_overrides.dart zstandard + - name: Get dependencies working-directory: zstandard run: flutter pub get diff --git a/.github/workflows/publish_workflow.yml b/.github/workflows/publish_workflow.yml new file mode 100644 index 00000000..79be625a --- /dev/null +++ b/.github/workflows/publish_workflow.yml @@ -0,0 +1,187 @@ +name: Publish Tagged Release + +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +concurrency: + group: zstandard-release-publication + cancel-in-progress: false + +permissions: + contents: read + +env: + FLUTTER_VERSION: '3.47.2' + +jobs: + validate_tag: + name: Validate immutable release tag + runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + version: ${{ steps.context.outputs.version }} + commit: ${{ steps.context.outputs.commit }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + fetch-depth: 0 + + - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 + with: + flutter-version: ${{ env.FLUTTER_VERSION }} + channel: stable + cache: true + + - name: Validate tag, release branch and package metadata + id: context + run: | + set -euo pipefail + version="${GITHUB_REF_NAME#v}" + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid stable release tag: $GITHUB_REF_NAME" >&2 + exit 1 + fi + git fetch origin "release/$version:refs/remotes/origin/release/$version" + commit="$(git rev-parse HEAD)" + branch_commit="$(git rev-parse "origin/release/$version")" + if [[ "$commit" != "$branch_commit" ]]; then + echo "Tag $GITHUB_REF_NAME must point to the tip of release/$version." >&2 + exit 1 + fi + dart scripts/check_pub_metadata.dart --release-version "$version" + git diff --check + for artifact in \ + zstandard_cli/lib/src/bin/libzstandard_macos.dylib \ + zstandard_cli/lib/src/bin/libzstandard_linux_x64.so \ + zstandard_cli/lib/src/bin/libzstandard_linux_arm64.so \ + zstandard_cli/lib/src/bin/zstandard_windows_x64.dll \ + zstandard_cli/lib/src/bin/zstandard_windows_arm64.dll \ + zstandard_web/blob/zstd.js \ + zstandard_web/blob/zstd_core.js \ + zstandard_web/blob/zstd_worker.js \ + zstandard_web/blob/zstd.wasm; do + test -s "$artifact" + done + cmp zstandard_web/blob/zstd.js zstandard_web/example/web/zstd.js + cmp zstandard_web/blob/zstd.js zstandard/example/web/zstd.js + cmp zstandard_web/blob/zstd_core.js zstandard_web/example/web/zstd_core.js + cmp zstandard_web/blob/zstd_core.js zstandard/example/web/zstd_core.js + cmp zstandard_web/blob/zstd_worker.js zstandard_web/example/web/zstd_worker.js + cmp zstandard_web/blob/zstd_worker.js zstandard/example/web/zstd_worker.js + cmp zstandard_web/blob/zstd.wasm zstandard_web/example/web/zstd.wasm + cmp zstandard_web/blob/zstd.wasm zstandard/example/web/zstd.wasm + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "commit=$commit" >> "$GITHUB_OUTPUT" + + publish: + name: Publish packages to pub.dev with OIDC + runs-on: ubuntu-latest + timeout-minutes: 180 + needs: [validate_tag] + environment: pub.dev + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ needs.validate_tag.outputs.commit }} + fetch-depth: 0 + + # setup-dart provisions the short-lived pub.dev credential from GitHub's + # OIDC token. Flutter is installed afterwards so its bundled Dart stays + # first on PATH while using the same Pub credentials. + - uses: dart-lang/setup-dart@6afc89df92d6eb3834022f73cd65adc8cdfcb92d + + - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 + with: + flutter-version: ${{ env.FLUTTER_VERSION }} + channel: stable + cache: true + + - name: Publish in dependency order, safely skipping immutable versions + env: + VERSION: ${{ needs.validate_tag.outputs.version }} + run: | + set -euo pipefail + publish_and_wait() { + local package="$1" + if [[ "$package" == zstandard_native || "$package" == zstandard_cli ]]; then + (cd "$package" && dart pub get) + else + (cd "$package" && flutter pub get) + fi + dart run scripts/publish_if_missing.dart "$package" "$VERSION" + dart run scripts/wait_for_pub_version.dart "$package" "$VERSION" 1200 + } + + publish_and_wait zstandard_platform_interface + publish_and_wait zstandard_native + for package in \ + zstandard_android zstandard_ios zstandard_linux zstandard_macos \ + zstandard_web zstandard_windows; do + publish_and_wait "$package" + done + publish_and_wait zstandard_cli + publish_and_wait zstandard + + verify: + name: Verify all published packages + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [validate_tag, publish] + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ needs.validate_tag.outputs.commit }} + + - name: Verify exact versions through the public API + env: + VERSION: ${{ needs.validate_tag.outputs.version }} + run: | + set -euo pipefail + for package in \ + zstandard_platform_interface zstandard_native zstandard_android \ + zstandard_ios zstandard_linux zstandard_macos zstandard_web \ + zstandard_windows zstandard_cli zstandard; do + dart run scripts/wait_for_pub_version.dart "$package" "$VERSION" 1200 + done + + github_release: + name: Create immutable GitHub release + runs-on: ubuntu-latest + timeout-minutes: 15 + needs: [validate_tag, verify] + permissions: + contents: write + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ needs.validate_tag.outputs.commit }} + + - name: Build native-library checksum manifest + run: | + set -euo pipefail + { + echo "Native CLI library checksums for $GITHUB_REF_NAME" + sha256sum zstandard_cli/lib/src/bin/* + sha256sum zstandard_web/blob/zstd.js zstandard_web/blob/zstd_core.js \ + zstandard_web/blob/zstd_worker.js zstandard_web/blob/zstd.wasm + } > checksums.txt + + - name: Create release from the validated tag + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + gh release upload "$GITHUB_REF_NAME" checksums.txt --repo "$GITHUB_REPOSITORY" --clobber + else + gh release create "$GITHUB_REF_NAME" checksums.txt \ + --repo "$GITHUB_REPOSITORY" \ + --verify-tag \ + --generate-notes \ + --title "Release $GITHUB_REF_NAME" + fi diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 00000000..bdbbb044 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,188 @@ +name: Pull Request Safety Gate + +on: + pull_request: + branches: + - '**' + +concurrency: + group: pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +env: + FLUTTER_VERSION: '3.47.2' + +jobs: + dart_packages: + name: Analyze and test ${{ matrix.package }} + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + package: + - zstandard_platform_interface + - zstandard_native + - zstandard_android + - zstandard_ios + - zstandard_linux + - zstandard_macos + - zstandard_web + - zstandard_windows + - zstandard_cli + - zstandard + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + + - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 + with: + flutter-version: ${{ env.FLUTTER_VERSION }} + channel: stable + cache: true + + - name: Resolve monorepo dependencies + run: dart scripts/create_local_overrides.dart "${{ matrix.package }}" + + - name: Get dependencies + working-directory: ${{ matrix.package }} + run: flutter pub get + + - name: Verify formatting + working-directory: ${{ matrix.package }} + run: dart format --output=none --set-exit-if-changed . + + - name: Analyze + working-directory: ${{ matrix.package }} + run: flutter analyze + + - name: Test + # Native/Linux need a built shared library. The three coverage packages + # are tested by the required coverage matrix below. + if: >- + matrix.package != 'zstandard_native' && + matrix.package != 'zstandard_linux' && + matrix.package != 'zstandard_cli' && + matrix.package != 'zstandard_platform_interface' && + matrix.package != 'zstandard' + working-directory: ${{ matrix.package }} + run: flutter test + + coverage: + name: Coverage (${{ matrix.package }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - package: zstandard_cli + flag: cli + threshold: 100 + - package: zstandard_platform_interface + flag: platform_interface + threshold: 85 + - package: zstandard + flag: zstandard + threshold: 76 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + + - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 + with: + flutter-version: ${{ env.FLUTTER_VERSION }} + channel: stable + cache: true + + - name: Resolve monorepo dependencies + run: dart scripts/create_local_overrides.dart "${{ matrix.package }}" + + - name: Get dependencies + working-directory: ${{ matrix.package }} + run: flutter pub get + + - name: Run CLI tests with coverage + if: matrix.package == 'zstandard_cli' + working-directory: ${{ matrix.package }} + run: dart test --coverage=coverage + + - name: Format CLI coverage + if: matrix.package == 'zstandard_cli' + working-directory: ${{ matrix.package }} + run: >- + dart run coverage:format_coverage --lcov -i coverage + -o coverage/lcov.info --packages=.dart_tool/package_config.json + --report-on=lib + + - name: Run Flutter tests with coverage + if: matrix.package != 'zstandard_cli' + working-directory: ${{ matrix.package }} + run: flutter test --coverage + + - name: Enforce line coverage + working-directory: ${{ matrix.package }} + run: >- + dart ../scripts/check_lcov_coverage.dart coverage/lcov.info + "${{ matrix.threshold }}" + + - name: Upload coverage to Codecov + if: >- + github.event.pull_request.head.repo.full_name == github.repository && + github.actor != 'dependabot[bot]' + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${{ matrix.package }}/coverage/lcov.info + flags: ${{ matrix.flag }} + disable_search: true + fail_ci_if_error: true + + repository_checks: + name: Repository checks + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + + - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 + with: + flutter-version: ${{ env.FLUTTER_VERSION }} + channel: stable + cache: true + + - name: Validate metadata and generated artifacts + run: | + set -euo pipefail + dart scripts/check_pub_metadata.dart + git diff --check + cmp zstandard_web/blob/zstd.js zstandard_web/example/web/zstd.js + cmp zstandard_web/blob/zstd.js zstandard/example/web/zstd.js + cmp zstandard_web/blob/zstd_core.js zstandard_web/example/web/zstd_core.js + cmp zstandard_web/blob/zstd_core.js zstandard/example/web/zstd_core.js + cmp zstandard_web/blob/zstd_worker.js zstandard_web/example/web/zstd_worker.js + cmp zstandard_web/blob/zstd_worker.js zstandard/example/web/zstd_worker.js + cmp zstandard_web/blob/zstd.wasm zstandard_web/example/web/zstd.wasm + cmp zstandard_web/blob/zstd.wasm zstandard/example/web/zstd.wasm + + - name: Validate shell scripts + run: bash -n scripts/*.sh zstandard_ios/scripts/*.sh zstandard_macos/scripts/*.sh + + safety_gate: + name: Pull Request Safety Gate + runs-on: ubuntu-latest + timeout-minutes: 5 + if: always() + needs: [dart_packages, coverage, repository_checks] + steps: + - name: Require every safety-gate job + env: + PACKAGE_RESULT: ${{ needs.dart_packages.result }} + COVERAGE_RESULT: ${{ needs.coverage.result }} + REPOSITORY_RESULT: ${{ needs.repository_checks.result }} + run: | + set -euo pipefail + [[ "$PACKAGE_RESULT" == success ]] + [[ "$COVERAGE_RESULT" == success ]] + [[ "$REPOSITORY_RESULT" == success ]] diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml index bb3f12cc..0e92b7de 100644 --- a/.github/workflows/release_workflow.yml +++ b/.github/workflows/release_workflow.yml @@ -1,4 +1,4 @@ -name: Task - Release +name: Task - Prepare Release on: workflow_dispatch: @@ -23,15 +23,9 @@ on: required: false type: string default: '-1' - resume: - description: 'Resume a partial release whose tag already exists' - required: true - type: boolean - default: false - concurrency: - # Publishing is irreversible and must never be cancelled by a newer run. - group: ${{ github.workflow }}-${{ github.ref }} + # Preparation and tag-driven publication share this repository-wide lock. + group: zstandard-release-publication cancel-in-progress: false defaults: @@ -55,10 +49,8 @@ jobs: timeout-minutes: 60 permissions: contents: write - outputs: - release_tag: ${{ steps.context.outputs.release_tag }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -68,7 +60,6 @@ jobs: id: context env: VERSION: ${{ inputs.version }} - RESUME: ${{ inputs.resume }} run: | set -euo pipefail if [[ ! "$GITHUB_REF_NAME" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -87,25 +78,16 @@ jobs: git fetch origin --tags --force tag="v$VERSION" if git ls-remote --exit-code --tags origin "refs/tags/$tag" >/dev/null 2>&1; then - if [[ "$RESUME" != 'true' ]]; then - echo "Tag $tag already exists. Set resume=true only for a partial release." >&2 - exit 1 - fi - echo "Resuming the existing immutable tag $tag." - elif [[ "$RESUME" == 'true' ]]; then - echo "Cannot resume: tag $tag does not exist on origin." >&2 + echo "Tag $tag already exists; release preparation is immutable." >&2 exit 1 fi - echo "release_tag=$tag" >> "$GITHUB_OUTPUT" - name: Verify preinstalled Flutter - if: ${{ !inputs.resume }} uses: ./.github/actions/setup-preinstalled-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} - name: Update root CHANGELOG.md - if: ${{ !inputs.resume }} env: VERSION: ${{ inputs.version }} TITLE: ${{ inputs.title }} @@ -124,7 +106,6 @@ jobs: fi - name: Copy CHANGELOG.md to all published packages - if: ${{ !inputs.resume }} run: | set -euo pipefail for package in \ @@ -135,13 +116,11 @@ jobs: done - name: Update package, SwiftPM and CocoaPods versions - if: ${{ !inputs.resume }} env: VERSION: ${{ inputs.version }} run: dart run .github/scripts/update_versions.dart "$VERSION" --release - name: Regenerate WebAssembly from canonical C source - if: ${{ !inputs.resume }} env: # The script uses this only for the read-only emsdk fetch. GITHUB_TOKEN: ${{ github.token }} @@ -158,8 +137,14 @@ jobs: git diff --check for generated in \ zstandard_web/blob/zstd.js \ + zstandard_web/blob/zstd_core.js \ + zstandard_web/blob/zstd_worker.js \ zstandard_web/example/web/zstd.js \ - zstandard/example/web/zstd.js; do + zstandard_web/example/web/zstd_core.js \ + zstandard_web/example/web/zstd_worker.js \ + zstandard/example/web/zstd.js \ + zstandard/example/web/zstd_core.js \ + zstandard/example/web/zstd_worker.js; do test -s "$generated" done for generated in \ @@ -170,11 +155,14 @@ jobs: done cmp zstandard_web/blob/zstd.js zstandard_web/example/web/zstd.js cmp zstandard_web/blob/zstd.js zstandard/example/web/zstd.js + cmp zstandard_web/blob/zstd_core.js zstandard_web/example/web/zstd_core.js + cmp zstandard_web/blob/zstd_core.js zstandard/example/web/zstd_core.js + cmp zstandard_web/blob/zstd_worker.js zstandard_web/example/web/zstd_worker.js + cmp zstandard_web/blob/zstd_worker.js zstandard/example/web/zstd_worker.js cmp zstandard_web/blob/zstd.wasm zstandard_web/example/web/zstd.wasm cmp zstandard_web/blob/zstd.wasm zstandard/example/web/zstd.wasm - name: Commit and push release preparation - if: ${{ !inputs.resume }} env: VERSION: ${{ inputs.version }} RELEASE_BRANCH: ${{ github.ref_name }} @@ -187,6 +175,7 @@ jobs: zstandard/pubspec.yaml zstandard/CHANGELOG.md \ zstandard_android/pubspec.yaml zstandard_android/CHANGELOG.md \ zstandard_cli/pubspec.yaml zstandard_cli/CHANGELOG.md \ + zstandard_cli/lib/src/cli_runner.dart \ zstandard_ios/pubspec.yaml zstandard_ios/CHANGELOG.md \ zstandard_ios/ios/zstandard_ios/Package.swift \ zstandard_ios/ios/zstandard_ios.podspec \ @@ -198,8 +187,11 @@ jobs: zstandard_platform_interface/pubspec.yaml zstandard_platform_interface/CHANGELOG.md \ zstandard_web/pubspec.yaml zstandard_web/CHANGELOG.md \ zstandard_web/blob/zstd.js zstandard_web/blob/zstd.wasm \ + zstandard_web/blob/zstd_core.js zstandard_web/blob/zstd_worker.js \ zstandard_web/example/web/zstd.js zstandard_web/example/web/zstd.wasm \ + zstandard_web/example/web/zstd_core.js zstandard_web/example/web/zstd_worker.js \ zstandard/example/web/zstd.js zstandard/example/web/zstd.wasm \ + zstandard/example/web/zstd_core.js zstandard/example/web/zstd_worker.js \ zstandard_windows/pubspec.yaml zstandard_windows/CHANGELOG.md if git diff --cached --quiet; then echo 'Release preparation is already committed.' @@ -210,14 +202,13 @@ jobs: cli_build_macos_precompiled_libs: name: Build CLI libraries - macOS universal - if: ${{ !inputs.resume }} runs-on: [self-hosted, macOS, ARM64] timeout-minutes: 60 needs: [update_files] permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -262,14 +253,13 @@ jobs: cli_build_linux_precompiled_libs: name: Build CLI libraries - Linux - if: ${{ !inputs.resume }} runs-on: [self-hosted, Linux] timeout-minutes: 60 needs: [update_files, cli_build_macos_precompiled_libs] permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -318,14 +308,13 @@ jobs: cli_build_windows_precompiled_libs: name: Build CLI libraries - Windows - if: ${{ !inputs.resume }} runs-on: [self-hosted, Windows, X64] timeout-minutes: 60 needs: [update_files, cli_build_linux_precompiled_libs] permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -382,12 +371,11 @@ jobs: candidate_static_checks: name: Candidate static and package checks - if: ${{ !inputs.resume }} runs-on: [self-hosted, Linux] timeout-minutes: 45 needs: [update_files, cli_build_windows_precompiled_libs] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -411,16 +399,18 @@ jobs: (cd zstandard_native && dart pub get && dart analyze) (cd zstandard_cli && dart pub get && dart analyze && dart test) (cd zstandard && flutter pub get && flutter analyze && flutter test) + for package in zstandard_platform_interface zstandard_native zstandard_android zstandard_ios zstandard_linux zstandard_macos zstandard_web zstandard_windows zstandard_cli zstandard; do + (cd "$package" && dart format --output=none --set-exit-if-changed .) + done git diff --check candidate_android: name: Candidate Android - AGP 9 and legacy - if: ${{ !inputs.resume }} runs-on: [self-hosted, Linux] timeout-minutes: 120 needs: [update_files, cli_build_windows_precompiled_libs] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -430,11 +420,11 @@ jobs: - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} - - uses: actions/setup-java@v5 + - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 with: distribution: temurin java-version: '17' - - uses: android-actions/setup-android@v4 + - uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 - name: Build, run native tests and run both Android suites run: | set -euo pipefail @@ -453,29 +443,29 @@ jobs: grep -F 'id "com.android.application" version "8.11.1" apply false' "zstandard_android/$example/android/settings.gradle" grep -F 'gradle-8.14-all.zip' "zstandard_android/$example/android/gradle/wrapper/gradle-wrapper.properties" fi - (cd "zstandard_android/$example" && flutter pub get && flutter clean && flutter pub get && flutter build apk --debug --split-per-abi --target-platform android-arm,android-arm64,android-x64) - bash scripts/check_android_apk_abis.sh "zstandard_android/$example/build/app/outputs/flutter-apk" debug + (cd "zstandard_android/$example" && flutter pub get && flutter clean && flutter pub get && flutter build apk --release --split-per-abi --target-platform android-arm,android-arm64,android-x64) + bash scripts/check_android_apk_abis.sh "zstandard_android/$example/build/app/outputs/flutter-apk" release done bash scripts/run_android_emulator_ci.sh bash -c ' set -euo pipefail for example in example example_legacy; do - cd "zstandard_android/$example" - timeout --foreground 10m flutter test integration_test/android_compression_integration_test.dart -d "emulator-${EMULATOR_PORT}" --reporter expanded - timeout --foreground 10m flutter test integration_test/android_properties_integration_test.dart -d "emulator-${EMULATOR_PORT}" --reporter expanded - cd android - timeout --foreground 15m bash ./gradlew --no-daemon connectedDebugAndroidTest --stacktrace - cd ../.. + ( + cd "zstandard_android/$example" + timeout --foreground 10m flutter test integration_test/android_compression_integration_test.dart -d "emulator-${EMULATOR_PORT}" --reporter expanded + timeout --foreground 10m flutter test integration_test/android_properties_integration_test.dart -d "emulator-${EMULATOR_PORT}" --reporter expanded + cd android + timeout --foreground 15m bash ./gradlew --no-daemon connectedDebugAndroidTest --stacktrace + ) done ' candidate_linux: name: Candidate Linux build and integration - if: ${{ !inputs.resume }} runs-on: [self-hosted, Linux] timeout-minutes: 60 needs: [update_files, cli_build_windows_precompiled_libs] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -493,6 +483,7 @@ jobs: trap 'rm -f zstandard_linux/pubspec_overrides.yaml zstandard_linux/example/pubspec_overrides.yaml' EXIT (cd zstandard_linux && flutter pub get && flutter analyze && flutter test) (cd zstandard_linux/example && flutter pub get) + (cd zstandard_linux/example && flutter build linux --release) bash scripts/test_linux_integration.sh cmake_build_dir=$(find zstandard_linux/example/build/linux -type f -name CMakeCache.txt -not -path '*/_deps/*' -print | sed 's#/CMakeCache.txt$##' | sort | tail -n 1) test -n "$cmake_build_dir" @@ -502,12 +493,11 @@ jobs: candidate_web: name: Candidate Web build and integration - if: ${{ !inputs.resume }} runs-on: [self-hosted, Linux] timeout-minutes: 90 needs: [update_files, cli_build_windows_precompiled_libs] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -517,7 +507,7 @@ jobs: - uses: ./.github/actions/setup-flutter with: flutter-version: ${{ env.FLUTTER_VERSION }} - - uses: browser-actions/setup-chrome@v2 + - uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 id: chrome with: chrome-version: stable @@ -531,18 +521,17 @@ jobs: printf 'dependency_overrides:\n zstandard_native:\n path: ../../zstandard_native\n zstandard_platform_interface:\n path: ../../zstandard_platform_interface\n' > zstandard_web/example/pubspec_overrides.yaml printf 'dependency_overrides:\n zstandard_native:\n path: ../zstandard_native\n zstandard_platform_interface:\n path: ../zstandard_platform_interface\n' > zstandard_web/pubspec_overrides.yaml trap 'rm -f zstandard_web/pubspec_overrides.yaml zstandard_web/example/pubspec_overrides.yaml' EXIT - (cd zstandard_web && flutter pub get && flutter analyze) + (cd zstandard_web && flutter pub get && flutter analyze && flutter test) (cd zstandard_web/example && flutter pub get && flutter build web --release) bash scripts/test_web_integration.sh candidate_windows: name: Candidate Windows build and integration - if: ${{ !inputs.resume }} runs-on: [self-hosted, Windows, X64] timeout-minutes: 75 needs: [update_files, cli_build_windows_precompiled_libs] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -577,6 +566,8 @@ jobs: Push-Location zstandard_windows/example flutter pub get if ($LASTEXITCODE -ne 0) { throw 'Windows example pub get failed.' } + flutter build windows --release + if ($LASTEXITCODE -ne 0) { throw 'Windows release build failed.' } flutter build windows --debug if ($LASTEXITCODE -ne 0) { throw 'Windows example build failed.' } $buildDir = (Resolve-Path 'build/windows/x64').Path @@ -601,9 +592,11 @@ jobs: candidate_apple: name: Candidate Apple (${{ matrix.platform }}, ${{ matrix.dependency_manager }}) - if: ${{ !inputs.resume }} runs-on: [self-hosted, macOS, ARM64] timeout-minutes: 90 + concurrency: + group: zstandard-apple-flutter-toolchain + cancel-in-progress: false needs: [update_files, cli_build_windows_precompiled_libs] strategy: fail-fast: false @@ -612,7 +605,7 @@ jobs: platform: [ios, macos] dependency_manager: [swiftpm, cocoapods] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 @@ -653,9 +646,13 @@ jobs: fi fi if [[ "$PLATFORM" == ios ]]; then + # Flutter does not support Release mode for the iOS simulator. + # Compile an unsigned device Release artifact, then let the next + # step build and execute the behavioral suite on the simulator. + (cd zstandard_ios/example && flutter build ios --release --no-codesign) (cd zstandard_ios/example && flutter build ios --simulator --debug --no-codesign) else - (cd zstandard_macos/example && flutter build macos --debug) + (cd zstandard_macos/example && flutter build macos --release) fi - name: Run Apple integration tests env: @@ -682,11 +679,14 @@ jobs: run: | rm -f "zstandard_${{ matrix.platform }}/pubspec_overrides.yaml" rm -f "zstandard_${{ matrix.platform }}/example/pubspec_overrides.yaml" + - name: Restore Flutter dependency-manager default + if: always() + run: flutter config --no-enable-swift-package-manager - tag_version: - name: Tag immutable candidate and create draft release + ready_to_tag: + name: Release candidate ready to tag runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 10 needs: - update_files - cli_build_macos_precompiled_libs @@ -698,464 +698,32 @@ jobs: - candidate_web - candidate_windows - candidate_apple - if: >- - ${{ always() && (inputs.resume || - (needs.update_files.result == 'success' && - needs.cli_build_macos_precompiled_libs.result == 'success' && - needs.cli_build_linux_precompiled_libs.result == 'success' && - needs.cli_build_windows_precompiled_libs.result == 'success' && - needs.candidate_static_checks.result == 'success' && - needs.candidate_android.result == 'success' && - needs.candidate_linux.result == 'success' && - needs.candidate_web.result == 'success' && - needs.candidate_windows.result == 'success' && - needs.candidate_apple.result == 'success')) }} - permissions: - contents: write - outputs: - tag: ${{ steps.tag.outputs.tag }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.ref }} fetch-depth: 0 - - name: Tag exact candidate and create draft release - id: tag - env: - VERSION: ${{ inputs.version }} - TITLE: ${{ inputs.title }} - CHANGELOG_BODY: ${{ inputs.changelog }} - RESUME: ${{ inputs.resume }} - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - tag="v$VERSION" - if [[ "$RESUME" == 'true' ]]; then - git fetch origin "refs/tags/$tag:refs/tags/$tag" - git checkout --detach "refs/tags/$tag" - else - git pull --ff-only origin "$GITHUB_REF_NAME" - git tag -a "$tag" -m "Release $VERSION" - git push origin "$tag" - fi - if ! gh release view "$tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then - printf '%s\n' "$CHANGELOG_BODY" > release-notes.md - gh release create "$tag" --repo "$GITHUB_REPOSITORY" --draft \ - --title "$TITLE" --notes-file release-notes.md - fi - { - echo "Checksums for native CLI libraries at $tag" - (cd zstandard_cli/lib/src/bin && sha256sum *) - } > checksums-native-libs.txt - gh release upload "$tag" checksums-native-libs.txt --repo "$GITHUB_REPOSITORY" --clobber - echo "tag=$tag" >> "$GITHUB_OUTPUT" - platform_interface_publish_dry_run: - name: Dry run - platform interface - runs-on: [self-hosted, macOS, ARM64] - timeout-minutes: 45 - needs: [tag_version] - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - fetch-depth: 0 - - uses: ./.github/actions/setup-preinstalled-flutter - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Get dependencies and dry run - working-directory: zstandard_platform_interface - run: flutter pub get && dart pub publish --dry-run - - platform_interface_publish: - name: Publish - platform interface - runs-on: [self-hosted, macOS, ARM64] - timeout-minutes: 45 - needs: [platform_interface_publish_dry_run] - permissions: - contents: write - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - fetch-depth: 0 - - uses: ./.github/actions/setup-preinstalled-flutter - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Publish package if required - run: dart run scripts/publish_if_missing.dart zstandard_platform_interface "${{ inputs.version }}" - - name: Upload package archive - run: zip -r zstandard_platform_interface.zip zstandard_platform_interface - - uses: softprops/action-gh-release@v2 - with: - tag_name: v${{ inputs.version }} - files: zstandard_platform_interface.zip - env: - GITHUB_TOKEN: ${{ github.token }} - - zstandard_native_publish_dry_run: - name: Dry run - native package - runs-on: [self-hosted, macOS, ARM64] - timeout-minutes: 45 - needs: [platform_interface_publish] - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - fetch-depth: 0 - - uses: ./.github/actions/setup-preinstalled-flutter - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Get dependencies and dry run - working-directory: zstandard_native - run: dart pub get && dart pub publish --dry-run - - zstandard_native_publish: - name: Publish - native package - runs-on: [self-hosted, macOS, ARM64] - timeout-minutes: 45 - needs: [zstandard_native_publish_dry_run] - permissions: - contents: write - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - fetch-depth: 0 - - uses: ./.github/actions/setup-preinstalled-flutter - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Publish package if required - run: dart run scripts/publish_if_missing.dart zstandard_native "${{ inputs.version }}" - - name: Wait for native package indexing - run: dart run scripts/wait_for_pub_version.dart zstandard_native "${{ inputs.version }}" 1200 - - name: Upload package archive - run: zip -r zstandard_native.zip zstandard_native - - uses: softprops/action-gh-release@v2 - with: - tag_name: v${{ inputs.version }} - files: zstandard_native.zip - env: - GITHUB_TOKEN: ${{ github.token }} - - platform_publish_dry_runs: - name: Dry run - ${{ matrix.package }} - runs-on: ${{ fromJSON(matrix.runs_on) }} - timeout-minutes: 60 - needs: [zstandard_native_publish] - strategy: - fail-fast: false - max-parallel: 3 - matrix: - include: - - package: zstandard_android - runs_on: '["self-hosted", "Linux"]' - setup: linux - - package: zstandard_ios - runs_on: '["self-hosted", "macOS", "ARM64"]' - setup: apple - - package: zstandard_macos - runs_on: '["self-hosted", "macOS", "ARM64"]' - setup: apple - - package: zstandard_linux - runs_on: '["self-hosted", "Linux"]' - setup: linux - - package: zstandard_web - runs_on: '["self-hosted", "Linux"]' - setup: linux - - package: zstandard_windows - runs_on: '["self-hosted", "Windows", "X64"]' - setup: windows - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - fetch-depth: 0 - - uses: ./.github/actions/setup-flutter - if: matrix.setup == 'linux' - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - uses: ./.github/actions/setup-preinstalled-flutter - if: matrix.setup == 'apple' - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - uses: ./.github/actions/setup-preinstalled-flutter-windows - if: matrix.setup == 'windows' - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Wait for native package indexing - run: dart run scripts/wait_for_pub_version.dart zstandard_native "${{ inputs.version }}" 1200 - - name: Get dependencies and dry run - working-directory: ${{ matrix.package }} - if: matrix.setup != 'windows' - run: flutter pub get && dart pub publish --dry-run - - name: Get dependencies and dry run (Windows) - working-directory: ${{ matrix.package }} - if: matrix.setup == 'windows' - shell: pwsh - run: flutter pub get; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; dart pub publish --dry-run - - platform_publishes: - name: Publish - ${{ matrix.package }} - runs-on: ${{ fromJSON(matrix.runs_on) }} - timeout-minutes: 60 - needs: [platform_publish_dry_runs] - strategy: - fail-fast: false - max-parallel: 3 - matrix: - include: - - package: zstandard_android - runs_on: '["self-hosted", "Linux"]' - setup: linux - - package: zstandard_ios - runs_on: '["self-hosted", "macOS", "ARM64"]' - setup: apple - - package: zstandard_macos - runs_on: '["self-hosted", "macOS", "ARM64"]' - setup: apple - - package: zstandard_linux - runs_on: '["self-hosted", "Linux"]' - setup: linux - - package: zstandard_web - runs_on: '["self-hosted", "Linux"]' - setup: linux - - package: zstandard_windows - runs_on: '["self-hosted", "Windows", "X64"]' - setup: windows - permissions: - contents: write - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - fetch-depth: 0 - - uses: ./.github/actions/setup-flutter - if: matrix.setup == 'linux' - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - uses: ./.github/actions/setup-preinstalled-flutter - if: matrix.setup == 'apple' - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - uses: ./.github/actions/setup-preinstalled-flutter-windows - if: matrix.setup == 'windows' - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Wait for native package indexing - shell: bash - run: dart run scripts/wait_for_pub_version.dart zstandard_native "${{ inputs.version }}" 1200 - - name: Publish package if required - shell: bash - run: dart run scripts/publish_if_missing.dart "${{ matrix.package }}" "${{ inputs.version }}" - - name: Archive package - if: matrix.setup != 'windows' - run: zip -r "${{ matrix.package }}.zip" "${{ matrix.package }}" - - name: Archive package (Windows) - if: matrix.setup == 'windows' - shell: pwsh - run: Compress-Archive -Path "${{ matrix.package }}" -DestinationPath "${{ matrix.package }}.zip" -Force - - uses: softprops/action-gh-release@v2 - with: - tag_name: v${{ inputs.version }} - files: ${{ matrix.package }}.zip - env: - GITHUB_TOKEN: ${{ github.token }} - - cli_publish_dry_run: - name: Dry run - CLI - runs-on: [self-hosted, Linux] - timeout-minutes: 60 - needs: [platform_publishes] - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - fetch-depth: 0 - - uses: ./.github/actions/setup-flutter - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Wait for platform package indexing - run: | - set -euo pipefail - for package in zstandard_android zstandard_ios zstandard_linux zstandard_macos zstandard_web zstandard_windows; do - dart run scripts/wait_for_pub_version.dart "$package" "${{ inputs.version }}" 1200 - done - - name: Get dependencies and dry run - working-directory: zstandard_cli - run: dart pub get && dart pub publish --dry-run - - cli_publish: - name: Publish - CLI - runs-on: [self-hosted, Linux] - timeout-minutes: 60 - needs: [cli_publish_dry_run] - permissions: - contents: write - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - fetch-depth: 0 - - uses: ./.github/actions/setup-flutter - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Publish package if required - run: dart run scripts/publish_if_missing.dart zstandard_cli "${{ inputs.version }}" - - name: Upload package archive - run: zip -r zstandard_cli.zip zstandard_cli - - uses: softprops/action-gh-release@v2 - with: - tag_name: v${{ inputs.version }} - files: zstandard_cli.zip - env: - GITHUB_TOKEN: ${{ github.token }} - - zstandard_publish_dry_run: - name: Dry run - main plugin - runs-on: [self-hosted, Linux] - timeout-minutes: 60 - needs: [cli_publish] - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - fetch-depth: 0 - - uses: ./.github/actions/setup-flutter - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Wait for all published packages - run: | - set -euo pipefail - for package in zstandard_platform_interface zstandard_native zstandard_android zstandard_ios zstandard_linux zstandard_macos zstandard_web zstandard_windows zstandard_cli; do - dart run scripts/wait_for_pub_version.dart "$package" "${{ inputs.version }}" 1200 - done - - name: Get dependencies and dry run - working-directory: zstandard - run: flutter pub get && dart pub publish --dry-run - - zstandard_publish: - name: Publish - main plugin - runs-on: [self-hosted, Linux] - timeout-minutes: 60 - needs: [zstandard_publish_dry_run] - permissions: - contents: write - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - fetch-depth: 0 - - uses: ./.github/actions/setup-flutter - with: - flutter-version: ${{ env.FLUTTER_VERSION }} - - name: Publish package if required - run: dart run scripts/publish_if_missing.dart zstandard "${{ inputs.version }}" - - name: Upload package archive - run: zip -r zstandard.zip zstandard - - uses: softprops/action-gh-release@v2 - with: - tag_name: v${{ inputs.version }} - files: zstandard.zip - env: - GITHUB_TOKEN: ${{ github.token }} - - verify_published_packages: - name: Verify pub.dev - ${{ matrix.package }} - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [zstandard_publish] - strategy: - fail-fast: false - matrix: - package: - - zstandard_platform_interface - - zstandard_native - - zstandard_android - - zstandard_ios - - zstandard_linux - - zstandard_macos - - zstandard_web - - zstandard_windows - - zstandard_cli - - zstandard - steps: - - uses: actions/checkout@v6 - with: - ref: refs/tags/v${{ inputs.version }} - - name: Verify immutable package version - run: dart run scripts/wait_for_pub_version.dart "${{ matrix.package }}" "${{ inputs.version }}" 1200 - - finalize_release: - name: Finalize GitHub release - runs-on: ubuntu-latest - timeout-minutes: 10 - needs: [verify_published_packages] - permissions: - contents: write - steps: - - name: Mark draft release as final - env: - GH_TOKEN: ${{ github.token }} - run: gh release edit "v${{ inputs.version }}" --repo "$GITHUB_REPOSITORY" --draft=false - - rollback_guide: - name: Release recovery guide - runs-on: ubuntu-latest - if: >- - ${{ always() && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }} - needs: - - update_files - - cli_build_macos_precompiled_libs - - cli_build_linux_precompiled_libs - - cli_build_windows_precompiled_libs - - candidate_static_checks - - candidate_android - - candidate_linux - - candidate_web - - candidate_windows - - candidate_apple - - tag_version - - platform_interface_publish_dry_run - - platform_interface_publish - - zstandard_native_publish_dry_run - - zstandard_native_publish - - platform_publish_dry_runs - - platform_publishes - - cli_publish_dry_run - - cli_publish - - zstandard_publish_dry_run - - zstandard_publish - - verify_published_packages - - finalize_release - steps: - - name: Write recovery instructions + - name: Verify exact candidate at the remote branch tip env: VERSION: ${{ inputs.version }} + RELEASE_BRANCH: ${{ github.ref_name }} run: | + set -euo pipefail + git fetch origin "$RELEASE_BRANCH" --tags + remote_sha="$(git rev-parse "origin/$RELEASE_BRANCH")" + git checkout --detach "$remote_sha" + dart scripts/check_pub_metadata.dart --release-version "$VERSION" + test -s zstandard_cli/lib/src/bin/libzstandard_macos.dylib + test -s zstandard_cli/lib/src/bin/libzstandard_linux_x64.so + test -s zstandard_cli/lib/src/bin/libzstandard_linux_arm64.so + test -s zstandard_cli/lib/src/bin/zstandard_windows_x64.dll + test -s zstandard_cli/lib/src/bin/zstandard_windows_arm64.dll { - echo '## Release recovery' - echo - echo "The release workflow did not complete for **$VERSION**. Published pub.dev versions are immutable." + echo '## Release candidate validated' echo - echo '1. Inspect the first failed job and fix the runner, dependency, or credential problem.' - echo '2. If the tag already exists, rerun this workflow with `resume: true`; already published packages are skipped safely.' - echo '3. If the tag does not exist, rerun with `resume: false` after fixing the cause.' - echo '4. Never force-move an existing release tag or republish a different archive under the same version.' + echo "Candidate: \`$remote_sha\`" echo - echo 'Publication order: platform interface → native → platform packages → CLI → main plugin.' + echo "Push the immutable tag \`v$VERSION\` at exactly this commit." + echo 'The tag-triggered Publish workflow will revalidate the tag and publish with OIDC.' } >> "$GITHUB_STEP_SUMMARY" - - notify_deploy_completed: - name: Notify deploy completed - runs-on: ubuntu-latest - needs: [finalize_release] - steps: - - name: Git Board Flow - deploy success notification - continue-on-error: true - uses: vypdev/copilot@v2 - with: - debug: ${{ vars.DEBUG }} - single-action: deployed_action - single-action-issue: ${{ inputs.issue }} - token: ${{ secrets.PAT }} diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index a21e3fb1..2b2f833e 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v5 + - uses: actions/stale@f7176fd3007623b69d27091f9b9d4ab7995f0a06 with: operations-per-run: 100 days-before-issue-stale: 30 diff --git a/.gitignore b/.gitignore index 17fa42b1..cf84e370 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ zstandard_cli/zstd/ .zstd_upstream/ +.DS_Store .idea/ */.idea/ */build/ @@ -29,3 +30,4 @@ zstandard_cli/zstd/ **/windows/flutter/generated_plugin_registrant.* **/windows/flutter/generated_plugins.cmake **/example/pubspec.lock +**/pubspec_overrides.yaml diff --git a/AUDIT_REPORT.md b/AUDIT_REPORT.md new file mode 100644 index 00000000..568a5adf --- /dev/null +++ b/AUDIT_REPORT.md @@ -0,0 +1,675 @@ +# Zstandard federated plugin — integrity audit and remediation report + +Audit snapshot: 2026-09-07 UTC + +Baseline: `develop` at `16a489fd9fb95076ebcad1c25792c90fcc0a4891` + +Scope: the complete repository, all ten packages, examples, native source, +generated Web assets, CI, release automation, documentation, and observable +GitHub configuration. + +Important: findings describe the baseline; their status describes the remediated +branch and its remote validation. The changes were committed and pushed to +`codex/integrity-audit`, and pull request 144 was opened. No package publication, +release publication, release/integration workflow, or ruleset mutation was +performed. + +## 1. Executive summary + +The repository contained real release-blocking defects: decompression trusted +frame metadata and had no output bound, native pointers crossed Dart isolate +boundaries, Web work ran on the browser thread, empty input did not consistently +produce a valid zstd frame, release publication was not safely separated from +preparation, and several test/CI paths could report misleading success. These +issues were reproduced and corrected after explicit authorization. + +The remediated tree now has one canonical zstd 1.5.7 source, a shared bounded +streaming native codec, byte-only isolate transfers, dedicated Web Workers, +deterministic Emscripten assets, safer CLI behavior, tested Apple dual dependency +systems, modern and legacy Android consumers, immutable Action pins, and a +prepare/tag/OIDC-publish/integrate release sequence. + +Local verification is strong on macOS/Apple Silicon, Android/ARM64, iOS +Simulator, and Chrome. Remote workflows additionally passed on Linux, Windows, +Android x86_64 emulation, and all package/platform matrices. Intel macOS runtime +and physical iOS-device execution were not available; both Apple modes did pass +their supported build and Simulator/macOS integration coverage. The repository +changes and current-commit Safety Gate are complete. The only release-governance +blocker intentionally deferred by the owner is GitHub ruleset 2541601, which is +disabled and contains stale required-check names. + +## 2. Repository state + +- Baseline branch was clean `develop`, exactly tracking `origin/develop` at + `16a489f`; remote is `https://github.com/vypdev/zstandard.git`. +- The remediated branch is `codex/integrity-audit`; it is pushed and proposed by + pull request 144. The source, tests, workflows, documentation, generated Web + artifacts, and this report are committed in reviewable thematic changes. +- `develop...master` was `76 6`; merge-base was + `1e6ca981f608cbb916ad92ecb726f99992221e44`. +- Local release branches: `release/1.3.30`, `release/1.3.31`, and + `release/1.4.1`; more historical release branches exist remotely. Latest + normal version tag is `v1.5.0`. A malformed tag named `v` also exists. +- Git object database: 3,179 loose objects/20.03 MiB plus one 109.37 MiB pack. + The historic pack retains old media even though the current tree no longer + does. +- No submodule, Git LFS object, unexpected symlink, provisioning profile, + certificate, keystore, private key, embedded credential, or unexpected + executable was found. +- Removed unreferenced tracked media (`media/publish.gif`, 54.2 MiB; + `media/update_files.gif`, 4.0 MiB; and its orphaned README). No tracked/current + file now exceeds 10 MiB. +- All audit-generated locks, package overrides, Pods, `.dart_tool`, `build`, + `.cxx`, coverage, and Apple `Classes/zstd` staging directories were cleaned. + +## 3. Package and dependency graph + +All expected packages exist and are version `1.5.0`; no orphan or cycle was +found. + +```text +zstandard +├── zstandard_platform_interface +├── zstandard_android ─┬─ zstandard_platform_interface +│ └─ zstandard_native +├── zstandard_ios ─────┬─ zstandard_platform_interface +│ └─ zstandard_native +├── zstandard_macos ───┬─ zstandard_platform_interface +│ └─ zstandard_native +├── zstandard_linux ───┬─ zstandard_platform_interface +│ └─ zstandard_native +├── zstandard_windows ─┬─ zstandard_platform_interface +│ └─ zstandard_native +└── zstandard_web ─────── zstandard_platform_interface + +zstandard_cli ─────────── zstandard_native +Apple SPM packages ────── repository zstandard-native product +Apple CocoaPods ───────── staged copy generated from zstandard_native +Web examples ──────────── four generated JS/WASM assets +``` + +Examples resolve their corresponding plugin by path. Local and CI validation use +ignored `pubspec_overrides.yaml` files generated by the cross-platform +`scripts/create_local_overrides.dart` entry point; the shell helper is only a +wrapper, and no override is included in the result. + +## 4. Canonical native-source integrity + +- The only persistent canonical C tree is `zstandard_native/src/zstd` (106 + files). Searches found no second committed zstd tree. +- Provenance is pinned in `zstandard_native/UPSTREAM_ZSTD.md:3-15`: upstream + zstd 1.5.7 base `d7ee3207cc0db53f78fc6a69babc80747b1b7658` plus upstream + allocation-null-check commit `3f8f9b3f89244638f10bca664c120fd28cb14efe`. +- `scripts/update_zstd.sh` defaults to those immutable revisions and records the + resolved revision. Apple CocoaPods staging is generated and removed; SwiftPM + directly targets `zstandard_native/src/zstd` (`Package.swift:20-55`). +- Required streaming and error symbols were present in strict Release output. +- Independent Apple-Clang builds passed Debug with `-Wall -Wextra -Wpedantic + -Werror`, Release with the same warning policy, ASan+UBSan, and TSan. The + native harness covered empty, one byte, UTF-8, zeros, deterministic random + 8 MiB, malformed/truncated data, repetition, and 8 threads × 100 iterations. +- Apple's ASan runtime does not support `detect_leaks=1`; memory-safety execution + passed with `detect_leaks=0`, while repeated-call tests and cleanup-path review + supplied complementary evidence. This is not equivalent to a Linux LSan run. + +## 5. Public API and FFI audit + +- `Zstandard.decompress` now accepts `maxOutputSize`, defaulting to 256 MiB + (`zstandard/lib/zstandard.dart:25-77`). A negative bound is rejected. +- Compatibility is preserved: existing third-party implementations can retain + `ZstandardPlatform.decompress`; implementations opt into + `BoundedZstandardPlatform` (`zstandard_platform_interface/lib/src/zstandard_platform.dart:16-70`). +- `ZstandardNativeCodec` owns all allocations and frees them in `finally` + (`zstandard_native/lib/src/zstandard_native_codec.dart:14-152`). It uses + `ZSTD_decompressStream`, supports unknown-size and concatenated frames, detects + truncation/malformed input, and checks the limit before appending output. +- Native platform classes pass only `Uint8List`/integers into `Isolate.run`; + allocation, symbol calls, and free happen within that isolate. No native + pointer crosses an isolate boundary. +- Empty compression calls real zstd and returns a valid non-empty frame. +- Size types use generated `Size`/`UnsignedLongLong` bindings appropriate to the + C signatures; pointer lifetimes and null/error checks were traced through all + paths. There is no public native handle or struct-layout exposure. +- Synchronous native work is moved off the caller isolate on Flutter native + platforms. CLI calls remain asynchronous at the Dart API but perform native + work in-process, which is appropriate for a command-line process. + +## 6. Platform-by-platform results + +### Android + +The AGP 9.1.0/Gradle 9.3.1 modern example and AGP 8.11.1/Gradle 8.14 legacy +example built Debug and Release split APKs for `armeabi-v7a`, `arm64-v8a`, and +`x86_64`. APK/ELF inspection confirmed the expected ABI directories and zstd +symbols. Both examples passed 6 functional tests, 100 deterministic generative +round trips, and 10 JNI instrumentation checks on clean API 31 ARM64 emulation. +The modern example also passed on the connected API 36 ARM64 emulator. +Configuration-cache strict mode stored and then reused its entry. The test JNI +bridge is Debug-only (`zstandard_android/zstd_build/CMakeLists.txt:62-74`) and +was absent from Release symbols. The hosted Android workflow then repeated both +examples against workspace and exact package-config native sources on isolated +x86_64 API 35 emulators. + +### iOS + +CocoaPods and SwiftPM each compiled Debug for iPhone 16 Simulator and unsigned +Release for the generic device destination. Each dependency mode passed the +eight real Simulator integration cases, including exact byte round trips and +controlled invalid input. Exact release pins and local-source injection were +verified. A physical device was not available. Flutter/Xcode does not provide a +representative Simulator Release runtime, so device Release compilation plus +Simulator Debug execution was used and recorded separately. + +### macOS + +CocoaPods and SwiftPM each built Release and ran the eight integration cases on +Apple Silicon. A universal CLI dylib was separately produced and inspected as +`arm64` + `x86_64`, but the plugin was not run on Intel hardware. + +### Web + +Direct and federated Release web builds passed, as did `flutter build web +--wasm --release`. Chrome integration exercised small/large/empty/random data, +invalid input, unknown-size frames, concatenated frames, bounds, repetition, +properties, and event-loop yielding through a real Worker. Browser execution +ended with the required `All tests passed` marker and exit 0. + +### CLI + +All 34 tests passed against the real shared native library. File and binary +stdin/stdout round trips, empty/large inputs, concatenated frames, bounds, +invalid options/data, collision refusal, force behavior, exit codes, +concurrency, and symbol resolution were checked. An independent file round trip +also passed `cmp`. + +### Linux and Windows + +The current branch passed its dedicated Linux and Windows workflows. Linux built +the plugin shared object, ran CTest/native symbol checks, executed the Flutter +package against that exact built library, repeated resolution through an exact +package config, and passed its publish dry run. Windows passed analysis, MSVC +C++/package/integration tests, exact package-config source resolution, the +federated example, and its publish dry run. The results are current-tree runtime +evidence rather than historical baseline results. + +## 7. Test-quality assessment + +- Real native/browser/instrumentation tests now assert meaningful non-empty + compressed output and byte-for-byte decompressed equality. +- Deterministic property suites use 100 non-empty arrays on both Android + examples; CLI/main/interface property tests also passed. +- Malformed, truncated, random, unknown-size, concatenated, empty, large, + repeated, concurrent, and output-limit behavior is represented where the + platform can execute it. +- Two tests that only printed `WIP` were removed. Platform skips are no longer + disguised by a placeholder success. +- Emulator and Chrome scripts have bounded commands and cleanup traps. Final + checks found no temporary emulator, ChromeDriver listener on port 4444, or + audit-generated build directory. +- Main-package unit tests intentionally use a mock to test dispatch; real core + behavior is supplied by native, CLI, platform integration, instrumentation, + Apple, and browser suites rather than being inferred from those mocks. + +## 8. CI/CD audit + +- All 23 workflow/composite YAML files parsed; actionlint 1.7.12 reported no + issue. All shell scripts passed `bash -n`. `yamllint`, `shellcheck`, and local + PowerShell were unavailable; the Windows workflow supplied native command and + path validation, while equivalent syntax/logic inspection covered the other + unavailable local tools. +- Every external Action reference is pinned to a 40-character commit SHA. +- Untrusted pull requests receive read-only contents permission on + GitHub-hosted runners and aggregate into the stable `Pull Request Safety Gate` + context (`.github/workflows/pull_request.yml:1-103`). Self-hosted platform + workflows do not execute fork code with secrets/write authority. +- Platform workflows use timeouts, concurrency, unique artifacts, failure + propagation, and cleanup. Apple jobs serialize global Flutter configuration. +- Native source is tested both from the workspace and an exact package-config + resolution, preventing accidental reliance on stale pub cache state. +- Pull request 144 has a successful current-commit `Pull Request Safety Gate`. + Every platform/package workflow was dispatched from the remediated tree. When + all Linux self-hosted runners were offline, an ephemeral branch changed only + runner selection (plus isolated Android matrix parallelism) and supplied the + equivalent GitHub-hosted evidence; it was not proposed for merge. +- Pull-request coverage now runs as a required hosted matrix on every head. It + enforces CLI/platform-interface/main-package thresholds and uploads the three + corresponding Codecov flags for trusted same-repository branches. CLI reports + are scoped to its own `lib/`, excluding tests and dependency sources. + +## 9. Release-process audit + +The remediated process is intentionally split: + +1. `release_workflow.yml` validates `release/x.y.z`, synchronizes versions, + changelogs and artifacts, then builds/tests the candidate. +2. A reviewed immutable `vX.Y.Z` tag triggers `publish_workflow.yml`. +3. Tag identity is checked against the release branch tip. +4. OIDC grants `id-token: write` only to the publish job. +5. Publication order is interface + native, platform packages, CLI, then main; + each step performs a dry run, safely skips an exact already-published + version, and waits for pub.dev propagation before dependents continue. +6. Verification precedes GitHub release finalization. +7. `integrate_release.yml` separately opens/reuses reviewed PRs into `develop` + and `master`; it never merges or bypasses protection. + +The version updater was simulated with `1.5.1` and all ten package metadata +records matched. No tag, OIDC token, package, GitHub release, or integration PR +was created. + +## 10. Dependency and supply-chain findings + +- Internal constraints are consistently `^1.5.0`; the candidate updater changes + all of them together. Direct constraints resolved successfully. +- No dependency override remains in the deliverable. Package libraries do not + import dev-only dependencies at runtime. +- zstd and Emscripten inputs are immutable; Emscripten is pinned to 3.1.69. +- SwiftPM release dependencies are exact (`1.5.0`); local environment-path + injection is used only for audited workspace builds. +- Action pins are immutable. Remote tool execution is confined to named setup + steps and audited scripts. +- Pub reported newer transitive/incompatible versions, but no upgrade was made, + per audit instructions. `cli_util 0.6`, `ffigen 21`, analyzer/test/package + config, and related major/minor moves need separate compatibility PRs. +- No incompatible license or exposed credential was found. This was a source + and manifest review, not a commercial legal opinion or authenticated + vulnerability-database attestation. + +## 11. Documentation and pub.dev findings + +README, API, architecture, platform, security, testing, build, CI, release, and +runbook documentation were reconciled with the implementation. They now state +the 256 MiB bound, frame semantics, Worker/isolate behavior, Apple dual package +systems, Android tool lines, native-source ownership, CLI flags, and honest +platform limitations. + +`scripts/check_pub_metadata.dart` validates all ten packages. Cross-platform +local overrides let each package dry-run against the coordinated unpublished +monorepo graph: all ten `dart pub publish --dry-run` commands completed with zero +warnings other than the expected override notice. The `1.5.1` release simulation +and ordered wait logic cover the real dependency transition without publishing. + +## 12. Performance and robustness findings + +- Native decompression is streaming and bounded; it no longer allocates based on + untrusted frame metadata. Web streaming uses bounded chunks and frees its + stream/input/output structures. +- Flutter native calls use worker isolates and Web uses a dedicated Worker, + avoiding the UI/event-loop blocking observed in the baseline. +- One byte-list transfer/copy at Dart/native and Worker boundaries remains. It is + a deliberate ownership/safety tradeoff; no stable zero-copy API was exposed. +- Benchmarks now reject non-finite timings and report microseconds. They are + smoke/regression helpers, not statistically controlled cross-platform + performance claims. +- 8 MiB native, large Flutter/browser, 100-case generative, repeated, and + concurrent loads passed. Peak-RSS/allocation-count profiling and Android + low-memory process-kill recovery were not measured. +- Current Web SHA-256 values: + - `zstd.js`: `992ad9e5dcd8aa2bf8cd827043830f0cd3f4b41b8b63c3eff2ffe7ec9f4f383e` + - `zstd_core.js`: `f9f132a3c3f2b06ec939dd97ee5ae2ca3fa91b93a84abec60d1bf8d4c1c96081` + - `zstd_worker.js`: `46eff474d7fa9304dc8c97d764e339aca45f24d1e04254cadca16ab3b8f93c38` + - `zstd.wasm`: `7bee44e804382c3145523067ba3db2e2195b2dba72787de14cb7fc67f86a1ed1` + +## 13. Security findings + +- P0 memory/resource-exhaustion paths found in decompression were remediated + with bounded streaming and controlled `null` failure. +- Allocation failure before `memset` in vendored upstream code is guarded. +- Pointer ownership stays within the executing isolate; all reviewed native + allocations have cleanup paths. +- CLI refuses accidental input overwrite and existing-output replacement unless + explicitly forced. +- Web output bounds are enforced inside the Worker, not after an unbounded result + reaches Dart. +- Secret-pattern and sensitive-file searches found no credential. Workflow + permissions and fork trust boundaries were reduced. +- The disabled GitHub ruleset remains the sole deferred external + security/release-governance blocker; it was not modified, as requested. + +## 14. Full findings table + +| ID | Sev. | Area / affected | Exact location | Status | Release blocker | Confidence | +|---|---|---|---|---|---|---| +| F-001 | P0 | Unbounded decompression, native/all native platforms | `zstandard_native/lib/src/zstandard_native_codec.dart:65-149` | Resolved | Was yes | High | +| F-002 | P1 | Empty input was not a valid zstd frame | `zstandard_native/lib/src/zstandard_native_codec.dart:22-56` | Resolved | Was yes | High | +| F-003 | P0 | Native pointers crossed isolate boundaries | `zstandard_android/lib/zstandard_android.dart:65-78` and peer platform files | Resolved | Was yes | High | +| F-004 | P1 | Web compressed/decompressed on main thread | `scripts/build_web_wasm.sh:353-387` | Resolved | Was yes | High | +| F-005 | P0 | Web lacked streaming output bound | `scripts/build_web_wasm.sh:195-280` | Resolved | Was yes | High | +| F-006 | P2 | Native provenance/update was not reproducibly pinned | `zstandard_native/UPSTREAM_ZSTD.md:3-15` | Resolved | No | High | +| F-007 | P1 | CLI I/O, options, collision and exit behavior incomplete | `zstandard_cli/lib/src/cli_runner.dart:63-282` | Resolved | Was yes for CLI | High | +| F-008 | P1 | CLI library lookup/ABI validation fragile | `zstandard_cli/lib/src/utils/lib_loader.dart:11-85` | Resolved | Was yes for CLI | High | +| F-009 | P3 | Benchmarks could report non-finite/ambiguous values | `zstandard_cli/benchmark/benchmark_suite.dart:1-88` | Resolved | No | High | +| F-010 | P2 | Mutable/unsafe workflow execution paths | `.github/workflows/copilot_commit.yml:1` and `hotfix_workflow.yml:1` (deleted) | Resolved | Was yes | High | +| F-011 | P0 | Release preparation/publication/integration coupled unsafely | `.github/workflows/release_workflow.yml:1-724`; `publish_workflow.yml:1-176`; `integrate_release.yml:1-131` | Resolved in repo | Ruleset only | High | +| F-012 | P2 | Apple source-sync race could delete inputs mid-build | `zstandard_ios/scripts/sync_zstd.sh:1-90` and macOS peer | Resolved | Was yes | High | +| F-013 | P2 | CI could validate stale pub-cache native source | `scripts/create_local_overrides.dart`; platform workflow source matrices | Resolved | No | High | +| F-014 | P2 | Web runner could print success after a failed drive | `scripts/test_web_integration.sh:64-153` | Resolved | Was yes | High | +| F-015 | P2 | Android instrumentation JNI symbols shipped in Release | `zstandard_android/zstd_build/CMakeLists.txt:62-74` | Resolved | Was yes | High | +| F-016 | P3 | Unreferenced 58 MiB media inflated the tree | `media/publish.gif`, `media/update_files.gif` (deleted binaries) | Resolved | No | High | +| F-017 | P3 | Version regex consumed vertical whitespace | `.github/scripts/update_versions.dart:184-192` | Resolved | No | High | +| F-018 | P2 | Ruleset disabled with stale required contexts | GitHub ruleset `2541601`, `PR Checks` | Open/deferred by owner | **Yes** | High | +| F-019 | P2 | Supported-host/runtime validation unavailable | Platform workflow matrix and remote runs below | Resolved for supported CI matrix | No | High | +| F-020 | P3 | Legacy AGP line emits future-support warnings | `zstandard_android/example_legacy/android/settings.gradle:1-25` | Open/planned | No today | High | +| F-021 | Info | Published 1.5.0 graph diverged from coordinated source dry-run | each package `pubspec.yaml:3`; local override helper | Resolved for validation | No | High | +| F-022 | P3 | Placeholder `WIP` tests produced meaningless passes | `zstandard/test/tests_exist_elsewhere_test.dart:1` and Linux peer (deleted) | Resolved | No | High | +| F-023 | P3 | CLI used deprecated platform APIs | `zstandard_cli/lib/src/zstandard_cli_base.dart:30-44` | Resolved | No | High | +| F-024 | P3 | Root Android analysis treated legacy consumer in wrong context | `zstandard_android/analysis_options.yaml:2-6` | Resolved | No | High | +| F-025 | P3 | Malformed remote tag/release `v` | GitHub release and `refs/tags/v` | Resolved | No | High | +| F-026 | P1 | CLI package discovery failed under `flutter_test` | `zstandard_cli/lib/src/utils/lib_loader.dart` | Resolved | Was yes for CLI | High | +| F-027 | P2 | Monorepo validation was shell-dependent and dry-runs resolved old packages | `scripts/create_local_overrides.dart`; check workflows | Resolved | Was yes | High | +| F-028 | P2 | Linux package tests did not load the library built by CI | `.github/workflows/pr_check_linux.yml` | Resolved | Was yes for Linux | High | +| F-029 | P2 | Shared Apple concurrency canceled matrix rows | `.github/workflows/pr_check_ios.yml`; `pr_check_macos.yml` | Resolved | Was yes | High | +| F-030 | P2 | Android AVD creation assumed an implicit home | `scripts/run_android_emulator_ci.sh` | Resolved | Was yes for hosted CI | High | +| F-031 | P2 | Codecov PR data was stale and upload counts mismatched the base | `.github/workflows/pull_request.yml`; coverage workflows | Resolved | No | High | + +Finding records (evidence, reproduction, expectation, impact, root cause, fix, +and regression proof): + +- **F-001:** Baseline frame-size allocation was reproducible with unknown-size or + hostile content-size frames. Expected bounded controlled failure; actual could + trust metadata/allocate excessively, risking OOM/crash. Root cause was a + one-shot API. Fix: streaming decoder and pre-append limit. Regression: native, + CLI, Android, Apple and browser bounds/unknown/concat tests. Resolved, high + confidence. +- **F-002:** Compressing zero bytes returned an empty byte array rather than a + zstd frame. Expected non-empty valid frame and exact empty round trip; actual + was not interoperable. Fix: always call zstd. Regression passed across native, + CLI, Android, Apple and Web. Resolved, high confidence. +- **F-003:** Baseline platform implementations created/used FFI state around + isolate callbacks. Expected pointer ownership in one isolate; actual lifetime + could cross isolate boundaries, risking invalid memory access. Fix: shared + codec instantiated/called inside `Isolate.run` with byte data only. Real + round-trip, repetition and sanitizer tests passed. Resolved, high confidence. +- **F-004:** Baseline Web calls performed WASM work on the UI thread. Expected + responsive event loop; actual large work could freeze it. Fix: persistent + Worker broker with transferable buffers. Browser test proves a scheduled event + executes during compression. Resolved, high confidence. +- **F-005:** Baseline Web decompression did not enforce an in-worker output cap + or robustly handle unknown/concatenated frames. Expected streaming/bounded + semantics equal to native; actual risk was memory exhaustion or failed valid + frames. Fix and tests are at the cited lines. Resolved, high confidence. +- **F-006:** Source version could not be reproduced from a documented immutable + revision. Expected provenance and deterministic updater; actual audit trail was + incomplete. Fix: pinned base/backport and provenance file. Rebuild/checksum + reproduction passed. Resolved, high confidence. +- **F-007:** Baseline CLI lacked complete binary stdin/stdout, safe defaults, + collision policy, validated levels/limits and stable non-zero failures. + Expected normal Unix CLI semantics; impact was corruption or unusability. + Central parser/runner implemented; 34 tests and file `cmp` passed. Resolved, + high confidence. +- **F-008:** Library lookup depended on working directory/partial symbols. + Expected executable/package-relative resolution and full ABI check; actual + deployments could fail late. Resolver and symbol inspection tests now cover + required streaming APIs and macOS universal output. Resolved, high confidence. +- **F-009:** Timing division could produce infinite/ambiguous throughput. + Expected finite microsecond accounting; fix validates elapsed time/result. + Benchmark smoke execution passed. Resolved, high confidence. +- **F-010:** Mutable Actions and privileged automation expanded supply-chain and + fork risk. Expected immutable pins and least privilege; actual workflows could + execute changing code. Every external Action is SHA-pinned, trusted-actor + checks were tightened, and obsolete commit/hotfix workflows removed. actionlint + passed. Resolved, high confidence. +- **F-011:** Baseline release automation mixed moving-branch preparation, + publication and integration. Expected reviewed immutable tag publication, + dependency waits, idempotency and separate merges; actual partial/unsafe + publication was possible. Split workflows and dry-run-first publisher were + simulated at 1.5.1. No real publication was attempted. Repo fix complete; + remote rules remain blocking. +- **F-012:** Concurrent Apple jobs could delete shared `Classes/zstd` while + another compiled. Expected atomic/serialized staging; actual intermittent file + loss was reproducible in workflow structure. Atomic sync plus workflow locking + implemented. Both CocoaPods/SPM builds/integrations passed. Resolved, high + confidence. +- **F-013:** Native source fallback could silently select an older pub-cache + package. Expected exact workspace or package-config resolution; actual test + provenance was ambiguous. Explicit overrides and a two-source CI matrix now + verify both. Resolved, high confidence. +- **F-014:** Bash 3 scoping and incomplete status handling allowed the Web helper + to announce success without proving `flutter drive`. Expected exit 0 plus the + actual success marker; fixed runner requires both and always cleans logs/ + driver. Final real Chrome run passed. Resolved, high confidence. +- **F-015:** APK Release symbol inspection found the instrumentation bridge. + Expected test JNI only in Debug; actual production surface included it. + Generator-expression source selection fixed it. Debug retained both JNI + symbols, Release retained zstd streaming symbols and no test bridge; + instrumentation still passed. Resolved, high confidence. +- **F-016:** `git ls-files`/size inspection found two unreferenced GIFs totaling + about 58 MiB. Expected source-focused package repository; actual clone/history + bloat. Current-tree files were removed (recoverable from Git); historic pack + remains. Resolved for current tree, high confidence. +- **F-017:** A multiline version regex used broad whitespace and consumed the + following blank line. Expected one-line replacement; actual simulation created + unrelated formatting diffs. Horizontal whitespace restriction fixed it and + 1.5.1 simulation passed. Resolved, high confidence. +- **F-018:** Read-only GitHub API inspection returned ruleset 2541601 as disabled + and containing obsolete contexts including names with `(from pub cache)`. + Expected an active rule requiring the stable aggregate gate; actual protection + cannot enforce this tree. Fix requires repository-admin action: activate the + rule and require only `Pull Request Safety Gate` after its first successful + run. The current-commit gate now exists and passes; activation remains deferred + by explicit owner instruction and is the sole release-governance blocker. +- **F-019:** Initial host inventory contained only macOS ARM64 and Android ARM64. + The amended current-tree workflows subsequently passed on Linux, Windows, and + Android x86_64 as well as macOS, iOS Simulator, Web, CLI, interface, and main + package matrices. Intel macOS and physical iOS hardware were unavailable, but + their generic Release/device build targets passed. Resolved for the supported + CI matrix; residual hardware-specific risk is documented below. +- **F-020:** Flutter 3.47 warns that Gradle 8.14, AGP 8.11.1 and Kotlin 2.2.20 + support will soon be dropped. Expected a deliberately supported legacy line; + actual still builds/tests today but has a finite maintenance horizon. Keep it + as compatibility coverage and schedule a migration when Flutter raises its + floor. Open, non-blocking today, high confidence. +- **F-021:** Running dependent package dry-runs at unchanged version 1.5.0 + initially resolved already-published old dependencies, which lack the new + bounded API. Cross-platform local overrides now validate the coordinated + source graph; all ten dry-runs pass. Candidate-version simulation, ordered + publication, and propagation waits protect the real transition. Informational. +- **F-022:** Two tests only printed `WIP`. Expected meaningful assertions or an + honest skip; actual added green counts without validation. Files removed; main + now passes 35 meaningful tests and Linux reports all platform tests skipped on + macOS. Resolved, high confidence. +- **F-023:** Resolving current `platform 3.2.0` produced five deprecation infos. + Expected zero analyzer issues; actual used `LocalPlatform`. Migrated to + `NativePlatform.current`, preserving returned version semantics. Analyzer and + all 34 CLI tests passed. Resolved, high confidence. +- **F-024:** A clean root Android `flutter analyze` descended into the independent + legacy package before its SDK package config existed. Expected that package to + be analyzed in its own context; actual produced four false resolution errors. + Root exclusion plus explicit CI/example analysis fixed it. Both contexts pass. +- **F-025:** Tag inventory contained bare `v` and an empty-body release holding + one obsolete Windows ZIP. Expected semantic immutable tags; the repository had + no reference to it and GitHub reported only two historic downloads. After + explicit broad maintenance authorization, the release and its local/remote tag + were deleted. The tag can be reconstructed at commit `6e5042f` if needed. +- **F-026:** CLI tests under `flutter_test` could not use an API that only exists + in standalone-isolate contexts. The loader now parses `--packages=`, + `Platform.packageConfig`, `Isolate.packageConfig`, and ancestor configs. All 34 + real-library CLI tests pass. Resolved, high confidence. +- **F-027:** The original override helper assumed Bash and publish dry-runs could + validate the published graph rather than the coordinated source graph. A Dart + entry point is now used on every OS, including Windows; all ten package + dry-runs pass. Resolved, high confidence. +- **F-028:** Linux package tests could pass compilation without proving that the + just-built plugin shared object was loadable. The dedicated workflow now adds + that exact output to `LD_LIBRARY_PATH` and tests both workspace and exact + package-config source resolution. The hosted Linux run passed. Resolved, high + confidence. +- **F-029:** A shared job-level concurrency key allowed GitHub Actions to retain + only one pending Apple matrix row and cancel peers. Removing that key lets all + four CocoaPods/SwiftPM × source rows complete; the final iOS and macOS matrices + passed. Resolved, high confidence. +- **F-030:** `avdmanager` could exit successfully while placing the AVD outside + the path assumed by the runner. The helper now creates and exports + `ANDROID_USER_HOME`/`ANDROID_AVD_HOME`, verifies `config.ini`, and emits bounded + diagnostics on failure. All four hosted Android rows passed. Resolved, high + confidence. +- **F-031:** Codecov's PR comment remained pinned to an early commit with one + head upload versus three base uploads, while the live dashboard had advanced + only to another ancestor. A required PR coverage matrix now uploads all three + expected flags on every trusted head, applies the same thresholds without + secrets on forks, and scopes CLI LCOV to package sources. New resolver/error + tests raise CLI line coverage from 80.5% to 100% (213/213), which is now + enforced as the package threshold. Resolved, high confidence. + +## 15. Platform/test matrix + +| Platform | Build | Launch | Native round trip | Instrumentation/integration | Architectures | Status | +|---|---|---|---|---|---|---| +| Android modern | Debug + Release split APK | Emulator | Yes | 6 + 100 + 10 passed locally and in both CI source modes | ARM64 + x86_64 runtime; ARMv7 artifact | Confirmed pass | +| Android legacy | Debug + Release split APK | Headless emulator | Yes | 6 + 100 + 10 passed locally and in both CI source modes | ARM64 + x86_64 runtime; ARMv7 artifact | Confirmed pass | +| iOS CocoaPods | Simulator Debug + device Release compile | iPhone 16 Simulator | Yes | 8 passed | arm64 simulator/device compile | Confirmed pass; physical device unavailable | +| iOS SwiftPM | Simulator Debug + device Release compile | iPhone 16 Simulator | Yes | 8 passed | arm64 simulator/device compile | Confirmed pass; physical device unavailable | +| macOS CocoaPods | Release | macOS app | Yes | 8 passed | Apple Silicon | Confirmed pass; Intel unavailable | +| macOS SwiftPM | Release | macOS app | Yes | 8 passed | Apple Silicon | Confirmed pass; Intel unavailable | +| Web JS | Release | Chrome | Yes | Full browser suite passed | WASM | Confirmed pass | +| Flutter Web WASM | Release | Build only | Not launched in Dart-Wasm mode | Build passed | WASM | Build confirmed; dedicated launch not run | +| CLI macOS | Native dylib + Dart tests | Process | Yes | 34 passed + file E2E | arm64 runtime; universal dylib inspected | Confirmed pass | +| Linux | CMake/plugin + Flutter package | Test process | Yes | CTest, package, exact-source and publish dry run passed | x86_64 runtime; ARM64 target retained | Confirmed remote pass | +| Windows | MSVC/plugin + federated example | Test process | Yes | C++, package, integration, exact-source and publish dry run passed | x86_64 runtime; ARM64 target retained | Confirmed remote pass | + +## 16. Commands executed + +Representative commands (many were repeated after fixes): + +- Git/inventory: `git status`, `git branch -a -vv`, `git log`, `git tag`, + `git merge-base`, `git rev-list --left-right --count`, `git ls-files`, + `git check-ignore`, `git count-objects -vH`, file-size/symlink/duplicate/secret + searches with `find`, `rg`, `cmp`, `shasum`, `file`, `lipo`, and + `llvm-readelf`. +- Dart/Flutter for all ten packages and eight examples: `pub get`, `analyze`, + `test`, `dart format --set-exit-if-changed`, metadata checks, and safe publish + dry runs where dependency state allowed. +- Native: CMake Debug/Release strict builds, ASan+UBSan build/harness, TSan + build/harness, multi-thread and malformed-input harnesses, symbol inspection. +- Android: modern/legacy split Debug and Release APK builds, APK/aapt2/ELF + inspection, two Flutter integration suites per example, Gradle + `connectedDebugAndroidTest`, configuration-cache strict store/reuse, emulator + boot/readiness/install/test/shutdown. +- Apple: CocoaPods and SwiftPM resolution/build for iOS and macOS, generic-device + Release compilation, Simulator/macOS integration tests, package/podspec tests. +- Web: pinned Emscripten regeneration in clean temporary state, three-way asset + comparison/checksums, direct/federated Release builds, Dart-Wasm build, + ChromeDriver integration. +- CLI: 34 tests, real file compress/decompress/`cmp`, universal library build and + symbol/architecture inspection. +- CI/release: actionlint, Ruby YAML parse, `bash -n`, Action-pin/permission scan, + version `1.5.1` simulation, publication metadata validation, read-only GitHub + workflow/release/ruleset API inspection. + +## 17. Remote validation evidence + +All links below are immutable GitHub Actions run records for the remediated code. +The Linux-hosted contingency branch differed from the PR head only in `runs-on` +selection and isolated Android `max-parallel`; it was used because every +self-hosted Linux runner was offline and was deleted after validation. + +| Workflow | Result | Run | +|---|---|---| +| Pull Request Safety Gate | Passed | [34138922806](https://github.com/vypdev/zstandard/actions/runs/34138922806) | +| Windows | Passed, all jobs | [34135888149](https://github.com/vypdev/zstandard/actions/runs/34135888149) | +| macOS | Passed, all four dependency/source rows | [34135873437](https://github.com/vypdev/zstandard/actions/runs/34135873437) | +| iOS | Passed, all four dependency/source rows | [34137188984](https://github.com/vypdev/zstandard/actions/runs/34137188984) | +| Linux | Passed, including built-plugin runtime | [34137856488](https://github.com/vypdev/zstandard/actions/runs/34137856488) | +| Web | Passed, including deterministic WASM and real Chrome | [34137866872](https://github.com/vypdev/zstandard/actions/runs/34137866872) | +| Android | Passed, all four example/source rows | [34139066644](https://github.com/vypdev/zstandard/actions/runs/34139066644) | +| CLI | Passed | [34137851354](https://github.com/vypdev/zstandard/actions/runs/34137851354) | +| Platform interface | Passed | [34137861505](https://github.com/vypdev/zstandard/actions/runs/34137861505) | +| Main `zstandard` package | Passed | [34137871862](https://github.com/vypdev/zstandard/actions/runs/34137871862) | + +## 18. Commands not executed and why + +- No package publication, release workflow, tag-triggered OIDC publication, + integration workflow, or GitHub release was executed; validation stopped at + safe dry runs, as required. +- No ruleset/branch-protection mutation was performed; the owner explicitly + deferred activation. +- No Intel macOS runtime test: no Intel runner was available. Universal output + was built and inspected instead. +- No physical iOS deployment/signing: no physical device or signing target was + available. Simulator execution and generic-device Release compilation passed. +- No AAB: the examples are validation consumers, not shipping applications; all + plugin ABI packaging was exercised through split APKs. +- No local `shellcheck`, `yamllint`, or PowerShell syntax command: tools were + absent. YAML/actionlint/shell checks and native remote platform execution + supplied the available evidence. + +## 19. Environment matrix + +| Component | Observed | +|---|---| +| Host | macOS 26.5.1 (25F80), arm64 | +| Xcode | 26.6 (17F113), Apple Clang 21 | +| iOS SDK / selected simulator | 26.5 / iPhone 16 | +| Flutter / Dart | 3.47.2 / 3.13.2 stable | +| Swift / CocoaPods | Swift 6.3.3 / CocoaPods 1.16.2 | +| CMake | 4.4.0 | +| Emscripten | 3.1.69 pinned | +| Android | SDK/API 36 available; API 31 and 36 ARM64 emulators used; emulator 37.1.11 | +| Java | Corretto 17.0.9 for direct shell; Android Studio JDK 21 used by modern Flutter/Gradle | +| Browser | Chrome 152.0.7977.76 | +| ChromeDriver | 151.0.7922.34; passed locally despite one-major mismatch | +| actionlint | 1.7.12 official release binary in temporary audit directory | + +Android emitted an SDK XML v4/tool-reader-v3 warning, and the legacy Gradle line +emitted Flutter future-support warnings. Both builds and tests passed; align the +runner command-line tools and browser driver during normal runner maintenance. + +## 20. Release blocker + +The only intentional blocker is GitHub ruleset `2541601`: it is disabled and its +required contexts are stale. Pull request 144 now supplies a successful stable +`Pull Request Safety Gate`, so the owner can enable the rule later and require +that context without changing repository code. + +## 21. Remaining maintenance recommendations + +- Align ChromeDriver with Chrome and Android command-line tools with SDK XML v4 + during normal runner maintenance; neither mismatch caused a test failure. +- Track the deliberately retained legacy AGP line as Flutter raises its minimum + supported Gradle/AGP/Kotlin versions. +- Add Intel macOS and physical-iOS periodic hardware coverage if those devices + are operational support commitments rather than build-compatibility targets. + +## 22. Suggested release sequence after review and ruleset activation + +1. Review and merge pull request 144 under the enabled Safety Gate. +2. Create `release/x.y.z`, run preparation, review generated version/artifact + changes, and merge/approve according to policy. +3. Create the immutable tag, allow OIDC publication, verify pub.dev, then run the + separate integration-PR workflow. + +## 23. Residual risks + +- Hardware not represented by the runner matrix (notably Intel macOS and a + physical iOS device) can still expose loader, signing, or registration defects + beyond the successful universal/device builds. +- The 256 MiB default prevents unbounded decompression but remains a substantial + per-call allocation ceiling; applications handling hostile input may choose a + smaller value. +- Native and Worker paths still copy bytes at ownership boundaries. +- Apple leak detection was not a Linux LSan run; process RSS and allocator-count + baselines were not captured. +- Generated Web assets are deterministic in the audited toolchain, but future + Emscripten/toolchain updates require checksum review and browser retesting. +- Exact dependency pins improve release reproducibility but require deliberate + coordinated version updates. +- Historical Git objects still contain the removed large media until an + explicitly approved history rewrite, which is not recommended solely for this + release. + +## 24. Final readiness verdict + +**REMEDIATED AND VALIDATED — RULESET ACTIVATION DEFERRED** + +All identified repository release blockers were corrected, committed, proposed +through pull request 144, and exercised by the supported local/remote workflow +matrix. The malformed historical release/tag was also removed. No failing or +queued validation remains. Ruleset 2541601 is the only intentional outstanding +release-governance action; publication must remain gated until the owner enables +it with the successful stable `Pull Request Safety Gate` context. + +Authoritative references used to validate changing behavior: the +[Zstandard API manual](https://facebook.github.io/zstd/zstd_manual.html), +[Flutter SwiftPM guidance for app developers](https://docs.flutter.dev/packages-and-plugins/swift-package-manager/for-app-developers), +[Flutter SwiftPM guidance for plugin authors](https://docs.flutter.dev/packages-and-plugins/swift-package-manager/for-plugin-authors), +[Dart automated publishing/OIDC documentation](https://dart.dev/tools/pub/automated-publishing), +and [GitHub's workflow-token trigger behavior](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow). diff --git a/CHANGELOG.md b/CHANGELOG.md index fecde1ab..fcc08b63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,21 @@ ## Unreleased +- Added bounded decompression (256 MiB default) with support for unknown-size + and concatenated frames across native, Web, CLI, and extension APIs. +- Moved native work to byte-safe isolates and WebAssembly work to a dedicated + Web Worker; empty input now always produces a valid zstd frame. +- Reworked CLI argument parsing, stdin/stdout support, collision protection, + library resolution, exported-symbol tests, and finite benchmark reporting. +- Pinned native provenance, dependency sources, Apple versions, Emscripten, + third-party Actions, and aligned SwiftPM/CocoaPods native source sets. +- Split release preparation from immutable tag-triggered OIDC publishing and + added a GitHub-hosted, read-only safety gate for untrusted pull requests and + a separate reviewed release-integration workflow. - Fixed intermittent iOS and macOS build failures caused by deleting synced zstd sources during compilation. - Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12 and migrated Android builds to Built-in Kotlin on AGP 9+. +- Removed obsolete Android Jetifier settings, stale workflow paths, and 58 MiB of unreferenced repository media. +- Removed placeholder `WIP` tests, isolated the legacy Android example's analysis, + and migrated the CLI away from deprecated platform APIs. - Added Swift Package Manager support for iOS and macOS while retaining CocoaPods compatibility. ## 1.5.0 - Dependencies Updated diff --git a/Package.swift b/Package.swift index f7dc5311..881908fb 100644 --- a/Package.swift +++ b/Package.swift @@ -32,6 +32,8 @@ let package = Package( cSettings: [ .headerSearchPath("."), .headerSearchPath("include"), + // Required when Xcode precompiles zstd.h as a Clang module; + // internal zstd translation units consume these declarations. .define("ZSTD_STATIC_LINKING_ONLY"), .define("ZSTD_DISABLE_ASM"), ], @@ -43,7 +45,13 @@ let package = Package( "-Xlinker", "-u", "-Xlinker", "_ZSTD_compress", "-Xlinker", "-u", "-Xlinker", "_ZSTD_decompress", "-Xlinker", "-u", "-Xlinker", "_ZSTD_compressBound", + "-Xlinker", "-u", "-Xlinker", "_ZSTD_isError", "-Xlinker", "-u", "-Xlinker", "_ZSTD_getFrameContentSize", + "-Xlinker", "-u", "-Xlinker", "_ZSTD_createDStream", + "-Xlinker", "-u", "-Xlinker", "_ZSTD_initDStream", + "-Xlinker", "-u", "-Xlinker", "_ZSTD_decompressStream", + "-Xlinker", "-u", "-Xlinker", "_ZSTD_freeDStream", + "-Xlinker", "-u", "-Xlinker", "_ZSTD_DStreamOutSize", ]) ] ), diff --git a/README.md b/README.md index 53d5587c..62ebcd37 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,9 @@ void main() async { Uint8List? compressed = await zstandard.compress(originalData, 3); - Uint8List? decompressed = await zstandard.decompress(compressed ?? Uint8List(0)); + final decompressed = compressed == null + ? null + : await zstandard.decompress(compressed, maxOutputSize: 64 * 1024 * 1024); } ``` @@ -73,7 +75,9 @@ void main() async { Uint8List? compressed = await originalData.compress(); - Uint8List? decompressed = await compressed.decompress(); + final decompressed = await compressed.decompress( + maxOutputSize: 64 * 1024 * 1024, + ); } ``` @@ -81,7 +85,7 @@ void main() async { ```bash # Compress a file with a specified compression level -dart run zstandard_cli:compress myfile.txt 3 +dart run zstandard_cli:compress --level 3 myfile.txt # Decompress a file dart run zstandard_cli:decompress myfile.txt.zstd @@ -151,4 +155,4 @@ void main() async { ## License -This project uses code from the original [facebook/zstd](https://github.com/facebook/zstd/tree/dev/lib) repository. Please see the LICENSE file for more information. \ No newline at end of file +This project uses code from the original [facebook/zstd](https://github.com/facebook/zstd/tree/dev/lib) repository. Please see the LICENSE file for more information. diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md index 49a2cba6..c337a373 100644 --- a/docs/GLOSSARY.md +++ b/docs/GLOSSARY.md @@ -20,7 +20,7 @@ Definitions of terms and acronyms used in the Zstandard plugin and CLI documenta **Federated plugin** — A Flutter plugin that delegates to platform-specific implementations (e.g. zstandard_android, zstandard_ios) rather than implementing everything in one package. The **zstandard_native** package holds the shared C source (facebook/zstd) used by all native implementations and the CLI; it is published to pub.dev as part of the release set. -**Frame** — A Zstandard-compressed unit of data with a header and optional checksum. The API compresses and decompresses one frame at a time (or a single buffer that may contain a frame). +**Frame** — A Zstandard-compressed unit of data with a header and optional checksum. Compression emits one frame; decompression accepts a complete buffer containing one or more concatenated frames, including frames without a declared content size. **Isolate** — A Dart concurrency unit. The native implementations may run compression/decompression in a separate isolate so the UI thread is not blocked. diff --git a/docs/api/cli-api.md b/docs/api/cli-api.md index c6fb56d4..855e8a0e 100644 --- a/docs/api/cli-api.md +++ b/docs/api/cli-api.md @@ -1,104 +1,56 @@ # CLI API Reference -The **zstandard_cli** package provides Zstandard compression and decompression for **pure Dart** applications targeting **macOS, Windows, and Linux** (no Flutter). It uses FFI with precompiled native zstd libraries. +`package:zstandard_cli/zstandard_cli.dart` provides a pure-Dart desktop API +for macOS, Windows, and Linux. It bundles x64 and arm64 native libraries. -## Import +## Dart API ```dart -import 'package:zstandard_cli/zstandard_cli.dart'; -``` - -## ZstandardCLI Class +final codec = ZstandardCLI(); -### Constructor +Future compress( + Uint8List data, { + int compressionLevel = 3, +}) -```dart -ZstandardCLI() +Future decompress( + Uint8List data, { + int maxOutputSize = 256 * 1024 * 1024, +}) ``` -Creates a new instance. Each instance shares the same underlying native library (loaded once per process). +Compression accepts levels 1–22 and emits a valid frame for empty input. +Decompression is streaming, supports unknown-size and concatenated frames, +and returns `null` for invalid, truncated, or oversized input. The equivalent +options are available on the `Uint8List?` extensions. -### getPlatformVersion - -```dart -Future getPlatformVersion() -``` +`getPlatformVersion()` returns a diagnostic OS string. -Returns a string describing the current platform (e.g. `"macOS 14.0"`, `"Windows 10"`, `"Linux ..."`). Useful for CLI output or debugging. +## Command-line entry points -### compress +```text +zstandard-compress [options] + -l, --level LEVEL + -o, --output PATH + -f, --force -```dart -Future compress(Uint8List data, {int compressionLevel = 3}) +zstandard-decompress [options] + -m, --max-output-size BYTES + -o, --output PATH + -f, --force ``` -Compresses `data` using Zstandard. - -- **data**: Bytes to compress. Empty input returns the same empty `Uint8List` as per implementation. -- **compressionLevel**: Optional; default **3**. Range 1–22. -- **Returns**: Compressed bytes, or `null` on failure. - -### decompress - -```dart -Future decompress(Uint8List data) -``` - -Decompresses Zstandard-compressed `data`. - -- **data**: Compressed bytes (full zstd frame). -- **Returns**: Decompressed bytes, or `null` on failure. - ---- - -## Extensions (zstandard_cli) - -The package also defines **ZstandardExt** on `Uint8List?`: - -### compress - -```dart -Future compress({int compressionLevel = 3}) -``` - -Same as `ZstandardCLI().compress(this, compressionLevel: compressionLevel)`. Returns `null` if the receiver is `null`. - -### decompress - -```dart -Future decompress() -``` - -Same as `ZstandardCLI().decompress(this)`. Returns `null` if the receiver is `null`. - -**Example:** - -```dart -final data = Uint8List.fromList([1, 2, 3, 4, 5]); -final compressed = await data.compress(compressionLevel: 5); -final decompressed = await compressed?.decompress(); -``` - -## Command-Line Entry Points - -When used as a CLI (e.g. `dart run zstandard_cli:compress` / `zstandard_cli:decompress`), the package provides: - -- **compress**: Reads a file (or stdin), compresses with a given level, writes to file (or stdout). Usage: `dart run zstandard_cli:compress `. -- **decompress**: Reads a compressed file, decompresses, writes output. Usage: `dart run zstandard_cli:decompress `. - -See the package README and [Platforms — CLI](../platforms/cli.md) for exact usage and file naming. - -## Platform Support - -| Platform | Architectures | Precompiled library | -|----------|----------------|----------------------| -| macOS | x64, arm64 | Yes | -| Windows | x64, arm64 | Yes | -| Linux | x64, arm64 | Yes | +Both commands also accept `--help` and `--version`. `-` reads stdin; without +an explicit output, stdin writes to stdout. Compression appends `.zstd` to a +file name. Decompression strips `.zstd`, or appends `.out` when there is no +matching suffix. Input aliases are never overwritten, and existing output +files require `--force`. -The library is loaded at runtime from the package’s resources based on the current platform and architecture. +Exit codes are 0 (success), 1 (I/O or codec failure), and 2 (usage error or +refused overwrite). -## See Also +The native library path is resolved from the package URI rather than the +working directory. Set `ZSTANDARD_CLI_LIBRARY` to an explicit library path for +custom deployment or testing. -- [Platforms — CLI](../platforms/cli.md) -- [Main API](main-api.md) — Flutter plugin API +See the [CLI platform guide](../platforms/cli.md). diff --git a/docs/api/extensions.md b/docs/api/extensions.md index 31bd49f1..7d858ffc 100644 --- a/docs/api/extensions.md +++ b/docs/api/extensions.md @@ -1,71 +1,26 @@ # Extensions API Reference -The **zstandard** package adds extension methods on `Uint8List?` so you can call `compress` and `decompress` directly on byte data. - -## Import - -```dart -import 'package:zstandard/zstandard.dart'; -``` - -The extensions are exported from the main library. - -## ZstandardExt on Uint8List? - -Extension on nullable `Uint8List`. If the receiver is `null`, both methods return `null` without calling the platform. - -### compress +Import `package:zstandard/zstandard.dart` to add these methods to +`Uint8List?`. A null receiver returns `null` without invoking a platform. ```dart Future compress({int compressionLevel = 3}) -``` - -Compresses this byte list using the default (or specified) compression level. - -- **compressionLevel**: Optional; defaults to **3**. Range 1–22. -- **Returns**: Compressed bytes, or `null` if the receiver is `null` or compression failed. - -**Example:** - -```dart -final data = Uint8List.fromList([10, 20, 30, 40, 50]); -final compressed = await data.compress(); -final compressedHigh = await data.compress(compressionLevel: 10); -``` -### decompress - -```dart -Future decompress() +Future decompress({ + int maxOutputSize = Zstandard.defaultMaxDecompressedSize, +}) ``` -Decompresses this byte list, which must be Zstandard-compressed data. - -- **Returns**: Decompressed bytes, or `null` if the receiver is `null` or decompression failed. - -**Example:** +The methods have the same behavior as `Zstandard.compress` and +`Zstandard.decompress`. In particular, decompression defaults to a 256 MiB +output limit and supports unknown-size and concatenated frames. ```dart -final compressed = await data.compress(compressionLevel: 3); -final decompressed = await compressed?.decompress(); -``` - -## Null Safety - -- On `null` receiver, `compress()` and `decompress()` return `null` and do not throw. -- Always check the result for `null` when the source might be null or when the operation can fail. - -**Example:** - -```dart -Uint8List? maybeData = ...; -final compressed = await maybeData.compress(); -if (compressed != null) { - final back = await compressed.decompress(); +final compressed = await data.compress(compressionLevel: 5); +final original = await compressed.decompress(maxOutputSize: 8 * 1024 * 1024); +if (original == null) { + // Null receiver, invalid input, output-limit violation, or codec failure. } ``` -## See Also - -- [Main API](main-api.md) — `Zstandard` class -- [Usage Examples](../guides/usage-examples.md) +See the [main API](main-api.md) for the execution and error contract. diff --git a/docs/api/main-api.md b/docs/api/main-api.md index 9b3b53ec..2cfd1f3d 100644 --- a/docs/api/main-api.md +++ b/docs/api/main-api.md @@ -1,102 +1,75 @@ # Main API Reference -The main package **zstandard** exposes a single public class and re-exports the extension methods. Applications should only depend on this package. +Applications should import `package:zstandard/zstandard.dart`. It exposes the +cross-platform `Zstandard` singleton and the `Uint8List?` extensions. -## Zstandard Class - -**Library:** `package:zstandard/zstandard.dart` - -### Constructor +## `Zstandard` ```dart factory Zstandard() ``` -Creates or returns the singleton instance. Use this to obtain the shared `Zstandard` instance. +Returns the process singleton. `instance` exposes the registered platform +implementation for diagnostics and tests; application code normally calls the +methods below. -**Example:** +### `compress` ```dart -final zstandard = Zstandard(); -``` - -### Instance Property - -```dart -ZstandardPlatform get instance +Future compress(Uint8List data, int compressionLevel) ``` -Returns the currently registered platform implementation. Typically you do not need to access this; use `compress` and `decompress` on the `Zstandard` instance instead. Useful for testing (mock the platform) or for calling `getPlatformVersion`. +Creates one valid zstd frame. Empty input is supported. Levels 1–22 are +portable across the official implementations; an invalid level or codec +failure returns `null`. -### getPlatformVersion +### `decompress` ```dart -Future getPlatformVersion() +Future decompress( + Uint8List data, { + int maxOutputSize = Zstandard.defaultMaxDecompressedSize, +}) ``` -Returns a platform-specific version or identifier string (e.g. for display or debugging). May be `null` if the platform does not provide one. - -### compress - -```dart -Future compress(Uint8List data, int compressionLevel) -``` +Decompresses complete zstd input, including concatenated frames and frames +without a declared content size. The default maximum output is 256 MiB. +Malformed, truncated, empty, or oversized input returns `null`. -Compresses `data` using Zstandard with the given `compressionLevel`. +Official platforms enforce the limit while producing output, before an +oversized result can be allocated. For source compatibility, third-party +platform implementations that only implement the original `decompress` +method are post-checked; their allocation cannot be bounded by the main +package. Such implementations should add `BoundedZstandardPlatform` before +handling untrusted data. -- **data**: Raw bytes to compress. Can be any length; empty input is allowed (behavior is platform-dependent). -- **compressionLevel**: Integer from **1** (fastest, least compression) to **22** (slowest, best compression). Default in extensions is **3**. -- **Returns**: Compressed bytes as `Uint8List`, or `null` if compression failed. - -**Example:** +Choose a smaller budget whenever the application has a known protocol limit: ```dart -final zstandard = Zstandard(); -final bytes = Uint8List.fromList([1, 2, 3, 4, 5]); -final compressed = await zstandard.compress(bytes, 3); -if (compressed != null) { - // use compressed +final decompressed = await Zstandard().decompress( + compressed, + maxOutputSize: 8 * 1024 * 1024, +); +if (decompressed == null) { + // Invalid/truncated frame, codec error, or more than 8 MiB of output. } ``` -### decompress - -```dart -Future decompress(Uint8List data) -``` - -Decompresses Zstandard-compressed `data`. - -- **data**: Bytes produced by `compress` (or any valid zstd frame). -- **Returns**: Decompressed bytes as `Uint8List`, or `null` if decompression failed (e.g. invalid or corrupted input). - -**Example:** +### `getPlatformVersion` ```dart -final decompressed = await zstandard.decompress(compressed!); -if (decompressed != null) { - // use decompressed -} +Future getPlatformVersion() ``` -## Compression Levels - -| Level | Typical use | Speed | Ratio | -|-------|------------------|--------|--------| -| 1 | Real-time, low latency | Fastest | Lower | -| 3 | Default balance | Fast | Good | -| 10–19 | High compression | Slower | Higher | -| 20–22 | Maximum ratio | Slowest | Best | - -Invalid levels (e.g. < 1 or > 22) may be accepted or rejected depending on the platform; avoid them for portability. - -## Threading and Performance +Returns a platform identifier intended for diagnostics, or `null` when it is +not available. -- All methods return `Future`s. Heavy work may be offloaded to a background isolate on native platforms to avoid blocking the UI. -- For large data, prefer using the main plugin API (which can use isolates) rather than blocking the main thread. +## Execution model -## See Also +The official native implementations run byte-oriented codec work in a worker +isolate. The web implementation sends it to a dedicated Web Worker. Results +are asynchronous and `null` represents an expected operation failure; setup +errors such as using an unsupported platform can still throw. -- [Extensions](extensions.md) — `compress` and `decompress` on `Uint8List?` -- [Platform Interface](platform-interface.md) — Contract implemented by each platform -- [Compression Levels Guide](../guides/compression-levels.md) +See [Extensions](extensions.md), [Platform interface](platform-interface.md), +and [Security](../guides/security.md). diff --git a/docs/api/platform-interface.md b/docs/api/platform-interface.md index 7854a68f..b068f35d 100644 --- a/docs/api/platform-interface.md +++ b/docs/api/platform-interface.md @@ -1,76 +1,41 @@ # Platform Interface API Reference -The **zstandard_platform_interface** package defines the contract that every platform implementation (Android, iOS, macOS, Linux, Windows, Web) must satisfy. Application code typically uses the main **zstandard** package and does not depend on this package directly. +`zstandard_platform_interface` defines the federated plugin contract. +Applications should use the main `zstandard` package. -## ZstandardPlatform +## `ZstandardPlatform` -**Library:** `package:zstandard_platform_interface/zstandard_platform_interface.dart` - -Abstract base class for all platform implementations. Extends `PlatformInterface` from the plugin_platform_interface package. - -### instance (static getter) +Implementations extend `ZstandardPlatform`, provide `getPlatformVersion`, +`compress`, and the legacy-compatible `decompress`, then register themselves: ```dart -static ZstandardPlatform get instance +ZstandardPlatform.instance = MyZstandardPlatform(); ``` -Returns the current platform implementation. Defaults to `MethodChannelZstandardPlatform`. +The default `MethodChannelZstandardPlatform` only implements platform-version +lookup. Codec operations throw `UnimplementedError` until an implementation is +registered. -### instance (static setter) +## Bounded decompression capability -```dart -static set instance(ZstandardPlatform instance) -``` - -Sets the platform implementation. Only instances created with the correct token (from this package) can be set. Platform packages call this in their `registerWith()`. - -### getPlatformVersion +New and official implementations should additionally implement: ```dart -Future getPlatformVersion() +abstract interface class BoundedZstandardPlatform { + Future decompressWithOptions( + Uint8List data, { + int maxOutputSize = ZstandardPlatform.defaultMaxDecompressedSize, + }); +} ``` -Returns a platform-specific version or identifier string. Base implementation throws `UnimplementedError`. - -### compress - -```dart -Future compress(Uint8List data, int compressionLevel) -``` - -Compresses `data` at the given `compressionLevel` (1–22). Base implementation throws `UnimplementedError`. - -### decompress - -```dart -Future decompress(Uint8List data) -``` - -Decompresses Zstandard-compressed `data`. Base implementation throws `UnimplementedError`. - ---- - -## MethodChannelZstandardPlatform - -Default implementation used when no native implementation is registered (e.g. in tests or unsupported platforms). - -- **getPlatformVersion()**: Implemented; invokes the method channel `plugins.flutter.io/zstandard` with method `getPlatformVersion`. -- **compress()**: Not implemented; throws `UnimplementedError`. -- **decompress()**: Not implemented; throws `UnimplementedError`. - -So in environments where only the method channel is available, only `getPlatformVersion` is usable unless a test sets a mock platform. - -## Implementing the Interface - -Platform packages: - -1. Extend `ZstandardPlatform`. -2. Implement `getPlatformVersion`, `compress`, and `decompress`. -3. In registration, set `ZstandardPlatform.instance = MyPlatform()` (with the token from the interface). - -See [Architecture — Platform Interface](../architecture/platform-interface.md) for the registration flow. +`ZstandardPlatform.defaultMaxDecompressedSize` is 256 MiB. Implementations +must enforce the limit during decompression, not only after allocating the +result. Keeping this as a separate optional interface avoids a source-breaking +method addition for existing third-party platform implementations. -## See Also +The main package uses the bounded capability when present. Its compatibility +fallback calls legacy `decompress` and rejects an oversized returned value, +but cannot prevent the third-party implementation's earlier allocation. -- [Architecture — Platform Interface](../architecture/platform-interface.md) -- [Main API](main-api.md) +See [Platform interface architecture](../architecture/platform-interface.md). diff --git a/docs/architecture/ffi-implementation.md b/docs/architecture/ffi-implementation.md index 0fe46c5a..601d5fba 100644 --- a/docs/architecture/ffi-implementation.md +++ b/docs/architecture/ffi-implementation.md @@ -1,70 +1,50 @@ -# FFI Implementation +# Native FFI Implementation -Native platforms (Android, iOS, macOS, Linux, Windows) use Dart’s **FFI (Foreign Function Interface)** to call the Zstandard C library directly. This document describes the shared pattern used across these implementations. +`zstandard_native` is the single native-code package. It contains the pinned +upstream C source, generated Dart bindings, and the shared +`ZstandardNativeCodec` used by Android, iOS, macOS, Linux, Windows, and the +desktop CLI. -## Overview +## Compression -Each native platform package: +The codec validates levels 1–22, allocates at least one source byte so an empty +input is safe to pass through FFI, obtains `ZSTD_compressBound`, compresses into +a native buffer, copies the exact result into Dart memory, and frees both +allocations in `finally`. Empty input therefore produces a normal zstd frame. -1. Ships or builds the official [facebook/zstd](https://github.com/facebook/zstd) C library for that platform. -2. Generates Dart FFI bindings (e.g. with `ffigen`) for the zstd functions used by the plugin. -3. Opens the native library at runtime and calls `ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, and `ZSTD_getFrameContentSize` from Dart. -4. Manages memory (allocate output buffers, copy bytes, free) using `package:ffi` (e.g. `malloc.allocate` / `malloc.free`). +## Decompression -## Shared C API Usage +The codec uses this streaming surface: -The plugin uses a minimal subset of the zstd C API: - -| C function | Purpose | -|------------|---------| -| `ZSTD_compressBound(srcSize)` | Upper bound for compressed size; used to allocate the destination buffer. | -| `ZSTD_compress(dst, dstCapacity, src, srcSize, compressionLevel)` | Compresses `src` into `dst`; returns compressed size or an error code. | -| `ZSTD_getFrameContentSize(src, srcSize)` | Gets original size from a zstd frame (or -1 / -2 if unknown/error). | -| `ZSTD_decompress(dst, dstCapacity, src, srcSize)` | Decompresses `src` into `dst`; returns decompressed size or an error code. | - -## Binding Generation - -Bindings are typically generated with **ffigen** from the zstd headers. The generated Dart file (e.g. `zstandard_linux_bindings_generated.dart`) exposes a class that wraps the `DynamicLibrary` and provides typed Dart methods for the C functions above. - -Example pattern: - -```dart -final DynamicLibrary _dylib = DynamicLibrary.open('libzstandard_linux_plugin.so'); -final ZstandardLinuxBindings _bindings = ZstandardLinuxBindings(_dylib); +```text +ZSTD_createDStream ZSTD_initDStream +ZSTD_DStreamOutSize ZSTD_decompressStream +ZSTD_isError ZSTD_freeDStream ``` -Library names and loading differ per platform (e.g. `.so` on Linux/Android, `.dylib` on macOS/iOS, `.dll` on Windows). Each platform’s `lib/` code opens the appropriate library and instantiates the bindings once. - -## Memory Management - -Compression and decompression follow the same pattern across native implementations: - -1. **Allocate** input buffer with `malloc.allocate(size)` and copy `Uint8List` into it. -2. **Allocate** output buffer: - - Compression: size = `ZSTD_compressBound(srcSize)` - - Decompression: size = from `ZSTD_getFrameContentSize`; frames with an unknown or invalid advertised size are rejected by the one-shot API instead of guessing an allocation. -3. **Call** `ZSTD_compress` or `ZSTD_decompress`. -4. **Copy** result into a new `Uint8List` (only the written length). -5. **Free** both buffers in a `finally` block so memory is always released. - -All platforms use this pattern to avoid leaks and to stay safe with Dart’s GC and native memory. - -## Platform-Specific Details - -- **Android**: Native code is built as part of the Android project; the Dart plugin loads the library via the engine (e.g. `DynamicLibrary.open('libzstandard_android_plugin.so')` or similar as configured). -- **iOS / macOS**: zstd is built as a static library or framework and linked into the app; the plugin opens the corresponding dynamic library or uses the linked symbols as per project setup. -- **Linux**: CMake builds `libzstandard_linux_plugin.so`; the plugin loads it by name. -- **Windows**: CMake builds `zstandard_windows_plugin.dll` (or similar); the plugin loads it by name. +Input is copied into native memory once. Output is produced into the zstd +recommended chunk size and copied into a `BytesBuilder`. Before accepting each +chunk, the implementation verifies that the cumulative result cannot exceed +`maxOutputSize`. It accepts content-size-omitting and concatenated frames and +rejects errors, truncation, or a no-progress loop. All stream state and buffers +are freed on every return path. -Each platform’s README and `docs/platforms/` guide should describe how to build and where the library is placed. +The older one-shot `ZSTD_decompress` and frame-size symbols remain exported for +low-level compatibility, but public byte decompression does not trust frame +metadata to choose an unbounded allocation. -## Error Handling +## Linking and source resolution -- `ZSTD_compress` and `ZSTD_decompress` return negative values on error. The Dart code checks for `result > 0` and returns `null` otherwise (or throws, depending on the package’s public API contract). -- `ZSTD_getFrameContentSize` returns -1 (unknown) or -2 (error). Implementations use the generated Dart constants, reject unknown/error sizes, and validate the Zstandard frame magic before allocating native buffers. +Native build systems compile only upstream `common`, `compress`, and +`decompress` C files from the exact `zstandard_native` package resolved by the +application. Repository checkouts may use the adjacent package; published +consumers resolve it through Dart's package configuration. Build scripts never +select an arbitrary matching Pub-cache directory. -## Related Documentation +SwiftPM force-loads the complete FFI symbol surface because its static linker +could otherwise remove Dart-only entry points. CocoaPods builds the equivalent +C set in an embedded framework. CLI tests open every required symbol; runtime +roundtrips exercise the same codec. -- [Overview](overview.md) -- [Platform Interface](platform-interface.md) -- [Isolate Pattern](isolate-pattern.md) — Offloading compression/decompression to isolates on native platforms +See [Native isolate pattern](isolate-pattern.md) and +[`UPSTREAM_ZSTD.md`](../../zstandard_native/UPSTREAM_ZSTD.md). diff --git a/docs/architecture/isolate-pattern.md b/docs/architecture/isolate-pattern.md index e5bd5cd9..f292a60c 100644 --- a/docs/architecture/isolate-pattern.md +++ b/docs/architecture/isolate-pattern.md @@ -1,64 +1,35 @@ -# Isolate Pattern for Async Compression - -On native platforms (Android, iOS, macOS, Linux, Windows), the plugin can run compression and decompression in a **background isolate** so that CPU-heavy work does not block the UI thread. This document describes the pattern used in the native implementations. - -## Motivation - -- **ZSTD_compress** and **ZSTD_decompress** are CPU-bound and can take noticeable time for large inputs. -- Dart is single-threaded per isolate. Running zstd on the main isolate would cause frame drops and jank in a Flutter app. -- **Isolates** allow running Dart code (and FFI calls) on a separate thread. The main isolate sends input and receives the result asynchronously. - -## Design - -The native platform packages (e.g. zstandard_linux) implement: - -1. **Synchronous FFI wrappers** - Top-level functions such as `compress(...)` and `decompress(...)` that call the bindings and return the result size. These run in whichever isolate calls them. - -2. **A long-lived helper isolate** - Created once (lazily) and used for all async requests. It holds the FFI bindings and runs only zstd calls. - -3. **Async entry points** - `compressAsync` and `decompressAsync` (or similar) that: - - Allocate buffers in the **main** isolate (or a sending isolate), - - Send a request to the helper isolate (e.g. via `SendPort`), - - The helper isolate runs `ZSTD_compress` or `ZSTD_decompress` and sends back the result (e.g. written size or error), - - The main isolate copies the result into a new `Uint8List` and completes the `Future`. - -4. **Public API** - The platform’s `compress`/`decompress` methods that implement `ZstandardPlatform` may use the synchronous wrappers on the main isolate for small inputs, or use the async path for large inputs. The exact policy (e.g. threshold) is implementation-defined. Alternatively, all work may be offloaded to the helper isolate for simplicity. - -## Communication - -- **Request types**: e.g. `_CompressRequest` and `_DecompressRequest` holding: request id, pointers to src/dst buffers, sizes, compression level (for compress). -- **Response types**: e.g. `_CompressResponse` and `_DecompressResponse` holding request id and result (size or error code). -- **Ports**: The main isolate has a `ReceivePort` that listens for: - - The helper’s `SendPort` (once, at startup), - - Response objects. Each response is matched to a `Completer` by request id; the completer is completed with the result. -- **Helper isolate**: Has a `ReceivePort` that listens for request objects, runs the appropriate zstd call, and sends the corresponding response back. - -## Memory and Pointers - -- **Pointers** (e.g. `Pointer`) cannot be sent between isolates; only simple values and some Dart objects can. So in the typical design: - - Buffers are allocated in the **helper** isolate (or in shared memory if the implementation uses it). The main isolate sends only the **data** (e.g. as `Uint8List`); the helper isolate allocates, compresses/decompresses, and sends the result back as bytes (or a copy). -- Alternatively, the main isolate allocates buffers and sends a **copy** of the data; the helper isolate allocates its own buffers, copies the input, runs zstd, and sends back the output bytes. The exact approach depends on the package implementation. - -## Usage in the Plugin - -Application code does not see isolates directly. It only calls: - -- `Zstandard().compress(data, level)` -- `Zstandard().decompress(data)` -- or the extension methods on `Uint8List?` - -The platform implementation (e.g. `ZstandardLinux`) implements these as `Future` and may use the isolate-based async path internally so that the UI stays responsive. - -## Web - -The web implementation does **not** use this isolate pattern. It uses the JS/WASM API on the main thread. For large data on web, consider doing the work in a Web Worker if the implementation supports it, or chunking input to avoid long main-thread blocks. - -## Related Documentation - -- [Overview](overview.md) -- [FFI Implementation](ffi-implementation.md) -- [Platform Interface](platform-interface.md) +# Native Isolate Pattern + +All official native platform implementations keep CPU-heavy zstd work away +from the Flutter UI isolate with `Isolate.run`. + +## Safety boundary + +The public platform method captures a Dart-owned `Uint8List` and scalar +options. Inside the worker isolate it constructs or uses the shared +`ZstandardNativeCodec`, allocates native buffers, performs the FFI calls, +copies the result into a new Dart byte list, and frees every native allocation +before completion. + +Native `Pointer` values are never sent between isolates. This matters because +pointer lifetime, ownership, and concurrent access cannot be inferred from a +numeric address. The historical `compressAsync` and `decompressAsync` +pointer helpers remain only for source compatibility: they are deprecated and +execute synchronously in their caller's isolate before their Future completes. + +```text +UI isolate Worker isolate +----------- -------------- +Uint8List + options ───────▶ allocate native input/output + call zstd +new Uint8List ◀────── copy result and free native memory +``` + +Compression emits one frame. Decompression uses `ZSTD_decompressStream`, so +it can consume unknown-size and concatenated frames while checking the output +budget after every produced chunk. A no-progress condition with exhausted +input is treated as a truncated frame. + +Applications only await `Zstandard().compress` or `decompress`; they do not +manage isolates or pointers. For the browser execution model, see +[Web implementation](web-implementation.md). diff --git a/docs/architecture/performance.md b/docs/architecture/performance.md index e715a40d..9e8cbe10 100644 --- a/docs/architecture/performance.md +++ b/docs/architecture/performance.md @@ -25,7 +25,7 @@ Higher levels use more CPU and memory during compression; decompression memory a ## Platform behaviour - **Native (Android, iOS, macOS, Windows, Linux)**: Work runs in a **background isolate** by default, so the UI thread is not blocked. Throughput is comparable to the underlying zstd C library; some builds disable assembly optimizations for portability (e.g. Android, iOS). -- **Web**: Runs on the **main thread** (no isolates). For large data, prefer smaller chunks or offload to a Web Worker if you implement it. Throughput is generally lower than native. +- **Web**: Runs in a dedicated **Web Worker** through transferable byte buffers, keeping the browser UI thread responsive. Throughput is generally lower than native. - **CLI**: Runs in the **current isolate**; suitable for CLI/server where blocking is acceptable. Throughput is similar to native. See the [platform guides](../platforms/) for platform-specific performance notes. @@ -33,7 +33,7 @@ See the [platform guides](../platforms/) for platform-specific performance notes ## Optimization techniques 1. **Choose the right level**: Use 1–3 for speed, 10+ for size when CPU and time allow. -2. **Chunk large data**: Process in fixed-size chunks to limit peak memory and (on web) keep the UI responsive. See [Advanced usage](../guides/advanced-usage.md). +2. **Chunk large data**: Process in fixed-size, independently framed chunks to limit peak memory. See [Advanced usage](../guides/advanced-usage.md). 3. **Reuse the instance**: `Zstandard()` is a singleton; no need to cache it. Same for `ZstandardCLI()`. 4. **Limit concurrency**: Many simultaneous compress/decompress calls increase peak memory; batch or limit parallelism if needed. 5. **Avoid compressing very small payloads**: Frame overhead can make compressed output larger than input; consider a size threshold below which you skip compression. diff --git a/docs/architecture/platform-interface.md b/docs/architecture/platform-interface.md index 40d1e7ca..97fc7483 100644 --- a/docs/architecture/platform-interface.md +++ b/docs/architecture/platform-interface.md @@ -12,6 +12,12 @@ All platform implementations must extend `ZstandardPlatform` and implement: | `compress` | `Future compress(Uint8List data, int compressionLevel)` | Compresses `data` with the given level (1–22). Returns compressed bytes or `null` on failure. | | `decompress` | `Future decompress(Uint8List data)` | Decompresses zstd-compressed `data`. Returns decompressed bytes or `null` on failure. | +Official implementations also implement the separate +`BoundedZstandardPlatform` capability. Its `decompressWithOptions` method +accepts `maxOutputSize` and must enforce it before allocating oversized output. +The separate interface preserves source compatibility for third-party +implementations of the original contract. + ## Abstract Base Class `ZstandardPlatform` extends `PlatformInterface` from plugin_platform_interface: @@ -57,7 +63,7 @@ sequenceDiagram ## Implementing a New Platform 1. Create a new package (e.g. `zstandard_fuchsia`) that depends on `zstandard_platform_interface`. -2. Implement a class that extends `ZstandardPlatform` and implements `getPlatformVersion`, `compress`, and `decompress`. +2. Implement a class that extends `ZstandardPlatform`, implements its three methods, and implements `BoundedZstandardPlatform` for production-safe decompression. 3. Expose a `registerWith()` (or similar) that sets `ZstandardPlatform.instance = YourPlatform()` using the token from the interface package. 4. In the main plugin’s `ZstandardImpl`, add detection for the new platform and call your `registerWith()` when that platform is active. diff --git a/docs/architecture/web-implementation.md b/docs/architecture/web-implementation.md index 0be98510..c8ed1815 100644 --- a/docs/architecture/web-implementation.md +++ b/docs/architecture/web-implementation.md @@ -1,74 +1,47 @@ # Web Implementation -The web platform cannot use Dart FFI. Instead, **zstandard_web** uses JavaScript and WebAssembly: the Zstandard C library is compiled to WASM with Emscripten, and Dart calls into it via JS interop. - -## Architecture +The web platform uses JavaScript interop and a dedicated Web Worker around a +zstd WebAssembly module. It does not use Dart FFI. ```mermaid -graph LR - Dart[zstandard_web Dart] - JS[zstd.js] - WASM[zstd.wasm] - Dart -->|dart:js_interop| JS - JS --> WASM +flowchart LR + Dart[Flutter / Dart] -->|Promise + transferable buffer| Broker[zstd.js] + Broker -->|postMessage| Worker[zstd_worker.js] + Worker --> Core[zstd_core.js] + Core --> WASM[zstd.wasm] ``` -- **zstd.js**: Emscripten-generated JS glue that loads and initializes the WASM module and exposes functions like `compressData` and `decompressData`. -- **zstd.wasm**: Compiled zstd C code (compression, decompression, bounds, frame size). -- **Dart**: Uses `dart:js_interop` (and `package:web`) to call the JS functions and pass `Uint8List` data. - -## Differences from Native Implementations - -| Aspect | Native (FFI) | Web (JS/WASM) | -|--------|----------------|----------------| -| Entry point | C functions via FFI | JS functions `compressData` / `decompressData` | -| Threading | Can use Dart isolates | No isolates for WASM; work runs on main thread or in workers if implemented | -| Memory | Dart + native malloc/free | JS typed arrays + WASM linear memory | -| Build | CMake / Gradle / Xcode | Emscripten (emcc) build of zstd | - -The web implementation does not use the same isolate-based async helpers as native; it awaits the JS interop calls that run the WASM compression/decompression. - -## Required Setup in the App - -1. **Assets**: Copy `zstd.js` and `zstd.wasm` into the Flutter web project (e.g. `web/` directory). The exact paths are documented in [Platforms — Web](../platforms/web.md). -2. **HTML**: Include the script in your `web/index.html` so the WASM module is loaded before the app runs: - ```html - - ``` -3. **Initialization**: The Emscripten module must be loaded and ready before any call to `compress` or `decompress`. The web implementation assumes the global functions exist when invoked. - -## JS API Contract +- `zstd.js` exposes `compressData` and `decompressData` and manages request + IDs, failures, and transferable buffers. +- `zstd_worker.js` runs outside the browser UI thread and loads the generated + core. +- `zstd_core.js` owns the Emscripten runtime, validates inputs and allocations, + streams decompression within the caller's output budget, and always copies + results before freeing WASM memory. +- `zstd.wasm` contains the pinned common/compress/decompress zstd source set. -The Dart code expects the following in the global scope (or on a known object): +All four files must be deployed together. Only `zstd.js` is loaded from the +application HTML; it locates its Worker relative to its own script URL. -- **compressData(inputData, compressionLevel)** - - `inputData`: `Uint8Array` - - `compressionLevel`: number (1–22) - - Returns: `Uint8Array` (compressed) or `null` on error +## Interop contract -- **decompressData(compressedData)** - - `compressedData`: `Uint8Array` - - Returns: `Uint8Array` (decompressed) or `null` on error - -Dart converts between `Uint8List` and JS typed arrays via `dart:js_interop` so that the same `Uint8List` API is used across all platforms. - -## Building zstd.js and zstd.wasm - -zstd is built with Emscripten. High-level steps: - -1. Install and activate [Emscripten SDK](https://emscripten.org/). -2. Clone the [facebook/zstd](https://github.com/facebook/zstd) repository. -3. Run `emcc` on the zstd sources with flags for WASM, exported functions (`ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, `ZSTD_getFrameContentSize`, `malloc`, `free`), and output name. -4. Add the wrapper functions `compressData` and `decompressData` in `zstd.js` (or a separate script) that allocate buffers, call the exported C functions, and return the result or null. - -Detailed commands and wrapper code are in the [zstandard_web README](https://github.com/vypdev/zstandard/tree/master/zstandard_web) and in [Platforms — Web](../platforms/web.md). +```text +compressData(Uint8Array, level) -> Promise +decompressData(Uint8Array, maxOutputSize) -> Promise +``` -## Small Data Handling +The Dart layer validates level and output-limit arguments, awaits the Promise, +and maps JavaScript or Worker failures to `null`. Empty compression is valid. +Decompression uses `ZSTD_decompressStream` and checks every produced chunk +against the caller's limit (256 MiB by default), so unknown-size and +concatenated frames do not require a speculative whole-output allocation. -Some web implementations may return the original data unchanged when the input is very small (e.g. < 9 bytes) because zstd has a minimum frame size. The Dart implementation may handle this by returning the input as-is for both compress and decompress in those cases. See the package source and [Platforms — Web](../platforms/web.md) for the exact behavior. +## Rebuilding -## Related Documentation +`scripts/build_web_wasm.sh` pins both the emsdk commit and Emscripten version, +compiles only the canonical source in `zstandard_native/src/zstd`, normalizes +WASM metadata, and synchronizes each output copy. Generated artifacts are +tested in release mode through ChromeDriver. -- [Overview](overview.md) -- [Platform Interface](platform-interface.md) -- [Platforms — Web](../platforms/web.md) +See [Web platform guide](../platforms/web.md) and the package +[`zstandard_web` README](../../zstandard_web/README.md). diff --git a/docs/deployment/RUNBOOK.md b/docs/deployment/RUNBOOK.md index a475762a..2fe2c9db 100644 --- a/docs/deployment/RUNBOOK.md +++ b/docs/deployment/RUNBOOK.md @@ -1,105 +1,78 @@ -# Deployment and recovery runbook - -This runbook describes how to run a release, what the pipeline does, and how to recover when a release fails. - -## Prerequisites - -- **Pub.dev**: Each runner that publishes (macOS, Linux, Windows) must have run `dart pub login` so that `dart pub publish` can authenticate. Prefer pub.dev trusted publishing with GitHub OIDC when it is configured for all packages. -- **Secrets** (in GitHub repo settings): `PAT` is only used by the optional deployment notification; `GITHUB_TOKEN` creates the tag, draft release, and release assets. -- **Variables** (optional): `DEBUG`, `OPEN_ROUTER_MODEL` (or similar) if your notification/tooling uses them. - -## Running a release - -1. Create or check out a branch named exactly `release/x.y.z`, for example `release/1.5.1`. -2. Update **CHANGELOG.md** at the repo root with the new version and user-facing changes. -3. In GitHub: **Actions → Task - Release → Run workflow**. -4. Fill inputs: - - **version**: Semver, e.g. `1.5.1` (do not include `v`; it must match the release branch). - - **title**: Short release title. - - **changelog**: Summary (or paste from CHANGELOG). - - **issue**: Launcher/issue reference (e.g. `-1` if not used). - - **resume**: Leave disabled for a new release. Enable it only when a previous run already created `v` and stopped during publication. -5. Run the workflow. It will: - - Validate the release branch, version, and remote tag state. - - Copy CHANGELOG into all packages. - - Update all `pubspec.yaml` versions/dependencies, pin SwiftPM to the exact release, and update both CocoaPods podspec versions. - - Regenerate WebAssembly from the canonical C source and verify all committed copies are synchronized. - - Build and verify macOS universal, Linux x86_64/arm64, and Windows x64/ARM64 CLI libraries. - - Run candidate checks for Android (AGP 9 and legacy), Linux, Web, Windows, and Apple with both SwiftPM and CocoaPods. Apple jobs use ARM64; Intel Apple execution is not claimed. - - Create one immutable tag and a draft GitHub release. Every later job checks out that tag, never a moving branch. - - Publish to pub.dev in order: **platform_interface → zstandard_native → platform packages (parallel) → CLI → zstandard**, with one common dry-run barrier for all platform packages. - - Skip already published package versions safely when resuming, verify all ten versions through the pub.dev API, finalize the draft release, and then notify. - -The release workflow only prepares and publishes `release/x.y.z`. A separate integration workflow brings the completed release into `develop` and `master`; this workflow does not alter either branch or their rulesets. - -## Dependency order (for manual publish) - -If you must publish manually (e.g. after a partial failure), use this order: - -```text -zstandard_platform_interface - → zstandard_native - → zstandard_android, zstandard_ios, zstandard_web, zstandard_macos, zstandard_windows, zstandard_linux - → zstandard_cli - → zstandard -``` - -**zstandard_native** contains the shared C source; all native platform packages and the CLI depend on it, so it must be published before them. - -From repo root, with credentials configured: - -```bash -# 1. Platform interface -cd zstandard_platform_interface && dart pub publish -f && cd ../.. - -# 2. Native (shared C source — required by platform packages and CLI) -cd zstandard_native && dart pub publish -f && cd ../.. - -# 3. Platforms (any order after zstandard_native) -for pkg in zstandard_android zstandard_ios zstandard_web zstandard_macos zstandard_windows zstandard_linux; do - (cd $pkg && dart pub publish -f) && cd ../.. -done +# Deployment and Recovery Runbook -# 4. CLI -cd zstandard_cli && dart pub publish -f && cd ../.. +## One-time configuration -# 5. Main plugin -cd zstandard && dart pub publish -f && cd ../.. -``` - -## When a release fails - -- **pub.dev does not allow deleting or overwriting published versions.** Fix the cause and rerun the workflow with the same branch/version and `resume: true`. The workflow checks pub.dev before publishing and never blindly repeats an immutable upload. If the candidate itself must change after the tag was created, use a new patch version; do not move the tag. - -- The workflow includes a **Release recovery guide** job that runs when any preparation, candidate, publication, verification, or finalization job fails. It writes the recovery mode to the GitHub Actions job summary. +1. Configure automated publishing on pub.dev for all ten packages, this GitHub + repository, and stable `vX.Y.Z` tags. +2. Create and protect the GitHub `pub.dev` environment. Require a reviewer if + that matches the project's release policy. +3. Protect stable tags and release/development branches. Keep self-hosted + platform workflows restricted to trusted pushes or manual dispatch. +4. Allow GitHub Actions to create pull requests. After the integration + workflow opens them, a maintainer must approve the resulting safety-gate + runs, which GitHub initially places in an approval-required state. -- **Common causes of failure** - - **Credentials**: Runner not logged in to pub.dev. On each publishing runner (macOS, Linux, Windows), run `dart pub login` and ensure the account has publish rights for the packages. - - **Dependency not found**: A package (e.g. `zstandard_platform_interface`) was just published and pub.dev has not indexed it yet. The workflow waits up to ~10 minutes (with backoff) and verifies via the pub.dev API; if it still fails, wait a bit and re-run the same version. - - **Tests or analyze before the tag**: Fix the release branch and rerun with `resume: false` while no tag exists. - - **Tests or analyze after the tag**: The candidate is immutable; create a new patch release. +No `dart pub login` credential or persistent pub token is required. The tagged +workflow exchanges GitHub's short-lived OIDC identity for publication access. -## Building precompiled CLI libraries (release workflow) +## Prepare a release -The canonical source is **`zstandard_native/src/zstd/`**. The release workflow builds directly from that path through each CMake builder; it does not copy or maintain a second `zstd` tree in `zstandard_cli`. The macOS job cross-compiles x86_64 and arm64 on Apple Silicon and joins them into one universal library. Linux and Windows jobs assert both architecture markers before committing artifacts. +1. Create `release/X.Y.Z` and update the root changelog. +2. Push the branch and run **Task - Prepare Release** on that exact branch. +3. Enter version `X.Y.Z` without `v`; complete the title/changelog inputs. +4. Wait for all native-library builds, release builds, integration tests, and + metadata checks. +5. Review the workflow summary and record the reported candidate SHA. -The Linux release job installs the AArch64 cross compiler through the shared CI dependency action before building. The Windows job detects the installed Visual Studio CMake generator, so it is not coupled to a specific edition or to a hard-coded Visual Studio 2022 path. The runner still needs the C++ desktop workload, CMake, and ARM64 build tools. +The workflow commits its generated versions and native artifacts to the +release branch. It refuses an existing stable tag, so there is no resume mode +that can bypass candidate validation. -## Updating the zstd (C library) version +## Publish -The canonical zstd C source lives in **`zstandard_native/src/zstd/`**. To upgrade: +After approval, create `vX.Y.Z` at the exact remote branch tip and push it: -1. From the repo root, run: - ```bash - ./scripts/update_zstd.sh # latest from dev (upstream default) - ./scripts/update_zstd.sh v1.5.7 # or a specific tag/branch - ``` - This fetches from [facebook/zstd](https://github.com/facebook/zstd) and updates `zstandard_native/src/zstd/`. -2. Run `zstandard_ios/scripts/sync_zstd.sh` and `zstandard_macos/scripts/sync_zstd.sh` (from repo root) to refresh the ignored compatibility trees used by CocoaPods. Swift Package Manager consumes the canonical source through the repository-level `Package.swift` and does not need copied Apple trees. -3. Optionally run `./scripts/regenerate_bindings.sh` and commit any changed `*_bindings_generated.dart` files. -4. Commit the changes. For releases, the workflow builds precompiled CLI libraries; see `.github/workflows/release_workflow.yml` for how each runner obtains the zstd source (e.g. from the repo or a pinned ref). - -## Related docs +```bash +git fetch origin release/X.Y.Z +test "$(git rev-parse origin/release/X.Y.Z)" = "EXPECTED_CANDIDATE_SHA" +git tag -s vX.Y.Z EXPECTED_CANDIDATE_SHA +git push origin vX.Y.Z +``` -- [Release process](../development/release-process.md) – versioning and pre-release checklist. -- [SECURITY.md](../../SECURITY.md) – reporting vulnerabilities and CI security practices. +**Publish Tagged Release** independently rechecks the tag/branch identity, +metadata, five CLI native libraries, and four synchronized Web artifacts. It +then publishes sequentially in dependency order, waits for public indexing, +runs a publication dry-run immediately before each unpublished package, +verifies all ten versions, and creates the GitHub release with checksums. + +## Recovery + +- Preparation failure: fix the release branch and rerun preparation. No tag or + package has been published. +- Publication job interrupted: rerun the same tag workflow. The helper skips + a package version only after pub.dev reports it publicly, then continues in + dependency order. +- Dependency not indexed: the workflow waits up to 20 minutes per package. If + pub.dev remains unavailable, rerun later with the same immutable tag. +- Incorrect tagged content: never move or recreate the tag and never attempt + to overwrite a pub.dev version. Prepare a new patch release. +- GitHub release failure after all packages publish: rerun the workflow; its + release creation/upload is idempotent. + +After publication, run **Integrate Published Release** with the same version. +It revalidates the final release and opens or reuses pull requests from the +release branch into `master` and `develop`. Review and merge both normally; +the workflow does not merge or bypass branch protection. Do not delete the +release branch until both pull requests are complete. + +## Updating upstream zstd + +`scripts/update_zstd.sh` defaults to the repository's pinned upstream commit; +an explicit tag or commit can be passed for an intentional upgrade. Review the +diff, update `zstandard_native/UPSTREAM_ZSTD.md`, regenerate bindings when the +public header changes, regenerate WebAssembly, and rerun every platform gate. +Apple CocoaPods sync scripts create ignored compatibility copies; SwiftPM uses +the canonical repository target directly. + +See [Release process](../development/release-process.md) and +[`SECURITY.md`](../../SECURITY.md). diff --git a/docs/development/apple-dependencies.md b/docs/development/apple-dependencies.md index 42674c59..ab24b4c5 100644 --- a/docs/development/apple-dependencies.md +++ b/docs/development/apple-dependencies.md @@ -32,14 +32,13 @@ The iOS and macOS manifests depend on the repository-level SwiftPM facade: ```swift .package( url: "https://github.com/vypdev/zstandard.git", - branch: "develop" + exact: "1.5.0" ) ``` -The development branch is used while the facade is being integrated into the -next release. Before publishing a plugin release, replace it with an -immutable release tag (or revision) that contains the matching -`zstandard_native` source and `Package.swift`. +The dependency is always immutable and matches the plugin package version. +Release preparation updates both manifests to the requested exact version; +mutable branches are never used in a published manifest. The SwiftPM target deliberately excludes unsupported or unnecessary upstream directories and disables assembly for the Apple build. It also preserves the @@ -58,8 +57,10 @@ native integration tests, not only by manifest parsing. Every Apple change must be tested through both dependency managers on the ARM64 self-hosted macOS runner: -- iOS simulator builds and integration tests with Swift Package Manager; -- iOS simulator builds and integration tests with CocoaPods; +- unsigned iOS device Release builds plus simulator Debug builds and + integration tests with Swift Package Manager; +- unsigned iOS device Release builds plus simulator Debug builds and + integration tests with CocoaPods; - macOS application builds and integration tests with Swift Package Manager; and - macOS application builds and integration tests with CocoaPods. @@ -70,27 +71,24 @@ runner's existing graphical session. macOS applications still require a WindowServer session, so the macOS runner must remain logged in even though no manual interaction is needed. -The matrix also exercises the platform packages with the Dart -`zstandard_native` dependency resolved from the Pub cache. In the workspace -rows, SwiftPM receives `ZSTANDARD_NATIVE_PACKAGE_PATH` and uses the checkout's -repository-level package; in the Pub-cache rows, SwiftPM resolves the remote -repository package while CocoaPods syncs the native C source from the cached -`zstandard_native` package. This catches accidental reliance on the monorepo -checkout and ensures that published platform packages still locate the native -dependency. +The matrix also exercises the platform packages with the current Dart +`zstandard_native` package staged outside the repository. In the workspace +rows, SwiftPM receives `ZSTANDARD_NATIVE_PACKAGE_PATH` for the checkout root. +In the package-config rows, Dart, CocoaPods, and SwiftPM all receive the exact +external path to the same candidate source. This catches accidental reliance +on the monorepo's sibling-directory layout without compiling against an older +version from the runner's Pub cache. The workflows use Flutter 3.47.2, which is new enough for Flutter's default SwiftPM integration. CocoaPods jobs explicitly disable SwiftPM so that both -paths are tested independently. - -The SwiftPM pub-cache integration jobs also create a temporary, ignored -`FlutterFramework` package copy next to the plugin package. Flutter's -integration-test project inspection evaluates a plugin manifest from its -resolved source path, while the generated framework package lives under the -example application's ephemeral SwiftPM directory and may be regenerated -while the test starts. The copy makes the manifest's standard -`../FlutterFramework` dependency resolvable in that test mode and is removed -at the end of the job. +paths are tested independently. Jobs that change this persistent Flutter +preference share a non-cancelling concurrency group across iOS, macOS, and +release validation, and cleanup restores CocoaPods mode. + +Flutter generates its ephemeral `FlutterFramework` dependency beside the +example application's generated plugin package. The platform plugin itself +remains in the checkout in both source-layout rows, so no CI-only manifest or +framework copy is required. ## Migration policy diff --git a/docs/development/building.md b/docs/development/building.md index efadc216..f9b5123d 100644 --- a/docs/development/building.md +++ b/docs/development/building.md @@ -65,7 +65,7 @@ If you are developing or modifying a platform package’s native code: - The checked-in Apple examples target iOS 15.0 and macOS 12.0, matching the minimum deployment targets enforced by the Flutter 3.47.2 SDK used in CI. - In SwiftPM mode the native target is statically linked into the app and Dart FFI resolves its symbols from the process. In CocoaPods mode the plugin framework is embedded and loaded by path. -The Apple workflows run the example applications through a matrix covering both dependency managers (`swiftpm` and `cocoapods`) and both native-source locations (the repository workspace and the pub cache). Each workflow limits its native matrix to one job at a time, while runner availability determines whether the iOS and macOS workflows run concurrently. There is no cross-workflow concurrency group that could discard queued matrix jobs. Each native job builds the example, launches it through Flutter's integration-test runner, and executes the integration tests on the iOS simulator or macOS desktop. The runner uses its preinstalled Apple Silicon Flutter SDK; no SDK download action is used. +The Apple workflows run the example applications through a matrix covering both dependency managers (`swiftpm` and `cocoapods`) and both native-source layouts (the repository workspace and an exact external path selected through Dart package configuration). Jobs that switch Flutter's global dependency-manager preference share a non-cancelling cross-workflow lock, so iOS and macOS cannot race on the persistent SDK. Each native job builds the example, launches it through Flutter's integration-test runner, and executes the integration tests on the iOS simulator or macOS desktop. The preference is restored to CocoaPods mode during cleanup. The runner uses its preinstalled Apple Silicon Flutter SDK; no SDK download action is used. ### Linux diff --git a/docs/development/ci-cd.md b/docs/development/ci-cd.md index 11761ffe..fe54fd06 100644 --- a/docs/development/ci-cd.md +++ b/docs/development/ci-cd.md @@ -1,140 +1,71 @@ # CI/CD -This document describes the continuous integration and deployment setup for the Zstandard plugin and CLI, including GitHub Actions workflows and how to use them. - -## Overview - -The repository uses **GitHub Actions** for: - -- **PR checks**: Analyze and test each package on pull requests (open or new commits; except to protected branches). -- **Release workflow**: Version bumping, building precompiled CLI libraries, tagging, and publishing to pub.dev. -- **Hotfix workflow**: Expedited fixes and releases when needed. - -Workflows are in [`.github/workflows/`](https://github.com/vypdev/zstandard/tree/master/.github/workflows). - -## PR check workflows - -Each package has a dedicated workflow that runs on pull requests (open or new commits to the PR branch) to non-protected branches: - -| Workflow file | Package | Runner | Steps | -|---------------|---------|--------|--------| -| `pr_check_zstandard.yml` | zstandard | self-hosted Linux | Analyze, Test (with coverage), Publish dry run | -| `pr_check_android.yml` | zstandard_android | self-hosted Linux | Analyze, Build APK, Android emulator integration tests, Publish dry run | -| `pr_check_ios.yml` | zstandard_ios | self-hosted macOS ARM64 | Analyze, build, headless simulator launch, SwiftPM and CocoaPods integration tests from checkout and Pub cache, Publish dry run | -| `pr_check_macos.yml` | zstandard_macos | self-hosted macOS ARM64 | Analyze, build, launch, SwiftPM and CocoaPods integration tests from checkout and Pub cache, Publish dry run | -| `pr_check_linux.yml` | zstandard_linux | self-hosted Linux | Analyze, Build Linux app, Linux integration tests, Publish dry run | -| `pr_check_windows.yml` | zstandard_windows | self-hosted Windows X64 | Analyze, build, native C++ tests, Dart tests, Windows integration tests, Publish dry run | -| `pr_check_web.yml` | zstandard_web | self-hosted Linux | Analyze, Build Web app, ChromeDriver integration tests, Publish dry run | -| `pr_check_cli.yml` | zstandard_cli | self-hosted Linux | Analyze, Test (with coverage), Publish dry run | -| `pr_check_platform_interface.yml` | zstandard_platform_interface | self-hosted Linux | Analyze, Test (with coverage), Publish dry run | - -There is no dedicated PR check workflow for **zstandard_native** (it has no Dart tests; it mainly ships C source and bindings). It is published in the release workflow after `platform_interface` and before the platform packages that depend on it. - -Pull request checks run for pull requests targeting any branch except `master`. Push checks run on `develop` so the branch itself is continuously verified. - -**Concurrency**: Only the latest run per branch/PR is kept; in-progress runs are cancelled when new commits are pushed to the PR. - -### Coverage - -- Flutter packages: `flutter test --coverage` produces `coverage/lcov.info` in the package directory. -- CLI package: `dart test --coverage=coverage` then `dart run coverage:format_coverage` to produce lcov. -- Coverage is uploaded to **Codecov** via the `codecov/codecov-action@v4` step. Upload failures are job failures because coverage reporting is part of the check contract. - -### Self-hosted Linux runner contract - -The Linux workflows target the repository labels `[self-hosted, Linux]`. The runner account must be able to use passwordless `sudo` so the local composite action at [`.github/actions/setup-linux-dependencies/action.yml`](../../.github/actions/setup-linux-dependencies/action.yml) can install the required packages. - -The release workflow enables the action's `arm64` option, which installs the -`aarch64-linux-gnu` cross compiler before producing the Linux ARM64 CLI -artifact; it does not rely on a manually prepared compiler on the runner. - -The action installs the common native toolchain (`build-essential`, Clang, CMake, Ninja, `pkg-config`, and `curl`). Desktop jobs additionally install GTK and Xvfb; coverage jobs install `lcov` and `bc`. - -Before installing anything, the action repairs an interrupted Debian package -transaction with `dpkg --configure -a` and `apt-get -f install`, then runs the -installation under a host-local lock and verifies the result with -`apt-get check`. This is required because self-hosted runners preserve their -package database between jobs; a cancelled job must not make the next check -fail before the project is even built. If recovery itself fails on a runner, -inspect `dpkg --audit` and the package-manager logs on that host. - -Android runners must also provide: - -- Android SDK with `platform-tools` and the emulator on the SDK path. -- Java 17. `/dev/kvm` is optional because the current workflow explicitly uses software emulation. -- Network access for the API 30 `aosp_atd` x86_64 system image used by the emulator launcher. Android ATD is intentionally used because it removes background services that are irrelevant to these tests and reduces CPU/memory pressure on the software-only runner. - -The Android job builds the example APK before booting the emulator. The Linux job builds the example application before running its integration tests. The Web job installs a matching Chrome/ChromeDriver pair, builds the example, and runs the `flutter drive` integration suite under Xvfb. A missing dependency or failed build is an explicit failure; platform integration tests are not silently skipped in CI. - -Android integration tests verify non-empty payloads, exact -`decompress(compress(input))` round-trips, empty input, corruption handling, -all documented compression levels, and deterministic property-based cases. -The Android instrumentation suite repeats the round-trip and invalid-frame -checks directly through JNI against the same shared library loaded by Dart -FFI. The level-22 test uses a small payload because the CI emulator is -software-only; large-content behavior is covered separately. - -PR checks pin Flutter 3.47.2. This is intentional: the organisation-level `FLUTTER_VERSION` variable previously selected Flutter 3.41.4, which is below the minimum required by the current root and Android packages. - -Apple workflows use the self-hosted ARM64 Mac runner and validate both native dependency managers. iOS simulators are booted through `simctl` without opening the Simulator UI; the iOS job invokes the single integration file explicitly because the example intentionally keeps all iOS checks in one test process. macOS tests run the built app in the runner's logged-in graphical session. Windows uses self-hosted X64 runners and requires a Windows desktop session plus Visual Studio/CMake for native builds and launched integration tests. - -### Self-hosted Windows runner contract - -Windows jobs target `[self-hosted, Windows, X64]`. The workflow installs and caches the pinned Flutter SDK on each runner, then enables Windows desktop support and verifies CMake. Each runner must provide a Visual Studio C++ desktop toolchain and an interactive desktop session because Flutter's Windows integration tests launch the compiled example application. - -The Windows workflow builds and tests the implementation package from the repository and from the published `zstandard_native` package in the Pub cache. Each path runs the native C++ zstd round-trip tests, Dart package tests, and launched Flutter integration tests. A separate job runs the federated `zstandard/example` application to verify the public plugin entry point on Windows. - -## Release workflow - -**File**: [`.github/workflows/release_workflow.yml`](https://github.com/vypdev/zstandard/blob/master/.github/workflows/release_workflow.yml) - -Triggered manually (**workflow_dispatch**) with inputs such as version, title, changelog, and optional issue reference. - -**Main phases**: - -1. **Prepare candidate**: Require `release/x.y.z`, bump all package versions/dependencies, pin SwiftPM to the exact version, update CocoaPods podspecs, copy CHANGELOG, and regenerate WebAssembly from `zstandard_native/src/zstd/`. -2. **Build precompiled CLI libraries** (on platform-specific runners): - - **macOS**: Cross-compile x86_64 and ARM64 from the canonical C source, merge with `lipo` into a universal `libzstandard_macos.dylib`, and assert both slices; commit. - - **Linux**: Build x86_64 and ARM64 `.so` files from the canonical C source and assert ELF machine headers; commit. - - **Windows**: Detect the installed Visual Studio generator, build x64 and ARM64 DLLs, and assert PE machine headers; commit. -3. **Candidate verification**: Build and execute Android (AGP 9 and legacy), Linux, Web, Windows, and both Apple dependency-manager variants. Candidate jobs use local overrides until their pub.dev versions exist. -4. **Tag and release**: Create one immutable git tag and a draft GitHub release with changelog and checksums. -5. **Publish**: Publish packages to pub.dev in dependency order: **platform_interface → zstandard_native** (shared C source) **→ platform implementations** (android, ios, macos, linux, windows, web) **→ zstandard_cli → zstandard**. -6. **Verify and finalize**: Verify all ten package versions through the pub.dev API, finalize the draft release, and notify. A separate workflow integrates the release branch into `develop` and `master`. - -The workflow uses **self-hosted** runners for macOS, Linux, and Windows to build native binaries and run platform-specific steps. - -## Hotfix workflow - -**File**: [`.github/workflows/hotfix_workflow.yml`](https://github.com/vypdev/zstandard/blob/master/.github/workflows/hotfix_workflow.yml) - -Used for expedited fixes (e.g. security or critical bugs). Typically triggered manually and may skip some steps or use a shorter path to release. See the workflow file and team docs for details. - -## Running checks locally - -To mimic CI locally: - -- **Analyze**: `flutter analyze` or `dart analyze` in each package. -- **Test**: `flutter test` or `dart test` in each package. -- **Build**: `flutter build apk --debug`, `flutter build linux --debug`, or `flutter build web --release` for the platform examples before integration tests. -- **Test with coverage**: `flutter test --coverage` (Flutter) or `dart test --coverage=coverage` then format (CLI). -- **All packages**: Use the [test scripts](../../scripts/) (e.g. `./scripts/test_all.sh` or `scripts\test_all.bat`). - -## Build automation scripts - -Scripts under [**scripts/**](https://github.com/vypdev/zstandard/tree/master/scripts) help build native libraries and run tests locally: - -- `build_macos.sh`, `build_linux.sh`, `build_windows.bat`: Build precompiled zstd libraries for the CLI. -- `build_android.sh`, `build_ios.sh`: Build or prepare the Android/iOS plugin. -- Swift Package Manager consumes the canonical zstd C source through the repository-level `Package.swift`; the sync scripts refresh only the ignored `Classes/zstd/` compatibility trees used by CocoaPods. -- `regenerate_bindings.sh`: Regenerate FFI bindings (ffigen) for all platform packages after zstd source updates. -- `test_all.sh` / `test_all.bat`: Run tests in all packages. -- `coverage_report.sh` / `coverage_report.bat`: Generate coverage reports. - -See the script contents and [Building](building.md) for requirements (CMake, NDK, Xcode, etc.). - -## See also - -- [Release process](release-process.md) -- [Testing](testing.md) -- [Building](building.md) +## Pull requests + +`pull_request.yml` is the untrusted-code safety gate. It runs on GitHub-hosted +Ubuntu with read-only contents permission, resolves monorepo dependencies, +formats and analyzes all ten packages, executes every available VM/unit test, +and validates metadata, generated Web artifacts, and shell syntax. A required +hosted matrix collects coverage for the CLI, platform interface, and main +package on every pull-request head, enforces their package-specific thresholds, +and uploads the three matching Codecov flags for trusted same-repository pull +requests. Forks run the same coverage gates without receiving the upload token. +Its final +`Pull Request Safety Gate` job is the stable required-check context; repository +rules should require that one context on both `master` and `develop` rather +than coupling protection to individual matrix-entry names. + +Untrusted pull-request code is never sent to persistent self-hosted runners. +The platform-heavy workflows (`pr_check_*.yml`, retained under their historic +names) run only on trusted pushes to `develop` or by explicit manual dispatch. +They provide Android emulator/instrumentation, Linux desktop, ChromeDriver, +Windows desktop/native, and both SwiftPM and CocoaPods Apple integration. They +also run publish dry-runs and coverage where applicable. + +Every third-party action reference is pinned to a full commit SHA. Copilot +automation uses `pull_request_target` or issue metadata without checking out +untrusted pull-request code, grants read-only repository contents, and only +uses its PAT for OWNER, MEMBER, or COLLABORATOR actors. + +## Release workflows + +- `release_workflow.yml` (**Task - Prepare Release**) runs only by manual + dispatch on `release/X.Y.Z`. It mutates the release branch, builds native + artifacts, and validates the complete candidate. It neither tags nor + publishes. +- `publish_workflow.yml` (**Publish Tagged Release**) runs only for a stable + `vX.Y.Z` tag, validates that it points to the matching release-branch tip, + publishes all packages with OIDC on GitHub-hosted Ubuntu, verifies pub.dev, + and creates the GitHub release. +- `integrate_release.yml` (**Integrate Published Release**) is a separate + manual workflow. It verifies the completed publication and opens reviewed + pull requests into `master` and `develop`; it never merges or bypasses + branch protection. + +Both workflows share one non-cancelling concurrency group, preventing two +publication state machines from overlapping. + +## Runner trust and requirements + +Self-hosted Linux, Windows, and macOS runners are persistent trusted release +infrastructure. They must not be exposed to fork pull requests. Linux needs +the native/GTK toolchain, Android SDK/Java 17, Chrome, and optional AArch64 +cross-tools. Windows needs Visual Studio C++/CMake and an interactive desktop. +Apple needs Xcode, simulator runtimes, CocoaPods, and a logged-in WindowServer +session for macOS integration. + +The local composite actions verify the pinned Flutter 3.47.2 toolchain. +Platform workflows build release candidates as well as debug test hosts; +missing infrastructure is a failure rather than a silent skip. + +## Local equivalents + +Use `dart scripts/create_local_overrides.dart ` before resolving an +individual monorepo package; the `.sh` wrapper remains available on Unix-like +systems. Run `dart format --output=none +--set-exit-if-changed .`, the appropriate analyzer/test command, and the +platform integration script. `scripts/check_pub_metadata.dart` validates the +publication contract; `scripts/build_web_wasm.sh` regenerates Web artifacts. + +See [Testing](testing.md), [Self-hosted CI](self-hosted-ci.md), and +[Release process](release-process.md). diff --git a/docs/development/emulator-setup.md b/docs/development/emulator-setup.md index 257023a8..375bf90d 100644 --- a/docs/development/emulator-setup.md +++ b/docs/development/emulator-setup.md @@ -6,7 +6,7 @@ This document describes how to set up Android emulators, iOS simulators, Linux d ### CI (GitHub Actions) -The push and release workflows use `scripts/run_android_emulator_ci.sh` on the `[self-hosted, Linux]` runner to install the required SDK components, create an API 31 `google_atd` `pixel_4` AVD, and run the Android integration tests. The Android Test Device image avoids Google Play services that are not needed by this package's tests and is suitable for headless execution. The generated AVD disables GPS because the tests do not use location, and the launcher disables the emulator's `GnssGrpcV1` feature so it does not start an unnecessary host GNSS socket. Vulkan is disabled explicitly because this software-only runner has experienced emulator crashes while initializing the Vulkan/SwiftShader backend; `swiftshader_indirect` remains enabled for GLES rendering. The workflow builds the APK first and explicitly uses software emulation because the current self-hosted runner does not expose `/dev/kvm`. The launcher intentionally starts the emulator directly instead of using a generic action whose unconditional input-unlock step is unreliable while this slow software image is still bringing up Android services. Before Flutter starts, `scripts/wait_for_android_ci_services.sh` requires the package, input, and settings services plus the package manager to pass a single combined `adb` probe repeatedly for 90 seconds; this avoids relying on the image's early `boot_completed` signal without multiplying slow ADB round trips. Every ADB operation has its own timeout so a stalled ADB daemon cannot consume the whole job silently; `ANDROID_ADB_TIMEOUT_SECONDS` can override the 30-second default. If boot or readiness times out, the launcher prints the emulator log and the readiness script prints Android logcat output. +The push and release workflows use `scripts/run_android_emulator_ci.sh` on the `[self-hosted, Linux]` runner to install the required SDK components, create an API 30 `aosp_atd` `pixel_2` AVD, and run the Android integration tests. The Android Test Device image avoids Google Play services that are not needed by this package's tests and is suitable for headless execution. The generated AVD disables GPS because the tests do not use location, and the launcher disables the emulator's `GnssGrpcV1` feature so it does not start an unnecessary host GNSS socket. Vulkan is disabled explicitly because this software-only runner has experienced emulator crashes while initializing the Vulkan/SwiftShader backend; `swiftshader_indirect` remains enabled for GLES rendering. The workflow builds the APK first and explicitly uses software emulation because the current self-hosted runner does not expose `/dev/kvm`. The launcher intentionally starts the emulator directly instead of using a generic action whose unconditional input-unlock step is unreliable while this slow software image is still bringing up Android services. Before Flutter starts, `scripts/wait_for_android_ci_services.sh` requires the package, input, and settings services plus the package manager to pass a single combined `adb` probe repeatedly for 90 seconds; this avoids relying on the image's early `boot_completed` signal without multiplying slow ADB round trips. Every ADB operation has its own timeout so a stalled ADB daemon cannot consume the whole job silently; `ANDROID_ADB_TIMEOUT_SECONDS` can override the 30-second default. If boot or readiness times out, the launcher prints the emulator log and the readiness script prints Android logcat output. ### Local: Prerequisites diff --git a/docs/development/release-process.md b/docs/development/release-process.md index c13b5100..2a7b2f9c 100644 --- a/docs/development/release-process.md +++ b/docs/development/release-process.md @@ -1,78 +1,102 @@ # Release Process -This document outlines how releases of the Zstandard plugin and CLI are prepared and published. The project uses a centralized version and CHANGELOG across all packages. - -## Versioning - -- All packages (zstandard, zstandard_platform_interface, **zstandard_native**, zstandard_android, zstandard_ios, zstandard_macos, zstandard_linux, zstandard_windows, zstandard_web, zstandard_cli) share the **same version number** (e.g. 1.5.0). **zstandard_native** contains the shared C source and is published so that platform packages and the CLI can depend on it from pub.dev. -- Follow [semantic versioning](https://semver.org/): MAJOR.MINOR.PATCH. Bump: - - **MAJOR** for incompatible API changes. - - **MINOR** for new backward-compatible features. - - **PATCH** for backward-compatible bug fixes. - -## Pre-Release Checklist - -- [ ] All tests pass (`flutter test` / `dart test` in each package). -- [ ] `flutter analyze` (or `dart analyze`) reports no errors in the packages you are releasing. -- [ ] CHANGELOG.md is updated with user-facing changes for the release. -- [ ] Version in root and in each package’s `pubspec.yaml` is updated to the new version. -- [ ] Inter-package dependencies use the new version (e.g. `zstandard_android` depends on `zstandard_platform_interface: ^x.y.z` and `zstandard_native: ^x.y.z`). -- [ ] Every published package description is between 50 and 180 characters and uses HTTPS metadata URLs. -- [ ] `dart pub publish --dry-run` is clean for every package; verify that the iOS and macOS archives contain their `Package.swift` manifests. -- [ ] Run `pana` against a copy of the main package and review its report before publishing. - -## Release Workflow (CI) - -The project uses a **Release** workflow (GitHub Actions “Task - Release”) that runs from a `release/x.y.z` branch and: - -1. **Validates** that the branch matches the requested version and that the tag is available. -2. **Copies** CHANGELOG.md into each package (including zstandard_native). -3. **Updates** `version:` and dependency versions in every package’s `pubspec.yaml`, pins Apple SwiftPM to the exact release, and synchronizes CocoaPods podspec versions. -4. **Regenerates and verifies** WebAssembly from the canonical `zstandard_native/src/zstd/` source. -5. **Builds and verifies** CLI libraries for macOS universal, Linux x86_64/arm64, and Windows x64/ARM64. -6. **Runs candidate builds and integration tests** for Android (AGP 9 and legacy), Linux, Web, Windows, and Apple (SwiftPM and CocoaPods on ARM64 macOS). -7. **Creates** one immutable git tag and a draft GitHub release. All publication jobs check out that tag. -8. **Publishes** packages to pub.dev in dependency order: **platform_interface → zstandard_native** (shared C source) **→ platform implementations** (android, ios, macos, linux, windows, web) **→ zstandard_cli → zstandard**. - -The workflow is typically triggered manually (workflow_dispatch) with inputs such as: - -- **version**: e.g. `1.5.0` -- **title**: Release title -- **changelog**: Summary of changes -- **issue**: Optional launcher issue reference -- **resume**: Set to `true` only to continue a partial release whose immutable tag already exists. - -## Manual Steps (if not using full automation) - -If you need to release without the full workflow: - -1. Create `release/x.y.z` and update **CHANGELOG.md** at the repo root with the new version and list of changes. -2. Update **version** in every package’s **pubspec.yaml** to the new version. -3. Update **dependency versions** in each package that depends on another (e.g. `zstandard_android` depends on `zstandard_platform_interface: ^X.Y.Z` — set to the new version). -4. Copy **CHANGELOG.md** into each package’s directory if the project keeps a copy per package. -5. **Publish** in dependency order: - - `zstandard_platform_interface` - - `zstandard_native` (platform packages and CLI depend on it) - - Platform packages (android, ios, macos, linux, windows, web) - - `zstandard_cli` - - `zstandard` -6. **Tag** the release only after candidate checks: `git tag vX.Y.Z` (e.g. `v1.5.0`) and push the tag. -7. **Create** a draft GitHub release, upload checksums/package archives, and finalize it only after pub.dev verification. - -After publication, a separate workflow integrates the release branch into `develop` and `master`. The release workflow does not alter either branch or their rulesets. - -## Publishing to pub.dev - -- Use `dart pub publish` (or `flutter pub publish`) from each package directory. Confirm the package name and version when prompted. -- Ensure you are logged in (`dart pub login`) and have permissions to publish the package. Prefer pub.dev trusted publishing with GitHub OIDC once configured for all packages. -- Publish in order so that dependencies are available: platform_interface first, then **zstandard_native**, then platform implementations, then zstandard and zstandard_cli. - -## After Release - -- Bump the development version in `pubspec.yaml` files if the project uses a separate “next” version (e.g. 1.3.30+1 or 1.5.0-dev). -- Add an “Unreleased” or “Next” section in CHANGELOG.md for the next release. -- Announce the release (e.g. GitHub release notes, changelog link) as appropriate. - -## Hotfixes - -For critical fixes, the project may use a **hotfix** workflow (see `.github/workflows/hotfix_workflow.yml` and issue templates). Follow the same versioning and publish order; use a PATCH bump (e.g. 1.5.0 → 1.4.1). +All ten published packages use one stable `MAJOR.MINOR.PATCH` version. A +release has three deliberately separate stages: candidate preparation, +tag-triggered publication, and reviewed branch integration. + +## Candidate checklist + +- Update the root changelog with user-facing changes. +- Run format, analysis, tests, native integration tests, metadata checks, and + `dart pub publish --dry-run` for every package. +- Confirm `zstandard_native/UPSTREAM_ZSTD.md` records the exact upstream zstd + version, commit, and any backport. +- Confirm all four generated Web files are synchronized across the package and + examples. +- Confirm the Apple SwiftPM manifests and CocoaPods podspecs build the same + common/compress/decompress source set. + +## Stage 1: prepare and validate + +Create and push `release/X.Y.Z`, then run **Task - Prepare Release** from that +branch with version `X.Y.Z`. The workflow refuses existing tags and mismatched +branch/version values. It: + +1. updates changelogs, package/dependency versions, podspecs, and exact SwiftPM + versions; +2. regenerates the pinned Web Worker/WASM artifacts; +3. builds and commits macOS universal, Linux x64/arm64, and Windows x64/arm64 + CLI libraries; +4. runs static, release-build, and native integration candidate gates on the + trusted platform runners; and +5. reports the exact remote release-branch commit that is ready to tag. + +The preparation workflow does not create a tag and cannot publish. Re-run it +only while the release tag does not exist. + +## Stage 2: immutable tag and OIDC publication + +After reviewing the successful candidate, push `vX.Y.Z` at exactly the commit +reported by the workflow: + +```bash +git fetch origin release/X.Y.Z +git tag -s vX.Y.Z origin/release/X.Y.Z +git push origin vX.Y.Z +``` + +The **Publish Tagged Release** workflow revalidates that the tag equals the +tip of `origin/release/X.Y.Z`, publishes through GitHub OIDC in this order, and +waits for pub.dev indexing between dependency layers: + +```text +zstandard_platform_interface + → zstandard_native + → zstandard_android → zstandard_ios → zstandard_linux + → zstandard_macos → zstandard_web → zstandard_windows + → zstandard_cli + → zstandard +``` + +It then verifies all ten public versions and creates an immutable GitHub +release containing checksums for the native and Web artifacts. No dirty source +archives are attached. + +Publication is idempotent for a workflow retry: versions already visible on +pub.dev are skipped. Before each unpublished package is sent, its dependencies +are resolved and `dart pub publish --dry-run` must pass. A tag never moves. If +the tagged candidate is wrong, prepare a new patch version. + +## Stage 3: reviewed branch integration + +After publication succeeds, run **Integrate Published Release** with `X.Y.Z`. +It verifies that the stable tag equals `release/X.Y.Z`, that the GitHub Release +is final, and that all ten versions are visible on pub.dev. It then opens or +reuses pull requests from the release branch into `master` and `develop`. + +The workflow never merges a pull request and never bypasses a ruleset. Keep +the release branch until both pull requests have passed their normal review +and required checks and have been merged. + +## Required external configuration + +For every pub.dev package, configure automated publishing for this GitHub +repository and the stable `vX.Y.Z` tag pattern. Protect the GitHub `pub.dev` +environment with appropriate reviewer rules. The workflow uses +`id-token: write`; no persistent pub token belongs on a runner. + +Allow GitHub Actions to create pull requests in the repository Actions +settings. The integration workflow uses only its short-lived `GITHUB_TOKEN` +with contents-read and pull-requests-write permissions. GitHub places checks +triggered by an automation-created pull request in an approval-required state; +a maintainer must approve those runs from the pull request before merging. + +Protect `master`, `develop`, release branches, and stable tags with active +GitHub rulesets. Require pull requests and the stable `Pull Request Safety +Gate` status context on both long-lived branches, plus deletion and +non-fast-forward protection. Remove obsolete per-matrix context names before +enabling a ruleset. The integration workflow only opens reviewed pull +requests; it cannot merge or bypass those protections. + +See the [deployment runbook](../deployment/RUNBOOK.md) and +[CI/CD](ci-cd.md). diff --git a/docs/development/self-hosted-ci.md b/docs/development/self-hosted-ci.md index 7df13518..55f50e0e 100644 --- a/docs/development/self-hosted-ci.md +++ b/docs/development/self-hosted-ci.md @@ -17,7 +17,8 @@ The expected baseline is: - Flutter 3.47.2, selected explicitly by the workflows through [`.github/actions/setup-flutter/action.yml`](../../.github/actions/setup-flutter/action.yml). The action also marks the shared Flutter checkout as a Git safe directory, which is required by the self-hosted runner image. - `build-essential`, Clang, CMake, Ninja, `pkg-config`, and `curl`. - GTK 3 development headers and Xvfb for Linux/Web desktop jobs. -- `lcov` and `bc` for coverage jobs. +- Dart's coverage formatter; thresholds are evaluated by + `scripts/check_lcov_coverage.dart` without host-specific LCOV tooling. ## Android job requirements @@ -26,23 +27,38 @@ In addition to the common Linux baseline, Android jobs require the emulator runt - Android SDK root exposed as `ANDROID_SDK_ROOT` or `ANDROID_HOME`. - An Android SDK location with writable command-line tools and `platform-tools/adb`. The workflow bootstraps the command-line tools and platform tools when the runner image does not already provide them. - Java 17. -- KVM is useful for accelerating the x86_64 API 31 emulator, but is not required by the current workflow because the runner does not expose `/dev/kvm`; the workflow explicitly uses software emulation. +- KVM is useful for accelerating the x86_64 API 30 emulator, but is not required by the current workflow because the runner does not expose `/dev/kvm`; the workflow explicitly uses software emulation. The workflow also calls [`scripts/check_android_ci_prerequisites.sh`](../../scripts/check_android_ci_prerequisites.sh), which discovers SDKs in the standard Linux locations when the environment variable is missing. A runner should still export `ANDROID_SDK_ROOT` explicitly so all Android tooling uses the same installation. -The workflow first bootstraps the Android SDK tools, then runs `flutter build apk --debug`, starts an API 31 `google_atd` `pixel_4` emulator with software acceleration through `scripts/run_android_emulator_ci.sh`, waits for the Android `package`, `input`, and `settings` services to remain stable across repeated checks, and runs every file in `zstandard_android/example/integration_test/`. The Android Test Device image is intentionally used for headless testing because these package tests do not require Google Play services. GPS is disabled in the generated AVD because location is outside this test scope, and the launcher disables the emulator's `GnssGrpcV1` feature so it does not start an unnecessary host GNSS socket. Vulkan is disabled explicitly because this runner uses software rendering and the Vulkan/SwiftShader path has crashed during emulator startup. A missing SDK or platform tools fails before the test starts with a diagnostic message. Software emulation is slower than KVM and is given a longer job/boot timeout. The launcher does not run optional input or animation setup commands before the Android system services are ready, and each ADB operation is bounded by a timeout to prevent an offline daemon from hanging the job; if boot or readiness fails, it prints the emulator log and Android logcat for diagnosis. +The workflow first bootstraps the Android SDK tools, then builds debug APKs for every supported Flutter ABI, starts an API 30 `aosp_atd` `pixel_2` emulator through `scripts/run_android_emulator_ci.sh`, waits for the Android `package`, `input`, and `settings` services to remain stable across repeated checks, and runs the compression, property, and native instrumentation suites. The Android Test Device image is intentionally used for headless testing because these package tests do not require Google Play services. GPS is disabled in the generated AVD because location is outside this test scope, and the launcher disables the emulator's `GnssGrpcV1` feature so it does not start an unnecessary host GNSS socket. Vulkan is disabled explicitly because this runner uses software rendering and the Vulkan/SwiftShader path has crashed during emulator startup. A missing SDK or platform tools fails before the test starts with a diagnostic message. Software emulation is slower than KVM and is given a longer job/boot timeout. The launcher does not run optional input or animation setup commands before the Android system services are ready, and each ADB operation is bounded by a timeout to prevent an offline daemon from hanging the job; if boot or readiness fails, it prints the emulator log and Android logcat for diagnosis. -The two Android instrumentation variants run sequentially. They share the runner's ADB daemon, and concurrent software emulators can leave one device offline even after it has booted successfully. The pub-cache variant still runs when the repository-source variant fails, so both dependency resolution paths remain observable. +The launcher also initializes and exports `ANDROID_USER_HOME` and +`ANDROID_AVD_HOME` explicitly. This is required on clean hosted images where +`avdmanager` does not create the default `$HOME/.android/avd` hierarchy before +returning. + +The Android matrix runs sequentially because all entries share the runner's ADB daemon and emulator port. It covers both example projects and both native-source layouts: the normal workspace path and an exact external path recorded in Dart's package configuration. ## Linux job requirements -The workflow builds `zstandard_linux/example` with `flutter build linux --debug`. It then runs the single desktop integration-test file under Xvfb. The same sequence is repeated after removing the repository copy of `zstandard_native`, which verifies resolution from the published package cache. +The workflow builds `zstandard_linux/example` with `flutter build linux --debug`, runs native CTest coverage, executes package tests with the built plugin on `LD_LIBRARY_PATH`, and executes the desktop integration suite under Xvfb. The sequence is repeated with the current `zstandard_native` package staged outside the checkout and selected through `package_config.json`, proving that CMake does not rely on the monorepo-relative fallback. + +## Apple job requirements + +iOS and macOS jobs target `[self-hosted, macOS, ARM64]` and use the pinned, +preinstalled Flutter SDK. Each runner process must execute one job at a time. +Do not add a shared job-level Actions concurrency group to the iOS and macOS +matrices: GitHub retains only one pending job per group and cancels the other +matrix combinations instead of queuing them. Matrix `max-parallel: 1` provides +per-workflow serialization, while the runner process serializes work across +the two workflows. ## Windows job requirements Windows jobs target `[self-hosted, Windows, X64]`. The workflow installs and caches the pinned Flutter SDK on each runner, enables Windows desktop support, verifies Git and CMake, and checks that the runner can create symbolic links. The runner must provide a Visual Studio C++ desktop toolchain and Windows Developer Mode (or equivalent `SeCreateSymbolicLinkPrivilege` for the runner service account). Integration tests launch a real Windows Flutter application, so the runner must have an interactive desktop session; a headless service session is not sufficient for this job. -The workflow builds `zstandard_windows/example` in Debug mode, runs the generated CTest target (including a native zstd compression/decompression round trip), runs the Windows package tests with the built DLL on `PATH`, and launches the example for Flutter integration tests. The same sequence is repeated after removing the repository copy of `zstandard_native`, proving that the native C source can be resolved from the Pub cache. A separate federated-example job builds `zstandard/example` and runs its integration suite. +The workflow builds `zstandard_windows/example` in Debug mode, runs the generated native test target (including a zstd compression/decompression round trip), runs the Windows package tests with the built DLL on `PATH`, and launches the example for Flutter integration tests. The same sequence is repeated with the current `zstandard_native` package staged outside the checkout and selected through `package_config.json`. A separate federated-example job builds `zstandard/example` and runs its integration suite. ## Web job requirements @@ -60,5 +76,5 @@ The workflow fails if ChromeDriver cannot be installed, started, or used. There - Analyze jobs prove dependency resolution and static analysis with the repository Flutter version. - Build steps prove that the platform project and native/WASM integration compile. - Android, Linux, and Web integration tests execute the real platform implementation and its compression/decompression round trips. -- Pub-cache jobs prove that published `zstandard_native` artifacts can be consumed instead of an in-repository path. +- Package-config rows prove that native build systems consume the exact resolved package path instead of accidentally relying on a sibling checkout directory. - Coverage and publish dry runs remain package-level quality gates. diff --git a/docs/development/testing.md b/docs/development/testing.md index b4ac25da..b36a192b 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -151,7 +151,10 @@ Linux and Windows keep unit tests in `test/` that run only when the host OS matc ### CLI -- The existing tests in `zstandard_cli/test/` are a good reference: small/large/empty data, repeated values, min/max compression level. Add tests for invalid compression levels and platform detection if desired. +- The tests in `zstandard_cli/test/` cover small, large, empty, invalid and + concatenated inputs; output limits; compression-level boundaries; platform + and ABI-aware library selection; filesystem collisions; stdin/stdout; exit + codes; and concurrent calls. ## Coverage @@ -164,6 +167,13 @@ flutter test --coverage View the generated `coverage/lcov.info` with a tool like `lcov` or your IDE. Aim for high coverage on the main plugin and platform interface; platform-specific code may have lower coverage when run on a single host. +Pull requests collect three independent Codecov flags on every head: +`cli` (required 100%), `platform_interface` (minimum 85%), and `zstandard` +(minimum 76%). CLI reports are restricted to `zstandard_cli/lib`, so test files +and dependency sources cannot inflate or dilute its result. The stable Safety +Gate requires all three local thresholds even when a fork cannot access the +Codecov upload token. + ## Mutation testing Mutation testing measures test quality by mutating source code and checking whether tests detect the changes. A mutation score of 90% or above is required. diff --git a/docs/guides/advanced-usage.md b/docs/guides/advanced-usage.md index 5df48234..436f87bf 100644 --- a/docs/guides/advanced-usage.md +++ b/docs/guides/advanced-usage.md @@ -46,7 +46,11 @@ If you stored chunks as [length, bytes, length, bytes, ...], read back the same 4. Repeat until the stream ends. ```dart -Future decompressFileChunked(String path, String outPath) async { +Future decompressFileChunked( + String path, + String outPath, + int maxDecodedChunkSize, +) async { final file = File(path); final out = File(outPath); final z = Zstandard(); @@ -60,7 +64,10 @@ Future decompressFileChunked(String path, String outPath) async { if (offset + length > bytes.length) break; final chunk = Uint8List.sublistView(bytes, offset, offset + length); offset += length; - final decompressed = await z.decompress(chunk); + final decompressed = await z.decompress( + chunk, + maxOutputSize: maxDecodedChunkSize, + ); if (decompressed == null) throw Exception('Decompression failed'); sink.add(decompressed); } @@ -106,10 +113,10 @@ final results = await Future.wait(futures); ## Web platform -On web, compression and decompression run on the main thread (no isolates). For large data: +On web, compression and decompression run in the packaged Web Worker. For large data: -- Prefer smaller chunks to keep the UI responsive. -- Consider moving work to a Web Worker and calling the same API from there if you run Dart in the worker. +- Prefer smaller chunks to reduce per-request Worker/WASM memory. +- Limit concurrent calls so several large WASM operations do not grow memory at once. - Lower compression levels reduce CPU time and improve responsiveness. ## CLI and batch processing diff --git a/docs/guides/best-practices.md b/docs/guides/best-practices.md index 1ec13556..a027f5e4 100644 --- a/docs/guides/best-practices.md +++ b/docs/guides/best-practices.md @@ -13,13 +13,13 @@ This guide summarizes recommended practices, a production checklist, and common 7. **Choose the right level**: level 3 for general use; 1 for speed; 10+ for size when CPU and time allow. 8. **Validate decompressed content** when data comes from untrusted sources; the API only guarantees valid zstd output, not safe application-level content. 9. **Run tests and analyze** before release: `flutter test`, `flutter analyze` (or `dart test` / `dart analyze` for the CLI package). -10. **Pin package versions** in `pubspec.yaml` (e.g. `zstandard: ^1.3.0`) and update in a controlled way. +10. **Use a deliberate dependency constraint** in `pubspec.yaml` (for example, `zstandard: ^1.5.0`) and test upgrades. ## Don'ts 1. **Don't ignore null results** — using a null result as if it were data can lead to crashes or wrong behaviour. 2. **Don't use compression levels outside 1–22** — behaviour is implementation-defined and may differ by platform. -3. **Don't decompress untrusted data without a size limit** — cap input size to what you are willing to allocate. +3. **Don't decompress untrusted data without an output limit** — pass the smallest safe `maxOutputSize` and also cap input size. 4. **Don't assume decompress throws** on invalid input — it typically returns null; handle null. 5. **Don't load entire very large files into memory** if you can avoid it; use chunked reading and compression. 6. **Don't run many concurrent compress/decompress operations** without limiting concurrency; memory usage can grow quickly. @@ -61,9 +61,10 @@ await send(c); // Bad: no size limit on untrusted input final d = await z.decompress(userBytes); -// Good: reject or cap size before calling -if (userBytes.length > maxDecompressSize) return reject(); -final d = await z.decompress(userBytes); +// Good: cap both compressed input and produced output +if (userBytes.length > maxCompressedSize) return reject(); +final d = await z.decompress(userBytes, maxOutputSize: maxDecodedSize); +if (d == null) return reject(); ``` **Assuming exceptions** diff --git a/docs/guides/performance-tips.md b/docs/guides/performance-tips.md index 24ec44ca..b496137b 100644 --- a/docs/guides/performance-tips.md +++ b/docs/guides/performance-tips.md @@ -10,13 +10,13 @@ Suggestions to get the best performance and resource usage when using the Zstand ## Data size - **Small data** (e.g. < 100 bytes): Compression may not reduce size (zstd has frame overhead). Consider skipping compression for very small payloads. -- **Large data**: The plugin may use a background isolate on native platforms to avoid blocking the UI. For very large inputs (e.g. tens of MB), consider **chunking**: compress chunks and store/transmit separately, or use streaming if the API supports it in the future. +- **Large data**: Native packages use worker isolates and Web uses a dedicated Worker. For very large inputs, consider application-level chunking to bound peak memory. - **Empty data**: Handled quickly; no need to avoid. ## Memory - Compress and decompress allocate buffers (input + output). For very large inputs, peak memory is roughly proportional to input size plus compressed/decompressed size. Chunking reduces peak usage. -- On native platforms, work may run in an isolate; the main isolate only holds the input and result bytes, which helps keep UI responsive. +- On native platforms, work runs in an isolate; the main isolate holds the input and result bytes while native allocations stay in the worker. ## Reuse @@ -25,8 +25,8 @@ Suggestions to get the best performance and resource usage when using the Zstand ## Platform-specific -- **Web**: No isolates; compression/decompression run on the main thread. For large data on web, consider chunking or moving work to a Web Worker if you implement it. -- **Native (Android, iOS, macOS, Linux, Windows)**: The implementation may offload work to an isolate; you get non-blocking behavior without extra code. +- **Web**: Compression/decompression run in the packaged Web Worker. The request broker remains on the UI thread. +- **Native (Android, iOS, macOS, Linux, Windows)**: Public codec work runs in an isolate without extra application code. ## Measuring diff --git a/docs/guides/security.md b/docs/guides/security.md index 8d8df989..80c47a66 100644 --- a/docs/guides/security.md +++ b/docs/guides/security.md @@ -1,68 +1,63 @@ # Security -This guide covers security considerations when using the Zstandard plugin and CLI: input validation, handling untrusted data, memory safety, and how to report vulnerabilities. +## Treat output size as the primary decompression budget -## Input validation - -### Compression - -- **Input data**: The plugin accepts `Uint8List` for compression. No inherent size limit is enforced by the API; very large inputs may cause high memory usage or platform-specific limits. Validate or cap input size in your application when processing user-controlled data. -- **Compression level**: Valid levels are **1–22**. Native platform implementations reject levels outside this range with `null`; validate the level (e.g. clamp to 1–22) before calling `compress`. +Compressed input can expand dramatically. Every official implementation +therefore accepts `maxOutputSize` and defaults it to 256 MiB: ```dart -int safeLevel(int level) { - if (level < 1) return 1; - if (level > 22) return 22; - return level; -} -final compressed = await zstandard.compress(data, safeLevel(userLevel)); +final decoded = await Zstandard().decompress( + untrustedFrame, + maxOutputSize: 8 * 1024 * 1024, +); +if (decoded == null) return rejectInput(); ``` -### Decompression - -- **Untrusted compressed data**: Data that is not a valid Zstandard frame (random bytes, truncated data, or crafted payloads) will typically result in a **null** return from `decompress`, not an exception. The native zstd library is designed to fail safely on invalid input. -- **Bomb resistance**: Zstandard frames contain size information; native one-shot decompression allocates the advertised output size and rejects frames without a usable size instead of guessing a multiplier. Very large stored sizes in a malicious frame could still lead to large allocations. Prefer validating or limiting input size when decompressing data from untrusted sources. - -## Handling untrusted data - -When decompressing data from untrusted sources (network, user uploads, third-party files): - -1. **Check for null**: Always treat a null result as failure and do not use the result. - -```dart -final decompressed = await zstandard.decompress(receivedBytes); -if (decompressed == null) { - // Invalid or malicious input; reject - return; -} -// Only use decompressed after null check -``` - -2. **Limit input size**: Reject or refuse to decompress payloads above a size you are willing to allocate (e.g. cap at 10 MB or 100 MB depending on your use case). - -3. **Validate after decompression**: If the decompressed data has a known format (JSON, protocol buffer, etc.), validate it before use. The plugin only guarantees that the bytes are a valid zstd decompression result, not that the content is safe for your application. - -4. **Avoid trusting compressed size blindly**: If you expose decompressed size or progress to users, ensure it comes from the library’s result (e.g. length of the returned `Uint8List`) rather than from unvalidated metadata. - -## Memory safety - -- **Native code**: The plugin uses the official Zstandard C library via FFI (and WebAssembly on web). The library is widely used and maintained; buffer overflows and similar issues in zstd are addressed by upstream. -- **Dart/Flutter**: The Dart API uses `Uint8List`; no raw pointers are exposed. Memory is managed by the Dart VM and the native allocator used by zstd. -- **Large allocations**: Compression and decompression allocate memory proportional to input and output. To avoid out-of-memory conditions, limit input size and consider processing large files in chunks if supported by your workflow (see [Advanced usage](advanced-usage.md)). +Native and Web implementations enforce the budget chunk-by-chunk with zstd's +streaming decoder, including for concatenated frames and frames whose content +size is unknown. Malformed, truncated, empty, or oversized input returns +`null`. + +Set the lowest bound allowed by the application protocol. Also cap compressed +input size and concurrent operations: an output bound does not limit the bytes +already held as input, zstd's internal workspace, or the aggregate memory of +several simultaneous calls. + +Third-party `ZstandardPlatform` implementations should implement +`BoundedZstandardPlatform`. The main package post-checks results from legacy +implementations for compatibility, but it cannot undo an oversized allocation +that a third-party decoder already made. + +## Validate all external values + +- Accept only compression levels 1–22. Official implementations return `null` + outside this range. +- Treat `null` as operation failure; do not continue with fallback bytes that + could be confused with authenticated content. +- Validate the decompressed application format, schema, lengths, and integrity. + A valid zstd frame does not make JSON, images, archives, or protocol messages + trustworthy. +- Authenticate data separately when integrity or origin matters. Compression + is not encryption or authentication. + +## Memory and execution safety + +Public native APIs exchange Dart-owned `Uint8List` values with worker isolates; +native pointers are allocated, used, copied, and freed within the same isolate. +Deprecated pointer-level helpers are compatibility APIs and should not be used +for new code. Web transfers copied buffers to a dedicated Worker and frees WASM +allocations after copying each result. + +Compression itself has no input-size limit. Large compression input and high +levels can consume substantial CPU and memory, so apply request, file, and +concurrency limits before invoking the codec. ## Vulnerability reporting -If you discover a security vulnerability in this plugin, its dependencies, or the way it uses the Zstandard library: - -1. **Do not** open a public GitHub issue for security-sensitive findings. -2. Report privately to the maintainers (e.g. via the repository’s contact or security policy, if stated). -3. Include a clear description, steps to reproduce, and impact if possible. -4. Allow a reasonable time for a fix before any public disclosure. - -For issues in the **upstream Zstandard library** (Facebook/Meta), follow the [Zstandard project’s security policy](https://github.com/facebook/zstd/security). - -## See also +Do not disclose a suspected vulnerability in a public issue. Follow the +repository `SECURITY.md` and include a reproducer, affected versions, and impact +when possible. Upstream zstd vulnerabilities should also follow the +[upstream security policy](https://github.com/facebook/zstd/security). -- [Error handling](error-handling.md) — null semantics and failure handling -- [Advanced usage](advanced-usage.md) — large data and memory considerations -- [Best practices](best-practices.md) — production checklist +See [Error handling](error-handling.md) and +[Best practices](best-practices.md). diff --git a/docs/platforms/android.md b/docs/platforms/android.md index bdf3ffdd..d3e98c43 100644 --- a/docs/platforms/android.md +++ b/docs/platforms/android.md @@ -16,7 +16,7 @@ Add the main plugin to your app; the Android implementation is included via the ```yaml dependencies: - zstandard: ^1.3.29 + zstandard: ^1.5.0 ``` No additional Gradle or native setup is required for normal use. The plugin registers the Android implementation automatically when running on Android. @@ -33,8 +33,8 @@ AGP 8.11.1 compatibility example. ## Architecture - **Native layer**: The facebook/zstd C library is built as part of the Android project (e.g. via CMake or Android NDK) and exposed as a shared library (e.g. `libzstandard_android_plugin.so`). -- **Dart layer**: The package uses Dart FFI to open the library and generated bindings to call `ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, and `ZSTD_getFrameContentSize`. -- **Isolates**: The implementation may use a helper isolate for async compression/decompression to avoid blocking the UI thread. +- **Dart layer**: The package uses Dart FFI and the shared native codec for one-frame compression and bounded streaming decompression. +- **Isolates**: Public compression/decompression runs in a worker isolate with Dart-owned byte buffers. ## Usage diff --git a/docs/platforms/cli.md b/docs/platforms/cli.md index 5c136a3d..b54cff6d 100644 --- a/docs/platforms/cli.md +++ b/docs/platforms/cli.md @@ -1,108 +1,59 @@ # CLI Platform Guide -The **zstandard_cli** package provides Zstandard compression and decompression for **pure Dart** applications (no Flutter) on **macOS, Windows, and Linux**. It uses FFI with precompiled native zstd libraries and supports both in-code API and command-line entry points. +`zstandard_cli` is a pure-Dart API and command-line package for macOS, Windows, +and Linux. Bundled native libraries support x64 and arm64. -## Support - -| Platform | x64 | arm64 | Precompiled | -|----------|-----|-------|-------------| -| macOS | Yes | Yes | Yes | -| Windows | Yes | Yes | Yes | -| Linux | Yes | Yes | Yes | - -## Installation - -Add the package to your Dart project (not Flutter): +## Installation and API ```yaml dependencies: - zstandard_cli: ^1.3.29 -``` - -## Usage in Code - -```dart -import 'package:zstandard_cli/zstandard_cli.dart'; - -void main() async { - final cli = ZstandardCLI(); - final data = Uint8List.fromList([1, 2, 3, 4, 5]); - - final compressed = await cli.compress(data, compressionLevel: 3); - final decompressed = await cli.decompress(compressed ?? Uint8List(0)); -} + zstandard_cli: ^1.5.0 ``` -With extensions: - ```dart -final compressed = await data.compress(compressionLevel: 3); -final decompressed = await compressed?.decompress(); -``` - -## Command-Line Usage - -Compress a file with a given compression level: - -```bash -dart run zstandard_cli:compress +final codec = ZstandardCLI(); +final compressed = await codec.compress(data, compressionLevel: 3); +final decompressed = compressed == null + ? null + : await codec.decompress( + compressed, + maxOutputSize: 16 * 1024 * 1024, + ); ``` -Example: `dart run zstandard_cli:compress myfile.txt 3` +Empty input produces a valid frame. Streaming decompression supports frames +without a declared size and concatenated frames, with a 256 MiB default output +budget. -Decompress a file: +## Commands ```bash -dart run zstandard_cli:decompress +dart run zstandard_cli:compress --level 5 input.bin +dart run zstandard_cli:decompress --max-output-size 16777216 input.bin.zstd ``` -Example: `dart run zstandard_cli:decompress myfile.txt.zstd` - -Output file names and default paths are defined by the package (e.g. compressed files may get a `.zstd` suffix). See the package README for exact behavior. +Use `--output PATH` to select output and `--force` to replace an existing +file. Input `-` reads stdin and defaults to stdout, enabling binary pipelines. +Run either command with `--help` for its exact options and naming rules. -## Architecture +The commands return 0 on success, 1 on I/O/codec failure, and 2 on invalid +usage or unsafe overwrite. An output that aliases the input is always refused. -- **Precompiled libraries**: The package ships with native zstd libraries per platform/architecture (e.g. in `lib/src/bin/` or similar). At runtime, the correct library is loaded based on the current platform and CPU architecture. -- **FFI**: Dart opens the library with `DynamicLibrary` and uses generated bindings to call `ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, and `ZSTD_getFrameContentSize`. -- **No Flutter**: No dependency on Flutter; suitable for server or CLI Dart apps. +## Native loading and deployment -## API Summary +The loader selects a packaged library using the package URI, not the current +working directory. It also supports compiled-executable adjacency and an +explicit `ZSTANDARD_CLI_LIBRARY` override. Windows CLI binaries link the MSVC +runtime statically to avoid an otherwise undeclared redistributable dependency. -- **ZstandardCLI()** — Create an instance. -- **compress(Uint8List data, {int compressionLevel = 3})** — Compress; returns `Future`. -- **decompress(Uint8List data)** — Decompress; returns `Future`. -- **getPlatformVersion()** — Returns a string like `"macOS 14.0"` or `"Linux ..."`. -- **Extensions** on `Uint8List?`: `compress({int compressionLevel = 3})`, `decompress()`. - -See [CLI API Reference](../api/cli-api.md) for full details. +Public API operations pass bytes to the shared native codec and never expose +FFI allocation ownership. Callers doing concurrent large operations should +still cap their own concurrency and output budgets. ## Testing -From the package directory: - -```bash -dart test -``` - -The package has a solid set of unit tests (small/large/empty data, compression levels, roundtrip). Run them on the target platform to ensure the native library loads and behaves correctly. - -## Performance characteristics - -- **No isolates**: Runs in the current isolate; suitable for CLI or server where blocking is acceptable. -- **Throughput**: Comparable to native zstd; level 1–3 fastest, level 22 slowest. -- **Memory**: Proportional to input and output; precompiled libs are built with standard zstd options. - -## Known limitations - -- **Desktop only**: macOS, Windows, Linux. For mobile or web, use the main **zstandard** Flutter plugin. -- **Precompiled binaries**: You depend on the package’s shipped libraries; for custom builds or other platforms you would need to build and load your own library (see [Building](development/building.md) and `scripts/build_*.sh`). - -## Troubleshooting - -- **Library not found**: Ensure you are on a supported platform and architecture. Check that the package’s native library for that platform/arch is present and that `openZstdLibrary()` (or equivalent) can find it. -- **Compress/decompress returns null**: Check that input is valid (e.g. non-empty for cases where the implementation requires it, valid zstd frame for decompress). See [Common Issues](../troubleshooting/common-issues.md). - -## See Also +Run `dart test` and `dart analyze` on each target OS. The test suite verifies +the bundled symbol surface and the native roundtrip in addition to argument, +file-safety, and loading behavior. -- [API — CLI](../api/cli-api.md) -- [Architecture — FFI Implementation](../architecture/ffi-implementation.md) +See [CLI API](../api/cli-api.md) and [FFI architecture](../architecture/ffi-implementation.md). diff --git a/docs/platforms/ios.md b/docs/platforms/ios.md index 343ea2a7..3395a519 100644 --- a/docs/platforms/ios.md +++ b/docs/platforms/ios.md @@ -16,7 +16,7 @@ Add the main plugin to your app; the iOS implementation is included via the fede ```yaml dependencies: - zstandard: ^1.3.29 + zstandard: ^1.5.0 ``` No additional setup is required for normal use. The plugin registers the iOS implementation automatically when running on iOS. @@ -24,8 +24,8 @@ No additional setup is required for normal use. The plugin registers the iOS imp ## Architecture - **Native layer**: The canonical facebook/zstd C library lives in `zstandard_native/src/zstd/`. Swift Package Manager statically links it through the repository-level `Package.swift`; CocoaPods synchronizes it into the generated `ios/Classes/zstd/` directory and embeds the plugin framework. Dart FFI resolves the symbols from the application process in SwiftPM builds or from the framework in CocoaPods builds. -- **Dart layer**: The package uses Dart FFI and generated bindings to call `ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, and `ZSTD_getFrameContentSize`. -- **Isolates**: The implementation may use a helper isolate for async compression/decompression to avoid blocking the UI thread. +- **Dart layer**: The package uses Dart FFI and the shared native codec for one-frame compression and bounded streaming decompression. +- **Isolates**: Public compression/decompression runs in a worker isolate with Dart-owned byte buffers. ## Usage diff --git a/docs/platforms/linux.md b/docs/platforms/linux.md index f77b3f8a..760d5d4a 100644 --- a/docs/platforms/linux.md +++ b/docs/platforms/linux.md @@ -15,7 +15,7 @@ Add the main plugin to your app; the Linux implementation is included via the fe ```yaml dependencies: - zstandard: ^1.3.29 + zstandard: ^1.5.0 ``` No additional setup is required for normal use. The plugin registers the Linux implementation automatically when running on Linux. @@ -23,8 +23,8 @@ No additional setup is required for normal use. The plugin registers the Linux i ## Architecture - **Native layer**: The facebook/zstd C library is built with CMake (e.g. under `linux/` or `src/`) and produces a shared library `libzstandard_linux_plugin.so` that the Dart plugin loads via FFI. -- **Dart layer**: The package uses Dart FFI and generated bindings to call `ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, and `ZSTD_getFrameContentSize`. -- **Isolates**: The implementation may use a helper isolate for async compression/decompression. +- **Dart layer**: The package uses Dart FFI and the shared native codec for one-frame compression and bounded streaming decompression. +- **Isolates**: Public compression/decompression runs in a worker isolate with Dart-owned byte buffers. ## Usage diff --git a/docs/platforms/macos.md b/docs/platforms/macos.md index be64408d..18ffbf89 100644 --- a/docs/platforms/macos.md +++ b/docs/platforms/macos.md @@ -15,7 +15,7 @@ Add the main plugin to your app; the macOS implementation is included via the fe ```yaml dependencies: - zstandard: ^1.3.29 + zstandard: ^1.5.0 ``` No additional setup is required for normal use. The plugin registers the macOS implementation automatically when running on macOS. @@ -23,8 +23,8 @@ No additional setup is required for normal use. The plugin registers the macOS i ## Architecture - **Native layer**: The canonical facebook/zstd C library lives in `zstandard_native/src/zstd/`. Swift Package Manager statically links it through the repository-level `Package.swift`; CocoaPods synchronizes it into the generated `macos/Classes/zstd/` directory and embeds the plugin framework. Dart FFI resolves the symbols from the application process in SwiftPM builds or from the framework in CocoaPods builds. -- **Dart layer**: The package uses Dart FFI and generated bindings to call `ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, and `ZSTD_getFrameContentSize`. -- **Isolates**: The implementation may use a helper isolate for async compression/decompression. +- **Dart layer**: The package uses Dart FFI and the shared native codec for one-frame compression and bounded streaming decompression. +- **Isolates**: Public compression/decompression runs in a worker isolate with Dart-owned byte buffers. ## Usage diff --git a/docs/platforms/web.md b/docs/platforms/web.md index b1ab27f9..198a8c5b 100644 --- a/docs/platforms/web.md +++ b/docs/platforms/web.md @@ -1,111 +1,79 @@ # Web Platform Guide -The **zstandard_web** package provides the web implementation of the Zstandard Flutter plugin using JavaScript and WebAssembly (zstd compiled with Emscripten). It does not use Dart FFI. - -## Support - -| Environment | Support | -|-------------|---------| -| Browser (Chrome, Firefox, Safari, Edge) | Yes | -| WebAssembly | Yes (zstd.wasm) | +The federated web implementation runs the canonical zstd C source as +WebAssembly in a dedicated browser Worker. ## Installation -1. Add the main plugin to your app: +Add the main package and copy all four generated assets from +`zstandard_web/blob/` to the application's `web/` directory: ```yaml dependencies: zstandard: ^1.5.0 ``` -2. **Copy web assets** into your Flutter web project: - - **zstd.js** — Emscripten-generated JS that loads and wraps the WASM module. - - **zstd.wasm** — Compiled Zstandard C library. - - These files are provided by the zstandard_web package (e.g. under `blob/` or as documented in the package README). Copy them into your app’s `web/` directory (e.g. `web/zstd.js`, `web/zstd.wasm`). +```text +web/zstd.js +web/zstd_worker.js +web/zstd_core.js +web/zstd.wasm +``` -3. **Include the script** in your `web/index.html`: +Load the request broker before Flutter: ```html - - - - ... - - ``` -The script must load before your Flutter app so that `compressData` and `decompressData` are available when the Dart code runs. - -## Architecture +The files must share a directory because the broker locates +`zstd_worker.js`, and the Worker in turn loads `zstd_core.js` and `zstd.wasm` +with relative URLs. -- **zstd.js** loads **zstd.wasm** and exposes global functions `compressData(inputData, compressionLevel)` and `decompressData(compressedData)`. -- **Dart** uses `dart:js_interop` (and `package:web`) to call these functions and convert between `Uint8List` and JS typed arrays. -- There is no FFI and no background isolate; compression/decompression run on the main thread. For large data, consider chunking or moving work to a Web Worker if the implementation supports it. - -## Usage - -Use the main package API; the web implementation is used automatically when running on web: - -```dart -import 'package:zstandard/zstandard.dart'; - -final zstandard = Zstandard(); -final compressed = await zstandard.compress(data, 3); -final decompressed = await zstandard.decompress(compressed!); -``` - -Or use the extensions: +## Usage and failure contract ```dart final compressed = await data.compress(compressionLevel: 3); -final decompressed = await compressed?.decompress(); +final decompressed = await compressed.decompress( + maxOutputSize: 16 * 1024 * 1024, +); ``` -## Building zstd.js and zstd.wasm - -If you need to rebuild the WebAssembly artifacts: - -1. Install and activate the [Emscripten SDK](https://emscripten.org/). -2. Clone the [facebook/zstd](https://github.com/facebook/zstd) repository. -3. Run `emcc` on the zstd C sources with: - - WASM output - - Exported functions: `ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, `ZSTD_getFrameContentSize`, `malloc`, `free` -4. Add the wrapper functions `compressData` and `decompressData` in `zstd.js` (or a separate script) that allocate buffers, call the C functions, and return the result or null. - -Detailed commands and wrapper code are in the [zstandard_web README](https://github.com/vypdev/zstandard/tree/master/zstandard_web). +Empty input compresses to a valid frame; short values are never returned +unchanged. Unknown-size and concatenated frames are supported. Invalid, +truncated, oversized, or otherwise failed operations return `null`, +consistently with native platforms. -## Small Data Behavior +## Architecture and performance -For very small inputs (e.g. less than 9 bytes), the implementation may return the data unchanged for compress or decompress, as zstd has a minimum frame size. Check the package source for exact behavior. +`zstd.js` transfers a copy of each input buffer to a lazily created Web Worker, +so zstd CPU work does not block browser rendering. The Worker owns the +Emscripten module. Decompression uses zstd's streaming API and rejects output +as soon as it would exceed `maxOutputSize`. The default output budget is +256 MiB; choose a smaller protocol-specific value when possible. -## Testing +The WASM heap can grow to accommodate valid operations, so input and output +still determine peak memory. Limit concurrency for large payloads. -- **Unit tests**: From the package directory: `flutter test` (some tests may require a browser or mock the JS API). -- **Integration tests**: The example app has web integration tests (e.g. `example/integration_test/zstandard_web_integration_test.dart`) that run in the browser. +## Generation and testing -## Performance characteristics - -- **Single-threaded**: Compression and decompression run on the main thread (no isolates on web). Large payloads can block the UI. -- **Throughput**: Generally slower than native; level 1–3 are faster than high levels. -- **Memory**: WASM heap usage scales with input and output; consider smaller chunks for large data. - -## Known limitations +```bash +./scripts/build_web_wasm.sh +./scripts/test_web_integration.sh +``` -- Requires `zstd.js` and `zstd.wasm` to be deployed with your app and loaded before use. -- No isolate-based offloading; heavy work runs on the main thread. For large data, consider chunking or a Web Worker (see [Advanced usage](../guides/advanced-usage.md)). -- Behavior may differ slightly from native (e.g. small-data handling, error codes). Decompress failure may throw instead of returning null. +The build is pinned and synchronizes every committed copy. Integration tests +build the example in release mode and exercise the real Worker/WASM boundary +through ChromeDriver, including event-loop yielding and output limits. ## Troubleshooting -- **compressData / decompressData is not defined**: Ensure `zstd.js` is included in `index.html` and loads before the Flutter app. Check the browser console for script errors. -- **WASM load failed**: Ensure `zstd.wasm` is served from the same origin or with correct CORS and that the path in `zstd.js` is correct. -- **Null from compress/decompress**: Check that input is valid and that the JS functions return a typed array or null; see [Common Issues](../troubleshooting/common-issues.md). - -## See Also +- Undefined `compressData`: ensure `zstd.js` loads before Flutter. +- Worker/WASM network failure: deploy all four assets together with compatible + same-origin/CORS and Content-Security-Policy rules. +- `null` decompression: validate the frame and ensure the configured output + budget is sufficient and safe for the application. -- [Architecture — Web Implementation](../architecture/web-implementation.md) -- [API — Main](../api/main-api.md) +See [Web architecture](../architecture/web-implementation.md). diff --git a/docs/platforms/windows.md b/docs/platforms/windows.md index f5f02d66..87562a1a 100644 --- a/docs/platforms/windows.md +++ b/docs/platforms/windows.md @@ -15,7 +15,7 @@ Add the main plugin to your app; the Windows implementation is included via the ```yaml dependencies: - zstandard: ^1.3.29 + zstandard: ^1.5.0 ``` No additional setup is required for normal use. The plugin registers the Windows implementation automatically when running on Windows. @@ -23,8 +23,8 @@ No additional setup is required for normal use. The plugin registers the Windows ## Architecture - **Native layer**: The facebook/zstd C library is built with CMake under the package’s `windows/` (or `src/`) and produces a DLL (e.g. `zstandard_windows_plugin.dll`) that the Dart plugin loads via FFI. -- **Dart layer**: The package uses Dart FFI and generated bindings to call `ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, and `ZSTD_getFrameContentSize`. -- **Isolates**: The implementation may use a helper isolate for async compression/decompression. +- **Dart layer**: The package uses Dart FFI and the shared native codec for one-frame compression and bounded streaming decompression. +- **Isolates**: Public compression/decompression runs in a worker isolate with Dart-owned byte buffers. ## Usage diff --git a/docs/troubleshooting/debugging.md b/docs/troubleshooting/debugging.md index 99b4f754..85444b98 100644 --- a/docs/troubleshooting/debugging.md +++ b/docs/troubleshooting/debugging.md @@ -40,7 +40,7 @@ The plugin may log errors or debug info. Check whether the implementation uses ` When opening an issue, include: -- Package and version (e.g. zstandard 1.3.29) +- Package and version (for example, zstandard 1.5.0) - Platform (Android, iOS, macOS, Windows, Linux, web, CLI) - Flutter/Dart version - Minimal code that reproduces the issue diff --git a/docs/troubleshooting/platform-issues.md b/docs/troubleshooting/platform-issues.md index 16fed314..023f6196 100644 --- a/docs/troubleshooting/platform-issues.md +++ b/docs/troubleshooting/platform-issues.md @@ -29,7 +29,7 @@ Issues that are specific to one platform or environment. - **WASM or JS errors**: Ensure `zstd.js` and `zstd.wasm` are served from the same origin (or CORS is set correctly) and that the path in the script matches. Check the browser console and network tab. - **compressData/decompressData undefined**: The script must load before the Flutter app. Put `` in `` and ensure it loads without errors. -- **Slow or blocking**: Web runs on the main thread. For large data, consider chunking or offloading to a Web Worker if you implement it. +- **Slow operations**: Web already runs zstd in a dedicated Worker. For large data, use smaller independently framed chunks and limit concurrency to reduce Worker/WASM memory pressure. ## CLI (macOS, Windows, Linux) diff --git a/media/README.md b/media/README.md deleted file mode 100644 index acae8835..00000000 --- a/media/README.md +++ /dev/null @@ -1,7 +0,0 @@ -Refresh the gif with: - -```bash -brew install ffmpeg - -ffmpeg -i input.mov -filter_complex "[0:v]setpts=PTS/4,fps=1,scale=480:-1,split[s1][s2];[s1]palettegen[p];[s2][p]paletteuse" -t 40 output.gif -``` \ No newline at end of file diff --git a/media/publish.gif b/media/publish.gif deleted file mode 100644 index 4568f319..00000000 Binary files a/media/publish.gif and /dev/null differ diff --git a/media/update_files.gif b/media/update_files.gif deleted file mode 100644 index 674b06cd..00000000 Binary files a/media/update_files.gif and /dev/null differ diff --git a/scripts/build_web_wasm.sh b/scripts/build_web_wasm.sh index b84f2ff9..4f11b517 100755 --- a/scripts/build_web_wasm.sh +++ b/scripts/build_web_wasm.sh @@ -99,15 +99,15 @@ emcc -O3 \ -I. -Icommon -Icompress -Idecompress \ -s WASM=1 \ -s EXPORT_NAME="zstdWasmModule" \ - -s EXPORTED_FUNCTIONS="['_ZSTD_compress','_ZSTD_decompress','_ZSTD_isError','_malloc','_free','_ZSTD_getFrameContentSize','_ZSTD_compressBound']" \ + -s EXPORTED_FUNCTIONS="['_ZSTD_compress','_ZSTD_isError','_malloc','_free','_ZSTD_compressBound','_ZSTD_createDStream','_ZSTD_initDStream','_ZSTD_decompressStream','_ZSTD_freeDStream','_ZSTD_DStreamOutSize']" \ -s EXPORTED_RUNTIME_METHODS="['HEAPU8']" \ -s INITIAL_MEMORY=134217728 \ -s ALLOW_MEMORY_GROWTH=1 \ -s MAXIMUM_MEMORY=2147483648 \ - -o zstd_generated.js + -o zstd_core_generated.js -if [[ ! -f zstd_generated.js || ! -f zstd_generated.wasm ]]; then - echo "Error: emcc did not produce zstd_generated.js / zstd_generated.wasm" +if [[ ! -f zstd_core_generated.js || ! -f zstd_core_generated.wasm ]]; then + echo "Error: emcc did not produce zstd_core_generated.js / zstd_core_generated.wasm" exit 1 fi @@ -124,12 +124,12 @@ fi --strip-producers \ --strip-target-features \ --remove-unused-names \ - zstd_generated.wasm \ + zstd_core_generated.wasm \ -o zstd_normalized.wasm -mv zstd_normalized.wasm zstd_generated.wasm +mv zstd_normalized.wasm zstd_core_generated.wasm # Append the compressData/decompressData wrappers required by the web plugin (see zstandard_web/README.md). -cat >> zstd_generated.js << 'WRAPPER_JS' +cat >> zstd_core_generated.js << 'WRAPPER_JS' // Promise that resolves when the module is ready let moduleReady = new Promise((resolve) => { @@ -148,92 +148,263 @@ let moduleReady = new Promise((resolve) => { async function compressData(inputData, compressionLevel) { await moduleReady; - - let inputPtr = Module._malloc(inputData.length); - Module.HEAPU8.set(inputData, inputPtr); - - let outputBufferSize = Number(Module._ZSTD_compressBound(inputData.length)); - let outputPtr = Module._malloc(outputBufferSize); - - let compressedSize = Number(Module._ZSTD_compress( - outputPtr, - outputBufferSize, - inputPtr, - inputData.length, - compressionLevel - )); - - if (Module._ZSTD_isError(compressedSize) !== 0 || compressedSize <= 0) { - console.error('Compression error, error code: ', compressedSize); - Module._free(inputPtr); - Module._free(outputPtr); + + if (!(inputData instanceof Uint8Array) + || !Number.isInteger(compressionLevel) + || compressionLevel < 1 + || compressionLevel > 22) { return null; - } else { - let compressedData = new Uint8Array(Module.HEAPU8.buffer, outputPtr, compressedSize); - let out = compressedData.slice(0); - Module._free(inputPtr); - Module._free(outputPtr); - return out; + } + + let inputPtr = 0; + let outputPtr = 0; + try { + inputPtr = Module._malloc(Math.max(inputData.length, 1)); + if (!inputPtr) return null; + Module.HEAPU8.set(inputData, inputPtr); + + const outputBufferSize = Number(Module._ZSTD_compressBound(inputData.length)); + if (!Number.isSafeInteger(outputBufferSize) || outputBufferSize <= 0) { + return null; + } + outputPtr = Module._malloc(outputBufferSize); + if (!outputPtr) return null; + + const compressedSize = Number(Module._ZSTD_compress( + outputPtr, + outputBufferSize, + inputPtr, + inputData.length, + compressionLevel + )); + if (Module._ZSTD_isError(compressedSize) !== 0 || compressedSize <= 0) { + console.error('Compression error, error code: ', compressedSize); + return null; + } + return new Uint8Array( + Module.HEAPU8.buffer, + outputPtr, + compressedSize + ).slice(); + } finally { + if (inputPtr) Module._free(inputPtr); + if (outputPtr) Module._free(outputPtr); } } -async function decompressData(compressedData) { +async function decompressData(compressedData, maxOutputSize = 256 * 1024 * 1024) { await moduleReady; - - let compressedPtr = Module._malloc(compressedData.length); - Module.HEAPU8.set(compressedData, compressedPtr); - - // ZSTD_getFrameContentSize returns these unsigned 64-bit sentinel values. - // JavaScript rounds them to the same Number values returned by Emscripten. - const ZSTD_CONTENTSIZE_UNKNOWN = 0xffffffffffffffff; - const ZSTD_CONTENTSIZE_ERROR = 0xfffffffffffffffe; - let decompressedSize = Number(Module._ZSTD_getFrameContentSize(compressedPtr, compressedData.length)); - if (decompressedSize === ZSTD_CONTENTSIZE_ERROR) { - console.error('Error in obtaining the original size of the data'); - Module._free(compressedPtr); + + if (!(compressedData instanceof Uint8Array) + || !Number.isSafeInteger(maxOutputSize) + || maxOutputSize < 0 + || compressedData.length === 0) { return null; } - const outputBufferSize = decompressedSize === ZSTD_CONTENTSIZE_UNKNOWN - ? compressedData.length * 20 - : decompressedSize; - let decompressedPtr = Module._malloc(outputBufferSize); - - let resultSize = Number(Module._ZSTD_decompress( - decompressedPtr, - outputBufferSize, - compressedPtr, - compressedData.length - )); - - if (Module._ZSTD_isError(resultSize) !== 0 || resultSize < 0) { - console.error('Decompression error, error code: ', resultSize); - Module._free(compressedPtr); - Module._free(decompressedPtr); - return null; - } else { - let decompressedData = new Uint8Array(Module.HEAPU8.buffer, decompressedPtr, resultSize); - let out = decompressedData.slice(0); - Module._free(compressedPtr); - Module._free(decompressedPtr); - return out; + // ZSTD_inBuffer and ZSTD_outBuffer contain three wasm32 size_t/pointer + // fields each. Keep their allocation and field access local to the Worker. + const bufferStructSize = 3 * Uint32Array.BYTES_PER_ELEMENT; + let compressedPtr = 0; + let inputBufferPtr = 0; + let outputBufferPtr = 0; + let outputChunkPtr = 0; + let stream = 0; + try { + compressedPtr = Module._malloc(compressedData.length); + inputBufferPtr = Module._malloc(bufferStructSize); + outputBufferPtr = Module._malloc(bufferStructSize); + stream = Module._ZSTD_createDStream(); + if (!compressedPtr || !inputBufferPtr || !outputBufferPtr || !stream) { + return null; + } + Module.HEAPU8.set(compressedData, compressedPtr); + + const initialization = Number(Module._ZSTD_initDStream(stream)); + if (Module._ZSTD_isError(initialization) !== 0) return null; + + const recommendedChunkSize = Number(Module._ZSTD_DStreamOutSize()); + if (!Number.isSafeInteger(recommendedChunkSize) + || recommendedChunkSize <= 0) { + return null; + } + const outputChunkSize = Math.max( + 1, + Math.min(recommendedChunkSize, Math.max(maxOutputSize, 1)), + ); + outputChunkPtr = Module._malloc(outputChunkSize); + if (!outputChunkPtr) return null; + + const writeField = (structPtr, field, value) => { + Module.HEAPU32[(structPtr >>> 2) + field] = value; + }; + const readField = (structPtr, field) => + Module.HEAPU32[(structPtr >>> 2) + field]; + + writeField(inputBufferPtr, 0, compressedPtr); + writeField(inputBufferPtr, 1, compressedData.length); + writeField(inputBufferPtr, 2, 0); + + const chunks = []; + let totalLength = 0; + let previousInputPosition = -1; + let previousOutputLength = -1; + while (true) { + writeField(outputBufferPtr, 0, outputChunkPtr); + writeField(outputBufferPtr, 1, outputChunkSize); + writeField(outputBufferPtr, 2, 0); + + const remaining = Number(Module._ZSTD_decompressStream( + stream, + outputBufferPtr, + inputBufferPtr, + )); + if (Module._ZSTD_isError(remaining) !== 0) return null; + + const inputPosition = readField(inputBufferPtr, 2); + const produced = readField(outputBufferPtr, 2); + if (inputPosition > compressedData.length + || produced > outputChunkSize + || totalLength + produced > maxOutputSize) { + return null; + } + if (produced > 0) { + chunks.push(new Uint8Array( + Module.HEAPU8.buffer, + outputChunkPtr, + produced, + ).slice()); + totalLength += produced; + } + + const allInputConsumed = inputPosition === compressedData.length; + if (remaining === 0 && allInputConsumed) { + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + return result; + } + + const madeProgress = inputPosition !== previousInputPosition + || totalLength !== previousOutputLength; + if (!madeProgress || (allInputConsumed && produced === 0)) { + return null; + } + previousInputPosition = inputPosition; + previousOutputLength = totalLength; + } + } finally { + if (stream) Module._ZSTD_freeDStream(stream); + if (compressedPtr) Module._free(compressedPtr); + if (inputBufferPtr) Module._free(inputBufferPtr); + if (outputBufferPtr) Module._free(outputBufferPtr); + if (outputChunkPtr) Module._free(outputChunkPtr); } } WRAPPER_JS +# Keep CPU-heavy zstd calls away from the browser UI thread. The public +# zstd.js file is only a small request broker; the worker loads the generated +# core and transfers byte buffers without an additional structured-clone copy. +cat > zstd_worker_generated.js << 'WORKER_JS' +'use strict'; + +importScripts('zstd_core.js'); + +self.onmessage = async (event) => { + const { id, operation, inputBuffer, option } = event.data; + try { + const input = new Uint8Array(inputBuffer); + const result = operation === 'compress' + ? await compressData(input, option) + : await decompressData(input, option); + if (result === null) { + self.postMessage({ id, result: null }); + } else { + self.postMessage({ id, result }, [result.buffer]); + } + } catch (_) { + self.postMessage({ id, result: null }); + } +}; +WORKER_JS + +cat > zstd_client_generated.js << 'CLIENT_JS' +'use strict'; + +(() => { + const scriptUrl = document.currentScript?.src + ?? new URL('zstd.js', document.baseURI).href; + const workerUrl = new URL('zstd_worker.js', scriptUrl).href; + let worker = null; + let nextRequestId = 1; + const pending = new Map(); + + function failPendingRequests() { + for (const resolve of pending.values()) resolve(null); + pending.clear(); + worker?.terminate(); + worker = null; + } + + function getWorker() { + if (worker !== null) return worker; + worker = new Worker(workerUrl); + worker.onmessage = (event) => { + const resolve = pending.get(event.data.id); + if (resolve === undefined) return; + pending.delete(event.data.id); + resolve(event.data.result); + }; + worker.onerror = failPendingRequests; + worker.onmessageerror = failPendingRequests; + return worker; + } + + function run(operation, inputData, option) { + return new Promise((resolve) => { + const id = nextRequestId++; + try { + const inputCopy = inputData.slice(); + pending.set(id, resolve); + getWorker().postMessage( + { id, operation, inputBuffer: inputCopy.buffer, option }, + [inputCopy.buffer], + ); + } catch (_) { + pending.delete(id); + resolve(null); + } + }); + } + + globalThis.compressData = (inputData, compressionLevel) => + run('compress', inputData, compressionLevel); + globalThis.decompressData = (compressedData, maxOutputSize) => + run('decompress', compressedData, maxOutputSize); +})(); +CLIENT_JS + mkdir -p "$OUT_BLOB" "$OUT_EXAMPLE_WEB" "$OUT_ZSTANDARD_EXAMPLE_WEB" # Replace the wasm filename in the generated JS to match what we'll copy -sed -i.bak 's/zstd_generated\.wasm/zstd.wasm/g' zstd_generated.js -rm -f zstd_generated.js.bak - -cp zstd_generated.wasm "$OUT_BLOB/zstd.wasm" -cp zstd_generated.wasm "$OUT_EXAMPLE_WEB/zstd.wasm" -cp zstd_generated.wasm "$OUT_ZSTANDARD_EXAMPLE_WEB/zstd.wasm" -cp zstd_generated.js "$OUT_BLOB/zstd.js" -cp zstd_generated.js "$OUT_EXAMPLE_WEB/zstd.js" -cp zstd_generated.js "$OUT_ZSTANDARD_EXAMPLE_WEB/zstd.js" -rm -f "$ZSTD_ROOT/zstd_generated.js" "$ZSTD_ROOT/zstd_generated.wasm" +sed -i.bak 's/zstd_core_generated\.wasm/zstd.wasm/g' zstd_core_generated.js +rm -f zstd_core_generated.js.bak + +for destination in "$OUT_BLOB" "$OUT_EXAMPLE_WEB" "$OUT_ZSTANDARD_EXAMPLE_WEB"; do + cp zstd_core_generated.wasm "$destination/zstd.wasm" + cp zstd_core_generated.js "$destination/zstd_core.js" + cp zstd_worker_generated.js "$destination/zstd_worker.js" + cp zstd_client_generated.js "$destination/zstd.js" +done +rm -f \ + "$ZSTD_ROOT/zstd_core_generated.js" \ + "$ZSTD_ROOT/zstd_core_generated.wasm" \ + "$ZSTD_ROOT/zstd_worker_generated.js" \ + "$ZSTD_ROOT/zstd_client_generated.js" echo "Done. zstd.js and zstd.wasm have been written to:" echo " - $OUT_BLOB/" diff --git a/scripts/check_lcov_coverage.dart b/scripts/check_lcov_coverage.dart new file mode 100644 index 00000000..b9e8fe31 --- /dev/null +++ b/scripts/check_lcov_coverage.dart @@ -0,0 +1,57 @@ +import 'dart:io'; + +void main(List arguments) { + if (arguments.length != 2) { + stderr.writeln( + 'Usage: dart scripts/check_lcov_coverage.dart ', + ); + exitCode = 2; + return; + } + + final coverageFile = File(arguments[0]); + final threshold = double.tryParse(arguments[1]); + if (!coverageFile.existsSync()) { + stderr.writeln('Coverage file does not exist: ${coverageFile.path}'); + exitCode = 2; + return; + } + if (threshold == null || + !threshold.isFinite || + threshold < 0 || + threshold > 100) { + stderr.writeln('Coverage threshold must be a number from 0 to 100.'); + exitCode = 2; + return; + } + + var hit = 0; + var found = 0; + for (final line in coverageFile.readAsLinesSync()) { + if (!line.startsWith('DA:')) continue; + final fields = line.substring(3).split(','); + if (fields.length < 2) continue; + final count = int.tryParse(fields[1]); + if (count == null) continue; + found++; + if (count > 0) hit++; + } + + if (found == 0) { + stderr.writeln( + 'Coverage file contains no line records: ${coverageFile.path}', + ); + exitCode = 2; + return; + } + + final coverage = hit * 100 / found; + stdout.writeln( + 'Line coverage: ${coverage.toStringAsFixed(2)}% ($hit/$found; ' + 'required ${threshold.toStringAsFixed(2)}%)', + ); + if (coverage + 1e-9 < threshold) { + stderr.writeln('Coverage is below the required threshold.'); + exitCode = 1; + } +} diff --git a/scripts/check_performance_regression.dart b/scripts/check_performance_regression.dart index 82092459..2ff921c0 100644 --- a/scripts/check_performance_regression.dart +++ b/scripts/check_performance_regression.dart @@ -22,7 +22,9 @@ void main(List args) { } if (baselinePath == null || currentPath == null) { - print('Usage: dart run scripts/check_performance_regression.dart --baseline=BASELINE.json --current=CURRENT.json [--threshold=0.10]'); + print( + 'Usage: dart run scripts/check_performance_regression.dart --baseline=BASELINE.json --current=CURRENT.json [--threshold=0.10]', + ); exit(1); } @@ -37,40 +39,60 @@ void main(List args) { exit(1); } - final baseline = jsonDecode(baselineFile.readAsStringSync()) as Map; - final current = jsonDecode(currentFile.readAsStringSync()) as Map; + final baseline = + jsonDecode(baselineFile.readAsStringSync()) as Map; + final current = + jsonDecode(currentFile.readAsStringSync()) as Map; - final baselineResults = (baseline['results'] as List).cast>(); - final currentResults = (current['results'] as List).cast>(); + final baselineResults = (baseline['results'] as List) + .cast>(); + final currentResults = (current['results'] as List) + .cast>(); if (baselineResults.length != currentResults.length) { - print('Result count mismatch: baseline ${baselineResults.length}, current ${currentResults.length}'); + print( + 'Result count mismatch: baseline ${baselineResults.length}, current ${currentResults.length}', + ); exit(1); } var failed = false; for (var i = 0; i < baselineResults.length; i++) { final name = baselineResults[i]['name'] as String; - final baseCompress = (baselineResults[i]['compress_throughput_mbps'] as num).toDouble(); - final baseDecompress = (baselineResults[i]['decompress_throughput_mbps'] as num).toDouble(); - final currCompress = (currentResults[i]['compress_throughput_mbps'] as num).toDouble(); - final currDecompress = (currentResults[i]['decompress_throughput_mbps'] as num).toDouble(); + final baseCompress = (baselineResults[i]['compress_throughput_mbps'] as num) + .toDouble(); + final baseDecompress = + (baselineResults[i]['decompress_throughput_mbps'] as num).toDouble(); + final currCompress = (currentResults[i]['compress_throughput_mbps'] as num) + .toDouble(); + final currDecompress = + (currentResults[i]['decompress_throughput_mbps'] as num).toDouble(); - final compressRegress = baseCompress > 0 ? (baseCompress - currCompress) / baseCompress : 0.0; - final decompressRegress = baseDecompress > 0 ? (baseDecompress - currDecompress) / baseDecompress : 0.0; + final compressRegress = baseCompress > 0 + ? (baseCompress - currCompress) / baseCompress + : 0.0; + final decompressRegress = baseDecompress > 0 + ? (baseDecompress - currDecompress) / baseDecompress + : 0.0; if (compressRegress > threshold) { - print('REGRESSION $name: compress ${currCompress.toStringAsFixed(2)} MB/s (baseline $baseCompress, ${(compressRegress * 100).toStringAsFixed(1)}% regression)'); + print( + 'REGRESSION $name: compress ${currCompress.toStringAsFixed(2)} MB/s (baseline $baseCompress, ${(compressRegress * 100).toStringAsFixed(1)}% regression)', + ); failed = true; } if (decompressRegress > threshold) { - print('REGRESSION $name: decompress ${currDecompress.toStringAsFixed(2)} MB/s (baseline $baseDecompress, ${(decompressRegress * 100).toStringAsFixed(1)}% regression)'); + print( + 'REGRESSION $name: decompress ${currDecompress.toStringAsFixed(2)} MB/s (baseline $baseDecompress, ${(decompressRegress * 100).toStringAsFixed(1)}% regression)', + ); failed = true; } } if (failed) { - print('Performance regression detected (threshold ${(threshold * 100).toInt()}%)'); + print( + 'Performance regression detected (threshold ${(threshold * 100).toInt()}%)', + ); exit(1); } print('No performance regression detected.'); diff --git a/scripts/check_pub_metadata.dart b/scripts/check_pub_metadata.dart index 5f11bddc..97ae85e1 100644 --- a/scripts/check_pub_metadata.dart +++ b/scripts/check_pub_metadata.dart @@ -99,6 +99,20 @@ void main(List args) { } } + final cliPubspec = File('zstandard_cli/pubspec.yaml').readAsStringSync(); + final cliVersion = RegExp( + r'^version:\s*(\d+\.\d+\.\d+)\s*$', + multiLine: true, + ).firstMatch(cliPubspec)?.group(1); + final cliRunner = File('zstandard_cli/lib/src/cli_runner.dart'); + if (cliVersion == null || + !cliRunner.existsSync() || + !cliRunner.readAsStringSync().contains( + "const String zstandardCliVersion = '$cliVersion';", + )) { + errors.add('zstandard_cli --version must match its pubspec version'); + } + for (final manifest in [ 'zstandard_ios/ios/zstandard_ios/Package.swift', 'zstandard_macos/macos/zstandard_macos/Package.swift', @@ -138,6 +152,30 @@ void main(List args) { 'Canonical zstd source is missing zstandard_native/src/zstd/zstd.h', ); } + final provenance = File('zstandard_native/UPSTREAM_ZSTD.md'); + if (!provenance.existsSync()) { + errors.add('zstandard_native is missing UPSTREAM_ZSTD.md'); + } else { + final contents = provenance.readAsStringSync(); + if (!contents.contains(RegExp(r'\b[0-9a-f]{40}\b')) || + !contents.contains('1.5.7')) { + errors.add( + 'UPSTREAM_ZSTD.md must record an exact upstream commit and version', + ); + } + } + + for (final generated in [ + 'zstandard_web/blob/zstd.js', + 'zstandard_web/blob/zstd_core.js', + 'zstandard_web/blob/zstd_worker.js', + 'zstandard_web/blob/zstd.wasm', + ]) { + final file = File(generated); + if (!file.existsSync() || file.lengthSync() == 0) { + errors.add('Missing generated WebAssembly artifact: $generated'); + } + } for (final duplicate in [ 'zstandard_ios/ios/zstandard_ios/Sources/zstd', 'zstandard_macos/macos/zstandard_macos/Sources/zstd', diff --git a/scripts/collect_all_coverage.sh b/scripts/collect_all_coverage.sh index 660f3e61..7377b570 100644 --- a/scripts/collect_all_coverage.sh +++ b/scripts/collect_all_coverage.sh @@ -18,7 +18,7 @@ done if [ -d "zstandard_cli" ]; then (cd zstandard_cli && dart test --coverage=coverage 2>/dev/null) || true - (cd zstandard_cli && dart run coverage:format_coverage --lcov -i coverage -o coverage/lcov.info --packages=.dart_tool/package_config.json 2>/dev/null) || true + (cd zstandard_cli && dart run coverage:format_coverage --lcov -i coverage -o coverage/lcov.info --packages=.dart_tool/package_config.json --report-on=lib 2>/dev/null) || true if [ -f "zstandard_cli/coverage/lcov.info" ]; then cp zstandard_cli/coverage/lcov.info coverage_all/zstandard_cli.lcov.info 2>/dev/null || true fi diff --git a/scripts/coverage_report.bat b/scripts/coverage_report.bat index 3f6adbe7..748d8104 100644 --- a/scripts/coverage_report.bat +++ b/scripts/coverage_report.bat @@ -21,7 +21,7 @@ if exist zstandard_cli ( echo ---- Coverage: zstandard_cli ---- cd zstandard_cli dart test --coverage=coverage - dart run coverage:format_coverage --lcov -i coverage -o coverage/lcov.info --packages=.dart_tool/package_config.json + dart run coverage:format_coverage --lcov -i coverage -o coverage/lcov.info --packages=.dart_tool/package_config.json --report-on=lib cd "%ROOT%" ) diff --git a/scripts/coverage_report.sh b/scripts/coverage_report.sh index f383fa03..3cedc023 100755 --- a/scripts/coverage_report.sh +++ b/scripts/coverage_report.sh @@ -24,7 +24,7 @@ done # CLI: dart test --coverage then format to lcov if [[ -d "zstandard_cli" ]]; then echo "---- Coverage: zstandard_cli ----" - (cd zstandard_cli && dart test --coverage=coverage 2>/dev/null && dart run coverage:format_coverage --lcov -i coverage -o coverage/lcov.info --packages=.dart_tool/package_config.json 2>/dev/null) || true + (cd zstandard_cli && dart test --coverage=coverage 2>/dev/null && dart run coverage:format_coverage --lcov -i coverage -o coverage/lcov.info --packages=.dart_tool/package_config.json --report-on=lib 2>/dev/null) || true if [[ -f "zstandard_cli/coverage/lcov.info" ]]; then LCOV_ARGS+=("--add-tracefile" "zstandard_cli/coverage/lcov.info") fi diff --git a/scripts/create_local_overrides.dart b/scripts/create_local_overrides.dart new file mode 100644 index 00000000..28b8b545 --- /dev/null +++ b/scripts/create_local_overrides.dart @@ -0,0 +1,105 @@ +import 'dart:io'; + +void main(List arguments) { + if (arguments.isEmpty || arguments.length > 2) { + stderr.writeln( + 'Usage: dart scripts/create_local_overrides.dart ' + ' [external-native-path]', + ); + exitCode = 2; + return; + } + + final repositoryRoot = File.fromUri(Platform.script).parent.parent; + final package = arguments.first; + final externalNative = arguments.length == 2 ? arguments[1] : null; + + final packageNative = externalNative ?? '../zstandard_native'; + final exampleNative = externalNative ?? '../../zstandard_native'; + + switch (package) { + case 'zstandard_native': + case 'zstandard_platform_interface': + return; + case 'zstandard_cli': + _writeOverrides(repositoryRoot, '$package/pubspec_overrides.yaml', { + 'zstandard_native': packageNative, + }); + case 'zstandard_android': + case 'zstandard_ios': + case 'zstandard_linux': + case 'zstandard_macos': + case 'zstandard_web': + case 'zstandard_windows': + final dependencies = { + 'zstandard_native': packageNative, + 'zstandard_platform_interface': '../zstandard_platform_interface', + }; + _writeOverrides( + repositoryRoot, + '$package/pubspec_overrides.yaml', + dependencies, + ); + final example = Directory.fromUri( + repositoryRoot.uri.resolve('$package/example/'), + ); + if (example.existsSync()) { + _writeOverrides( + repositoryRoot, + '$package/example/pubspec_overrides.yaml', + { + 'zstandard_native': exampleNative, + 'zstandard_platform_interface': + '../../zstandard_platform_interface', + }, + ); + } + final legacyExample = Directory.fromUri( + repositoryRoot.uri.resolve('$package/example_legacy/'), + ); + if (package == 'zstandard_android' && legacyExample.existsSync()) { + _writeOverrides( + repositoryRoot, + '$package/example_legacy/pubspec_overrides.yaml', + { + 'zstandard_native': exampleNative, + 'zstandard_platform_interface': + '../../zstandard_platform_interface', + }, + ); + } + case 'zstandard': + _writeOverrides(repositoryRoot, '$package/pubspec_overrides.yaml', { + for (final dependency in const [ + 'zstandard_android', + 'zstandard_ios', + 'zstandard_linux', + 'zstandard_macos', + 'zstandard_native', + 'zstandard_platform_interface', + 'zstandard_web', + 'zstandard_windows', + ]) + dependency: '../$dependency', + }); + default: + stderr.writeln('Unsupported package: $package'); + exitCode = 2; + } +} + +void _writeOverrides( + Directory repositoryRoot, + String relativePath, + Map dependencies, +) { + final output = StringBuffer('dependency_overrides:\n'); + for (final MapEntry(:key, :value) in dependencies.entries) { + final portablePath = value.replaceAll('\\', '/').replaceAll("'", "''"); + output + ..writeln(' $key:') + ..writeln(" path: '$portablePath'"); + } + File.fromUri(repositoryRoot.uri.resolve(relativePath)) + .writeAsStringSync(output.toString()); +} diff --git a/scripts/create_local_overrides.sh b/scripts/create_local_overrides.sh new file mode 100755 index 00000000..f0a5f5d7 --- /dev/null +++ b/scripts/create_local_overrides.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Create package-local dependency overrides for monorepo validation. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec dart "$SCRIPT_DIR/create_local_overrides.dart" "$@" diff --git a/scripts/generate_quality_report.dart b/scripts/generate_quality_report.dart index 7862a216..2d642f3d 100644 --- a/scripts/generate_quality_report.dart +++ b/scripts/generate_quality_report.dart @@ -33,15 +33,23 @@ void main() { buffer.writeln('## Test Packages'); buffer.writeln(''); final packages = [ - 'zstandard', 'zstandard_platform_interface', 'zstandard_android', - 'zstandard_ios', 'zstandard_macos', 'zstandard_linux', 'zstandard_windows', - 'zstandard_web', 'zstandard_cli' + 'zstandard', + 'zstandard_platform_interface', + 'zstandard_android', + 'zstandard_ios', + 'zstandard_macos', + 'zstandard_linux', + 'zstandard_windows', + 'zstandard_web', + 'zstandard_cli', ]; for (final p in packages) { final testDir = Directory('$root/$p/test'); final pubspec = File('$root/$p/pubspec.yaml'); if (pubspec.existsSync()) { - buffer.writeln('- $p: ${testDir.existsSync() ? "has test/" : "no test/"}'); + buffer.writeln( + '- $p: ${testDir.existsSync() ? "has test/" : "no test/"}', + ); } } diff --git a/scripts/publish_if_missing.dart b/scripts/publish_if_missing.dart index bcb8908b..af2c6fc6 100644 --- a/scripts/publish_if_missing.dart +++ b/scripts/publish_if_missing.dart @@ -5,6 +5,19 @@ import 'dart:convert'; import 'dart:io'; +const _packages = { + 'zstandard', + 'zstandard_android', + 'zstandard_cli', + 'zstandard_ios', + 'zstandard_linux', + 'zstandard_macos', + 'zstandard_native', + 'zstandard_platform_interface', + 'zstandard_web', + 'zstandard_windows', +}; + Future main(List args) async { if (args.length != 2) { stderr.writeln( @@ -16,6 +29,12 @@ Future main(List args) async { final package = args[0]; final version = args[1]; + if (!_packages.contains(package) || + !RegExp(r'^\d+\.\d+\.\d+$').hasMatch(version)) { + stderr.writeln('Package or stable version is not part of this release.'); + exitCode = 2; + return; + } final client = HttpClient()..connectionTimeout = const Duration(seconds: 15); var alreadyPublished = false; try { @@ -24,13 +43,18 @@ Future main(List args) async { ); final response = await request.close(); final body = await response.transform(utf8.decoder).join(); - if (response.statusCode == HttpStatus.ok) { - final json = jsonDecode(body) as Map; - final versions = (json['versions'] as List? ?? const []); - alreadyPublished = versions.any( - (entry) => entry is Map && entry['version'] == version, + if (response.statusCode != HttpStatus.ok) { + stderr.writeln( + 'pub.dev returned HTTP ${response.statusCode} for $package.', ); + exitCode = 1; + return; } + final json = jsonDecode(body) as Map; + final versions = (json['versions'] as List? ?? const []); + alreadyPublished = versions.any( + (entry) => entry is Map && entry['version'] == version, + ); } catch (error) { stderr.writeln('Could not query pub.dev before publishing: $error'); exitCode = 1; @@ -44,12 +68,26 @@ Future main(List args) async { return; } - final result = await Process.run( - 'dart', - ['pub', 'publish', '-f'], - workingDirectory: package, - runInShell: true, - ); + final dryRun = await Process.run('dart', [ + 'pub', + 'publish', + '--dry-run', + ], workingDirectory: package); + stdout.write(dryRun.stdout); + stderr.write(dryRun.stderr); + if (dryRun.exitCode != 0) { + stderr.writeln( + 'Dry-run validation failed; $package $version was not published.', + ); + exitCode = dryRun.exitCode; + return; + } + + final result = await Process.run('dart', [ + 'pub', + 'publish', + '-f', + ], workingDirectory: package); stdout.write(result.stdout); stderr.write(result.stderr); exitCode = result.exitCode; diff --git a/scripts/run_android_emulator_ci.sh b/scripts/run_android_emulator_ci.sh index db758238..d4220a91 100644 --- a/scripts/run_android_emulator_ci.sh +++ b/scripts/run_android_emulator_ci.sh @@ -55,6 +55,12 @@ export ANDROID_HOME="$sdk_root" export ANDROID_SDK_ROOT="$sdk_root" export PATH="$sdk_root/platform-tools:$sdk_root/emulator:$PATH" +android_user_home="${ANDROID_USER_HOME:-$HOME/.android}" +avd_home="${ANDROID_AVD_HOME:-$android_user_home/avd}" +mkdir -p "$avd_home" +export ANDROID_USER_HOME="$android_user_home" +export ANDROID_AVD_HOME="$avd_home" + echo "Installing Android emulator prerequisites..." set +o pipefail yes 2>/dev/null | "$sdkmanager_bin" --licenses >/dev/null @@ -74,7 +80,12 @@ echo no | "$avdmanager_bin" create avd \ --package "system-images;android-${api_level};${target};${arch}" \ --device pixel_2 -avd_config="$HOME/.android/avd/${avd_name}.avd/config.ini" +avd_config="$avd_home/${avd_name}.avd/config.ini" +if [[ ! -f "$avd_config" ]]; then + echo "AVD manager did not create the expected configuration: ${avd_config}" >&2 + find "$avd_home" -maxdepth 2 -type f -print >&2 || true + exit 1 +fi # Location is outside the scope of these integration tests. printf 'hw.cpu.ncore=2\nhw.gps=no\n' >> "$avd_config" diff --git a/scripts/test_web_integration.sh b/scripts/test_web_integration.sh index beff2db5..e5199108 100755 --- a/scripts/test_web_integration.sh +++ b/scripts/test_web_integration.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Run web tests in Chrome: unit tests (flutter test -d chrome) and integration -# tests (flutter drive with ChromeDriver + web-server). +# Run package-level artifact tests on the Dart VM and browser integration tests +# with ChromeDriver plus Flutter's web-server device. # # Usage: from repo root, ./scripts/test_web_integration.sh # @@ -12,28 +12,19 @@ # npx @puppeteer/browsers install chromedriver@stable # See: https://docs.flutter.dev/testing/integration-tests#web -set -e +set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" EXIT_CODE=0 -CHROMEDRIVER_PID="" CHROMEDRIVER_PORT=4444 +WEB_SERVER_PORT="${ZSTANDARD_WEB_PORT:-0}" -# If we start ChromeDriver, stop it on exit. -cleanup_chromedriver() { - if [[ -n "$CHROMEDRIVER_PID" ]] && kill -0 "$CHROMEDRIVER_PID" 2>/dev/null; then - kill "$CHROMEDRIVER_PID" 2>/dev/null || true - wait "$CHROMEDRIVER_PID" 2>/dev/null || true - fi -} -trap cleanup_chromedriver EXIT - -# --- Unit tests (Chrome), if the package has any --- +# --- Package-level artifact tests, if the package has any --- WEB_TEST_COUNT=$(find "$ROOT/zstandard_web/test" -name "*_test.dart" 2>/dev/null | wc -l | tr -d ' ') if [[ "$WEB_TEST_COUNT" -gt 0 ]]; then - echo "Running zstandard_web unit tests in Chrome..." - if (cd "$ROOT/zstandard_web" && flutter test -d chrome --coverage 2>/dev/null || flutter test -d chrome); then + echo "Running zstandard_web package tests..." + if (cd "$ROOT/zstandard_web" && flutter test --coverage); then echo "Web unit tests passed." else EXIT_CODE=1 @@ -44,7 +35,7 @@ else fi # --- Integration tests (flutter drive + ChromeDriver + web-server) --- -if [[ -n "$ZSTANDARD_SKIP_WEB" ]]; then +if [[ -n "${ZSTANDARD_SKIP_WEB:-}" ]]; then echo "Web integration tests skipped (ZSTANDARD_SKIP_WEB=1)." elif [[ -d "$ROOT/zstandard_web/example/integration_test" ]] && [[ -d "$ROOT/zstandard_web/example/test_driver" ]]; then if ! command -v chromedriver &>/dev/null; then @@ -70,14 +61,19 @@ elif [[ -d "$ROOT/zstandard_web/example/integration_test" ]] && [[ -d "$ROOT/zst } run_web_driver() { - local driver_pid="" - local driver_log="" + # These must outlive the function body because the EXIT trap runs when + # the exported-function subshell exits (Bash 3.2 included). + driver_pid="" + driver_log="" cleanup_driver() { if [[ -n "$driver_pid" ]] && kill -0 "$driver_pid" 2>/dev/null; then kill "$driver_pid" 2>/dev/null || true wait "$driver_pid" 2>/dev/null || true fi + if [[ -n "$driver_log" ]]; then + rm -f "$driver_log" + fi } trap cleanup_driver EXIT @@ -102,23 +98,31 @@ elif [[ -d "$ROOT/zstandard_web/example/integration_test" ]] && [[ -d "$ROOT/zst sleep 2 echo "Running zstandard_web example integration tests (flutter drive -d web-server)..." - local chrome_args=() if [[ -n "${CHROME_EXECUTABLE:-}" ]]; then - chrome_args+=("--chrome-binary=$CHROME_EXECUTABLE") + flutter drive \ + --driver=test_driver/integration_test.dart \ + --target=integration_test/zstandard_web_integration_test.dart \ + --driver-port="$CHROMEDRIVER_PORT" \ + --chrome-binary="$CHROME_EXECUTABLE" \ + --web-browser-flag=--disable-dev-shm-usage \ + --web-browser-flag=--disable-gpu \ + -d web-server \ + --web-port="$WEB_SERVER_PORT" + else + flutter drive \ + --driver=test_driver/integration_test.dart \ + --target=integration_test/zstandard_web_integration_test.dart \ + --driver-port="$CHROMEDRIVER_PORT" \ + --web-browser-flag=--disable-dev-shm-usage \ + --web-browser-flag=--disable-gpu \ + -d web-server \ + --web-port="$WEB_SERVER_PORT" fi - flutter drive \ - --driver=test_driver/integration_test.dart \ - --target=integration_test/zstandard_web_integration_test.dart \ - --driver-port="$CHROMEDRIVER_PORT" \ - "${chrome_args[@]}" \ - --web-browser-flag=--disable-dev-shm-usage \ - --web-browser-flag=--disable-gpu \ - -d web-server \ - --web-port=8080 } export -f chrome_driver_ready run_web_driver export CHROMEDRIVER_PORT + export WEB_SERVER_PORT DRIVE_OUTPUT=$(mktemp -t flutter_drive_XXXXXX.txt) if [[ "$(uname -s)" == "Linux" ]] && command -v xvfb-run >/dev/null 2>&1; then @@ -132,8 +136,11 @@ elif [[ -d "$ROOT/zstandard_web/example/integration_test" ]] && [[ -d "$ROOT/zst else DRIVE_EXIT=$? fi - # Flutter drive can exit 0 even when compilation fails; detect known failure output. - if [[ $DRIVE_EXIT -ne 0 ]] || grep -qE "Failed to compile|Dart compiler exited unexpectedly|SessionNotCreatedException|Unable to start a WebDriver session" "$DRIVE_OUTPUT"; then + # Flutter drive can exit 0 even when setup or compilation fails. Require + # its positive completion marker as well as the process exit status. + if [[ $DRIVE_EXIT -ne 0 ]] || \ + grep -qE "Failed to compile|Dart compiler exited unexpectedly|SessionNotCreatedException|Unable to start a WebDriver session|unbound variable" "$DRIVE_OUTPUT" || \ + ! grep -q "All tests passed" "$DRIVE_OUTPUT"; then EXIT_CODE=1 echo "Web integration tests failed." cat "$DRIVE_OUTPUT" diff --git a/scripts/update_zstd.sh b/scripts/update_zstd.sh index 8e7293d5..1602d915 100755 --- a/scripts/update_zstd.sh +++ b/scripts/update_zstd.sh @@ -1,32 +1,45 @@ #!/usr/bin/env bash # Update the canonical zstd source in zstandard_native/src/zstd/ from the official repo. # Usage: from repo root, run: ./scripts/update_zstd.sh -# Optional: ./scripts/update_zstd.sh v1.5.6 (tag or branch; default: dev) +# Optional: ./scripts/update_zstd.sh # # Requires: git. After this, run zstandard_ios/scripts/sync_zstd.sh and # zstandard_macos/scripts/sync_zstd.sh (from repo root), and optionally # ./scripts/regenerate_bindings.sh. -set -e +set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" ZSTD_DIR="$ROOT/zstandard_native/src/zstd" -REF="${1:-dev}" +DEFAULT_REF="d7ee3207cc0db53f78fc6a69babc80747b1b7658" +CALLOC_FIX="3f8f9b3f89244638f10bca664c120fd28cb14efe" +REF="${1:-$DEFAULT_REF}" echo "Fetching zstd from https://github.com/facebook/zstd.git (ref: $REF)..." -TMP="$ROOT/.zstd_upstream" -rm -rf "$TMP" -git clone --depth 1 --branch "$REF" https://github.com/facebook/zstd.git "$TMP" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +git -C "$TMP" init -q +git -C "$TMP" remote add origin https://github.com/facebook/zstd.git +git -C "$TMP" fetch --depth 1 origin "$REF" +git -C "$TMP" checkout --detach FETCH_HEAD +RESOLVED_REF="$(git -C "$TMP" rev-parse HEAD)" + +# Keep the known allocator null-dereference fix when updating from an older +# revision. A revision that already contains it needs no local patch. +if ! grep -q 'if (ptr != NULL)' "$TMP/lib/common/allocations.h"; then + git -C "$TMP" fetch --depth 1 origin "$CALLOC_FIX" + git -C "$TMP" show --format= "$CALLOC_FIX" -- lib/common/allocations.h | + git -C "$TMP" apply +fi mkdir -p "$ZSTD_DIR" echo "Copying lib/ into $ZSTD_DIR ..." # `include/` contains the SwiftPM public-header bridges maintained by this # repository; preserve it while replacing the upstream source tree. rsync -a --delete --exclude='include/' "$TMP/lib/" "$ZSTD_DIR/" -rm -rf "$TMP" if [[ ! -f "$ZSTD_DIR/zstd.h" ]]; then echo "Error: zstd.h not found after copy." exit 1 fi -echo "Done. zstandard_native/src/zstd/ is now in sync with facebook/zstd @ $REF." +echo "Done. zstandard_native/src/zstd/ is based on facebook/zstd @ $RESOLVED_REF." echo "Next: run zstandard_ios/scripts/sync_zstd.sh and zstandard_macos/scripts/sync_zstd.sh (from repo root), and optionally ./scripts/regenerate_bindings.sh" diff --git a/zstandard/CHANGELOG.md b/zstandard/CHANGELOG.md index ed338300..8dc6c83c 100644 --- a/zstandard/CHANGELOG.md +++ b/zstandard/CHANGELOG.md @@ -1,7 +1,13 @@ ## Unreleased +- Added `maxOutputSize` to decompression and extension APIs with a 256 MiB + default, while preserving third-party platform-interface compatibility. +- Standardized null failure semantics and documented unknown-size, + concatenated, and empty-frame behavior across platforms. - Improved pub.dev package metadata and added regression checks for publication requirements, including Apple Swift Package Manager manifests. +- Removed the placeholder `WIP` test; the package now reports only meaningful + unit and property checks. ## 1.5.0 - Dependencies Updated diff --git a/zstandard/README.md b/zstandard/README.md index 374a6bc6..055089b3 100644 --- a/zstandard/README.md +++ b/zstandard/README.md @@ -26,7 +26,9 @@ void act() async { Uint8List? compressed = await zstandard.compress(original); - Uint8List? decompressed = await zstandard.decompress(compressed ?? Uint8List(0)); + final decompressed = compressed == null + ? null + : await zstandard.decompress(compressed); } ``` @@ -38,10 +40,16 @@ void act() async { Uint8List? compressed = await original.compress(); - Uint8List? decompressed = await compressed.decompress(); + final decompressed = await compressed.decompress(); } ``` +Compression always emits a valid zstd frame, including for empty input. +Decompression supports concatenated frames and frames without a declared +content size. It returns `null` for malformed or truncated input and limits +output to 256 MiB by default. Pass `maxOutputSize` to `decompress` (including +the extension method) to choose a smaller application-specific budget. + Below are examples of the plugin in action across different platforms.

diff --git a/zstandard/analysis_options.yaml b/zstandard/analysis_options.yaml index a5744c1c..e46654b9 100644 --- a/zstandard/analysis_options.yaml +++ b/zstandard/analysis_options.yaml @@ -1,3 +1,6 @@ +analyzer: + exclude: + - build/** include: package:flutter_lints/flutter.yaml # Additional information about this file can be found at diff --git a/zstandard/example/analysis_options.yaml b/zstandard/example/analysis_options.yaml index 0d290213..bf8d4218 100644 --- a/zstandard/example/analysis_options.yaml +++ b/zstandard/example/analysis_options.yaml @@ -7,6 +7,15 @@ # The following line activates a set of recommended lints for Flutter apps, # packages, and plugins designed to encourage good coding practices. +analyzer: + exclude: + - build/** + - android/** + - ios/** + - web/** + - windows/** + - macos/** + - linux/** include: package:flutter_lints/flutter.yaml linter: diff --git a/zstandard/example/android/app/build.gradle b/zstandard/example/android/app/build.gradle index 18ede477..f3ae5054 100644 --- a/zstandard/example/android/app/build.gradle +++ b/zstandard/example/android/app/build.gradle @@ -15,7 +15,7 @@ android { } defaultConfig { - // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). + // Stable identifier used only by this example application. applicationId = "com.example.zstandard_example" // You can update the following values to match your application needs. // For more information, see: https://flutter.dev/to/review-gradle-config. @@ -27,8 +27,8 @@ android { buildTypes { release { - // TODO: Add your own signing config for the release build. - // Signing with the debug keys for now, so `flutter run --release` works. + // Examples use debug signing so CI can build and run Release + // without production signing credentials. signingConfig = signingConfigs.debug } } diff --git a/zstandard/example/android/gradle.properties b/zstandard/example/android/gradle.properties index f534322e..1fc98750 100644 --- a/zstandard/example/android/gradle.properties +++ b/zstandard/example/android/gradle.properties @@ -1,5 +1,4 @@ org.gradle.jvmargs=-Xmx4G -XX:MaxMetaspaceSize=2G -XX:+HeapDumpOnOutOfMemoryError android.useAndroidX=true -android.enableJetifier=true android.builtInKotlin=true android.newDsl=false diff --git a/zstandard/example/ios/Podfile b/zstandard/example/ios/Podfile index e51a31d9..a419e423 100644 --- a/zstandard/example/ios/Podfile +++ b/zstandard/example/ios/Podfile @@ -1,5 +1,5 @@ # Uncomment this line to define a global platform for your project -# platform :ios, '13.0' +# platform :ios, '15.0' # CocoaPods analytics sends network stats synchronously affecting flutter build latency. ENV['COCOAPODS_DISABLE_STATS'] = 'true' diff --git a/zstandard/example/ios/Runner.xcodeproj/project.pbxproj b/zstandard/example/ios/Runner.xcodeproj/project.pbxproj index 0cdaf628..cd446399 100644 --- a/zstandard/example/ios/Runner.xcodeproj/project.pbxproj +++ b/zstandard/example/ios/Runner.xcodeproj/project.pbxproj @@ -455,7 +455,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 13.0; + IPHONEOS_DEPLOYMENT_TARGET = 15.0; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = iphoneos; SUPPORTED_PLATFORMS = iphoneos; @@ -584,7 +584,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 13.0; + IPHONEOS_DEPLOYMENT_TARGET = 15.0; MTL_ENABLE_DEBUG_INFO = YES; ONLY_ACTIVE_ARCH = YES; SDKROOT = iphoneos; @@ -635,7 +635,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 13.0; + IPHONEOS_DEPLOYMENT_TARGET = 15.0; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = iphoneos; SUPPORTED_PLATFORMS = iphoneos; diff --git a/zstandard/example/lib/main.dart b/zstandard/example/lib/main.dart index 437cd6ba..da08341b 100644 --- a/zstandard/example/lib/main.dart +++ b/zstandard/example/lib/main.dart @@ -17,55 +17,53 @@ class MyApp extends StatefulWidget { } class _MyAppState extends State { - final Uint8List _originalData = Uint8List.fromList( - [ - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30 - ], - ); + final Uint8List _originalData = Uint8List.fromList([ + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + ]); Uint8List? _compressedData; @@ -89,8 +87,7 @@ class _MyAppState extends State { // We also handle the message potentially returning null. try { platformVersion = - await _zstandard.getPlatformVersion() ?? - 'Unknown platform version'; + await _zstandard.getPlatformVersion() ?? 'Unknown platform version'; } on PlatformException { platformVersion = 'Failed to get platform version.'; } @@ -130,9 +127,7 @@ class _MyAppState extends State { Widget build(BuildContext context) { return MaterialApp( home: Scaffold( - appBar: AppBar( - title: const Text('Plugin example app'), - ), + appBar: AppBar(title: const Text('Plugin example app')), body: Center( child: Padding( padding: const EdgeInsets.all(15.0), diff --git a/zstandard/example/macos/Podfile b/zstandard/example/macos/Podfile index b52666a1..b15254e3 100644 --- a/zstandard/example/macos/Podfile +++ b/zstandard/example/macos/Podfile @@ -1,4 +1,4 @@ -platform :osx, '10.15' +platform :osx, '12.0' # CocoaPods analytics sends network stats synchronously affecting flutter build latency. ENV['COCOAPODS_DISABLE_STATS'] = 'true' diff --git a/zstandard/example/macos/Runner.xcodeproj/project.pbxproj b/zstandard/example/macos/Runner.xcodeproj/project.pbxproj index c61e8019..9f6ce0cc 100644 --- a/zstandard/example/macos/Runner.xcodeproj/project.pbxproj +++ b/zstandard/example/macos/Runner.xcodeproj/project.pbxproj @@ -557,7 +557,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 10.15; + MACOSX_DEPLOYMENT_TARGET = 12.0; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = macosx; SWIFT_COMPILATION_MODE = wholemodule; @@ -639,7 +639,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 10.15; + MACOSX_DEPLOYMENT_TARGET = 12.0; MTL_ENABLE_DEBUG_INFO = YES; ONLY_ACTIVE_ARCH = YES; SDKROOT = macosx; @@ -689,7 +689,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 10.15; + MACOSX_DEPLOYMENT_TARGET = 12.0; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = macosx; SWIFT_COMPILATION_MODE = wholemodule; diff --git a/zstandard/example/test/widget_test.dart b/zstandard/example/test/widget_test.dart index ec4dcbcf..05129612 100644 --- a/zstandard/example/test/widget_test.dart +++ b/zstandard/example/test/widget_test.dart @@ -18,8 +18,8 @@ void main() { // Verify that platform version is retrieved. expect( find.byWidgetPredicate( - (Widget widget) => widget is Text && - widget.data!.startsWith('Running on:'), + (Widget widget) => + widget is Text && widget.data!.startsWith('Running on:'), ), findsOneWidget, ); diff --git a/zstandard/example/web/zstd.js b/zstandard/example/web/zstd.js index 82898927..d1963b65 100644 --- a/zstandard/example/web/zstd.js +++ b/zstandard/example/web/zstd.js @@ -1,90 +1,53 @@ -var Module=typeof zstdWasmModule!="undefined"?zstdWasmModule:{};var ENVIRONMENT_IS_WEB=typeof window=="object";var ENVIRONMENT_IS_WORKER=typeof importScripts=="function";var ENVIRONMENT_IS_NODE=typeof process=="object"&&typeof process.versions=="object"&&typeof process.versions.node=="string"&&process.type!="renderer";if(ENVIRONMENT_IS_NODE){}var moduleOverrides=Object.assign({},Module);var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("fs");var nodePath=require("path");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);var ret=fs.readFileSync(filename);return ret};readAsync=(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);return new Promise((resolve,reject)=>{fs.readFile(filename,binary?undefined:"utf8",(err,data)=>{if(err)reject(err);else resolve(binary?data.buffer:data)})})};if(!Module["thisProgram"]&&process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}arguments_=process.argv.slice(2);if(typeof module!="undefined"){module["exports"]=Module}quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){if(ENVIRONMENT_IS_WORKER){scriptDirectory=self.location.href}else if(typeof document!="undefined"&&document.currentScript){scriptDirectory=document.currentScript.src}if(scriptDirectory.startsWith("blob:")){scriptDirectory=""}else{scriptDirectory=scriptDirectory.substr(0,scriptDirectory.replace(/[?#].*/,"").lastIndexOf("/")+1)}{if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=url=>{if(isFileURI(url)){return new Promise((resolve,reject)=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=()=>{if(xhr.status==200||xhr.status==0&&xhr.response){resolve(xhr.response);return}reject(xhr.status)};xhr.onerror=reject;xhr.send(null)})}return fetch(url,{credentials:"same-origin"}).then(response=>{if(response.ok){return response.arrayBuffer()}return Promise.reject(new Error(response.status+" : "+response.url))})}}}else{}var out=Module["print"]||console.log.bind(console);var err=Module["printErr"]||console.error.bind(console);Object.assign(Module,moduleOverrides);moduleOverrides=null;if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];var wasmBinary=Module["wasmBinary"];var wasmMemory;var ABORT=false;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;function updateMemoryViews(){var b=wasmMemory.buffer;Module["HEAP8"]=HEAP8=new Int8Array(b);Module["HEAP16"]=HEAP16=new Int16Array(b);Module["HEAPU8"]=HEAPU8=new Uint8Array(b);Module["HEAPU16"]=HEAPU16=new Uint16Array(b);Module["HEAP32"]=HEAP32=new Int32Array(b);Module["HEAPU32"]=HEAPU32=new Uint32Array(b);Module["HEAPF32"]=HEAPF32=new Float32Array(b);Module["HEAPF64"]=HEAPF64=new Float64Array(b)}var __ATPRERUN__=[];var __ATINIT__=[];var __ATPOSTRUN__=[];var runtimeInitialized=false;function preRun(){var preRuns=Module["preRun"];if(preRuns){if(typeof preRuns=="function")preRuns=[preRuns];preRuns.forEach(addOnPreRun)}callRuntimeCallbacks(__ATPRERUN__)}function initRuntime(){runtimeInitialized=true;callRuntimeCallbacks(__ATINIT__)}function postRun(){var postRuns=Module["postRun"];if(postRuns){if(typeof postRuns=="function")postRuns=[postRuns];postRuns.forEach(addOnPostRun)}callRuntimeCallbacks(__ATPOSTRUN__)}function addOnPreRun(cb){__ATPRERUN__.unshift(cb)}function addOnInit(cb){__ATINIT__.unshift(cb)}function addOnPostRun(cb){__ATPOSTRUN__.unshift(cb)}var runDependencies=0;var runDependencyWatcher=null;var dependenciesFulfilled=null;function addRunDependency(id){runDependencies++;Module["monitorRunDependencies"]?.(runDependencies)}function removeRunDependency(id){runDependencies--;Module["monitorRunDependencies"]?.(runDependencies);if(runDependencies==0){if(runDependencyWatcher!==null){clearInterval(runDependencyWatcher);runDependencyWatcher=null}if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);throw e}var dataURIPrefix="data:application/octet-stream;base64,";var isDataURI=filename=>filename.startsWith(dataURIPrefix);var isFileURI=filename=>filename.startsWith("file://");function findWasmBinary(){var f="zstd.wasm";if(!isDataURI(f)){return locateFile(f)}return f}var wasmBinaryFile;function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}function getBinaryPromise(binaryFile){if(!wasmBinary){return readAsync(binaryFile).then(response=>new Uint8Array(response),()=>getBinarySync(binaryFile))}return Promise.resolve().then(()=>getBinarySync(binaryFile))}function instantiateArrayBuffer(binaryFile,imports,receiver){return getBinaryPromise(binaryFile).then(binary=>WebAssembly.instantiate(binary,imports)).then(receiver,reason=>{err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)})}function instantiateAsync(binary,binaryFile,imports,callback){if(!binary&&typeof WebAssembly.instantiateStreaming=="function"&&!isDataURI(binaryFile)&&!isFileURI(binaryFile)&&!ENVIRONMENT_IS_NODE&&typeof fetch=="function"){return fetch(binaryFile,{credentials:"same-origin"}).then(response=>{var result=WebAssembly.instantiateStreaming(response,imports);return result.then(callback,function(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation");return instantiateArrayBuffer(binaryFile,imports,callback)})})}return instantiateArrayBuffer(binaryFile,imports,callback)}function getWasmImports(){return{a:wasmImports}}function createWasm(){var info=getWasmImports();function receiveInstance(instance,module){wasmExports=instance.exports;wasmMemory=wasmExports["c"];updateMemoryViews();addOnInit(wasmExports["d"]);removeRunDependency("wasm-instantiate");return wasmExports}addRunDependency("wasm-instantiate");function receiveInstantiationResult(result){receiveInstance(result["instance"])}if(Module["instantiateWasm"]){try{return Module["instantiateWasm"](info,receiveInstance)}catch(e){err(`Module.instantiateWasm callback failed with error: ${e}`);return false}}wasmBinaryFile??=findWasmBinary();instantiateAsync(wasmBinary,wasmBinaryFile,info,receiveInstantiationResult);return{}}var callRuntimeCallbacks=callbacks=>{callbacks.forEach(f=>f(Module))};var noExitRuntime=Module["noExitRuntime"]||true;var __emscripten_memcpy_js=(dest,src,num)=>HEAPU8.copyWithin(dest,src,src+num);var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var b=wasmMemory.buffer;var pages=(size-b.byteLength+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var wasmImports={b:__emscripten_memcpy_js,a:_emscripten_resize_heap};var wasmExports=createWasm();var ___wasm_call_ctors=()=>(___wasm_call_ctors=wasmExports["d"])();var _malloc=Module["_malloc"]=a0=>(_malloc=Module["_malloc"]=wasmExports["f"])(a0);var _free=Module["_free"]=a0=>(_free=Module["_free"]=wasmExports["g"])(a0);var _ZSTD_isError=Module["_ZSTD_isError"]=a0=>(_ZSTD_isError=Module["_ZSTD_isError"]=wasmExports["h"])(a0);var _ZSTD_compressBound=Module["_ZSTD_compressBound"]=a0=>(_ZSTD_compressBound=Module["_ZSTD_compressBound"]=wasmExports["i"])(a0);var _ZSTD_compress=Module["_ZSTD_compress"]=(a0,a1,a2,a3,a4)=>(_ZSTD_compress=Module["_ZSTD_compress"]=wasmExports["j"])(a0,a1,a2,a3,a4);var _ZSTD_getFrameContentSize=Module["_ZSTD_getFrameContentSize"]=(a0,a1)=>(_ZSTD_getFrameContentSize=Module["_ZSTD_getFrameContentSize"]=wasmExports["k"])(a0,a1);var _ZSTD_decompress=Module["_ZSTD_decompress"]=(a0,a1,a2,a3)=>(_ZSTD_decompress=Module["_ZSTD_decompress"]=wasmExports["l"])(a0,a1,a2,a3);var calledRun;var calledPrerun;dependenciesFulfilled=function runCaller(){if(!calledRun)run();if(!calledRun)dependenciesFulfilled=runCaller};function run(){if(runDependencies>0){return}if(!calledPrerun){calledPrerun=1;preRun();if(runDependencies>0){return}}function doRun(){if(calledRun)return;calledRun=1;Module["calledRun"]=1;if(ABORT)return;initRuntime();Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].pop()()}}run(); - -// Promise that resolves when the module is ready -let moduleReady = new Promise((resolve) => { - if (typeof Module !== 'undefined' && Module.calledRun) { - // Module already initialized - resolve(); - } else { - // Wait for module initialization - const originalOnRuntimeInitialized = Module.onRuntimeInitialized || function() {}; - Module.onRuntimeInitialized = function() { - originalOnRuntimeInitialized(); - resolve(); - }; +'use strict'; + +(() => { + const scriptUrl = document.currentScript?.src + ?? new URL('zstd.js', document.baseURI).href; + const workerUrl = new URL('zstd_worker.js', scriptUrl).href; + let worker = null; + let nextRequestId = 1; + const pending = new Map(); + + function failPendingRequests() { + for (const resolve of pending.values()) resolve(null); + pending.clear(); + worker?.terminate(); + worker = null; } -}); - -async function compressData(inputData, compressionLevel) { - await moduleReady; - - let inputPtr = Module._malloc(inputData.length); - Module.HEAPU8.set(inputData, inputPtr); - - let outputBufferSize = Number(Module._ZSTD_compressBound(inputData.length)); - let outputPtr = Module._malloc(outputBufferSize); - - let compressedSize = Number(Module._ZSTD_compress( - outputPtr, - outputBufferSize, - inputPtr, - inputData.length, - compressionLevel - )); - if (Module._ZSTD_isError(compressedSize) !== 0 || compressedSize <= 0) { - console.error('Compression error, error code: ', compressedSize); - Module._free(inputPtr); - Module._free(outputPtr); - return null; - } else { - let compressedData = new Uint8Array(Module.HEAPU8.buffer, outputPtr, compressedSize); - let out = compressedData.slice(0); - Module._free(inputPtr); - Module._free(outputPtr); - return out; + function getWorker() { + if (worker !== null) return worker; + worker = new Worker(workerUrl); + worker.onmessage = (event) => { + const resolve = pending.get(event.data.id); + if (resolve === undefined) return; + pending.delete(event.data.id); + resolve(event.data.result); + }; + worker.onerror = failPendingRequests; + worker.onmessageerror = failPendingRequests; + return worker; } -} - -async function decompressData(compressedData) { - await moduleReady; - - let compressedPtr = Module._malloc(compressedData.length); - Module.HEAPU8.set(compressedData, compressedPtr); - // ZSTD_getFrameContentSize returns these unsigned 64-bit sentinel values. - // JavaScript rounds them to the same Number values returned by Emscripten. - const ZSTD_CONTENTSIZE_UNKNOWN = 0xffffffffffffffff; - const ZSTD_CONTENTSIZE_ERROR = 0xfffffffffffffffe; - let decompressedSize = Number(Module._ZSTD_getFrameContentSize(compressedPtr, compressedData.length)); - if (decompressedSize === ZSTD_CONTENTSIZE_ERROR) { - console.error('Error in obtaining the original size of the data'); - Module._free(compressedPtr); - return null; + function run(operation, inputData, option) { + return new Promise((resolve) => { + const id = nextRequestId++; + try { + const inputCopy = inputData.slice(); + pending.set(id, resolve); + getWorker().postMessage( + { id, operation, inputBuffer: inputCopy.buffer, option }, + [inputCopy.buffer], + ); + } catch (_) { + pending.delete(id); + resolve(null); + } + }); } - const outputBufferSize = decompressedSize === ZSTD_CONTENTSIZE_UNKNOWN - ? compressedData.length * 20 - : decompressedSize; - let decompressedPtr = Module._malloc(outputBufferSize); - - let resultSize = Number(Module._ZSTD_decompress( - decompressedPtr, - outputBufferSize, - compressedPtr, - compressedData.length - )); - - if (Module._ZSTD_isError(resultSize) !== 0 || resultSize < 0) { - console.error('Decompression error, error code: ', resultSize); - Module._free(compressedPtr); - Module._free(decompressedPtr); - return null; - } else { - let decompressedData = new Uint8Array(Module.HEAPU8.buffer, decompressedPtr, resultSize); - let out = decompressedData.slice(0); - Module._free(compressedPtr); - Module._free(decompressedPtr); - return out; - } -} + globalThis.compressData = (inputData, compressionLevel) => + run('compress', inputData, compressionLevel); + globalThis.decompressData = (compressedData, maxOutputSize) => + run('decompress', compressedData, maxOutputSize); +})(); diff --git a/zstandard/example/web/zstd.wasm b/zstandard/example/web/zstd.wasm index 69a3403f..4731172c 100755 Binary files a/zstandard/example/web/zstd.wasm and b/zstandard/example/web/zstd.wasm differ diff --git a/zstandard/example/web/zstd_core.js b/zstandard/example/web/zstd_core.js new file mode 100644 index 00000000..6e4d1bda --- /dev/null +++ b/zstandard/example/web/zstd_core.js @@ -0,0 +1,175 @@ +var Module=typeof zstdWasmModule!="undefined"?zstdWasmModule:{};var ENVIRONMENT_IS_WEB=typeof window=="object";var ENVIRONMENT_IS_WORKER=typeof importScripts=="function";var ENVIRONMENT_IS_NODE=typeof process=="object"&&typeof process.versions=="object"&&typeof process.versions.node=="string"&&process.type!="renderer";if(ENVIRONMENT_IS_NODE){}var moduleOverrides=Object.assign({},Module);var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("fs");var nodePath=require("path");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);var ret=fs.readFileSync(filename);return ret};readAsync=(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);return new Promise((resolve,reject)=>{fs.readFile(filename,binary?undefined:"utf8",(err,data)=>{if(err)reject(err);else resolve(binary?data.buffer:data)})})};if(!Module["thisProgram"]&&process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}arguments_=process.argv.slice(2);if(typeof module!="undefined"){module["exports"]=Module}quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){if(ENVIRONMENT_IS_WORKER){scriptDirectory=self.location.href}else if(typeof document!="undefined"&&document.currentScript){scriptDirectory=document.currentScript.src}if(scriptDirectory.startsWith("blob:")){scriptDirectory=""}else{scriptDirectory=scriptDirectory.substr(0,scriptDirectory.replace(/[?#].*/,"").lastIndexOf("/")+1)}{if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=url=>{if(isFileURI(url)){return new Promise((resolve,reject)=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=()=>{if(xhr.status==200||xhr.status==0&&xhr.response){resolve(xhr.response);return}reject(xhr.status)};xhr.onerror=reject;xhr.send(null)})}return fetch(url,{credentials:"same-origin"}).then(response=>{if(response.ok){return response.arrayBuffer()}return Promise.reject(new Error(response.status+" : "+response.url))})}}}else{}var out=Module["print"]||console.log.bind(console);var err=Module["printErr"]||console.error.bind(console);Object.assign(Module,moduleOverrides);moduleOverrides=null;if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];var wasmBinary=Module["wasmBinary"];var wasmMemory;var ABORT=false;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;function updateMemoryViews(){var b=wasmMemory.buffer;Module["HEAP8"]=HEAP8=new Int8Array(b);Module["HEAP16"]=HEAP16=new Int16Array(b);Module["HEAPU8"]=HEAPU8=new Uint8Array(b);Module["HEAPU16"]=HEAPU16=new Uint16Array(b);Module["HEAP32"]=HEAP32=new Int32Array(b);Module["HEAPU32"]=HEAPU32=new Uint32Array(b);Module["HEAPF32"]=HEAPF32=new Float32Array(b);Module["HEAPF64"]=HEAPF64=new Float64Array(b)}var __ATPRERUN__=[];var __ATINIT__=[];var __ATPOSTRUN__=[];var runtimeInitialized=false;function preRun(){var preRuns=Module["preRun"];if(preRuns){if(typeof preRuns=="function")preRuns=[preRuns];preRuns.forEach(addOnPreRun)}callRuntimeCallbacks(__ATPRERUN__)}function initRuntime(){runtimeInitialized=true;callRuntimeCallbacks(__ATINIT__)}function postRun(){var postRuns=Module["postRun"];if(postRuns){if(typeof postRuns=="function")postRuns=[postRuns];postRuns.forEach(addOnPostRun)}callRuntimeCallbacks(__ATPOSTRUN__)}function addOnPreRun(cb){__ATPRERUN__.unshift(cb)}function addOnInit(cb){__ATINIT__.unshift(cb)}function addOnPostRun(cb){__ATPOSTRUN__.unshift(cb)}var runDependencies=0;var runDependencyWatcher=null;var dependenciesFulfilled=null;function addRunDependency(id){runDependencies++;Module["monitorRunDependencies"]?.(runDependencies)}function removeRunDependency(id){runDependencies--;Module["monitorRunDependencies"]?.(runDependencies);if(runDependencies==0){if(runDependencyWatcher!==null){clearInterval(runDependencyWatcher);runDependencyWatcher=null}if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);throw e}var dataURIPrefix="data:application/octet-stream;base64,";var isDataURI=filename=>filename.startsWith(dataURIPrefix);var isFileURI=filename=>filename.startsWith("file://");function findWasmBinary(){var f="zstd.wasm";if(!isDataURI(f)){return locateFile(f)}return f}var wasmBinaryFile;function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}function getBinaryPromise(binaryFile){if(!wasmBinary){return readAsync(binaryFile).then(response=>new Uint8Array(response),()=>getBinarySync(binaryFile))}return Promise.resolve().then(()=>getBinarySync(binaryFile))}function instantiateArrayBuffer(binaryFile,imports,receiver){return getBinaryPromise(binaryFile).then(binary=>WebAssembly.instantiate(binary,imports)).then(receiver,reason=>{err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)})}function instantiateAsync(binary,binaryFile,imports,callback){if(!binary&&typeof WebAssembly.instantiateStreaming=="function"&&!isDataURI(binaryFile)&&!isFileURI(binaryFile)&&!ENVIRONMENT_IS_NODE&&typeof fetch=="function"){return fetch(binaryFile,{credentials:"same-origin"}).then(response=>{var result=WebAssembly.instantiateStreaming(response,imports);return result.then(callback,function(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation");return instantiateArrayBuffer(binaryFile,imports,callback)})})}return instantiateArrayBuffer(binaryFile,imports,callback)}function getWasmImports(){return{a:wasmImports}}function createWasm(){var info=getWasmImports();function receiveInstance(instance,module){wasmExports=instance.exports;wasmMemory=wasmExports["c"];updateMemoryViews();addOnInit(wasmExports["d"]);removeRunDependency("wasm-instantiate");return wasmExports}addRunDependency("wasm-instantiate");function receiveInstantiationResult(result){receiveInstance(result["instance"])}if(Module["instantiateWasm"]){try{return Module["instantiateWasm"](info,receiveInstance)}catch(e){err(`Module.instantiateWasm callback failed with error: ${e}`);return false}}wasmBinaryFile??=findWasmBinary();instantiateAsync(wasmBinary,wasmBinaryFile,info,receiveInstantiationResult);return{}}var callRuntimeCallbacks=callbacks=>{callbacks.forEach(f=>f(Module))};var noExitRuntime=Module["noExitRuntime"]||true;var __emscripten_memcpy_js=(dest,src,num)=>HEAPU8.copyWithin(dest,src,src+num);var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var b=wasmMemory.buffer;var pages=(size-b.byteLength+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var wasmImports={b:__emscripten_memcpy_js,a:_emscripten_resize_heap};var wasmExports=createWasm();var ___wasm_call_ctors=()=>(___wasm_call_ctors=wasmExports["d"])();var _malloc=Module["_malloc"]=a0=>(_malloc=Module["_malloc"]=wasmExports["f"])(a0);var _free=Module["_free"]=a0=>(_free=Module["_free"]=wasmExports["g"])(a0);var _ZSTD_isError=Module["_ZSTD_isError"]=a0=>(_ZSTD_isError=Module["_ZSTD_isError"]=wasmExports["h"])(a0);var _ZSTD_compressBound=Module["_ZSTD_compressBound"]=a0=>(_ZSTD_compressBound=Module["_ZSTD_compressBound"]=wasmExports["i"])(a0);var _ZSTD_compress=Module["_ZSTD_compress"]=(a0,a1,a2,a3,a4)=>(_ZSTD_compress=Module["_ZSTD_compress"]=wasmExports["j"])(a0,a1,a2,a3,a4);var _ZSTD_createDStream=Module["_ZSTD_createDStream"]=()=>(_ZSTD_createDStream=Module["_ZSTD_createDStream"]=wasmExports["k"])();var _ZSTD_freeDStream=Module["_ZSTD_freeDStream"]=a0=>(_ZSTD_freeDStream=Module["_ZSTD_freeDStream"]=wasmExports["l"])(a0);var _ZSTD_DStreamOutSize=Module["_ZSTD_DStreamOutSize"]=()=>(_ZSTD_DStreamOutSize=Module["_ZSTD_DStreamOutSize"]=wasmExports["m"])();var _ZSTD_initDStream=Module["_ZSTD_initDStream"]=a0=>(_ZSTD_initDStream=Module["_ZSTD_initDStream"]=wasmExports["n"])(a0);var _ZSTD_decompressStream=Module["_ZSTD_decompressStream"]=(a0,a1,a2)=>(_ZSTD_decompressStream=Module["_ZSTD_decompressStream"]=wasmExports["o"])(a0,a1,a2);var calledRun;var calledPrerun;dependenciesFulfilled=function runCaller(){if(!calledRun)run();if(!calledRun)dependenciesFulfilled=runCaller};function run(){if(runDependencies>0){return}if(!calledPrerun){calledPrerun=1;preRun();if(runDependencies>0){return}}function doRun(){if(calledRun)return;calledRun=1;Module["calledRun"]=1;if(ABORT)return;initRuntime();Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].pop()()}}run(); + +// Promise that resolves when the module is ready +let moduleReady = new Promise((resolve) => { + if (typeof Module !== 'undefined' && Module.calledRun) { + // Module already initialized + resolve(); + } else { + // Wait for module initialization + const originalOnRuntimeInitialized = Module.onRuntimeInitialized || function() {}; + Module.onRuntimeInitialized = function() { + originalOnRuntimeInitialized(); + resolve(); + }; + } +}); + +async function compressData(inputData, compressionLevel) { + await moduleReady; + + if (!(inputData instanceof Uint8Array) + || !Number.isInteger(compressionLevel) + || compressionLevel < 1 + || compressionLevel > 22) { + return null; + } + + let inputPtr = 0; + let outputPtr = 0; + try { + inputPtr = Module._malloc(Math.max(inputData.length, 1)); + if (!inputPtr) return null; + Module.HEAPU8.set(inputData, inputPtr); + + const outputBufferSize = Number(Module._ZSTD_compressBound(inputData.length)); + if (!Number.isSafeInteger(outputBufferSize) || outputBufferSize <= 0) { + return null; + } + outputPtr = Module._malloc(outputBufferSize); + if (!outputPtr) return null; + + const compressedSize = Number(Module._ZSTD_compress( + outputPtr, + outputBufferSize, + inputPtr, + inputData.length, + compressionLevel + )); + if (Module._ZSTD_isError(compressedSize) !== 0 || compressedSize <= 0) { + console.error('Compression error, error code: ', compressedSize); + return null; + } + return new Uint8Array( + Module.HEAPU8.buffer, + outputPtr, + compressedSize + ).slice(); + } finally { + if (inputPtr) Module._free(inputPtr); + if (outputPtr) Module._free(outputPtr); + } +} + +async function decompressData(compressedData, maxOutputSize = 256 * 1024 * 1024) { + await moduleReady; + + if (!(compressedData instanceof Uint8Array) + || !Number.isSafeInteger(maxOutputSize) + || maxOutputSize < 0 + || compressedData.length === 0) { + return null; + } + + // ZSTD_inBuffer and ZSTD_outBuffer contain three wasm32 size_t/pointer + // fields each. Keep their allocation and field access local to the Worker. + const bufferStructSize = 3 * Uint32Array.BYTES_PER_ELEMENT; + let compressedPtr = 0; + let inputBufferPtr = 0; + let outputBufferPtr = 0; + let outputChunkPtr = 0; + let stream = 0; + try { + compressedPtr = Module._malloc(compressedData.length); + inputBufferPtr = Module._malloc(bufferStructSize); + outputBufferPtr = Module._malloc(bufferStructSize); + stream = Module._ZSTD_createDStream(); + if (!compressedPtr || !inputBufferPtr || !outputBufferPtr || !stream) { + return null; + } + Module.HEAPU8.set(compressedData, compressedPtr); + + const initialization = Number(Module._ZSTD_initDStream(stream)); + if (Module._ZSTD_isError(initialization) !== 0) return null; + + const recommendedChunkSize = Number(Module._ZSTD_DStreamOutSize()); + if (!Number.isSafeInteger(recommendedChunkSize) + || recommendedChunkSize <= 0) { + return null; + } + const outputChunkSize = Math.max( + 1, + Math.min(recommendedChunkSize, Math.max(maxOutputSize, 1)), + ); + outputChunkPtr = Module._malloc(outputChunkSize); + if (!outputChunkPtr) return null; + + const writeField = (structPtr, field, value) => { + Module.HEAPU32[(structPtr >>> 2) + field] = value; + }; + const readField = (structPtr, field) => + Module.HEAPU32[(structPtr >>> 2) + field]; + + writeField(inputBufferPtr, 0, compressedPtr); + writeField(inputBufferPtr, 1, compressedData.length); + writeField(inputBufferPtr, 2, 0); + + const chunks = []; + let totalLength = 0; + let previousInputPosition = -1; + let previousOutputLength = -1; + while (true) { + writeField(outputBufferPtr, 0, outputChunkPtr); + writeField(outputBufferPtr, 1, outputChunkSize); + writeField(outputBufferPtr, 2, 0); + + const remaining = Number(Module._ZSTD_decompressStream( + stream, + outputBufferPtr, + inputBufferPtr, + )); + if (Module._ZSTD_isError(remaining) !== 0) return null; + + const inputPosition = readField(inputBufferPtr, 2); + const produced = readField(outputBufferPtr, 2); + if (inputPosition > compressedData.length + || produced > outputChunkSize + || totalLength + produced > maxOutputSize) { + return null; + } + if (produced > 0) { + chunks.push(new Uint8Array( + Module.HEAPU8.buffer, + outputChunkPtr, + produced, + ).slice()); + totalLength += produced; + } + + const allInputConsumed = inputPosition === compressedData.length; + if (remaining === 0 && allInputConsumed) { + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + return result; + } + + const madeProgress = inputPosition !== previousInputPosition + || totalLength !== previousOutputLength; + if (!madeProgress || (allInputConsumed && produced === 0)) { + return null; + } + previousInputPosition = inputPosition; + previousOutputLength = totalLength; + } + } finally { + if (stream) Module._ZSTD_freeDStream(stream); + if (compressedPtr) Module._free(compressedPtr); + if (inputBufferPtr) Module._free(inputBufferPtr); + if (outputBufferPtr) Module._free(outputBufferPtr); + if (outputChunkPtr) Module._free(outputChunkPtr); + } +} diff --git a/zstandard/example/web/zstd_worker.js b/zstandard/example/web/zstd_worker.js new file mode 100644 index 00000000..38e076eb --- /dev/null +++ b/zstandard/example/web/zstd_worker.js @@ -0,0 +1,20 @@ +'use strict'; + +importScripts('zstd_core.js'); + +self.onmessage = async (event) => { + const { id, operation, inputBuffer, option } = event.data; + try { + const input = new Uint8Array(inputBuffer); + const result = operation === 'compress' + ? await compressData(input, option) + : await decompressData(input, option); + if (result === null) { + self.postMessage({ id, result: null }); + } else { + self.postMessage({ id, result }, [result.buffer]); + } + } catch (_) { + self.postMessage({ id, result: null }); + } +}; diff --git a/zstandard/lib/src/zstandard_ext.dart b/zstandard/lib/src/zstandard_ext.dart index e287c10d..d91838e1 100644 --- a/zstandard/lib/src/zstandard_ext.dart +++ b/zstandard/lib/src/zstandard_ext.dart @@ -26,9 +26,11 @@ extension ZstandardExt on Uint8List? { /// Decompresses this byte list (must be Zstandard-compressed data). /// /// Returns null if the receiver is null or decompression failed. - Future decompress() async { + Future decompress({ + int maxOutputSize = Zstandard.defaultMaxDecompressedSize, + }) async { var data = this; if (data == null) return null; - return Zstandard().decompress(data); + return Zstandard().decompress(data, maxOutputSize: maxOutputSize); } } diff --git a/zstandard/lib/zstandard.dart b/zstandard/lib/zstandard.dart index 64659f94..3ae6f00b 100644 --- a/zstandard/lib/zstandard.dart +++ b/zstandard/lib/zstandard.dart @@ -21,6 +21,10 @@ export 'src/zstandard_ext.dart'; /// /// See also [ZstandardExt] for extension methods on [Uint8List?]. class Zstandard { + /// Default maximum output accepted by [decompress]. + static const int defaultMaxDecompressedSize = + ZstandardPlatform.defaultMaxDecompressedSize; + static Zstandard? _instance; Zstandard._internal(); @@ -52,7 +56,26 @@ class Zstandard { /// Decompresses Zstandard-compressed [data]. /// - /// Returns the decompressed bytes, or null if decompression failed - /// (e.g. invalid or corrupted input). - Future decompress(Uint8List data) => instance.decompress(data); + /// Returns the decompressed bytes, or null if decompression failed, the + /// input is incomplete, or the output would exceed [maxOutputSize]. + Future decompress( + Uint8List data, { + int maxOutputSize = defaultMaxDecompressedSize, + }) async { + if (maxOutputSize < 0) return null; + final platform = instance; + if (platform is BoundedZstandardPlatform) { + return (platform as BoundedZstandardPlatform).decompressWithOptions( + data, + maxOutputSize: maxOutputSize, + ); + } + // Compatibility path for third-party implementations of the original + // platform interface. Official implementations enforce the limit while + // streaming and therefore never allocate the oversized result. + final result = await platform.decompress(data); + return result != null && result.lengthInBytes <= maxOutputSize + ? result + : null; + } } diff --git a/zstandard/test/error_handling_test.dart b/zstandard/test/error_handling_test.dart index bfa1fb3c..ced49744 100644 --- a/zstandard/test/error_handling_test.dart +++ b/zstandard/test/error_handling_test.dart @@ -6,13 +6,25 @@ import 'package:zstandard/zstandard.dart'; import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; /// Mock that returns null for decompress (simulates corrupted/invalid input). -class MockDecompressFails with MockPlatformInterfaceMixin implements ZstandardPlatform { +class MockDecompressFails + with MockPlatformInterfaceMixin + implements ZstandardPlatform { @override Future getPlatformVersion() => Future.value('Mock 1.0'); @override Future compress(Uint8List data, int compressionLevel) async { - return Uint8List.fromList([0x28, 0xb5, 0x2f, 0xfd, 0x00, 0x00, 0x01, 0x00, 0x00]); + return Uint8List.fromList([ + 0x28, + 0xb5, + 0x2f, + 0xfd, + 0x00, + 0x00, + 0x01, + 0x00, + 0x00, + ]); } @override @@ -20,12 +32,15 @@ class MockDecompressFails with MockPlatformInterfaceMixin implements ZstandardPl } /// Mock that returns null for compress (simulates compression failure). -class MockCompressFails with MockPlatformInterfaceMixin implements ZstandardPlatform { +class MockCompressFails + with MockPlatformInterfaceMixin + implements ZstandardPlatform { @override Future getPlatformVersion() => Future.value('Mock 1.0'); @override - Future compress(Uint8List data, int compressionLevel) async => null; + Future compress(Uint8List data, int compressionLevel) async => + null; @override Future decompress(Uint8List data) async => diff --git a/zstandard/test/properties_test.dart b/zstandard/test/properties_test.dart index ccf71925..930adb80 100644 --- a/zstandard/test/properties_test.dart +++ b/zstandard/test/properties_test.dart @@ -7,7 +7,9 @@ import 'package:zstandard/zstandard.dart'; import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; /// Mock platform that implements identity roundtrip for property tests. -class RoundtripMockPlatform with MockPlatformInterfaceMixin implements ZstandardPlatform { +class RoundtripMockPlatform + with MockPlatformInterfaceMixin + implements ZstandardPlatform { @override Future getPlatformVersion() => Future.value('mock'); @@ -33,46 +35,45 @@ void main() { addTearDown(() { ZstandardPlatform.instance = saved; }); - forAll( - binary(minLength: 0, maxLength: 2000), - (List data) async { - final input = Uint8List.fromList(data); - final compressed = await Zstandard().compress(input, 3); - if (compressed == null) return; - final decompressed = await Zstandard().decompress(compressed); - expect(decompressed, isNotNull); - expect(decompressed!.length, input.length); - expect(List.from(decompressed), data); - }, - maxExamples: 200, - ); + forAll(binary(minLength: 0, maxLength: 2000), (List data) async { + final input = Uint8List.fromList(data); + final compressed = await Zstandard().compress(input, 3); + if (compressed == null) return; + final decompressed = await Zstandard().decompress(compressed); + expect(decompressed, isNotNull); + expect(decompressed!.length, input.length); + expect(List.from(decompressed), data); + }, maxExamples: 200); }); - property('determinism with mock: same input and level produce same output', () { - final saved = ZstandardPlatform.instance; - Zstandard().instance; - ZstandardPlatform.instance = RoundtripMockPlatform(); - addTearDown(() { - ZstandardPlatform.instance = saved; - }); - forAll( - combine2( - binary(minLength: 1, maxLength: 500), - integer(min: 1, max: 22), - ), - (tuple) async { - final data = tuple.$1; - final level = tuple.$2; - final input = Uint8List.fromList(data); - final compressed1 = await Zstandard().compress(input, level); - final compressed2 = await Zstandard().compress(input, level); - expect(compressed1, isNotNull); - expect(compressed2, isNotNull); - expect(compressed1!.length, compressed2!.length); - expect(List.from(compressed1), List.from(compressed2)); - }, - maxExamples: 100, - ); - }); + property( + 'determinism with mock: same input and level produce same output', + () { + final saved = ZstandardPlatform.instance; + Zstandard().instance; + ZstandardPlatform.instance = RoundtripMockPlatform(); + addTearDown(() { + ZstandardPlatform.instance = saved; + }); + forAll( + combine2( + binary(minLength: 1, maxLength: 500), + integer(min: 1, max: 22), + ), + (tuple) async { + final data = tuple.$1; + final level = tuple.$2; + final input = Uint8List.fromList(data); + final compressed1 = await Zstandard().compress(input, level); + final compressed2 = await Zstandard().compress(input, level); + expect(compressed1, isNotNull); + expect(compressed2, isNotNull); + expect(compressed1!.length, compressed2!.length); + expect(List.from(compressed1), List.from(compressed2)); + }, + maxExamples: 100, + ); + }, + ); }); } diff --git a/zstandard/test/tests_exist_elsewhere_test.dart b/zstandard/test/tests_exist_elsewhere_test.dart deleted file mode 100644 index 1478dc75..00000000 --- a/zstandard/test/tests_exist_elsewhere_test.dart +++ /dev/null @@ -1,11 +0,0 @@ -// ignore_for_file: avoid_print - -import 'package:flutter_test/flutter_test.dart'; - -void main() { - test('Tell the user where to find the real tests', () { - print('---'); - print('WIP'); - print('---'); - }); -} \ No newline at end of file diff --git a/zstandard/test/zstandard_advanced_test.dart b/zstandard/test/zstandard_advanced_test.dart index d8980214..fcd5a21d 100644 --- a/zstandard/test/zstandard_advanced_test.dart +++ b/zstandard/test/zstandard_advanced_test.dart @@ -6,7 +6,9 @@ import 'package:zstandard/zstandard.dart'; import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; /// Mock that echoes data for roundtrip-style tests. -class EchoMockPlatform with MockPlatformInterfaceMixin implements ZstandardPlatform { +class EchoMockPlatform + with MockPlatformInterfaceMixin + implements ZstandardPlatform { @override Future getPlatformVersion() => Future.value('EchoMock 1.0'); @@ -147,7 +149,9 @@ void main() { test('multiple compressions in parallel', () async { final futures = List.generate(10, (i) { - final data = Uint8List.fromList(List.generate(1000, (j) => (i + j) % 256)); + final data = Uint8List.fromList( + List.generate(1000, (j) => (i + j) % 256), + ); return data.compress(compressionLevel: 3); }); final results = await Future.wait(futures); diff --git a/zstandard/test/zstandard_test.dart b/zstandard/test/zstandard_test.dart index 0f1d794c..2dfe2eb1 100644 --- a/zstandard/test/zstandard_test.dart +++ b/zstandard/test/zstandard_test.dart @@ -6,7 +6,9 @@ import 'package:zstandard/zstandard.dart'; import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; import 'package:zstandard/src/platform_manager.dart'; -class MockZstandardPlatform with MockPlatformInterfaceMixin implements ZstandardPlatform { +class MockZstandardPlatform + with MockPlatformInterfaceMixin + implements ZstandardPlatform { @override Future getPlatformVersion() => Future.value('MockPlatform 1.0'); @@ -21,6 +23,31 @@ class MockZstandardPlatform with MockPlatformInterfaceMixin implements Zstandard } } +class MockBoundedZstandardPlatform + with MockPlatformInterfaceMixin + implements ZstandardPlatform, BoundedZstandardPlatform { + int? receivedMaxOutputSize; + + @override + Future getPlatformVersion() => Future.value('BoundedMock 1.0'); + + @override + Future compress(Uint8List data, int compressionLevel) async => + data; + + @override + Future decompress(Uint8List data) async => data; + + @override + Future decompressWithOptions( + Uint8List data, { + int maxOutputSize = ZstandardPlatform.defaultMaxDecompressedSize, + }) async { + receivedMaxOutputSize = maxOutputSize; + return Uint8List.fromList([1, 2, 3]); + } +} + void main() { TestWidgetsFlutterBinding.ensureInitialized(); @@ -68,6 +95,38 @@ void main() { expect(decompressed, Uint8List.fromList([1, 2, 3, 4, 5])); }); + test( + 'legacy platform result is rejected when it exceeds the limit', + () async { + final result = await Zstandard().decompress( + Uint8List.fromList([0x7f]), + maxOutputSize: 4, + ); + expect(result, isNull); + }, + ); + + test('bounded platform receives the output limit', () async { + final platform = MockBoundedZstandardPlatform(); + ZstandardPlatform.instance = platform; + + final result = await Zstandard().decompress( + Uint8List.fromList([0x7f]), + maxOutputSize: 1234, + ); + + expect(result, Uint8List.fromList([1, 2, 3])); + expect(platform.receivedMaxOutputSize, 1234); + }); + + test('negative decompression limit is rejected', () async { + final result = await Zstandard().decompress( + Uint8List.fromList([0x7f]), + maxOutputSize: -1, + ); + expect(result, isNull); + }); + test('instance returns registered platform', () { final z = Zstandard(); expect(z.instance, isA()); diff --git a/zstandard_android/CHANGELOG.md b/zstandard_android/CHANGELOG.md index df7cc766..9b62e4ab 100644 --- a/zstandard_android/CHANGELOG.md +++ b/zstandard_android/CHANGELOG.md @@ -1,11 +1,19 @@ ## Unreleased +- Added byte-safe worker-isolate execution and bounded streaming decompression + for unknown-size and concatenated frames. - Updated the minimum supported SDK version to Flutter 3.44/Dart 3.12. - Migrated Android builds to Built-in Kotlin on AGP 9+. - Removed legacy Kotlin task configuration so the consuming app owns Kotlin compiler settings. - Added an AGP 8.11.1/Gradle 8.14 legacy example alongside the AGP 9.1.0 example. +- Removed obsolete Jetifier configuration and gave the legacy example and its + instrumentation sources a distinct, consistent package identity. +- Restricted the instrumentation-only JNI bridge to Debug builds so test + symbols are absent from production binaries. +- Analyze the independent legacy consumer in its own package context instead + of treating it as part of the plugin package. - Test debug and release APKs for armeabi-v7a, arm64-v8a, and x86_64, and run both Dart integration and native JNI instrumentation round-trip tests in CI. diff --git a/zstandard_android/README.md b/zstandard_android/README.md index 50685339..47dab528 100644 --- a/zstandard_android/README.md +++ b/zstandard_android/README.md @@ -10,7 +10,7 @@ Add the main plugin to your app; this package is included automatically via the ```yaml dependencies: - zstandard: ^1.3.29 + zstandard: ^1.5.0 ``` No extra Gradle or native setup is required for normal use. @@ -42,12 +42,15 @@ final decompressed = await compressed?.decompress(); - **ZstandardAndroid()** — Creates the Android platform implementation. - **compress(Uint8List data, int compressionLevel)** — Compresses `data` (level 1–22). Returns compressed bytes or `null`. -- **decompress(Uint8List data)** — Decompresses zstd-compressed data. Returns decompressed bytes or `null`. +- **decompressWithOptions(Uint8List data, {int maxOutputSize})** — Decompresses complete, concatenated, or unknown-size zstd frames with a bounded output (256 MiB by default). Invalid, truncated, or oversized input returns `null`. - **getPlatformVersion()** — Returns a platform identifier string. ## Architecture -This package uses Dart FFI to load `libzstandard_android.so` and call the Zstandard C API (`ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, `ZSTD_getFrameContentSize`). Heavy work may run in a background isolate to keep the UI responsive. +This package uses Dart FFI to load `libzstandard_android.so`. Public operations +send Dart-owned bytes to a worker isolate; native pointers never cross an +isolate boundary. Decompression uses zstd's streaming API and enforces the +configured output limit while producing bytes. The plugin does not pin an Android Gradle Plugin version. The consuming app owns that choice. The repository contains two Android consumers for CI: diff --git a/zstandard_android/analysis_options.yaml b/zstandard_android/analysis_options.yaml index a5744c1c..87f15a8e 100644 --- a/zstandard_android/analysis_options.yaml +++ b/zstandard_android/analysis_options.yaml @@ -1,3 +1,9 @@ +analyzer: + exclude: + - build/** + - android/** + # This is an independent consumer package and is analyzed separately. + - example_legacy/** include: package:flutter_lints/flutter.yaml # Additional information about this file can be found at diff --git a/zstandard_android/android/src/androidTest/kotlin/com/landamessenger/zstandard_android/ZstandardAndroidComprehensiveTest.kt b/zstandard_android/android/src/androidTest/kotlin/dev/vyp/zstandard_android/ZstandardAndroidComprehensiveTest.kt similarity index 81% rename from zstandard_android/android/src/androidTest/kotlin/com/landamessenger/zstandard_android/ZstandardAndroidComprehensiveTest.kt rename to zstandard_android/android/src/androidTest/kotlin/dev/vyp/zstandard_android/ZstandardAndroidComprehensiveTest.kt index a8dd31f3..0e26ad16 100644 --- a/zstandard_android/android/src/androidTest/kotlin/com/landamessenger/zstandard_android/ZstandardAndroidComprehensiveTest.kt +++ b/zstandard_android/android/src/androidTest/kotlin/dev/vyp/zstandard_android/ZstandardAndroidComprehensiveTest.kt @@ -2,9 +2,9 @@ package dev.vyp.zstandard_android import androidx.test.platform.app.InstrumentationRegistry import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.flutter.embedding.engine.plugins.FlutterPlugin import io.flutter.plugin.common.MethodChannel import org.junit.Assert.assertNotNull -import org.junit.Assert.assertTrue import org.junit.Test import org.junit.runner.RunWith @@ -18,14 +18,14 @@ class ZstandardAndroidComprehensiveTest { @Test fun pluginImplementsFlutterPlugin() { - val plugin = ZstandardAndroidPlugin() - assertTrue(plugin is io.flutter.embedding.engine.plugins.FlutterPlugin) + val plugin: FlutterPlugin = ZstandardAndroidPlugin() + assertNotNull(plugin) } @Test fun pluginImplementsMethodCallHandler() { - val plugin = ZstandardAndroidPlugin() - assertTrue(plugin is MethodChannel.MethodCallHandler) + val plugin: MethodChannel.MethodCallHandler = ZstandardAndroidPlugin() + assertNotNull(plugin) } @Test diff --git a/zstandard_android/android/src/androidTest/kotlin/com/landamessenger/zstandard_android/ZstandardAndroidInstrumentedTest.kt b/zstandard_android/android/src/androidTest/kotlin/dev/vyp/zstandard_android/ZstandardAndroidInstrumentedTest.kt similarity index 78% rename from zstandard_android/android/src/androidTest/kotlin/com/landamessenger/zstandard_android/ZstandardAndroidInstrumentedTest.kt rename to zstandard_android/android/src/androidTest/kotlin/dev/vyp/zstandard_android/ZstandardAndroidInstrumentedTest.kt index b37c5cbb..7851c749 100644 --- a/zstandard_android/android/src/androidTest/kotlin/com/landamessenger/zstandard_android/ZstandardAndroidInstrumentedTest.kt +++ b/zstandard_android/android/src/androidTest/kotlin/dev/vyp/zstandard_android/ZstandardAndroidInstrumentedTest.kt @@ -7,8 +7,8 @@ import org.junit.Test import org.junit.runner.RunWith /** - * Basic instrumented tests for the zstandard_android plugin. Native loading and - * compression/decompression are covered by the native round-trip test class. + * Basic Android-context tests for the zstandard_android plugin. Native loading + * and compression are covered by the native round-trip test class. */ @RunWith(AndroidJUnit4::class) class ZstandardAndroidInstrumentedTest { diff --git a/zstandard_android/example/analysis_options.yaml b/zstandard_android/example/analysis_options.yaml index 0d290213..5671d4c5 100644 --- a/zstandard_android/example/analysis_options.yaml +++ b/zstandard_android/example/analysis_options.yaml @@ -7,6 +7,10 @@ # The following line activates a set of recommended lints for Flutter apps, # packages, and plugins designed to encourage good coding practices. +analyzer: + exclude: + - build/** + - android/** include: package:flutter_lints/flutter.yaml linter: diff --git a/zstandard_android/example/android/app/build.gradle b/zstandard_android/example/android/app/build.gradle index abfe2bc4..05acd14b 100644 --- a/zstandard_android/example/android/app/build.gradle +++ b/zstandard_android/example/android/app/build.gradle @@ -15,7 +15,7 @@ android { } defaultConfig { - // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). + // Stable identifier used only by this example application. applicationId = "dev.vyp.zstandard_android_example" // You can update the following values to match your application needs. // For more information, see: https://flutter.dev/to/review-gradle-config. @@ -27,8 +27,8 @@ android { buildTypes { release { - // TODO: Add your own signing config for the release build. - // Signing with the debug keys for now, so `flutter run --release` works. + // Examples use debug signing so CI can build and run Release + // without production signing credentials. signingConfig = signingConfigs.debug } } diff --git a/zstandard_android/example/android/gradle.properties b/zstandard_android/example/android/gradle.properties index f534322e..1fc98750 100644 --- a/zstandard_android/example/android/gradle.properties +++ b/zstandard_android/example/android/gradle.properties @@ -1,5 +1,4 @@ org.gradle.jvmargs=-Xmx4G -XX:MaxMetaspaceSize=2G -XX:+HeapDumpOnOutOfMemoryError android.useAndroidX=true -android.enableJetifier=true android.builtInKotlin=true android.newDsl=false diff --git a/zstandard_android/example/lib/main.dart b/zstandard_android/example/lib/main.dart index b8a752f3..0b1587b1 100644 --- a/zstandard_android/example/lib/main.dart +++ b/zstandard_android/example/lib/main.dart @@ -17,55 +17,53 @@ class MyApp extends StatefulWidget { } class _MyAppState extends State { - final Uint8List _originalData = Uint8List.fromList( - [ - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30 - ], - ); + final Uint8List _originalData = Uint8List.fromList([ + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + ]); Uint8List? _compressedData; @@ -123,9 +121,7 @@ class _MyAppState extends State { Widget build(BuildContext context) { return MaterialApp( home: Scaffold( - appBar: AppBar( - title: const Text('Plugin example app'), - ), + appBar: AppBar(title: const Text('Plugin example app')), body: Center( child: Padding( padding: const EdgeInsets.all(15.0), diff --git a/zstandard_android/example_legacy/analysis_options.yaml b/zstandard_android/example_legacy/analysis_options.yaml index 0d290213..5671d4c5 100644 --- a/zstandard_android/example_legacy/analysis_options.yaml +++ b/zstandard_android/example_legacy/analysis_options.yaml @@ -7,6 +7,10 @@ # The following line activates a set of recommended lints for Flutter apps, # packages, and plugins designed to encourage good coding practices. +analyzer: + exclude: + - build/** + - android/** include: package:flutter_lints/flutter.yaml linter: diff --git a/zstandard_android/example_legacy/android/app/build.gradle b/zstandard_android/example_legacy/android/app/build.gradle index 1fed67ba..6ceba600 100644 --- a/zstandard_android/example_legacy/android/app/build.gradle +++ b/zstandard_android/example_legacy/android/app/build.gradle @@ -6,7 +6,7 @@ plugins { } android { - namespace = "dev.vyp.zstandard_android_example" + namespace = "dev.vyp.zstandard_android_legacy_example" compileSdk = flutter.compileSdkVersion ndkVersion = flutter.ndkVersion @@ -16,8 +16,8 @@ android { } defaultConfig { - // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). - applicationId = "dev.vyp.zstandard_android_example" + // Stable identifier used only by this example application. + applicationId = "dev.vyp.zstandard_android_legacy_example" // You can update the following values to match your application needs. // For more information, see: https://flutter.dev/to/review-gradle-config. minSdk = flutter.minSdkVersion @@ -28,8 +28,8 @@ android { buildTypes { release { - // TODO: Add your own signing config for the release build. - // Signing with the debug keys for now, so `flutter run --release` works. + // Examples use debug signing so CI can build and run Release + // without production signing credentials. signingConfig = signingConfigs.debug } } diff --git a/zstandard_android/example_legacy/android/app/src/main/AndroidManifest.xml b/zstandard_android/example_legacy/android/app/src/main/AndroidManifest.xml index 7a32fa74..01314f7a 100644 --- a/zstandard_android/example_legacy/android/app/src/main/AndroidManifest.xml +++ b/zstandard_android/example_legacy/android/app/src/main/AndroidManifest.xml @@ -1,6 +1,6 @@ { - final Uint8List _originalData = Uint8List.fromList( - [ - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30, - 10, - 20, - 30, - 4, - 3, - 3, - 10, - 20, - 30 - ], - ); + final Uint8List _originalData = Uint8List.fromList([ + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + 10, + 20, + 30, + 4, + 3, + 3, + 10, + 20, + 30, + ]); Uint8List? _compressedData; @@ -123,9 +121,7 @@ class _MyAppState extends State { Widget build(BuildContext context) { return MaterialApp( home: Scaffold( - appBar: AppBar( - title: const Text('Plugin example app'), - ), + appBar: AppBar(title: const Text('Plugin example app')), body: Center( child: Padding( padding: const EdgeInsets.all(15.0), diff --git a/zstandard_android/images/sample.png b/zstandard_android/images/sample.png index 5e6379b1..ea683936 100644 Binary files a/zstandard_android/images/sample.png and b/zstandard_android/images/sample.png differ diff --git a/zstandard_android/lib/zstandard_android.dart b/zstandard_android/lib/zstandard_android.dart index e006e7ab..e48f46d2 100644 --- a/zstandard_android/lib/zstandard_android.dart +++ b/zstandard_android/lib/zstandard_android.dart @@ -1,12 +1,12 @@ -import 'dart:async'; import 'dart:ffi'; import 'dart:io'; import 'dart:isolate'; -import 'package:ffi/ffi.dart'; import 'package:flutter/services.dart'; import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; +import 'package:zstandard_native/zstandard_native.dart' + show ZstandardNativeCodec; import 'package:zstandard_native/zstandard_native_bindings.dart'; export 'zstandard_ext.dart'; @@ -21,6 +21,7 @@ final DynamicLibrary _dylib = () { }(); final ZstandardNativeBindings _bindings = ZstandardNativeBindings(_dylib); +final ZstandardNativeCodec _codec = ZstandardNativeCodec(_bindings); bool _hasZstdFrameMagic(Uint8List data) { if (data.lengthInBytes < 4) { @@ -33,22 +34,12 @@ bool _hasZstdFrameMagic(Uint8List data) { (magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START; } -bool _isUnavailableContentSize(int size) { - // The generated bindings expose these C unsigned values as -1 and -2. - // Keep the unsigned representations as a compatibility guard for older - // generated bindings and runtimes that returned the raw 64-bit bit pattern. - return size == ZSTD_CONTENTSIZE_UNKNOWN || - size == ZSTD_CONTENTSIZE_ERROR || - size == 0xffffffffffffffff || - size == 0xfffffffffffffffe; -} - /// Android implementation of [ZstandardPlatform] using FFI and the native zstd library. /// -/// Loads libzstandard_android.so and uses ZSTD_compress, ZSTD_decompress, -/// ZSTD_compressBound, and ZSTD_getFrameContentSize. The main [zstandard] -/// plugin registers this implementation automatically on Android. -class ZstandardAndroid extends ZstandardPlatform { +/// Loads libzstandard_android.so and uses the shared byte-oriented codec. The +/// main [zstandard] plugin registers this implementation automatically. +class ZstandardAndroid extends ZstandardPlatform + implements BoundedZstandardPlatform { /// Creates the Android platform implementation. ZstandardAndroid(); @@ -70,87 +61,21 @@ class ZstandardAndroid extends ZstandardPlatform { } @override - Future compress(Uint8List data, int compressionLevel) async { - if (compressionLevel < 1 || compressionLevel > 22) { - return null; - } - - final int srcSize = data.lengthInBytes; - final Pointer src = malloc.allocate( - srcSize > 0 ? srcSize : 1, - ); - src.asTypedList(srcSize).setAll(0, data); - - final int dstCapacity = _bindings.ZSTD_compressBound(srcSize); - if (_bindings.ZSTD_isError(dstCapacity) != 0 || dstCapacity <= 0) { - malloc.free(src); - return null; - } - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int compressedSize = _bindings.ZSTD_compress( - dst.cast(), - dstCapacity, - src.cast(), - srcSize, - compressionLevel, - ); - - if (_bindings.ZSTD_isError(compressedSize) == 0 && compressedSize > 0) { - return Uint8List.fromList(dst.asTypedList(compressedSize)); - } else { - return null; - } - } finally { - malloc.free(src); - malloc.free(dst); - } - } + Future compress(Uint8List data, int compressionLevel) => + Isolate.run(() => _codec.compress(data, compressionLevel)); @override - Future decompress(Uint8List data) async { - // Avoid entering the native decoder for arbitrary input. Apart from - // being cheaper, this prevents malformed data from reaching an ABI - // boundary while the frame header is already known to be invalid. - if (!_hasZstdFrameMagic(data)) { - return null; - } - - final int compressedSize = data.lengthInBytes; - final Pointer src = malloc.allocate(compressedSize); - src.asTypedList(compressedSize).setAll(0, data); - - final int decompressedSizeExpected = _bindings.ZSTD_getFrameContentSize( - src.cast(), - compressedSize, - ); - if (_isUnavailableContentSize(decompressedSizeExpected)) { - malloc.free(src); - return null; - } - final int dstCapacity = decompressedSizeExpected > 0 - ? decompressedSizeExpected - : 1; - final Pointer dst = malloc.allocate(dstCapacity); + Future decompress(Uint8List data) => decompressWithOptions(data); - try { - final int decompressedSize = _bindings.ZSTD_decompress( - dst.cast(), - dstCapacity, - src.cast(), - compressedSize, - ); - - if (_bindings.ZSTD_isError(decompressedSize) != 0) { - return null; - } - return Uint8List.fromList(dst.asTypedList(decompressedSize)); - } finally { - malloc.free(src); - malloc.free(dst); - } - } + @override + Future decompressWithOptions( + Uint8List data, { + int maxOutputSize = ZstandardPlatform.defaultMaxDecompressedSize, + }) => Isolate.run( + () => _hasZstdFrameMagic(data) + ? _codec.decompress(data, maxOutputSize: maxOutputSize) + : null, + ); } int compress( @@ -168,182 +93,21 @@ int decompress( int compressedSize, ) => _bindings.ZSTD_decompress(dst, dstCapacity, src, compressedSize); +@Deprecated('Use ZstandardAndroid.compress with Dart-owned bytes instead.') Future compressAsync( Pointer dst, int dstCapacity, Pointer src, int srcSize, int compressionLevel, -) async { - final SendPort helperIsolateSendPort = await _getHelperIsolateSendPort(); - final int requestId = _nextCompressRequestId++; - final _CompressRequest request = _CompressRequest( - requestId, - dst, - dstCapacity, - src, - srcSize, - compressionLevel, - ); - final Completer completer = Completer(); - _compressRequests[requestId] = completer; - helperIsolateSendPort.send(request); - return completer.future; -} +) => Future.sync( + () => compress(dst, dstCapacity, src, srcSize, compressionLevel), +); +@Deprecated('Use ZstandardAndroid.decompress with Dart-owned bytes instead.') Future decompressAsync( Pointer dst, int dstCapacity, Pointer src, int compressedSize, -) async { - final SendPort helperIsolateSendPort = await _getHelperIsolateSendPort(); - final int requestId = _nextDecompressRequestId++; - final _DecompressRequest request = _DecompressRequest( - requestId, - dst, - dstCapacity, - src, - compressedSize, - ); - final Completer completer = Completer(); - _decompressRequests[requestId] = completer; - helperIsolateSendPort.send(request); - return completer.future; -} - -// ==== Communication between isolates for asynchronous compression and decompression ==== // - -/// Application for compression. -class _CompressRequest { - final int id; - final Pointer dst; - final int dstCapacity; - final Pointer src; - final int srcSize; - final int compressionLevel; - - const _CompressRequest( - this.id, - this.dst, - this.dstCapacity, - this.src, - this.srcSize, - this.compressionLevel, - ); -} - -/// Response with the compression result. -class _CompressResponse { - final int id; - final int result; - - const _CompressResponse(this.id, this.result); -} - -/// Request for decompression. -class _DecompressRequest { - final int id; - final Pointer dst; - final int dstCapacity; - final Pointer src; - final int compressedSize; - - const _DecompressRequest( - this.id, - this.dst, - this.dstCapacity, - this.src, - this.compressedSize, - ); -} - -/// Response with the result of the decompression. -class _DecompressResponse { - final int id; - final int result; - - const _DecompressResponse(this.id, this.result); -} - -/// Counters to identify compression and decompression requests. -int _nextCompressRequestId = 0; -int _nextDecompressRequestId = 0; - -/// Mapping of requests to completers for compression and decompression. -final Map> _compressRequests = >{}; -final Map> _decompressRequests = >{}; - -/// Port for sending requests to the auxiliary isolate. -// Start the worker lazily so importing the plugin cannot keep test processes -// alive when the async helper API is not used. -Future? _helperIsolateSendPort; - -Future _getHelperIsolateSendPort() => - _helperIsolateSendPort ??= () async { - final Completer completer = Completer(); - final ReceivePort receivePort = ReceivePort() - ..listen((dynamic data) { - if (data is SendPort) { - completer.complete(data); - return; - } - if (data is _CompressResponse) { - final Completer completer = _compressRequests[data.id]!; - _compressRequests.remove(data.id); - completer.complete(data.result); - return; - } - if (data is _DecompressResponse) { - final Completer completer = _decompressRequests[data.id]!; - _decompressRequests.remove(data.id); - completer.complete(data.result); - return; - } - throw UnsupportedError( - 'Message type not supported: ${data.runtimeType}', - ); - }); - - await Isolate.spawn((SendPort sendPort) async { - final ReceivePort helperReceivePort = ReceivePort() - ..listen((dynamic data) { - if (data is _CompressRequest) { - final int result = _bindings.ZSTD_compress( - data.dst, - data.dstCapacity, - data.src, - data.srcSize, - data.compressionLevel, - ); - final _CompressResponse response = _CompressResponse( - data.id, - result, - ); - sendPort.send(response); - return; - } - if (data is _DecompressRequest) { - final int result = _bindings.ZSTD_decompress( - data.dst, - data.dstCapacity, - data.src, - data.compressedSize, - ); - final _DecompressResponse response = _DecompressResponse( - data.id, - result, - ); - sendPort.send(response); - return; - } - throw UnsupportedError( - 'Message type not supported: ${data.runtimeType}', - ); - }); - - sendPort.send(helperReceivePort.sendPort); - }, receivePort.sendPort); - - return completer.future; - }(); +) => Future.sync(() => decompress(dst, dstCapacity, src, compressedSize)); diff --git a/zstandard_android/lib/zstandard_ext.dart b/zstandard_android/lib/zstandard_ext.dart index 38711c6c..b6f5d983 100644 --- a/zstandard_android/lib/zstandard_ext.dart +++ b/zstandard_android/lib/zstandard_ext.dart @@ -9,9 +9,12 @@ extension ZstandardExt on Uint8List? { return ZstandardAndroid().compress(data, compressionLevel); } - Future decompress() async { + Future decompress({int maxOutputSize = 256 * 1024 * 1024}) async { var data = this; if (data == null) return null; - return ZstandardAndroid().decompress(data); + return ZstandardAndroid().decompressWithOptions( + data, + maxOutputSize: maxOutputSize, + ); } } diff --git a/zstandard_android/zstd_build/CMakeLists.txt b/zstandard_android/zstd_build/CMakeLists.txt index 8430dd15..37505c19 100644 --- a/zstandard_android/zstd_build/CMakeLists.txt +++ b/zstandard_android/zstd_build/CMakeLists.txt @@ -14,18 +14,8 @@ if(EXISTS "${_LOCAL_ZSTD}/common") set(ZSTD_ROOT "${_LOCAL_ZSTD}") message(STATUS "Using local zstandard_native: ${ZSTD_ROOT}") else() - # 2. Look for zstandard_native in pub-cache (versioned dirs only, e.g. - # zstandard_native-1.4.0; do not match zstandard_native2 etc.). - file(GLOB NATIVE_PARENT_DIRS "${_PLUGIN_PARENT}/zstandard_native-*") - if(NATIVE_PARENT_DIRS) - list(GET NATIVE_PARENT_DIRS 0 NATIVE_PKG_DIR) - set(ZSTD_CANDIDATE "${NATIVE_PKG_DIR}/src/zstd") - if(EXISTS "${ZSTD_CANDIDATE}/common") - set(ZSTD_ROOT "${ZSTD_CANDIDATE}") - message(STATUS "Using zstandard_native from pub-cache: ${ZSTD_ROOT}") - endif() - endif() - # 3. Resolve zstandard_native from package_config.json (plugin or any parent, e.g. example app) + # 2. Resolve the exact dependency selected by pub. Never choose the first + # matching cache directory, which can silently compile a stale version. set(_SEARCH_PKG "${CMAKE_CURRENT_SOURCE_DIR}") foreach(_PKG_IGNORE RANGE 10) get_filename_component(_PARENT "${_SEARCH_PKG}/.." ABSOLUTE) @@ -54,7 +44,7 @@ else() set(_SEARCH_PKG "${_PARENT}") endforeach() if(NOT ZSTD_ROOT OR NOT EXISTS "${ZSTD_ROOT}/common") - message(FATAL_ERROR "zstd source not found. Ensure zstandard_native dependency is available (repo: ../../zstandard_native/src/zstd or pub-cache).") + message(FATAL_ERROR "zstd source not found. Set ZSTD_ROOT or run flutter pub get so package_config.json resolves zstandard_native.") endif() endif() @@ -67,16 +57,16 @@ file(GLOB ZSTD_SRC "${ZSTD_ROOT}/decompress/*.c" ) -# JNI entry points are compiled into the same Android shared object so -# instrumentation tests can exercise the exact library that Dart FFI loads. -# They are not part of the Dart-facing API. +add_library(zstandard_android SHARED ${ZSTD_SRC}) + +# JNI entry points are compiled into the Debug shared object so instrumentation +# tests exercise the exact library that Dart FFI loads. Production Profile and +# Release binaries must not expose test-only JNI symbols. if(ANDROID) - list(APPEND ZSTD_SRC - "${CMAKE_CURRENT_SOURCE_DIR}/../android/src/main/cpp/zstandard_android_test_bridge.c") + target_sources(zstandard_android PRIVATE + "$<$:${CMAKE_CURRENT_SOURCE_DIR}/../android/src/main/cpp/zstandard_android_test_bridge.c>") endif() -add_library(zstandard_android SHARED ${ZSTD_SRC}) - # The CI instrumentation suite also exercises the maximum compression level # on a software-only emulator. Keep debug native builds optimized enough that # this behavioral check does not spend minutes in the high-compression search. diff --git a/zstandard_cli/CHANGELOG.md b/zstandard_cli/CHANGELOG.md index 14965197..5ed19bff 100644 --- a/zstandard_cli/CHANGELOG.md +++ b/zstandard_cli/CHANGELOG.md @@ -1,3 +1,14 @@ +## Unreleased + +- Added bounded streaming decompression for unknown-size and concatenated + frames, and valid compression of empty input. +- Added structured CLI options, stdin/stdout pipelines, safe output naming and + overwrite refusal, stable exit codes, and CWD-independent library loading. +- Added complete native-symbol validation, finite microsecond benchmarks, and + static MSVC runtime linkage for bundled Windows libraries. +- Migrated platform-version detection from deprecated `LocalPlatform` APIs to + `NativePlatform.current`. + ## 1.5.0 - Dependencies Updated - Updated direct dependencies diff --git a/zstandard_cli/README.md b/zstandard_cli/README.md index cc2e456b..a75c8326 100644 --- a/zstandard_cli/README.md +++ b/zstandard_cli/README.md @@ -2,89 +2,81 @@ # zstandard_cli -The command-line implementation of [`zstandard`](https://pub.dev/packages/zstandard). +Pure-Dart Zstandard API and command-line tools for macOS, Windows, and Linux, +with bundled native libraries for x64 and arm64. -Zstandard (zstd) is a fast compression algorithm developed by Meta (formerly Facebook) for real-time scenarios. It provides a flexible range of compression levels, enabling both high-speed and high-compression-ratio options. This makes it ideal for applications needing efficient data storage, transmission, and backup solutions. - -`zstandard_cli` is a Dart package that binds to the high-performance Zstandard compression library, enabling both in-code and command-line compression and decompression. It leverages FFI to directly access native Zstandard functionality, allowing efficient data processing in Dart applications, from in-memory data compression to file handling via the CLI. - -**Available on macOS, Windows, and Linux desktops only**. - -| | [macOS](https://flutter.dev/desktop) | [Windows](https://flutter.dev/desktop) | [Linux](https://flutter.dev/desktop) | -|:-----------:|:------------------------------------:|:--------------------------------------:|:------------------------------------:| -| x64 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| arm64 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | -| Precompiled | Yes | Yes | Yes | - -> **Note:** This is a pure Dart package for desktop usage. For Flutter, please see the [zstandard](https://pub.dev/packages/zstandard) plugin. - -## Basic Usage +## Dart API ```dart -void main() async { - var cli = ZstandardCLI(); - - final originalData = Uint8List.fromList([...]); - - final compressed = await cli.compress(originalData, compressionLevel: 3); - - final decompressed = await cli.decompress(compressed ?? Uint8List(0)); -} +final codec = ZstandardCLI(); +final compressed = await codec.compress(data, compressionLevel: 3); +final decompressed = compressed == null + ? null + : await codec.decompress( + compressed, + maxOutputSize: 64 * 1024 * 1024, + ); ``` -With extensions: +The `Uint8List?` extensions expose the same `compress` and `decompress` +options. Compression levels are 1–22 and empty input produces a valid frame. +Decompression supports concatenated frames and frames without a declared +content size. It returns `null` on failure and enforces a 256 MiB output limit +by default. -```dart -void main() async { - final originalData = Uint8List.fromList([...]); +## Command line - final compressed = await originalData.compress(compressionLevel: 3); +After `dart pub global activate zstandard_cli`, use the installed executables: - final decompressed = await compressed.decompress(); -} +```bash +zstandard-compress --level 5 input.bin +zstandard-decompress input.bin.zstd ``` -## CLI Usage +The same commands can be run from a package checkout: ```bash -dart run zstandard_cli:compress any_file 3 +dart run zstandard_cli:compress --level 5 input.bin +dart run zstandard_cli:decompress input.bin.zstd +``` + +Common options: -dart run zstandard_cli:decompress any_file.zstd +```text +-o, --output PATH output path, or - for stdout +-f, --force replace an existing output file +-h, --help show help + --version show the package version ``` -## API +Compression also accepts `-l, --level 1..22`. Decompression accepts +`-m, --max-output-size BYTES`. Use `-` as the input to read stdin; it defaults +to stdout, so pipelines remain binary-clean. File compression appends `.zstd`. +File decompression strips `.zstd`, or appends `.out` when the input has no such +suffix. Existing outputs are refused unless `--force` is present; an output +that aliases the input is always refused. -- **ZstandardCLI()** — Creates a CLI instance. The native library is loaded once per process. -- **compress(Uint8List data, {int compressionLevel = 3})** — Compresses `data` (level 1–22). Returns compressed bytes or `null`. -- **decompress(Uint8List data)** — Decompresses zstd-compressed data. Returns decompressed bytes or `null`. -- **getPlatformVersion()** — Returns a string like `"macOS 14.0"` or `"Windows 10"`. +Exit codes are 0 for success, 1 for I/O or codec failure, and 2 for invalid +arguments or a refused overwrite. -Extensions on `Uint8List?`: **compress({int compressionLevel = 3})** and **decompress()**; they return `null` when the receiver is null. +## Native library resolution -## Testing +The package resolves the library through the active Dart package +configuration, independent of the process working directory. +`ZSTANDARD_CLI_LIBRARY` can name an explicit library for testing or custom +deployment; an executable-adjacent library is also supported for compiled +applications. Windows bundles use the static MSVC runtime and do not require a +matching redistributable solely for this library. -From the package directory: +## Testing ```bash dart test +dart analyze ``` -Tests run only on supported platforms (macOS, Windows, Linux). They cover small/large/empty data, compression levels, and null-safe extensions. - -## Troubleshooting - -- **Library not found**: Ensure you are on macOS, Windows, or Linux (x64 or arm64). Update the package with `dart pub upgrade zstandard_cli`. -- **Compress/decompress returns null**: Check that input is valid; for decompress, ensure the data is a complete zstd frame. - -See the [documentation](https://github.com/vypdev/zstandard/tree/master/docs) for more. - ---- - -The images provided below illustrate how to use `zstandard_cli` for compression and decompression on different platforms. +Tests verify empty, unknown-size and concatenated frames, output bounds, +symbol exports, CWD-independent loading, file collisions, and CLI exit codes. -

-

-

-

-

-

+See the repository [CLI guide](https://github.com/vypdev/zstandard/blob/master/docs/platforms/cli.md) +for more detail. diff --git a/zstandard_cli/benchmark/benchmark_suite.dart b/zstandard_cli/benchmark/benchmark_suite.dart index 8c2a0dfc..b88408bd 100644 --- a/zstandard_cli/benchmark/benchmark_suite.dart +++ b/zstandard_cli/benchmark/benchmark_suite.dart @@ -1,6 +1,7 @@ // ignore_for_file: avoid_print /// Reusable benchmark suite for zstandard_cli. Outputs JSON for regression detection. /// Run: dart run benchmark/benchmark_suite.dart [--output=path.json] +library; import 'dart:convert'; import 'dart:io'; @@ -51,39 +52,58 @@ Future> runAll({int runs = 5}) async { final decompressed = await cli.decompress(compressed); if (decompressed == null || decompressed.length != data.length) continue; - int compressSumMs = 0; - int decompressSumMs = 0; + var compressSumUs = 0; + var decompressSumUs = 0; for (var i = 0; i < runs; i++) { final sw = Stopwatch()..start(); final c = await cli.compress(data, compressionLevel: level); sw.stop(); - compressSumMs += sw.elapsedMilliseconds; - if (c == null) continue; + compressSumUs += + sw.elapsedMicroseconds == 0 ? 1 : sw.elapsedMicroseconds; + if (c == null) { + throw StateError('Compression failed for ${size}B at level $level'); + } sw.reset(); sw.start(); - await cli.decompress(c); + final d = await cli.decompress(c); sw.stop(); - decompressSumMs += sw.elapsedMilliseconds; + decompressSumUs += + sw.elapsedMicroseconds == 0 ? 1 : sw.elapsedMicroseconds; + if (d == null || !_bytesEqual(d, data)) { + throw StateError('Roundtrip failed for ${size}B at level $level'); + } } - final compressMs = compressSumMs / runs; - final decompressMs = decompressSumMs / runs; + final compressSeconds = + compressSumUs / runs / Duration.microsecondsPerSecond; + final decompressSeconds = + decompressSumUs / runs / Duration.microsecondsPerSecond; final sizeMb = size / (1024 * 1024); - final compressMbS = sizeMb / (compressMs / 1000); - final decompressMbS = sizeMb / (decompressMs / 1000); - results.add(BenchmarkResult( - name: '${size}B_L$level', - compressThroughputMBps: compressMbS, - decompressThroughputMBps: decompressMbS, - iterations: runs, - dataSizeBytes: size, - level: level, - )); + final compressMbS = sizeMb / compressSeconds; + final decompressMbS = sizeMb / decompressSeconds; + results.add( + BenchmarkResult( + name: '${size}B_L$level', + compressThroughputMBps: compressMbS, + decompressThroughputMBps: decompressMbS, + iterations: runs, + dataSizeBytes: size, + level: level, + ), + ); } } return results; } +bool _bytesEqual(Uint8List first, Uint8List second) { + if (first.length != second.length) return false; + for (var index = 0; index < first.length; index++) { + if (first[index] != second[index]) return false; + } + return true; +} + void main(List args) async { final outputIndex = args.indexWhere((a) => a.startsWith('--output=')); final outputPath = outputIndex >= 0 diff --git a/zstandard_cli/benchmark/compression_benchmark.dart b/zstandard_cli/benchmark/compression_benchmark.dart index 7370a3f9..bf94c416 100644 --- a/zstandard_cli/benchmark/compression_benchmark.dart +++ b/zstandard_cli/benchmark/compression_benchmark.dart @@ -5,6 +5,7 @@ /// /// Use for regression detection: run before/after changes and compare /// throughput (MB/s) and roundtrip correctness. +library; import 'dart:typed_data'; @@ -38,26 +39,35 @@ void main() async { // Timed runs const runs = 5; - int compressSumMs = 0; - int decompressSumMs = 0; + var compressSumUs = 0; + var decompressSumUs = 0; for (var i = 0; i < runs; i++) { final sw = Stopwatch()..start(); final c = await cli.compress(data, compressionLevel: level); sw.stop(); - compressSumMs += sw.elapsedMilliseconds; - if (c == null) continue; + compressSumUs += + sw.elapsedMicroseconds == 0 ? 1 : sw.elapsedMicroseconds; + if (c == null) { + throw StateError('Compression failed for ${size}B at level $level'); + } sw.reset(); sw.start(); - await cli.decompress(c); + final d = await cli.decompress(c); sw.stop(); - decompressSumMs += sw.elapsedMilliseconds; + decompressSumUs += + sw.elapsedMicroseconds == 0 ? 1 : sw.elapsedMicroseconds; + if (d == null || !_bytesEqual(d, data)) { + throw StateError('Roundtrip failed for ${size}B at level $level'); + } } - final compressMs = compressSumMs / runs; - final decompressMs = decompressSumMs / runs; + final compressSeconds = + compressSumUs / runs / Duration.microsecondsPerSecond; + final decompressSeconds = + decompressSumUs / runs / Duration.microsecondsPerSecond; final sizeMb = size / (1024 * 1024); - final compressMbS = sizeMb / (compressMs / 1000); - final decompressMbS = sizeMb / (decompressMs / 1000); + final compressMbS = sizeMb / compressSeconds; + final decompressMbS = sizeMb / decompressSeconds; final key = '${size}B_L$level'; results[key] = 'compress ${compressMbS.toStringAsFixed(2)} MB/s, decompress ${decompressMbS.toStringAsFixed(2)} MB/s'; @@ -68,3 +78,11 @@ void main() async { print('Done. Use these numbers as baseline for regression detection.'); } + +bool _bytesEqual(Uint8List first, Uint8List second) { + if (first.length != second.length) return false; + for (var index = 0; index < first.length; index++) { + if (first[index] != second[index]) return false; + } + return true; +} diff --git a/zstandard_cli/bin/compress.dart b/zstandard_cli/bin/compress.dart index 65cbd814..491c8641 100644 --- a/zstandard_cli/bin/compress.dart +++ b/zstandard_cli/bin/compress.dart @@ -1,52 +1,6 @@ import 'dart:io'; -import 'package:zstandard_cli/src/utils/constants.dart'; -import 'package:zstandard_cli/src/utils/size_utils.dart'; -import 'package:zstandard_cli/zstandard_cli.dart'; +import 'package:zstandard_cli/src/cli_runner.dart'; -void main(List args) async { - print('===================== 📦 zstandard_cli =====================\n'); - - final cli = ZstandardCLI(); - print('Running on: ${await cli.getPlatformVersion()}\n'); - - final filePath = args.firstOrNull; - - int compressionLevel = 3; - if (args.length > 1) { - String cl = args[1] as String? ?? '3'; - compressionLevel = int.tryParse(cl) ?? 3; - } - - if (filePath == null) { - print('Path not found: $filePath'); - return; - } - final file = File(filePath); - if (!file.existsSync()) { - print('File not found: $filePath'); - return; - } - - print('Compressing ${file.path} ${await getFileSize(file)}'); - - final compressed = await cli.compress( - file.readAsBytesSync(), - compressionLevel: compressionLevel, - ); - if (compressed == null) { - print('Error compressing: $filePath'); - return; - } - - final compressedFilePath = '$filePath$extension'; - final compressedFile = File(compressedFilePath); - compressedFile.writeAsBytesSync(compressed); - if (!compressedFile.existsSync()) { - print('Compressed file not found: $compressedFilePath'); - return; - } - - print( - 'Compressed ${compressedFile.path} ${await getFileSize(compressedFile)} \n'); -} +Future main(List arguments) async => + exitCode = await runCompressCommand(arguments); diff --git a/zstandard_cli/bin/decompress.dart b/zstandard_cli/bin/decompress.dart index 42bf9665..5057d942 100644 --- a/zstandard_cli/bin/decompress.dart +++ b/zstandard_cli/bin/decompress.dart @@ -1,48 +1,6 @@ import 'dart:io'; -import 'package:zstandard_cli/src/utils/constants.dart'; -import 'package:zstandard_cli/src/utils/size_utils.dart'; -import 'package:zstandard_cli/zstandard_cli.dart'; +import 'package:zstandard_cli/src/cli_runner.dart'; -void main(List args) async { - print('===================== 📦 zstandard_cli =====================\n'); - - final cli = ZstandardCLI(); - print('Running on: ${await cli.getPlatformVersion()}\n'); - - final filePath = args.firstOrNull; - - if (filePath == null) { - print('Path not found: $filePath'); - return; - } - final file = File(filePath); - if (!file.existsSync()) { - print('File not found: $filePath'); - return; - } - - print('Decompressing ${file.path} ${await getFileSize(file)}'); - - final compressed = await cli.decompress( - file.readAsBytesSync(), - ); - if (compressed == null) { - print('Error decompressing: $filePath'); - return; - } - - final decompressedFilePath = filePath.endsWith(extension) - ? filePath.substring(0, filePath.length - extension.length) - : filePath; - - final decompressedFile = File(decompressedFilePath); - decompressedFile.writeAsBytesSync(compressed); - if (!decompressedFile.existsSync()) { - print('Decompressed file not found: $decompressedFilePath'); - return; - } - - print( - 'Decompressed ${decompressedFile.path} ${await getFileSize(decompressedFile)} \n'); -} +Future main(List arguments) async => + exitCode = await runDecompressCommand(arguments); diff --git a/zstandard_cli/builders/windows_arm/CMakeLists.txt b/zstandard_cli/builders/windows_arm/CMakeLists.txt index 11bfa298..874b6a3b 100644 --- a/zstandard_cli/builders/windows_arm/CMakeLists.txt +++ b/zstandard_cli/builders/windows_arm/CMakeLists.txt @@ -1,7 +1,10 @@ # The Flutter tooling requires that developers have CMake 3.10 or later # installed. You should not increase this version, as doing so will cause # the plugin to fail to compile for some customers of the plugin. -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.15) + +cmake_policy(SET CMP0091 NEW) +set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") project(zstandard_windows_library VERSION 0.0.1 LANGUAGES C) diff --git a/zstandard_cli/builders/windows_x64/CMakeLists.txt b/zstandard_cli/builders/windows_x64/CMakeLists.txt index 11bfa298..874b6a3b 100644 --- a/zstandard_cli/builders/windows_x64/CMakeLists.txt +++ b/zstandard_cli/builders/windows_x64/CMakeLists.txt @@ -1,7 +1,10 @@ # The Flutter tooling requires that developers have CMake 3.10 or later # installed. You should not increase this version, as doing so will cause # the plugin to fail to compile for some customers of the plugin. -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.15) + +cmake_policy(SET CMP0091 NEW) +set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") project(zstandard_windows_library VERSION 0.0.1 LANGUAGES C) diff --git a/zstandard_cli/lib/src/cli_runner.dart b/zstandard_cli/lib/src/cli_runner.dart new file mode 100644 index 00000000..6292f09d --- /dev/null +++ b/zstandard_cli/lib/src/cli_runner.dart @@ -0,0 +1,279 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:path/path.dart' as path; + +import 'utils/constants.dart'; +import 'zstandard_cli_base.dart'; + +/// Version of the command-line interface. +const String zstandardCliVersion = '1.5.0'; + +/// Runs the compression command and returns a process exit code. +Future runCompressCommand( + List arguments, { + Stream>? standardInput, + IOSink? standardOutput, + IOSink? standardError, +}) => + _runCommand( + _Operation.compress, + arguments, + standardInput: standardInput, + standardOutput: standardOutput, + standardError: standardError, + ); + +/// Runs the decompression command and returns a process exit code. +Future runDecompressCommand( + List arguments, { + Stream>? standardInput, + IOSink? standardOutput, + IOSink? standardError, +}) => + _runCommand( + _Operation.decompress, + arguments, + standardInput: standardInput, + standardOutput: standardOutput, + standardError: standardError, + ); + +enum _Operation { compress, decompress } + +final class _CommandOptions { + _CommandOptions({ + required this.input, + required this.output, + required this.force, + required this.compressionLevel, + required this.maxOutputSize, + required this.showHelp, + required this.showVersion, + }); + + final String? input; + final String? output; + final bool force; + final int compressionLevel; + final int maxOutputSize; + final bool showHelp; + final bool showVersion; +} + +Future _runCommand( + _Operation operation, + List arguments, { + Stream>? standardInput, + IOSink? standardOutput, + IOSink? standardError, +}) async { + final inputStream = standardInput ?? stdin; + final outputSink = standardOutput ?? stdout; + final errorSink = standardError ?? stderr; + + final _CommandOptions options; + try { + options = _parseArguments(operation, arguments); + } on FormatException catch (error) { + errorSink.writeln('Error: ${error.message}'); + errorSink.writeln(_usage(operation)); + return 2; + } + + if (options.showHelp) { + outputSink.writeln(_usage(operation)); + return 0; + } + if (options.showVersion) { + outputSink.writeln('zstandard_cli $zstandardCliVersion'); + return 0; + } + + final inputPath = options.input!; + final outputPath = options.output ?? _defaultOutputPath(operation, inputPath); + + if (inputPath != '-' && outputPath != '-') { + final outputFile = File(outputPath); + final aliasesInput = outputFile.existsSync() && + File(inputPath).existsSync() && + FileSystemEntity.identicalSync(inputPath, outputPath); + if (_samePath(inputPath, outputPath) || aliasesInput) { + errorSink.writeln( + 'Error: refusing to overwrite the input file. Choose another output ' + 'with --output.', + ); + return 2; + } + if (outputFile.existsSync() && !options.force) { + errorSink.writeln( + 'Error: output already exists: $outputPath (use --force to replace it)', + ); + return 2; + } + } + + try { + final input = inputPath == '-' + ? await _readAll(inputStream) + : await File(inputPath).readAsBytes(); + final cli = ZstandardCLI(); + final Uint8List? result; + if (operation == _Operation.compress) { + result = await cli.compress( + input, + compressionLevel: options.compressionLevel, + ); + } else { + result = await cli.decompress( + input, + maxOutputSize: options.maxOutputSize, + ); + } + + if (result == null) { + errorSink.writeln( + 'Error: ${operation.name}ion failed. The input may be invalid or the ' + 'configured limit may be too small.', + ); + return 1; + } + + if (outputPath == '-') { + outputSink.add(result); + await outputSink.flush(); + } else { + await File(outputPath).writeAsBytes(result, flush: true); + errorSink.writeln( + '${operation.name}ed ${result.length} bytes to $outputPath', + ); + } + return 0; + } on FileSystemException catch (error) { + errorSink.writeln( + 'Error: ${error.message}${error.path == null ? '' : ': ${error.path}'}', + ); + return 1; + } on Object catch (error) { + errorSink.writeln('Error: $error'); + return 1; + } +} + +_CommandOptions _parseArguments(_Operation operation, List arguments) { + String? input; + String? output; + var force = false; + var compressionLevel = 3; + var maxOutputSize = 256 * 1024 * 1024; + var showHelp = false; + var showVersion = false; + var positionalOnly = false; + + String valueAfter(int index, String option) { + if (index + 1 >= arguments.length) { + throw FormatException('$option requires a value'); + } + return arguments[index + 1]; + } + + for (var index = 0; index < arguments.length; index++) { + final argument = arguments[index]; + if (!positionalOnly && argument == '--') { + positionalOnly = true; + } else if (!positionalOnly && (argument == '-h' || argument == '--help')) { + showHelp = true; + } else if (!positionalOnly && argument == '--version') { + showVersion = true; + } else if (!positionalOnly && (argument == '-f' || argument == '--force')) { + force = true; + } else if (!positionalOnly && + (argument == '-o' || argument == '--output')) { + output = valueAfter(index, argument); + index++; + } else if (!positionalOnly && + operation == _Operation.compress && + (argument == '-l' || argument == '--level')) { + final rawLevel = valueAfter(index, argument); + compressionLevel = int.tryParse(rawLevel) ?? + (throw FormatException('invalid compression level: $rawLevel')); + index++; + } else if (!positionalOnly && + operation == _Operation.decompress && + (argument == '-m' || argument == '--max-output-size')) { + final rawLimit = valueAfter(index, argument); + maxOutputSize = int.tryParse(rawLimit) ?? + (throw FormatException('invalid maximum output size: $rawLimit')); + index++; + } else if (!positionalOnly && argument.startsWith('-') && argument != '-') { + throw FormatException('unknown option: $argument'); + } else if (input == null) { + input = argument; + } else { + throw const FormatException('exactly one input path is required'); + } + } + + if (!showHelp && !showVersion && input == null) { + throw const FormatException('an input path or - for stdin is required'); + } + if (compressionLevel < 1 || compressionLevel > 22) { + throw const FormatException('compression level must be between 1 and 22'); + } + if (maxOutputSize <= 0) { + throw const FormatException( + 'maximum output size must be greater than zero', + ); + } + if (input == '-' && output == null) output = '-'; + + return _CommandOptions( + input: input, + output: output, + force: force, + compressionLevel: compressionLevel, + maxOutputSize: maxOutputSize, + showHelp: showHelp, + showVersion: showVersion, + ); +} + +String _defaultOutputPath(_Operation operation, String input) { + if (operation == _Operation.compress) return '$input$extension'; + if (input.endsWith(extension) && input.length > extension.length) { + return input.substring(0, input.length - extension.length); + } + return '$input.out'; +} + +bool _samePath(String first, String second) => + path.equals(path.canonicalize(first), path.canonicalize(second)); + +Future _readAll(Stream> source) async { + final builder = BytesBuilder(copy: false); + await for (final chunk in source) { + builder.add(chunk); + } + return builder.takeBytes(); +} + +String _usage(_Operation operation) { + final executable = operation == _Operation.compress + ? 'zstandard-compress' + : 'zstandard-decompress'; + final operationOptions = operation == _Operation.compress + ? ' -l, --level LEVEL Compression level (1-22; default: 3)\n' + : ' -m, --max-output-size BYTES Maximum decompressed size (default: 268435456)\n'; + return '''Usage: $executable [options] + +Use - as input to read stdin. Stdin defaults to stdout; file input defaults to +$extension for compression and to the stripped extension (or .out) +for decompression. + +Options: + -o, --output PATH Output path, or - for stdout + -f, --force Replace an existing output file +$operationOptions -h, --help Show this help + --version Show the CLI version'''; +} diff --git a/zstandard_cli/lib/src/utils/lib_loader.dart b/zstandard_cli/lib/src/utils/lib_loader.dart index fd9da967..2357da64 100644 --- a/zstandard_cli/lib/src/utils/lib_loader.dart +++ b/zstandard_cli/lib/src/utils/lib_loader.dart @@ -1,27 +1,219 @@ +import 'dart:convert'; import 'dart:ffi'; import 'dart:io'; +import 'dart:isolate'; import 'package:path/path.dart' as path; -/// Returns the path to the zstd native library for the current platform. -/// Exposed for testing; [openZstdLibrary] uses this and opens the library. -String getZstdLibraryPath() { - if (Platform.isWindows) { - final String arch = Platform.version.contains('ARM64') ? "arm64" : "x64"; - return path.join(Directory.current.path, 'lib', 'src', 'bin', - 'zstandard_windows_$arch.dll'); - } else if (Platform.isMacOS) { - return path.join(Directory.current.path, 'lib', 'src', 'bin', - 'libzstandard_macos.dylib'); - } else if (Platform.isLinux) { - final String arch = - Platform.operatingSystemVersion.contains("aarch64") ? "arm64" : "x64"; - return path.join(Directory.current.path, 'lib', 'src', 'bin', - 'libzstandard_linux_$arch.so'); - } - throw UnsupportedError('Unsupported platform'); +/// Returns the bundled library name for a platform and ABI. +/// +/// Optional arguments keep platform selection deterministic in tests. This is +/// an internal `src` API; production callers should omit them. +String zstdLibraryFileName({String? operatingSystem, Abi? abi}) { + final selectedOperatingSystem = operatingSystem ?? Platform.operatingSystem; + final selectedAbi = abi ?? Abi.current(); + if (selectedOperatingSystem == 'windows') { + return switch (selectedAbi) { + Abi.windowsArm64 => 'zstandard_windows_arm64.dll', + Abi.windowsX64 => 'zstandard_windows_x64.dll', + _ => throw UnsupportedError('Unsupported Windows ABI: $selectedAbi'), + }; + } + if (selectedOperatingSystem == 'macos') return 'libzstandard_macos.dylib'; + if (selectedOperatingSystem == 'linux') { + return switch (selectedAbi) { + Abi.linuxArm64 => 'libzstandard_linux_arm64.so', + Abi.linuxX64 => 'libzstandard_linux_x64.so', + _ => throw UnsupportedError('Unsupported Linux ABI: $selectedAbi'), + }; + } + throw UnsupportedError('Unsupported platform: $selectedOperatingSystem'); +} + +/// Returns the legacy package-relative native library path. +/// +/// Prefer [resolveZstdLibraryPath], which is independent of the current +/// working directory. [packageRoot] exists for deterministic tests. +String getZstdLibraryPath({String? packageRoot}) => path.join( + packageRoot ?? Directory.current.path, + 'lib', + 'src', + 'bin', + zstdLibraryFileName(), + ); + +/// Resolves the shipped native library from the active package configuration. +/// +/// `ZSTANDARD_CLI_LIBRARY` may point to an explicitly managed library for AOT +/// deployments that do not ship package sources next to the executable. +Future resolveZstdLibraryPath({ + Map? environment, + Iterable? packageConfigs, + String? executableDirectory, + String? legacyPackageRoot, +}) async { + final override = + (environment ?? Platform.environment)['ZSTANDARD_CLI_LIBRARY']; + if (override != null && override.isNotEmpty) { + if (File(override).existsSync()) return path.normalize(override); + throw ArgumentError.value( + override, + 'ZSTANDARD_CLI_LIBRARY', + 'Native library does not exist', + ); + } + + for (final packageConfig + in packageConfigs ?? await zstdPackageConfigCandidates()) { + final candidate = await resolveZstdLibraryFromPackageConfig(packageConfig); + if (candidate != null && File(candidate).existsSync()) { + return path.normalize(candidate); + } + } + + final executableCandidate = path.join( + executableDirectory ?? File(Platform.resolvedExecutable).parent.path, + zstdLibraryFileName(), + ); + if (File(executableCandidate).existsSync()) { + return path.normalize(executableCandidate); + } + + final legacyCandidate = getZstdLibraryPath(packageRoot: legacyPackageRoot); + if (File(legacyCandidate).existsSync()) { + return path.normalize(legacyCandidate); + } + + throw StateError( + 'Unable to locate ${zstdLibraryFileName()}. Set ZSTANDARD_CLI_LIBRARY to ' + 'an absolute native-library path for compiled deployments.', + ); } -DynamicLibrary openZstdLibrary() { - return DynamicLibrary.open(getZstdLibraryPath()); +/// Opens the shipped or explicitly configured zstd library. +Future openZstdLibrary() async => + DynamicLibrary.open(await resolveZstdLibraryPath()); + +/// Returns package-config candidates visible to the current Dart process. +/// +/// Optional arguments expose otherwise host-controlled inputs to internal +/// tests without changing production resolution behavior. +Future> zstdPackageConfigCandidates({ + List? executableArguments, + Future Function()? isolatePackageConfigProvider, +}) async { + final candidates = []; + final seen = {}; + + void add(Uri? uri) { + if (uri == null || uri.scheme != 'file' || !seen.add(uri.toString())) { + return; + } + candidates.add(uri); + } + + void addPath(String configuredPath) { + final parsed = Uri.tryParse(configuredPath); + add( + parsed != null && parsed.scheme.isNotEmpty + ? parsed + : File(configuredPath).absolute.uri, + ); + } + + final configuredPath = Platform.packageConfig; + if (configuredPath != null && configuredPath.isNotEmpty) { + addPath(configuredPath); + } + + final arguments = executableArguments ?? Platform.executableArguments; + for (var index = 0; index < arguments.length; index++) { + final argument = arguments[index]; + if (argument.startsWith('--packages=')) { + addPath(argument.substring('--packages='.length)); + } else if (argument == '--packages' && index + 1 < arguments.length) { + addPath(arguments[++index]); + } + } + + try { + add( + isolatePackageConfigProvider == null + ? await Isolate.packageConfig + : await isolatePackageConfigProvider(), + ); + } on UnsupportedError { + // Some embedders, including flutter_tester, do not expose this VM API. + } + + if (Platform.script.scheme == 'file') { + _addAncestorPackageConfigs(File.fromUri(Platform.script).parent, add); + } + _addAncestorPackageConfigs(Directory.current, add); + + return candidates; +} + +void _addAncestorPackageConfigs(Directory start, void Function(Uri) add) { + var directory = start.absolute; + while (true) { + final packageConfig = File( + path.join(directory.path, '.dart_tool', 'package_config.json'), + ); + if (packageConfig.existsSync()) add(packageConfig.uri); + + final parent = directory.parent; + if (parent.path == directory.path) return; + directory = parent; + } +} + +/// Resolves the CLI library path declared by one package configuration. +/// +/// Exposed from this internal `src` library for deterministic parser tests. +Future resolveZstdLibraryFromPackageConfig( + Uri configUri, { + Future Function(File)? readConfig, +}) async { + final configFile = File.fromUri(configUri); + if (!configFile.existsSync()) return null; + + try { + final config = jsonDecode( + await (readConfig ?? (file) => file.readAsString())(configFile), + ); + if (config is! Map) return null; + final packages = config['packages']; + if (packages is! List) return null; + + for (final package in packages) { + if (package is! Map || + package['name'] != 'zstandard_cli') { + continue; + } + final rootValue = package['rootUri']; + final packageValue = package['packageUri']; + if (rootValue is! String || packageValue is! String) return null; + + var rootUri = configUri.resolve(rootValue); + if (!rootUri.path.endsWith('/')) { + rootUri = rootUri.replace(path: '${rootUri.path}/'); + } + final packageLibUri = rootUri.resolve(packageValue); + if (packageLibUri.scheme != 'file') return null; + + return path.join( + Directory.fromUri(packageLibUri).path, + 'src', + 'bin', + zstdLibraryFileName(), + ); + } + } on FormatException { + return null; + } on FileSystemException { + return null; + } + + return null; } diff --git a/zstandard_cli/lib/src/utils/size_utils.dart b/zstandard_cli/lib/src/utils/size_utils.dart index 855f8da2..29ca835c 100644 --- a/zstandard_cli/lib/src/utils/size_utils.dart +++ b/zstandard_cli/lib/src/utils/size_utils.dart @@ -23,4 +23,4 @@ Future getFileSize(File file) async { print('Error reading file size: $e'); return ''; } -} \ No newline at end of file +} diff --git a/zstandard_cli/lib/src/utils/zstandard_ext.dart b/zstandard_cli/lib/src/utils/zstandard_ext.dart index 82c17be2..c40914a2 100644 --- a/zstandard_cli/lib/src/utils/zstandard_ext.dart +++ b/zstandard_cli/lib/src/utils/zstandard_ext.dart @@ -19,9 +19,9 @@ extension ZstandardExt on Uint8List? { /// Decompresses this byte list (must be Zstandard-compressed data). /// /// Returns null if the receiver is null or decompression failed. - Future decompress() async { + Future decompress({int maxOutputSize = 256 * 1024 * 1024}) async { var data = this; if (data == null) return null; - return ZstandardCLI().decompress(data); + return ZstandardCLI().decompress(data, maxOutputSize: maxOutputSize); } } diff --git a/zstandard_cli/lib/src/zstandard_cli_base.dart b/zstandard_cli/lib/src/zstandard_cli_base.dart index c887abe9..027ec1fe 100644 --- a/zstandard_cli/lib/src/zstandard_cli_base.dart +++ b/zstandard_cli/lib/src/zstandard_cli_base.dart @@ -1,118 +1,50 @@ -import 'dart:ffi'; import 'dart:typed_data'; -import 'package:ffi/ffi.dart'; import 'package:platform/platform.dart'; -import 'package:zstandard_native/zstandard_native_bindings.dart'; +import 'package:zstandard_native/zstandard_native.dart'; import 'utils/lib_loader.dart'; import 'zstandard_interface.dart'; -/// Command-line and in-code Zstandard compression for macOS, Windows, and Linux. +/// Formats a native platform version without consulting host state. /// -/// Uses FFI with precompiled native zstd libraries. Supports [compress] and -/// [decompress] with the same semantics as the Flutter plugin. Use this -/// package in pure Dart (non-Flutter) desktop apps or CLI tools. -/// -/// Example: -/// ```dart -/// final cli = ZstandardCLI(); -/// final compressed = await cli.compress(data, compressionLevel: 3); -/// final decompressed = await cli.decompress(compressed!); -/// ``` +/// This internal helper keeps all platform branches deterministic in tests. +String zstdPlatformVersion(String? operatingSystem, String? version) { + if (operatingSystem == null) return 'Unknown platform'; + return switch (operatingSystem) { + NativePlatform.macOS => 'macOS $version', + NativePlatform.windows => 'Windows $version', + NativePlatform.linux => 'Linux $version', + _ => 'Unknown platform', + }; +} + +/// Command-line and in-code Zstandard compression for desktop Dart. class ZstandardCLI implements ZstandardInterface { - final ZstandardNativeBindings _bindings = - ZstandardNativeBindings(openZstdLibrary()); + static final Future _sharedCodec = _loadCodec(); + + static Future _loadCodec() async => + ZstandardNativeCodec(ZstandardNativeBindings(await openZstdLibrary())); @override Future compress( Uint8List data, { int compressionLevel = 3, - }) async { - if (data.isEmpty) return data; - final int srcSize = data.lengthInBytes; - final Pointer src = malloc.allocate(srcSize); - src.asTypedList(srcSize).setAll(0, data); - - final int dstCapacity = _bindings.ZSTD_compressBound(srcSize); - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int compressedSize = _bindings.ZSTD_compress( - dst.cast(), - dstCapacity, - src.cast(), - srcSize, - compressionLevel, - ); - - if (_bindings.ZSTD_isError(compressedSize) == 0 && compressedSize > 0) { - return Uint8List.fromList(dst.asTypedList(compressedSize)); - } else { - return null; - } - } finally { - malloc.free(src); - malloc.free(dst); - } - } + }) async => + (await _sharedCodec).compress(data, compressionLevel); @override - Future decompress(Uint8List data) async { - if (data.isEmpty) return data; - const int contentSizeUnknown = 0xffffffffffffffff; - const int contentSizeError = 0xfffffffffffffffe; - - final int compressedSize = data.lengthInBytes; - final Pointer src = malloc.allocate(compressedSize); - src.asTypedList(compressedSize).setAll(0, data); - - final int decompressedSizeExpected = - _bindings.ZSTD_getFrameContentSize(src.cast(), compressedSize); - if (decompressedSizeExpected == contentSizeError) { - malloc.free(src); - return null; - } - final int dstCapacity = - (decompressedSizeExpected != contentSizeUnknown && - decompressedSizeExpected > 0) - ? decompressedSizeExpected - : compressedSize * 20; - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int decompressedSize = _bindings.ZSTD_decompress( - dst.cast(), - dstCapacity, - src.cast(), - compressedSize, - ); - - if (_bindings.ZSTD_isError(decompressedSize) != 0) { - return null; - } - return Uint8List.fromList(dst.asTypedList(decompressedSize)); - } finally { - malloc.free(src); - malloc.free(dst); - } - } + Future decompress( + Uint8List data, { + int maxOutputSize = nativeDefaultMaxDecompressedSize, + }) async => + (await _sharedCodec).decompress(data, maxOutputSize: maxOutputSize); @override Future getPlatformVersion() { - final platform = LocalPlatform(); - - String version; - if (platform.isMacOS) { - version = 'macOS ${platform.version}'; - } else if (platform.isWindows) { - version = 'Windows ${platform.version}'; - } else if (platform.isLinux) { - version = 'Linux ${platform.version}'; - } else { - version = 'Unknown platform'; - } - - return Future.value(version); + final platform = NativePlatform.current; + return Future.value( + zstdPlatformVersion(platform?.operatingSystem, platform?.version), + ); } } diff --git a/zstandard_cli/lib/src/zstandard_interface.dart b/zstandard_cli/lib/src/zstandard_interface.dart index d7391463..d4fe1788 100644 --- a/zstandard_cli/lib/src/zstandard_interface.dart +++ b/zstandard_cli/lib/src/zstandard_interface.dart @@ -3,10 +3,10 @@ import 'dart:typed_data'; abstract class ZstandardInterface { Future getPlatformVersion(); - Future compress( + Future compress(Uint8List data, {int compressionLevel = 3}); + + Future decompress( Uint8List data, { - int compressionLevel = 3, + int maxOutputSize = 256 * 1024 * 1024, }); - - Future decompress(Uint8List data); } diff --git a/zstandard_cli/lib/zstandard_cli.dart b/zstandard_cli/lib/zstandard_cli.dart index da5b8a47..92930263 100644 --- a/zstandard_cli/lib/zstandard_cli.dart +++ b/zstandard_cli/lib/zstandard_cli.dart @@ -1,4 +1,4 @@ library; export 'src/utils/zstandard_ext.dart'; -export 'src/zstandard_cli_base.dart'; +export 'src/zstandard_cli_base.dart' show ZstandardCLI; diff --git a/zstandard_cli/pubspec.yaml b/zstandard_cli/pubspec.yaml index 2153d915..489f0fe3 100644 --- a/zstandard_cli/pubspec.yaml +++ b/zstandard_cli/pubspec.yaml @@ -2,6 +2,10 @@ name: zstandard_cli description: Command-line interface for compressing and decompressing files with Zstandard. version: 1.5.0 +executables: + zstandard-compress: compress + zstandard-decompress: decompress + homepage: https://vyp.dev repository: https://github.com/vypdev/zstandard/tree/master/zstandard_cli diff --git a/zstandard_cli/test/properties_test.dart b/zstandard_cli/test/properties_test.dart index a8b3f7dc..be12c17b 100644 --- a/zstandard_cli/test/properties_test.dart +++ b/zstandard_cli/test/properties_test.dart @@ -6,30 +6,23 @@ import 'package:test/test.dart'; import 'package:zstandard_cli/zstandard_cli.dart'; void main() { - final bool skipPlatform = !Platform.isMacOS && !Platform.isWindows && !Platform.isLinux; + final bool skipPlatform = + !Platform.isMacOS && !Platform.isWindows && !Platform.isLinux; group('Property-based tests', () { - property( - 'roundtrip: decompress(compress(x)) == x', - () { - forAll( - binary(minLength: 0, maxLength: 2000), - (List data) async { - if (skipPlatform) return; - final input = Uint8List.fromList(data); - final cli = ZstandardCLI(); - final compressed = await cli.compress(input, compressionLevel: 3); - if (compressed == null) return; - final decompressed = await cli.decompress(compressed); - expect(decompressed, isNotNull); - expect(decompressed!.length, input.length); - expect(List.from(decompressed), data); - }, - maxExamples: 200, - ); - }, - skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : null, - ); + property('roundtrip: decompress(compress(x)) == x', () { + forAll(binary(minLength: 0, maxLength: 2000), (List data) async { + if (skipPlatform) return; + final input = Uint8List.fromList(data); + final cli = ZstandardCLI(); + final compressed = await cli.compress(input, compressionLevel: 3); + if (compressed == null) return; + final decompressed = await cli.decompress(compressed); + expect(decompressed, isNotNull); + expect(decompressed!.length, input.length); + expect(List.from(decompressed), data); + }, maxExamples: 200); + }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : null); property( 'determinism: same input and level produce same compressed output', @@ -45,8 +38,14 @@ void main() { final level = tuple.$2; final input = Uint8List.fromList(data); final cli = ZstandardCLI(); - final compressed1 = await cli.compress(input, compressionLevel: level); - final compressed2 = await cli.compress(input, compressionLevel: level); + final compressed1 = await cli.compress( + input, + compressionLevel: level, + ); + final compressed2 = await cli.compress( + input, + compressionLevel: level, + ); expect(compressed1, isNotNull); expect(compressed2, isNotNull); expect(compressed1!.length, compressed2!.length); @@ -58,25 +57,17 @@ void main() { skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : null, ); - property( - 'compression level 22 output size <= level 1 for same input', - () { - forAll( - binary(minLength: 100, maxLength: 1000), - (List data) async { - if (skipPlatform) return; - final input = Uint8List.fromList(data); - final cli = ZstandardCLI(); - final c1 = await cli.compress(input, compressionLevel: 1); - final c22 = await cli.compress(input, compressionLevel: 22); - expect(c1, isNotNull); - expect(c22, isNotNull); - expect(c22!.length, lessThanOrEqualTo(c1!.length)); - }, - maxExamples: 50, - ); - }, - skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : null, - ); + property('compression level 22 output size <= level 1 for same input', () { + forAll(binary(minLength: 100, maxLength: 1000), (List data) async { + if (skipPlatform) return; + final input = Uint8List.fromList(data); + final cli = ZstandardCLI(); + final c1 = await cli.compress(input, compressionLevel: 1); + final c22 = await cli.compress(input, compressionLevel: 22); + expect(c1, isNotNull); + expect(c22, isNotNull); + expect(c22!.length, lessThanOrEqualTo(c1!.length)); + }, maxExamples: 50); + }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : null); }); } diff --git a/zstandard_cli/test/zstandard_cli_test.dart b/zstandard_cli/test/zstandard_cli_test.dart index 06c3e941..abd7c605 100644 --- a/zstandard_cli/test/zstandard_cli_test.dart +++ b/zstandard_cli/test/zstandard_cli_test.dart @@ -1,15 +1,22 @@ +import 'dart:convert'; +import 'dart:ffi'; import 'dart:io'; import 'dart:typed_data'; import 'package:leak_tracker/leak_tracker.dart'; import 'package:leak_tracker_testing/leak_tracker_testing.dart'; +import 'package:path/path.dart' as path; import 'package:test/test.dart'; +import 'package:zstandard_cli/src/cli_runner.dart'; +import 'package:zstandard_cli/src/zstandard_cli_base.dart' + show zstdPlatformVersion; import 'package:zstandard_cli/zstandard_cli.dart'; import 'package:zstandard_cli/src/utils/constants.dart'; import 'package:zstandard_cli/src/utils/lib_loader.dart'; void main() { - final bool skipPlatform = !Platform.isMacOS && !Platform.isWindows && !Platform.isLinux; + final bool skipPlatform = + !Platform.isMacOS && !Platform.isWindows && !Platform.isLinux; setUpAll(() { if (!LeakTracking.isStarted) { @@ -17,14 +24,293 @@ void main() { } }); + group('Native library resolver', () { + test('selects every supported platform and ABI deterministically', () { + expect( + zstdLibraryFileName( + operatingSystem: 'windows', + abi: Abi.windowsArm64, + ), + 'zstandard_windows_arm64.dll', + ); + expect( + zstdLibraryFileName( + operatingSystem: 'windows', + abi: Abi.windowsX64, + ), + 'zstandard_windows_x64.dll', + ); + expect( + zstdLibraryFileName( + operatingSystem: 'linux', + abi: Abi.linuxArm64, + ), + 'libzstandard_linux_arm64.so', + ); + expect( + zstdLibraryFileName( + operatingSystem: 'linux', + abi: Abi.linuxX64, + ), + 'libzstandard_linux_x64.so', + ); + expect( + zstdLibraryFileName(operatingSystem: 'macos', abi: Abi.macosArm64), + 'libzstandard_macos.dylib', + ); + expect( + () => zstdLibraryFileName( + operatingSystem: 'windows', + abi: Abi.linuxX64, + ), + throwsUnsupportedError, + ); + expect( + () => zstdLibraryFileName( + operatingSystem: 'linux', + abi: Abi.windowsX64, + ), + throwsUnsupportedError, + ); + expect( + () => zstdLibraryFileName( + operatingSystem: 'android', + abi: Abi.androidArm64, + ), + throwsUnsupportedError, + ); + }); + + test('honors an explicit existing library and rejects a missing one', + () async { + final directory = await Directory.systemTemp.createTemp('zstd_override'); + try { + final library = await File( + path.join(directory.path, zstdLibraryFileName()), + ).create(); + expect( + await resolveZstdLibraryPath( + environment: {'ZSTANDARD_CLI_LIBRARY': library.path}, + ), + path.normalize(library.path), + ); + expect( + resolveZstdLibraryPath( + environment: { + 'ZSTANDARD_CLI_LIBRARY': path.join(directory.path, 'missing'), + }, + ), + throwsA(isA()), + ); + } finally { + await directory.delete(recursive: true); + } + }); + + test('resolves package-config, executable, and legacy fallbacks', () async { + final directory = await Directory.systemTemp.createTemp('zstd_fallback'); + try { + final packageRoot = await Directory( + path.join(directory.path, 'package'), + ).create(); + final packageLibrary = await File( + path.join( + packageRoot.path, + 'lib', + 'src', + 'bin', + zstdLibraryFileName(), + ), + ).create(recursive: true); + final packageConfig = await File( + path.join(directory.path, '.dart_tool', 'package_config.json'), + ).create(recursive: true); + final rootWithoutTrailingSlash = + packageRoot.uri.toString().replaceFirst(RegExp(r'/$'), ''); + await packageConfig.writeAsString( + jsonEncode({ + 'configVersion': 2, + 'packages': [ + { + 'name': 'zstandard_cli', + 'rootUri': rootWithoutTrailingSlash, + 'packageUri': 'lib/', + }, + ], + }), + ); + + expect( + await resolveZstdLibraryFromPackageConfig(packageConfig.uri), + path.normalize(packageLibrary.path), + ); + expect( + await resolveZstdLibraryPath( + environment: const {}, + packageConfigs: [packageConfig.uri], + executableDirectory: path.join(directory.path, 'empty-executable'), + legacyPackageRoot: path.join(directory.path, 'empty-legacy'), + ), + path.normalize(packageLibrary.path), + ); + + final executableDirectory = await Directory( + path.join(directory.path, 'executable'), + ).create(); + final executableLibrary = await File( + path.join(executableDirectory.path, zstdLibraryFileName()), + ).create(); + expect( + await resolveZstdLibraryPath( + environment: const {}, + packageConfigs: const [], + executableDirectory: executableDirectory.path, + legacyPackageRoot: path.join(directory.path, 'empty-legacy'), + ), + path.normalize(executableLibrary.path), + ); + + final legacyRoot = await Directory( + path.join(directory.path, 'legacy'), + ).create(); + final legacyLibrary = await File( + getZstdLibraryPath(packageRoot: legacyRoot.path), + ).create(recursive: true); + expect( + await resolveZstdLibraryPath( + environment: const {}, + packageConfigs: const [], + executableDirectory: path.join(directory.path, 'empty-executable'), + legacyPackageRoot: legacyRoot.path, + ), + path.normalize(legacyLibrary.path), + ); + + expect( + await resolveZstdLibraryPath( + environment: const {}, + packageConfigs: const [], + ), + path.normalize(getZstdLibraryPath()), + ); + + expect( + resolveZstdLibraryPath( + environment: const {}, + packageConfigs: const [], + executableDirectory: path.join(directory.path, 'empty-executable'), + legacyPackageRoot: path.join(directory.path, 'empty-legacy'), + ), + throwsStateError, + ); + } finally { + await directory.delete(recursive: true); + } + }); + + test('rejects malformed and invalid package configurations', () async { + final directory = await Directory.systemTemp.createTemp('zstd_config'); + try { + final config = File(path.join(directory.path, 'package_config.json')); + + await config.writeAsString('{invalid'); + expect(await resolveZstdLibraryFromPackageConfig(config.uri), isNull); + + await config.writeAsString(jsonEncode(const [])); + expect(await resolveZstdLibraryFromPackageConfig(config.uri), isNull); + + await config.writeAsString(jsonEncode({'packages': const {}})); + expect(await resolveZstdLibraryFromPackageConfig(config.uri), isNull); + + await config.writeAsString( + jsonEncode({ + 'packages': [ + { + 'name': 'zstandard_cli', + 'rootUri': 42, + 'packageUri': 'lib/', + }, + ], + }), + ); + expect(await resolveZstdLibraryFromPackageConfig(config.uri), isNull); + + await config.writeAsString( + jsonEncode({ + 'packages': [ + { + 'name': 'zstandard_cli', + 'rootUri': 'https://example.invalid/package/', + 'packageUri': 'lib/', + }, + ], + }), + ); + expect(await resolveZstdLibraryFromPackageConfig(config.uri), isNull); + + expect( + await resolveZstdLibraryFromPackageConfig( + config.uri, + readConfig: (_) async => throw const FileSystemException( + 'simulated read failure', + ), + ), + isNull, + ); + } finally { + await directory.delete(recursive: true); + } + }); + + test('discovers split package arguments and tolerates unsupported VM APIs', + () async { + final directory = + await Directory.systemTemp.createTemp('zstd_candidates'); + try { + final config = await File( + path.join(directory.path, 'package_config.json'), + ).create(); + final candidates = await zstdPackageConfigCandidates( + executableArguments: [ + '--packages', + config.path, + '--packages=${config.uri}', + ], + isolatePackageConfigProvider: () async => + throw UnsupportedError('simulated embedder limitation'), + ); + + expect(candidates, contains(config.absolute.uri)); + expect( + candidates.where((candidate) => candidate == config.absolute.uri), + hasLength(1), + ); + } finally { + await directory.delete(recursive: true); + } + }); + }); + group('Zstandard CLI tests', () { - test('getPlatformVersion returns non-null string on supported platform', () async { - if (skipPlatform) return; - final cli = ZstandardCLI(); - final version = await cli.getPlatformVersion(); - expect(version, isNotNull); - expect(version, isNotEmpty); - }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false); + test('formats every native platform version deterministically', () { + expect(zstdPlatformVersion(null, null), 'Unknown platform'); + expect(zstdPlatformVersion('macos', '14.0'), 'macOS 14.0'); + expect(zstdPlatformVersion('windows', '11'), 'Windows 11'); + expect(zstdPlatformVersion('linux', '6.8'), 'Linux 6.8'); + expect(zstdPlatformVersion('android', '15'), 'Unknown platform'); + }); + + test( + 'getPlatformVersion returns non-null string on supported platform', + () async { + if (skipPlatform) return; + final cli = ZstandardCLI(); + final version = await cli.getPlatformVersion(); + expect(version, isNotNull); + expect(version, isNotEmpty); + }, + skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false, + ); test('getPlatformVersion starts with platform name', () async { if (skipPlatform) return; @@ -39,22 +325,72 @@ void main() { } }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false); - test('getZstdLibraryPath returns path with lib/src/bin and platform extension', () { - if (skipPlatform) return; - final libPath = getZstdLibraryPath(); - expect(libPath, contains('lib')); - expect(libPath, contains('src')); - expect(libPath, contains('bin')); - if (Platform.isWindows) { - expect(libPath, endsWith('.dll')); - expect(libPath, anyOf(contains('arm64'), contains('x64'))); - } else if (Platform.isMacOS) { - expect(libPath, endsWith('.dylib')); - } else if (Platform.isLinux) { - expect(libPath, endsWith('.so')); - expect(libPath, anyOf(contains('arm64'), contains('x64'))); - } - }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false); + test( + 'getZstdLibraryPath returns path with lib/src/bin and platform extension', + () { + if (skipPlatform) return; + final libPath = getZstdLibraryPath(); + expect(libPath, contains('lib')); + expect(libPath, contains('src')); + expect(libPath, contains('bin')); + if (Platform.isWindows) { + expect(libPath, endsWith('.dll')); + expect(libPath, anyOf(contains('arm64'), contains('x64'))); + } else if (Platform.isMacOS) { + expect(libPath, endsWith('.dylib')); + } else if (Platform.isLinux) { + expect(libPath, endsWith('.so')); + expect(libPath, anyOf(contains('arm64'), contains('x64'))); + } + }, + skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false, + ); + + test( + 'resolveZstdLibraryPath is independent of the working directory', + () async { + if (skipPlatform) return; + final originalDirectory = Directory.current; + final temporaryDirectory = await Directory.systemTemp.createTemp( + 'zstd_loader_test', + ); + try { + Directory.current = temporaryDirectory; + final libraryPath = await resolveZstdLibraryPath(); + expect(File(libraryPath).existsSync(), isTrue); + } finally { + Directory.current = originalDirectory; + await temporaryDirectory.delete(recursive: true); + } + }, + skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false, + ); + + test( + 'native library exports the complete supported symbol surface', + () async { + final library = DynamicLibrary.open(await resolveZstdLibraryPath()); + for (final symbol in [ + 'ZSTD_compress', + 'ZSTD_decompress', + 'ZSTD_compressBound', + 'ZSTD_getFrameContentSize', + 'ZSTD_isError', + 'ZSTD_createDStream', + 'ZSTD_initDStream', + 'ZSTD_decompressStream', + 'ZSTD_freeDStream', + 'ZSTD_DStreamOutSize', + ]) { + expect( + () => library.lookup>(symbol), + returnsNormally, + reason: '$symbol must be exported by the bundled library', + ); + } + }, + skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false, + ); test('ZstandardCLI can be instantiated on supported platform', () { if (skipPlatform) return; @@ -70,8 +406,9 @@ void main() { }); test('Compress and decompress large Uint8List', () async { - final Uint8List sample = - Uint8List.fromList(List.generate(100000, (i) => i % 256)); + final Uint8List sample = Uint8List.fromList( + List.generate(100000, (i) => i % 256), + ); final compressed = await sample.compress(compressionLevel: 3); final decompressed = await compressed.decompress(); expect(sample, isNot(equals(compressed))); @@ -83,11 +420,66 @@ void main() { final Uint8List sample = Uint8List(0); final compressed = await sample.compress(compressionLevel: 3); final decompressed = await compressed.decompress(); - expect(sample, equals(compressed)); - expect(sample.length, equals(compressed?.length)); + expect(compressed, isNotNull); + expect(compressed, isNotEmpty); expect(decompressed, equals(sample)); }); + test('decompresses frames without a declared content size', () async { + const frameWithoutContentSize = [ + 0x28, + 0xb5, + 0x2f, + 0xfd, + 0x04, + 0x58, + 0x91, + 0x00, + 0x00, + 0x75, + 0x6e, + 0x6b, + 0x6e, + 0x6f, + 0x77, + 0x6e, + 0x2d, + 0x73, + 0x69, + 0x7a, + 0x65, + 0x20, + 0x66, + 0x72, + 0x61, + 0x6d, + 0x65, + 0xab, + 0x02, + 0x28, + 0xf0, + ]; + final result = await ZstandardCLI().decompress( + Uint8List.fromList(frameWithoutContentSize), + ); + expect(result, equals('unknown-size frame'.codeUnits)); + }); + + test('decompression respects the configured output limit', () async { + final cli = ZstandardCLI(); + final compressed = await cli.compress(Uint8List(1024)); + expect(compressed, isNotNull); + expect(await cli.decompress(compressed!, maxOutputSize: 1023), isNull); + }); + + test('decompresses concatenated frames', () async { + final cli = ZstandardCLI(); + final first = await cli.compress(Uint8List.fromList([1, 2, 3])); + final second = await cli.compress(Uint8List.fromList([4, 5, 6])); + final frames = Uint8List.fromList([...first!, ...second!]); + expect(await cli.decompress(frames), equals([1, 2, 3, 4, 5, 6])); + }); + test('Compress and decompress Uint8List with repeated values', () async { final Uint8List sample = Uint8List.fromList(List.filled(1000, 42)); final compressed = await sample.compress(compressionLevel: 3); @@ -98,8 +490,18 @@ void main() { }); test('Compress and decompress with maximum compression level', () async { - final Uint8List sample = - Uint8List.fromList([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + final Uint8List sample = Uint8List.fromList([ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + ]); final compressed = await sample.compress(compressionLevel: 22); final decompressed = await compressed.decompress(); expect(sample, isNot(equals(compressed))); @@ -108,8 +510,18 @@ void main() { }); test('Compress and decompress with minimal compression level', () async { - final Uint8List sample = - Uint8List.fromList([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + final Uint8List sample = Uint8List.fromList([ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + ]); final compressed = await sample.compress(compressionLevel: 1); final decompressed = await compressed.decompress(); expect(sample, isNot(equals(compressed))); @@ -117,12 +529,43 @@ void main() { expect(decompressed, equals(sample)); }); - test('Compress with level 22 produces valid decompressible output', () async { - final Uint8List sample = Uint8List.fromList(List.filled(500, 7)); - final compressed = await sample.compress(compressionLevel: 22); + test( + 'Compress with level 22 produces valid decompressible output', + () async { + final Uint8List sample = Uint8List.fromList(List.filled(500, 7)); + final compressed = await sample.compress(compressionLevel: 22); + expect(compressed, isNotNull); + final decompressed = await compressed!.decompress(); + expect(decompressed, equals(sample)); + }, + ); + + test('invalid compression levels return null', () async { + final cli = ZstandardCLI(); + expect(await cli.compress(Uint8List(1), compressionLevel: 0), isNull); + expect(await cli.compress(Uint8List(1), compressionLevel: 23), isNull); + }); + + test('truncated frame returns null', () async { + final cli = ZstandardCLI(); + final compressed = await cli.compress(Uint8List.fromList([1, 2, 3])); expect(compressed, isNotNull); - final decompressed = await compressed!.decompress(); - expect(decompressed, equals(sample)); + expect( + await cli.decompress( + Uint8List.sublistView(compressed!, 0, compressed.length - 1), + ), + isNull, + ); + }); + + test('an exact decompression output limit succeeds', () async { + final cli = ZstandardCLI(); + final input = Uint8List(1024); + final compressed = await cli.compress(input); + expect( + await cli.decompress(compressed!, maxOutputSize: input.length), + equals(input), + ); }); test('Null extension receiver returns null from compress', () async { @@ -147,7 +590,9 @@ void main() { test('decompress random bytes returns null', () async { if (skipPlatform) return; - final random = Uint8List.fromList(List.generate(64, (i) => (i * 31) % 256)); + final random = Uint8List.fromList( + List.generate(64, (i) => (i * 31) % 256), + ); final cli = ZstandardCLI(); final result = await cli.decompress(random); expect(result, isNull); @@ -184,18 +629,255 @@ void main() { await tempDir.delete(recursive: true); }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false); + test( + 'extensionless CLI input is decompressed to a distinct .out file', + () async { + if (skipPlatform) return; + final directory = await Directory.systemTemp.createTemp( + 'zstd_cli_output', + ); + try { + final original = Uint8List.fromList([9, 8, 7, 6]); + final compressed = await ZstandardCLI().compress(original); + final input = File('${directory.path}/payload'); + await input.writeAsBytes(compressed!); + + expect(await runDecompressCommand([input.path]), 0); + expect(await input.readAsBytes(), equals(compressed)); + expect( + await File('${input.path}.out').readAsBytes(), + equals(original), + ); + } finally { + await directory.delete(recursive: true); + } + }, + skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false, + ); + + test( + 'CLI returns a usage error instead of replacing an existing output', + () async { + if (skipPlatform) return; + final directory = await Directory.systemTemp.createTemp( + 'zstd_cli_force', + ); + try { + final input = File('${directory.path}/input.bin'); + final output = File('${directory.path}/output.zstd'); + await input.writeAsBytes([1, 2, 3]); + await output.writeAsBytes([99]); + + expect( + await runCompressCommand([input.path, '--output', output.path]), + 2, + ); + expect(await output.readAsBytes(), equals([99])); + } finally { + await directory.delete(recursive: true); + } + }, + skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false, + ); + + test( + 'CLI validates arguments without loading the native library', + () async { + expect(await runCompressCommand([]), 2); + expect(await runCompressCommand(['--level', '0', 'input']), 2); + expect( + await runDecompressCommand(['--max-output-size', '0', 'input']), + 2, + ); + expect(await runCompressCommand(['--help']), 0); + expect(await runDecompressCommand(['--version']), 0); + }, + ); + + test('CLI stdin and stdout form a binary-clean roundtrip', () async { + if (skipPlatform) return; + final directory = await Directory.systemTemp.createTemp('zstd_cli_pipe'); + try { + final input = Uint8List.fromList(List.generate(256, (i) => i)); + final compressedFile = File('${directory.path}/compressed'); + final compressedSink = compressedFile.openWrite(); + expect( + await runCompressCommand( + ['-'], + standardInput: Stream>.value(input), + standardOutput: compressedSink, + ), + 0, + ); + await compressedSink.close(); + + final restoredFile = File('${directory.path}/restored'); + final restoredSink = restoredFile.openWrite(); + expect( + await runDecompressCommand( + ['-'], + standardInput: Stream>.value( + await compressedFile.readAsBytes(), + ), + standardOutput: restoredSink, + ), + 0, + ); + await restoredSink.close(); + expect(await restoredFile.readAsBytes(), equals(input)); + } finally { + await directory.delete(recursive: true); + } + }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false); + + test('CLI always refuses to overwrite its input', () async { + if (skipPlatform) return; + final directory = await Directory.systemTemp.createTemp('zstd_cli_alias'); + try { + final input = File('${directory.path}/input.bin'); + await input.writeAsBytes([1, 2, 3]); + expect( + await runCompressCommand([ + input.path, + '--output', + input.path, + '--force', + ]), + 2, + ); + expect(await input.readAsBytes(), equals([1, 2, 3])); + } finally { + await directory.delete(recursive: true); + } + }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false); + + test('CLI reports every invalid argument form as a usage error', () async { + final directory = await Directory.systemTemp.createTemp('zstd_cli_args'); + final errors = File(path.join(directory.path, 'stderr')).openWrite(); + try { + expect( + await runCompressCommand(['--output'], standardError: errors), + 2, + ); + expect( + await runCompressCommand( + ['--level', 'not-an-integer', 'input'], + standardError: errors, + ), + 2, + ); + expect( + await runDecompressCommand( + ['--max-output-size', 'not-an-integer', 'input'], + standardError: errors, + ), + 2, + ); + expect( + await runCompressCommand( + ['--unknown', 'input'], + standardError: errors, + ), + 2, + ); + expect( + await runCompressCommand( + ['first', 'second'], + standardError: errors, + ), + 2, + ); + expect( + await runCompressCommand( + ['--', '--literal-missing-file'], + standardError: errors, + ), + 1, + ); + } finally { + await errors.close(); + await directory.delete(recursive: true); + } + }); + + test('CLI converts codec, filesystem, and stream failures to exit code 1', + () async { + if (skipPlatform) return; + final directory = await Directory.systemTemp.createTemp('zstd_cli_error'); + final errors = File(path.join(directory.path, 'stderr')).openWrite(); + try { + final invalidInput = File(path.join(directory.path, 'invalid.zstd')); + await invalidInput.writeAsBytes([1, 2, 3, 4]); + expect( + await runDecompressCommand( + [invalidInput.path], + standardError: errors, + ), + 1, + ); + expect( + await runCompressCommand( + [path.join(directory.path, 'missing')], + standardError: errors, + ), + 1, + ); + expect( + await runCompressCommand( + ['-'], + standardInput: Stream>.error(StateError('read failed')), + standardError: errors, + ), + 1, + ); + } finally { + await errors.close(); + await directory.delete(recursive: true); + } + }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false); + + test('CLI strips the zstd extension for the default decompressed output', + () async { + if (skipPlatform) return; + final directory = + await Directory.systemTemp.createTemp('zstd_cli_suffix'); + final errors = File(path.join(directory.path, 'stderr')).openWrite(); + try { + final original = Uint8List.fromList([4, 8, 15, 16, 23, 42]); + final compressed = await ZstandardCLI().compress(original); + final input = File(path.join(directory.path, 'payload$extension')); + await input.writeAsBytes(compressed!); + + expect( + await runDecompressCommand([input.path], standardError: errors), + 0, + ); + expect( + await File(path.join(directory.path, 'payload')).readAsBytes(), + original, + ); + } finally { + await errors.close(); + await directory.delete(recursive: true); + } + }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false); + test('multiple compressions in parallel', () async { if (skipPlatform) return; final cli = ZstandardCLI(); final futures = List.generate(5, (i) { - final data = Uint8List.fromList(List.generate(200, (j) => (i + j) % 256)); + final data = Uint8List.fromList( + List.generate(200, (j) => (i + j) % 256), + ); return cli.compress(data, compressionLevel: 3); }); final results = await Future.wait(futures); expect(results.every((r) => r != null), isTrue); for (var i = 0; i < results.length; i++) { final decompressed = await cli.decompress(results[i]!); - final original = Uint8List.fromList(List.generate(200, (j) => (i + j) % 256)); + final original = Uint8List.fromList( + List.generate(200, (j) => (i + j) % 256), + ); expect(decompressed, equals(original)); } }, skip: skipPlatform ? 'Only runs on macOS, Windows, or Linux' : false); diff --git a/zstandard_ios/CHANGELOG.md b/zstandard_ios/CHANGELOG.md index d3621d95..433461d1 100644 --- a/zstandard_ios/CHANGELOG.md +++ b/zstandard_ios/CHANGELOG.md @@ -1,5 +1,7 @@ ## Unreleased +- Added byte-safe worker-isolate execution and bounded streaming decompression; + aligned CocoaPods and SwiftPM native sources and pinned SwiftPM releases. - Fixed intermittent iOS build failures caused by deleting synced zstd sources during compilation. - Added Swift Package Manager support while retaining CocoaPods compatibility. diff --git a/zstandard_ios/README.md b/zstandard_ios/README.md index a11129f4..22c847dd 100644 --- a/zstandard_ios/README.md +++ b/zstandard_ios/README.md @@ -10,7 +10,7 @@ Add the main plugin to your app; this package is included automatically via the ```yaml dependencies: - zstandard: ^1.3.29 + zstandard: ^1.5.0 ``` No extra setup is required for normal use. @@ -42,12 +42,16 @@ final decompressed = await compressed?.decompress(); - **ZstandardIOS()** — Creates the iOS platform implementation. - **compress(Uint8List data, int compressionLevel)** — Compresses `data` (level 1–22). Returns compressed bytes or `null`. -- **decompress(Uint8List data)** — Decompresses zstd-compressed data. Returns decompressed bytes or `null`. +- **decompressWithOptions(Uint8List data, {int maxOutputSize})** — Decompresses complete, concatenated, or unknown-size zstd frames with a bounded output (256 MiB by default). Invalid, truncated, or oversized input returns `null`. - **getPlatformVersion()** — Returns a platform identifier string. ## Architecture -This package uses Dart FFI with the native zstd C library. Swift Package Manager is the primary integration for Flutter 3.44 and newer and statically links the shared target into the app; CocoaPods remains supported for compatibility and embeds the plugin framework. Both paths use the same canonical C implementation from `zstandard_native`. +This package uses Dart FFI with the native zstd C library. Swift Package +Manager statically links the shared target into the app; CocoaPods remains +supported and embeds the plugin framework. Both paths compile the same +common/compress/decompress source set from `zstandard_native`. Public work runs +in an isolate using Dart-owned bytes and bounded streaming decompression. ## Testing diff --git a/zstandard_ios/analysis_options.yaml b/zstandard_ios/analysis_options.yaml index a5744c1c..1186c2bb 100644 --- a/zstandard_ios/analysis_options.yaml +++ b/zstandard_ios/analysis_options.yaml @@ -1,3 +1,7 @@ +analyzer: + exclude: + - build/** + - ios/** include: package:flutter_lints/flutter.yaml # Additional information about this file can be found at diff --git a/zstandard_ios/example/analysis_options.yaml b/zstandard_ios/example/analysis_options.yaml index 0d290213..a5a2c9f5 100644 --- a/zstandard_ios/example/analysis_options.yaml +++ b/zstandard_ios/example/analysis_options.yaml @@ -7,6 +7,10 @@ # The following line activates a set of recommended lints for Flutter apps, # packages, and plugins designed to encourage good coding practices. +analyzer: + exclude: + - build/** + - ios/** include: package:flutter_lints/flutter.yaml linter: diff --git a/zstandard_ios/images/sample.png b/zstandard_ios/images/sample.png index 9da2fb4f..332bc30c 100644 Binary files a/zstandard_ios/images/sample.png and b/zstandard_ios/images/sample.png differ diff --git a/zstandard_ios/ios/zstandard_ios.podspec b/zstandard_ios/ios/zstandard_ios.podspec index 28407724..d1014b5b 100644 --- a/zstandard_ios/ios/zstandard_ios.podspec +++ b/zstandard_ios/ios/zstandard_ios.podspec @@ -5,20 +5,24 @@ Pod::Spec.new do |s| s.cocoapods_version = '>= 1.11.0' # for script_phase :before_headers s.name = 'zstandard_ios' - s.version = '0.0.1' - s.summary = 'A new Flutter FFI plugin project.' + s.version = '1.5.0' + s.summary = 'iOS implementation of the Zstandard Flutter plugin.' s.description = <<-DESC -A new Flutter FFI plugin project. +Native Zstandard compression and bounded decompression for Flutter on iOS. DESC - s.homepage = 'http://example.com' + s.homepage = 'https://github.com/vypdev/zstandard' s.license = { :file => '../LICENSE' } - s.author = { 'Your Company' => 'email@example.com' } + s.author = { 'VypDev' => 'https://github.com/vypdev' } # Zstd C sources: synced from zstandard_native/src/zstd/ into Classes/zstd/ by # scripts/sync_zstd.sh (in this plugin). Must exist at pod install time so source_files glob finds them. s.source = { :path => '.' } - s.source_files = 'Classes/zstd/**/*.c', 'Classes/zstd/**/*.h', + s.source_files = 'Classes/zstd/common/*.c', 'Classes/zstd/common/*.h', + 'Classes/zstd/compress/*.c', 'Classes/zstd/compress/*.h', + 'Classes/zstd/decompress/*.c', 'Classes/zstd/decompress/*.h', + 'Classes/zstd/*.h', 'zstandard_ios/Sources/zstandard_ios/*.swift' + s.private_header_files = 'Classes/zstd/**/*.h' # zstd.h includes zstd_errors.h; both must be public so the module build finds them. s.public_header_files = 'Classes/zstd/zstd.h', 'Classes/zstd/zstd_errors.h' @@ -34,7 +38,8 @@ A new Flutter FFI plugin project. 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386', 'HEADER_SEARCH_PATHS' => '$(PODS_TARGET_SRCROOT)/Classes/zstd', 'CLANG_ALLOW_NON_MODULAR_INCLUDES_IN_FRAMEWORK_MODULES' => 'YES', - 'OTHER_CFLAGS' => '$(inherited) -DZSTD_STATIC_LINKING_ONLY -DZSTD_DISABLE_ASM -fvisibility=default', + 'OTHER_CFLAGS' => '$(inherited) -DZSTD_DISABLE_ASM -fvisibility=default', + 'GCC_WARN_INHIBIT_ALL_WARNINGS' => 'YES', 'DEAD_CODE_STRIPPING' => 'NO', 'STRIP_INSTALLED_PRODUCT' => 'NO', } diff --git a/zstandard_ios/ios/zstandard_ios/Package.swift b/zstandard_ios/ios/zstandard_ios/Package.swift index d4ebd176..d655ba5f 100644 --- a/zstandard_ios/ios/zstandard_ios/Package.swift +++ b/zstandard_ios/ios/zstandard_ios/Package.swift @@ -7,13 +7,10 @@ let nativePackageDependency: Package.Dependency = { return .package(name: "zstandard", path: localPath) } - // The repository root package exposes the canonical C implementation - // from zstandard_native/src/zstd without copying it into this plugin. - // Use the development branch until the first release containing this - // SwiftPM facade is tagged; release builds should pin that tag. + // Match the immutable repository tag carrying this plugin release. return .package( url: "https://github.com/vypdev/zstandard.git", - branch: "develop" + exact: "1.5.0" ) }() diff --git a/zstandard_ios/lib/zstandard_ext.dart b/zstandard_ios/lib/zstandard_ext.dart index a06223d1..9434f0df 100644 --- a/zstandard_ios/lib/zstandard_ext.dart +++ b/zstandard_ios/lib/zstandard_ext.dart @@ -9,9 +9,12 @@ extension ZstandardExt on Uint8List? { return ZstandardIOS().compress(data, compressionLevel); } - Future decompress() async { + Future decompress({int maxOutputSize = 256 * 1024 * 1024}) async { var data = this; if (data == null) return null; - return ZstandardIOS().decompress(data); + return ZstandardIOS().decompressWithOptions( + data, + maxOutputSize: maxOutputSize, + ); } } diff --git a/zstandard_ios/lib/zstandard_ios.dart b/zstandard_ios/lib/zstandard_ios.dart index 7fb0fe8b..dd274b28 100644 --- a/zstandard_ios/lib/zstandard_ios.dart +++ b/zstandard_ios/lib/zstandard_ios.dart @@ -1,13 +1,12 @@ -import 'dart:async'; import 'dart:ffi'; import 'dart:io'; import 'dart:isolate'; -import 'package:ffi/ffi.dart'; import 'package:flutter/services.dart'; -import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; - +import 'package:zstandard_native/zstandard_native.dart' + show ZstandardNativeCodec; import 'package:zstandard_native/zstandard_native_bindings.dart'; +import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; export 'zstandard_ext.dart'; @@ -16,12 +15,12 @@ const String _libName = 'zstandard_ios'; final DynamicLibrary _dylib = () { if (!Platform.isIOS) { throw UnsupportedError( - 'Platform not supported: ${Platform.operatingSystem}'); + 'Platform not supported: ${Platform.operatingSystem}', + ); } // CocoaPods embeds a dynamic framework. Flutter's SwiftPM integration - // links the same C target statically into the application, so those symbols - // are exposed through the process instead of a standalone framework. + // links the same C target statically into the application. try { return DynamicLibrary.open('$_libName.framework/$_libName'); } on ArgumentError { @@ -30,134 +29,52 @@ final DynamicLibrary _dylib = () { }(); final ZstandardNativeBindings _bindings = ZstandardNativeBindings(_dylib); +final ZstandardNativeCodec _codec = ZstandardNativeCodec(_bindings); bool _hasZstdFrameMagic(Uint8List data) { - if (data.lengthInBytes < 4) { - return false; - } - - final int magic = - data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); + if (data.lengthInBytes < 4) return false; + final magic = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); return magic == ZSTD_MAGICNUMBER || (magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START; } -bool _isUnavailableContentSize(int size) { - // The generated bindings expose these C unsigned values as -1 and -2. - // Keep the unsigned representations as a compatibility guard for older - // generated bindings and runtimes that returned the raw 64-bit bit pattern. - return size == ZSTD_CONTENTSIZE_UNKNOWN || - size == ZSTD_CONTENTSIZE_ERROR || - size == 0xffffffffffffffff || - size == 0xfffffffffffffffe; -} - -/// iOS implementation of [ZstandardPlatform] using FFI and the native zstd library. -/// -/// Resolves ZSTD_compress, ZSTD_decompress, ZSTD_compressBound, and -/// ZSTD_getFrameContentSize from the CocoaPods framework or the statically -/// linked SwiftPM application. The main [zstandard] plugin registers this -/// implementation automatically on iOS. -class ZstandardIOS extends ZstandardPlatform { +/// iOS implementation of [ZstandardPlatform] using the native zstd library. +class ZstandardIOS extends ZstandardPlatform + implements BoundedZstandardPlatform { /// Creates the iOS platform implementation. ZstandardIOS(); final methodChannel = const MethodChannel('plugins.flutter.io/zstandard'); - /// Registers this class as the default instance of [ZstandardPlatform]. - /// - /// Called by the main plugin when running on iOS. + /// Registers this implementation with the federated plugin. static void registerWith() { ZstandardPlatform.instance = ZstandardIOS(); } @override - Future getPlatformVersion() async { - final version = - await methodChannel.invokeMethod('getPlatformVersion'); - return version; - } + Future getPlatformVersion() => + methodChannel.invokeMethod('getPlatformVersion'); @override - Future compress(Uint8List data, int compressionLevel) async { - if (compressionLevel < 1 || compressionLevel > 22) { - return null; - } - - final int srcSize = data.lengthInBytes; - final Pointer src = - malloc.allocate(srcSize > 0 ? srcSize : 1); - src.asTypedList(srcSize).setAll(0, data); - - final int dstCapacity = _bindings.ZSTD_compressBound(srcSize); - if (_bindings.ZSTD_isError(dstCapacity) != 0 || dstCapacity <= 0) { - malloc.free(src); - return null; - } - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int compressedSize = _bindings.ZSTD_compress( - dst.cast(), - dstCapacity, - src.cast(), - srcSize, - compressionLevel, - ); - - if (_bindings.ZSTD_isError(compressedSize) == 0 && compressedSize > 0) { - return Uint8List.fromList(dst.asTypedList(compressedSize)); - } else { - return null; - } - } finally { - malloc.free(src); - malloc.free(dst); - } - } + Future compress(Uint8List data, int compressionLevel) => + Isolate.run(() => _codec.compress(data, compressionLevel)); @override - Future decompress(Uint8List data) async { - // Avoid entering the native decoder for arbitrary input. Apart from - // being cheaper, this prevents malformed data from reaching an ABI - // boundary while the frame header is already known to be invalid. - if (!_hasZstdFrameMagic(data)) { - return null; - } + Future decompress(Uint8List data) => decompressWithOptions(data); - final int compressedSize = data.lengthInBytes; - final Pointer src = malloc.allocate(compressedSize); - src.asTypedList(compressedSize).setAll(0, data); - - final int decompressedSizeExpected = - _bindings.ZSTD_getFrameContentSize(src.cast(), compressedSize); - if (_isUnavailableContentSize(decompressedSizeExpected)) { - malloc.free(src); - return null; - } - final int dstCapacity = - decompressedSizeExpected > 0 ? decompressedSizeExpected : 1; - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int decompressedSize = _bindings.ZSTD_decompress( - dst.cast(), - dstCapacity, - src.cast(), - compressedSize, + @override + Future decompressWithOptions( + Uint8List data, { + int maxOutputSize = ZstandardPlatform.defaultMaxDecompressedSize, + }) => + Isolate.run( + () => _hasZstdFrameMagic(data) + ? _codec.decompress(data, maxOutputSize: maxOutputSize) + : null, ); - - if (_bindings.ZSTD_isError(decompressedSize) != 0) { - return null; - } - return Uint8List.fromList(dst.asTypedList(decompressedSize)); - } finally { - malloc.free(src); - malloc.free(dst); - } - } } +/// Low-level synchronous compression for existing FFI consumers. int compress( Pointer dst, int dstCapacity, @@ -165,170 +82,36 @@ int compress( int srcSize, int compressionLevel, ) => - _bindings.ZSTD_compress( - dst, - dstCapacity, - src, - srcSize, - compressionLevel, - ); + _bindings.ZSTD_compress(dst, dstCapacity, src, srcSize, compressionLevel); +/// Low-level synchronous decompression for existing FFI consumers. int decompress( Pointer dst, int dstCapacity, Pointer src, int compressedSize, ) => - _bindings.ZSTD_decompress( - dst, - dstCapacity, - src, - compressedSize, - ); + _bindings.ZSTD_decompress(dst, dstCapacity, src, compressedSize); +/// Compatibility wrapper that completes after the pointer call returns. +@Deprecated('Use ZstandardIOS.compress with Dart-owned bytes instead.') Future compressAsync( Pointer dst, int dstCapacity, Pointer src, int srcSize, int compressionLevel, -) async { - final SendPort helperIsolateSendPort = await _getHelperIsolateSendPort(); - final int requestId = _nextCompressRequestId++; - final _CompressRequest request = _CompressRequest( - requestId, dst, dstCapacity, src, srcSize, compressionLevel); - final Completer completer = Completer(); - _compressRequests[requestId] = completer; - helperIsolateSendPort.send(request); - return completer.future; -} +) => + Future.sync( + () => compress(dst, dstCapacity, src, srcSize, compressionLevel), + ); +/// Compatibility wrapper that completes after the pointer call returns. +@Deprecated('Use ZstandardIOS.decompress with Dart-owned bytes instead.') Future decompressAsync( Pointer dst, int dstCapacity, Pointer src, int compressedSize, -) async { - final SendPort helperIsolateSendPort = await _getHelperIsolateSendPort(); - final int requestId = _nextDecompressRequestId++; - final _DecompressRequest request = - _DecompressRequest(requestId, dst, dstCapacity, src, compressedSize); - final Completer completer = Completer(); - _decompressRequests[requestId] = completer; - helperIsolateSendPort.send(request); - return completer.future; -} - -// ==== Communication between isolates for asynchronous compression and decompression ==== // - -/// Application for compression. -class _CompressRequest { - final int id; - final Pointer dst; - final int dstCapacity; - final Pointer src; - final int srcSize; - final int compressionLevel; - - const _CompressRequest(this.id, this.dst, this.dstCapacity, this.src, - this.srcSize, this.compressionLevel); -} - -/// Response with the compression result. -class _CompressResponse { - final int id; - final int result; - - const _CompressResponse(this.id, this.result); -} - -/// Request for decompression. -class _DecompressRequest { - final int id; - final Pointer dst; - final int dstCapacity; - final Pointer src; - final int compressedSize; - - const _DecompressRequest( - this.id, this.dst, this.dstCapacity, this.src, this.compressedSize); -} - -/// Response with the result of the decompression. -class _DecompressResponse { - final int id; - final int result; - - const _DecompressResponse(this.id, this.result); -} - -/// Counters to identify compression and decompression requests. -int _nextCompressRequestId = 0; -int _nextDecompressRequestId = 0; - -/// Mapping of requests to completers for compression and decompression. -final Map> _compressRequests = >{}; -final Map> _decompressRequests = >{}; - -/// Port for sending requests to the auxiliary isolate. -// Start the worker lazily so importing the plugin cannot keep test processes -// alive when the async helper API is not used. -Future? _helperIsolateSendPort; - -Future _getHelperIsolateSendPort() => - _helperIsolateSendPort ??= () async { - final Completer completer = Completer(); - final ReceivePort receivePort = ReceivePort() - ..listen((dynamic data) { - if (data is SendPort) { - completer.complete(data); - return; - } - if (data is _CompressResponse) { - final Completer completer = _compressRequests[data.id]!; - _compressRequests.remove(data.id); - completer.complete(data.result); - return; - } - if (data is _DecompressResponse) { - final Completer completer = _decompressRequests[data.id]!; - _decompressRequests.remove(data.id); - completer.complete(data.result); - return; - } - throw UnsupportedError( - 'Message type not supported: ${data.runtimeType}'); - }); - - await Isolate.spawn((SendPort sendPort) async { - final ReceivePort helperReceivePort = ReceivePort() - ..listen((dynamic data) { - if (data is _CompressRequest) { - final int result = _bindings.ZSTD_compress( - data.dst, - data.dstCapacity, - data.src, - data.srcSize, - data.compressionLevel); - final _CompressResponse response = - _CompressResponse(data.id, result); - sendPort.send(response); - return; - } - if (data is _DecompressRequest) { - final int result = _bindings.ZSTD_decompress( - data.dst, data.dstCapacity, data.src, data.compressedSize); - final _DecompressResponse response = - _DecompressResponse(data.id, result); - sendPort.send(response); - return; - } - throw UnsupportedError( - 'Message type not supported: ${data.runtimeType}'); - }); - - sendPort.send(helperReceivePort.sendPort); - }, receivePort.sendPort); - - return completer.future; - }(); +) => + Future.sync(() => decompress(dst, dstCapacity, src, compressedSize)); diff --git a/zstandard_ios/scripts/sync_zstd.sh b/zstandard_ios/scripts/sync_zstd.sh index b2ea888f..8e42447f 100644 --- a/zstandard_ios/scripts/sync_zstd.sh +++ b/zstandard_ios/scripts/sync_zstd.sh @@ -4,7 +4,8 @@ # does not use this generated directory. Works in repo and in the pub cache. # # Usage: ./scripts/sync_zstd.sh -# Resolves zstd from: 1) sibling zstandard_native (repo), 2) pub-cache sibling zstandard_native-*, 3) package_config.json (pub or repo). +# Resolves zstd from a repository sibling or the exact package selected in +# package_config.json. It never guesses among versioned Pub-cache directories. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -15,18 +16,7 @@ DEST="$PLUGIN_ROOT/ios/Classes/zstd" SRC="$PLUGIN_ROOT/../zstandard_native/src/zstd" if [[ ! -d "$SRC" || ! -f "$SRC/zstd.h" ]]; then SRC="" - # 2. Pub cache: sibling zstandard_native-* next to this plugin (e.g. .../pub.dev/zstandard_native-1.4.x) - CACHE_DIR="$PLUGIN_ROOT/.." - for NATIVE_PKG in "$CACHE_DIR"/zstandard_native-*; do - if [[ -d "$NATIVE_PKG" ]]; then - CANDIDATE="$NATIVE_PKG/src/zstd" - if [[ -d "$CANDIDATE" && -f "$CANDIDATE/zstd.h" ]]; then - SRC="$CANDIDATE" - break - fi - fi - done - # 3. package_config.json (walk up from plugin: plugin itself or app that depends on it) + # 2. package_config.json (walk up from plugin: plugin itself or app that depends on it) if [[ -z "$SRC" ]]; then SEARCH="$PLUGIN_ROOT" while [[ -n "$SEARCH" ]]; do diff --git a/zstandard_ios/test/podspec_test.dart b/zstandard_ios/test/podspec_test.dart index 80f964c4..a67eb4f6 100644 --- a/zstandard_ios/test/podspec_test.dart +++ b/zstandard_ios/test/podspec_test.dart @@ -10,6 +10,10 @@ void main() { expect(podspec, contains(":name => 'Sync zstd'")); expect(podspec, contains('zstandard_ios/Sources/zstandard_ios/*.swift')); expect(podspec, contains('-DZSTD_DISABLE_ASM')); + expect(podspec, contains("s.platform = :ios, '13.0'")); + expect(podspec, isNot(contains('legacy/'))); + expect(podspec, isNot(contains('dictBuilder/'))); + expect(podspec, isNot(contains('*.S'))); expect(podspec, contains(r'$(PODS_TARGET_SRCROOT)/Classes/zstd/zstd.h')); expect(podspec, isNot(contains('Remove synced zstd'))); expect(podspec, isNot(contains('rm -rf'))); diff --git a/zstandard_ios/test/spm_package_test.dart b/zstandard_ios/test/spm_package_test.dart index 12647295..f76bdf25 100644 --- a/zstandard_ios/test/spm_package_test.dart +++ b/zstandard_ios/test/spm_package_test.dart @@ -5,32 +5,40 @@ import 'package:flutter_test/flutter_test.dart'; void main() { test('Swift Package Manager consumes the shared native package', () { final manifest = File('ios/zstandard_ios/Package.swift').readAsStringSync(); + final pubspec = File('pubspec.yaml').readAsStringSync(); + final version = RegExp( + r'^version:\s*(\d+\.\d+\.\d+)$', + multiLine: true, + ).firstMatch(pubspec)!.group(1)!; final rootManifest = File('../Package.swift'); - final duplicateSourceDirectory = - Directory('ios/zstandard_ios/Sources/zstd'); + final duplicateSourceDirectory = Directory( + 'ios/zstandard_ios/Sources/zstd', + ); expect(manifest, contains('name: "zstandard_ios"')); expect(manifest, contains('name: "zstandard-ios"')); expect(manifest, contains('path: "Sources/zstandard_ios"')); expect(manifest, contains('ZSTANDARD_NATIVE_PACKAGE_PATH')); expect(manifest, contains('https://github.com/vypdev/zstandard.git')); - // Development checkouts use develop; the release workflow rewrites this - // to an exact immutable version before tagging a published archive. - expect( - manifest, - anyOf( - contains('branch: "develop"'), - matches(RegExp(r'exact:\s*"\d+\.\d+\.\d+"')), - ), - ); - if (manifest.contains('exact:')) { - expect(manifest, isNot(contains('branch: "develop"'))); - } + expect(manifest, contains('exact: "$version"')); + expect(manifest, isNot(contains('branch:'))); expect(manifest, contains('product(name: "zstandard-native"')); expect(duplicateSourceDirectory.existsSync(), isFalse); if (rootManifest.existsSync()) { - expect(rootManifest.readAsStringSync(), - contains('path: "zstandard_native/src/zstd"')); + final rootContents = rootManifest.readAsStringSync(); + expect(rootContents, contains('path: "zstandard_native/src/zstd"')); + for (final symbol in [ + '_ZSTD_compress', + '_ZSTD_decompress', + '_ZSTD_isError', + '_ZSTD_createDStream', + '_ZSTD_initDStream', + '_ZSTD_decompressStream', + '_ZSTD_freeDStream', + '_ZSTD_DStreamOutSize', + ]) { + expect(rootContents, contains(symbol)); + } } }); } diff --git a/zstandard_linux/CHANGELOG.md b/zstandard_linux/CHANGELOG.md index b92135b7..69100c37 100644 --- a/zstandard_linux/CHANGELOG.md +++ b/zstandard_linux/CHANGELOG.md @@ -1,8 +1,12 @@ ## Unreleased +- Added byte-safe worker-isolate execution and bounded streaming decompression + for unknown-size and concatenated frames. - Build and run the native Linux CMake test target in CI. - Extend Linux integration coverage to assert non-empty compression output and exact byte-for-byte decompression round-trips. +- Removed the placeholder `WIP` test so skipped macOS-hosted runs cannot be + mistaken for executed Linux behavior. ## 1.5.0 - Dependencies Updated diff --git a/zstandard_linux/README.md b/zstandard_linux/README.md index 3c9052b5..3c2f6496 100644 --- a/zstandard_linux/README.md +++ b/zstandard_linux/README.md @@ -10,7 +10,7 @@ Add the main plugin to your app; this package is included automatically via the ```yaml dependencies: - zstandard: ^1.3.29 + zstandard: ^1.5.0 ``` No extra setup is required for normal use. @@ -42,12 +42,15 @@ final decompressed = await compressed?.decompress(); - **ZstandardLinux()** — Creates the Linux platform implementation. - **compress(Uint8List data, int compressionLevel)** — Compresses `data` (level 1–22). Returns compressed bytes or `null`. -- **decompress(Uint8List data)** — Decompresses zstd-compressed data. Returns decompressed bytes or `null`. +- **decompressWithOptions(Uint8List data, {int maxOutputSize})** — Decompresses complete, concatenated, or unknown-size zstd frames with a bounded output (256 MiB by default). Invalid, truncated, or oversized input returns `null`. - **getPlatformVersion()** — Returns a platform identifier string. ## Architecture -This package uses Dart FFI to load `libzstandard_linux_plugin.so` and call the Zstandard C API. The native library is built by CMake when you build your Flutter Linux app. +This package uses Dart FFI to load `libzstandard_linux_plugin.so`; CMake builds +it with the Flutter application. Public work runs in a worker isolate using +Dart-owned bytes, and decompression enforces the configured output limit as it +streams. ## Testing diff --git a/zstandard_linux/analysis_options.yaml b/zstandard_linux/analysis_options.yaml index a5744c1c..612ee4bf 100644 --- a/zstandard_linux/analysis_options.yaml +++ b/zstandard_linux/analysis_options.yaml @@ -1,3 +1,7 @@ +analyzer: + exclude: + - build/** + - linux/** include: package:flutter_lints/flutter.yaml # Additional information about this file can be found at diff --git a/zstandard_linux/example/analysis_options.yaml b/zstandard_linux/example/analysis_options.yaml index 0d290213..cbe5dc5b 100644 --- a/zstandard_linux/example/analysis_options.yaml +++ b/zstandard_linux/example/analysis_options.yaml @@ -7,6 +7,10 @@ # The following line activates a set of recommended lints for Flutter apps, # packages, and plugins designed to encourage good coding practices. +analyzer: + exclude: + - build/** + - linux/** include: package:flutter_lints/flutter.yaml linter: diff --git a/zstandard_linux/lib/zstandard_ext.dart b/zstandard_linux/lib/zstandard_ext.dart index acd86a70..af965ca5 100644 --- a/zstandard_linux/lib/zstandard_ext.dart +++ b/zstandard_linux/lib/zstandard_ext.dart @@ -9,9 +9,12 @@ extension ZstandardExt on Uint8List? { return ZstandardLinux().compress(data, compressionLevel); } - Future decompress() async { + Future decompress({int maxOutputSize = 256 * 1024 * 1024}) async { var data = this; if (data == null) return null; - return ZstandardLinux().decompress(data); + return ZstandardLinux().decompressWithOptions( + data, + maxOutputSize: maxOutputSize, + ); } } diff --git a/zstandard_linux/lib/zstandard_linux.dart b/zstandard_linux/lib/zstandard_linux.dart index ba174c93..f190f4eb 100644 --- a/zstandard_linux/lib/zstandard_linux.dart +++ b/zstandard_linux/lib/zstandard_linux.dart @@ -1,153 +1,69 @@ -import 'dart:async'; import 'dart:ffi'; import 'dart:io'; import 'dart:isolate'; -import 'package:ffi/ffi.dart'; import 'package:flutter/services.dart'; -import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; - +import 'package:zstandard_native/zstandard_native.dart' + show ZstandardNativeCodec; import 'package:zstandard_native/zstandard_native_bindings.dart'; +import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; export 'zstandard_ext.dart'; const String _libName = 'zstandard_linux_plugin'; final DynamicLibrary _dylib = () { - if (Platform.isLinux) { - return DynamicLibrary.open('lib$_libName.so'); - } + if (Platform.isLinux) return DynamicLibrary.open('lib$_libName.so'); throw UnsupportedError('Platform not supported: ${Platform.operatingSystem}'); }(); final ZstandardNativeBindings _bindings = ZstandardNativeBindings(_dylib); +final ZstandardNativeCodec _codec = ZstandardNativeCodec(_bindings); bool _hasZstdFrameMagic(Uint8List data) { - if (data.lengthInBytes < 4) { - return false; - } - - final int magic = - data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); + if (data.lengthInBytes < 4) return false; + final magic = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); return magic == ZSTD_MAGICNUMBER || (magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START; } -bool _isUnavailableContentSize(int size) { - // The generated bindings expose these C unsigned values as -1 and -2. - // Keep the unsigned representations as a compatibility guard for older - // generated bindings and runtimes that returned the raw 64-bit bit pattern. - return size == ZSTD_CONTENTSIZE_UNKNOWN || - size == ZSTD_CONTENTSIZE_ERROR || - size == 0xffffffffffffffff || - size == 0xfffffffffffffffe; -} - -/// Linux implementation of [ZstandardPlatform] using FFI and the native zstd library. -/// -/// Uses [DynamicLibrary] to load libzstandard_linux_plugin.so and calls -/// ZSTD_compress, ZSTD_decompress, ZSTD_compressBound, and ZSTD_getFrameContentSize. -/// The main [zstandard] plugin registers this implementation automatically on Linux. -class ZstandardLinux extends ZstandardPlatform { +/// Linux implementation of [ZstandardPlatform] using the native zstd library. +class ZstandardLinux extends ZstandardPlatform + implements BoundedZstandardPlatform { /// Creates the Linux platform implementation. ZstandardLinux(); final methodChannel = const MethodChannel('plugins.flutter.io/zstandard'); - /// Registers this class as the default instance of [ZstandardPlatform]. - /// - /// Called by the main plugin when running on Linux. + /// Registers this implementation with the federated plugin. static void registerWith() { ZstandardPlatform.instance = ZstandardLinux(); } @override - Future getPlatformVersion() async { - final version = - await methodChannel.invokeMethod('getPlatformVersion'); - return version; - } + Future getPlatformVersion() => + methodChannel.invokeMethod('getPlatformVersion'); @override - Future compress(Uint8List data, int compressionLevel) async { - if (compressionLevel < 1 || compressionLevel > 22) { - return null; - } - - final int srcSize = data.lengthInBytes; - final Pointer src = - malloc.allocate(srcSize > 0 ? srcSize : 1); - src.asTypedList(srcSize).setAll(0, data); - - final int dstCapacity = _bindings.ZSTD_compressBound(srcSize); - if (_bindings.ZSTD_isError(dstCapacity) != 0 || dstCapacity <= 0) { - malloc.free(src); - return null; - } - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int compressedSize = _bindings.ZSTD_compress( - dst.cast(), - dstCapacity, - src.cast(), - srcSize, - compressionLevel, - ); - - if (_bindings.ZSTD_isError(compressedSize) == 0 && compressedSize > 0) { - return Uint8List.fromList(dst.asTypedList(compressedSize)); - } else { - return null; - } - } finally { - malloc.free(src); - malloc.free(dst); - } - } + Future compress(Uint8List data, int compressionLevel) => + Isolate.run(() => _codec.compress(data, compressionLevel)); @override - Future decompress(Uint8List data) async { - // Avoid entering the native decoder for arbitrary input. Apart from - // being cheaper, this prevents malformed data from reaching an ABI - // boundary while the frame header is already known to be invalid. - if (!_hasZstdFrameMagic(data)) { - return null; - } + Future decompress(Uint8List data) => decompressWithOptions(data); - final int compressedSize = data.lengthInBytes; - final Pointer src = malloc.allocate(compressedSize); - src.asTypedList(compressedSize).setAll(0, data); - - final int decompressedSizeExpected = - _bindings.ZSTD_getFrameContentSize(src.cast(), compressedSize); - if (_isUnavailableContentSize(decompressedSizeExpected)) { - malloc.free(src); - return null; - } - final int dstCapacity = - decompressedSizeExpected > 0 ? decompressedSizeExpected : 1; - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int decompressedSize = _bindings.ZSTD_decompress( - dst.cast(), - dstCapacity, - src.cast(), - compressedSize, + @override + Future decompressWithOptions( + Uint8List data, { + int maxOutputSize = ZstandardPlatform.defaultMaxDecompressedSize, + }) => + Isolate.run( + () => _hasZstdFrameMagic(data) + ? _codec.decompress(data, maxOutputSize: maxOutputSize) + : null, ); - - if (_bindings.ZSTD_isError(decompressedSize) != 0) { - return null; - } - return Uint8List.fromList(dst.asTypedList(decompressedSize)); - } finally { - malloc.free(src); - malloc.free(dst); - } - } } +/// Low-level synchronous compression for existing FFI consumers. int compress( Pointer dst, int dstCapacity, @@ -155,170 +71,36 @@ int compress( int srcSize, int compressionLevel, ) => - _bindings.ZSTD_compress( - dst, - dstCapacity, - src, - srcSize, - compressionLevel, - ); + _bindings.ZSTD_compress(dst, dstCapacity, src, srcSize, compressionLevel); +/// Low-level synchronous decompression for existing FFI consumers. int decompress( Pointer dst, int dstCapacity, Pointer src, int compressedSize, ) => - _bindings.ZSTD_decompress( - dst, - dstCapacity, - src, - compressedSize, - ); + _bindings.ZSTD_decompress(dst, dstCapacity, src, compressedSize); +/// Compatibility wrapper that completes after the pointer call returns. +@Deprecated('Use ZstandardLinux.compress with Dart-owned bytes instead.') Future compressAsync( Pointer dst, int dstCapacity, Pointer src, int srcSize, int compressionLevel, -) async { - final SendPort helperIsolateSendPort = await _getHelperIsolateSendPort(); - final int requestId = _nextCompressRequestId++; - final _CompressRequest request = _CompressRequest( - requestId, dst, dstCapacity, src, srcSize, compressionLevel); - final Completer completer = Completer(); - _compressRequests[requestId] = completer; - helperIsolateSendPort.send(request); - return completer.future; -} +) => + Future.sync( + () => compress(dst, dstCapacity, src, srcSize, compressionLevel), + ); +/// Compatibility wrapper that completes after the pointer call returns. +@Deprecated('Use ZstandardLinux.decompress with Dart-owned bytes instead.') Future decompressAsync( Pointer dst, int dstCapacity, Pointer src, int compressedSize, -) async { - final SendPort helperIsolateSendPort = await _getHelperIsolateSendPort(); - final int requestId = _nextDecompressRequestId++; - final _DecompressRequest request = - _DecompressRequest(requestId, dst, dstCapacity, src, compressedSize); - final Completer completer = Completer(); - _decompressRequests[requestId] = completer; - helperIsolateSendPort.send(request); - return completer.future; -} - -// ==== Communication between isolates for asynchronous compression and decompression ==== // - -/// Application for compression. -class _CompressRequest { - final int id; - final Pointer dst; - final int dstCapacity; - final Pointer src; - final int srcSize; - final int compressionLevel; - - const _CompressRequest(this.id, this.dst, this.dstCapacity, this.src, - this.srcSize, this.compressionLevel); -} - -/// Response with the compression result. -class _CompressResponse { - final int id; - final int result; - - const _CompressResponse(this.id, this.result); -} - -/// Request for decompression. -class _DecompressRequest { - final int id; - final Pointer dst; - final int dstCapacity; - final Pointer src; - final int compressedSize; - - const _DecompressRequest( - this.id, this.dst, this.dstCapacity, this.src, this.compressedSize); -} - -/// Response with the result of the decompression. -class _DecompressResponse { - final int id; - final int result; - - const _DecompressResponse(this.id, this.result); -} - -/// Counters to identify compression and decompression requests. -int _nextCompressRequestId = 0; -int _nextDecompressRequestId = 0; - -/// Mapping of requests to completers for compression and decompression. -final Map> _compressRequests = >{}; -final Map> _decompressRequests = >{}; - -/// Port for sending requests to the auxiliary isolate. -// Start the worker lazily so importing the plugin cannot keep test processes -// alive when the async helper API is not used. -Future? _helperIsolateSendPort; - -Future _getHelperIsolateSendPort() => - _helperIsolateSendPort ??= () async { - final Completer completer = Completer(); - final ReceivePort receivePort = ReceivePort() - ..listen((dynamic data) { - if (data is SendPort) { - completer.complete(data); - return; - } - if (data is _CompressResponse) { - final Completer completer = _compressRequests[data.id]!; - _compressRequests.remove(data.id); - completer.complete(data.result); - return; - } - if (data is _DecompressResponse) { - final Completer completer = _decompressRequests[data.id]!; - _decompressRequests.remove(data.id); - completer.complete(data.result); - return; - } - throw UnsupportedError( - 'Message type not supported: ${data.runtimeType}'); - }); - - await Isolate.spawn((SendPort sendPort) async { - final ReceivePort helperReceivePort = ReceivePort() - ..listen((dynamic data) { - if (data is _CompressRequest) { - final int result = _bindings.ZSTD_compress( - data.dst, - data.dstCapacity, - data.src, - data.srcSize, - data.compressionLevel); - final _CompressResponse response = - _CompressResponse(data.id, result); - sendPort.send(response); - return; - } - if (data is _DecompressRequest) { - final int result = _bindings.ZSTD_decompress( - data.dst, data.dstCapacity, data.src, data.compressedSize); - final _DecompressResponse response = - _DecompressResponse(data.id, result); - sendPort.send(response); - return; - } - throw UnsupportedError( - 'Message type not supported: ${data.runtimeType}'); - }); - - sendPort.send(helperReceivePort.sendPort); - }, receivePort.sendPort); - - return completer.future; - }(); +) => + Future.sync(() => decompress(dst, dstCapacity, src, compressedSize)); diff --git a/zstandard_linux/test/tests_exist_elsewhere_test.dart b/zstandard_linux/test/tests_exist_elsewhere_test.dart deleted file mode 100644 index 1478dc75..00000000 --- a/zstandard_linux/test/tests_exist_elsewhere_test.dart +++ /dev/null @@ -1,11 +0,0 @@ -// ignore_for_file: avoid_print - -import 'package:flutter_test/flutter_test.dart'; - -void main() { - test('Tell the user where to find the real tests', () { - print('---'); - print('WIP'); - print('---'); - }); -} \ No newline at end of file diff --git a/zstandard_linux/test/zstandard_linux_test.dart b/zstandard_linux/test/zstandard_linux_test.dart index 28b561f9..2d00ebd9 100644 --- a/zstandard_linux/test/zstandard_linux_test.dart +++ b/zstandard_linux/test/zstandard_linux_test.dart @@ -27,7 +27,9 @@ void main() { test('compress and decompress large data', () async { if (skipPlatform) return; - final data = Uint8List.fromList(List.generate(100000, (i) => i % 256)); + final data = Uint8List.fromList( + List.generate(100000, (i) => i % 256), + ); final compressed = await zstandard.compress(data, 3); expect(compressed, isNotNull); final decompressed = await zstandard.decompress(compressed!); @@ -63,7 +65,9 @@ void main() { test('decompress random bytes returns null', () async { if (skipPlatform) return; - final random = Uint8List.fromList(List.generate(64, (i) => (i * 31) % 256)); + final random = Uint8List.fromList( + List.generate(64, (i) => (i * 31) % 256), + ); final result = await zstandard.decompress(random); expect(result, isNull); }, skip: skipPlatform ? 'Only runs on Linux' : false); diff --git a/zstandard_linux/zstd_build/CMakeLists.txt b/zstandard_linux/zstd_build/CMakeLists.txt index e00fb17f..f123f8bb 100644 --- a/zstandard_linux/zstd_build/CMakeLists.txt +++ b/zstandard_linux/zstd_build/CMakeLists.txt @@ -33,17 +33,8 @@ else() message(STATUS "Using local zstandard_native: ${ZSTD_ROOT}") endif() if(NOT ZSTD_ROOT OR NOT EXISTS "${ZSTD_ROOT}/common") - # 2. Look for zstandard_native in pub-cache (versioned dirs only, e.g. zstandard_native-1.4.0) - file(GLOB NATIVE_PARENT_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/../../zstandard_native-*") - if(NATIVE_PARENT_DIRS) - list(GET NATIVE_PARENT_DIRS 0 NATIVE_PKG_DIR) - set(ZSTD_CANDIDATE "${NATIVE_PKG_DIR}/src/zstd") - if(EXISTS "${ZSTD_CANDIDATE}/common") - set(ZSTD_ROOT "${ZSTD_CANDIDATE}") - message(STATUS "Using zstandard_native from pub-cache: ${ZSTD_ROOT}") - endif() - endif() - # 3. Resolve zstandard_native from package_config.json (plugin or any parent, e.g. example app) + # 2. Resolve the exact dependency selected by pub. Never choose the first + # matching cache directory, which can silently compile a stale version. set(_SEARCH_PKG "${CMAKE_CURRENT_SOURCE_DIR}") foreach(_PKG_IGNORE RANGE 10) get_filename_component(_PARENT "${_SEARCH_PKG}/.." ABSOLUTE) @@ -72,7 +63,7 @@ else() set(_SEARCH_PKG "${_PARENT}") endforeach() if(NOT ZSTD_ROOT OR NOT EXISTS "${ZSTD_ROOT}/common") - message(FATAL_ERROR "zstd source not found. Set ZSTD_ROOT or ensure zstandard_native is available (repo: ../../zstandard_native/src/zstd or pub-cache).") + message(FATAL_ERROR "zstd source not found. Set ZSTD_ROOT or run flutter pub get so package_config.json resolves zstandard_native.") endif() endif() endif() diff --git a/zstandard_macos/CHANGELOG.md b/zstandard_macos/CHANGELOG.md index 6b507993..4e5b6ee1 100644 --- a/zstandard_macos/CHANGELOG.md +++ b/zstandard_macos/CHANGELOG.md @@ -1,5 +1,7 @@ ## Unreleased +- Added byte-safe worker-isolate execution and bounded streaming decompression; + aligned CocoaPods and SwiftPM native sources and pinned SwiftPM releases. - Fixed intermittent macOS build failures caused by deleting synced zstd sources during compilation. - Added Swift Package Manager support while retaining CocoaPods compatibility. diff --git a/zstandard_macos/README.md b/zstandard_macos/README.md index f820e37f..0c2e74a9 100644 --- a/zstandard_macos/README.md +++ b/zstandard_macos/README.md @@ -10,7 +10,7 @@ Add the main plugin to your app; this package is included automatically via the ```yaml dependencies: - zstandard: ^1.3.29 + zstandard: ^1.5.0 ``` No extra setup is required for normal use. @@ -42,12 +42,17 @@ final decompressed = await compressed?.decompress(); - **ZstandardMacOS()** — Creates the macOS platform implementation. - **compress(Uint8List data, int compressionLevel)** — Compresses `data` (level 1–22). Returns compressed bytes or `null`. -- **decompress(Uint8List data)** — Decompresses zstd-compressed data. Returns decompressed bytes or `null`. +- **decompressWithOptions(Uint8List data, {int maxOutputSize})** — Decompresses complete, concatenated, or unknown-size zstd frames with a bounded output (256 MiB by default). Invalid, truncated, or oversized input returns `null`. - **getPlatformVersion()** — Returns a platform identifier string. ## Architecture -This package uses Dart FFI with the native zstd C library. Swift Package Manager is the primary integration for Flutter 3.44 and newer and statically links the shared target into the app; CocoaPods remains supported for compatibility and embeds the plugin framework. Both paths use the same canonical C implementation from `zstandard_native`. Supports x64 and arm64 (Apple Silicon). +This package uses Dart FFI with the native zstd C library. Swift Package +Manager statically links the shared target into the app; CocoaPods remains +supported and embeds the plugin framework. Both paths compile the same +common/compress/decompress source set from `zstandard_native`. Public work runs +in an isolate using Dart-owned bytes and bounded streaming decompression. +Both x64 and arm64 are supported. ## Testing diff --git a/zstandard_macos/analysis_options.yaml b/zstandard_macos/analysis_options.yaml index a5744c1c..5e15f457 100644 --- a/zstandard_macos/analysis_options.yaml +++ b/zstandard_macos/analysis_options.yaml @@ -1,3 +1,7 @@ +analyzer: + exclude: + - build/** + - macos/** include: package:flutter_lints/flutter.yaml # Additional information about this file can be found at diff --git a/zstandard_macos/example/analysis_options.yaml b/zstandard_macos/example/analysis_options.yaml index 0d290213..5d3e697c 100644 --- a/zstandard_macos/example/analysis_options.yaml +++ b/zstandard_macos/example/analysis_options.yaml @@ -7,6 +7,10 @@ # The following line activates a set of recommended lints for Flutter apps, # packages, and plugins designed to encourage good coding practices. +analyzer: + exclude: + - build/** + - macos/** include: package:flutter_lints/flutter.yaml linter: diff --git a/zstandard_macos/images/sample.png b/zstandard_macos/images/sample.png index b338c0e3..d55a33f3 100644 Binary files a/zstandard_macos/images/sample.png and b/zstandard_macos/images/sample.png differ diff --git a/zstandard_macos/lib/zstandard_ext.dart b/zstandard_macos/lib/zstandard_ext.dart index 57eacf23..365da15e 100644 --- a/zstandard_macos/lib/zstandard_ext.dart +++ b/zstandard_macos/lib/zstandard_ext.dart @@ -9,9 +9,12 @@ extension ZstandardExt on Uint8List? { return ZstandardMacOS().compress(data, compressionLevel); } - Future decompress() async { + Future decompress({int maxOutputSize = 256 * 1024 * 1024}) async { var data = this; if (data == null) return null; - return ZstandardMacOS().decompress(data); + return ZstandardMacOS().decompressWithOptions( + data, + maxOutputSize: maxOutputSize, + ); } } diff --git a/zstandard_macos/lib/zstandard_macos.dart b/zstandard_macos/lib/zstandard_macos.dart index d1496487..a566040d 100644 --- a/zstandard_macos/lib/zstandard_macos.dart +++ b/zstandard_macos/lib/zstandard_macos.dart @@ -1,13 +1,12 @@ -import 'dart:async'; import 'dart:ffi'; import 'dart:io'; import 'dart:isolate'; -import 'package:ffi/ffi.dart'; import 'package:flutter/services.dart'; -import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; - +import 'package:zstandard_native/zstandard_native.dart' + show ZstandardNativeCodec; import 'package:zstandard_native/zstandard_native_bindings.dart'; +import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; export 'zstandard_ext.dart'; @@ -16,12 +15,12 @@ const String _libName = 'zstandard_macos'; final DynamicLibrary _dylib = () { if (!Platform.isMacOS) { throw UnsupportedError( - 'Platform not supported: ${Platform.operatingSystem}'); + 'Platform not supported: ${Platform.operatingSystem}', + ); } // CocoaPods embeds a dynamic framework. Flutter's SwiftPM integration - // links the same C target statically into the application, so those symbols - // are exposed through the process instead of a standalone framework. + // links the same C target statically into the application. try { return DynamicLibrary.open('$_libName.framework/$_libName'); } on ArgumentError { @@ -30,134 +29,52 @@ final DynamicLibrary _dylib = () { }(); final ZstandardNativeBindings _bindings = ZstandardNativeBindings(_dylib); +final ZstandardNativeCodec _codec = ZstandardNativeCodec(_bindings); bool _hasZstdFrameMagic(Uint8List data) { - if (data.lengthInBytes < 4) { - return false; - } - - final int magic = - data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); + if (data.lengthInBytes < 4) return false; + final magic = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); return magic == ZSTD_MAGICNUMBER || (magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START; } -bool _isUnavailableContentSize(int size) { - // The generated bindings expose these C unsigned values as -1 and -2. - // Keep the unsigned representations as a compatibility guard for older - // generated bindings and runtimes that returned the raw 64-bit bit pattern. - return size == ZSTD_CONTENTSIZE_UNKNOWN || - size == ZSTD_CONTENTSIZE_ERROR || - size == 0xffffffffffffffff || - size == 0xfffffffffffffffe; -} - -/// macOS implementation of [ZstandardPlatform] using FFI and the native zstd library. -/// -/// Resolves ZSTD_compress, ZSTD_decompress, ZSTD_compressBound, and -/// ZSTD_getFrameContentSize from the CocoaPods framework or the statically -/// linked SwiftPM application. The main [zstandard] plugin registers this -/// implementation automatically on macOS. -class ZstandardMacOS extends ZstandardPlatform { +/// macOS implementation of [ZstandardPlatform] using the native zstd library. +class ZstandardMacOS extends ZstandardPlatform + implements BoundedZstandardPlatform { /// Creates the macOS platform implementation. ZstandardMacOS(); final methodChannel = const MethodChannel('plugins.flutter.io/zstandard'); - /// Registers this class as the default instance of [ZstandardPlatform]. - /// - /// Called by the main plugin when running on macOS. + /// Registers this implementation with the federated plugin. static void registerWith() { ZstandardPlatform.instance = ZstandardMacOS(); } @override - Future getPlatformVersion() async { - final version = - await methodChannel.invokeMethod('getPlatformVersion'); - return version; - } + Future getPlatformVersion() => + methodChannel.invokeMethod('getPlatformVersion'); @override - Future compress(Uint8List data, int compressionLevel) async { - if (compressionLevel < 1 || compressionLevel > 22) { - return null; - } - - final int srcSize = data.lengthInBytes; - final Pointer src = - malloc.allocate(srcSize > 0 ? srcSize : 1); - src.asTypedList(srcSize).setAll(0, data); - - final int dstCapacity = _bindings.ZSTD_compressBound(srcSize); - if (_bindings.ZSTD_isError(dstCapacity) != 0 || dstCapacity <= 0) { - malloc.free(src); - return null; - } - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int compressedSize = _bindings.ZSTD_compress( - dst.cast(), - dstCapacity, - src.cast(), - srcSize, - compressionLevel, - ); - - if (_bindings.ZSTD_isError(compressedSize) == 0 && compressedSize > 0) { - return Uint8List.fromList(dst.asTypedList(compressedSize)); - } else { - return null; - } - } finally { - malloc.free(src); - malloc.free(dst); - } - } + Future compress(Uint8List data, int compressionLevel) => + Isolate.run(() => _codec.compress(data, compressionLevel)); @override - Future decompress(Uint8List data) async { - // Avoid entering the native decoder for arbitrary input. Apart from - // being cheaper, this prevents malformed data from reaching an ABI - // boundary while the frame header is already known to be invalid. - if (!_hasZstdFrameMagic(data)) { - return null; - } + Future decompress(Uint8List data) => decompressWithOptions(data); - final int compressedSize = data.lengthInBytes; - final Pointer src = malloc.allocate(compressedSize); - src.asTypedList(compressedSize).setAll(0, data); - - final int decompressedSizeExpected = - _bindings.ZSTD_getFrameContentSize(src.cast(), compressedSize); - if (_isUnavailableContentSize(decompressedSizeExpected)) { - malloc.free(src); - return null; - } - final int dstCapacity = - decompressedSizeExpected > 0 ? decompressedSizeExpected : 1; - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int decompressedSize = _bindings.ZSTD_decompress( - dst.cast(), - dstCapacity, - src.cast(), - compressedSize, + @override + Future decompressWithOptions( + Uint8List data, { + int maxOutputSize = ZstandardPlatform.defaultMaxDecompressedSize, + }) => + Isolate.run( + () => _hasZstdFrameMagic(data) + ? _codec.decompress(data, maxOutputSize: maxOutputSize) + : null, ); - - if (_bindings.ZSTD_isError(decompressedSize) != 0) { - return null; - } - return Uint8List.fromList(dst.asTypedList(decompressedSize)); - } finally { - malloc.free(src); - malloc.free(dst); - } - } } +/// Low-level synchronous compression for existing FFI consumers. int compress( Pointer dst, int dstCapacity, @@ -165,170 +82,36 @@ int compress( int srcSize, int compressionLevel, ) => - _bindings.ZSTD_compress( - dst, - dstCapacity, - src, - srcSize, - compressionLevel, - ); + _bindings.ZSTD_compress(dst, dstCapacity, src, srcSize, compressionLevel); +/// Low-level synchronous decompression for existing FFI consumers. int decompress( Pointer dst, int dstCapacity, Pointer src, int compressedSize, ) => - _bindings.ZSTD_decompress( - dst, - dstCapacity, - src, - compressedSize, - ); + _bindings.ZSTD_decompress(dst, dstCapacity, src, compressedSize); +/// Compatibility wrapper that completes after the pointer call returns. +@Deprecated('Use ZstandardMacOS.compress with Dart-owned bytes instead.') Future compressAsync( Pointer dst, int dstCapacity, Pointer src, int srcSize, int compressionLevel, -) async { - final SendPort helperIsolateSendPort = await _getHelperIsolateSendPort(); - final int requestId = _nextCompressRequestId++; - final _CompressRequest request = _CompressRequest( - requestId, dst, dstCapacity, src, srcSize, compressionLevel); - final Completer completer = Completer(); - _compressRequests[requestId] = completer; - helperIsolateSendPort.send(request); - return completer.future; -} +) => + Future.sync( + () => compress(dst, dstCapacity, src, srcSize, compressionLevel), + ); +/// Compatibility wrapper that completes after the pointer call returns. +@Deprecated('Use ZstandardMacOS.decompress with Dart-owned bytes instead.') Future decompressAsync( Pointer dst, int dstCapacity, Pointer src, int compressedSize, -) async { - final SendPort helperIsolateSendPort = await _getHelperIsolateSendPort(); - final int requestId = _nextDecompressRequestId++; - final _DecompressRequest request = - _DecompressRequest(requestId, dst, dstCapacity, src, compressedSize); - final Completer completer = Completer(); - _decompressRequests[requestId] = completer; - helperIsolateSendPort.send(request); - return completer.future; -} - -// ==== Communication between isolates for asynchronous compression and decompression ==== // - -/// Application for compression. -class _CompressRequest { - final int id; - final Pointer dst; - final int dstCapacity; - final Pointer src; - final int srcSize; - final int compressionLevel; - - const _CompressRequest(this.id, this.dst, this.dstCapacity, this.src, - this.srcSize, this.compressionLevel); -} - -/// Response with the compression result. -class _CompressResponse { - final int id; - final int result; - - const _CompressResponse(this.id, this.result); -} - -/// Request for decompression. -class _DecompressRequest { - final int id; - final Pointer dst; - final int dstCapacity; - final Pointer src; - final int compressedSize; - - const _DecompressRequest( - this.id, this.dst, this.dstCapacity, this.src, this.compressedSize); -} - -/// Response with the result of the decompression. -class _DecompressResponse { - final int id; - final int result; - - const _DecompressResponse(this.id, this.result); -} - -/// Counters to identify compression and decompression requests. -int _nextCompressRequestId = 0; -int _nextDecompressRequestId = 0; - -/// Mapping of requests to completers for compression and decompression. -final Map> _compressRequests = >{}; -final Map> _decompressRequests = >{}; - -/// Port for sending requests to the auxiliary isolate. -// Start the worker lazily so importing the plugin cannot keep test processes -// alive when the async helper API is not used. -Future? _helperIsolateSendPort; - -Future _getHelperIsolateSendPort() => - _helperIsolateSendPort ??= () async { - final Completer completer = Completer(); - final ReceivePort receivePort = ReceivePort() - ..listen((dynamic data) { - if (data is SendPort) { - completer.complete(data); - return; - } - if (data is _CompressResponse) { - final Completer completer = _compressRequests[data.id]!; - _compressRequests.remove(data.id); - completer.complete(data.result); - return; - } - if (data is _DecompressResponse) { - final Completer completer = _decompressRequests[data.id]!; - _decompressRequests.remove(data.id); - completer.complete(data.result); - return; - } - throw UnsupportedError( - 'Message type not supported: ${data.runtimeType}'); - }); - - await Isolate.spawn((SendPort sendPort) async { - final ReceivePort helperReceivePort = ReceivePort() - ..listen((dynamic data) { - if (data is _CompressRequest) { - final int result = _bindings.ZSTD_compress( - data.dst, - data.dstCapacity, - data.src, - data.srcSize, - data.compressionLevel); - final _CompressResponse response = - _CompressResponse(data.id, result); - sendPort.send(response); - return; - } - if (data is _DecompressRequest) { - final int result = _bindings.ZSTD_decompress( - data.dst, data.dstCapacity, data.src, data.compressedSize); - final _DecompressResponse response = - _DecompressResponse(data.id, result); - sendPort.send(response); - return; - } - throw UnsupportedError( - 'Message type not supported: ${data.runtimeType}'); - }); - - sendPort.send(helperReceivePort.sendPort); - }, receivePort.sendPort); - - return completer.future; - }(); +) => + Future.sync(() => decompress(dst, dstCapacity, src, compressedSize)); diff --git a/zstandard_macos/macos/zstandard_macos.podspec b/zstandard_macos/macos/zstandard_macos.podspec index 14f5e1a2..15855f74 100644 --- a/zstandard_macos/macos/zstandard_macos.podspec +++ b/zstandard_macos/macos/zstandard_macos.podspec @@ -5,14 +5,14 @@ Pod::Spec.new do |s| s.cocoapods_version = '>= 1.11.0' # for script_phase :before_headers s.name = 'zstandard_macos' - s.version = '0.0.1' - s.summary = 'A new Flutter FFI plugin project.' + s.version = '1.5.0' + s.summary = 'macOS implementation of the Zstandard Flutter plugin.' s.description = <<-DESC -A new Flutter FFI plugin project. +Native Zstandard compression and bounded decompression for Flutter on macOS. DESC - s.homepage = 'http://example.com' + s.homepage = 'https://github.com/vypdev/zstandard' s.license = { :file => '../LICENSE' } - s.author = { 'Your Company' => 'email@example.com' } + s.author = { 'VypDev' => 'https://github.com/vypdev' } # Zstd C sources: synced from zstandard_native/src/zstd/ into Classes/zstd/ by # scripts/sync_zstd.sh (in this plugin). Must exist at pod install time so source_files glob finds them. @@ -21,9 +21,6 @@ A new Flutter FFI plugin project. 'Classes/zstd/common/*.c', 'Classes/zstd/common/*.h', 'Classes/zstd/compress/*.c', 'Classes/zstd/compress/*.h', 'Classes/zstd/decompress/*.c', 'Classes/zstd/decompress/*.h', - 'Classes/zstd/decompress/*.S', - 'Classes/zstd/dictBuilder/*.c', 'Classes/zstd/dictBuilder/*.h', - 'Classes/zstd/legacy/*.c', 'Classes/zstd/legacy/*.h', 'Classes/zstd/*.h' s.private_header_files = 'Classes/zstd/**/*.h' @@ -32,13 +29,14 @@ A new Flutter FFI plugin project. s.dependency 'FlutterMacOS' - s.platform = :osx, '10.11' + s.platform = :osx, '10.15' # Export zstd C symbols so Dart FFI (DynamicLibrary.lookup) can find them in the framework. s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'HEADER_SEARCH_PATHS' => '$(PODS_TARGET_SRCROOT)/Classes/zstd', 'CLANG_ALLOW_NON_MODULAR_INCLUDES_IN_FRAMEWORK_MODULES' => 'YES', - 'OTHER_CFLAGS' => '$(inherited) -DZSTD_STATIC_LINKING_ONLY -DZSTD_DISABLE_ASM -fvisibility=default', + 'OTHER_CFLAGS' => '$(inherited) -DZSTD_DISABLE_ASM -fvisibility=default', + 'GCC_WARN_INHIBIT_ALL_WARNINGS' => 'YES', 'DEAD_CODE_STRIPPING' => 'NO', 'STRIP_INSTALLED_PRODUCT' => 'NO', } diff --git a/zstandard_macos/macos/zstandard_macos/Package.swift b/zstandard_macos/macos/zstandard_macos/Package.swift index 225bbabf..dcaed68c 100644 --- a/zstandard_macos/macos/zstandard_macos/Package.swift +++ b/zstandard_macos/macos/zstandard_macos/Package.swift @@ -7,11 +7,10 @@ let nativePackageDependency: Package.Dependency = { return .package(name: "zstandard", path: localPath) } - // Keep the C implementation in the repository-level SwiftPM package. - // Pin this to a release tag before publishing the plugin package. + // Match the immutable repository tag carrying this plugin release. return .package( url: "https://github.com/vypdev/zstandard.git", - branch: "develop" + exact: "1.5.0" ) }() diff --git a/zstandard_macos/scripts/sync_zstd.sh b/zstandard_macos/scripts/sync_zstd.sh index 9134e4c2..336d5645 100644 --- a/zstandard_macos/scripts/sync_zstd.sh +++ b/zstandard_macos/scripts/sync_zstd.sh @@ -4,7 +4,8 @@ # does not use this generated directory. Works in repo and in the pub cache. # # Usage: ./scripts/sync_zstd.sh -# Resolves zstd from: 1) sibling zstandard_native (repo), 2) pub-cache sibling zstandard_native-*, 3) package_config.json (pub or repo). +# Resolves zstd from a repository sibling or the exact package selected in +# package_config.json. It never guesses among versioned Pub-cache directories. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -15,18 +16,7 @@ DEST="$PLUGIN_ROOT/macos/Classes/zstd" SRC="$PLUGIN_ROOT/../zstandard_native/src/zstd" if [[ ! -d "$SRC" || ! -f "$SRC/zstd.h" ]]; then SRC="" - # 2. Pub cache: sibling zstandard_native-* next to this plugin (e.g. .../pub.dev/zstandard_native-1.4.x) - CACHE_DIR="$PLUGIN_ROOT/.." - for NATIVE_PKG in "$CACHE_DIR"/zstandard_native-*; do - if [[ -d "$NATIVE_PKG" ]]; then - CANDIDATE="$NATIVE_PKG/src/zstd" - if [[ -d "$CANDIDATE" && -f "$CANDIDATE/zstd.h" ]]; then - SRC="$CANDIDATE" - break - fi - fi - done - # 3. package_config.json (walk up from plugin: plugin itself or app that depends on it) + # 2. package_config.json (walk up from plugin: plugin itself or app that depends on it) if [[ -z "$SRC" ]]; then SEARCH="$PLUGIN_ROOT" while [[ -n "$SEARCH" ]]; do diff --git a/zstandard_macos/test/podspec_test.dart b/zstandard_macos/test/podspec_test.dart index 4259a002..33275ab3 100644 --- a/zstandard_macos/test/podspec_test.dart +++ b/zstandard_macos/test/podspec_test.dart @@ -8,8 +8,15 @@ void main() { expect(podspec, contains('s.script_phases = [')); expect(podspec, contains(":name => 'Sync zstd'")); - expect(podspec, contains('zstandard_macos/Sources/zstandard_macos/*.swift')); + expect( + podspec, + contains('zstandard_macos/Sources/zstandard_macos/*.swift'), + ); expect(podspec, contains('-DZSTD_DISABLE_ASM')); + expect(podspec, contains("s.platform = :osx, '10.15'")); + expect(podspec, isNot(contains('legacy/'))); + expect(podspec, isNot(contains('dictBuilder/'))); + expect(podspec, isNot(contains('*.S'))); expect(podspec, contains(r'$(PODS_TARGET_SRCROOT)/Classes/zstd/zstd.h')); expect(podspec, isNot(contains('Remove synced zstd'))); expect(podspec, isNot(contains('rm -rf'))); diff --git a/zstandard_macos/test/spm_package_test.dart b/zstandard_macos/test/spm_package_test.dart index 4cb07556..e7995772 100644 --- a/zstandard_macos/test/spm_package_test.dart +++ b/zstandard_macos/test/spm_package_test.dart @@ -6,32 +6,40 @@ void main() { test('Swift Package Manager consumes the shared native package', () { final manifest = File('macos/zstandard_macos/Package.swift').readAsStringSync(); + final pubspec = File('pubspec.yaml').readAsStringSync(); + final version = RegExp( + r'^version:\s*(\d+\.\d+\.\d+)$', + multiLine: true, + ).firstMatch(pubspec)!.group(1)!; final rootManifest = File('../Package.swift'); - final duplicateSourceDirectory = - Directory('macos/zstandard_macos/Sources/zstd'); + final duplicateSourceDirectory = Directory( + 'macos/zstandard_macos/Sources/zstd', + ); expect(manifest, contains('name: "zstandard_macos"')); expect(manifest, contains('name: "zstandard-macos"')); expect(manifest, contains('path: "Sources/zstandard_macos"')); expect(manifest, contains('ZSTANDARD_NATIVE_PACKAGE_PATH')); expect(manifest, contains('https://github.com/vypdev/zstandard.git')); - // Development checkouts use develop; the release workflow rewrites this - // to an exact immutable version before tagging a published archive. - expect( - manifest, - anyOf( - contains('branch: "develop"'), - matches(RegExp(r'exact:\s*"\d+\.\d+\.\d+"')), - ), - ); - if (manifest.contains('exact:')) { - expect(manifest, isNot(contains('branch: "develop"'))); - } + expect(manifest, contains('exact: "$version"')); + expect(manifest, isNot(contains('branch:'))); expect(manifest, contains('product(name: "zstandard-native"')); expect(duplicateSourceDirectory.existsSync(), isFalse); if (rootManifest.existsSync()) { - expect(rootManifest.readAsStringSync(), - contains('path: "zstandard_native/src/zstd"')); + final rootContents = rootManifest.readAsStringSync(); + expect(rootContents, contains('path: "zstandard_native/src/zstd"')); + for (final symbol in [ + '_ZSTD_compress', + '_ZSTD_decompress', + '_ZSTD_isError', + '_ZSTD_createDStream', + '_ZSTD_initDStream', + '_ZSTD_decompressStream', + '_ZSTD_freeDStream', + '_ZSTD_DStreamOutSize', + ]) { + expect(rootContents, contains(symbol)); + } } }); } diff --git a/zstandard_native/CHANGELOG.md b/zstandard_native/CHANGELOG.md index 14965197..9f2ff0fc 100644 --- a/zstandard_native/CHANGELOG.md +++ b/zstandard_native/CHANGELOG.md @@ -1,3 +1,10 @@ +## Unreleased + +- Added the shared byte-safe codec with bounded streaming decompression. +- Recorded the exact upstream zstd revision and applied upstream's allocation + null-check backport. +- Made the zstd update script deterministic and safe to rerun. + ## 1.5.0 - Dependencies Updated - Updated direct dependencies diff --git a/zstandard_native/README.md b/zstandard_native/README.md index 1ac2123e..b374916d 100644 --- a/zstandard_native/README.md +++ b/zstandard_native/README.md @@ -6,6 +6,8 @@ Native zstd C sources and FFI bindings used by the [zstandard](https://pub.dev/p - **`src/zstd/`** — C source code from [facebook/zstd](https://github.com/facebook/zstd) (common, compress, decompress). - **`lib/zstandard_native_bindings.dart`** — FFI bindings generated from `zstd.h` for use by all platform plugins. +- **`lib/src/zstandard_native_codec.dart`** — Shared bounded byte-oriented codec used by every native implementation. +- **`UPSTREAM_ZSTD.md`** — Exact upstream version, commit, and locally applied upstream backport. ## Usage @@ -13,7 +15,11 @@ This package is a dependency of the platform-specific zstandard plugins. End use ### Development (monorepo) -From the repository root, the main **example** app uses `dependency_overrides` in `zstandard/example/pubspec.yaml` so that all packages (including `zstandard_native`) resolve from path. Run `flutter pub get` and builds from the example. When publishing to pub.dev, publish **zstandard_native** first, then the platform packages and the main plugin (they depend on `zstandard_native: ^1.4.0`). +From the repository root, run `scripts/create_local_overrides.sh ` to +resolve the monorepo packages by path for local tests. Generated +`pubspec_overrides.yaml` files are ignored and must not be published. During a +release, publish `zstandard_native` after the platform interface and before +the platform packages and CLI. ### Regenerating bindings @@ -37,4 +43,6 @@ From the repository root: ./scripts/update_zstd.sh ``` -This updates `zstandard_native/src/zstd/` from the official facebook/zstd repository. +This refreshes `zstandard_native/src/zstd/` from the exact requested upstream +revision (the repository default is pinned). Update `UPSTREAM_ZSTD.md` whenever +the revision or the recorded backport changes. diff --git a/zstandard_native/UPSTREAM_ZSTD.md b/zstandard_native/UPSTREAM_ZSTD.md new file mode 100644 index 00000000..2f271a66 --- /dev/null +++ b/zstandard_native/UPSTREAM_ZSTD.md @@ -0,0 +1,17 @@ +# Vendored Zstandard provenance + +The canonical native sources under `src/zstd/` come from the official +[`facebook/zstd`](https://github.com/facebook/zstd) repository. + +- Base revision: `d7ee3207cc0db53f78fc6a69babc80747b1b7658` +- Reported library version: `1.5.7` +- Backported upstream fix: + `3f8f9b3f89244638f10bca664c120fd28cb14efe` (guard a failed custom + allocation before `memset`) +- Imported scope: upstream `lib/`, preserving this repository's + `src/zstd/include/` SwiftPM bridge headers + +`scripts/update_zstd.sh` uses the pinned base revision by default, applies the +backport when it is not already present, and records the resolved revision. +Review upstream release notes and security advisories before changing either +revision. diff --git a/zstandard_native/lib/src/zstandard_native_codec.dart b/zstandard_native/lib/src/zstandard_native_codec.dart new file mode 100644 index 00000000..27b696d2 --- /dev/null +++ b/zstandard_native/lib/src/zstandard_native_codec.dart @@ -0,0 +1,154 @@ +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:ffi/ffi.dart'; + +import '../zstandard_native_bindings.dart'; + +/// Default maximum number of bytes produced by one decompression operation. +const int nativeDefaultMaxDecompressedSize = 256 * 1024 * 1024; + +/// Memory-safe, bounded convenience wrapper around the generated zstd bindings. +/// +/// Instances do not retain native state and may be recreated in worker isolates. +final class ZstandardNativeCodec { + /// Creates a codec backed by [bindings]. + const ZstandardNativeCodec(this.bindings); + + /// The generated zstd bindings used by this codec. + final ZstandardNativeBindings bindings; + + /// Compresses [data] into one valid zstd frame. + Uint8List? compress(Uint8List data, int compressionLevel) { + if (compressionLevel < 1 || compressionLevel > 22) { + return null; + } + + Pointer? src; + Pointer? dst; + try { + final srcSize = data.lengthInBytes; + src = malloc.allocate(srcSize == 0 ? 1 : srcSize); + if (srcSize > 0) { + src.asTypedList(srcSize).setAll(0, data); + } + + final dstCapacity = bindings.ZSTD_compressBound(srcSize); + if (bindings.ZSTD_isError(dstCapacity) != 0 || dstCapacity <= 0) { + return null; + } + dst = malloc.allocate(dstCapacity); + + final compressedSize = bindings.ZSTD_compress( + dst.cast(), + dstCapacity, + src.cast(), + srcSize, + compressionLevel, + ); + if (bindings.ZSTD_isError(compressedSize) != 0 || compressedSize <= 0) { + return null; + } + return Uint8List.fromList(dst.asTypedList(compressedSize)); + } on ArgumentError { + // package:ffi reports a failed native allocation as ArgumentError. + return null; + } finally { + if (src != null) malloc.free(src); + if (dst != null) malloc.free(dst); + } + } + + /// Decompresses complete zstd/skippable frames without trusting frame sizes. + /// + /// Streaming decompression supports frames that omit their content size. The + /// operation fails before returning more than [maxOutputSize] bytes. + Uint8List? decompress( + Uint8List data, { + int maxOutputSize = nativeDefaultMaxDecompressedSize, + }) { + if (data.isEmpty || maxOutputSize < 0) { + return null; + } + + final stream = bindings.ZSTD_createDStream(); + if (stream == nullptr) { + return null; + } + + Pointer? src; + Pointer? input; + Pointer? output; + Pointer? chunk; + final result = BytesBuilder(copy: false); + + try { + src = malloc.allocate(data.lengthInBytes); + input = calloc(); + output = calloc(); + final recommendedOutputSize = bindings.ZSTD_DStreamOutSize(); + final chunkCapacity = + recommendedOutputSize > 0 ? recommendedOutputSize : 128 * 1024; + chunk = malloc.allocate(chunkCapacity); + + src.asTypedList(data.lengthInBytes).setAll(0, data); + input.ref + ..src = src.cast() + ..size = data.lengthInBytes + ..pos = 0; + output.ref + ..dst = chunk.cast() + ..size = chunkCapacity + ..pos = 0; + + final initialization = bindings.ZSTD_initDStream(stream); + if (bindings.ZSTD_isError(initialization) != 0) { + return null; + } + + var previousInputPosition = -1; + var previousOutputLength = -1; + while (true) { + output.ref + ..dst = chunk.cast() + ..size = chunkCapacity + ..pos = 0; + + final remaining = bindings.ZSTD_decompressStream(stream, output, input); + if (bindings.ZSTD_isError(remaining) != 0) { + return null; + } + + final produced = output.ref.pos; + if (result.length + produced > maxOutputSize) { + return null; + } + if (produced > 0) { + result.add(Uint8List.fromList(chunk.asTypedList(produced))); + } + + final allInputConsumed = input.ref.pos == input.ref.size; + if (remaining == 0 && allInputConsumed) { + return result.takeBytes(); + } + + final madeProgress = input.ref.pos != previousInputPosition || + result.length != previousOutputLength; + if (!madeProgress || (allInputConsumed && produced == 0)) { + // More input is required: the frame is truncated or incomplete. + return null; + } + previousInputPosition = input.ref.pos; + previousOutputLength = result.length; + } + } on ArgumentError { + return null; + } finally { + bindings.ZSTD_freeDStream(stream); + if (src != null) malloc.free(src); + if (input != null) calloc.free(input); + if (output != null) calloc.free(output); + if (chunk != null) malloc.free(chunk); + } + } +} diff --git a/zstandard_native/lib/zstandard_native.dart b/zstandard_native/lib/zstandard_native.dart index 8ded64ac..296dbc7e 100644 --- a/zstandard_native/lib/zstandard_native.dart +++ b/zstandard_native/lib/zstandard_native.dart @@ -1,4 +1,7 @@ /// Native zstd C sources and FFI bindings for the zstandard plugin. /// /// Export the generated bindings for use by platform implementations. +library; + export 'zstandard_native_bindings.dart'; +export 'src/zstandard_native_codec.dart'; diff --git a/zstandard_native/src/zstd/common/allocations.h b/zstandard_native/src/zstd/common/allocations.h index 5e899550..0b7576a5 100644 --- a/zstandard_native/src/zstd/common/allocations.h +++ b/zstandard_native/src/zstd/common/allocations.h @@ -36,7 +36,9 @@ MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) /* calloc implemented as malloc+memset; * not as efficient as calloc, but next best guess for custom malloc */ void* const ptr = customMem.customAlloc(customMem.opaque, size); - ZSTD_memset(ptr, 0, size); + if (ptr != NULL) { + ZSTD_memset(ptr, 0, size); + } return ptr; } return ZSTD_calloc(1, size); diff --git a/zstandard_platform_interface/CHANGELOG.md b/zstandard_platform_interface/CHANGELOG.md index 14965197..e05b52d7 100644 --- a/zstandard_platform_interface/CHANGELOG.md +++ b/zstandard_platform_interface/CHANGELOG.md @@ -1,3 +1,8 @@ +## Unreleased + +- Added the source-compatible `BoundedZstandardPlatform` capability and a + shared 256 MiB default decompression limit. + ## 1.5.0 - Dependencies Updated - Updated direct dependencies diff --git a/zstandard_platform_interface/README.md b/zstandard_platform_interface/README.md index 7b456d63..2ec2771f 100644 --- a/zstandard_platform_interface/README.md +++ b/zstandard_platform_interface/README.md @@ -16,6 +16,11 @@ platform-specific behavior, and when you register your plugin, set the default `ZstandardPlatform` by calling `ZstandardPlatform.instance = MyPlatformZstandard()`. +Implementations that can enforce a decompressed-output budget should also +implement `BoundedZstandardPlatform`. The main package detects this optional +capability while preserving source compatibility for existing third-party +implementations of `ZstandardPlatform`. + # Note on breaking changes Strongly prefer non-breaking changes (such as adding a method to the interface) @@ -25,4 +30,4 @@ See https://flutter.dev/go/platform-interface-breaking-changes for a discussion on why a less-clean interface is preferable to a breaking change. [1]: ../zstandard -[2]: lib/zstandard_platform_interface.dart \ No newline at end of file +[2]: lib/zstandard_platform_interface.dart diff --git a/zstandard_platform_interface/lib/src/zstandard_platform.dart b/zstandard_platform_interface/lib/src/zstandard_platform.dart index 7be6937b..b9f900df 100644 --- a/zstandard_platform_interface/lib/src/zstandard_platform.dart +++ b/zstandard_platform_interface/lib/src/zstandard_platform.dart @@ -12,6 +12,9 @@ import 'zstandard_platform_interface_method_channel.dart'; /// /// Application code should use the main [zstandard] package, not this interface. abstract class ZstandardPlatform extends PlatformInterface { + /// Default maximum output accepted by [decompress]. + static const int defaultMaxDecompressedSize = 256 * 1024 * 1024; + /// Constructs a [ZstandardPlatform]. ZstandardPlatform() : super(token: _token); @@ -56,3 +59,12 @@ abstract class ZstandardPlatform extends PlatformInterface { throw UnimplementedError('decompress() has not been implemented.'); } } + +/// Optional capability implemented by platforms that enforce an output limit. +abstract interface class BoundedZstandardPlatform { + /// Decompresses [data] while limiting the maximum produced byte count. + Future decompressWithOptions( + Uint8List data, { + int maxOutputSize = ZstandardPlatform.defaultMaxDecompressedSize, + }); +} diff --git a/zstandard_platform_interface/lib/src/zstandard_platform_interface_method_channel.dart b/zstandard_platform_interface/lib/src/zstandard_platform_interface_method_channel.dart index ddbc1877..fa173487 100644 --- a/zstandard_platform_interface/lib/src/zstandard_platform_interface_method_channel.dart +++ b/zstandard_platform_interface/lib/src/zstandard_platform_interface_method_channel.dart @@ -4,16 +4,16 @@ import 'package:flutter/services.dart'; import 'zstandard_platform.dart'; /// An implementation of [ZstandardPlatform] that uses method channels. -class MethodChannelZstandardPlatform - extends ZstandardPlatform { +class MethodChannelZstandardPlatform extends ZstandardPlatform { /// The method channel used to interact with the native platform. @visibleForTesting final methodChannel = const MethodChannel('plugins.flutter.io/zstandard'); @override Future getPlatformVersion() async { - final version = - await methodChannel.invokeMethod('getPlatformVersion'); + final version = await methodChannel.invokeMethod( + 'getPlatformVersion', + ); return version; } } diff --git a/zstandard_platform_interface/test/properties_test.dart b/zstandard_platform_interface/test/properties_test.dart index b1b3f241..738739ff 100644 --- a/zstandard_platform_interface/test/properties_test.dart +++ b/zstandard_platform_interface/test/properties_test.dart @@ -5,7 +5,9 @@ import 'package:kiri_check/kiri_check.dart'; import 'package:plugin_platform_interface/plugin_platform_interface.dart'; import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; -class RoundtripMockPlatform with MockPlatformInterfaceMixin implements ZstandardPlatform { +class RoundtripMockPlatform + with MockPlatformInterfaceMixin + implements ZstandardPlatform { @override Future getPlatformVersion() => Future.value('mock'); @@ -23,24 +25,25 @@ class RoundtripMockPlatform with MockPlatformInterfaceMixin implements Zstandard void main() { group('Property-based tests', () { property('mock roundtrip: decompress(compress(x)) == x', () { - forAll( - binary(minLength: 0, maxLength: 500), - (List data) async { - final saved = ZstandardPlatform.instance; - ZstandardPlatform.instance = RoundtripMockPlatform(); - try { - final input = Uint8List.fromList(data); - final compressed = await ZstandardPlatform.instance.compress(input, 3); - expect(compressed, isNotNull); - final decompressed = await ZstandardPlatform.instance.decompress(compressed!); - expect(decompressed, isNotNull); - expect(List.from(decompressed!), data); - } finally { - ZstandardPlatform.instance = saved; - } - }, - maxExamples: 100, - ); + forAll(binary(minLength: 0, maxLength: 500), (List data) async { + final saved = ZstandardPlatform.instance; + ZstandardPlatform.instance = RoundtripMockPlatform(); + try { + final input = Uint8List.fromList(data); + final compressed = await ZstandardPlatform.instance.compress( + input, + 3, + ); + expect(compressed, isNotNull); + final decompressed = await ZstandardPlatform.instance.decompress( + compressed!, + ); + expect(decompressed, isNotNull); + expect(List.from(decompressed!), data); + } finally { + ZstandardPlatform.instance = saved; + } + }, maxExamples: 100); }); }); } diff --git a/zstandard_platform_interface/test/zstandard_platform_interface_test.dart b/zstandard_platform_interface/test/zstandard_platform_interface_test.dart index 1de95159..ed002bee 100644 --- a/zstandard_platform_interface/test/zstandard_platform_interface_test.dart +++ b/zstandard_platform_interface/test/zstandard_platform_interface_test.dart @@ -5,7 +5,9 @@ import 'package:plugin_platform_interface/plugin_platform_interface.dart'; import 'package:zstandard_platform_interface/src/zstandard_platform_interface_method_channel.dart'; import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; -class MockZstandardPlatform with MockPlatformInterfaceMixin implements ZstandardPlatform { +class MockZstandardPlatform + with MockPlatformInterfaceMixin + implements ZstandardPlatform { @override Future getPlatformVersion() => Future.value('42'); @@ -131,7 +133,10 @@ void main() { expect(await ZstandardPlatform.instance.getPlatformVersion(), '42'); ZstandardPlatform.instance = MockB(); - expect(await ZstandardPlatform.instance.getPlatformVersion(), 'PlatformB'); + expect( + await ZstandardPlatform.instance.getPlatformVersion(), + 'PlatformB', + ); final data = Uint8List.fromList([1]); final c = await ZstandardPlatform.instance.compress(data, 3); expect(c, Uint8List.fromList([9, 9])); diff --git a/zstandard_web/CHANGELOG.md b/zstandard_web/CHANGELOG.md index 0550e473..cca7fd9c 100644 --- a/zstandard_web/CHANGELOG.md +++ b/zstandard_web/CHANGELOG.md @@ -1,5 +1,9 @@ ## Unreleased +- Moved zstd execution to a dedicated Web Worker with transferable buffers. +- Added consistent null semantics, bounded streaming decompression, + unknown-size and concatenated frame support, valid empty frames, and + release-browser integration coverage. - Pin the Emscripten toolchain used to regenerate the checked-in WebAssembly artifacts from the canonical `zstandard_native` C source. - Normalize WebAssembly metadata with the matching `wasm-opt` tool and validate diff --git a/zstandard_web/README.md b/zstandard_web/README.md index 805ea8ef..d0171acd 100644 --- a/zstandard_web/README.md +++ b/zstandard_web/README.md @@ -2,174 +2,90 @@ # zstandard_web -The web implementation of [`zstandard`](https://pub.dev/packages/zstandard). +Web implementation of [`zstandard`](https://pub.dev/packages/zstandard), built +from the same pinned zstd C source as the native packages. ## Installation -Copy `zstd.js` and `zstd.wasm` into your app's `web/` folder. In this repo they are built into `zstandard_web/blob/` and `zstandard_web/example/web/` (see Generation below). +Copy all four files from this package's `blob/` directory into the Flutter +application's `web/` directory: -Include the library inside the ``: +- `zstd.js` — small main-thread request broker; +- `zstd_worker.js` — dedicated Worker entry point; +- `zstd_core.js` — generated Emscripten runtime and safe wrappers; and +- `zstd.wasm` — compiled zstd implementation. + +Load only the broker before Flutter starts: ```html - - - ``` -## Usage +The four files must remain together at the same relative URL. The broker +starts the Worker lazily, transfers byte buffers to it, and keeps compression +and decompression away from the browser UI thread. -```dart -void act() async { - final zstandard = ZstandardWeb(); +## Usage - Uint8List original = Uint8List.fromList([...]); +Applications normally use the federated `zstandard` package. Direct use is +also available: - Uint8List? compressed = await zstandard.compress(original); - - Uint8List? decompressed = await zstandard.decompress(compressed ?? Uint8List(0)); -} +```dart +final codec = ZstandardWeb(); +final compressed = await codec.compress(data, 3); +final decompressed = compressed == null + ? null + : await codec.decompressWithOptions( + compressed, + maxOutputSize: 64 * 1024 * 1024, + ); ``` -

+Compression levels are 1–22. Empty input produces a valid frame. +Decompression streams unknown-size and concatenated frames, checking every +output chunk against the configured limit. It returns `null` for malformed or +truncated input, unsupported values, Worker/WASM failures, or oversized output. +The default limit is 256 MiB. ## Generation -The repo builds `zstd.js` and `zstd.wasm` from the **same** `zstandard_native/src/zstd/` C source used by Android, iOS, macOS, Windows, Linux, and the CLI. From the repository root: +From the repository root: ```bash ./scripts/build_web_wasm.sh ``` -This script temporarily clones [Emscripten SDK (emsdk)](https://github.com/emscripten-core/emsdk) at the pinned SDK revision used by CI, installs and activates Emscripten 3.1.69, compiles `zstandard_native/src/zstd/` with `emcc`, normalizes the WASM metadata with the matching `wasm-opt` tool, appends the `compressData`/`decompressData` wrappers, and writes `zstd.js` and `zstd.wasm` to the web package and examples. Requires `git` and a shell; the temporary emsdk directory is removed after the build. Set `EMSDK_DIR` to reuse an existing local SDK, or override `EMSDK_REF`/`EMSCRIPTEN_VERSION` when intentionally updating the toolchain. - -The JavaScript glue is expected to be byte-for-byte reproducible. The WASM module is validated and all generated copies are synchronized, but Emscripten/LLVM can emit different valid WASM layouts on different host architectures; CI therefore proves the freshly regenerated module through the ChromeDriver integration test instead of comparing raw WASM bytes with an artifact generated on another architecture. - -### Manual generation (optional) - -If you prefer to build by hand (e.g. from upstream facebook/zstd or a custom emsdk install): - -1. Install and activate [Emscripten SDK](https://github.com/emscripten-core/emsdk): - - ```bash - git clone https://github.com/emscripten-core/emsdk.git - cd emsdk - ./emsdk install latest - ./emsdk activate latest - source ./emsdk_env.sh - ``` - -2. From the **repo root** (so the single source `zstd/` is used), or from a clone of [facebook/zstd](https://github.com/facebook/zstd) (using `lib/` in place of `zstd/`), run: - - ```bash - # If using this repo's zstandard_native (from repo root): - cd zstandard_native/src/zstd - emcc -O3 \ - $(find common -name "*.c") \ - $(find compress -name "*.c") \ - $(find decompress -name "*.c") \ - -I. -Icommon -Icompress -Idecompress \ - -s WASM=1 \ - -s EXPORT_NAME="zstdWasmModule" \ - -s EXPORTED_FUNCTIONS="['_ZSTD_compress', '_ZSTD_decompress', '_malloc', '_free', '_ZSTD_getFrameContentSize', '_ZSTD_compressBound']" \ - -o zstd.js - ``` - - Then append the `compressData` and `decompressData` wrappers to the generated `zstd.js` (see below), and copy `zstd.js` and `zstd.wasm` into both `zstandard_web/blob/` and `zstandard_web/example/web/`. - -Add these methods to `zstd.js` (the script does this automatically): - -```js -function compressData(inputData, compressionLevel) { - let inputPtr = Module._malloc(inputData.length); - Module.HEAPU8.set(inputData, inputPtr); - - let outputBufferSize = Module._ZSTD_compressBound(inputData.length); - let outputPtr = Module._malloc(outputBufferSize); - - let compressedSize = Module._ZSTD_compress( - outputPtr, - outputBufferSize, - inputPtr, - inputData.length, - compressionLevel - ); - - if (compressedSize < 0) { - console.error('Compression error, error code: ', compressedSize); - return null; - } else { - let compressedData = new Uint8Array(Module.HEAPU8.buffer, outputPtr, compressedSize); - - Module._free(inputPtr); - Module._free(outputPtr); - - return compressedData; - } -} - -function decompressData(compressedData) { - let compressedPtr = Module._malloc(compressedData.length); - Module.HEAPU8.set(compressedData, compressedPtr); - - let decompressedSize = Module._ZSTD_getFrameContentSize(compressedPtr, compressedData.length); - if (decompressedSize === -1 || decompressedSize === -2) { - console.error('Error in obtaining the original size of the data'); - Module._free(compressedPtr); - return null; - } - - let decompressedPtr = Module._malloc(decompressedSize); - - let resultSize = Module._ZSTD_decompress( - decompressedPtr, - decompressedSize, - compressedPtr, - compressedData.length - ); - - if (resultSize < 0) { - console.error('Decompression error, error code: ', resultSize); - Module._free(compressedPtr); - Module._free(decompressedPtr); - return null; - } else { - let decompressedData = new Uint8Array(Module.HEAPU8.buffer, decompressedPtr, resultSize); - - Module._free(compressedPtr); - Module._free(decompressedPtr); - - return decompressedData; - } -} -``` - -## API - -- **ZstandardWeb()** — Creates the web platform implementation. -- **compress(Uint8List data, int compressionLevel)** — Compresses `data` (level 1–22). Returns compressed bytes or throws on failure. Inputs smaller than 9 bytes may be returned unchanged. -- **decompress(Uint8List data)** — Decompresses zstd-compressed data. Returns decompressed bytes or throws on failure. -- **getPlatformVersion()** — Returns the browser user agent string. +The script uses the pinned Emscripten SDK revision and version, compiles +`zstandard_native/src/zstd/`, strips host metadata, and synchronizes all four +outputs into the package and both examples. Set `EMSDK_DIR` to reuse an +existing SDK. Change `EMSDK_REF` or `EMSCRIPTEN_VERSION` only as an intentional +toolchain update. -## Architecture - -This package uses JavaScript interop and WebAssembly. It calls the global `compressData` and `decompressData` functions provided by `zstd.js`, which in turn use the compiled zstd C library in `zstd.wasm`. No Dart FFI; runs on the main thread. +The JavaScript outputs are expected to be byte-for-byte reproducible. CI also +validates the WASM module and executes the real Worker/WASM integration in +Chrome. ## Testing -From the package directory: - ```bash flutter test +./scripts/test_web_integration.sh ``` -Unit tests run only on web (skipped on other platforms). Full integration tests are in `example/integration_test/` and require ChromeDriver. Run them with `./scripts/test_web_integration.sh` from the repository root. +Run the second command from the repository root with ChromeDriver installed. +It builds the release example and exercises the generated artifacts in a real +browser. ## Troubleshooting -- **compressData / decompressData is not defined**: Ensure `zstd.js` is included in your `web/index.html` and loads before the Flutter app. -- **WASM load failed**: Ensure `zstd.wasm` is served from the same origin and the path is correct. Check the browser console and network tab. +- If `compressData` is undefined, load `zstd.js` before Flutter. +- If the Worker or WASM fails to load, deploy all four files together and + inspect their requests in the browser network panel. +- A `null` result is an operation failure; lower the input size, verify the + frame, or raise `maxOutputSize` only when the application can safely accept + the larger allocation. -See the [documentation](https://github.com/vypdev/zstandard/tree/master/docs) for more. +See the repository [Web platform guide](https://github.com/vypdev/zstandard/blob/master/docs/platforms/web.md) +for the full architecture and deployment contract. diff --git a/zstandard_web/analysis_options.yaml b/zstandard_web/analysis_options.yaml index a5744c1c..e46654b9 100644 --- a/zstandard_web/analysis_options.yaml +++ b/zstandard_web/analysis_options.yaml @@ -1,3 +1,6 @@ +analyzer: + exclude: + - build/** include: package:flutter_lints/flutter.yaml # Additional information about this file can be found at diff --git a/zstandard_web/blob/zstd.js b/zstandard_web/blob/zstd.js index 82898927..d1963b65 100644 --- a/zstandard_web/blob/zstd.js +++ b/zstandard_web/blob/zstd.js @@ -1,90 +1,53 @@ -var Module=typeof zstdWasmModule!="undefined"?zstdWasmModule:{};var ENVIRONMENT_IS_WEB=typeof window=="object";var ENVIRONMENT_IS_WORKER=typeof importScripts=="function";var ENVIRONMENT_IS_NODE=typeof process=="object"&&typeof process.versions=="object"&&typeof process.versions.node=="string"&&process.type!="renderer";if(ENVIRONMENT_IS_NODE){}var moduleOverrides=Object.assign({},Module);var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("fs");var nodePath=require("path");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);var ret=fs.readFileSync(filename);return ret};readAsync=(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);return new Promise((resolve,reject)=>{fs.readFile(filename,binary?undefined:"utf8",(err,data)=>{if(err)reject(err);else resolve(binary?data.buffer:data)})})};if(!Module["thisProgram"]&&process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}arguments_=process.argv.slice(2);if(typeof module!="undefined"){module["exports"]=Module}quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){if(ENVIRONMENT_IS_WORKER){scriptDirectory=self.location.href}else if(typeof document!="undefined"&&document.currentScript){scriptDirectory=document.currentScript.src}if(scriptDirectory.startsWith("blob:")){scriptDirectory=""}else{scriptDirectory=scriptDirectory.substr(0,scriptDirectory.replace(/[?#].*/,"").lastIndexOf("/")+1)}{if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=url=>{if(isFileURI(url)){return new Promise((resolve,reject)=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=()=>{if(xhr.status==200||xhr.status==0&&xhr.response){resolve(xhr.response);return}reject(xhr.status)};xhr.onerror=reject;xhr.send(null)})}return fetch(url,{credentials:"same-origin"}).then(response=>{if(response.ok){return response.arrayBuffer()}return Promise.reject(new Error(response.status+" : "+response.url))})}}}else{}var out=Module["print"]||console.log.bind(console);var err=Module["printErr"]||console.error.bind(console);Object.assign(Module,moduleOverrides);moduleOverrides=null;if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];var wasmBinary=Module["wasmBinary"];var wasmMemory;var ABORT=false;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;function updateMemoryViews(){var b=wasmMemory.buffer;Module["HEAP8"]=HEAP8=new Int8Array(b);Module["HEAP16"]=HEAP16=new Int16Array(b);Module["HEAPU8"]=HEAPU8=new Uint8Array(b);Module["HEAPU16"]=HEAPU16=new Uint16Array(b);Module["HEAP32"]=HEAP32=new Int32Array(b);Module["HEAPU32"]=HEAPU32=new Uint32Array(b);Module["HEAPF32"]=HEAPF32=new Float32Array(b);Module["HEAPF64"]=HEAPF64=new Float64Array(b)}var __ATPRERUN__=[];var __ATINIT__=[];var __ATPOSTRUN__=[];var runtimeInitialized=false;function preRun(){var preRuns=Module["preRun"];if(preRuns){if(typeof preRuns=="function")preRuns=[preRuns];preRuns.forEach(addOnPreRun)}callRuntimeCallbacks(__ATPRERUN__)}function initRuntime(){runtimeInitialized=true;callRuntimeCallbacks(__ATINIT__)}function postRun(){var postRuns=Module["postRun"];if(postRuns){if(typeof postRuns=="function")postRuns=[postRuns];postRuns.forEach(addOnPostRun)}callRuntimeCallbacks(__ATPOSTRUN__)}function addOnPreRun(cb){__ATPRERUN__.unshift(cb)}function addOnInit(cb){__ATINIT__.unshift(cb)}function addOnPostRun(cb){__ATPOSTRUN__.unshift(cb)}var runDependencies=0;var runDependencyWatcher=null;var dependenciesFulfilled=null;function addRunDependency(id){runDependencies++;Module["monitorRunDependencies"]?.(runDependencies)}function removeRunDependency(id){runDependencies--;Module["monitorRunDependencies"]?.(runDependencies);if(runDependencies==0){if(runDependencyWatcher!==null){clearInterval(runDependencyWatcher);runDependencyWatcher=null}if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);throw e}var dataURIPrefix="data:application/octet-stream;base64,";var isDataURI=filename=>filename.startsWith(dataURIPrefix);var isFileURI=filename=>filename.startsWith("file://");function findWasmBinary(){var f="zstd.wasm";if(!isDataURI(f)){return locateFile(f)}return f}var wasmBinaryFile;function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}function getBinaryPromise(binaryFile){if(!wasmBinary){return readAsync(binaryFile).then(response=>new Uint8Array(response),()=>getBinarySync(binaryFile))}return Promise.resolve().then(()=>getBinarySync(binaryFile))}function instantiateArrayBuffer(binaryFile,imports,receiver){return getBinaryPromise(binaryFile).then(binary=>WebAssembly.instantiate(binary,imports)).then(receiver,reason=>{err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)})}function instantiateAsync(binary,binaryFile,imports,callback){if(!binary&&typeof WebAssembly.instantiateStreaming=="function"&&!isDataURI(binaryFile)&&!isFileURI(binaryFile)&&!ENVIRONMENT_IS_NODE&&typeof fetch=="function"){return fetch(binaryFile,{credentials:"same-origin"}).then(response=>{var result=WebAssembly.instantiateStreaming(response,imports);return result.then(callback,function(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation");return instantiateArrayBuffer(binaryFile,imports,callback)})})}return instantiateArrayBuffer(binaryFile,imports,callback)}function getWasmImports(){return{a:wasmImports}}function createWasm(){var info=getWasmImports();function receiveInstance(instance,module){wasmExports=instance.exports;wasmMemory=wasmExports["c"];updateMemoryViews();addOnInit(wasmExports["d"]);removeRunDependency("wasm-instantiate");return wasmExports}addRunDependency("wasm-instantiate");function receiveInstantiationResult(result){receiveInstance(result["instance"])}if(Module["instantiateWasm"]){try{return Module["instantiateWasm"](info,receiveInstance)}catch(e){err(`Module.instantiateWasm callback failed with error: ${e}`);return false}}wasmBinaryFile??=findWasmBinary();instantiateAsync(wasmBinary,wasmBinaryFile,info,receiveInstantiationResult);return{}}var callRuntimeCallbacks=callbacks=>{callbacks.forEach(f=>f(Module))};var noExitRuntime=Module["noExitRuntime"]||true;var __emscripten_memcpy_js=(dest,src,num)=>HEAPU8.copyWithin(dest,src,src+num);var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var b=wasmMemory.buffer;var pages=(size-b.byteLength+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var wasmImports={b:__emscripten_memcpy_js,a:_emscripten_resize_heap};var wasmExports=createWasm();var ___wasm_call_ctors=()=>(___wasm_call_ctors=wasmExports["d"])();var _malloc=Module["_malloc"]=a0=>(_malloc=Module["_malloc"]=wasmExports["f"])(a0);var _free=Module["_free"]=a0=>(_free=Module["_free"]=wasmExports["g"])(a0);var _ZSTD_isError=Module["_ZSTD_isError"]=a0=>(_ZSTD_isError=Module["_ZSTD_isError"]=wasmExports["h"])(a0);var _ZSTD_compressBound=Module["_ZSTD_compressBound"]=a0=>(_ZSTD_compressBound=Module["_ZSTD_compressBound"]=wasmExports["i"])(a0);var _ZSTD_compress=Module["_ZSTD_compress"]=(a0,a1,a2,a3,a4)=>(_ZSTD_compress=Module["_ZSTD_compress"]=wasmExports["j"])(a0,a1,a2,a3,a4);var _ZSTD_getFrameContentSize=Module["_ZSTD_getFrameContentSize"]=(a0,a1)=>(_ZSTD_getFrameContentSize=Module["_ZSTD_getFrameContentSize"]=wasmExports["k"])(a0,a1);var _ZSTD_decompress=Module["_ZSTD_decompress"]=(a0,a1,a2,a3)=>(_ZSTD_decompress=Module["_ZSTD_decompress"]=wasmExports["l"])(a0,a1,a2,a3);var calledRun;var calledPrerun;dependenciesFulfilled=function runCaller(){if(!calledRun)run();if(!calledRun)dependenciesFulfilled=runCaller};function run(){if(runDependencies>0){return}if(!calledPrerun){calledPrerun=1;preRun();if(runDependencies>0){return}}function doRun(){if(calledRun)return;calledRun=1;Module["calledRun"]=1;if(ABORT)return;initRuntime();Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].pop()()}}run(); - -// Promise that resolves when the module is ready -let moduleReady = new Promise((resolve) => { - if (typeof Module !== 'undefined' && Module.calledRun) { - // Module already initialized - resolve(); - } else { - // Wait for module initialization - const originalOnRuntimeInitialized = Module.onRuntimeInitialized || function() {}; - Module.onRuntimeInitialized = function() { - originalOnRuntimeInitialized(); - resolve(); - }; +'use strict'; + +(() => { + const scriptUrl = document.currentScript?.src + ?? new URL('zstd.js', document.baseURI).href; + const workerUrl = new URL('zstd_worker.js', scriptUrl).href; + let worker = null; + let nextRequestId = 1; + const pending = new Map(); + + function failPendingRequests() { + for (const resolve of pending.values()) resolve(null); + pending.clear(); + worker?.terminate(); + worker = null; } -}); - -async function compressData(inputData, compressionLevel) { - await moduleReady; - - let inputPtr = Module._malloc(inputData.length); - Module.HEAPU8.set(inputData, inputPtr); - - let outputBufferSize = Number(Module._ZSTD_compressBound(inputData.length)); - let outputPtr = Module._malloc(outputBufferSize); - - let compressedSize = Number(Module._ZSTD_compress( - outputPtr, - outputBufferSize, - inputPtr, - inputData.length, - compressionLevel - )); - if (Module._ZSTD_isError(compressedSize) !== 0 || compressedSize <= 0) { - console.error('Compression error, error code: ', compressedSize); - Module._free(inputPtr); - Module._free(outputPtr); - return null; - } else { - let compressedData = new Uint8Array(Module.HEAPU8.buffer, outputPtr, compressedSize); - let out = compressedData.slice(0); - Module._free(inputPtr); - Module._free(outputPtr); - return out; + function getWorker() { + if (worker !== null) return worker; + worker = new Worker(workerUrl); + worker.onmessage = (event) => { + const resolve = pending.get(event.data.id); + if (resolve === undefined) return; + pending.delete(event.data.id); + resolve(event.data.result); + }; + worker.onerror = failPendingRequests; + worker.onmessageerror = failPendingRequests; + return worker; } -} - -async function decompressData(compressedData) { - await moduleReady; - - let compressedPtr = Module._malloc(compressedData.length); - Module.HEAPU8.set(compressedData, compressedPtr); - // ZSTD_getFrameContentSize returns these unsigned 64-bit sentinel values. - // JavaScript rounds them to the same Number values returned by Emscripten. - const ZSTD_CONTENTSIZE_UNKNOWN = 0xffffffffffffffff; - const ZSTD_CONTENTSIZE_ERROR = 0xfffffffffffffffe; - let decompressedSize = Number(Module._ZSTD_getFrameContentSize(compressedPtr, compressedData.length)); - if (decompressedSize === ZSTD_CONTENTSIZE_ERROR) { - console.error('Error in obtaining the original size of the data'); - Module._free(compressedPtr); - return null; + function run(operation, inputData, option) { + return new Promise((resolve) => { + const id = nextRequestId++; + try { + const inputCopy = inputData.slice(); + pending.set(id, resolve); + getWorker().postMessage( + { id, operation, inputBuffer: inputCopy.buffer, option }, + [inputCopy.buffer], + ); + } catch (_) { + pending.delete(id); + resolve(null); + } + }); } - const outputBufferSize = decompressedSize === ZSTD_CONTENTSIZE_UNKNOWN - ? compressedData.length * 20 - : decompressedSize; - let decompressedPtr = Module._malloc(outputBufferSize); - - let resultSize = Number(Module._ZSTD_decompress( - decompressedPtr, - outputBufferSize, - compressedPtr, - compressedData.length - )); - - if (Module._ZSTD_isError(resultSize) !== 0 || resultSize < 0) { - console.error('Decompression error, error code: ', resultSize); - Module._free(compressedPtr); - Module._free(decompressedPtr); - return null; - } else { - let decompressedData = new Uint8Array(Module.HEAPU8.buffer, decompressedPtr, resultSize); - let out = decompressedData.slice(0); - Module._free(compressedPtr); - Module._free(decompressedPtr); - return out; - } -} + globalThis.compressData = (inputData, compressionLevel) => + run('compress', inputData, compressionLevel); + globalThis.decompressData = (compressedData, maxOutputSize) => + run('decompress', compressedData, maxOutputSize); +})(); diff --git a/zstandard_web/blob/zstd.wasm b/zstandard_web/blob/zstd.wasm index 69a3403f..4731172c 100755 Binary files a/zstandard_web/blob/zstd.wasm and b/zstandard_web/blob/zstd.wasm differ diff --git a/zstandard_web/blob/zstd_core.js b/zstandard_web/blob/zstd_core.js new file mode 100644 index 00000000..6e4d1bda --- /dev/null +++ b/zstandard_web/blob/zstd_core.js @@ -0,0 +1,175 @@ +var Module=typeof zstdWasmModule!="undefined"?zstdWasmModule:{};var ENVIRONMENT_IS_WEB=typeof window=="object";var ENVIRONMENT_IS_WORKER=typeof importScripts=="function";var ENVIRONMENT_IS_NODE=typeof process=="object"&&typeof process.versions=="object"&&typeof process.versions.node=="string"&&process.type!="renderer";if(ENVIRONMENT_IS_NODE){}var moduleOverrides=Object.assign({},Module);var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("fs");var nodePath=require("path");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);var ret=fs.readFileSync(filename);return ret};readAsync=(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);return new Promise((resolve,reject)=>{fs.readFile(filename,binary?undefined:"utf8",(err,data)=>{if(err)reject(err);else resolve(binary?data.buffer:data)})})};if(!Module["thisProgram"]&&process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}arguments_=process.argv.slice(2);if(typeof module!="undefined"){module["exports"]=Module}quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){if(ENVIRONMENT_IS_WORKER){scriptDirectory=self.location.href}else if(typeof document!="undefined"&&document.currentScript){scriptDirectory=document.currentScript.src}if(scriptDirectory.startsWith("blob:")){scriptDirectory=""}else{scriptDirectory=scriptDirectory.substr(0,scriptDirectory.replace(/[?#].*/,"").lastIndexOf("/")+1)}{if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=url=>{if(isFileURI(url)){return new Promise((resolve,reject)=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=()=>{if(xhr.status==200||xhr.status==0&&xhr.response){resolve(xhr.response);return}reject(xhr.status)};xhr.onerror=reject;xhr.send(null)})}return fetch(url,{credentials:"same-origin"}).then(response=>{if(response.ok){return response.arrayBuffer()}return Promise.reject(new Error(response.status+" : "+response.url))})}}}else{}var out=Module["print"]||console.log.bind(console);var err=Module["printErr"]||console.error.bind(console);Object.assign(Module,moduleOverrides);moduleOverrides=null;if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];var wasmBinary=Module["wasmBinary"];var wasmMemory;var ABORT=false;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;function updateMemoryViews(){var b=wasmMemory.buffer;Module["HEAP8"]=HEAP8=new Int8Array(b);Module["HEAP16"]=HEAP16=new Int16Array(b);Module["HEAPU8"]=HEAPU8=new Uint8Array(b);Module["HEAPU16"]=HEAPU16=new Uint16Array(b);Module["HEAP32"]=HEAP32=new Int32Array(b);Module["HEAPU32"]=HEAPU32=new Uint32Array(b);Module["HEAPF32"]=HEAPF32=new Float32Array(b);Module["HEAPF64"]=HEAPF64=new Float64Array(b)}var __ATPRERUN__=[];var __ATINIT__=[];var __ATPOSTRUN__=[];var runtimeInitialized=false;function preRun(){var preRuns=Module["preRun"];if(preRuns){if(typeof preRuns=="function")preRuns=[preRuns];preRuns.forEach(addOnPreRun)}callRuntimeCallbacks(__ATPRERUN__)}function initRuntime(){runtimeInitialized=true;callRuntimeCallbacks(__ATINIT__)}function postRun(){var postRuns=Module["postRun"];if(postRuns){if(typeof postRuns=="function")postRuns=[postRuns];postRuns.forEach(addOnPostRun)}callRuntimeCallbacks(__ATPOSTRUN__)}function addOnPreRun(cb){__ATPRERUN__.unshift(cb)}function addOnInit(cb){__ATINIT__.unshift(cb)}function addOnPostRun(cb){__ATPOSTRUN__.unshift(cb)}var runDependencies=0;var runDependencyWatcher=null;var dependenciesFulfilled=null;function addRunDependency(id){runDependencies++;Module["monitorRunDependencies"]?.(runDependencies)}function removeRunDependency(id){runDependencies--;Module["monitorRunDependencies"]?.(runDependencies);if(runDependencies==0){if(runDependencyWatcher!==null){clearInterval(runDependencyWatcher);runDependencyWatcher=null}if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);throw e}var dataURIPrefix="data:application/octet-stream;base64,";var isDataURI=filename=>filename.startsWith(dataURIPrefix);var isFileURI=filename=>filename.startsWith("file://");function findWasmBinary(){var f="zstd.wasm";if(!isDataURI(f)){return locateFile(f)}return f}var wasmBinaryFile;function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}function getBinaryPromise(binaryFile){if(!wasmBinary){return readAsync(binaryFile).then(response=>new Uint8Array(response),()=>getBinarySync(binaryFile))}return Promise.resolve().then(()=>getBinarySync(binaryFile))}function instantiateArrayBuffer(binaryFile,imports,receiver){return getBinaryPromise(binaryFile).then(binary=>WebAssembly.instantiate(binary,imports)).then(receiver,reason=>{err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)})}function instantiateAsync(binary,binaryFile,imports,callback){if(!binary&&typeof WebAssembly.instantiateStreaming=="function"&&!isDataURI(binaryFile)&&!isFileURI(binaryFile)&&!ENVIRONMENT_IS_NODE&&typeof fetch=="function"){return fetch(binaryFile,{credentials:"same-origin"}).then(response=>{var result=WebAssembly.instantiateStreaming(response,imports);return result.then(callback,function(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation");return instantiateArrayBuffer(binaryFile,imports,callback)})})}return instantiateArrayBuffer(binaryFile,imports,callback)}function getWasmImports(){return{a:wasmImports}}function createWasm(){var info=getWasmImports();function receiveInstance(instance,module){wasmExports=instance.exports;wasmMemory=wasmExports["c"];updateMemoryViews();addOnInit(wasmExports["d"]);removeRunDependency("wasm-instantiate");return wasmExports}addRunDependency("wasm-instantiate");function receiveInstantiationResult(result){receiveInstance(result["instance"])}if(Module["instantiateWasm"]){try{return Module["instantiateWasm"](info,receiveInstance)}catch(e){err(`Module.instantiateWasm callback failed with error: ${e}`);return false}}wasmBinaryFile??=findWasmBinary();instantiateAsync(wasmBinary,wasmBinaryFile,info,receiveInstantiationResult);return{}}var callRuntimeCallbacks=callbacks=>{callbacks.forEach(f=>f(Module))};var noExitRuntime=Module["noExitRuntime"]||true;var __emscripten_memcpy_js=(dest,src,num)=>HEAPU8.copyWithin(dest,src,src+num);var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var b=wasmMemory.buffer;var pages=(size-b.byteLength+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var wasmImports={b:__emscripten_memcpy_js,a:_emscripten_resize_heap};var wasmExports=createWasm();var ___wasm_call_ctors=()=>(___wasm_call_ctors=wasmExports["d"])();var _malloc=Module["_malloc"]=a0=>(_malloc=Module["_malloc"]=wasmExports["f"])(a0);var _free=Module["_free"]=a0=>(_free=Module["_free"]=wasmExports["g"])(a0);var _ZSTD_isError=Module["_ZSTD_isError"]=a0=>(_ZSTD_isError=Module["_ZSTD_isError"]=wasmExports["h"])(a0);var _ZSTD_compressBound=Module["_ZSTD_compressBound"]=a0=>(_ZSTD_compressBound=Module["_ZSTD_compressBound"]=wasmExports["i"])(a0);var _ZSTD_compress=Module["_ZSTD_compress"]=(a0,a1,a2,a3,a4)=>(_ZSTD_compress=Module["_ZSTD_compress"]=wasmExports["j"])(a0,a1,a2,a3,a4);var _ZSTD_createDStream=Module["_ZSTD_createDStream"]=()=>(_ZSTD_createDStream=Module["_ZSTD_createDStream"]=wasmExports["k"])();var _ZSTD_freeDStream=Module["_ZSTD_freeDStream"]=a0=>(_ZSTD_freeDStream=Module["_ZSTD_freeDStream"]=wasmExports["l"])(a0);var _ZSTD_DStreamOutSize=Module["_ZSTD_DStreamOutSize"]=()=>(_ZSTD_DStreamOutSize=Module["_ZSTD_DStreamOutSize"]=wasmExports["m"])();var _ZSTD_initDStream=Module["_ZSTD_initDStream"]=a0=>(_ZSTD_initDStream=Module["_ZSTD_initDStream"]=wasmExports["n"])(a0);var _ZSTD_decompressStream=Module["_ZSTD_decompressStream"]=(a0,a1,a2)=>(_ZSTD_decompressStream=Module["_ZSTD_decompressStream"]=wasmExports["o"])(a0,a1,a2);var calledRun;var calledPrerun;dependenciesFulfilled=function runCaller(){if(!calledRun)run();if(!calledRun)dependenciesFulfilled=runCaller};function run(){if(runDependencies>0){return}if(!calledPrerun){calledPrerun=1;preRun();if(runDependencies>0){return}}function doRun(){if(calledRun)return;calledRun=1;Module["calledRun"]=1;if(ABORT)return;initRuntime();Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].pop()()}}run(); + +// Promise that resolves when the module is ready +let moduleReady = new Promise((resolve) => { + if (typeof Module !== 'undefined' && Module.calledRun) { + // Module already initialized + resolve(); + } else { + // Wait for module initialization + const originalOnRuntimeInitialized = Module.onRuntimeInitialized || function() {}; + Module.onRuntimeInitialized = function() { + originalOnRuntimeInitialized(); + resolve(); + }; + } +}); + +async function compressData(inputData, compressionLevel) { + await moduleReady; + + if (!(inputData instanceof Uint8Array) + || !Number.isInteger(compressionLevel) + || compressionLevel < 1 + || compressionLevel > 22) { + return null; + } + + let inputPtr = 0; + let outputPtr = 0; + try { + inputPtr = Module._malloc(Math.max(inputData.length, 1)); + if (!inputPtr) return null; + Module.HEAPU8.set(inputData, inputPtr); + + const outputBufferSize = Number(Module._ZSTD_compressBound(inputData.length)); + if (!Number.isSafeInteger(outputBufferSize) || outputBufferSize <= 0) { + return null; + } + outputPtr = Module._malloc(outputBufferSize); + if (!outputPtr) return null; + + const compressedSize = Number(Module._ZSTD_compress( + outputPtr, + outputBufferSize, + inputPtr, + inputData.length, + compressionLevel + )); + if (Module._ZSTD_isError(compressedSize) !== 0 || compressedSize <= 0) { + console.error('Compression error, error code: ', compressedSize); + return null; + } + return new Uint8Array( + Module.HEAPU8.buffer, + outputPtr, + compressedSize + ).slice(); + } finally { + if (inputPtr) Module._free(inputPtr); + if (outputPtr) Module._free(outputPtr); + } +} + +async function decompressData(compressedData, maxOutputSize = 256 * 1024 * 1024) { + await moduleReady; + + if (!(compressedData instanceof Uint8Array) + || !Number.isSafeInteger(maxOutputSize) + || maxOutputSize < 0 + || compressedData.length === 0) { + return null; + } + + // ZSTD_inBuffer and ZSTD_outBuffer contain three wasm32 size_t/pointer + // fields each. Keep their allocation and field access local to the Worker. + const bufferStructSize = 3 * Uint32Array.BYTES_PER_ELEMENT; + let compressedPtr = 0; + let inputBufferPtr = 0; + let outputBufferPtr = 0; + let outputChunkPtr = 0; + let stream = 0; + try { + compressedPtr = Module._malloc(compressedData.length); + inputBufferPtr = Module._malloc(bufferStructSize); + outputBufferPtr = Module._malloc(bufferStructSize); + stream = Module._ZSTD_createDStream(); + if (!compressedPtr || !inputBufferPtr || !outputBufferPtr || !stream) { + return null; + } + Module.HEAPU8.set(compressedData, compressedPtr); + + const initialization = Number(Module._ZSTD_initDStream(stream)); + if (Module._ZSTD_isError(initialization) !== 0) return null; + + const recommendedChunkSize = Number(Module._ZSTD_DStreamOutSize()); + if (!Number.isSafeInteger(recommendedChunkSize) + || recommendedChunkSize <= 0) { + return null; + } + const outputChunkSize = Math.max( + 1, + Math.min(recommendedChunkSize, Math.max(maxOutputSize, 1)), + ); + outputChunkPtr = Module._malloc(outputChunkSize); + if (!outputChunkPtr) return null; + + const writeField = (structPtr, field, value) => { + Module.HEAPU32[(structPtr >>> 2) + field] = value; + }; + const readField = (structPtr, field) => + Module.HEAPU32[(structPtr >>> 2) + field]; + + writeField(inputBufferPtr, 0, compressedPtr); + writeField(inputBufferPtr, 1, compressedData.length); + writeField(inputBufferPtr, 2, 0); + + const chunks = []; + let totalLength = 0; + let previousInputPosition = -1; + let previousOutputLength = -1; + while (true) { + writeField(outputBufferPtr, 0, outputChunkPtr); + writeField(outputBufferPtr, 1, outputChunkSize); + writeField(outputBufferPtr, 2, 0); + + const remaining = Number(Module._ZSTD_decompressStream( + stream, + outputBufferPtr, + inputBufferPtr, + )); + if (Module._ZSTD_isError(remaining) !== 0) return null; + + const inputPosition = readField(inputBufferPtr, 2); + const produced = readField(outputBufferPtr, 2); + if (inputPosition > compressedData.length + || produced > outputChunkSize + || totalLength + produced > maxOutputSize) { + return null; + } + if (produced > 0) { + chunks.push(new Uint8Array( + Module.HEAPU8.buffer, + outputChunkPtr, + produced, + ).slice()); + totalLength += produced; + } + + const allInputConsumed = inputPosition === compressedData.length; + if (remaining === 0 && allInputConsumed) { + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + return result; + } + + const madeProgress = inputPosition !== previousInputPosition + || totalLength !== previousOutputLength; + if (!madeProgress || (allInputConsumed && produced === 0)) { + return null; + } + previousInputPosition = inputPosition; + previousOutputLength = totalLength; + } + } finally { + if (stream) Module._ZSTD_freeDStream(stream); + if (compressedPtr) Module._free(compressedPtr); + if (inputBufferPtr) Module._free(inputBufferPtr); + if (outputBufferPtr) Module._free(outputBufferPtr); + if (outputChunkPtr) Module._free(outputChunkPtr); + } +} diff --git a/zstandard_web/blob/zstd_worker.js b/zstandard_web/blob/zstd_worker.js new file mode 100644 index 00000000..38e076eb --- /dev/null +++ b/zstandard_web/blob/zstd_worker.js @@ -0,0 +1,20 @@ +'use strict'; + +importScripts('zstd_core.js'); + +self.onmessage = async (event) => { + const { id, operation, inputBuffer, option } = event.data; + try { + const input = new Uint8Array(inputBuffer); + const result = operation === 'compress' + ? await compressData(input, option) + : await decompressData(input, option); + if (result === null) { + self.postMessage({ id, result: null }); + } else { + self.postMessage({ id, result }, [result.buffer]); + } + } catch (_) { + self.postMessage({ id, result: null }); + } +}; diff --git a/zstandard_web/example/analysis_options.yaml b/zstandard_web/example/analysis_options.yaml index 0d290213..2a2b57e8 100644 --- a/zstandard_web/example/analysis_options.yaml +++ b/zstandard_web/example/analysis_options.yaml @@ -7,6 +7,10 @@ # The following line activates a set of recommended lints for Flutter apps, # packages, and plugins designed to encourage good coding practices. +analyzer: + exclude: + - build/** + - web/** include: package:flutter_lints/flutter.yaml linter: diff --git a/zstandard_web/example/integration_test/zstandard_web_integration_test.dart b/zstandard_web/example/integration_test/zstandard_web_integration_test.dart index fb6753be..fbb45650 100644 --- a/zstandard_web/example/integration_test/zstandard_web_integration_test.dart +++ b/zstandard_web/example/integration_test/zstandard_web_integration_test.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:typed_data'; import 'package:flutter/widgets.dart'; @@ -35,17 +36,19 @@ void main() { expect(version, isNotNull); }); - test('compress and decompress roundtrip for data shorter than 9 bytes', - () async { - final data = Uint8List.fromList([1, 2, 3, 4, 5]); - expect(data, isNotEmpty); - final compressed = await zstandardWeb.compress(data, 3); - expect(compressed, isNotNull); - expect(compressed!.isNotEmpty, isTrue); - expect(compressed, isNot(equals(data))); - final decompressed = await zstandardWeb.decompress(compressed); - expect(decompressed, equals(data)); - }); + test( + 'compress and decompress roundtrip for data shorter than 9 bytes', + () async { + final data = Uint8List.fromList([1, 2, 3, 4, 5]); + expect(data, isNotEmpty); + final compressed = await zstandardWeb.compress(data, 3); + expect(compressed, isNotNull); + expect(compressed!.isNotEmpty, isTrue); + expect(compressed, isNot(equals(data))); + final decompressed = await zstandardWeb.decompress(compressed); + expect(decompressed, equals(data)); + }, + ); test('compress and decompress small data', () async { final data = Uint8List.fromList(List.generate(10, (int i) => i)); @@ -58,8 +61,9 @@ void main() { }); test('compress and decompress large data', () async { - final data = - Uint8List.fromList(List.generate(100000, (int i) => i % 256)); + final data = Uint8List.fromList( + List.generate(100000, (int i) => i % 256), + ); expect(data, isNotEmpty); final compressed = await zstandardWeb.compress(data, 3); expect(compressed, isNotNull); @@ -89,23 +93,108 @@ void main() { } }); - test('decompress corrupted data throws', () async { + test('decompress corrupted data returns null', () async { final corrupted = Uint8List.fromList([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + expect(await zstandardWeb.decompress(corrupted), isNull); + }); + + test('decompress random bytes returns null', () async { + final random = Uint8List.fromList( + List.generate(64, (int i) => (i * 31) % 256), + ); + expect(await zstandardWeb.decompress(random), isNull); + }); + + test('decompresses a frame without a declared content size', () async { + const frameWithoutContentSize = [ + 0x28, + 0xb5, + 0x2f, + 0xfd, + 0x04, + 0x58, + 0x91, + 0x00, + 0x00, + 0x75, + 0x6e, + 0x6b, + 0x6e, + 0x6f, + 0x77, + 0x6e, + 0x2d, + 0x73, + 0x69, + 0x7a, + 0x65, + 0x20, + 0x66, + 0x72, + 0x61, + 0x6d, + 0x65, + 0xab, + 0x02, + 0x28, + 0xf0, + ]; expect( - () async => await zstandardWeb.decompress(corrupted), - throwsException, + await zstandardWeb.decompress( + Uint8List.fromList(frameWithoutContentSize), + ), + equals('unknown-size frame'.codeUnits), ); }); - test('decompress random bytes throws', () async { - final random = - Uint8List.fromList(List.generate(64, (int i) => (i * 31) % 256)); + test('decompresses concatenated frames', () async { + final first = Uint8List.fromList('first frame'.codeUnits); + final second = Uint8List.fromList('second frame'.codeUnits); + final compressedFirst = await zstandardWeb.compress(first, 3); + final compressedSecond = await zstandardWeb.compress(second, 3); + expect(compressedFirst, isNotNull); + expect(compressedSecond, isNotNull); + + final concatenated = Uint8List.fromList([ + ...compressedFirst!, + ...compressedSecond!, + ]); + expect( + await zstandardWeb.decompress(concatenated), + equals([...first, ...second]), + ); + }); + + test('enforces the decompressed output limit', () async { + final compressed = await zstandardWeb.compress(Uint8List(1024), 3); + expect(compressed, isNotNull); expect( - () async => await zstandardWeb.decompress(random), - throwsException, + await zstandardWeb.decompressWithOptions( + compressed!, + maxOutputSize: 1023, + ), + isNull, + ); + expect( + await zstandardWeb.decompressWithOptions( + compressed, + maxOutputSize: 1024, + ), + equals(Uint8List(1024)), ); }); + test('compression yields the browser event loop to a worker', () async { + var timerRan = false; + Timer.run(() => timerRan = true); + + final compressed = zstandardWeb.compress(Uint8List(1024 * 1024), 3); + await Future.delayed(Duration.zero); + + expect(timerRan, isTrue); + expect(await compressed, isNotNull); + }); + test('compress and decompress do not leak', () async { final data = Uint8List.fromList(List.generate(10, (int i) => i)); expect(data, isNotEmpty); @@ -122,25 +211,18 @@ void main() { }); group('Property-based tests', () { - property( - 'roundtrip: decompress(compress(x)) == x', - () { - forAll( - binary(minLength: 1, maxLength: 1000), - (List data) async { - final input = Uint8List.fromList(data); - expect(input, isNotEmpty); - final z = ZstandardWeb(); - final compressed = await z.compress(input, 3); - expect(compressed, isNotNull); - expect(compressed!.isNotEmpty, isTrue); - final decompressed = await z.decompress(compressed); - expect(decompressed, isNotNull); - expect(List.from(decompressed!), data); - }, - maxExamples: 100, - ); - }, - ); + property('roundtrip: decompress(compress(x)) == x', () { + forAll(binary(minLength: 1, maxLength: 1000), (List data) async { + final input = Uint8List.fromList(data); + expect(input, isNotEmpty); + final z = ZstandardWeb(); + final compressed = await z.compress(input, 3); + expect(compressed, isNotNull); + expect(compressed!.isNotEmpty, isTrue); + final decompressed = await z.decompress(compressed); + expect(decompressed, isNotNull); + expect(List.from(decompressed!), data); + }, maxExamples: 100); + }); }); } diff --git a/zstandard_web/example/pubspec.yaml b/zstandard_web/example/pubspec.yaml index 4325309d..602f2762 100644 --- a/zstandard_web/example/pubspec.yaml +++ b/zstandard_web/example/pubspec.yaml @@ -8,7 +8,8 @@ environment: dependencies: flutter: sdk: flutter - zstandard_platform_interface: ^1.0.0 + zstandard_platform_interface: + path: ../../zstandard_platform_interface zstandard_web: path: ../ cupertino_icons: ^1.0.8 diff --git a/zstandard_web/example/web/zstd.js b/zstandard_web/example/web/zstd.js index 82898927..d1963b65 100644 --- a/zstandard_web/example/web/zstd.js +++ b/zstandard_web/example/web/zstd.js @@ -1,90 +1,53 @@ -var Module=typeof zstdWasmModule!="undefined"?zstdWasmModule:{};var ENVIRONMENT_IS_WEB=typeof window=="object";var ENVIRONMENT_IS_WORKER=typeof importScripts=="function";var ENVIRONMENT_IS_NODE=typeof process=="object"&&typeof process.versions=="object"&&typeof process.versions.node=="string"&&process.type!="renderer";if(ENVIRONMENT_IS_NODE){}var moduleOverrides=Object.assign({},Module);var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("fs");var nodePath=require("path");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);var ret=fs.readFileSync(filename);return ret};readAsync=(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);return new Promise((resolve,reject)=>{fs.readFile(filename,binary?undefined:"utf8",(err,data)=>{if(err)reject(err);else resolve(binary?data.buffer:data)})})};if(!Module["thisProgram"]&&process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}arguments_=process.argv.slice(2);if(typeof module!="undefined"){module["exports"]=Module}quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){if(ENVIRONMENT_IS_WORKER){scriptDirectory=self.location.href}else if(typeof document!="undefined"&&document.currentScript){scriptDirectory=document.currentScript.src}if(scriptDirectory.startsWith("blob:")){scriptDirectory=""}else{scriptDirectory=scriptDirectory.substr(0,scriptDirectory.replace(/[?#].*/,"").lastIndexOf("/")+1)}{if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=url=>{if(isFileURI(url)){return new Promise((resolve,reject)=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=()=>{if(xhr.status==200||xhr.status==0&&xhr.response){resolve(xhr.response);return}reject(xhr.status)};xhr.onerror=reject;xhr.send(null)})}return fetch(url,{credentials:"same-origin"}).then(response=>{if(response.ok){return response.arrayBuffer()}return Promise.reject(new Error(response.status+" : "+response.url))})}}}else{}var out=Module["print"]||console.log.bind(console);var err=Module["printErr"]||console.error.bind(console);Object.assign(Module,moduleOverrides);moduleOverrides=null;if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];var wasmBinary=Module["wasmBinary"];var wasmMemory;var ABORT=false;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;function updateMemoryViews(){var b=wasmMemory.buffer;Module["HEAP8"]=HEAP8=new Int8Array(b);Module["HEAP16"]=HEAP16=new Int16Array(b);Module["HEAPU8"]=HEAPU8=new Uint8Array(b);Module["HEAPU16"]=HEAPU16=new Uint16Array(b);Module["HEAP32"]=HEAP32=new Int32Array(b);Module["HEAPU32"]=HEAPU32=new Uint32Array(b);Module["HEAPF32"]=HEAPF32=new Float32Array(b);Module["HEAPF64"]=HEAPF64=new Float64Array(b)}var __ATPRERUN__=[];var __ATINIT__=[];var __ATPOSTRUN__=[];var runtimeInitialized=false;function preRun(){var preRuns=Module["preRun"];if(preRuns){if(typeof preRuns=="function")preRuns=[preRuns];preRuns.forEach(addOnPreRun)}callRuntimeCallbacks(__ATPRERUN__)}function initRuntime(){runtimeInitialized=true;callRuntimeCallbacks(__ATINIT__)}function postRun(){var postRuns=Module["postRun"];if(postRuns){if(typeof postRuns=="function")postRuns=[postRuns];postRuns.forEach(addOnPostRun)}callRuntimeCallbacks(__ATPOSTRUN__)}function addOnPreRun(cb){__ATPRERUN__.unshift(cb)}function addOnInit(cb){__ATINIT__.unshift(cb)}function addOnPostRun(cb){__ATPOSTRUN__.unshift(cb)}var runDependencies=0;var runDependencyWatcher=null;var dependenciesFulfilled=null;function addRunDependency(id){runDependencies++;Module["monitorRunDependencies"]?.(runDependencies)}function removeRunDependency(id){runDependencies--;Module["monitorRunDependencies"]?.(runDependencies);if(runDependencies==0){if(runDependencyWatcher!==null){clearInterval(runDependencyWatcher);runDependencyWatcher=null}if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);throw e}var dataURIPrefix="data:application/octet-stream;base64,";var isDataURI=filename=>filename.startsWith(dataURIPrefix);var isFileURI=filename=>filename.startsWith("file://");function findWasmBinary(){var f="zstd.wasm";if(!isDataURI(f)){return locateFile(f)}return f}var wasmBinaryFile;function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}function getBinaryPromise(binaryFile){if(!wasmBinary){return readAsync(binaryFile).then(response=>new Uint8Array(response),()=>getBinarySync(binaryFile))}return Promise.resolve().then(()=>getBinarySync(binaryFile))}function instantiateArrayBuffer(binaryFile,imports,receiver){return getBinaryPromise(binaryFile).then(binary=>WebAssembly.instantiate(binary,imports)).then(receiver,reason=>{err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)})}function instantiateAsync(binary,binaryFile,imports,callback){if(!binary&&typeof WebAssembly.instantiateStreaming=="function"&&!isDataURI(binaryFile)&&!isFileURI(binaryFile)&&!ENVIRONMENT_IS_NODE&&typeof fetch=="function"){return fetch(binaryFile,{credentials:"same-origin"}).then(response=>{var result=WebAssembly.instantiateStreaming(response,imports);return result.then(callback,function(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation");return instantiateArrayBuffer(binaryFile,imports,callback)})})}return instantiateArrayBuffer(binaryFile,imports,callback)}function getWasmImports(){return{a:wasmImports}}function createWasm(){var info=getWasmImports();function receiveInstance(instance,module){wasmExports=instance.exports;wasmMemory=wasmExports["c"];updateMemoryViews();addOnInit(wasmExports["d"]);removeRunDependency("wasm-instantiate");return wasmExports}addRunDependency("wasm-instantiate");function receiveInstantiationResult(result){receiveInstance(result["instance"])}if(Module["instantiateWasm"]){try{return Module["instantiateWasm"](info,receiveInstance)}catch(e){err(`Module.instantiateWasm callback failed with error: ${e}`);return false}}wasmBinaryFile??=findWasmBinary();instantiateAsync(wasmBinary,wasmBinaryFile,info,receiveInstantiationResult);return{}}var callRuntimeCallbacks=callbacks=>{callbacks.forEach(f=>f(Module))};var noExitRuntime=Module["noExitRuntime"]||true;var __emscripten_memcpy_js=(dest,src,num)=>HEAPU8.copyWithin(dest,src,src+num);var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var b=wasmMemory.buffer;var pages=(size-b.byteLength+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var wasmImports={b:__emscripten_memcpy_js,a:_emscripten_resize_heap};var wasmExports=createWasm();var ___wasm_call_ctors=()=>(___wasm_call_ctors=wasmExports["d"])();var _malloc=Module["_malloc"]=a0=>(_malloc=Module["_malloc"]=wasmExports["f"])(a0);var _free=Module["_free"]=a0=>(_free=Module["_free"]=wasmExports["g"])(a0);var _ZSTD_isError=Module["_ZSTD_isError"]=a0=>(_ZSTD_isError=Module["_ZSTD_isError"]=wasmExports["h"])(a0);var _ZSTD_compressBound=Module["_ZSTD_compressBound"]=a0=>(_ZSTD_compressBound=Module["_ZSTD_compressBound"]=wasmExports["i"])(a0);var _ZSTD_compress=Module["_ZSTD_compress"]=(a0,a1,a2,a3,a4)=>(_ZSTD_compress=Module["_ZSTD_compress"]=wasmExports["j"])(a0,a1,a2,a3,a4);var _ZSTD_getFrameContentSize=Module["_ZSTD_getFrameContentSize"]=(a0,a1)=>(_ZSTD_getFrameContentSize=Module["_ZSTD_getFrameContentSize"]=wasmExports["k"])(a0,a1);var _ZSTD_decompress=Module["_ZSTD_decompress"]=(a0,a1,a2,a3)=>(_ZSTD_decompress=Module["_ZSTD_decompress"]=wasmExports["l"])(a0,a1,a2,a3);var calledRun;var calledPrerun;dependenciesFulfilled=function runCaller(){if(!calledRun)run();if(!calledRun)dependenciesFulfilled=runCaller};function run(){if(runDependencies>0){return}if(!calledPrerun){calledPrerun=1;preRun();if(runDependencies>0){return}}function doRun(){if(calledRun)return;calledRun=1;Module["calledRun"]=1;if(ABORT)return;initRuntime();Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].pop()()}}run(); - -// Promise that resolves when the module is ready -let moduleReady = new Promise((resolve) => { - if (typeof Module !== 'undefined' && Module.calledRun) { - // Module already initialized - resolve(); - } else { - // Wait for module initialization - const originalOnRuntimeInitialized = Module.onRuntimeInitialized || function() {}; - Module.onRuntimeInitialized = function() { - originalOnRuntimeInitialized(); - resolve(); - }; +'use strict'; + +(() => { + const scriptUrl = document.currentScript?.src + ?? new URL('zstd.js', document.baseURI).href; + const workerUrl = new URL('zstd_worker.js', scriptUrl).href; + let worker = null; + let nextRequestId = 1; + const pending = new Map(); + + function failPendingRequests() { + for (const resolve of pending.values()) resolve(null); + pending.clear(); + worker?.terminate(); + worker = null; } -}); - -async function compressData(inputData, compressionLevel) { - await moduleReady; - - let inputPtr = Module._malloc(inputData.length); - Module.HEAPU8.set(inputData, inputPtr); - - let outputBufferSize = Number(Module._ZSTD_compressBound(inputData.length)); - let outputPtr = Module._malloc(outputBufferSize); - - let compressedSize = Number(Module._ZSTD_compress( - outputPtr, - outputBufferSize, - inputPtr, - inputData.length, - compressionLevel - )); - if (Module._ZSTD_isError(compressedSize) !== 0 || compressedSize <= 0) { - console.error('Compression error, error code: ', compressedSize); - Module._free(inputPtr); - Module._free(outputPtr); - return null; - } else { - let compressedData = new Uint8Array(Module.HEAPU8.buffer, outputPtr, compressedSize); - let out = compressedData.slice(0); - Module._free(inputPtr); - Module._free(outputPtr); - return out; + function getWorker() { + if (worker !== null) return worker; + worker = new Worker(workerUrl); + worker.onmessage = (event) => { + const resolve = pending.get(event.data.id); + if (resolve === undefined) return; + pending.delete(event.data.id); + resolve(event.data.result); + }; + worker.onerror = failPendingRequests; + worker.onmessageerror = failPendingRequests; + return worker; } -} - -async function decompressData(compressedData) { - await moduleReady; - - let compressedPtr = Module._malloc(compressedData.length); - Module.HEAPU8.set(compressedData, compressedPtr); - // ZSTD_getFrameContentSize returns these unsigned 64-bit sentinel values. - // JavaScript rounds them to the same Number values returned by Emscripten. - const ZSTD_CONTENTSIZE_UNKNOWN = 0xffffffffffffffff; - const ZSTD_CONTENTSIZE_ERROR = 0xfffffffffffffffe; - let decompressedSize = Number(Module._ZSTD_getFrameContentSize(compressedPtr, compressedData.length)); - if (decompressedSize === ZSTD_CONTENTSIZE_ERROR) { - console.error('Error in obtaining the original size of the data'); - Module._free(compressedPtr); - return null; + function run(operation, inputData, option) { + return new Promise((resolve) => { + const id = nextRequestId++; + try { + const inputCopy = inputData.slice(); + pending.set(id, resolve); + getWorker().postMessage( + { id, operation, inputBuffer: inputCopy.buffer, option }, + [inputCopy.buffer], + ); + } catch (_) { + pending.delete(id); + resolve(null); + } + }); } - const outputBufferSize = decompressedSize === ZSTD_CONTENTSIZE_UNKNOWN - ? compressedData.length * 20 - : decompressedSize; - let decompressedPtr = Module._malloc(outputBufferSize); - - let resultSize = Number(Module._ZSTD_decompress( - decompressedPtr, - outputBufferSize, - compressedPtr, - compressedData.length - )); - - if (Module._ZSTD_isError(resultSize) !== 0 || resultSize < 0) { - console.error('Decompression error, error code: ', resultSize); - Module._free(compressedPtr); - Module._free(decompressedPtr); - return null; - } else { - let decompressedData = new Uint8Array(Module.HEAPU8.buffer, decompressedPtr, resultSize); - let out = decompressedData.slice(0); - Module._free(compressedPtr); - Module._free(decompressedPtr); - return out; - } -} + globalThis.compressData = (inputData, compressionLevel) => + run('compress', inputData, compressionLevel); + globalThis.decompressData = (compressedData, maxOutputSize) => + run('decompress', compressedData, maxOutputSize); +})(); diff --git a/zstandard_web/example/web/zstd.wasm b/zstandard_web/example/web/zstd.wasm index 69a3403f..4731172c 100755 Binary files a/zstandard_web/example/web/zstd.wasm and b/zstandard_web/example/web/zstd.wasm differ diff --git a/zstandard_web/example/web/zstd_core.js b/zstandard_web/example/web/zstd_core.js new file mode 100644 index 00000000..6e4d1bda --- /dev/null +++ b/zstandard_web/example/web/zstd_core.js @@ -0,0 +1,175 @@ +var Module=typeof zstdWasmModule!="undefined"?zstdWasmModule:{};var ENVIRONMENT_IS_WEB=typeof window=="object";var ENVIRONMENT_IS_WORKER=typeof importScripts=="function";var ENVIRONMENT_IS_NODE=typeof process=="object"&&typeof process.versions=="object"&&typeof process.versions.node=="string"&&process.type!="renderer";if(ENVIRONMENT_IS_NODE){}var moduleOverrides=Object.assign({},Module);var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_NODE){var fs=require("fs");var nodePath=require("path");scriptDirectory=__dirname+"/";readBinary=filename=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);var ret=fs.readFileSync(filename);return ret};readAsync=(filename,binary=true)=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);return new Promise((resolve,reject)=>{fs.readFile(filename,binary?undefined:"utf8",(err,data)=>{if(err)reject(err);else resolve(binary?data.buffer:data)})})};if(!Module["thisProgram"]&&process.argv.length>1){thisProgram=process.argv[1].replace(/\\/g,"/")}arguments_=process.argv.slice(2);if(typeof module!="undefined"){module["exports"]=Module}quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){if(ENVIRONMENT_IS_WORKER){scriptDirectory=self.location.href}else if(typeof document!="undefined"&&document.currentScript){scriptDirectory=document.currentScript.src}if(scriptDirectory.startsWith("blob:")){scriptDirectory=""}else{scriptDirectory=scriptDirectory.substr(0,scriptDirectory.replace(/[?#].*/,"").lastIndexOf("/")+1)}{if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=url=>{if(isFileURI(url)){return new Promise((resolve,reject)=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=()=>{if(xhr.status==200||xhr.status==0&&xhr.response){resolve(xhr.response);return}reject(xhr.status)};xhr.onerror=reject;xhr.send(null)})}return fetch(url,{credentials:"same-origin"}).then(response=>{if(response.ok){return response.arrayBuffer()}return Promise.reject(new Error(response.status+" : "+response.url))})}}}else{}var out=Module["print"]||console.log.bind(console);var err=Module["printErr"]||console.error.bind(console);Object.assign(Module,moduleOverrides);moduleOverrides=null;if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];var wasmBinary=Module["wasmBinary"];var wasmMemory;var ABORT=false;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;function updateMemoryViews(){var b=wasmMemory.buffer;Module["HEAP8"]=HEAP8=new Int8Array(b);Module["HEAP16"]=HEAP16=new Int16Array(b);Module["HEAPU8"]=HEAPU8=new Uint8Array(b);Module["HEAPU16"]=HEAPU16=new Uint16Array(b);Module["HEAP32"]=HEAP32=new Int32Array(b);Module["HEAPU32"]=HEAPU32=new Uint32Array(b);Module["HEAPF32"]=HEAPF32=new Float32Array(b);Module["HEAPF64"]=HEAPF64=new Float64Array(b)}var __ATPRERUN__=[];var __ATINIT__=[];var __ATPOSTRUN__=[];var runtimeInitialized=false;function preRun(){var preRuns=Module["preRun"];if(preRuns){if(typeof preRuns=="function")preRuns=[preRuns];preRuns.forEach(addOnPreRun)}callRuntimeCallbacks(__ATPRERUN__)}function initRuntime(){runtimeInitialized=true;callRuntimeCallbacks(__ATINIT__)}function postRun(){var postRuns=Module["postRun"];if(postRuns){if(typeof postRuns=="function")postRuns=[postRuns];postRuns.forEach(addOnPostRun)}callRuntimeCallbacks(__ATPOSTRUN__)}function addOnPreRun(cb){__ATPRERUN__.unshift(cb)}function addOnInit(cb){__ATINIT__.unshift(cb)}function addOnPostRun(cb){__ATPOSTRUN__.unshift(cb)}var runDependencies=0;var runDependencyWatcher=null;var dependenciesFulfilled=null;function addRunDependency(id){runDependencies++;Module["monitorRunDependencies"]?.(runDependencies)}function removeRunDependency(id){runDependencies--;Module["monitorRunDependencies"]?.(runDependencies);if(runDependencies==0){if(runDependencyWatcher!==null){clearInterval(runDependencyWatcher);runDependencyWatcher=null}if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);throw e}var dataURIPrefix="data:application/octet-stream;base64,";var isDataURI=filename=>filename.startsWith(dataURIPrefix);var isFileURI=filename=>filename.startsWith("file://");function findWasmBinary(){var f="zstd.wasm";if(!isDataURI(f)){return locateFile(f)}return f}var wasmBinaryFile;function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}function getBinaryPromise(binaryFile){if(!wasmBinary){return readAsync(binaryFile).then(response=>new Uint8Array(response),()=>getBinarySync(binaryFile))}return Promise.resolve().then(()=>getBinarySync(binaryFile))}function instantiateArrayBuffer(binaryFile,imports,receiver){return getBinaryPromise(binaryFile).then(binary=>WebAssembly.instantiate(binary,imports)).then(receiver,reason=>{err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)})}function instantiateAsync(binary,binaryFile,imports,callback){if(!binary&&typeof WebAssembly.instantiateStreaming=="function"&&!isDataURI(binaryFile)&&!isFileURI(binaryFile)&&!ENVIRONMENT_IS_NODE&&typeof fetch=="function"){return fetch(binaryFile,{credentials:"same-origin"}).then(response=>{var result=WebAssembly.instantiateStreaming(response,imports);return result.then(callback,function(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation");return instantiateArrayBuffer(binaryFile,imports,callback)})})}return instantiateArrayBuffer(binaryFile,imports,callback)}function getWasmImports(){return{a:wasmImports}}function createWasm(){var info=getWasmImports();function receiveInstance(instance,module){wasmExports=instance.exports;wasmMemory=wasmExports["c"];updateMemoryViews();addOnInit(wasmExports["d"]);removeRunDependency("wasm-instantiate");return wasmExports}addRunDependency("wasm-instantiate");function receiveInstantiationResult(result){receiveInstance(result["instance"])}if(Module["instantiateWasm"]){try{return Module["instantiateWasm"](info,receiveInstance)}catch(e){err(`Module.instantiateWasm callback failed with error: ${e}`);return false}}wasmBinaryFile??=findWasmBinary();instantiateAsync(wasmBinary,wasmBinaryFile,info,receiveInstantiationResult);return{}}var callRuntimeCallbacks=callbacks=>{callbacks.forEach(f=>f(Module))};var noExitRuntime=Module["noExitRuntime"]||true;var __emscripten_memcpy_js=(dest,src,num)=>HEAPU8.copyWithin(dest,src,src+num);var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var b=wasmMemory.buffer;var pages=(size-b.byteLength+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var wasmImports={b:__emscripten_memcpy_js,a:_emscripten_resize_heap};var wasmExports=createWasm();var ___wasm_call_ctors=()=>(___wasm_call_ctors=wasmExports["d"])();var _malloc=Module["_malloc"]=a0=>(_malloc=Module["_malloc"]=wasmExports["f"])(a0);var _free=Module["_free"]=a0=>(_free=Module["_free"]=wasmExports["g"])(a0);var _ZSTD_isError=Module["_ZSTD_isError"]=a0=>(_ZSTD_isError=Module["_ZSTD_isError"]=wasmExports["h"])(a0);var _ZSTD_compressBound=Module["_ZSTD_compressBound"]=a0=>(_ZSTD_compressBound=Module["_ZSTD_compressBound"]=wasmExports["i"])(a0);var _ZSTD_compress=Module["_ZSTD_compress"]=(a0,a1,a2,a3,a4)=>(_ZSTD_compress=Module["_ZSTD_compress"]=wasmExports["j"])(a0,a1,a2,a3,a4);var _ZSTD_createDStream=Module["_ZSTD_createDStream"]=()=>(_ZSTD_createDStream=Module["_ZSTD_createDStream"]=wasmExports["k"])();var _ZSTD_freeDStream=Module["_ZSTD_freeDStream"]=a0=>(_ZSTD_freeDStream=Module["_ZSTD_freeDStream"]=wasmExports["l"])(a0);var _ZSTD_DStreamOutSize=Module["_ZSTD_DStreamOutSize"]=()=>(_ZSTD_DStreamOutSize=Module["_ZSTD_DStreamOutSize"]=wasmExports["m"])();var _ZSTD_initDStream=Module["_ZSTD_initDStream"]=a0=>(_ZSTD_initDStream=Module["_ZSTD_initDStream"]=wasmExports["n"])(a0);var _ZSTD_decompressStream=Module["_ZSTD_decompressStream"]=(a0,a1,a2)=>(_ZSTD_decompressStream=Module["_ZSTD_decompressStream"]=wasmExports["o"])(a0,a1,a2);var calledRun;var calledPrerun;dependenciesFulfilled=function runCaller(){if(!calledRun)run();if(!calledRun)dependenciesFulfilled=runCaller};function run(){if(runDependencies>0){return}if(!calledPrerun){calledPrerun=1;preRun();if(runDependencies>0){return}}function doRun(){if(calledRun)return;calledRun=1;Module["calledRun"]=1;if(ABORT)return;initRuntime();Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].pop()()}}run(); + +// Promise that resolves when the module is ready +let moduleReady = new Promise((resolve) => { + if (typeof Module !== 'undefined' && Module.calledRun) { + // Module already initialized + resolve(); + } else { + // Wait for module initialization + const originalOnRuntimeInitialized = Module.onRuntimeInitialized || function() {}; + Module.onRuntimeInitialized = function() { + originalOnRuntimeInitialized(); + resolve(); + }; + } +}); + +async function compressData(inputData, compressionLevel) { + await moduleReady; + + if (!(inputData instanceof Uint8Array) + || !Number.isInteger(compressionLevel) + || compressionLevel < 1 + || compressionLevel > 22) { + return null; + } + + let inputPtr = 0; + let outputPtr = 0; + try { + inputPtr = Module._malloc(Math.max(inputData.length, 1)); + if (!inputPtr) return null; + Module.HEAPU8.set(inputData, inputPtr); + + const outputBufferSize = Number(Module._ZSTD_compressBound(inputData.length)); + if (!Number.isSafeInteger(outputBufferSize) || outputBufferSize <= 0) { + return null; + } + outputPtr = Module._malloc(outputBufferSize); + if (!outputPtr) return null; + + const compressedSize = Number(Module._ZSTD_compress( + outputPtr, + outputBufferSize, + inputPtr, + inputData.length, + compressionLevel + )); + if (Module._ZSTD_isError(compressedSize) !== 0 || compressedSize <= 0) { + console.error('Compression error, error code: ', compressedSize); + return null; + } + return new Uint8Array( + Module.HEAPU8.buffer, + outputPtr, + compressedSize + ).slice(); + } finally { + if (inputPtr) Module._free(inputPtr); + if (outputPtr) Module._free(outputPtr); + } +} + +async function decompressData(compressedData, maxOutputSize = 256 * 1024 * 1024) { + await moduleReady; + + if (!(compressedData instanceof Uint8Array) + || !Number.isSafeInteger(maxOutputSize) + || maxOutputSize < 0 + || compressedData.length === 0) { + return null; + } + + // ZSTD_inBuffer and ZSTD_outBuffer contain three wasm32 size_t/pointer + // fields each. Keep their allocation and field access local to the Worker. + const bufferStructSize = 3 * Uint32Array.BYTES_PER_ELEMENT; + let compressedPtr = 0; + let inputBufferPtr = 0; + let outputBufferPtr = 0; + let outputChunkPtr = 0; + let stream = 0; + try { + compressedPtr = Module._malloc(compressedData.length); + inputBufferPtr = Module._malloc(bufferStructSize); + outputBufferPtr = Module._malloc(bufferStructSize); + stream = Module._ZSTD_createDStream(); + if (!compressedPtr || !inputBufferPtr || !outputBufferPtr || !stream) { + return null; + } + Module.HEAPU8.set(compressedData, compressedPtr); + + const initialization = Number(Module._ZSTD_initDStream(stream)); + if (Module._ZSTD_isError(initialization) !== 0) return null; + + const recommendedChunkSize = Number(Module._ZSTD_DStreamOutSize()); + if (!Number.isSafeInteger(recommendedChunkSize) + || recommendedChunkSize <= 0) { + return null; + } + const outputChunkSize = Math.max( + 1, + Math.min(recommendedChunkSize, Math.max(maxOutputSize, 1)), + ); + outputChunkPtr = Module._malloc(outputChunkSize); + if (!outputChunkPtr) return null; + + const writeField = (structPtr, field, value) => { + Module.HEAPU32[(structPtr >>> 2) + field] = value; + }; + const readField = (structPtr, field) => + Module.HEAPU32[(structPtr >>> 2) + field]; + + writeField(inputBufferPtr, 0, compressedPtr); + writeField(inputBufferPtr, 1, compressedData.length); + writeField(inputBufferPtr, 2, 0); + + const chunks = []; + let totalLength = 0; + let previousInputPosition = -1; + let previousOutputLength = -1; + while (true) { + writeField(outputBufferPtr, 0, outputChunkPtr); + writeField(outputBufferPtr, 1, outputChunkSize); + writeField(outputBufferPtr, 2, 0); + + const remaining = Number(Module._ZSTD_decompressStream( + stream, + outputBufferPtr, + inputBufferPtr, + )); + if (Module._ZSTD_isError(remaining) !== 0) return null; + + const inputPosition = readField(inputBufferPtr, 2); + const produced = readField(outputBufferPtr, 2); + if (inputPosition > compressedData.length + || produced > outputChunkSize + || totalLength + produced > maxOutputSize) { + return null; + } + if (produced > 0) { + chunks.push(new Uint8Array( + Module.HEAPU8.buffer, + outputChunkPtr, + produced, + ).slice()); + totalLength += produced; + } + + const allInputConsumed = inputPosition === compressedData.length; + if (remaining === 0 && allInputConsumed) { + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + return result; + } + + const madeProgress = inputPosition !== previousInputPosition + || totalLength !== previousOutputLength; + if (!madeProgress || (allInputConsumed && produced === 0)) { + return null; + } + previousInputPosition = inputPosition; + previousOutputLength = totalLength; + } + } finally { + if (stream) Module._ZSTD_freeDStream(stream); + if (compressedPtr) Module._free(compressedPtr); + if (inputBufferPtr) Module._free(inputBufferPtr); + if (outputBufferPtr) Module._free(outputBufferPtr); + if (outputChunkPtr) Module._free(outputChunkPtr); + } +} diff --git a/zstandard_web/example/web/zstd_worker.js b/zstandard_web/example/web/zstd_worker.js new file mode 100644 index 00000000..38e076eb --- /dev/null +++ b/zstandard_web/example/web/zstd_worker.js @@ -0,0 +1,20 @@ +'use strict'; + +importScripts('zstd_core.js'); + +self.onmessage = async (event) => { + const { id, operation, inputBuffer, option } = event.data; + try { + const input = new Uint8Array(inputBuffer); + const result = operation === 'compress' + ? await compressData(input, option) + : await decompressData(input, option); + if (result === null) { + self.postMessage({ id, result: null }); + } else { + self.postMessage({ id, result }, [result.buffer]); + } + } catch (_) { + self.postMessage({ id, result: null }); + } +}; diff --git a/zstandard_web/images/sample.png b/zstandard_web/images/sample.png index 18884159..c5cd681c 100644 Binary files a/zstandard_web/images/sample.png and b/zstandard_web/images/sample.png differ diff --git a/zstandard_web/lib/zstandard_ext.dart b/zstandard_web/lib/zstandard_ext.dart index 465add0a..d772e005 100644 --- a/zstandard_web/lib/zstandard_ext.dart +++ b/zstandard_web/lib/zstandard_ext.dart @@ -9,9 +9,12 @@ extension ZstandardExt on Uint8List? { return ZstandardWeb().compress(data, compressionLevel); } - Future decompress() async { + Future decompress({int maxOutputSize = 256 * 1024 * 1024}) async { var data = this; if (data == null) return null; - return ZstandardWeb().decompress(data); + return ZstandardWeb().decompressWithOptions( + data, + maxOutputSize: maxOutputSize, + ); } } diff --git a/zstandard_web/lib/zstandard_web.dart b/zstandard_web/lib/zstandard_web.dart index 09fb67ff..fe30358d 100644 --- a/zstandard_web/lib/zstandard_web.dart +++ b/zstandard_web/lib/zstandard_web.dart @@ -11,11 +11,13 @@ export 'zstandard_ext.dart'; /// Web implementation of [ZstandardPlatform] using JavaScript and WebAssembly. /// -/// Calls the global `compressData` and `decompressData` functions provided by -/// zstd.js / zstd.wasm. Requires zstd.js and zstd.wasm to be loaded in the -/// page (e.g. via a script tag in index.html). The main [zstandard] plugin -/// registers this implementation automatically on web. -class ZstandardWeb extends ZstandardPlatform { +/// Calls the asynchronous `compressData` and `decompressData` broker functions +/// provided by `zstd.js`. The broker transfers work to `zstd_worker.js`, which +/// loads `zstd_core.js` and `zstd.wasm` away from the browser UI thread. All +/// four files must be deployed together. The main [zstandard] plugin registers +/// this implementation automatically on web. +class ZstandardWeb extends ZstandardPlatform + implements BoundedZstandardPlatform { /// Creates the web platform implementation. /// /// [debugWindow] is visible for testing to override the window object. @@ -37,28 +39,43 @@ class ZstandardWeb extends ZstandardPlatform { @override Future compress(Uint8List data, int compressionLevel) async { - var promise = html.window.callMethodVarArgs('compressData'.toJS, [ - data.toJS, - compressionLevel.toJS, - ]) as JSPromise; - var compressedData = (await promise.toDart) as JSUint8Array?; - if (compressedData != null) { - return compressedData.toDart; - } else { - throw Exception("Error compressing."); + if (compressionLevel < 1 || compressionLevel > 22) return null; + try { + var promise = html.window.callMethodVarArgs('compressData'.toJS, [ + data.toJS, + compressionLevel.toJS, + ]) as JSPromise; + var compressedData = (await promise.toDart) as JSUint8Array?; + if (compressedData != null) { + return compressedData.toDart; + } + } on Object { + return null; } + return null; } @override - Future decompress(Uint8List data) async { - var promise = html.window.callMethodVarArgs('decompressData'.toJS, [ - data.toJS, - ]) as JSPromise; - var decompressedData = (await promise.toDart) as JSUint8Array?; - if (decompressedData != null) { - return decompressedData.toDart; - } else { - throw Exception("Error decompressing."); + Future decompress(Uint8List data) => decompressWithOptions(data); + + @override + Future decompressWithOptions( + Uint8List data, { + int maxOutputSize = ZstandardPlatform.defaultMaxDecompressedSize, + }) async { + if (maxOutputSize < 0) return null; + try { + var promise = html.window.callMethodVarArgs('decompressData'.toJS, [ + data.toJS, + maxOutputSize.toJS, + ]) as JSPromise; + var decompressedData = (await promise.toDart) as JSUint8Array?; + if (decompressedData != null) { + return decompressedData.toDart; + } + } on Object { + return null; } + return null; } } diff --git a/zstandard_web/test/wasm_assets_test.dart b/zstandard_web/test/wasm_assets_test.dart new file mode 100644 index 00000000..71a0e0b0 --- /dev/null +++ b/zstandard_web/test/wasm_assets_test.dart @@ -0,0 +1,80 @@ +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('committed web artifacts are synchronized and bounded', () async { + final packageJavaScript = File('blob/zstd.js'); + final packageCore = File('blob/zstd_core.js'); + final packageWorker = File('blob/zstd_worker.js'); + final packageWasm = File('blob/zstd.wasm'); + final webExampleJavaScript = File('example/web/zstd.js'); + final webExampleCore = File('example/web/zstd_core.js'); + final webExampleWorker = File('example/web/zstd_worker.js'); + final webExampleWasm = File('example/web/zstd.wasm'); + final rootExampleJavaScript = File('../zstandard/example/web/zstd.js'); + final rootExampleCore = File('../zstandard/example/web/zstd_core.js'); + final rootExampleWorker = File('../zstandard/example/web/zstd_worker.js'); + final rootExampleWasm = File('../zstandard/example/web/zstd.wasm'); + + for (final artifact in [ + packageJavaScript, + packageCore, + packageWorker, + packageWasm, + webExampleJavaScript, + webExampleCore, + webExampleWorker, + webExampleWasm, + rootExampleJavaScript, + rootExampleCore, + rootExampleWorker, + rootExampleWasm, + ]) { + expect(artifact.existsSync(), isTrue, reason: artifact.path); + expect(artifact.lengthSync(), greaterThan(0), reason: artifact.path); + } + + expect( + await webExampleJavaScript.readAsBytes(), + equals(await packageJavaScript.readAsBytes()), + ); + expect( + await rootExampleJavaScript.readAsBytes(), + equals(await packageJavaScript.readAsBytes()), + ); + expect( + await webExampleCore.readAsBytes(), + equals(await packageCore.readAsBytes()), + ); + expect( + await rootExampleCore.readAsBytes(), + equals(await packageCore.readAsBytes()), + ); + expect( + await webExampleWorker.readAsBytes(), + equals(await packageWorker.readAsBytes()), + ); + expect( + await rootExampleWorker.readAsBytes(), + equals(await packageWorker.readAsBytes()), + ); + expect( + await webExampleWasm.readAsBytes(), + equals(await packageWasm.readAsBytes()), + ); + expect( + await rootExampleWasm.readAsBytes(), + equals(await packageWasm.readAsBytes()), + ); + + final client = await packageJavaScript.readAsString(); + final core = await packageCore.readAsString(); + expect(client, contains('new Worker(workerUrl)')); + expect(client, contains('[inputCopy.buffer]')); + expect(core, contains('Module._ZSTD_decompressStream')); + expect(core, contains('Module._ZSTD_createDStream')); + expect(core, contains('maxOutputSize = 256 * 1024 * 1024')); + expect(core, isNot(contains('compressedData.length * 20'))); + }); +} diff --git a/zstandard_windows/CHANGELOG.md b/zstandard_windows/CHANGELOG.md index 14965197..d43dfa28 100644 --- a/zstandard_windows/CHANGELOG.md +++ b/zstandard_windows/CHANGELOG.md @@ -1,3 +1,8 @@ +## Unreleased + +- Added byte-safe worker-isolate execution and bounded streaming decompression + for unknown-size and concatenated frames. + ## 1.5.0 - Dependencies Updated - Updated direct dependencies diff --git a/zstandard_windows/README.md b/zstandard_windows/README.md index eb967520..949539f2 100644 --- a/zstandard_windows/README.md +++ b/zstandard_windows/README.md @@ -10,7 +10,7 @@ Add the main plugin to your app; this package is included automatically via the ```yaml dependencies: - zstandard: ^1.3.29 + zstandard: ^1.5.0 ``` No extra setup is required for normal use. @@ -42,12 +42,15 @@ final decompressed = await compressed?.decompress(); - **ZstandardWindows()** — Creates the Windows platform implementation. - **compress(Uint8List data, int compressionLevel)** — Compresses `data` (level 1–22). Returns compressed bytes or `null`. -- **decompress(Uint8List data)** — Decompresses zstd-compressed data. Returns decompressed bytes or `null`. +- **decompressWithOptions(Uint8List data, {int maxOutputSize})** — Decompresses complete, concatenated, or unknown-size zstd frames with a bounded output (256 MiB by default). Invalid, truncated, or oversized input returns `null`. - **getPlatformVersion()** — Returns a platform identifier string. ## Architecture -This package uses Dart FFI to load `zstandard_windows.dll` and call the Zstandard C API. The DLL is built by CMake when you build your Flutter Windows app. Supports x64 and arm64. +This package uses Dart FFI to load `zstandard_windows.dll`; CMake builds it +with the Flutter application. Public work runs in a worker isolate using +Dart-owned bytes, and decompression enforces the configured output limit as it +streams. Both x64 and arm64 are supported. ## Testing diff --git a/zstandard_windows/analysis_options.yaml b/zstandard_windows/analysis_options.yaml index a5744c1c..ba427235 100644 --- a/zstandard_windows/analysis_options.yaml +++ b/zstandard_windows/analysis_options.yaml @@ -1,3 +1,7 @@ +analyzer: + exclude: + - build/** + - windows/** include: package:flutter_lints/flutter.yaml # Additional information about this file can be found at diff --git a/zstandard_windows/example/analysis_options.yaml b/zstandard_windows/example/analysis_options.yaml index 0d290213..6bd848b7 100644 --- a/zstandard_windows/example/analysis_options.yaml +++ b/zstandard_windows/example/analysis_options.yaml @@ -7,6 +7,10 @@ # The following line activates a set of recommended lints for Flutter apps, # packages, and plugins designed to encourage good coding practices. +analyzer: + exclude: + - build/** + - windows/** include: package:flutter_lints/flutter.yaml linter: diff --git a/zstandard_windows/lib/zstandard_ext.dart b/zstandard_windows/lib/zstandard_ext.dart index 9400744e..5fb524bc 100644 --- a/zstandard_windows/lib/zstandard_ext.dart +++ b/zstandard_windows/lib/zstandard_ext.dart @@ -9,9 +9,12 @@ extension ZstandardExt on Uint8List? { return ZstandardWindows().compress(data, compressionLevel); } - Future decompress() async { + Future decompress({int maxOutputSize = 256 * 1024 * 1024}) async { var data = this; if (data == null) return null; - return ZstandardWindows().decompress(data); + return ZstandardWindows().decompressWithOptions( + data, + maxOutputSize: maxOutputSize, + ); } } diff --git a/zstandard_windows/lib/zstandard_windows.dart b/zstandard_windows/lib/zstandard_windows.dart index d2fbecf8..3fc27254 100644 --- a/zstandard_windows/lib/zstandard_windows.dart +++ b/zstandard_windows/lib/zstandard_windows.dart @@ -1,153 +1,69 @@ -import 'dart:async'; import 'dart:ffi'; import 'dart:io'; import 'dart:isolate'; -import 'package:ffi/ffi.dart'; import 'package:flutter/services.dart'; -import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; - +import 'package:zstandard_native/zstandard_native.dart' + show ZstandardNativeCodec; import 'package:zstandard_native/zstandard_native_bindings.dart'; +import 'package:zstandard_platform_interface/zstandard_platform_interface.dart'; export 'zstandard_ext.dart'; const String _libName = 'zstandard_windows'; final DynamicLibrary _dylib = () { - if (Platform.isWindows) { - return DynamicLibrary.open('$_libName.dll'); - } + if (Platform.isWindows) return DynamicLibrary.open('$_libName.dll'); throw UnsupportedError('Platform not supported: ${Platform.operatingSystem}'); }(); final ZstandardNativeBindings _bindings = ZstandardNativeBindings(_dylib); +final ZstandardNativeCodec _codec = ZstandardNativeCodec(_bindings); bool _hasZstdFrameMagic(Uint8List data) { - if (data.lengthInBytes < 4) { - return false; - } - - final int magic = - data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); + if (data.lengthInBytes < 4) return false; + final magic = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); return magic == ZSTD_MAGICNUMBER || (magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START; } -bool _isUnavailableContentSize(int size) { - // The generated bindings expose these C unsigned values as -1 and -2. - // Keep the unsigned representations as a compatibility guard for older - // generated bindings and runtimes that returned the raw 64-bit bit pattern. - return size == ZSTD_CONTENTSIZE_UNKNOWN || - size == ZSTD_CONTENTSIZE_ERROR || - size == 0xffffffffffffffff || - size == 0xfffffffffffffffe; -} - -/// Windows implementation of [ZstandardPlatform] using FFI and the native zstd library. -/// -/// Loads zstandard_windows.dll and uses ZSTD_compress, ZSTD_decompress, -/// ZSTD_compressBound, and ZSTD_getFrameContentSize. The main [zstandard] -/// plugin registers this implementation automatically on Windows. -class ZstandardWindows extends ZstandardPlatform { +/// Windows implementation of [ZstandardPlatform] using the native zstd library. +class ZstandardWindows extends ZstandardPlatform + implements BoundedZstandardPlatform { /// Creates the Windows platform implementation. ZstandardWindows(); final methodChannel = const MethodChannel('plugins.flutter.io/zstandard'); - /// Registers this class as the default instance of [ZstandardPlatform]. - /// - /// Called by the main plugin when running on Windows. + /// Registers this implementation with the federated plugin. static void registerWith() { ZstandardPlatform.instance = ZstandardWindows(); } @override - Future getPlatformVersion() async { - final version = - await methodChannel.invokeMethod('getPlatformVersion'); - return version; - } + Future getPlatformVersion() => + methodChannel.invokeMethod('getPlatformVersion'); @override - Future compress(Uint8List data, int compressionLevel) async { - if (compressionLevel < 1 || compressionLevel > 22) { - return null; - } - - final int srcSize = data.lengthInBytes; - final Pointer src = - malloc.allocate(srcSize > 0 ? srcSize : 1); - src.asTypedList(srcSize).setAll(0, data); - - final int dstCapacity = _bindings.ZSTD_compressBound(srcSize); - if (_bindings.ZSTD_isError(dstCapacity) != 0 || dstCapacity <= 0) { - malloc.free(src); - return null; - } - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int compressedSize = _bindings.ZSTD_compress( - dst.cast(), - dstCapacity, - src.cast(), - srcSize, - compressionLevel, - ); - - if (_bindings.ZSTD_isError(compressedSize) == 0 && compressedSize > 0) { - return Uint8List.fromList(dst.asTypedList(compressedSize)); - } else { - return null; - } - } finally { - malloc.free(src); - malloc.free(dst); - } - } + Future compress(Uint8List data, int compressionLevel) => + Isolate.run(() => _codec.compress(data, compressionLevel)); @override - Future decompress(Uint8List data) async { - // Avoid entering the native decoder for arbitrary input. Apart from - // being cheaper, this prevents malformed data from reaching an ABI - // boundary while the frame header is already known to be invalid. - if (!_hasZstdFrameMagic(data)) { - return null; - } + Future decompress(Uint8List data) => decompressWithOptions(data); - final int compressedSize = data.lengthInBytes; - final Pointer src = malloc.allocate(compressedSize); - src.asTypedList(compressedSize).setAll(0, data); - - final int decompressedSizeExpected = - _bindings.ZSTD_getFrameContentSize(src.cast(), compressedSize); - if (_isUnavailableContentSize(decompressedSizeExpected)) { - malloc.free(src); - return null; - } - final int dstCapacity = - decompressedSizeExpected > 0 ? decompressedSizeExpected : 1; - final Pointer dst = malloc.allocate(dstCapacity); - - try { - final int decompressedSize = _bindings.ZSTD_decompress( - dst.cast(), - dstCapacity, - src.cast(), - compressedSize, + @override + Future decompressWithOptions( + Uint8List data, { + int maxOutputSize = ZstandardPlatform.defaultMaxDecompressedSize, + }) => + Isolate.run( + () => _hasZstdFrameMagic(data) + ? _codec.decompress(data, maxOutputSize: maxOutputSize) + : null, ); - - if (_bindings.ZSTD_isError(decompressedSize) != 0) { - return null; - } - return Uint8List.fromList(dst.asTypedList(decompressedSize)); - } finally { - malloc.free(src); - malloc.free(dst); - } - } } +/// Low-level synchronous compression for existing FFI consumers. int compress( Pointer dst, int dstCapacity, @@ -155,170 +71,36 @@ int compress( int srcSize, int compressionLevel, ) => - _bindings.ZSTD_compress( - dst, - dstCapacity, - src, - srcSize, - compressionLevel, - ); + _bindings.ZSTD_compress(dst, dstCapacity, src, srcSize, compressionLevel); +/// Low-level synchronous decompression for existing FFI consumers. int decompress( Pointer dst, int dstCapacity, Pointer src, int compressedSize, ) => - _bindings.ZSTD_decompress( - dst, - dstCapacity, - src, - compressedSize, - ); + _bindings.ZSTD_decompress(dst, dstCapacity, src, compressedSize); +/// Compatibility wrapper that completes after the pointer call returns. +@Deprecated('Use ZstandardWindows.compress with Dart-owned bytes instead.') Future compressAsync( Pointer dst, int dstCapacity, Pointer src, int srcSize, int compressionLevel, -) async { - final SendPort helperIsolateSendPort = await _getHelperIsolateSendPort(); - final int requestId = _nextCompressRequestId++; - final _CompressRequest request = _CompressRequest( - requestId, dst, dstCapacity, src, srcSize, compressionLevel); - final Completer completer = Completer(); - _compressRequests[requestId] = completer; - helperIsolateSendPort.send(request); - return completer.future; -} +) => + Future.sync( + () => compress(dst, dstCapacity, src, srcSize, compressionLevel), + ); +/// Compatibility wrapper that completes after the pointer call returns. +@Deprecated('Use ZstandardWindows.decompress with Dart-owned bytes instead.') Future decompressAsync( Pointer dst, int dstCapacity, Pointer src, int compressedSize, -) async { - final SendPort helperIsolateSendPort = await _getHelperIsolateSendPort(); - final int requestId = _nextDecompressRequestId++; - final _DecompressRequest request = - _DecompressRequest(requestId, dst, dstCapacity, src, compressedSize); - final Completer completer = Completer(); - _decompressRequests[requestId] = completer; - helperIsolateSendPort.send(request); - return completer.future; -} - -// ==== Communication between isolates for asynchronous compression and decompression ==== // - -/// Application for compression. -class _CompressRequest { - final int id; - final Pointer dst; - final int dstCapacity; - final Pointer src; - final int srcSize; - final int compressionLevel; - - const _CompressRequest(this.id, this.dst, this.dstCapacity, this.src, - this.srcSize, this.compressionLevel); -} - -/// Response with the compression result. -class _CompressResponse { - final int id; - final int result; - - const _CompressResponse(this.id, this.result); -} - -/// Request for decompression. -class _DecompressRequest { - final int id; - final Pointer dst; - final int dstCapacity; - final Pointer src; - final int compressedSize; - - const _DecompressRequest( - this.id, this.dst, this.dstCapacity, this.src, this.compressedSize); -} - -/// Response with the result of the decompression. -class _DecompressResponse { - final int id; - final int result; - - const _DecompressResponse(this.id, this.result); -} - -/// Counters to identify compression and decompression requests. -int _nextCompressRequestId = 0; -int _nextDecompressRequestId = 0; - -/// Mapping of requests to completers for compression and decompression. -final Map> _compressRequests = >{}; -final Map> _decompressRequests = >{}; - -/// Port for sending requests to the auxiliary isolate. -// Start the worker lazily so importing the plugin cannot keep test processes -// alive when the async helper API is not used. -Future? _helperIsolateSendPort; - -Future _getHelperIsolateSendPort() => - _helperIsolateSendPort ??= () async { - final Completer completer = Completer(); - final ReceivePort receivePort = ReceivePort() - ..listen((dynamic data) { - if (data is SendPort) { - completer.complete(data); - return; - } - if (data is _CompressResponse) { - final Completer completer = _compressRequests[data.id]!; - _compressRequests.remove(data.id); - completer.complete(data.result); - return; - } - if (data is _DecompressResponse) { - final Completer completer = _decompressRequests[data.id]!; - _decompressRequests.remove(data.id); - completer.complete(data.result); - return; - } - throw UnsupportedError( - 'Message type not supported: ${data.runtimeType}'); - }); - - await Isolate.spawn((SendPort sendPort) async { - final ReceivePort helperReceivePort = ReceivePort() - ..listen((dynamic data) { - if (data is _CompressRequest) { - final int result = _bindings.ZSTD_compress( - data.dst, - data.dstCapacity, - data.src, - data.srcSize, - data.compressionLevel); - final _CompressResponse response = - _CompressResponse(data.id, result); - sendPort.send(response); - return; - } - if (data is _DecompressRequest) { - final int result = _bindings.ZSTD_decompress( - data.dst, data.dstCapacity, data.src, data.compressedSize); - final _DecompressResponse response = - _DecompressResponse(data.id, result); - sendPort.send(response); - return; - } - throw UnsupportedError( - 'Message type not supported: ${data.runtimeType}'); - }); - - sendPort.send(helperReceivePort.sendPort); - }, receivePort.sendPort); - - return completer.future; - }(); +) => + Future.sync(() => decompress(dst, dstCapacity, src, compressedSize)); diff --git a/zstandard_windows/test/zstandard_windows_test.dart b/zstandard_windows/test/zstandard_windows_test.dart index 44efab3c..7510aed3 100644 --- a/zstandard_windows/test/zstandard_windows_test.dart +++ b/zstandard_windows/test/zstandard_windows_test.dart @@ -29,7 +29,9 @@ void main() { test('compress and decompress large data', () async { if (skipPlatform) return; - final data = Uint8List.fromList(List.generate(100000, (i) => i % 256)); + final data = Uint8List.fromList( + List.generate(100000, (i) => i % 256), + ); expect(data, isNotEmpty); final compressed = await zstandard.compress(data, 3); expect(compressed, isNotNull); @@ -70,7 +72,9 @@ void main() { test('decompress random bytes returns null', () async { if (skipPlatform) return; - final random = Uint8List.fromList(List.generate(64, (i) => (i * 31) % 256)); + final random = Uint8List.fromList( + List.generate(64, (i) => (i * 31) % 256), + ); final result = await zstandard.decompress(random); expect(result, isNull); }, skip: skipPlatform ? 'Only runs on Windows' : false); diff --git a/zstandard_windows/zstd_build/CMakeLists.txt b/zstandard_windows/zstd_build/CMakeLists.txt index 2371854c..03c329a9 100644 --- a/zstandard_windows/zstd_build/CMakeLists.txt +++ b/zstandard_windows/zstd_build/CMakeLists.txt @@ -34,17 +34,8 @@ else() message(STATUS "Using local zstandard_native: ${ZSTD_ROOT}") endif() if(NOT ZSTD_ROOT OR NOT EXISTS "${ZSTD_ROOT}/common") - # 2. Look for zstandard_native in pub-cache (versioned dirs only, e.g. zstandard_native-1.4.0) - file(GLOB NATIVE_PARENT_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/../../zstandard_native-*") - if(NATIVE_PARENT_DIRS) - list(GET NATIVE_PARENT_DIRS 0 NATIVE_PKG_DIR) - set(ZSTD_CANDIDATE "${NATIVE_PKG_DIR}/src/zstd") - if(EXISTS "${ZSTD_CANDIDATE}/common") - set(ZSTD_ROOT "${ZSTD_CANDIDATE}") - message(STATUS "Using zstandard_native from pub-cache: ${ZSTD_ROOT}") - endif() - endif() - # 3. Resolve zstandard_native from package_config.json (plugin or any parent, e.g. example app) + # 2. Resolve the exact dependency selected by pub. Never choose the first + # matching cache directory, which can silently compile a stale version. set(_PKG_CONFIG_FOUND OFF) set(_SEARCH_PKG "${CMAKE_CURRENT_SOURCE_DIR}") foreach(_PKG_IGNORE RANGE 10) @@ -75,7 +66,7 @@ else() set(_SEARCH_PKG "${_PARENT}") endforeach() if(NOT ZSTD_ROOT OR NOT EXISTS "${ZSTD_ROOT}/common") - message(FATAL_ERROR "zstd source not found. Set ZSTD_ROOT or ensure zstandard_native is available (repo: ../../zstandard_native/src/zstd or pub-cache).") + message(FATAL_ERROR "zstd source not found. Set ZSTD_ROOT or run flutter pub get so package_config.json resolves zstandard_native.") endif() endif() endif()