diff --git a/.github/workflows/build-ipa.yml b/.github/workflows/build-ipa.yml index ec04950..0acadb8 100644 --- a/.github/workflows/build-ipa.yml +++ b/.github/workflows/build-ipa.yml @@ -2,19 +2,11 @@ name: Build XForge IPA # Builds the unsigned XForge IPA. # -# There is no rootfs-provisioning job here. The Alpine root — including the build -# toolchain, when the published root carries one — is built once by the `rootfs` -# workflow, published as a pinned release asset and downloaded here; see -# EmbeddedLinux/build-rootfs.sh. Provisioning a Swift toolchain into every CI run -# is what used to make this job slow and its artifact enormous. What arrives now -# is a fakefs ZIP the app only unzips: a guest that can build an iOS app on first -# launch, with nothing to install on the device. -# -# The engine (OpenMinis/ish-arm64) is built here, on macOS, because it can only -# be: the aarch64 gadget sources alias registers with `.req` and only clang's -# integrated assembler accepts that, so a GNU binutils toolchain cannot assemble -# them at all. `EmbeddedLinux/build-ish-core.sh` probes for the capability and -# fails with an explanation rather than a wall of assembler errors. +# There is no guest to provision and no engine to build: the compiler is linked +# into the app from the toolchain bundle (Vendor/NativeToolchain), which +# `native-toolchain.yml` builds in its own workflow. A checkout without the bundle +# still builds — the bridge compiles to a "not available" stub — so the archive +# here is the app itself, and the Toolchain screen reports what is missing. on: push: @@ -31,6 +23,14 @@ on: description: Run simulator unit tests before archiving type: boolean default: false + with_toolchain: + description: Install the published native toolchain bundle and link it into the app + type: boolean + default: false + toolchain_tag: + description: Toolchain release tag to install (default - the newest toolchain-* release, or "artifact" for the newest workflow artifact) + type: string + default: "" permissions: contents: write @@ -48,248 +48,115 @@ jobs: env: APP_PATH: build/XForge.xcarchive/Products/Applications/XForge.app DERIVED_DATA_PATH: build/DerivedData - ENGINE_ROOT: Vendor/ish-arm64 - ENGINE_LIB_DIR: Vendor/ish-arm64-build/lib # Must match XFORGE_MARKETING_VERSION in project.yml; asserted below so a # version bump cannot silently produce a mislabelled release. EXPECTED_VERSION: "0.5.0" - # The pinned rootfs. Changing this tag is how the bundled Alpine root is - # upgraded; `make rootfs` (or the `rootfs` workflow) builds and publishes - # a new one. The sha256 is asserted below, so a tag that is moved — or an - # upload that was truncated — fails the build instead of shipping silently. - # - # rootfs-v5 is the *provisioned* root: plain Alpine plus xtool, swiftly, the - # Swift toolchain and the Darwin SDK, so the guest can build without first - # installing anything. Its manifest says how it was built, and the checks - # below hold it to that. - ROOTFS_TAG: rootfs-v5 - ROOTFS_ASSET: alpine-rootfs.zip SPM_CLONES_DIR: .spm steps: - name: Check out the repository uses: actions/checkout@v4 - - name: Download the pinned Alpine rootfs - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - # The same script any other build calls (and that Xcode users run by - # hand): it reads the pinned tag from this file, downloads the asset, - # verifies the published sha256 and checks it is a fakefs ZIP. - bash EmbeddedLinux/fetch-rootfs.sh - - - name: Verify the downloaded rootfs - run: | - set -euo pipefail - cd Support/Resources - # Read the checksum file and verify, rather than piping a listing into - # grep: `producer | grep -q` closes the pipe early, the producer dies - # of SIGPIPE, and under `set -o pipefail` a check that passed then - # reports failure. - shasum -a 256 -c "$ROOTFS_ASSET.sha256" - - unzip -Z1 "$ROOTFS_ASSET" > /tmp/rootfs-entries.txt - grep -q '^alpine-rootfs/meta\.db$' /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs archive has no meta.db'; exit 1; } - grep -q '^alpine-rootfs/data/$' /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs archive has no data/ directory'; exit 1; } - grep -q '^alpine-rootfs/data/bin/sh$' /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs archive has no /bin/sh'; exit 1; } - # SQLite sidecars are transient; the builder excludes them, and a - # regression there would put a stale WAL into every install. - ! grep -qE '\.db-(wal|shm)$' /tmp/rootfs-entries.txt - - # The glibc compatibility layer must be in the root *and indexed in - # meta.db*: it is installed before the fakefs conversion precisely so - # the engine can resolve it. Checking the ZIP listing alone would pass - # for a root where the files exist on disk but the database has no rows - # for them, which the guest reads as "not there at all". - grep -q '^alpine-rootfs/data/opt/glibc/usr/lib/aarch64-linux-gnu/ld-linux-aarch64\.so\.1$' \ - /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs has no glibc layer'; exit 1; } - grep -q '^alpine-rootfs/data/usr/local/share/xforge/glibc\.env$' \ - /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs has no glibc.env marker'; exit 1; } - unzip -p "$ROOTFS_ASSET" alpine-rootfs/meta.db > /tmp/rootfs-meta.db - indexed="$(sqlite3 /tmp/rootfs-meta.db \ - "SELECT COUNT(*) FROM paths WHERE CAST(path AS TEXT) LIKE '%aarch64-linux-gnu%';")" - [ "${indexed:-0}" -gt 0 ] \ - || { echo "::error::the glibc layer is not indexed in meta.db ($indexed paths)"; exit 1; } - echo "glibc layers paths indexed: $indexed" - - # The guest boots /sbin/init, which reads /etc/inittab and respawns - # /sbin/xforge-login root on the console — the script that starts root's - # login shell in /etc/passwd. That is the whole Terminal tab, and the - # shell it names is part of the root's contract, so both are checked - # here: a root pinned to a tag that predates them fails the build - # instead of shipping a terminal that cannot start a shell. - unzip -p "$ROOTFS_ASSET" alpine-rootfs/data/etc/inittab > /tmp/rootfs-inittab.txt - grep -qE '^tty1::respawn:/sbin/xforge-login root$' /tmp/rootfs-inittab.txt \ - || { echo '::error::the rootfs does not respawn xforge-login on the console'; exit 1; } - unzip -p "$ROOTFS_ASSET" alpine-rootfs/data/usr/local/share/xforge/rootfs-manifest.txt \ - > /tmp/rootfs-manifest.txt - ROOTFS_SHELL="$(awk '/^[[:space:]]*shell:/ { print $2 }' /tmp/rootfs-manifest.txt)" - [ -n "$ROOTFS_SHELL" ] \ - || { echo '::error::the rootfs manifest does not name a login shell'; exit 1; } - unzip -Z1 "$ROOTFS_ASSET" > /tmp/rootfs-files.txt - for path in \ - alpine-rootfs/data/sbin/init \ - alpine-rootfs/data/bin/login \ - alpine-rootfs/data/etc/inittab \ - alpine-rootfs/data/etc/init.d/rcS \ - alpine-rootfs/data/sbin/xforge-login \ - "alpine-rootfs/data${ROOTFS_SHELL}" \ - alpine-rootfs/data/usr/share/terminfo/x/xterm-256color ; do - grep -qx "$path" /tmp/rootfs-files.txt \ - || { echo "::error::the rootfs has no ${path#alpine-rootfs/data}"; exit 1; } - done - echo "console: /sbin/xforge-login root → $ROOTFS_SHELL" - - # The root's own manifest also says whether the build toolchain is baked - # in, and this is the other half of what the root is for: a guest that - # can build an iOS app without installing anything first. The checks are - # held to the manifest's claim, so the *plain* root still passes and a - # provisioned root that lost its toolchain does not — and the SDK is - # checked through meta.db as well as the listing, because a file the - # engine cannot see is a file the guest does not have. - TOOLCHAIN_CLAIM="$(awk '/^[[:space:]]*toolchain:/ { sub(/^[[:space:]]*toolchain:[[:space:]]*/, ""); print; exit }' /tmp/rootfs-manifest.txt)" - echo "toolchain: ${TOOLCHAIN_CLAIM:-}" - case "$TOOLCHAIN_CLAIM" in - ""|"not provisioned"*) - echo "::error::the pinned rootfs ($ROOTFS_TAG) ships no build toolchain — - the guest would have to install Swift and xtool itself on the device. Bump - ROOTFS_TAG to a provisioned root, or say so deliberately here." - exit 1 - ;; - *) - for path in \ - alpine-rootfs/data/usr/local/bin/xtool \ - alpine-rootfs/data/usr/local/bin/swift \ - alpine-rootfs/data/usr/local/bin/swiftc \ - alpine-rootfs/data/usr/local/bin/swiftly \ - alpine-rootfs/data/opt/xtool/usr/bin/xtool ; do - grep -qx "$path" /tmp/rootfs-entries.txt \ - || { echo "::error::the rootfs claims a toolchain but has no ${path#alpine-rootfs/data}"; exit 1; } - done - grep -qE '^alpine-rootfs/data/root/\.local/share/swiftly/toolchains/[^/]+/usr/bin/swift-frontend$' \ - /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs has no Swift toolchain'; exit 1; } - # By shape, and *unanchored* at the end, for two reasons that each - # cost a build: Swift 6.4 ships linux and linux-musl variants, so - # naming one exact directory fails a good root; and a `.swiftmodule` - # is a *directory* in Swift 6, whose entry ends in a slash — a - # pattern anchored with `$` finds nothing. verify-rootfs.sh asks the - # same two questions the same way. - grep -qE '^alpine-rootfs/data/root/\.local/share/swiftly/toolchains/[^/]+/usr/lib/swift/.*/libswiftCore\.so' \ - /tmp/rootfs-entries.txt \ - || { echo '::error::the Swift toolchain has no Linux stdlib'; exit 1; } - grep -qE '^alpine-rootfs/data/root/\.local/share/swiftly/toolchains/[^/]+/usr/lib/swift/.*/Swift\.swiftmodule' \ - /tmp/rootfs-entries.txt \ - || { echo '::error::the Swift toolchain has no Linux stdlib interface'; exit 1; } - - SDK_PATH="$(awk '/^[[:space:]]*darwin-sdk-path:/ { print $2; exit }' /tmp/rootfs-manifest.txt)" - [ -n "$SDK_PATH" ] \ - || { echo '::error::the rootfs claims a toolchain but names no Darwin SDK path'; exit 1; } - grep -qE "^alpine-rootfs/data${SDK_PATH}/" /tmp/rootfs-entries.txt \ - || { echo "::error::the rootfs has no Darwin SDK at $SDK_PATH"; exit 1; } - sdk_indexed="$(sqlite3 /tmp/rootfs-meta.db \ - "SELECT COUNT(*) FROM paths WHERE CAST(path AS TEXT) LIKE '${SDK_PATH%/}/%';")" - [ "${sdk_indexed:-0}" -gt 0 ] \ - || { echo "::error::the Darwin SDK is in the zip but not indexed in meta.db ($sdk_indexed paths)"; exit 1; } - echo "toolchain: xtool, swift, and the Darwin SDK at $SDK_PATH ($sdk_indexed indexed paths)" - ;; - esac - - echo "rootfs: $(du -h "$ROOTFS_ASSET" | cut -f1), $(wc -l < /tmp/rootfs-entries.txt) entries" - - - name: Check out the embedded Linux engine - run: | - set -euo pipefail - git submodule update --init --depth 1 "$ENGINE_ROOT" - # The engine's own submodules are deliberately skipped: deps/libarchive - # and deps/libapps feed tools this app does not link, and deps/linux is - # only used with -Dkernel=linux, which XForge does not build. - git -C "$ENGINE_ROOT" log --oneline -1 - - name: Install the build tools run: | set -euo pipefail - # xcodegen generates the Xcode project from project.yml; meson/ninja - # build the engine. libarchive is not needed at all: the app unpacks the - # fakefs ZIP with ZIPFoundation instead of importing an archive. - # llvm/lld provide the aarch64-linux cross-compiler the guest VDSO - # needs (`clang -target aarch64-linux-gnu -fuse-ld=lld`) — Apple's - # clang cannot link that. shellcheck validates the pinned zsign - # installer, hence it is installed with the build tools. - required=(xcodegen meson ninja llvm lld shellcheck) - missing=() - for formula in "${required[@]}"; do - if ! brew list --versions "$formula" >/dev/null 2>&1; then - missing+=("$formula") - fi - done - if [ "${#missing[@]}" -gt 0 ]; then - brew install "${missing[@]}" + # xcodegen generates the Xcode project from project.yml. Nothing else is + # needed: the compiler comes from the toolchain bundle, and the app + # unpacks archives with ZIPFoundation. + if ! brew list --versions xcodegen >/dev/null 2>&1; then + brew install xcodegen else echo "Homebrew tools already installed" fi - - name: Identify engine cache inputs - id: engine-revision - run: echo "value=$(git -C "$ENGINE_ROOT" rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - name: Identify the iOS compiler toolchain id: toolchain-versions run: | { - echo "value=$(xcodebuild -version | tr '\n' ' ')-$(clang --version | head -n 1)-$(meson --version)" + echo "value=$(xcodebuild -version | tr '\n' ' ')-$(clang --version | head -n 1)" } >> "$GITHUB_OUTPUT" - - name: Cache the embedded Linux engine - id: engine-cache - uses: actions/cache@v4 - with: - path: ${{ env.ENGINE_LIB_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-engine-${{ hashFiles('.gitmodules', 'EmbeddedLinux/build-ish-core.sh', 'project.yml') }}-${{ steps.engine-revision.outputs.value }}-${{ steps.toolchain-versions.outputs.value }} - - - name: Build the embedded Linux engine (ish-arm64, iOS arm64) - if: steps.engine-cache.outputs.cache-hit != 'true' + # The build that proves the compiler really is in the app: the bundle is + # installed where the generated xcconfig points, so the bridge compiles with + # XFORGE_HAS_LLVM / XFORGE_HAS_SWIFT_FRONTEND and the app *links* the merged + # archive — the step that no other workflow exercises. + # + # From the published release, not a workflow artifact: the toolchain build + # takes ~90 minutes and its output only changes when an LLVM or Swift revision + # does, so making the app build depend on a toolchain run would put that cost + # on every app build for no reason. The tag names its contents, so "the newest + # `toolchain-*` release" is a well-defined thing to ask for. + - name: Install the native toolchain bundle + if: ${{ inputs.with_toolchain }} + env: + GH_TOKEN: ${{ github.token }} + REQUESTED_TAG: ${{ inputs.toolchain_tag }} run: | set -euo pipefail - bash EmbeddedLinux/build-ish-core.sh - # Prove the engine is really there before spending 20 minutes on an - # archive that would fail to link. - for lib in libish.a libish_emu.a libfakefs.a; do - test -f "$ENGINE_LIB_DIR/$lib" \ - || { echo "::error::the engine build did not produce $lib"; exit 1; } - done - # The arm64 backend must be the one that was built: app and engine - # share struct layouts, so the wrong guest arch links but misbehaves. - nm -gU "$ENGINE_LIB_DIR/libish_emu.a" > /tmp/engine-symbols.txt 2>/dev/null || true - if grep -qi 'guest_arm64\|gadget' /tmp/engine-symbols.txt; then - echo "arm64 gadget backend present in libish_emu.a" + ARCHIVE_NAME="XForgeNativeToolchain-arm64-ios.tar.gz" + tag="$REQUESTED_TAG" + + if [ "$tag" = "artifact" ]; then + # Escape hatch for testing a bundle that was just built and has not been + # published: the newest artifact of that name. + echo "installing the newest workflow artifact" + gh run download --repo "$GITHUB_REPOSITORY" \ + --name XForgeNativeToolchain-arm64-ios --dir .toolchain-artifact else - echo "::error::libish_emu.a has no arm64 gadget symbols — wrong guest_arch?" - exit 1 + if [ -z "$tag" ]; then + # `gh release list` is newest-first, so the first toolchain-* tag is + # the newest published bundle. + tag="$(gh release list --repo "$GITHUB_REPOSITORY" --limit 100 \ + --json tagName --jq '[.[] | select(.tagName | startswith("toolchain-"))][0].tagName')" + fi + if [ -z "$tag" ] || [ "$tag" = "null" ]; then + echo "::error::no published toolchain release found." + echo "Run the 'Native iOS Toolchain' workflow by hand (with_swift=true) and let it publish, or pass toolchain_tag." + exit 1 + fi + echo "installing the toolchain bundle from release $tag" + gh release download "$tag" --repo "$GITHUB_REPOSITORY" \ + --pattern "XForgeNativeToolchain-*.tar.gz" --dir .toolchain-artifact fi - - name: Validate the bundled guest tools + bash NativeToolchain/install-bundle.sh ".toolchain-artifact/$ARCHIVE_NAME" + cat Vendor/NativeToolchain/manifest.txt + cat Support/NativeToolchain.generated.xcconfig + + # The manifest is what the app bundle's compiler is made of, so say what it is + # before building with it: the SDK the archives and the bridge were compiled + # against, and the Swift revision the frontend speaks. Both have to line up + # with what the app will actually compile against on a device — the Darwin SDK + # it installs at runtime — and neither is visible anywhere else in this log. + - name: Check the toolchain bundle against this Xcode + if: ${{ inputs.with_toolchain }} run: | set -euo pipefail - # zsign is built inside Alpine on first use. These two files are the - # guest-side installer and the tiny patch that adds -Q/--password-file - # to the pinned upstream CLI (so a P12 password never appears in argv). - bash -n EmbeddedLinux/install-zsign.sh - shellcheck -S error EmbeddedLinux/install-zsign.sh - python3 -m py_compile EmbeddedLinux/patch-zsign-password-file.py - rm -rf EmbeddedLinux/__pycache__ - grep -q '^ZSIGN_COMMIT=614caa8d1ca949e260e5746144aa52d27a4b08d6$' \ - EmbeddedLinux/install-zsign.sh - grep -q '^ZSIGN_SHA256=a373dc5ddbf81ba5c435a48c14b4a2857506c3dde02ec62e0136b6968797384b$' \ - EmbeddedLinux/install-zsign.sh - grep -q 'install-zsign.sh' project.yml - grep -q 'patch-zsign-password-file.py' project.yml + MANIFEST="Vendor/NativeToolchain/manifest.txt" + get() { sed -n "s/^$1=//p" "$MANIFEST"; } + + echo "bundle: $(get bundle_tag)" + echo " built with Xcode $(get xcode), iPhoneOS SDK $(get ios_sdk)" + echo " LLVM commit $(get llvm_commit)" + + runner_sdk="$(xcrun --sdk iphoneos --show-sdk-version)" + if [ "$(get ios_sdk)" != "$runner_sdk" ]; then + echo "::warning::the bundle was built against iPhoneOS SDK $(get ios_sdk) but this runner has $runner_sdk" + fi + + if ! grep -q '^swift_frontend=1' "$MANIFEST"; then + echo "this bundle has no Swift frontend; Swift targets will report it missing" + exit 0 + fi + + # The frontend's own version matters, and the runner's is not it: this + # Swift is what *bootstrapped* the frontend's build, while the frontend + # itself comes from the ref below and has to match the Swift version + # inside the Darwin SDK the app installs on device. Printing both is the + # point — a mismatch between them is expected and is not a problem; a + # mismatch between the bundle and the Darwin SDK is, and this step cannot + # see that one. + echo " Swift frontend built from $(get swift_ref)" + echo " this runner's Swift (used to bootstrap it): $(xcrun swift --version 2>/dev/null | head -n 1)" - name: Prepare the native toolchain configuration run: | @@ -353,52 +220,22 @@ jobs: *arm64*) ;; *) echo "::error::the app binary is not arm64: $binary_info"; exit 1 ;; esac - # NOTE: do not try to prove the engine is linked by scanning the app - # binary for its symbols. Xcode's Release configuration strips, and on - # a stripped binary `nm` reports "no symbols" and finds nothing — so a - # symbol check fails for a perfectly good build. The engine's presence - # is proven where it can be observed instead: the engine libraries are - # asserted after they are built, and the guest is exercised by the - # smoke harness. What is worth checking here is that the app is - # self-consistent — the rootfs it needs is inside it. + # NOTE: do not try to prove the compiler is linked by scanning the app + # binary for LLVM symbols. Xcode's Release configuration strips, and on a + # stripped binary `nm` reports "no symbols" and finds nothing — so a + # symbol check fails for a perfectly good build. What the toolchain can do + # is checked where it is observable: the Toolchain screen's smoke test, on + # a device. size_mb=$(( $(stat -f%z "$APP_PATH/XForge") / 1048576 )) echo "app binary: ${binary_info#*: } (${size_mb} MB)" test "$size_mb" -gt 1 \ || { echo '::error::the app binary is implausibly small'; exit 1; } - # The rootfs must be inside the app, and must be the fakefs ZIP the app - # knows how to unpack. - ROOTFS="$APP_PATH/alpine-rootfs.zip" - test -f "$ROOTFS" \ - || { echo '::error::the app does not bundle alpine-rootfs.zip'; exit 1; } - unzip -Z1 "$ROOTFS" > /tmp/bundled-rootfs.txt - grep -q '^alpine-rootfs/meta\.db$' /tmp/bundled-rootfs.txt \ - || { echo '::error::the bundled rootfs has no meta.db'; exit 1; } - grep -q '^alpine-rootfs/data/bin/sh$' /tmp/bundled-rootfs.txt \ - || { echo '::error::the bundled rootfs has no /bin/sh'; exit 1; } - - # No engine build leftovers in the bundle. + # No build leftovers in the bundle. strays="$(find "$APP_PATH" \( -name '*.a' -o -name 'meson-*' -o -name 'build.ninja' \) -print -quit)" [ -z "$strays" ] \ || { echo "::error::build leftovers in the app bundle: $strays"; exit 1; } - # Exactly one rootfs, and it is the one the app reads. This is a real - # regression: the app used to bundle a tarball it imported at runtime, - # that file stayed committed after the switch to a fakefs ZIP, and the - # IPA shipped both — 4 MB of dead weight nothing ever opened. - rootfs_files="$(find "$APP_PATH" -maxdepth 1 \ - \( -name '*.tar.gz' -o -name '*.tar.xz' -o -name 'alpine-rootfs*.zip' \) -print)" - count="$(printf '%s\n' "$rootfs_files" | grep -c . || true)" - [ "$count" = "1" ] || { - echo "::error::expected exactly one rootfs in the bundle, found $count:" - printf ' %s\n' $rootfs_files - exit 1 - } - case "$rootfs_files" in - */alpine-rootfs.zip) ;; - *) echo "::error::the bundled rootfs is not alpine-rootfs.zip: $rootfs_files"; exit 1 ;; - esac - # Identity and the icon wiring that makes the app installable. PLIST="$APP_PATH/Info.plist" test -f "$PLIST" @@ -433,9 +270,6 @@ jobs: echo '### Built app' echo echo "- binary: ${binary_info#*: } (${size_mb} MB)" - echo "- bundled rootfs: $ROOTFS_TAG ($(du -h "$ROOTFS" | cut -f1))" - echo "- guest toolchain: $(unzip -p "$ROOTFS" alpine-rootfs/data/usr/local/share/xforge/rootfs-manifest.txt \ - | awk '/^[[:space:]]*toolchain:/ { sub(/^[[:space:]]*toolchain:[[:space:]]*/, ""); print; exit }')" echo "- version: $(get CFBundleShortVersionString)" } >> "$GITHUB_STEP_SUMMARY" @@ -454,11 +288,9 @@ jobs: ditto -c -k --keepParent dist/Payload "$IPA" unzip -t "$IPA" >/dev/null - # The rootfs is the difference between an app that works on first - # launch and one that has nothing to boot. unzip -Z1 "$IPA" > /tmp/ipa-contents.txt - grep -q 'Payload/XForge.app/alpine-rootfs\.zip$' /tmp/ipa-contents.txt \ - || { echo '::error::the IPA does not contain the rootfs'; exit 1; } + grep -q '^Payload/XForge\.app/XForge$' /tmp/ipa-contents.txt \ + || { echo '::error::the IPA does not contain the app executable'; exit 1; } # Exactly one IPA, and it is the version the app says it is. count="$(find dist -maxdepth 1 -name '*.ipa' | wc -l | tr -d ' ')" @@ -467,12 +299,11 @@ jobs: test -f "$IPA" || { echo "::error::$IPA was not produced"; exit 1; } echo "IPA: $IPA ($(du -h "$IPA" | cut -f1))" - # A provisioned root makes this a ~1.6 GB IPA, and GitHub refuses a - # release asset over 2 GiB. Failing here — with the number — is better - # than a publish step that errors out after the whole build. + # GitHub refuses a release asset over 2 GiB. The app is tens of MB, so + # this only catches a runaway bundle — before the publish step does. ipa_bytes="$(stat -f%z "$IPA")" - if [ "$ipa_bytes" -gt 1900000000 ]; then - echo "::error::the IPA is $((ipa_bytes / 1048576)) MB; GitHub refuses a release asset over 2 GiB" + if [ "$ipa_bytes" -gt 2000000000 ]; then + echo "::error::the IPA is $((ipa_bytes / 1048576)) MB, over GitHub's 2 GiB release limit" exit 1 fi echo "ipa=$IPA" >> "$GITHUB_ENV" @@ -500,7 +331,6 @@ jobs: test -f "$FINAL_PLIST" || { echo '::error::IPA app has no Info.plist'; exit 1; } test -x "$FINAL_APP/XForge" || { echo '::error::IPA app has no executable XForge binary'; exit 1; } test -f "$FINAL_APP/Assets.car" || { echo '::error::IPA app has no compiled asset catalog'; exit 1; } - test -f "$FINAL_APP/alpine-rootfs.zip" || { echo '::error::IPA app has no bundled rootfs'; exit 1; } get_final() { /usr/libexec/PlistBuddy -c "Print :$1" "$FINAL_PLIST"; } [ "$(get_final CFBundleIdentifier)" = "com.r0gueee.xforge" ] || { echo "::error::wrong bundle identifier in final IPA"; exit 1; } @@ -521,11 +351,8 @@ jobs: exit 1 } - # Check the rootfs inside the final IPA, not only the archive source. - unzip -t "$FINAL_APP/alpine-rootfs.zip" >/dev/null - unzip -Z1 "$FINAL_APP/alpine-rootfs.zip" > /tmp/final-rootfs-entries.txt - grep -qx 'alpine-rootfs/meta.db' /tmp/final-rootfs-entries.txt || { echo '::error::final IPA rootfs has no meta.db'; exit 1; } - grep -qx 'alpine-rootfs/data/bin/sh' /tmp/final-rootfs-entries.txt || { echo '::error::final IPA rootfs has no /bin/sh'; exit 1; } + # The app must be self-contained; a missing executable or Info.plist is + # what makes an IPA uninstallable, and both were checked above. # Reject common malformed packaging layouts. [ ! -d "$VERIFY_DIR/Payload/Payload" ] || { echo '::error::IPA contains a nested Payload/Payload directory'; exit 1; } diff --git a/.github/workflows/build-rootfs.yml b/.github/workflows/build-rootfs.yml deleted file mode 100644 index ac4e7a1..0000000 --- a/.github/workflows/build-rootfs.yml +++ /dev/null @@ -1,248 +0,0 @@ -name: Publish the Alpine rootfs - -# Builds the Alpine root filesystem that XForge bundles, and publishes it as a -# pinned release asset for `build-ipa.yml` to download. -# -# Run this by hand when the root changes (a new Alpine release, a change to the -# root's configuration, a new Swift or xtool release). It is deliberately NOT part -# of the IPA build: rebuilding the root on every IPA run is what used to make that -# job slow, and the root changes far less often than the app. -# -# What goes in it is `provision`: -# -# all (default) plain Alpine *plus the build toolchain* — xtool, swiftly, the -# Swift toolchain and the Darwin Swift SDK — installed in a chroot of the -# root being built, so the guest boots ready to build an iOS app. ~1.6 GB. -# none plain Alpine only; the guest installs the toolchain itself on demand. -# -# The output is a ZIP of an *already-converted* fakefs — a `data/` tree plus a -# `meta.db` — produced by the engine's own tools/fakefsify. The app only unzips -# it, so first launch costs an unzip rather than a multi-minute import. -# -# Builds on Linux rather than macOS on purpose: the conversion tool must run on -# the build machine (it is not cross-compiled), and provisioning runs the guest's -# own tools (arm64 Linux binaries) — both need an aarch64 Linux host, which the -# arm64 runner is natively. - -on: - workflow_dispatch: - inputs: - tag: - description: Release tag to publish under (e.g. rootfs-v5) - type: string - required: true - alpine_version: - description: Alpine release series - type: string - default: "3.21" - provision: - description: Bake the build toolchain (xtool, Swift, Darwin SDK) into the root - type: choice - options: - - all - - none - default: all - provision_sdk: - description: Include the Darwin Swift SDK (~400 MB download, ~1.3 GB installed) - type: boolean - default: true - darwin_sdk_tag: - description: Release the Darwin Swift SDK comes from - type: string - default: darwin-sdk-7 - default_shell: - description: Login shell root's console starts (must be installed by console_packages) - type: string - default: "/bin/bash" - console_packages: - description: Space-separated packages baked in for the console session - type: string - default: "bash coreutils less ncurses-terminfo" - -permissions: - contents: write - -jobs: - rootfs: - name: Build and publish the rootfs - runs-on: ubuntu-24.04-arm - # Provisioning downloads the Swift toolchain, unpacks the Darwin SDK and - # installs both inside a chroot, then converts ~5 GB of filesystem to fakefs. - # The unprovisioned root takes a couple of minutes; this takes the rest. - timeout-minutes: 240 - steps: - - name: Check out the repository - uses: actions/checkout@v4 - - - name: Make sure there is room for a provisioned root - if: inputs.provision == 'all' - run: | - set -euo pipefail - # A provisioned root is ~5 GB on disk before it is packed, and the - # conversion used to want another 11 GB: the tree, the tar of it, and - # the fakefs root it converts to. The builder no longer holds the tree - # and the fakefs root at the same time (the tree is deleted once it is - # packed), which brings the peak to ~7 GB — but it is worth both saying - # so and taking back the ~2 GB of preinstalled SDKs a runner has and - # this job does not use. - df -h / - sudo apt-get clean || true - for dir in /usr/share/dotnet /opt/ghc /usr/local/share/boost /usr/lib/jvm \ - /usr/local/lib/android /opt/hostedtoolcache; do - sudo rm -rf "$dir" || true - done - df -h / - - - name: Check out the engine (for tools/fakefsify) - run: | - set -euo pipefail - git submodule update --init --depth 1 Vendor/ish-arm64 - # fakefsify links libarchive, so its submodule is required too. - git -C Vendor/ish-arm64 submodule update --init --depth 1 deps/libarchive - - - name: Install the build tools - run: | - set -euo pipefail - # meson/ninja build fakefsify; libarchive-dev is what it links - # against (the engine's meson looks for it via pkg-config); zip packs - # the result; shellcheck guards the builder scripts; sqlite3 reads the - # meta.db of the packed root, which is where the published root's paths - # and modes are actually checked. - sudo apt-get update -qq - sudo apt-get install -y -qq meson ninja-build libarchive-dev zip unzip shellcheck sqlite3 - - - name: Check the rootfs builder - run: | - set -euo pipefail - bash -n EmbeddedLinux/build-rootfs.sh - bash -n EmbeddedLinux/verify-rootfs.sh - shellcheck -S error EmbeddedLinux/build-rootfs.sh EmbeddedLinux/verify-rootfs.sh - # The console program is /bin/sh (busybox ash), and it is installed into - # the guest rather than run here — parse it and lint it, but as POSIX sh, - # which is the dialect it is written in. install-toolchain.sh is in the - # same dialect: it is the guest's own installer, and this build runs it - # in a chroot of the root being built. - sh -n EmbeddedLinux/xforge-login - shellcheck -s sh -S error EmbeddedLinux/xforge-login - sh -n EmbeddedLinux/install-toolchain.sh - shellcheck -s sh -S error EmbeddedLinux/install-toolchain.sh - # The toolchain half of the verification: a packed fakefs root cannot be - # chrooted and run, so "the toolchain is really in the artifact" is a - # structural check — and a structural check that looks for the wrong - # path, or that believes the manifest, passes everything. This drives - # verify-rootfs.sh against roots that are complete except for one thing - # and asserts it fails, naming that thing. - bash -n Tools/test-verify-toolchain.sh - shellcheck -S error Tools/test-verify-toolchain.sh - bash Tools/test-verify-toolchain.sh - - - name: Build the rootfs - env: - XFORGE_ALPINE_VERSION: ${{ inputs.alpine_version }} - XFORGE_PROVISION: ${{ inputs.provision }} - XFORGE_PROVISION_SDK: ${{ inputs.provision_sdk && '1' || '0' }} - XFORGE_DARWIN_SDK_TAG: ${{ inputs.darwin_sdk_tag }} - XFORGE_DEFAULT_SHELL: ${{ inputs.default_shell }} - XFORGE_CONSOLE_PACKAGES: ${{ inputs.console_packages }} - run: | - set -euo pipefail - # As root: the guest's installer and apk run in a chroot of the tree - # being built (installing the glibc layer unpacks Ubuntu packages with - # ar/zstd and writes to /lib, /usr/lib and /opt/glibc; provisioning runs - # install-toolchain.sh itself), and a chroot needs mounts. Keep the work - # directory and the output separate so the resulting files are owned by - # the runner, not root. - sudo -E env "PATH=$PATH" \ - XFORGE_ALPINE_VERSION="$XFORGE_ALPINE_VERSION" \ - XFORGE_PROVISION="$XFORGE_PROVISION" \ - XFORGE_PROVISION_SDK="$XFORGE_PROVISION_SDK" \ - XFORGE_DARWIN_SDK_TAG="$XFORGE_DARWIN_SDK_TAG" \ - XFORGE_DEFAULT_SHELL="$XFORGE_DEFAULT_SHELL" \ - XFORGE_CONSOLE_PACKAGES="$XFORGE_CONSOLE_PACKAGES" \ - bash EmbeddedLinux/build-rootfs.sh "$PWD/dist/rootfs" - sudo chown -R "$(id -u):$(id -g)" dist - ls -la dist/rootfs - - - name: Verify the root boots what XForge expects - run: | - set -euo pipefail - # XForge boots pid 1 as /sbin/init and drives the guest's single console, - # where /etc/inittab respawns /sbin/xforge-login root — the script that - # execs root's login shell from /etc/passwd. A root that cannot do that - # has no usable Terminal, and nothing else would notice: it fails here - # rather than on a device, and on the *published artifact*, not on the - # tree it was made from. - # - # For a provisioned root the same run also checks the toolchain the - # manifest claims: xtool, a Swift toolchain with a stdlib, and the Darwin - # SDK at the path it recorded — including that the engine can *see* the - # SDK, i.e. that it is indexed in meta.db. A fakefs root cannot be - # chrooted, so those checks are structural; the "it compiles" check ran - # on the tree during the build, where install-toolchain.sh verify could - # run a real compilation. - # - # The checker unpacks the ZIP and runs the guest's own console program in - # a chroot, so "root's shell starts" is answered by the root itself. - # Needs root for the chroot (the runner is aarch64, same as the root). - sudo -E env "PATH=$PATH" \ - bash EmbeddedLinux/verify-rootfs.sh "$PWD/dist/rootfs/alpine-rootfs.zip" - - - name: Report the artifact - run: | - set -euo pipefail - ZIP=dist/rootfs/alpine-rootfs.zip - SIZE="$(stat -c %s "$ZIP")" - # GitHub refuses a release asset over 2 GiB, and the IPA that carries - # this root is a few MB larger than the root itself. A provisioned root - # is ~1.6 GB, so the margin exists — but it is not large, and finding out - # at the upload is worse than finding out here. - if [ "$SIZE" -gt 1900000000 ]; then - echo "::error::$ZIP is $((SIZE / 1048576)) MB; an IPA bundling it would not fit in a 2 GiB release asset" - exit 1 - fi - # `data/` because the archive is a fakefs: its tree is under - # alpine-rootfs/data/, and the manifest is a file in that tree. - unzip -p "$ZIP" alpine-rootfs/data/usr/local/share/xforge/rootfs-manifest.txt \ - > /tmp/rootfs-manifest.txt - { - echo "### Rootfs: $((SIZE / 1048576)) MB" - echo - echo '```' - cat /tmp/rootfs-manifest.txt - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - - name: Publish the release - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - TAG="${{ inputs.tag }}" - # Create the release if it is new; uploading with --clobber makes a - # re-run of the same tag replace the assets rather than fail. - gh release view "$TAG" --repo "${{ github.repository }}" >/dev/null 2>&1 || \ - gh release create "$TAG" \ - --repo "${{ github.repository }}" \ - --title "XForge Alpine rootfs (${{ inputs.alpine_version }})" \ - --notes "Alpine aarch64 fakefs for the XForge embedded Linux engine, built by EmbeddedLinux/build-rootfs.sh. Tools: ${{ inputs.provision == 'all' && format('xtool, swiftly, Swift{0}', inputs.provision_sdk && format(' and the {0} Darwin SDK', inputs.darwin_sdk_tag) || '') || 'plain Alpine only (the guest provisions itself)' }}. Pinned by build-ipa.yml via ROOTFS_TAG." - - gh release upload "$TAG" \ - --repo "${{ github.repository }}" \ - dist/rootfs/alpine-rootfs.zip \ - dist/rootfs/alpine-rootfs.zip.sha256 \ - --clobber - - - name: Summary - run: | - set -euo pipefail - { - echo "### Published \`${{ inputs.tag }}\`" - echo - echo "- asset: \`alpine-rootfs.zip\`" - echo "- size: $(du -h dist/rootfs/alpine-rootfs.zip | cut -f1)" - echo "- toolchain: ${{ inputs.provision == 'all' && 'provisioned' || 'none' }}" - echo "- darwin SDK: ${{ inputs.provision == 'all' && inputs.provision_sdk && inputs.darwin_sdk_tag || 'not included' }}" - echo "- sha256: $(cut -d' ' -f1 < dist/rootfs/alpine-rootfs.zip.sha256)" - echo - echo "Set \`ROOTFS_TAG\` in .github/workflows/build-ipa.yml to this tag." - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ios-build.yml b/.github/workflows/ios-build.yml deleted file mode 100644 index 4bf9b19..0000000 --- a/.github/workflows/ios-build.yml +++ /dev/null @@ -1,1357 +0,0 @@ -name: iOS Build -run-name: iOS Build ${{ inputs.build_id || github.ref_name }} - -on: - # Pushing a tag named ios-build/ starts the same job without the - # GitHub API, for environments where only git access is available. A push - # event runs the workflow file from the tagged commit, and the parameters - # come from builder.json there. The tag is deleted when the job ends. - push: - tags: - - 'ios-build/*' - workflow_dispatch: - inputs: - build_id: - description: 'Unique build identifier' - required: true - type: string - snapshot_ref: - description: 'Ref holding the working-tree snapshot to build (empty builds this branch)' - required: false - type: string - default: '' - ios_path: - description: 'Path to iOS project (e.g., "ios" for React Native)' - required: false - type: string - default: '.' - scheme: - description: 'Xcode scheme to build (auto-detected if empty)' - required: false - type: string - default: '' - use_signing: - description: 'Enable code signing (requires secrets to be configured)' - required: false - type: boolean - default: false - configuration: - description: 'Build configuration (Debug is faster, Release for production)' - required: false - type: string - default: 'Debug' - flutter_version: - description: 'Flutter version to use (e.g., 3.24.0). Empty means latest stable.' - required: false - type: string - default: '' - jdk_version: - description: 'JDK version for Kotlin Multiplatform Gradle builds (e.g., 17).' - required: false - type: string - default: '17' - # One input for the profile fields that are not inputs of their own, so - # the workflow stays under the ten-input limit of workflow_dispatch. - profile: - description: 'Selected builder.json profile as JSON: {"name": "...", "env": {...}, "distribution": "..."}' - required: false - type: string - default: '{}' - # The tenth and last input workflow_dispatch allows; combine before adding. - build_number: - description: 'CFBundleVersion to stamp on the build: N, or X.Y.Z+N to set the marketing version too. Empty leaves the project as is.' - required: false - type: string - default: '' - -jobs: - build: - runs-on: macos-latest - permissions: - contents: write # deletes the trigger tag - timeout-minutes: 90 # a cold React Native/Expo build on the free runner takes 30-60 minutes - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - # The dispatch always runs the workflow from the default branch, so the - # sources come from a snapshot ref pushed by the CLI instead. - - name: Checkout working-tree snapshot - if: inputs.snapshot_ref != '' - env: - SNAPSHOT_REF: ${{ inputs.snapshot_ref }} - run: | - set -e - git fetch --depth=1 origin "+$SNAPSHOT_REF:refs/heads/snapshot" - git checkout --force snapshot - echo "Building $(git rev-parse --short HEAD)" - - # Dispatch inputs arrive with their declared defaults; a tag push has none, - # so the values come from builder.json in the tagged commit (defaultProfile - # included) and the build id is the tag name after the prefix. - - name: Resolve parameters - id: params - env: - IN_BUILD_ID: ${{ inputs.build_id }} - IN_IOS_PATH: ${{ inputs.ios_path }} - IN_SCHEME: ${{ inputs.scheme }} - IN_USE_SIGNING: ${{ inputs.use_signing }} - IN_CONFIGURATION: ${{ inputs.configuration }} - IN_FLUTTER_VERSION: ${{ inputs.flutter_version }} - IN_JDK_VERSION: ${{ inputs.jdk_version }} - IN_PROFILE: ${{ inputs.profile }} - IN_BUILD_NUMBER: ${{ inputs.build_number }} - run: | - set -e - PROFILE="" - if [ "$GITHUB_EVENT_NAME" != "workflow_dispatch" ] && [ -f builder.json ]; then - PROFILE=$(jq -r '.defaultProfile // empty' builder.json) - if [ -n "$PROFILE" ] && [ "$(jq -r --arg p "$PROFILE" '.profiles[$p] != null' builder.json)" != "true" ]; then - echo "::error::defaultProfile \"$PROFILE\" is not defined under profiles in builder.json" - exit 1 - fi - fi - param() { # name dispatch-value jq-path default - local v="" - if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - v="$2" - elif [ -f builder.json ]; then - v=$(jq -r --arg p "$PROFILE" "$3 // empty" builder.json) - fi - v="${v:-$4}" - echo "$1=$v" >> "$GITHUB_OUTPUT" - echo "$1=$v" - } - # Profile fields override ios.*. A selected profile signs exactly - # when it has a distribution (ios.signing is the no-profile path), - # and its configuration defaults to Debug for development and - # Release for every other distribution. - param build_id "$IN_BUILD_ID" '""' "${GITHUB_REF_NAME##*/}" - param ios_path "$IN_IOS_PATH" '.ios.path' '.' - param scheme "$IN_SCHEME" '(.profiles[$p].scheme // .ios.scheme)' '' - param use_signing "$IN_USE_SIGNING" '(if $p != "" then ((.profiles[$p].distribution // "") != "") else .ios.signing end)' 'false' - param configuration "$IN_CONFIGURATION" '(.profiles[$p].configuration // (if $p != "" and (.profiles[$p].distribution // "") != "" then (if .profiles[$p].distribution == "development" then "Debug" else "Release" end) else .ios.configuration end))' 'Debug' - param flutter_version "$IN_FLUTTER_VERSION" '.flutter.version' '' - param jdk_version "$IN_JDK_VERSION" '.kmp.jdkVersion' '17' - # A build number only comes from `builder ios release`, never builder.json. - param build_number "$IN_BUILD_NUMBER" '""' '' - - # The rest of the profile: name, distribution (selects the signing - # set) and env, exported from here on so dependency installs see it. - if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - PROFILE_JSON="$IN_PROFILE" - elif [ -f builder.json ]; then - PROFILE_JSON=$(jq -c --arg p "$PROFILE" '{name: $p, env: (.profiles[$p].env // {}), distribution: (.profiles[$p].distribution // "")}' builder.json) - fi - [ -n "${PROFILE_JSON:-}" ] || PROFILE_JSON='{}' - PROFILE=$(jq -r '.name // ""' <<< "$PROFILE_JSON") - DISTRIBUTION=$(jq -r '.distribution // ""' <<< "$PROFILE_JSON") - # The CLI sends the canonical name; a tag build reads whatever builder.json says. - case "$DISTRIBUTION" in internal) DISTRIBUTION=ad-hoc ;; esac - # The suffix of the IOS_* secrets a distribution is signed with; no - # distribution means the legacy unsuffixed secrets. Same table in runner.sh. - signing_set() { - case "$1" in - '') echo '' ;; - development) echo DEVELOPMENT ;; - ad-hoc|internal) echo AD_HOC ;; - store) echo STORE ;; - enterprise) echo ENTERPRISE ;; - *) return 1 ;; - esac - } - if ! SIGNING_SET=$(signing_set "$DISTRIBUTION"); then - echo "::error::distribution \"$DISTRIBUTION\" must be development, ad-hoc (or internal), store or enterprise"; exit 1 - fi - echo "profile=$PROFILE" >> "$GITHUB_OUTPUT" - echo "distribution=$DISTRIBUTION" >> "$GITHUB_OUTPUT" - echo "signing_set=$SIGNING_SET" >> "$GITHUB_OUTPUT" - echo "profile=${PROFILE:-(none)}" - echo "distribution=$DISTRIBUTION" - echo "signing_set=$SIGNING_SET" - # Values are base64 per entry so newlines and quotes survive, and the - # GITHUB_ENV heredoc gets a random delimiter so no value line can end - # it early. Names are checked so a value cannot smuggle in a second variable. - if [ "$(jq -r '.env // {} | type' <<< "$PROFILE_JSON")" != "object" ]; then - echo "::error::the profile's env must be a JSON object of variable names to string values"; exit 1 - fi - while IFS=' ' read -r key encoded; do - name=$(printf '%s' "$key" | base64 --decode) - if ! [[ "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then - echo "::error::env name \"$name\" is not a valid environment variable name"; exit 1 - fi - delim="BUILDER_ENV_${RANDOM}${RANDOM}${RANDOM}" - { - echo "$name<<$delim" - printf '%s' "$encoded" | base64 --decode - echo - echo "$delim" - } >> "$GITHUB_ENV" - echo "env: $name" - done < <(jq -r '.env // {} | to_entries[] | "\(.key | @base64) \(.value | tostring | @base64)"' <<< "$PROFILE_JSON") - - - name: Setup Xcode - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: latest-stable - - - name: Restore DerivedData cache - uses: actions/cache/restore@v6 - id: cache-deriveddata - with: - path: DerivedData - key: deriveddata-${{ github.run_id }} - restore-keys: | - deriveddata- - - - name: Detect project type - id: detect - run: | - if [ -f "pubspec.yaml" ]; then - echo "type=flutter" >> $GITHUB_OUTPUT - elif [ -f "package.json" ] && grep -q '"expo"' package.json; then - echo "type=expo" >> $GITHUB_OUTPUT - elif [ -f "package.json" ] && grep -q '"react-native"' package.json; then - echo "type=reactnative" >> $GITHUB_OUTPUT - elif grep -rqlE 'kotlin\("multiplatform"\)|org\.jetbrains\.kotlin\.multiplatform|id\("org.jetbrains.kotlin.multiplatform"\)' --include='*.gradle.kts' --include='*.gradle' --include='libs.versions.toml' . 2>/dev/null; then - echo "type=kmp" >> $GITHUB_OUTPUT - else - echo "type=native" >> $GITHUB_OUTPUT - fi - - # A gitignored *.xcconfig next to a committed template is a common iOS - # convention. The snapshot then carries the template but not the config, - # and xcodebuild fails with "Unable to open base configuration reference - # file", which says nothing about which file is missing. - - name: Materialize missing xcconfig files - run: | - set -e - find . -path ./DerivedData -prune -o -type f \ - \( -name '*.xcconfig.template' -o -name '*.xcconfig.example' \ - -o -name '*.xcconfig.sample' -o -name '*.template.xcconfig' \ - -o -name '*.example.xcconfig' -o -name '*.sample.xcconfig' \) -print \ - | while IFS= read -r template; do - case "$template" in - *.xcconfig.template|*.xcconfig.example|*.xcconfig.sample) target="${template%.*}" ;; - *) target="$(dirname "$template")/$(basename "$template" | sed -E 's/\.(template|example|sample)\.xcconfig$/.xcconfig/')" ;; - esac - if [ ! -f "$target" ]; then - cp "$template" "$target" - echo "::warning::$target was missing (gitignored?); copied $template over it. Any placeholder values in it are what the build sees." - fi - done - - - name: Stage the guest rootfs (XForge) - # XForge bundles an Alpine rootfs (1.6 GB with the build toolchain in it) - # that is published as a release asset rather than committed, and the app - # target lists it as a resource — - # so it has to be on disk *before* XcodeGen walks the project, or the app - # is built without a Linux guest. The script reads the pinned tag from - # .github/workflows/build-ipa.yml and verifies the sha256. It is a no-op in - # a repository that has no such script. - run: | - set -e - # -f, not -x: a script that lost its executable bit must still run rather - # than make this step a silent no-op (which is how an app with no Linux - # guest got built and shipped to an artifact once already). - if [ -f EmbeddedLinux/fetch-rootfs.sh ]; then - bash EmbeddedLinux/fetch-rootfs.sh - fi - - - name: Generate Xcode project (XcodeGen) - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - run: | - set -e - # A managed Expo project has no iOS directory at all until the - # prebuild step below creates it. - if [ ! -d "$IOS_PATH" ]; then - echo "No '$IOS_PATH' directory yet — skipping." - exit 0 - fi - cd "$IOS_PATH" - - # Projects generated by XcodeGen usually keep the .xcodeproj out of git, - # so the checkout has a project.yml manifest but nothing to build. - MANIFEST="" - for f in project.yml project.yaml; do - if [ -f "$f" ] && grep -qE '^targets:' "$f"; then - MANIFEST="$f" - break - fi - done - - if [ -z "$MANIFEST" ]; then - echo "No XcodeGen manifest — skipping." - elif [ -n "$(find . -maxdepth 1 -name '*.xcodeproj' -print -quit)" ]; then - echo "Found a committed .xcodeproj — skipping generation." - else - echo "Found $MANIFEST and no .xcodeproj — generating with XcodeGen." - brew install xcodegen - xcodegen generate - fi - - # Whatever the previous step could not fill in is named here, instead of - # surfacing as xcodebuild's "Unable to open base configuration reference". - - name: Check base configuration files - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - run: | - set -e - if [ ! -d "$IOS_PATH" ]; then - echo "No '$IOS_PATH' directory yet — skipping." - exit 0 - fi - MISSING="" - for pbx in $(find "$IOS_PATH" -maxdepth 2 -name project.pbxproj -not -path '*/Pods.xcodeproj/*'); do - for id in $(grep -o 'baseConfigurationReference = [A-Za-z0-9]*' "$pbx" | awk '{print $3}' | sort -u); do - ref=$(grep -m1 "^[[:space:]]*$id .*isa = PBXFileReference" "$pbx" || true) - name=$(printf '%s' "$ref" | sed -n 's/.*[ \t]path = "\{0,1\}\([^";]*\)"\{0,1\};.*/\1/p') - [ -n "$name" ] || continue - base=$(basename "$name") - # CocoaPods writes its xcconfigs during pod install and Flutter - # writes Generated.xcconfig during pub get — both happen later in - # this job, so they are legitimately absent right now. - case "$name" in *Pods/*|*"Target Support Files"*) continue ;; esac - case "$base" in Pods-*|Generated.xcconfig) continue ;; esac - if [ -z "$(find . -name "$base" -print -quit)" ]; then - MISSING="$MISSING $base" - fi - done - done - if [ -n "$MISSING" ]; then - echo "::error::The project has base configuration files that are not in the repository:$MISSING. They are probably gitignored — commit them, or commit a .xcconfig.template next to each so the build can fill them in." - exit 1 - fi - - - name: Setup Flutter - if: steps.detect.outputs.type == 'flutter' - uses: subosito/flutter-action@v2 - with: - flutter-version: ${{ steps.params.outputs.flutter_version || '' }} - channel: stable - cache: true - - - name: Flutter pub get - if: steps.detect.outputs.type == 'flutter' - run: flutter pub get - - # setup-node cannot make `with:` keys conditional, so the version and the - # version file are both computed here; it ignores an empty version file. - - name: Resolve JS toolchain - id: js - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - run: | - set -e - # >>> js toolchain (keep identical across ios-build.yml, ios-share.yml, runner.sh) - # The package manager the project actually uses: the packageManager field - # first, then the lockfile, then npm. Guessing npm is not safe — `npm install` - # in a pnpm or yarn workspace fails with - # 'EUNSUPPORTEDPROTOCOL Unsupported URL Type "workspace:"'. - js_detect_manager() { - if [ -z "${JS_MANAGER:-}" ]; then - case "$(jq -r '.packageManager // empty' package.json 2>/dev/null || true)" in - pnpm|pnpm@*) JS_MANAGER=pnpm ;; - yarn|yarn@*) JS_MANAGER=yarn ;; - bun|bun@*) JS_MANAGER=bun ;; - npm|npm@*) JS_MANAGER=npm ;; - *) - if [ -f pnpm-lock.yaml ]; then JS_MANAGER=pnpm - elif [ -f yarn.lock ]; then JS_MANAGER=yarn - elif [ -f bun.lock ] || [ -f bun.lockb ]; then JS_MANAGER=bun - else JS_MANAGER=npm # package-lock.json, or no lockfile at all - fi - ;; - esac - fi - case "$JS_MANAGER" in - pnpm) JS_EXEC='pnpm exec' ;; - yarn) JS_EXEC=yarn ;; - bun) JS_EXEC=bunx ;; - *) JS_EXEC=npx ;; - esac - export JS_MANAGER JS_EXEC - echo "Package manager: $JS_MANAGER" - } - - # The Node version the project asks for: .nvmrc or .node-version, else the - # first comparator of engines.node, else 22. A leading "v" or range prefix - # is dropped: nvm, n and setup-node all take "20.11.1" and "24.x" as they are. - js_node_version() { - js_version="" - for js_file in .nvmrc .node-version; do - if [ -f "$js_file" ]; then - js_version=$(tr -d ' \t\r' < "$js_file" | head -n1) - break - fi - done - if [ -z "$js_version" ]; then - js_version=$(jq -r '.engines.node // empty' package.json 2>/dev/null || true) - js_version=$(printf '%s' "${js_version%%|*}" | awk '{print $1}') - fi - js_version=$(printf '%s' "$js_version" | sed -E 's/^(>=|<=|>|<|=|\^|~|v)+//') - case "$js_version" in - [0-9]*|lts/*) ;; - *) js_version=22 ;; - esac - printf '%s\n' "$js_version" - } - - # Put the manager on PATH first, then install — unless a cache already restored - # node_modules, in which case the manager is still wanted for `expo prebuild` - # and for the React Native build phases that shell out to it. - js_install() { - js_detect_manager - case "$JS_MANAGER" in - pnpm|yarn) - # corepack installs the version packageManager pins. Node 16.10-24 - # bundle it; Node 25+ and some provider images do not. - command -v corepack >/dev/null 2>&1 || npm install -g corepack || true - corepack enable || true - command -v "$JS_MANAGER" >/dev/null 2>&1 || npm install -g "$JS_MANAGER" - ;; - bun) - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - export BUN_INSTALL="${BUN_INSTALL:-$HOME/.bun}" - export PATH="$BUN_INSTALL/bin:$PATH" - # A step's PATH does not survive into the next GitHub Actions step. - if [ -n "${GITHUB_PATH:-}" ]; then echo "$BUN_INSTALL/bin" >> "$GITHUB_PATH"; fi - fi - ;; - esac - if [ "${JS_DEPS_CACHED:-}" = "true" ]; then - echo "node_modules restored from cache; skipping install" - return 0 - fi - case "$JS_MANAGER" in - pnpm) - if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; else pnpm install; fi - ;; - yarn) - if [ -f .yarnrc.yml ]; then yarn install --immutable - elif [ -f yarn.lock ]; then yarn install --frozen-lockfile - else yarn install - fi - ;; - bun) - if [ -f bun.lock ] || [ -f bun.lockb ]; then bun install --frozen-lockfile; else bun install; fi - ;; - *) - if [ -f package-lock.json ]; then npm ci; else npm install; fi - ;; - esac - } - # <<< js toolchain - js_detect_manager - NODE_VERSION_FILE="" - for f in .nvmrc .node-version; do - if [ -f "$f" ]; then NODE_VERSION_FILE="$f"; break; fi - done - NODE_VERSION="" - if [ -n "$NODE_VERSION_FILE" ]; then - echo "Node version: from $NODE_VERSION_FILE" - else - NODE_VERSION=$(js_node_version) - echo "Node version: $NODE_VERSION" - fi - { - echo "manager=$JS_MANAGER" - echo "exec=$JS_EXEC" - echo "node_version=$NODE_VERSION" - echo "node_version_file=$NODE_VERSION_FILE" - } >> "$GITHUB_OUTPUT" - - - name: Setup Node.js - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - uses: actions/setup-node@v7 - with: - node-version: ${{ steps.js.outputs.node_version }} - node-version-file: ${{ steps.js.outputs.node_version_file }} - - # pnpm links its store into node_modules/.pnpm, so caching node_modules - # stays correct for every manager. The key carries the manager name - # because the same lockfiles can be installed by different ones. - - name: Restore node_modules cache - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - uses: actions/cache@v6 - id: node-modules-cache - with: - path: node_modules - key: node-modules-${{ runner.os }}-${{ steps.js.outputs.manager }}-${{ hashFiles('package-lock.json', 'yarn.lock', 'pnpm-lock.yaml', 'bun.lock', 'bun.lockb') }} - restore-keys: | - node-modules-${{ runner.os }}-${{ steps.js.outputs.manager }}- - - # Runs even on a cache hit: the manager itself still has to be on PATH for - # the prebuild and build steps that follow. - - name: Install JS dependencies - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - env: - JS_MANAGER: ${{ steps.js.outputs.manager }} - JS_DEPS_CACHED: ${{ steps.node-modules-cache.outputs.cache-hit }} - run: | - set -e - # >>> js toolchain (keep identical across ios-build.yml, ios-share.yml, runner.sh) - # The package manager the project actually uses: the packageManager field - # first, then the lockfile, then npm. Guessing npm is not safe — `npm install` - # in a pnpm or yarn workspace fails with - # 'EUNSUPPORTEDPROTOCOL Unsupported URL Type "workspace:"'. - js_detect_manager() { - if [ -z "${JS_MANAGER:-}" ]; then - case "$(jq -r '.packageManager // empty' package.json 2>/dev/null || true)" in - pnpm|pnpm@*) JS_MANAGER=pnpm ;; - yarn|yarn@*) JS_MANAGER=yarn ;; - bun|bun@*) JS_MANAGER=bun ;; - npm|npm@*) JS_MANAGER=npm ;; - *) - if [ -f pnpm-lock.yaml ]; then JS_MANAGER=pnpm - elif [ -f yarn.lock ]; then JS_MANAGER=yarn - elif [ -f bun.lock ] || [ -f bun.lockb ]; then JS_MANAGER=bun - else JS_MANAGER=npm # package-lock.json, or no lockfile at all - fi - ;; - esac - fi - case "$JS_MANAGER" in - pnpm) JS_EXEC='pnpm exec' ;; - yarn) JS_EXEC=yarn ;; - bun) JS_EXEC=bunx ;; - *) JS_EXEC=npx ;; - esac - export JS_MANAGER JS_EXEC - echo "Package manager: $JS_MANAGER" - } - - # The Node version the project asks for: .nvmrc or .node-version, else the - # first comparator of engines.node, else 22. A leading "v" or range prefix - # is dropped: nvm, n and setup-node all take "20.11.1" and "24.x" as they are. - js_node_version() { - js_version="" - for js_file in .nvmrc .node-version; do - if [ -f "$js_file" ]; then - js_version=$(tr -d ' \t\r' < "$js_file" | head -n1) - break - fi - done - if [ -z "$js_version" ]; then - js_version=$(jq -r '.engines.node // empty' package.json 2>/dev/null || true) - js_version=$(printf '%s' "${js_version%%|*}" | awk '{print $1}') - fi - js_version=$(printf '%s' "$js_version" | sed -E 's/^(>=|<=|>|<|=|\^|~|v)+//') - case "$js_version" in - [0-9]*|lts/*) ;; - *) js_version=22 ;; - esac - printf '%s\n' "$js_version" - } - - # Put the manager on PATH first, then install — unless a cache already restored - # node_modules, in which case the manager is still wanted for `expo prebuild` - # and for the React Native build phases that shell out to it. - js_install() { - js_detect_manager - case "$JS_MANAGER" in - pnpm|yarn) - # corepack installs the version packageManager pins. Node 16.10-24 - # bundle it; Node 25+ and some provider images do not. - command -v corepack >/dev/null 2>&1 || npm install -g corepack || true - corepack enable || true - command -v "$JS_MANAGER" >/dev/null 2>&1 || npm install -g "$JS_MANAGER" - ;; - bun) - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - export BUN_INSTALL="${BUN_INSTALL:-$HOME/.bun}" - export PATH="$BUN_INSTALL/bin:$PATH" - # A step's PATH does not survive into the next GitHub Actions step. - if [ -n "${GITHUB_PATH:-}" ]; then echo "$BUN_INSTALL/bin" >> "$GITHUB_PATH"; fi - fi - ;; - esac - if [ "${JS_DEPS_CACHED:-}" = "true" ]; then - echo "node_modules restored from cache; skipping install" - return 0 - fi - case "$JS_MANAGER" in - pnpm) - if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; else pnpm install; fi - ;; - yarn) - if [ -f .yarnrc.yml ]; then yarn install --immutable - elif [ -f yarn.lock ]; then yarn install --frozen-lockfile - else yarn install - fi - ;; - bun) - if [ -f bun.lock ] || [ -f bun.lockb ]; then bun install --frozen-lockfile; else bun install; fi - ;; - *) - if [ -f package-lock.json ]; then npm ci; else npm install; fi - ;; - esac - } - # <<< js toolchain - js_install - - # A managed Expo project keeps no ios/ directory in git — `expo prebuild` - # generates it from app.json / app.config.js. It needs node_modules, so it - # runs here, and it must run before the Pods steps: the Podfile it writes - # is what `pod install` later reads. - - name: Expo prebuild - if: steps.detect.outputs.type == 'expo' - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - JS_EXEC: ${{ steps.js.outputs.exec }} - CI: '1' # prebuild answers its own prompts instead of hanging - run: | - set -e - if [ -n "$(find "$IOS_PATH" -maxdepth 1 \( -name '*.xcodeproj' -o -name '*.xcworkspace' \) -print -quit 2>/dev/null)" ]; then - echo "'$IOS_PATH' already holds an Xcode project (ejected Expo) — skipping prebuild." - exit 0 - fi - - # Without a bundle identifier prebuild asks for one interactively, - # which on a runner means waiting for the job timeout. - BUNDLE_ID=$(${JS_EXEC:-npx} expo config --type public --json 2>/dev/null | jq -r '.ios.bundleIdentifier // empty' 2>/dev/null || true) - if [ -z "$BUNDLE_ID" ]; then - for f in app.json app.config.json; do - [ -f "$f" ] || continue - BUNDLE_ID=$(jq -r '(.expo.ios.bundleIdentifier // .ios.bundleIdentifier) // empty' "$f" 2>/dev/null || true) - if [ -n "$BUNDLE_ID" ]; then break; fi - done - fi - if [ -z "$BUNDLE_ID" ]; then - echo "::error::This managed Expo project has no ios.bundleIdentifier. Set it in app.json (expo.ios.bundleIdentifier) or app.config.js, commit that, and build again — 'expo prebuild' cannot run unattended without it." - exit 1 - fi - echo "Prebuilding $BUNDLE_ID into '$IOS_PATH'..." - - ${JS_EXEC:-npx} expo prebuild --platform ios --no-install - if [ -z "$(find "$IOS_PATH" -maxdepth 1 -name '*.xcodeproj' -print -quit 2>/dev/null)" ]; then - echo "::error::'expo prebuild' produced no Xcode project in '$IOS_PATH'. If the project generates it elsewhere, set ios.path in builder.json." - exit 1 - fi - - - name: Setup JDK - if: steps.detect.outputs.type == 'kmp' - uses: actions/setup-java@v6 - with: - distribution: temurin - java-version: ${{ steps.params.outputs.jdk_version || '17' }} - - - name: Setup Gradle - if: steps.detect.outputs.type == 'kmp' - uses: gradle/actions/setup-gradle@v6 - - # A managed Expo project has no Podfile.lock before the prebuild above, so - # hashFiles returns an empty string and the key degrades to the prefix. - # That still restores and saves a Pods directory, and the `pod install` in - # the build step syncs it, so an empty hash is not an error here. - - name: Restore Pods cache - uses: actions/cache@v6 - id: pods-cache - with: - path: | - ${{ steps.params.outputs.ios_path }}/Pods - ~/.cocoapods/repos - key: pods-${{ runner.os }}-${{ hashFiles(format('{0}/Podfile.lock', steps.params.outputs.ios_path)) }} - restore-keys: | - pods-${{ runner.os }}- - - # One set of secrets per distribution, IOS_*_; the unsuffixed names - # serve builds without a profile. A secret that does not exist arrives empty. - - name: Install certificate and provisioning profile - if: steps.params.outputs.use_signing == 'true' - env: - SIGNING_SET: ${{ steps.params.outputs.signing_set }} - DISTRIBUTION: ${{ steps.params.outputs.distribution }} - CONFIGURATION: ${{ steps.params.outputs.configuration }} - IOS_CERTIFICATE: ${{ secrets.IOS_CERTIFICATE }} - IOS_CERTIFICATE_PASSWORD: ${{ secrets.IOS_CERTIFICATE_PASSWORD }} - IOS_PROVISIONING_PROFILE: ${{ secrets.IOS_PROVISIONING_PROFILE }} - IOS_EXTENSION_PROFILES: ${{ secrets.IOS_EXTENSION_PROFILES }} - IOS_CERTIFICATE_DEVELOPMENT: ${{ secrets.IOS_CERTIFICATE_DEVELOPMENT }} - IOS_CERTIFICATE_PASSWORD_DEVELOPMENT: ${{ secrets.IOS_CERTIFICATE_PASSWORD_DEVELOPMENT }} - IOS_PROVISIONING_PROFILE_DEVELOPMENT: ${{ secrets.IOS_PROVISIONING_PROFILE_DEVELOPMENT }} - IOS_EXTENSION_PROFILES_DEVELOPMENT: ${{ secrets.IOS_EXTENSION_PROFILES_DEVELOPMENT }} - IOS_CERTIFICATE_AD_HOC: ${{ secrets.IOS_CERTIFICATE_AD_HOC }} - IOS_CERTIFICATE_PASSWORD_AD_HOC: ${{ secrets.IOS_CERTIFICATE_PASSWORD_AD_HOC }} - IOS_PROVISIONING_PROFILE_AD_HOC: ${{ secrets.IOS_PROVISIONING_PROFILE_AD_HOC }} - IOS_EXTENSION_PROFILES_AD_HOC: ${{ secrets.IOS_EXTENSION_PROFILES_AD_HOC }} - IOS_CERTIFICATE_STORE: ${{ secrets.IOS_CERTIFICATE_STORE }} - IOS_CERTIFICATE_PASSWORD_STORE: ${{ secrets.IOS_CERTIFICATE_PASSWORD_STORE }} - IOS_PROVISIONING_PROFILE_STORE: ${{ secrets.IOS_PROVISIONING_PROFILE_STORE }} - IOS_EXTENSION_PROFILES_STORE: ${{ secrets.IOS_EXTENSION_PROFILES_STORE }} - IOS_CERTIFICATE_ENTERPRISE: ${{ secrets.IOS_CERTIFICATE_ENTERPRISE }} - IOS_CERTIFICATE_PASSWORD_ENTERPRISE: ${{ secrets.IOS_CERTIFICATE_PASSWORD_ENTERPRISE }} - IOS_PROVISIONING_PROFILE_ENTERPRISE: ${{ secrets.IOS_PROVISIONING_PROFILE_ENTERPRISE }} - IOS_EXTENSION_PROFILES_ENTERPRISE: ${{ secrets.IOS_EXTENSION_PROFILES_ENTERPRISE }} - run: | - set -e - fail() { echo "::error::$*" >&2; exit 1; } - - # Picks the set's IOS_*_ secrets into the unsuffixed names, or with - # no distribution takes the unsuffixed ones as they are (password - # optional). A suffixed set needs all three, since builder signing setup - # always writes a password. - select_signing_set() { - if [ -z "$SIGNING_SET" ]; then - if [ -z "${IOS_CERTIFICATE:-}" ] || [ -z "${IOS_PROVISIONING_PROFILE:-}" ]; then - fail "No signing secrets: ios.signing needs IOS_CERTIFICATE, IOS_CERTIFICATE_PASSWORD and IOS_PROVISIONING_PROFILE; a build profile with a distribution reads its own IOS_*_ secrets instead (builder signing setup --distribution writes them)." - fi - IOS_CERTIFICATE_PASSWORD="${IOS_CERTIFICATE_PASSWORD:-}" - IOS_EXTENSION_PROFILES="${IOS_EXTENSION_PROFILES:-}" - SIGNING_SET_USED=legacy - else - local cert="IOS_CERTIFICATE_$SIGNING_SET" pass="IOS_CERTIFICATE_PASSWORD_$SIGNING_SET" prof="IOS_PROVISIONING_PROFILE_$SIGNING_SET" ext="IOS_EXTENSION_PROFILES_$SIGNING_SET" - local missing="" name - for name in "$cert" "$pass" "$prof"; do - [ -n "${!name:-}" ] || missing="${missing:+$missing, }$name" - done - [ -z "$missing" ] || fail "Signing set $SIGNING_SET for distribution $DISTRIBUTION is missing $missing. Run builder signing setup --distribution $DISTRIBUTION (builder ios build does it too when an App Store Connect key is configured)." - IOS_CERTIFICATE="${!cert}" - IOS_CERTIFICATE_PASSWORD="${!pass}" - IOS_PROVISIONING_PROFILE="${!prof}" - IOS_EXTENSION_PROFILES="${!ext:-}" - SIGNING_SET_USED="$SIGNING_SET" - fi - echo "Signing set: $SIGNING_SET_USED" - } - - # Decodes the set's extension profiles (IOS_EXTENSION_PROFILES, a JSON object - # of extension bundle id to base64 .mobileprovision) into $1, installs them - # next to the app's, and prints a JSON object of bundle id to profile name - # for apply_signing_to_app_target and write_export_options. - install_extension_profiles() { - local dir="$1" json="${IOS_EXTENSION_PROFILES:-}" i=0 id encoded path uuid name map='{}' - [ -n "$json" ] || json='{}' - if [ "$(jq -r 'type' <<< "$json" 2>/dev/null)" != "object" ]; then - fail "IOS_EXTENSION_PROFILES${SIGNING_SET:+_$SIGNING_SET} must be a JSON object of extension bundle id to base64 .mobileprovision, as builder signing setup writes it." - fi - mkdir -p "$dir" "$HOME/Library/MobileDevice/Provisioning Profiles" - while IFS=' ' read -r id encoded; do - id=$(printf '%s' "$id" | base64 --decode) - i=$((i + 1)) - path="$dir/extension-$i.mobileprovision" - printf '%s' "$encoded" | base64 --decode > "$path" - security cms -D -i "$path" > "$path.plist" - uuid=$(plutil -extract UUID raw -o - "$path.plist") - name=$(plutil -extract Name raw -o - "$path.plist") - cp "$path" "$HOME/Library/MobileDevice/Provisioning Profiles/$uuid.mobileprovision" - echo "$uuid" >> "$dir/installed" - map=$(jq -c --arg id "$id" --arg name "$name" '. + {($id): $name}' <<< "$map") - echo " extension: $id -> '$name' ($uuid)" >&2 - done < <(jq -r 'to_entries[] | "\(.key | @base64) \(.value)"' <<< "$json") - printf '%s' "$map" - } - - # The profile in the set must be the type the build profile asked for, - # or the IPA would not be what the profile promised. Compared - # canonically: the export method says app-store for store, and a tag - # build may say internal for ad-hoc. - check_signing_set() { - [ "$SIGNING_SET_USED" != legacy ] || return 0 - local have="$1" want="$DISTRIBUTION" - case "$have" in app-store) have=store ;; esac - case "$want" in internal) want=ad-hoc ;; esac - if [ "$have" != "$want" ]; then - fail "IOS_PROVISIONING_PROFILE_$SIGNING_SET holds a $1 provisioning profile, but the build profile asks for distribution $want. Run builder signing setup --distribution $want, or set the profile's distribution to $have." - fi - } - - # The export method has to match the profile, or -exportArchive fails - # and App Store Connect rejects the IPA. These legacy names are the only - # ones older Xcodes (pinned or self-hosted runners) understand, and - # Xcode 16 still takes them. - detect_export_method() { - if [ "$(plutil -extract ProvisionsAllDevices raw -o - "$1" 2>/dev/null)" = "true" ]; then - echo enterprise - elif plutil -extract ProvisionedDevices xml1 -o /dev/null "$1" >/dev/null 2>&1; then - if [ "$(plutil -extract Entitlements.get-task-allow raw -o - "$1" 2>/dev/null)" = "true" ]; then - echo development - else - echo ad-hoc - fi - else - echo app-store - fi - } - - # The certificate names that can sign for a profile of this type, the - # current one first: keychains still hold pre-2021 iPhone Developer / - # iPhone Distribution certificates that sign the same profiles. - signing_identities() { - case "$1" in - development) printf '%s\n' "Apple Development" "iPhone Developer" ;; - ad-hoc|app-store|enterprise) printf '%s\n' "Apple Distribution" "iPhone Distribution" ;; - *) return 1 ;; - esac - } - - # The identity to archive with: the first of those names in security - # find-identity's output ($2). Without an explicit CODE_SIGN_IDENTITY the - # archive keeps the project's default, which Xcode refuses to pair with a - # distribution profile ("No signing certificate iOS Development found"). - signing_identity() { - local names name - names=$(signing_identities "$1") || return 1 - while IFS= read -r name; do - case "$2" in *"$name: "*) echo "$name"; return 0 ;; esac - done <<< "$names" - return 2 - } - - select_signing_set - - # Read the profile first: its type is checked against the build - # profile before any keychain exists or the certificate is imported. - PROFILE_PATH=$RUNNER_TEMP/profile.mobileprovision - echo "$IOS_PROVISIONING_PROFILE" | base64 --decode > "$PROFILE_PATH" - PROFILE_PLIST=$RUNNER_TEMP/profile.plist - security cms -D -i "$PROFILE_PATH" > "$PROFILE_PLIST" - PROFILE_UUID=$(plutil -extract UUID raw -o - "$PROFILE_PLIST") - - # xcodebuild will not pick a team on its own on a CI machine, so read - # the team and profile name out of the profile and pass them to the - # build as manual signing settings. - TEAM_ID=$(plutil -extract TeamIdentifier.0 raw -o - "$PROFILE_PLIST") - PROFILE_NAME=$(plutil -extract Name raw -o - "$PROFILE_PLIST") - APP_ID=$(plutil -extract Entitlements.application-identifier raw -o - "$PROFILE_PLIST") - PROFILE_BUNDLE_ID=${APP_ID#"$TEAM_ID".} - - EXPORT_METHOD=$(detect_export_method "$PROFILE_PLIST") - check_signing_set "$EXPORT_METHOD" - - # A Debug archive carries get-task-allow=true, which no distribution - # profile grants: the export fails, or an IPA that App Store Connect - # rejects comes out. Say so now instead of after the whole build. - if [ "$EXPORT_METHOD" != "development" ] && [ "$CONFIGURATION" = "Debug" ]; then - echo "::error::The provisioning profile is an $EXPORT_METHOD profile, but the build configuration is Debug. A Debug build is signed with get-task-allow, which distribution profiles do not allow and App Store Connect rejects. Drop the profile's \"configuration\" (a distribution build defaults to Release) or set \"configuration\": \"Release\", or build with a development profile." - exit 1 - fi - - # Create temporary keychain - KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db - KEYCHAIN_PASSWORD=$(openssl rand -base64 32) - - security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH" - security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH" - security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH" - - # Import certificate - CERTIFICATE_PATH=$RUNNER_TEMP/certificate.p12 - echo "$IOS_CERTIFICATE" | base64 --decode > "$CERTIFICATE_PATH" - security import "$CERTIFICATE_PATH" -P "$IOS_CERTIFICATE_PASSWORD" -A -t cert -f pkcs12 -k "$KEYCHAIN_PATH" - security set-key-partition-list -S apple-tool:,apple: -k "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH" - security list-keychain -d user -s "$KEYCHAIN_PATH" - - # The certificate has to be the kind the profile asks for, under - # whichever of its names it carries. Say so here instead of letting - # xcodebuild discover it after the whole archive. - IDENTITIES=$(security find-identity -v -p codesigning "$KEYCHAIN_PATH") - echo "$IDENTITIES" - CODE_SIGN_IDENTITY=$(signing_identity "$EXPORT_METHOD" "$IDENTITIES") || fail "IOS_CERTIFICATE${SIGNING_SET:+_$SIGNING_SET} holds no $(signing_identities "$EXPORT_METHOD" | paste -sd '/' -) certificate, which an $EXPORT_METHOD profile must be signed with. Run builder signing setup --distribution ${DISTRIBUTION:-} to issue the right one." - - # Install the provisioning profiles: the app's, then its extensions' - mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles - cp "$PROFILE_PATH" ~/Library/MobileDevice/Provisioning\ Profiles/"$PROFILE_UUID".mobileprovision - EXTENSION_PROFILES=$(install_extension_profiles "$RUNNER_TEMP/extensions") - - echo "KEYCHAIN_PATH=$KEYCHAIN_PATH" >> $GITHUB_ENV - echo "DEVELOPMENT_TEAM=$TEAM_ID" >> $GITHUB_ENV - echo "PROVISIONING_PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV - echo "PROFILE_BUNDLE_ID=$PROFILE_BUNDLE_ID" >> $GITHUB_ENV - echo "EXTENSION_PROFILES=$EXTENSION_PROFILES" >> $GITHUB_ENV - echo "EXPORT_METHOD=$EXPORT_METHOD" >> $GITHUB_ENV - echo "CODE_SIGN_IDENTITY=$CODE_SIGN_IDENTITY" >> $GITHUB_ENV - echo "SIGNING_SET_USED=$SIGNING_SET_USED" >> $GITHUB_ENV - echo "Certificate and provisioning profile installed" - echo " team: $TEAM_ID" - echo " profile: $PROFILE_NAME ($PROFILE_UUID)" - echo " app id: $PROFILE_BUNDLE_ID" - echo " set: $SIGNING_SET_USED" - echo " export: $EXPORT_METHOD" - echo " identity: $CODE_SIGN_IDENTITY" - echo "The build writes these into the app target's build settings; its PRODUCT_BUNDLE_IDENTIFIER must match the app id above. Extension targets get the profiles listed above by bundle id." - - - name: Build IPA - env: - BUILD_ID: ${{ steps.params.outputs.build_id }} - IOS_PATH: ${{ steps.params.outputs.ios_path }} - SCHEME: ${{ steps.params.outputs.scheme }} - PROJECT_TYPE: ${{ steps.detect.outputs.type }} - USE_SIGNING: ${{ steps.params.outputs.use_signing }} - CONFIGURATION: ${{ steps.params.outputs.configuration }} - BUILD_NUMBER: ${{ steps.params.outputs.build_number }} - DISTRIBUTION: ${{ steps.params.outputs.distribution }} - run: | - set -e - fail() { echo "::error::$*" >&2; exit 1; } - - # Writes the manual signing settings (team, profile and identity from the - # environment) into the application targets of ./*.xcodeproj that the - # profile's app id ($1, "*" wildcards) covers, and into every extension target - # with the EXTENSION_PROFILES entry (bundle id to name) covering its bundle - # id, as an XML plist Xcode reads. On the xcodebuild command line they would - # apply to every target, and a CocoaPods framework target refuses a profile. - apply_signing_to_app_target() { - local projects=(*.xcodeproj) out - [ -d "${projects[0]}" ] || fail "No .xcodeproj in $PWD to apply the signing settings to" - out=$(python3 - "$1" "${projects[@]}" 2>&1 <<'PY' - import json, os, subprocess, sys - app_id, projects = sys.argv[1], sys.argv[2:] - settings = {'CODE_SIGN_STYLE': 'Manual', 'DEVELOPMENT_TEAM': os.environ['DEVELOPMENT_TEAM'], - 'PROVISIONING_PROFILE_SPECIFIER': os.environ['PROVISIONING_PROFILE_NAME'], - 'CODE_SIGN_IDENTITY': os.environ['CODE_SIGN_IDENTITY']} - extension_profiles = json.loads(os.environ.get('EXTENSION_PROFILES') or '{}') - # The same list as ExtensionProductTypes in internal/xcodeproj. - extension_types = {'com.apple.product-type.app-extension', 'com.apple.product-type.app-extension.messages', - 'com.apple.product-type.extensionkit-extension', 'com.apple.product-type.application.watchapp2', - 'com.apple.product-type.watchkit2-extension', 'com.apple.product-type.application.on-demand-install-capable'} - - def covers(pattern, bundle_id): - if pattern.endswith('*'): - return bundle_id.startswith(pattern[:-1]) - return bundle_id == pattern - - apps, extensions, plists = [], [], {} - for project in projects: - path = os.path.join(project, 'project.pbxproj') - plists[project] = json.loads(subprocess.check_output(['plutil', '-convert', 'json', '-o', '-', path])) - objects = plists[project]['objects'] - for target in objects.values(): - kind = target.get('productType') - if target.get('isa') != 'PBXNativeTarget' or (kind != 'com.apple.product-type.application' and kind not in extension_types): - continue - configs = [objects[c] for c in objects[target['buildConfigurationList']]['buildConfigurations']] - ids = sorted({c.setdefault('buildSettings', {}).get('PRODUCT_BUNDLE_IDENTIFIER', '') for c in configs}) - (apps if kind == 'com.apple.product-type.application' else extensions).append((project, target['name'], configs, ids)) - if not apps: - sys.exit('No application target in %s to apply the signing settings to' % ', '.join(projects)) - chosen = apps if len(apps) == 1 else [a for a in apps if any(covers(app_id, i) for i in a[3])] - if not chosen: - found = '; '.join('%s in %s (%s)' % (name, project, ', '.join(i or '?' for i in ids)) for project, name, _, ids in apps) - sys.exit('No application target has the PRODUCT_BUNDLE_IDENTIFIER the provisioning profile covers (%s): %s' % (app_id, found)) - chosen = [a + (settings['PROVISIONING_PROFILE_SPECIFIER'],) for a in chosen] - # An extension is signed with the most specific profile covering it; the app's never does. - missing = [] - for project, name, configs, ids in extensions: - names = [extension_profiles[p] for p in sorted(extension_profiles, key=len, reverse=True) if any(covers(p, i) for i in ids)] - if names: - chosen.append((project, name, configs, ids, names[0])) - else: - missing.append('%s in %s (%s)' % (name, project, ', '.join(i or '?' for i in ids))) - if missing: - sys.exit('No provisioning profile for extension target %s. Add each bundle id to ios.extensions in builder.json and run builder signing setup --distribution %s.' % ('; '.join(missing), os.environ.get('DISTRIBUTION') or '')) - for project, name, configs, _, profile in chosen: - for config in configs: - build = config['buildSettings'] - # A conditional setting (CODE_SIGN_IDENTITY[sdk=iphoneos*]) would win over the plain one. - for key in [k for k in build if k.split('[')[0] in settings]: - del build[key] - build.update(settings) - build['PROVISIONING_PROFILE_SPECIFIER'] = profile - print('Signing settings applied to target %s in %s: %s (profile %s)' % (name, project, ', '.join(c['name'] for c in configs), profile)) - for project in sorted({a[0] for a in chosen}): - subprocess.run(['plutil', '-convert', 'xml1', '-o', os.path.join(project, 'project.pbxproj'), '-'], - input=json.dumps(plists[project]).encode(), check=True) - PY - ) || fail "$out" - echo "$out" - } - - # BUILD_NUMBER is "N" or "X.Y.Z+N" (pubspec convention); sets build_number, - # build_name and version_settings for xcodebuild. Given a target and scheme - # it also rewrites an Info.plist that hardcodes CFBundleVersion, which - # CURRENT_PROJECT_VERSION never reaches; identical in ios-build.yml and runner.sh. - apply_build_number() { - build_number="" build_name="" version_settings="" - [ -n "${BUILD_NUMBER:-}" ] || return 0 - if ! [[ "$BUILD_NUMBER" =~ ^([0-9]+(\.[0-9]+)*\+)?[0-9]+(\.[0-9]+)*$ ]]; then - echo "BUILD_NUMBER must be N or X.Y.Z+N, got '$BUILD_NUMBER'" >&2 - return 1 - fi - build_number="${BUILD_NUMBER##*+}" - case "$BUILD_NUMBER" in *+*) build_name="${BUILD_NUMBER%+*}" ;; esac - version_settings="CURRENT_PROJECT_VERSION=$build_number${build_name:+ MARKETING_VERSION=$build_name}" - echo "Build number: $build_number${build_name:+ (version $build_name)}" - [ $# -gt 0 ] || return 0 - plist=$(xcodebuild "$@" -showBuildSettings -json 2>/dev/null \ - | jq -r 'map(.buildSettings) - | map(select(.PRODUCT_TYPE == "com.apple.product-type.application" and .INFOPLIST_FILE != null and .INFOPLIST_FILE != "")) - | map(.SRCROOT + "/" + .INFOPLIST_FILE) | first // empty' || true) - if [ -z "$plist" ] || [ ! -f "$plist" ]; then - echo "No Info.plist file in the app target; CFBundleVersion comes from CURRENT_PROJECT_VERSION" - return 0 - fi - current=$(plutil -extract CFBundleVersion raw -o - "$plist" 2>/dev/null || true) - case "$current" in - '$(CURRENT_PROJECT_VERSION)'|'${CURRENT_PROJECT_VERSION}'|'$(FLUTTER_BUILD_NUMBER)'|'${FLUTTER_BUILD_NUMBER}') - echo "$plist reads CFBundleVersion from $current" ;; - *) - echo "$plist hardcodes CFBundleVersion '$current'; setting $build_number in the plist" - plutil -replace CFBundleVersion -string "$build_number" "$plist" ;; - esac - [ -n "$build_name" ] || return 0 - current=$(plutil -extract CFBundleShortVersionString raw -o - "$plist" 2>/dev/null || true) - case "$current" in - '$(MARKETING_VERSION)'|'${MARKETING_VERSION}'|'$(FLUTTER_BUILD_NAME)'|'${FLUTTER_BUILD_NAME}') ;; - *) - echo "$plist hardcodes CFBundleShortVersionString '$current'; setting $build_name in the plist" - plutil -replace CFBundleShortVersionString -string "$build_name" "$plist" ;; - esac - } - - # Writes the export options ($1) with the same manual signing as the archive: - # the app's real bundle id (APP_BUNDLE_ID) maps to its profile and every - # extension keeps its own from EXTENSION_PROFILES. - write_export_options() { - python3 - "$1" <<'PY' - import json, os, plistlib, sys - options = {'method': os.environ['EXPORT_METHOD'], 'signingStyle': 'manual', - 'teamID': os.environ['DEVELOPMENT_TEAM'], - 'provisioningProfiles': {os.environ['APP_BUNDLE_ID']: os.environ['PROVISIONING_PROFILE_NAME']}} - options['provisioningProfiles'].update(json.loads(os.environ.get('EXTENSION_PROFILES') or '{}')) - # Distribution exports keep the version numbers the archive was built with; - # Xcode would otherwise renumber the build on export. - if options['method'] != 'development': - options['manageAppVersionAndBuildNumber'] = False - with open(sys.argv[1], 'wb') as out: - plistlib.dump(options, out) - PY - } - - # Navigate to iOS project - cd "$IOS_PATH" - - # Find workspace or project - WORKSPACE=$(find . -maxdepth 1 -name "*.xcworkspace" | head -1) - PROJECT=$(find . -maxdepth 1 -name "*.xcodeproj" | head -1) - - # Install pods if Podfile exists (always run to ensure sync with Podfile.lock) - if [ -f "Podfile" ]; then - echo "Running pod install..." - pod install - WORKSPACE=$(find . -maxdepth 1 -name "*.xcworkspace" | head -1) - fi - - if [ -n "$WORKSPACE" ]; then - TARGET_FLAG=-workspace - TARGET_PATH=$WORKSPACE - else - TARGET_FLAG=-project - TARGET_PATH=$PROJECT - fi - - # Determine scheme (only needed for native projects) - if [ -z "$SCHEME" ] && [ "$PROJECT_TYPE" = "native" ]; then - if [ -n "$WORKSPACE" ]; then - BASE=$(basename "$WORKSPACE" .xcworkspace) - else - BASE=$(basename "$PROJECT" .xcodeproj) - fi - - SCHEMES=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" -list -json 2>/dev/null | jq -r '(.workspace // .project).schemes[]?' || true) - if [ -z "$SCHEMES" ]; then - SCHEMES=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" -list 2>/dev/null | sed -n '/Schemes:/,$p' | tail -n +2 | sed 's/^[[:space:]]*//' | sed '/^$/d' || true) - fi - echo "Schemes:" - printf '%s\n' "$SCHEMES" | sed 's/^/ /' - - # In a workspace, the scheme list also holds package and pod - # schemes, and they sort ahead of the app as often as not. The - # scheme named after the project is the app in nearly every repo. - SCHEME=$(printf '%s\n' "$SCHEMES" | grep -Fx "$BASE" | head -1 || true) - if [ -z "$SCHEME" ]; then - SCHEME=$(printf '%s\n' "$SCHEMES" | grep -v '^Pods-' | while IFS= read -r s; do - [ -n "$s" ] || continue - # A scheme that belongs to a local Swift package is not the app. - [ -z "$(find . -maxdepth 4 -path "*/$s/Package.swift" -print -quit)" ] || continue - printf '%s\n' "$s" - break - done) - fi - echo "Building with scheme: $SCHEME (configuration: $CONFIGURATION)" - fi - - if [ "$PROJECT_TYPE" = "native" ]; then - if [ -z "$WORKSPACE" ] && [ -z "$PROJECT" ]; then - echo "::error::No .xcworkspace or .xcodeproj found in '$IOS_PATH'. If the project is generated (XcodeGen/Tuist), commit it or add a generation step." - exit 1 - fi - if [ -z "$SCHEME" ]; then - echo "::error::Could not determine a scheme. Pass one via the 'scheme' input, or make sure the project has a shared scheme." - exit 1 - fi - fi - - # Use custom DerivedData path for caching - DERIVED_DATA_PATH="${GITHUB_WORKSPACE}/DerivedData" - - if [ "$PROJECT_TYPE" = "flutter" ]; then - cd "$GITHUB_WORKSPACE" - apply_build_number - FLUTTER_VERSION_FLAGS="" - if [ -n "$build_number" ]; then - FLUTTER_VERSION_FLAGS="--build-number=$build_number${build_name:+ --build-name=$build_name}" - fi - if [ "$CONFIGURATION" = "Debug" ]; then - echo "Building Flutter iOS Debug..." - flutter build ios --debug --no-codesign $FLUTTER_VERSION_FLAGS - else - echo "Building Flutter iOS Release..." - flutter build ios --release --no-codesign $FLUTTER_VERSION_FLAGS - fi - cd "$IOS_PATH" - - if [ "$USE_SIGNING" = "true" ]; then - # 'flutter build ios' never signs, so archive the Runner project it - # generated. The IPA step below exports build/App.xcarchive - # whenever signing is on. - ARCHIVE_CMD="xcodebuild" - if [ -n "$WORKSPACE" ]; then - ARCHIVE_CMD="$ARCHIVE_CMD -workspace '$WORKSPACE'" - elif [ -n "$PROJECT" ]; then - ARCHIVE_CMD="$ARCHIVE_CMD -project '$PROJECT'" - else - echo "::error::Signing requested but no Runner workspace/project found in '$IOS_PATH'." - exit 1 - fi - - # Flutter wrote the build number into Generated.xcconfig; this - # only catches a Runner Info.plist that hardcodes it. - apply_build_number "$TARGET_FLAG" "$TARGET_PATH" -scheme "${SCHEME:-Runner}" - ARCHIVE_CMD="$ARCHIVE_CMD -scheme '${SCHEME:-Runner}'" - ARCHIVE_CMD="$ARCHIVE_CMD -configuration '$CONFIGURATION'" - ARCHIVE_CMD="$ARCHIVE_CMD -destination 'generic/platform=iOS'" - ARCHIVE_CMD="$ARCHIVE_CMD -derivedDataPath '$DERIVED_DATA_PATH'" - ARCHIVE_CMD="$ARCHIVE_CMD COMPILER_INDEX_STORE_ENABLE=NO $version_settings" - ARCHIVE_CMD="$ARCHIVE_CMD -quiet" - ARCHIVE_CMD="$ARCHIVE_CMD -archivePath '$GITHUB_WORKSPACE/build/App.xcarchive' archive" - - # The manual signing settings go on the Runner target, after - # flutter build ios has regenerated the project. - apply_signing_to_app_target "$PROFILE_BUNDLE_ID" - echo "Running: $ARCHIVE_CMD" - eval $ARCHIVE_CMD - fi - elif [ "$PROJECT_TYPE" = "reactnative" ] || [ "$PROJECT_TYPE" = "expo" ] || [ "$PROJECT_TYPE" = "kmp" ]; then - # Use xcodebuild directly for React Native/Expo and Kotlin Multiplatform. - # For KMP, the Xcode build phase (or CocoaPods) invokes Gradle to - # compile the shared Kotlin framework, so the JDK must be on PATH. - if [ "$PROJECT_TYPE" = "kmp" ]; then - echo "Building Kotlin Multiplatform iOS ($CONFIGURATION)..." - if [ -f "${GITHUB_WORKSPACE}/gradlew" ]; then - chmod +x "${GITHUB_WORKSPACE}/gradlew" - fi - else - echo "Building React Native iOS ($CONFIGURATION)..." - fi - BUILD_CMD="xcodebuild" - if [ -n "$WORKSPACE" ]; then - BUILD_CMD="$BUILD_CMD -workspace $WORKSPACE" - elif [ -n "$PROJECT" ]; then - BUILD_CMD="$BUILD_CMD -project $PROJECT" - fi - - # Auto-detect scheme from workspace/project name (e.g., MyApp.xcworkspace -> MyApp) - # unless an explicit scheme was provided. - if [ -z "$SCHEME" ]; then - if [ -n "$WORKSPACE" ]; then - SCHEME=$(basename "$WORKSPACE" .xcworkspace) - elif [ -n "$PROJECT" ]; then - SCHEME=$(basename "$PROJECT" .xcodeproj) - fi - fi - - apply_build_number "$TARGET_FLAG" "$TARGET_PATH" -scheme "$SCHEME" - BUILD_CMD="$BUILD_CMD -scheme '$SCHEME'" - BUILD_CMD="$BUILD_CMD -configuration '$CONFIGURATION'" - BUILD_CMD="$BUILD_CMD -destination 'generic/platform=iOS'" - BUILD_CMD="$BUILD_CMD -derivedDataPath '$DERIVED_DATA_PATH'" - BUILD_CMD="$BUILD_CMD COMPILER_INDEX_STORE_ENABLE=NO $version_settings" - BUILD_CMD="$BUILD_CMD -quiet" - - if [ "$USE_SIGNING" = "true" ]; then - # Must archive, not build: the IPA step below exports - # build/App.xcarchive whenever signing is on. The signing settings - # go on the app target now that pod install has generated the project. - apply_signing_to_app_target "$PROFILE_BUNDLE_ID" - BUILD_CMD="$BUILD_CMD -archivePath '$GITHUB_WORKSPACE/build/App.xcarchive' archive" - else - BUILD_CMD="$BUILD_CMD CODE_SIGNING_ALLOWED=NO build" - fi - - echo "Running: $BUILD_CMD" - eval $BUILD_CMD - else - BUILD_CMD="xcodebuild" - if [ -n "$WORKSPACE" ]; then - BUILD_CMD="$BUILD_CMD -workspace $WORKSPACE" - elif [ -n "$PROJECT" ]; then - BUILD_CMD="$BUILD_CMD -project $PROJECT" - fi - - apply_build_number "$TARGET_FLAG" "$TARGET_PATH" -scheme "$SCHEME" - BUILD_CMD="$BUILD_CMD -scheme '$SCHEME'" - BUILD_CMD="$BUILD_CMD -configuration '$CONFIGURATION'" - BUILD_CMD="$BUILD_CMD -destination 'generic/platform=iOS'" - BUILD_CMD="$BUILD_CMD -derivedDataPath '$DERIVED_DATA_PATH'" - BUILD_CMD="$BUILD_CMD $version_settings" - - # Speed optimization flags - BUILD_CMD="$BUILD_CMD COMPILER_INDEX_STORE_ENABLE=NO" - BUILD_CMD="$BUILD_CMD DEBUG_INFORMATION_FORMAT=dwarf" - BUILD_CMD="$BUILD_CMD ONLY_ACTIVE_ARCH=YES" - BUILD_CMD="$BUILD_CMD -quiet" - - # Xcode 16+ native compilation caching - BUILD_CMD="$BUILD_CMD SWIFT_ENABLE_COMPILE_CACHE=YES" - BUILD_CMD="$BUILD_CMD CLANG_ENABLE_COMPILE_CACHE=YES" - - if [ "$USE_SIGNING" = "true" ]; then - # For signed builds, use archive. Automatic signing cannot resolve a - # team on a runner (no Xcode account), so sign the app target - # manually against the profile installed above. - apply_signing_to_app_target "$PROFILE_BUNDLE_ID" - BUILD_CMD="$BUILD_CMD -archivePath '$GITHUB_WORKSPACE/build/App.xcarchive' archive" - else - # For unsigned builds, use faster 'build' action - BUILD_CMD="$BUILD_CMD CODE_SIGNING_ALLOWED=NO build" - fi - - echo "Running: $BUILD_CMD" - eval $BUILD_CMD - fi - - # Create IPA - if [ "$USE_SIGNING" = "true" ]; then - # For signed builds, use xcodebuild -exportArchive. - # Everything below is relative to the repo root, which is also what - # the upload-artifact step ("build/*.ipa") resolves against — do not - # use "cd .." here, it breaks when ios_path is "." (native at root). - cd "$GITHUB_WORKSPACE" - - # The export has to use the same manual signing as the archive, and - # it needs the app's real bundle id to map it to the profile. - APP_BUNDLE_ID=$(plutil -extract ApplicationProperties.CFBundleIdentifier raw -o - build/App.xcarchive/Info.plist) - echo "Exporting $APP_BUNDLE_ID with profile '$PROVISIONING_PROFILE_NAME' (team $DEVELOPMENT_TEAM), method $EXPORT_METHOD" - export APP_BUNDLE_ID - write_export_options ExportOptions.plist - - xcodebuild -exportArchive \ - -archivePath build/App.xcarchive \ - -exportOptionsPlist ExportOptions.plist \ - -exportPath build/export - - # Move IPA to expected location - mv build/export/*.ipa "build/$BUILD_ID.ipa" - echo "Created signed IPA: build/$BUILD_ID.ipa" - else - # For unsigned builds, find .app and create IPA - if [ "$PROJECT_TYPE" = "flutter" ]; then - # Flutter outputs to build/ios/iphoneos/ - APP_PATH=$(find "${GITHUB_WORKSPACE}/build/ios/iphoneos" -name "*.app" -type d | head -1) - else - # Native and React Native/Expo use DerivedData. Ask the project - # which bundle this scheme produces: the first .app under - # Build/Products is the wrong one whenever the repo has more than - # one app target. - APP_PATH=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" \ - -scheme "$SCHEME" \ - -configuration "$CONFIGURATION" \ - -destination 'generic/platform=iOS' \ - -derivedDataPath "$DERIVED_DATA_PATH" \ - -showBuildSettings -json 2>/dev/null \ - | jq -r 'map(.buildSettings) - | map(select(.PRODUCT_TYPE == "com.apple.product-type.application" - and (.TARGET_BUILD_DIR | contains("-iphoneos")))) - | map(.TARGET_BUILD_DIR + "/" + .FULL_PRODUCT_NAME) - | first // empty' || true) - if [ ! -d "$APP_PATH" ]; then - APP_PATH=$(find "$DERIVED_DATA_PATH/Build/Products/${CONFIGURATION}-iphoneos" -maxdepth 1 -name "*.app" -type d -print -quit 2>/dev/null || true) - fi - fi - - if [ -z "$APP_PATH" ]; then - echo "Error: Could not find .app" - exit 1 - fi - echo "Found app at: $APP_PATH" - cd "$GITHUB_WORKSPACE" - mkdir -p build/Payload - cp -R "$APP_PATH" build/Payload/ - cd build - zip -rq "$BUILD_ID.ipa" Payload - rm -rf Payload - echo "Created unsigned IPA: build/$BUILD_ID.ipa" - fi - - - name: Upload IPA artifact - uses: actions/upload-artifact@v7 - with: - name: ipa - path: build/*.ipa - retention-days: 7 - if-no-files-found: error - - # Flutter builds through its own tree, so DerivedData can be absent and a - # save would fail on a path that never existed. - - name: Check for DerivedData - id: dd - if: always() - run: | - if [ -d DerivedData ]; then echo "exists=yes" >> $GITHUB_OUTPUT; fi - - - name: Save DerivedData cache - uses: actions/cache/save@v6 - if: always() && steps.dd.outputs.exists == 'yes' - with: - path: DerivedData - key: deriveddata-${{ github.run_id }} - - - name: Cleanup signing - if: always() && steps.params.outputs.use_signing == 'true' - run: | - if [ -n "$KEYCHAIN_PATH" ] && [ -f "$KEYCHAIN_PATH" ]; then - security delete-keychain "$KEYCHAIN_PATH" || true - fi - - - name: Build summary - if: always() - env: - USE_SIGNING: ${{ steps.params.outputs.use_signing }} - CONFIGURATION: ${{ steps.params.outputs.configuration }} - PROFILE: ${{ steps.params.outputs.profile }} - BUILD_NUMBER: ${{ steps.params.outputs.build_number }} - run: | - echo "## Build Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- **Build ID:** ${{ steps.params.outputs.build_id }}" >> $GITHUB_STEP_SUMMARY - echo "- **Status:** ${{ job.status }}" >> $GITHUB_STEP_SUMMARY - if [ -n "$PROFILE" ]; then - echo "- **Profile:** $PROFILE" >> $GITHUB_STEP_SUMMARY - fi - echo "- **Configuration:** $CONFIGURATION" >> $GITHUB_STEP_SUMMARY - if [ -n "$BUILD_NUMBER" ]; then - echo "- **Build number:** $BUILD_NUMBER" >> $GITHUB_STEP_SUMMARY - fi - echo "- **DerivedData Cache:** ${{ steps.cache-deriveddata.outputs.cache-hit == 'true' && 'Hit' || 'Miss' }}" >> $GITHUB_STEP_SUMMARY - echo "- **Pods Cache:** ${{ steps.pods-cache.outputs.cache-hit == 'true' && 'Hit' || 'Miss' }}" >> $GITHUB_STEP_SUMMARY - echo "- **Node Modules Cache:** ${{ steps.node-modules-cache.outputs.cache-hit == 'true' && 'Hit' || 'N/A' }}" >> $GITHUB_STEP_SUMMARY - if [ "$USE_SIGNING" = "true" ]; then - echo "- **Signing:** Signed (${EXPORT_METHOD:-unknown}, set ${SIGNING_SET_USED:-unknown})" >> $GITHUB_STEP_SUMMARY - else - echo "- **Signing:** Unsigned (sign locally with AltStore/Sideloadly)" >> $GITHUB_STEP_SUMMARY - fi - if [ -f build/*.ipa ]; then - IPA_PATH=$(find build -name "*.ipa" -type f | head -1) - echo "- **IPA Size:** $(ls -lh "$IPA_PATH" | awk '{print $5}')" >> $GITHUB_STEP_SUMMARY - fi - - # The tag exists only to start this run, so it leaves with it. Deleting - # a tag fires no push event, so this cannot start another run. - - name: Delete trigger tag - if: always() && github.ref_type == 'tag' - run: git push origin ":refs/tags/$GITHUB_REF_NAME" diff --git a/.github/workflows/ios-share.yml b/.github/workflows/ios-share.yml deleted file mode 100644 index 4beb1dc..0000000 --- a/.github/workflows/ios-share.yml +++ /dev/null @@ -1,692 +0,0 @@ -name: iOS Simulator -run-name: iOS Simulator ${{ inputs.build_id || github.ref_name }} - -# Builds the app for the simulator and then makes that simulator usable from -# the MobAI app, so the build can be tried by hand from a machine that is not a -# Mac. The job stays alive while the simulator is in use and ends when it is -# released or left unused. -# -# Simulator builds are never signed, so none of the signing setup in -# ios-build.yml applies here. - -on: - # Pushing a tag named ios-share/ starts the same job without the - # GitHub API, for environments where only git access is available. A push - # event runs the workflow file from the tagged commit, and the parameters - # come from builder.json there. The tag is deleted when the job ends. - push: - tags: - - 'ios-share/*' - workflow_dispatch: - inputs: - build_id: - description: 'Unique build identifier' - required: true - type: string - snapshot_ref: - description: 'Ref holding the working-tree snapshot to build (empty builds this branch)' - required: false - type: string - default: '' - ios_path: - description: 'Path to iOS project (e.g., "ios" for React Native)' - required: false - type: string - default: '.' - scheme: - description: 'Xcode scheme to build (auto-detected if empty)' - required: false - type: string - default: '' - duration: - description: 'How long the simulator stays available while unused (e.g. 30m)' - required: false - type: string - default: '30m' - flutter_version: - description: 'Flutter version to use (e.g., 3.24.0). Empty means latest stable.' - required: false - type: string - default: '' - jdk_version: - description: 'JDK version for Kotlin Multiplatform Gradle builds (e.g., 17).' - required: false - type: string - default: '17' - -jobs: - simulator: - runs-on: macos-latest - permissions: - contents: write # deletes the trigger tag - # Bounds the session: a simulator nobody released must not hold a runner - # indefinitely. Comfortably above the default duration. - timeout-minutes: 90 - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - # The dispatch always runs the workflow from the default branch, so the - # sources come from a snapshot ref pushed by the CLI instead. - - name: Checkout working-tree snapshot - if: inputs.snapshot_ref != '' - env: - SNAPSHOT_REF: ${{ inputs.snapshot_ref }} - run: | - set -e - git fetch --depth=1 origin "+$SNAPSHOT_REF:refs/heads/snapshot" - git checkout --force snapshot - echo "Building $(git rev-parse --short HEAD)" - - # Dispatch inputs arrive with their declared defaults. A tag push has no - # inputs, so the values come from builder.json in the tagged commit and - # the build id is the tag name after the prefix. - - name: Resolve parameters - id: params - env: - IN_BUILD_ID: ${{ inputs.build_id }} - IN_IOS_PATH: ${{ inputs.ios_path }} - IN_SCHEME: ${{ inputs.scheme }} - IN_DURATION: ${{ inputs.duration }} - IN_FLUTTER_VERSION: ${{ inputs.flutter_version }} - IN_JDK_VERSION: ${{ inputs.jdk_version }} - run: | - set -e - param() { # name dispatch-value jq-path default - local v="" - if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - v="$2" - elif [ -f builder.json ]; then - v=$(jq -r "$3 // empty" builder.json) - fi - v="${v:-$4}" - echo "$1=$v" >> "$GITHUB_OUTPUT" - echo "$1=$v" - } - param build_id "$IN_BUILD_ID" '""' "${GITHUB_REF_NAME##*/}" - param ios_path "$IN_IOS_PATH" '.ios.path' '.' - param scheme "$IN_SCHEME" '.ios.scheme' '' - param duration "$IN_DURATION" '""' '30m' - param flutter_version "$IN_FLUTTER_VERSION" '.flutter.version' '' - param jdk_version "$IN_JDK_VERSION" '.kmp.jdkVersion' '17' - - # Starts the simulator booting in the background (cached, so later runs - # boot fast) while the app builds. - - name: Install mobai-ci + boot simulator - uses: MobAI-App/mobai-ci@v1 - with: - boot-sim: true # UDID lands in $MOBAI_SIM_UDID - - - name: Setup Xcode - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: latest-stable - - - name: Restore DerivedData cache - uses: actions/cache/restore@v6 - with: - path: DerivedData - key: deriveddata-sim-${{ github.run_id }} - restore-keys: | - deriveddata-sim- - - - name: Detect project type - id: detect - run: | - if [ -f "pubspec.yaml" ]; then - echo "type=flutter" >> $GITHUB_OUTPUT - elif [ -f "package.json" ] && grep -q '"expo"' package.json; then - echo "type=expo" >> $GITHUB_OUTPUT - elif [ -f "package.json" ] && grep -q '"react-native"' package.json; then - echo "type=reactnative" >> $GITHUB_OUTPUT - elif grep -rqlE 'kotlin\("multiplatform"\)|org\.jetbrains\.kotlin\.multiplatform|id\("org.jetbrains.kotlin.multiplatform"\)' --include='*.gradle.kts' --include='*.gradle' --include='libs.versions.toml' . 2>/dev/null; then - echo "type=kmp" >> $GITHUB_OUTPUT - else - echo "type=native" >> $GITHUB_OUTPUT - fi - - # A gitignored *.xcconfig next to a committed template is a common iOS - # convention. The snapshot then carries the template but not the config, - # and xcodebuild fails with "Unable to open base configuration reference - # file", which says nothing about which file is missing. - - name: Materialize missing xcconfig files - run: | - set -e - find . -path ./DerivedData -prune -o -type f \ - \( -name '*.xcconfig.template' -o -name '*.xcconfig.example' \ - -o -name '*.xcconfig.sample' -o -name '*.template.xcconfig' \ - -o -name '*.example.xcconfig' -o -name '*.sample.xcconfig' \) -print \ - | while IFS= read -r template; do - case "$template" in - *.xcconfig.template|*.xcconfig.example|*.xcconfig.sample) target="${template%.*}" ;; - *) target="$(dirname "$template")/$(basename "$template" | sed -E 's/\.(template|example|sample)\.xcconfig$/.xcconfig/')" ;; - esac - if [ ! -f "$target" ]; then - cp "$template" "$target" - echo "::warning::$target was missing (gitignored?); copied $template over it. Any placeholder values in it are what the build sees." - fi - done - - - name: Generate Xcode project (XcodeGen) - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - run: | - set -e - # A managed Expo project has no iOS directory at all until the - # prebuild step below creates it. - if [ ! -d "$IOS_PATH" ]; then - echo "No '$IOS_PATH' directory yet — skipping." - exit 0 - fi - cd "$IOS_PATH" - - MANIFEST="" - for f in project.yml project.yaml; do - if [ -f "$f" ] && grep -qE '^targets:' "$f"; then - MANIFEST="$f" - break - fi - done - - if [ -z "$MANIFEST" ]; then - echo "No XcodeGen manifest — skipping." - elif [ -n "$(find . -maxdepth 1 -name '*.xcodeproj' -print -quit)" ]; then - echo "Found a committed .xcodeproj — skipping generation." - else - echo "Found $MANIFEST and no .xcodeproj — generating with XcodeGen." - brew install xcodegen - xcodegen generate - fi - - # Whatever the previous step could not fill in is named here, instead of - # surfacing as xcodebuild's "Unable to open base configuration reference". - - name: Check base configuration files - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - run: | - set -e - if [ ! -d "$IOS_PATH" ]; then - echo "No '$IOS_PATH' directory yet — skipping." - exit 0 - fi - MISSING="" - for pbx in $(find "$IOS_PATH" -maxdepth 2 -name project.pbxproj -not -path '*/Pods.xcodeproj/*'); do - for id in $(grep -o 'baseConfigurationReference = [A-Za-z0-9]*' "$pbx" | awk '{print $3}' | sort -u); do - ref=$(grep -m1 "^[[:space:]]*$id .*isa = PBXFileReference" "$pbx" || true) - name=$(printf '%s' "$ref" | sed -n 's/.*[ \t]path = "\{0,1\}\([^";]*\)"\{0,1\};.*/\1/p') - [ -n "$name" ] || continue - base=$(basename "$name") - # CocoaPods writes its xcconfigs during pod install and Flutter - # writes Generated.xcconfig during pub get — both happen later in - # this job, so they are legitimately absent right now. - case "$name" in *Pods/*|*"Target Support Files"*) continue ;; esac - case "$base" in Pods-*|Generated.xcconfig) continue ;; esac - if [ -z "$(find . -name "$base" -print -quit)" ]; then - MISSING="$MISSING $base" - fi - done - done - if [ -n "$MISSING" ]; then - echo "::error::The project has base configuration files that are not in the repository:$MISSING. They are probably gitignored — commit them, or commit a .xcconfig.template next to each so the build can fill them in." - exit 1 - fi - - - name: Setup Flutter - if: steps.detect.outputs.type == 'flutter' - uses: subosito/flutter-action@v2 - with: - flutter-version: ${{ steps.params.outputs.flutter_version || '' }} - channel: stable - cache: true - - - name: Flutter pub get - if: steps.detect.outputs.type == 'flutter' - run: flutter pub get - - # setup-node cannot make `with:` keys conditional, so the version and the - # version file are both computed here; it ignores an empty version file. - - name: Resolve JS toolchain - id: js - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - run: | - set -e - # >>> js toolchain (keep identical across ios-build.yml, ios-share.yml, runner.sh) - # The package manager the project actually uses: the packageManager field - # first, then the lockfile, then npm. Guessing npm is not safe — `npm install` - # in a pnpm or yarn workspace fails with - # 'EUNSUPPORTEDPROTOCOL Unsupported URL Type "workspace:"'. - js_detect_manager() { - if [ -z "${JS_MANAGER:-}" ]; then - case "$(jq -r '.packageManager // empty' package.json 2>/dev/null || true)" in - pnpm|pnpm@*) JS_MANAGER=pnpm ;; - yarn|yarn@*) JS_MANAGER=yarn ;; - bun|bun@*) JS_MANAGER=bun ;; - npm|npm@*) JS_MANAGER=npm ;; - *) - if [ -f pnpm-lock.yaml ]; then JS_MANAGER=pnpm - elif [ -f yarn.lock ]; then JS_MANAGER=yarn - elif [ -f bun.lock ] || [ -f bun.lockb ]; then JS_MANAGER=bun - else JS_MANAGER=npm # package-lock.json, or no lockfile at all - fi - ;; - esac - fi - case "$JS_MANAGER" in - pnpm) JS_EXEC='pnpm exec' ;; - yarn) JS_EXEC=yarn ;; - bun) JS_EXEC=bunx ;; - *) JS_EXEC=npx ;; - esac - export JS_MANAGER JS_EXEC - echo "Package manager: $JS_MANAGER" - } - - # The Node version the project asks for: .nvmrc or .node-version, else the - # first comparator of engines.node, else 22. A leading "v" or range prefix - # is dropped: nvm, n and setup-node all take "20.11.1" and "24.x" as they are. - js_node_version() { - js_version="" - for js_file in .nvmrc .node-version; do - if [ -f "$js_file" ]; then - js_version=$(tr -d ' \t\r' < "$js_file" | head -n1) - break - fi - done - if [ -z "$js_version" ]; then - js_version=$(jq -r '.engines.node // empty' package.json 2>/dev/null || true) - js_version=$(printf '%s' "${js_version%%|*}" | awk '{print $1}') - fi - js_version=$(printf '%s' "$js_version" | sed -E 's/^(>=|<=|>|<|=|\^|~|v)+//') - case "$js_version" in - [0-9]*|lts/*) ;; - *) js_version=22 ;; - esac - printf '%s\n' "$js_version" - } - - # Put the manager on PATH first, then install — unless a cache already restored - # node_modules, in which case the manager is still wanted for `expo prebuild` - # and for the React Native build phases that shell out to it. - js_install() { - js_detect_manager - case "$JS_MANAGER" in - pnpm|yarn) - # corepack installs the version packageManager pins. Node 16.10-24 - # bundle it; Node 25+ and some provider images do not. - command -v corepack >/dev/null 2>&1 || npm install -g corepack || true - corepack enable || true - command -v "$JS_MANAGER" >/dev/null 2>&1 || npm install -g "$JS_MANAGER" - ;; - bun) - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - export BUN_INSTALL="${BUN_INSTALL:-$HOME/.bun}" - export PATH="$BUN_INSTALL/bin:$PATH" - # A step's PATH does not survive into the next GitHub Actions step. - if [ -n "${GITHUB_PATH:-}" ]; then echo "$BUN_INSTALL/bin" >> "$GITHUB_PATH"; fi - fi - ;; - esac - if [ "${JS_DEPS_CACHED:-}" = "true" ]; then - echo "node_modules restored from cache; skipping install" - return 0 - fi - case "$JS_MANAGER" in - pnpm) - if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; else pnpm install; fi - ;; - yarn) - if [ -f .yarnrc.yml ]; then yarn install --immutable - elif [ -f yarn.lock ]; then yarn install --frozen-lockfile - else yarn install - fi - ;; - bun) - if [ -f bun.lock ] || [ -f bun.lockb ]; then bun install --frozen-lockfile; else bun install; fi - ;; - *) - if [ -f package-lock.json ]; then npm ci; else npm install; fi - ;; - esac - } - # <<< js toolchain - js_detect_manager - NODE_VERSION_FILE="" - for f in .nvmrc .node-version; do - if [ -f "$f" ]; then NODE_VERSION_FILE="$f"; break; fi - done - NODE_VERSION="" - if [ -n "$NODE_VERSION_FILE" ]; then - echo "Node version: from $NODE_VERSION_FILE" - else - NODE_VERSION=$(js_node_version) - echo "Node version: $NODE_VERSION" - fi - { - echo "manager=$JS_MANAGER" - echo "exec=$JS_EXEC" - echo "node_version=$NODE_VERSION" - echo "node_version_file=$NODE_VERSION_FILE" - } >> "$GITHUB_OUTPUT" - - - name: Setup Node.js - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - uses: actions/setup-node@v7 - with: - node-version: ${{ steps.js.outputs.node_version }} - node-version-file: ${{ steps.js.outputs.node_version_file }} - - # Runs even on a cache hit: the manager itself still has to be on PATH for - # the prebuild and build steps that follow. - - name: Install JS dependencies - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - env: - JS_MANAGER: ${{ steps.js.outputs.manager }} - run: | - set -e - # >>> js toolchain (keep identical across ios-build.yml, ios-share.yml, runner.sh) - # The package manager the project actually uses: the packageManager field - # first, then the lockfile, then npm. Guessing npm is not safe — `npm install` - # in a pnpm or yarn workspace fails with - # 'EUNSUPPORTEDPROTOCOL Unsupported URL Type "workspace:"'. - js_detect_manager() { - if [ -z "${JS_MANAGER:-}" ]; then - case "$(jq -r '.packageManager // empty' package.json 2>/dev/null || true)" in - pnpm|pnpm@*) JS_MANAGER=pnpm ;; - yarn|yarn@*) JS_MANAGER=yarn ;; - bun|bun@*) JS_MANAGER=bun ;; - npm|npm@*) JS_MANAGER=npm ;; - *) - if [ -f pnpm-lock.yaml ]; then JS_MANAGER=pnpm - elif [ -f yarn.lock ]; then JS_MANAGER=yarn - elif [ -f bun.lock ] || [ -f bun.lockb ]; then JS_MANAGER=bun - else JS_MANAGER=npm # package-lock.json, or no lockfile at all - fi - ;; - esac - fi - case "$JS_MANAGER" in - pnpm) JS_EXEC='pnpm exec' ;; - yarn) JS_EXEC=yarn ;; - bun) JS_EXEC=bunx ;; - *) JS_EXEC=npx ;; - esac - export JS_MANAGER JS_EXEC - echo "Package manager: $JS_MANAGER" - } - - # The Node version the project asks for: .nvmrc or .node-version, else the - # first comparator of engines.node, else 22. A leading "v" or range prefix - # is dropped: nvm, n and setup-node all take "20.11.1" and "24.x" as they are. - js_node_version() { - js_version="" - for js_file in .nvmrc .node-version; do - if [ -f "$js_file" ]; then - js_version=$(tr -d ' \t\r' < "$js_file" | head -n1) - break - fi - done - if [ -z "$js_version" ]; then - js_version=$(jq -r '.engines.node // empty' package.json 2>/dev/null || true) - js_version=$(printf '%s' "${js_version%%|*}" | awk '{print $1}') - fi - js_version=$(printf '%s' "$js_version" | sed -E 's/^(>=|<=|>|<|=|\^|~|v)+//') - case "$js_version" in - [0-9]*|lts/*) ;; - *) js_version=22 ;; - esac - printf '%s\n' "$js_version" - } - - # Put the manager on PATH first, then install — unless a cache already restored - # node_modules, in which case the manager is still wanted for `expo prebuild` - # and for the React Native build phases that shell out to it. - js_install() { - js_detect_manager - case "$JS_MANAGER" in - pnpm|yarn) - # corepack installs the version packageManager pins. Node 16.10-24 - # bundle it; Node 25+ and some provider images do not. - command -v corepack >/dev/null 2>&1 || npm install -g corepack || true - corepack enable || true - command -v "$JS_MANAGER" >/dev/null 2>&1 || npm install -g "$JS_MANAGER" - ;; - bun) - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - export BUN_INSTALL="${BUN_INSTALL:-$HOME/.bun}" - export PATH="$BUN_INSTALL/bin:$PATH" - # A step's PATH does not survive into the next GitHub Actions step. - if [ -n "${GITHUB_PATH:-}" ]; then echo "$BUN_INSTALL/bin" >> "$GITHUB_PATH"; fi - fi - ;; - esac - if [ "${JS_DEPS_CACHED:-}" = "true" ]; then - echo "node_modules restored from cache; skipping install" - return 0 - fi - case "$JS_MANAGER" in - pnpm) - if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; else pnpm install; fi - ;; - yarn) - if [ -f .yarnrc.yml ]; then yarn install --immutable - elif [ -f yarn.lock ]; then yarn install --frozen-lockfile - else yarn install - fi - ;; - bun) - if [ -f bun.lock ] || [ -f bun.lockb ]; then bun install --frozen-lockfile; else bun install; fi - ;; - *) - if [ -f package-lock.json ]; then npm ci; else npm install; fi - ;; - esac - } - # <<< js toolchain - js_install - - # A managed Expo project keeps no ios/ directory in git — `expo prebuild` - # generates it from app.json / app.config.js. The simulator build needs - # the same native project the IPA build does. - - name: Expo prebuild - if: steps.detect.outputs.type == 'expo' - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - JS_EXEC: ${{ steps.js.outputs.exec }} - CI: '1' # prebuild answers its own prompts instead of hanging - run: | - set -e - if [ -n "$(find "$IOS_PATH" -maxdepth 1 \( -name '*.xcodeproj' -o -name '*.xcworkspace' \) -print -quit 2>/dev/null)" ]; then - echo "'$IOS_PATH' already holds an Xcode project (ejected Expo) — skipping prebuild." - exit 0 - fi - - # Without a bundle identifier prebuild asks for one interactively, - # which on a runner means waiting for the job timeout. - BUNDLE_ID=$(${JS_EXEC:-npx} expo config --type public --json 2>/dev/null | jq -r '.ios.bundleIdentifier // empty' 2>/dev/null || true) - if [ -z "$BUNDLE_ID" ]; then - for f in app.json app.config.json; do - [ -f "$f" ] || continue - BUNDLE_ID=$(jq -r '(.expo.ios.bundleIdentifier // .ios.bundleIdentifier) // empty' "$f" 2>/dev/null || true) - if [ -n "$BUNDLE_ID" ]; then break; fi - done - fi - if [ -z "$BUNDLE_ID" ]; then - echo "::error::This managed Expo project has no ios.bundleIdentifier. Set it in app.json (expo.ios.bundleIdentifier) or app.config.js, commit that, and share again — 'expo prebuild' cannot run unattended without it." - exit 1 - fi - echo "Prebuilding $BUNDLE_ID into '$IOS_PATH'..." - - ${JS_EXEC:-npx} expo prebuild --platform ios --no-install - if [ -z "$(find "$IOS_PATH" -maxdepth 1 -name '*.xcodeproj' -print -quit 2>/dev/null)" ]; then - echo "::error::'expo prebuild' produced no Xcode project in '$IOS_PATH'. If the project generates it elsewhere, set ios.path in builder.json." - exit 1 - fi - - # KMP builds through xcodebuild like a native project, but its Xcode run - # script phase shells out to Gradle, which needs a JDK on PATH. - - name: Setup JDK - if: steps.detect.outputs.type == 'kmp' - uses: actions/setup-java@v6 - with: - distribution: temurin - java-version: ${{ steps.params.outputs.jdk_version || '17' }} - - - name: Setup Gradle - if: steps.detect.outputs.type == 'kmp' - uses: gradle/actions/setup-gradle@v6 - - - name: Build for the simulator - id: build - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - SCHEME: ${{ steps.params.outputs.scheme }} - PROJECT_TYPE: ${{ steps.detect.outputs.type }} - run: | - set -e - - if [ "$PROJECT_TYPE" = "flutter" ]; then - # Flutter drives xcodebuild itself and puts the bundle in a known - # place, so there is nothing to detect here. - flutter build ios --simulator --debug - APP_PATH=$(find build/ios/iphonesimulator -maxdepth 1 -name '*.app' -print -quit) - else - cd "$IOS_PATH" - - # CocoaPods generates the .xcworkspace and many repos gitignore it, - # so install before looking for one. - if [ -f Podfile ]; then - command -v pod >/dev/null 2>&1 || gem install cocoapods --no-document - pod install - fi - - WORKSPACE=$(find . -maxdepth 1 -name '*.xcworkspace' -print -quit) - PROJECT=$(find . -maxdepth 1 -name '*.xcodeproj' -print -quit) - - if [ -n "$WORKSPACE" ]; then - TARGET_FLAG=-workspace - TARGET_PATH=$WORKSPACE - BASE=$(basename "$WORKSPACE" .xcworkspace) - elif [ -n "$PROJECT" ]; then - TARGET_FLAG=-project - TARGET_PATH=$PROJECT - BASE=$(basename "$PROJECT" .xcodeproj) - else - echo "::error::No .xcworkspace or .xcodeproj found in $IOS_PATH" - exit 1 - fi - - if [ -z "$SCHEME" ]; then - SCHEMES=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" -list -json 2>/dev/null | jq -r '(.workspace // .project).schemes[]?' || true) - if [ -z "$SCHEMES" ]; then - SCHEMES=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" -list 2>/dev/null | sed -n '/Schemes:/,$p' | tail -n +2 | sed 's/^[[:space:]]*//' | sed '/^$/d' || true) - fi - echo "Schemes:" - printf '%s\n' "$SCHEMES" | sed 's/^/ /' - - # In a workspace, the scheme list also holds package and pod - # schemes, and they sort ahead of the app as often as not. The - # scheme named after the project is the app in nearly every repo. - SCHEME=$(printf '%s\n' "$SCHEMES" | grep -Fx "$BASE" | head -1 || true) - if [ -z "$SCHEME" ]; then - SCHEME=$(printf '%s\n' "$SCHEMES" | grep -v '^Pods-' | while IFS= read -r s; do - [ -n "$s" ] || continue - # A scheme that belongs to a local Swift package is not the app. - [ -z "$(find . -maxdepth 4 -path "*/$s/Package.swift" -print -quit)" ] || continue - printf '%s\n' "$s" - break - done) - fi - fi - - if [ -z "$SCHEME" ]; then - echo "::error::Could not determine a scheme in $IOS_PATH. Pass one with the 'scheme' input." - exit 1 - fi - echo "Scheme: $SCHEME" - - # Build for the simulator this job is about to install on, rather - # than letting xcodebuild pick the first of several matches. - if [ -n "$MOBAI_SIM_UDID" ]; then - DEST_FLAG=-destination - DEST_VALUE="id=$MOBAI_SIM_UDID" - else - DEST_FLAG=-sdk - DEST_VALUE=iphonesimulator - fi - - xcodebuild "$TARGET_FLAG" "$TARGET_PATH" \ - -scheme "$SCHEME" \ - -configuration Debug \ - "$DEST_FLAG" "$DEST_VALUE" \ - -derivedDataPath "$GITHUB_WORKSPACE/DerivedData" \ - COMPILER_INDEX_STORE_ENABLE=NO \ - CODE_SIGNING_ALLOWED=NO \ - build - - # Ask the project which bundle this scheme produces: the first .app - # under Build/Products is the wrong one whenever the repo has more - # than one app target. - APP_PATH=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" \ - -scheme "$SCHEME" \ - -configuration Debug \ - "$DEST_FLAG" "$DEST_VALUE" \ - -derivedDataPath "$GITHUB_WORKSPACE/DerivedData" \ - -showBuildSettings -json 2>/dev/null \ - | jq -r 'map(.buildSettings) - | map(select(.PRODUCT_TYPE == "com.apple.product-type.application" - and (.TARGET_BUILD_DIR | contains("-iphonesimulator")))) - | map(.TARGET_BUILD_DIR + "/" + .FULL_PRODUCT_NAME) - | first // empty' || true) - if [ ! -d "$APP_PATH" ]; then - APP_PATH=$(find "$GITHUB_WORKSPACE/DerivedData/Build/Products/Debug-iphonesimulator" -maxdepth 1 -name '*.app' -print -quit 2>/dev/null || true) - fi - cd "$GITHUB_WORKSPACE" - fi - - if [ -z "$APP_PATH" ]; then - echo "::error::Build produced no .app" - exit 1 - fi - echo "Built $APP_PATH" - echo "app_path=$APP_PATH" >> $GITHUB_OUTPUT - - # Flutter builds through its own tree, so DerivedData can be absent and a - # save would fail on a path that never existed. - - name: Check for DerivedData - id: dd - if: always() - run: | - if [ -d DerivedData ]; then echo "exists=yes" >> $GITHUB_OUTPUT; fi - - # Saved before the sharing step, which blocks for as long as the - # simulator is in use: a job that hits the timeout there would otherwise - # leave the next run with a cold DerivedData. - - name: Save DerivedData cache - if: always() && steps.dd.outputs.exists == 'yes' - uses: actions/cache/save@v6 - with: - path: DerivedData - key: deriveddata-sim-${{ github.run_id }} - - # Installs the build on the simulator and publishes it to the MobAI app. - # The step (and the job) stay running until the simulator is released - # there or goes unused. - - name: Share the simulator - env: - MOBAI_API_KEY: ${{ secrets.MOBAI_API_KEY }} - run: | - mobai-ci share \ - --device "$MOBAI_SIM_UDID" \ - --app "${{ steps.build.outputs.app_path }}" \ - --duration "${{ steps.params.outputs.duration }}" \ - --label "${{ github.repository }} · ${{ steps.params.outputs.build_id }}" \ - --wait-device 4m --startup-timeout 5m - - # The tag exists only to start this run, so it leaves with it. Deleting - # a tag fires no push event, so this cannot start another run. - - name: Delete trigger tag - if: always() && github.ref_type == 'tag' - run: git push origin ":refs/tags/$GITHUB_REF_NAME" diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 900c15d..9fd897f 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -1,7 +1,63 @@ name: Native iOS Toolchain +# Builds the compiler that XForge links into the app: Clang + Mach-O LLD for +# iPhoneOS, and — when `with_swift` is set — Swift's frontend libraries. +# +# Two things about the Swift half are worth knowing before editing it: +# +# * Every bundle is built from Swift's LLVM fork (`swiftlang/llvm-project` at the +# branch paired with the Swift version), never from upstream. Swift is developed +# against that fork (the `swiftasync` calling convention, the coroutine passes, +# the C++ API it compiles against), and — the part that decided it — upstream's +# clang is a different *API generation*: value-typed `DiagnosticOptions` and +# reference-taking constructors, against the fork's reference-counted options and +# pointer-taking ones. The app's C bridge is compiled against whichever bundle is +# installed, so it can only be written for one generation. Pinning the fork for +# both bundles is also what removes the old "unpinned checkout of llvm-project +# main" this workflow used to have. +# * `-DSWIFT_INSTALL_COMPONENTS=` is load-bearing. Swift's CMake writes that as a +# *cache* variable, so an empty value on the command line survives its default +# and no component is ever installed — which means `swift_install_in_component` +# returns early and its `install(TARGETS ...)` never runs. That rule is what +# aborts the configure when Swift is cross-configured for iOS: CMake refuses +# `install(TARGETS)` for an executable without a BUNDLE destination on an iOS +# target, and Swift's host tools have no such destination. +# +# The Swift version must match the Swift runtime inside the darwin SDK the app +# compiles against (that SDK is built from an Xcode; Xcode 26.6 ships Swift +# 6.2.4), otherwise the frontend cannot read the SDK's standard-library modules. + on: workflow_dispatch: + inputs: + with_swift: + description: Build the Swift frontend libraries as well + type: boolean + default: false + swift_ref: + description: swiftlang/swift tag or branch + type: string + default: swift-6.2.4-RELEASE + swift_llvm_ref: + description: swiftlang/llvm-project branch paired with swift_ref + type: string + default: swift/release/6.2 + configure_only: + description: Stop after configuring the Swift build (cheap iteration) + type: boolean + default: false + deployment_target: + description: Minimum iOS version + type: string + default: "17.0" + cache_salt: + description: Bump to discard the cached iOS builds (do this when the configure recipe below changes) + type: string + default: "3" + publish: + description: Publish the finished bundle as a release asset + type: boolean + default: true pull_request: paths: - "App/NativeToolchain/**" @@ -16,24 +72,191 @@ on: jobs: llvm-ios: runs-on: macos-15 - timeout-minutes: 180 + timeout-minutes: 350 + # `contents: write` is for the publish step at the bottom: the bundle is worth + # more as a release asset than as a workflow artifact, because artifacts expire + # (7-90 days, at GitHub's discretion) and cannot be fetched by tag. + permissions: + contents: write + env: + IOS_DEPLOYMENT_TARGET: ${{ inputs.deployment_target || '17.0' }} + SWIFT_REF: ${{ inputs.swift_ref || 'swift-6.2.4-RELEASE' }} + SWIFT_LLVM_REF: ${{ inputs.swift_llvm_ref || 'swift/release/6.2' }} steps: - uses: actions/checkout@v4 - name: Install build tools run: brew install cmake ninja + # One LLVM for every bundle this workflow builds: Swift's fork, at the branch + # paired with the Swift version. Upstream's clang is a *different API + # generation* from the fork's (reference-counted options and pointer-taking + # printers versus value options and references), and the C bridge that the app + # compiles against the bundle can only be written for one of them — so the + # bundle must always be the same generation. Pinning it also removes the + # "unpinned checkout of llvm-project main" this workflow used to have. - name: Checkout LLVM uses: actions/checkout@v4 with: - repository: llvm/llvm-project + repository: swiftlang/llvm-project + ref: ${{ env.SWIFT_LLVM_REF }} path: llvm-project fetch-depth: 1 + - name: Checkout Swift + if: ${{ inputs.with_swift }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift + ref: ${{ env.SWIFT_REF }} + path: swift + fetch-depth: 1 + + # Swift's configure step executes `cmark --version` and generates + # documentation with it, so a native cmark has to exist before the + # cross-configure runs. + - name: Checkout cmark + if: ${{ inputs.with_swift }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift-cmark + ref: release/6.2 + path: swift-cmark + fetch-depth: 1 + + # Only needed to *find* the sources at configure time; the macro and ASTGen + # libraries are switched off, because a macro plugin is a separate process + # and iOS has no way to run one. + - name: Checkout swift-syntax + if: ${{ inputs.with_swift }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift-syntax + ref: release/6.2 + path: swift-syntax + fetch-depth: 1 + + - name: Checkout swift-experimental-string-processing + if: ${{ inputs.with_swift }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift-experimental-string-processing + ref: release/6.2 + path: swift-experimental-string-processing + fetch-depth: 1 + + - name: Identify the iOS compiler toolchain + id: toolchain-versions + run: | + { + echo "value=$(xcodebuild -version | tr '\n' ' ')-$(xcrun --sdk iphoneos --show-sdk-version)" + } >> "$GITHUB_OUTPUT" + + # The bundle's *name*, decided from what goes into it rather than from when it + # was built: the LLVM commit, the Swift revision (or the fact that it is not + # included) and the deployment target decide which archives exist, and the + # SDK decides what they were compiled against. Two builds of the same inputs + # therefore land on the same tag, and a build that could not be substituted + # for another can never shadow it — which is what makes + # `build-ipa.yml`'s "the newest `toolchain-*` release" a safe rule. + - name: Name the bundle + id: bundle + shell: bash + run: | + set -euxo pipefail + LLVM_SHA="$(git -C llvm-project rev-parse --short=12 HEAD)" + if [ "${{ inputs.with_swift }}" = "true" ]; then + # `/` is legal in a tag and awkward in a URL and a shell word. + SWIFT_MARK="swift-$(printf '%s' "$SWIFT_REF" | tr '/' '-')" + else + SWIFT_MARK="noswift" + fi + SDK_VERSION="$(xcrun --sdk iphoneos --show-sdk-version)" + TAG="toolchain-${LLVM_SHA}-${SWIFT_MARK}-ios${IOS_DEPLOYMENT_TARGET}-sdk${SDK_VERSION}" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "bundle tag: $TAG" + + # A restored build tree carries the timestamps of the run that saved it, and + # those are *older* than the sources this run has just checked out. Ninja + # compares timestamps, so the tree it just restored would be rebuilt in full + # — the one thing the cache exists to prevent. The fix is to age the sources + # instead of refreshing the tree: "every file in the tree is newer than every + # source" then holds for whatever the cache restores, and it holds for ninja's + # generator check as well, because build.ninja is an *output* of that edge and + # any source newer than it sends ninja into a re-configure. + # + # Stamping the tree with `find … -exec touch` (what this workflow used to do, + # after configure) is unsound: the files are stamped in batches milliseconds + # apart, so whether a generator *input* ends up after build.ninja is decided + # by directory order. When it does, ninja re-runs CMake, CMake rewrites + # llvm/Config/*.h, and every object that includes them is dirty again — a + # full 2601-edge rebuild that the restore paid seconds to avoid. + - name: Age the source checkouts + shell: bash + run: | + set -euxo pipefail + for dir in llvm-project swift swift-cmark swift-syntax \ + swift-experimental-string-processing; do + [ -d "$dir" ] && find "$dir" -exec touch -h -t 202001010000 {} + 2>/dev/null || true + done + + # The expensive half is LLVM's cross-build and (when enabled) Swift's. The + # key deliberately does not include the workflow file: editing this file — + # which is what iterating on it looks like — must not force a rebuild of a + # compiler that did not change. `cache_salt` is the way to ask for one. + # + # Bump it whenever the *recipe* changes too. A cache entry is immutable: once + # a partial tree has been stored under a key, every later run gets a "hit" + # for it, the save below is skipped (the key already exists, so the upload + # would be refused anyway) and the key can never be repaired. That is the + # trap the old `always()`-gated save walked into — it stored whatever existed + # even when the build it was saving had just failed. + - name: Restore the LLVM builds + id: llvm-cache + uses: actions/cache/restore@v4 + with: + path: | + llvm-host-build + llvm-build + cmark-host-build + cmark-ios-build + key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.toolchain-versions.outputs.value }}-${{ env.SWIFT_LLVM_REF }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '3' }} + + # Swift's own build is the longer half and is cached separately, because it + # is saved *after* the LLVM half has been built and Swift's has too. Its key + # deliberately ignores `cache_salt`: that salt exists to discard an LLVM tree + # when the LLVM source changes, and the Swift tree is a function of the Swift + # refs and the SDK alone — discarding it too would throw away the 45-minute + # half for a change to the other one. The `-vN` suffix is this half's own + # salt: bump it when the Swift configure or build recipe changes, because a + # stored tree can never be replaced under the key it was stored with. + - name: Restore the Swift build + if: ${{ inputs.with_swift }} + id: swift-cache + uses: actions/cache/restore@v4 + with: + path: swift-build + key: ${{ runner.os }}-${{ runner.arch }}-swift-${{ env.SWIFT_REF }}-${{ env.SWIFT_LLVM_REF }}-${{ steps.toolchain-versions.outputs.value }}-${{ env.IOS_DEPLOYMENT_TARGET }}-v2 + - name: Build host TableGen tools shell: bash run: | set -euxo pipefail + # All four, not just the TableGen pair: `llvm-ar` is what the packaging + # step merges the archives with, and a restored tree that has the two + # TableGen tools but not it used to pass this check and fail three seconds + # into staging ("llvm-host-build/bin/llvm-ar: No such file or directory"). + if [ -x llvm-host-build/bin/llvm-tblgen ] \ + && [ -x llvm-host-build/bin/clang-tblgen ] \ + && [ -x llvm-host-build/bin/llvm-ar ] \ + && [ -x llvm-host-build/bin/llvm-nm ]; then + echo "host tools already built" + mkdir -p llvm-build/bin + for tool in llvm-host-build/bin/*; do + ln -sf "$PWD/$tool" "llvm-build/bin/$(basename "$tool")" + done + exit 0 + fi cmake -S llvm-project/llvm -B llvm-host-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_PROJECTS="clang" \ @@ -46,12 +269,107 @@ jobs: -DLLVM_ENABLE_ZSTD=OFF \ -DLLVM_ENABLE_LIBXML2=OFF \ -DLLVM_ENABLE_CURL=OFF - ninja -C llvm-host-build llvm-tblgen clang-tblgen + ninja -C llvm-host-build llvm-tblgen clang-tblgen llvm-ar llvm-nm + + # Swift's CMake resolves its native tools through LLVM_TOOLS_BINARY_DIR, + # which for the iOS build is a directory with no tools in it + # (LLVM_BUILD_TOOLS=OFF). Mirroring the host tools there means every + # path it might take finds them. + mkdir -p llvm-build/bin + for tool in llvm-host-build/bin/*; do + ln -sf "$PWD/$tool" "llvm-build/bin/$(basename "$tool")" + done + + - name: Build native cmark + if: ${{ inputs.with_swift }} + shell: bash + run: | + set -euxo pipefail + if [ -x cmark-host-build/src/cmark-gfm ]; then + echo "native cmark already cached" + exit 0 + fi + cmake -S swift-cmark -B cmark-host-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMARK_TESTS=OFF + cmake --build cmark-host-build --target cmark-gfm + mkdir -p cmark-host-build/src + ln -sf cmark-host-build/src/cmark-gfm cmark-host-build/src/cmark + + # Gated exactly like the Swift checkouts and the Swift stages: it is only + # needed when the frontend is, and on a PR/push run the cmark sources are not + # checked out at all — which is how this step failed its first two runs. + - name: Build cmark for iPhoneOS + if: ${{ inputs.with_swift }} + shell: bash + run: | + set -euxo pipefail + if [ -f cmark-ios-build/src/libcmark-gfm.a ]; then + echo "iOS cmark already cached" + exit 0 + fi + SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + # CMAKE_SYSTEM_NAME=Darwin, not iOS, for the same reason the Swift + # configure uses it: on iOS CMake refuses \`install(TARGETS ...)\` for an + # executable without a BUNDLE destination, and cmark installs its + # command-line tool. The target is still iOS — sysroot, architecture and + # compiler target below say so — and nothing is installed here anyway. + # Static archives are cmark's default; naming CMARK_TESTS/CMARK_SHARED here + # only produces "manually-specified variables were not used" warnings. + cmake -S swift-cmark -B cmark-ios-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_SYSTEM_NAME=Darwin \ + -DCMAKE_OSX_SYSROOT="$SDK" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ + -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET + # cmark-gfm-extensions is a separate archive and swift's Markup references + # symbols from both. + # The *library* targets are libcmark-gfm and libcmark-gfm-extensions; the + # bare names are the command-line tool, which only the host build needs. + ninja -C cmark-ios-build libcmark-gfm libcmark-gfm-extensions + find cmark-ios-build -name 'libcmark-gfm*.a' + + # Skipping the configure below is only sound if the restored tree really is + # the configured-and-built one it was saved from. Checking is cheap, it has to + # happen *before* the step it gates (a step's output does not exist yet when + # the condition above it is evaluated), and the alternative is a confusing + # failure — no build.ninja — where the old workflow had a slow-but-working path. + - name: Verify the restored LLVM build tree + if: ${{ steps.llvm-cache.outputs.cache-hit == 'true' }} + id: llvm-tree + shell: bash + run: | + set -euxo pipefail + MISSING="" + for file in llvm-build/build.ninja llvm-build/CMakeCache.txt \ + llvm-build/include/llvm/Config/llvm-config.h \ + llvm-host-build/bin/llvm-tblgen llvm-host-build/bin/clang-tblgen; do + [ -e "$file" ] || MISSING="$MISSING $file" + done + OBJECTS="$(find llvm-build -name '*.o' -print -quit)" + if [ -n "$MISSING" ] || [ -z "$OBJECTS" ]; then + echo "::warning::restored LLVM tree is not usable (missing:$MISSING, objects:$([ -n "$OBJECTS" ] && echo present || echo none)); configuring from scratch" + echo "reusable=false" >> "$GITHUB_OUTPUT" + else + echo "restored LLVM tree is configured and built; the configure below is skipped" + echo "reusable=true" >> "$GITHUB_OUTPUT" + fi - name: Configure LLVM for iPhoneOS arm64 + if: ${{ steps.llvm-cache.outputs.cache-hit != 'true' || steps.llvm-tree.outputs.reusable != 'true' }} shell: bash run: | set -euxo pipefail + # CLANG_BUILD_TOOLS is ON on purpose, and nothing builds that target. + # Swift's CMake assumes the unified build it is configured against has a + # `clang` executable target — lib/Tooling/libSwiftScan/CMakeLists.txt + # does `add_dependencies(libSwiftScan clang …)` — and generation fails + # outright when the target does not exist ("The dependency target + # \"clang\" of target \"libSwiftScan\" does not exist"), even though + # neither target is ever built here. Defining it is cheaper than + # disabling Swift's subdirectories one generation error at a time. SDK="$(xcrun --sdk iphoneos --show-sdk-path)" cmake -S llvm-project/llvm -B llvm-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ @@ -59,11 +377,11 @@ jobs: -DCMAKE_MACOSX_BUNDLE=OFF \ -DCMAKE_OSX_SYSROOT="$SDK" \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=17.0 \ - -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios17.0 \ - -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios17.0 \ - -DLLVM_HOST_TRIPLE=arm64-apple-ios17.0 \ - -DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-ios17.0 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ + -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_HOST_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ -DLLVM_NATIVE_TOOL_DIR="$PWD/llvm-host-build/bin" \ -DLLVM_ENABLE_PROJECTS="clang;lld" \ -DLLVM_TARGETS_TO_BUILD=AArch64 \ @@ -80,11 +398,30 @@ jobs: -DLLVM_ENABLE_CURL=OFF \ -DLLVM_ENABLE_RTTI=ON \ -DCLANG_INCLUDE_TESTS=OFF \ - -DCLANG_BUILD_TOOLS=OFF \ + -DCLANG_BUILD_TOOLS=ON \ -DLLD_INCLUDE_TESTS=OFF \ -DLLD_BUILD_TOOLS=OFF + # Cheap by construction when the tree above was restored: the sources are all + # older than the objects, so ninja has nothing to do. It is also where the + # cache is proven — a restore that does not come back built shows up here as + # a 2601-edge rebuild in the log, not as a mystery. + # + # The target list is not "everything the app might use" but the closure the + # bundle has to satisfy, because the smoke link force-loads every archive: + # each member's undefined symbols must resolve inside the bundle. Two of these + # are invisible to clang's own libraries and only Swift references them, which + # is why they were missing: + # * LLVMOrcJIT — swift's JIT (SwiftMaterializationUnit.cpp) and LLJIT; + # * clangDependencyScanning — ClangImporter's explicit-module dependency + # scanning (ModuleDepCollector/ScanAndUpdateArgs). + # Their transitive dependencies (JITLink, ExecutionEngine, RuntimeDyld, + # OrcShared/OrcTargetProcess, clangTooling, clangCAS, …) come with them. + # Adding a target here changes the delivered bundle, so bump `cache_salt` + # when you do: the cache key cannot see this list. - name: Build Clang and Mach-O LLD libraries + id: llvm-build + if: ${{ !inputs.configure_only }} run: | set -euxo pipefail ninja -C llvm-build \ @@ -98,20 +435,217 @@ jobs: clangAST \ clangLex \ clangBasic \ + clangDependencyScanning \ lldMachO \ - lldCommon + lldCommon \ + LLVMOrcJIT + + # The step this workflow exists for. Everything above is shared with the + # LLVM-only bundle; everything below only happens when Swift is asked for. + # Saved here, before the step most likely to fail, so that a failed Swift + # configure does not cost another LLVM cross-build. + # + # Gated on the build having *succeeded*, not on `always()`: saving a tree + # ninja never finished stores a partial one under a key that can never be + # rewritten, and every later run then "hits" it and rebuilds anyway. + - name: Save the LLVM builds + if: ${{ steps.llvm-build.outcome == 'success' && steps.llvm-cache.outputs.cache-hit != 'true' }} + # A refused save must not fail a build that succeeded: an entry whose key + # already exists cannot be reserved, and the other workflow that stored it + # stored the same tree. The cache is an optimisation; the build is the work. + continue-on-error: true + uses: actions/cache/save@v4 + with: + key: ${{ steps.llvm-cache.outputs.cache-primary-key }} + path: | + llvm-host-build + llvm-build + cmark-host-build + cmark-ios-build + + # Same contract as the LLVM tree check above, and the same reason: the + # configure is skipped when the tree came back built, so a tree that came back + # *unbuilt* would fail on the missing archive instead of just rebuilding. + - name: Verify the restored Swift build tree + if: ${{ inputs.with_swift && steps.swift-cache.outputs.cache-hit == 'true' }} + id: swift-tree + shell: bash + run: | + set -euxo pipefail + MISSING="" + for file in swift-build/build.ninja swift-build/CMakeCache.txt \ + swift-build/lib/libswiftFrontendTool.a; do + [ -e "$file" ] || MISSING="$MISSING $file" + done + OBJECTS="$(find swift-build -name '*.o' -print -quit)" + if [ -n "$MISSING" ] || [ -z "$OBJECTS" ]; then + echo "::warning::restored Swift tree is not usable (missing:$MISSING, objects:$([ -n "$OBJECTS" ] && echo present || echo none)); configuring from scratch" + echo "reusable=false" >> "$GITHUB_OUTPUT" + else + echo "restored Swift tree is configured and built; the configure below is skipped" + echo "reusable=true" >> "$GITHUB_OUTPUT" + fi + + - name: Configure the Swift frontend for iPhoneOS + if: ${{ inputs.with_swift && (steps.swift-cache.outputs.cache-hit != 'true' || steps.swift-tree.outputs.reusable != 'true') }} + shell: bash + run: | + set -euxo pipefail + SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + HOST_SWIFT_BIN="$(dirname "$(xcrun -f swiftc)")" + + # CMAKE_SYSTEM_NAME=Darwin, not iOS, and that is deliberate. Swift's own + # iOS builds are configured exactly this way (build-script-impl passes + # the sysroot and SWIFT_HOST_VARIANT_SDK and never sets the system + # name), because its CMake branches on `CMAKE_SYSTEM_NAME STREQUAL + # "Darwin"` for Apple: + # * lib/Basic/CMakeLists.txt does `find_package(UUID REQUIRED)` on + # anything else — a Linux dependency for Swift-on-Linux builds; + # * SwiftSharedCMakeConfig's Libdispatch.cmake errors out with + # "libdispatch requires a newer clang compiler" unless it is Darwin + # (on Darwin, dispatch comes from the SDK). + # The target is still iOS: the sysroot, the architectures, the + # deployment target and the compiler target triple below say so. + cmake -S swift -B swift-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_SYSTEM_NAME=Darwin \ + -DCMAKE_MACOSX_BUNDLE=OFF \ + -DCMAKE_OSX_SYSROOT="$SDK" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ + -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_Swift_COMPILER="$(xcrun -f swiftc)" \ + -DCMAKE_Swift_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_DIR="$PWD/llvm-build/lib/cmake/llvm" \ + -DClang_DIR="$PWD/llvm-build/lib/cmake/clang" \ + -DLLVM_HOST_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_TABLEGEN="$PWD/llvm-host-build/bin/llvm-tblgen" \ + -DCLANG_TABLEGEN="$PWD/llvm-host-build/bin/clang-tblgen" \ + -DSWIFT_NATIVE_LLVM_TOOLS_PATH="$PWD/llvm-host-build/bin" \ + -DSWIFT_NATIVE_CLANG_TOOLS_PATH="$PWD/llvm-host-build/bin" \ + -DSWIFT_NATIVE_SWIFT_TOOLS_PATH="$HOST_SWIFT_BIN" \ + -DSWIFT_PATH_TO_CMARK_BUILD="$PWD/cmark-host-build" \ + -DSWIFT_PATH_TO_CMARK_SOURCE="$PWD/swift-cmark" \ + -DSWIFT_PATH_TO_STRING_PROCESSING_SOURCE="$PWD/swift-experimental-string-processing" \ + -DSWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE="$PWD/swift-syntax" \ + -DSWIFT_HOST_VARIANT=iphoneos \ + -DSWIFT_HOST_VARIANT_SDK=IOS \ + -DSWIFT_HOST_VARIANT_ARCH=arm64 \ + -DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS="$IOS_DEPLOYMENT_TARGET" \ + -DSWIFT_BUILD_DYNAMIC_STDLIB=OFF \ + -DSWIFT_BUILD_STATIC_STDLIB=OFF \ + -DSWIFT_BUILD_DYNAMIC_SDK_OVERLAY=OFF \ + -DSWIFT_BUILD_STATIC_SDK_OVERLAY=OFF \ + -DSWIFT_BUILD_STDLIB_EXTRA_TOOLCHAIN_CONTENT=OFF \ + -DSWIFT_BUILD_SOURCEKIT=OFF \ + -DSWIFT_BUILD_SWIFT_SYNTAX=OFF \ + -DSWIFT_BUILD_REMOTE_MIRROR=OFF \ + -DSWIFT_BUILD_PERF_TESTSUITE=OFF \ + -DSWIFT_INCLUDE_TESTS=OFF \ + -DSWIFT_INCLUDE_DOCS=OFF \ + -DSWIFT_INSTALL_COMPONENTS= + + # Proof that the configure produced what the app will link, before the + # hours of building. + test -f swift-build/lib/swift/host/libswiftFrontendTool.a \ + || test -f swift-build/lib/libswiftFrontendTool.a \ + || echo "::warning::swiftFrontendTool archive not found where expected; listing swift-build/lib" + ls -la swift-build/lib | head -40 || true + + - name: Build the Swift frontend libraries + id: swift-build + if: ${{ inputs.with_swift && !inputs.configure_only }} + shell: bash + run: | + set -euxo pipefail + # The static library holding `swift::performFrontend`. Its transitive + # dependencies (swiftAST, swiftSema, the SIL and IRGen libraries, the + # Clang importer and Swift's own Clang/LLVM) come along with it; the + # swift/driver executable libraries are deliberately not built, because + # nothing on the device runs a driver. + ninja -C swift-build swiftFrontendTool + + - name: Save the Swift build + if: ${{ inputs.with_swift && !inputs.configure_only && steps.swift-build.outcome == 'success' && steps.swift-cache.outputs.cache-hit != 'true' }} + continue-on-error: true + uses: actions/cache/save@v4 + with: + key: ${{ steps.swift-cache.outputs.cache-primary-key }} + path: swift-build + + - name: Verify the Swift frontend + if: ${{ inputs.with_swift && !inputs.configure_only }} + shell: bash + run: | + set -euxo pipefail + find swift-build/lib -maxdepth 3 -type f -name 'libswift*.a' | sort > /tmp/swift-archives.txt + test -s /tmp/swift-archives.txt || { echo "::error::no Swift libraries were produced"; exit 1; } + echo "Swift archives: $(wc -l < /tmp/swift-archives.txt)" + + # Every archive must be arm64: a bundle that mixes architectures links + # and then misbehaves, which is worse than one that fails here. + while IFS= read -r library; do + archs="$(lipo -archs "$library" 2>/dev/null || true)" + case " $archs " in + *" arm64 "*) ;; + *) echo "::error::$library is not arm64 ($archs)"; exit 1 ;; + esac + done < /tmp/swift-archives.txt + echo "every Swift archive is arm64" + + # `performFrontend` lives in libswiftFrontendTool.a (FrontendTool.cpp). + # The search is reported, not enforced: the gate that matters is the + # link below, which fails if the symbol is missing for real. An earlier + # version of this step failed the job because `nm` did not print the + # symbol for a library that had just been built. + FRONTEND_ARCHIVE="$PWD/swift-build/lib/libswiftFrontendTool.a" + test -f "$FRONTEND_ARCHIVE" || { echo "::error::libswiftFrontendTool.a was not produced"; exit 1; } + NM="" + if [ -x llvm-host-build/bin/llvm-nm ]; then NM="$PWD/llvm-host-build/bin/llvm-nm"; else NM="nm"; fi + echo "--- $NM -gU $FRONTEND_ARCHIVE | grep -c performFrontend" + SYMBOLS="$("$NM" -gU "$FRONTEND_ARCHIVE" 2>&1 || true)" + if printf '%s\n' "$SYMBOLS" | grep -q performFrontend; then + echo "performFrontend is in libswiftFrontendTool.a" + else + echo "::warning::no performFrontend symbol in libswiftFrontendTool.a; the link below decides" + printf '%s\n' "$SYMBOLS" | head -20 + fi - name: Compile-check XForge bridge against LLVM headers + if: ${{ !inputs.configure_only }} shell: bash run: | set -euxo pipefail SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + # Accumulated in a *string*, not an array. `shell: bash` on a macOS runner + # is macOS's bash 3.2, where `set -u` plus an empty `"${ARRAY[@]}"` is an + # unbound-variable error — and the arrays here are empty on every run that + # does not build the Swift half, which is every pull request and push run. + # The step died on `EXTRA_DEFINES[@]: unbound variable` before compiling + # anything, so the half of CI that runs unattended was red for a reason + # that had nothing to do with the bridge. + EXTRA_FLAGS="" + if [ "${{ inputs.with_swift }}" = "true" ]; then + # Exercises the Swift half of the bridge: with the define and the + # headers present, NativeToolchainBridge.mm compiles the code path + # that calls swift::performFrontend instead of the stub. + EXTRA_FLAGS="$EXTRA_FLAGS -DXFORGE_HAS_SWIFT_FRONTEND=1 -I swift/include" + if [ -d swift-build/include ]; then + EXTRA_FLAGS="$EXTRA_FLAGS -I swift-build/include" + fi + if [ -d swift-build/tools/swift/include ]; then + EXTRA_FLAGS="$EXTRA_FLAGS -I swift-build/tools/swift/include" + fi + fi xcrun --sdk iphoneos clang++ \ -std=c++20 \ -arch arm64 \ - -miphoneos-version-min=17.0 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ -isysroot "$SDK" \ -DXFORGE_HAS_LLVM=1 \ + $EXTRA_FLAGS \ -I llvm-project/llvm/include \ -I llvm-build/include \ -I llvm-project/clang/include \ @@ -121,29 +655,160 @@ jobs: -c App/NativeToolchain/NativeToolchainBridge.mm \ -o /tmp/NativeToolchainBridge.o + # The iteration substrate. Caches were the wrong tool for this: the entries + # either never stored (a 6-second "save" of a multi-gigabyte tree) or were + # evicted, so a workflow that restored them found nothing and failed steps this + # one passes. Artifacts are not evicted, and this is only what a check needs — + # the static archives and the generated headers — not the object files and + # CMake metadata that make up most of the trees. + # + # Uploaded with `always()` so a failure *after* the builds still publishes them: + # that is exactly the case the fast loop exists to iterate on. + - name: Package the build trees + if: ${{ always() && !inputs.configure_only }} + shell: bash + run: | + set -euxo pipefail + mkdir -p build-trees + ITEMS=() + for item in \ + llvm-build/lib \ + swift-build/lib \ + cmark-ios-build \ + llvm-build/include \ + llvm-build/tools/clang/include \ + swift-build/include \ + swift-build/tools/swift/include; do + [ -d "$item" ] && ITEMS+=("$item") + done + if [ "${#ITEMS[@]}" -eq 0 ]; then + echo "nothing to package" + exit 0 + fi + tar -cf build-trees/build-trees.tar "${ITEMS[@]}" + du -sh build-trees/build-trees.tar + + - name: Upload the build trees + if: ${{ always() && !inputs.configure_only }} + uses: actions/upload-artifact@v4 + with: + name: XForgeBuildTrees + path: build-trees/build-trees.tar + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + - name: Stage native toolchain SDK + id: stage + if: ${{ !inputs.configure_only }} shell: bash run: | set -euxo pipefail - mkdir -p dist/include dist/lib - cp -R llvm-project/llvm/include/llvm dist/include/ - cp -R llvm-build/include/llvm dist/include/ - cp -R llvm-project/clang/include/clang dist/include/ - if [ -d llvm-build/tools/clang/include/clang ]; then - cp -R llvm-build/tools/clang/include/clang dist/include/ + # Two header roots, searched in this order, and they must not be merged: + # the build tree generates a *file* at `swift/bridging` (the C++ interop + # header, included as ) while the sources have a + # *directory* of the same name (`include/swift/Bridging/`, holding + # ASTGen.h and MacroEvaluation.h). On a case-insensitive filesystem that + # is a single path, so copying the build tree over the source tree fails + # with "cp: .../swift/bridging: Is a directory" and stages nothing. + # + # `-L` is load-bearing too: the build tree mirrors headers as symlinks + # into this checkout, and a bundle that ships those links is dead + # anywhere but this runner. + mkdir -p dist/include-generated dist/include dist/lib + stage_headers() { cp -R -L "$1" "$2"; } + + # Every top-level header root the sources ship, not just the one named + # after the project. `llvm/include/llvm-c/DataTypes.h` is included from + # `llvm/include/llvm/Support/DataTypes.h`, and staging only `llvm/` left it + # out of the bundle: the app build failed on it while the bridge + # compile-check passed, because that check compiles against the source + # tree rather than against what is shipped. Same for `clang-c`, and for + # anything a future revision grows. + for source_root in llvm-project/llvm/include \ + llvm-project/clang/include \ + llvm-project/lld/include; do + for dir in "$source_root"/*/; do + stage_headers "${dir%/}" dist/include/ + done + done + + # The generated headers, likewise per top-level directory: the build tree + # writes `llvm/Config/…` and `clang/…`, and `llvm-c` if a revision ever + # generates one. + for generated_root in llvm-build/include llvm-build/tools/clang/include; do + [ -d "$generated_root" ] || continue + for dir in "$generated_root"/*/; do + stage_headers "${dir%/}" dist/include-generated/ + done + done + + if [ "${{ inputs.with_swift }}" = "true" ]; then + stage_headers swift/include/swift dist/include/ + for swift_headers in swift-build/include/swift swift-build/tools/swift/include/swift; do + if [ -d "$swift_headers" ]; then + stage_headers "$swift_headers" dist/include-generated/ + fi + done + fi + + for required in dist/include/llvm dist/include/llvm-c dist/include/clang dist/include/lld; do + [ -d "$required" ] || { echo "::error::missing staged header root $required"; exit 1; } + done + + # A symlink here would point into the runner's checkout. + if [ -n "$(find dist/include dist/include-generated -type l -print -quit)" ]; then + echo "::error::staged headers still contain symlinks:" + find dist/include dist/include-generated -type l -print + exit 1 fi - cp -R llvm-project/lld/include/lld dist/include/ - # Flatten the static dependency graph into one archive. XForge then - # links a single library instead of depending on LLVM's internal - # archive ordering. - find llvm-build/lib -maxdepth 1 -type f -name '*.a' -print | sort > /tmp/llvm-archives.txt - test -s /tmp/llvm-archives.txt + # Archives are staged *unmerged*, deliberately. + # + # This used to flatten every library into one archive with + # `libtool -static`, which was wrong: libtool de-duplicates members by name + # ("duplicate member name 'X86.cpp.o' from libclangCodeGen.a and …"), and a + # dropped member is a silently missing object. That is what produced + # undefined JIT symbols referenced from an object that was demonstrably in + # the archive. Swift's libraries collide the same way. + # + # The app already links a list — `prepare-xcode.sh` globs `lib/*.a` into + # XFORGE_NATIVE_LDFLAGS — so the merge only ever saved a line of shell. + find llvm-build/lib -maxdepth 1 -type f -name '*.a' -print | sort > /tmp/archives.txt + if [ -d cmark-ios-build ]; then + find cmark-ios-build -type f -name 'libcmark-gfm*.a' -print | sort >> /tmp/archives.txt + fi + if [ "${{ inputs.with_swift }}" = "true" ]; then + find swift-build/lib -maxdepth 3 -type f -name '*.a' -print | sort >> /tmp/archives.txt + fi + test -s /tmp/archives.txt ARCHIVES=() - while IFS= read -r archive; do - ARCHIVES+=("$archive") - done < /tmp/llvm-archives.txt - /usr/bin/libtool -static -o dist/lib/libXForgeNativeToolchain.a "${ARCHIVES[@]}" + while IFS= read -r archive; do ARCHIVES+=("$archive"); done < /tmp/archives.txt + cp /tmp/archives.txt dist/lib/XForgeToolchain.libraries.txt + + # One archive, merged with GNU MRI rather than `libtool -static`. + # libtool de-duplicates members by name — the log warns "duplicate member + # name 'X86.cpp.o' from libclangCodeGen.a and …" for clang's libraries and + # the same happens across Swift's — and a dropped member is a silently + # missing object, which is how the JIT symbols came out undefined while + # their object was demonstrably in the archive. `addlib` copies every + # member, duplicates included. + AR="$PWD/llvm-host-build/bin/llvm-ar" + MRI="$(mktemp)" + echo "create dist/lib/libXForgeNativeToolchain.a" > "$MRI" + for archive in "${ARCHIVES[@]}"; do echo "addlib $archive" >> "$MRI"; done + echo "save" >> "$MRI" + echo "end" >> "$MRI" + "$AR" -M < "$MRI" + + # The guard: a merge that loses members is an error here, not a link + # failure two steps later that looks like a missing library. + MERGED=$("$AR" t dist/lib/libXForgeNativeToolchain.a | wc -l | tr -d ' ') + EXPECTED=$(for archive in "${ARCHIVES[@]}"; do "$AR" t "$archive"; done | wc -l | tr -d ' ') + echo "merged members: $MERGED, sources: $EXPECTED in ${#ARCHIVES[@]} archives" + [ "$MERGED" -ge "$EXPECTED" ] \ + || { echo "::error::the merge dropped members ($MERGED < $EXPECTED)"; exit 1; } + ls -la dist/lib/libXForgeNativeToolchain.a cat > /tmp/native_link_smoke.mm <<'EOF' #include "NativeToolchainBridge.h" @@ -153,10 +818,33 @@ jobs: EOF SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + + # Compile the bridge against the *staged* roots and link the result. This + # is the check that the shipped bundle is self-contained — the earlier + # compile-check compiles against the source tree, which says nothing about + # what was staged. A header missing here is a header missing from every + # app that links this bundle, and finding that in a 20-minute app build + # instead of here is how `llvm-c/` stayed missing. + BRIDGE_DEFINES=(-DXFORGE_HAS_LLVM=1) + if [ "${{ inputs.with_swift }}" = "true" ]; then + BRIDGE_DEFINES+=(-DXFORGE_HAS_SWIFT_FRONTEND=1) + fi + xcrun --sdk iphoneos clang++ \ + -std=c++20 \ + -arch arm64 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ + -isysroot "$SDK" \ + "${BRIDGE_DEFINES[@]}" \ + -I dist/include \ + -I dist/include-generated \ + -I App/NativeToolchain \ + -c App/NativeToolchain/NativeToolchainBridge.mm \ + -o /tmp/StageBundleBridge.o + xcrun --sdk iphoneos clang++ \ -std=c++20 \ -arch arm64 \ - -miphoneos-version-min=17.0 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ -isysroot "$SDK" \ -I App/NativeToolchain \ -c /tmp/native_link_smoke.mm \ @@ -164,32 +852,118 @@ jobs: xcrun --sdk iphoneos clang++ \ -arch arm64 \ - -miphoneos-version-min=17.0 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ -isysroot "$SDK" \ /tmp/native_link_smoke.o \ - /tmp/NativeToolchainBridge.o \ - dist/lib/libXForgeNativeToolchain.a \ + /tmp/StageBundleBridge.o \ + $(for library in dist/lib/*.a; do printf ' -Wl,-force_load,%s' "$library"; done) \ -lc++ -lz -liconv -lsqlite3 \ -framework Foundation \ - -o /tmp/XForgeNativeToolchainSmoke + -o /tmp/XForgeNativeToolchainSmoke \ + 2> /tmp/native-link-errors.txt || { + echo "::error::linking the toolchain into an app-shaped target failed" + # ld prints one ` "", referenced from:` line per undefined + # symbol. `tail -60` showed the last of them and hid the rest, which + # is how a whole missing library stayed a mystery for a run: print + # every one, deduplicated, so the next iteration is not another guess. + grep 'referenced from:' /tmp/native-link-errors.txt \ + | sed 's/, referenced from:$//' | sort -u || true + tail -20 /tmp/native-link-errors.txt + exit 1 + } file /tmp/XForgeNativeToolchainSmoke - file /tmp/NativeToolchainBridge.o - cp /tmp/NativeToolchainBridge.o dist/ + file /tmp/StageBundleBridge.o + cp /tmp/StageBundleBridge.o dist/ + # The manifest is what the consumers read, so everything they have to + # agree with is in it: `bundle_tag` is the release this can be fetched + # from, `xcode`/`ios_sdk` are the SDK the archives were compiled against + # (a different SDK is a different bundle), and `swift_ref` is the Swift + # version the frontend speaks — the app's own Swift must match it, because + # the frontend reads the SDK's standard-library modules. cat > dist/manifest.txt </dev/null 2>&1; then + echo "release $TAG already exists; replacing its asset" + else + gh release create "$TAG" --repo "$GITHUB_REPOSITORY" \ + --title "Native toolchain $TAG" \ + --notes-file dist/manifest.txt + fi + gh release upload "$TAG" "$ARCHIVE" --repo "$GITHUB_REPOSITORY" --clobber + + # The point of publishing is that someone else can consume it, so assert + # what the consumer will see rather than what we just sent. + gh release view "$TAG" --repo "$GITHUB_REPOSITORY" \ + --json tagName,assets \ + --jq '.assets[] | select(.name == "XForgeNativeToolchain-arm64-ios.tar.gz") | "\(.name): \(.size) bytes"' + { + echo '### Published toolchain bundle' + echo + echo "- tag: \`$TAG\`" + echo "- asset: \`$ARCHIVE\` ($((bytes / 1048576)) MB)" + echo "- install in an app build: \`with_toolchain: true\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index ee46c35..76f5d84 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,17 @@ XForge.xcodeproj/ # Where build-ipa.yml clones package checkouts (`clonedSourcePackagesDirPath`). .spm/ +# The native toolchain bundle. Installing it (`make toolchain-release`) unpacks +# hundreds of megabytes of LLVM/Swift archives into the checkout; they are +# reproducible from the release the manifest names, and committing them is never +# what anyone means to do. The directory and the gcc-style placeholder stay, so a +# clone has somewhere to install into. +Vendor/NativeToolchain/* +!Vendor/NativeToolchain/.gitkeep + +# Where build-ipa.yml unpacks a fetched toolchain bundle. +.toolchain-artifact/ + # macOS .DS_Store @@ -19,20 +30,8 @@ XForge.xcodeproj/ .vscode/* !.vscode/extensions.json -# Embedded ish-arm64 engine -# (Vendor/ish-arm64 is a submodule/gitlink, so it is tracked; only its build -# output below is ignored.) -# Static libraries produced by EmbeddedLinux/build-ish-core.sh. -Vendor/ish-arm64-build/ # Generated by XcodeGen from project.yml's `info.properties` — the spec is the # single source of truth for the app's Info.plist. Support/Info.plist -# The bundled rootfs is a build product, not a source file: CI downloads it from -# the pinned `rootfs-v1` release and `make rootfs` builds it locally, so a clean -# checkout compiles and bundles one either way. Nothing here is tracked — an -# earlier revision committed a 4 MB tarball, and it stayed in the bundle long -# after the app stopped reading it (the IPA shipped both roots). -Support/Resources/* -!Support/Resources/.gitkeep diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index ab22d80..0000000 --- a/.gitmodules +++ /dev/null @@ -1,5 +0,0 @@ -[submodule "Vendor/ish-arm64"] - path = Vendor/ish-arm64 - url = https://github.com/OpenMinis/ish-arm64.git - branch = master - shallow = true diff --git a/App/Build/BuildManager.swift b/App/Build/BuildManager.swift index 5d0a5b7..d4b06ab 100644 --- a/App/Build/BuildManager.swift +++ b/App/Build/BuildManager.swift @@ -2,8 +2,8 @@ import Foundation import Combine /// Orchestrates the on-device IPA build pipeline. Exposes a `PipelineSnapshot` that -/// drives the GUI's stage UI. Toolchain checks, dependency resolution, compilation, -/// packaging, and SDK tooling all run inside Alpine via `BuildExecutor`. +/// drives the GUI's stage UI. Every stage runs in this process, through the +/// toolchain linked into the app (`NativeBuildExecutor`). @MainActor final class BuildManager: ObservableObject { let project: Project @@ -76,33 +76,27 @@ final class BuildManager: ObservableObject { } catch { markFailed(.provision, error) } } + /// The Darwin SDK is installed into the app's own container. + /// + /// It used to be installed by `swift sdk install` inside the guest; the SDK + /// bundle is the same artifact either way, so this only had to stop going + /// through a guest to get it. private func ensureSDK() async { let executor = makeExecutor() markRunning(.sdk) do { - // Ask the guest before the network. Every published rootfs ships a - // darwin SDK, and the release lookup below is a network call — so a - // device that is offline (or rate-limited by the GitHub API) failed a - // build at this stage for an SDK it already had. The probe mirrors - // ToolchainManager's: `swift sdk list` is a program, so its output goes - // to the pipe rather than /dev/null, which this engine kills. - let vm = XForgeEnvironment.makeVM() - if !vm.isBooted { try await vm.boot() } - let installed = try await vm.run( - "command -v swift >/dev/null 2>&1 && swift sdk list 2>&1 | grep -qi darwin", - environment: nil - ) { _ in } - if installed == 0 { - appendConsole("▸ darwin SDK: already installed in the guest") + if NativeSDK.isInstalled { + appendConsole("▸ darwin SDK: already installed") markSucceeded(.sdk) return } - // Resolve the published asset first: the darwin SDK lives under its own - // `darwin-sdk-*` release series, so `releases/latest/download/…` 404s. let url = try await XForgeReleases.darwinSDKURL() appendConsole("▸ darwin SDK: \(url.lastPathComponent)") try await executor.installSDK(from: .hostedRemote(url)) + guard NativeSDK.isInstalled else { + throw NativeSDKError.missingBundle + } markSucceeded(.sdk) } catch { markFailed(.sdk, error) } } @@ -110,21 +104,22 @@ final class BuildManager: ObservableObject { /// Make sure the project exists in the guest and record the app identity the /// build should produce. Previously this stage only printed a line and /// reported success unconditionally, even after provision had failed. + /// Record the app identity the build should produce. + /// + /// A project is a directory in the app container now, so there is nothing to + /// create inside a guest and nothing to copy into one; what remains is the + /// check that the directory is really there and the metadata dump that makes a + /// failed build reproducible. private func configure() async { markRunning(.configure) do { - let vm = XForgeEnvironment.makeVM() - if !vm.isBooted { try await vm.boot() } - guard project.hasSafeRootPath else { throw ProjectValidationError.unsafePath } - let dir = project.rootPath - let mkdir = try await vm.run( - "mkdir -p \(GuestShell.quote(dir))", - environment: nil - ) { _ in } - guard mkdir == 0 else { throw BuildError.stepFailed("mkdir \(dir)", mkdir) } + let root = project.rootURL + guard FileManager.default.fileExists(atPath: root.path) else { + throw NativeBuildError.missingProjectDirectory(root) + } let metadata = BuildMetadata( bundleIdentifier: appInfo.bundleIdentifier, @@ -134,14 +129,14 @@ final class BuildManager: ObservableObject { minimumOSVersion: appInfo.minimumOSVersion, configuration: configuration.rawValue ) - let temp = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-app-\(UUID().uuidString).json") - try JSONEncoder().encode(metadata).write(to: temp, options: .atomic) - defer { try? FileManager.default.removeItem(at: temp) } - try await vm.copyIn(hostURL: temp, to: "\(dir)/xforge-app.json") + try FileManager.default.createDirectory( + at: root.appendingPathComponent(".xforge-build", isDirectory: true), + withIntermediateDirectories: true + ) + let record = root.appendingPathComponent(".xforge-build/build.json") + try JSONEncoder().encode(metadata).write(to: record, options: .atomic) appendConsole("▸ bundle \(appInfo.bundleIdentifier) · \(configuration.rawValue)") - appendConsole("▸ wrote app identity to \(dir)/xforge-app.json") markSucceeded(.configure) } catch { markFailed(.configure, error) @@ -187,8 +182,8 @@ final class BuildManager: ObservableObject { } catch { markFailed(.compile, error) } } - /// Packaging belongs to xtool inside Alpine. The host receives only the final - /// IPA for export; it never runs build or SDK tooling. + /// The executor packages the IPA as the last step of compiling, so this stage + /// only has to accept the result. private func package() async { guard let compiled = compiledURL, compiled.pathExtension.lowercased() == "ipa" else { @@ -197,7 +192,7 @@ final class BuildManager: ObservableObject { } markRunning(.package) snapshot.lastIpa = compiled - appendConsole("✓ .ipa packaged by xtool inside Alpine") + appendConsole("✓ .ipa packaged by the native toolchain") markSucceeded(.package) } @@ -297,9 +292,9 @@ enum BuildError: LocalizedError { case .noArtifact: return "The build did not produce an artifact." case .notProvisioned: - return "The Alpine build toolchain is not installed. Run `sh /root/install-toolchain.sh all` in Terminal." + return "The native compiler toolchain is not linked into this build of XForge." case .stepFailed(let step, let status): - return "\(step) failed inside the embedded Linux (exit \(status))." + return "\(step) failed (exit \(status))." } } } diff --git a/App/Build/EmbeddedLinuxExecutor.swift b/App/Build/EmbeddedLinuxExecutor.swift deleted file mode 100644 index ee695da..0000000 --- a/App/Build/EmbeddedLinuxExecutor.swift +++ /dev/null @@ -1,211 +0,0 @@ -import Foundation - -/// Thread-safe accumulator for output delivered from the VM's `@Sendable` callback. -private final class ExecOutputBox: @unchecked Sendable { - private let lock = NSLock() - private var text = "" - func append(_ chunk: String) { lock.lock(); text += chunk; lock.unlock() } - var value: String { lock.lock(); defer { lock.unlock() }; return text } -} - -/// BuildExecutor backed by the embedded Linux VM. -/// -/// Commands run in the shared Alpine guest through its interactive login terminal. -/// Every guest command's exit status is checked — a build that did not run must -/// never report success. -@MainActor -final class EmbeddedLinuxExecutor: BuildExecutor { - let vm: LinuxVM - private let stagingDir: URL - private(set) var stagedOutputs: [URL] = [] - - init(vm: LinuxVM, stagingDir: URL) { - self.vm = vm - self.stagingDir = stagingDir - } - - func bootstrap() -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - Task { - do { - continuation.yield(.plan("Booting embedded Alpine Linux…")) - await vm.prepareRootfs() - try await vm.boot() - - // Whether the root carries the toolchain is the root's own - // business: the published one has xtool and Swift baked in - // (EmbeddedLinux/build-rootfs.sh, XFORGE_PROVISION=all), and a - // plain one installs them on demand with - // `install-toolchain.sh`. This check reports what the guest - // can actually do rather than what it was supposed to arrive - // with, and the message names the installer rather than - // calling a plain root broken. - continuation.yield(.plan("Checking the Alpine toolchain…")) - guard try await buildEnvironmentIsReady() else { - continuation.yield(.failed( - "The Alpine build toolchain is not ready. " - + "Run `sh /root/install-toolchain.sh all` in the Terminal tab." - )) - continuation.finish() - return - } - - continuation.yield(.finished) - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - } - } - - func installSDK(from source: SDKSource) async throws { - switch source { - case .bundled(let path): - // Already inside the guest filesystem. - let status = try await vm.run( - "swift sdk install \(GuestShell.quote(path))", - environment: nil - ) { _ in } - guard status == 0 else { throw BuildError.stepFailed("swift sdk install", status) } - case .hostedRemote(let url): - // Alpine downloads, extracts, and installs the published SDK directly - // into its rootfs. The iOS host only provides the resolved asset URL. - try await SDKInstaller.install(vm: vm, remoteURL: url) { _, _ in } - } - } - - /// Verify only; this must never invoke the installer. The executable and apk - /// checks make a stale or partially restored installation fail clearly. - /// - /// Whether the guest has the toolchain XForge builds with: the apk build - /// dependencies, Swift, and xtool. - /// - /// These are what the bundled rootfs *is* expected to carry — the published - /// root is built with `XFORGE_PROVISION=all` and arrives with them installed - /// (`EmbeddedLinux/build-rootfs.sh`) — but they are also exactly what the - /// guest's own installer puts there (`install-toolchain.sh`), so a plain root - /// that has been provisioned by hand answers yes too, and a false result is - /// answered with the installer rather than treated as a broken release. - /// - /// Every probe here is a separate guest process, and their output goes to a - /// *file*, never `/dev/null`: this engine kills a forked guest program whose - /// stdout/stderr is `/dev/null` (found with the engine-smoke harness — - /// `swift --version >/dev/null 2>&1` died where the unredirected form ran - /// fine). A probe killed that way reports "not installed" for a toolchain - /// that is, which is worse than a noisy log line. - private func buildEnvironmentIsReady() async throws -> Bool { - let silence = ">/tmp/xforge-probe.log 2>&1" - let status = try await vm.run( - "apk info -e clang lld cmake ninja git \(silence) && " - + "swift --version \(silence) && xtool --version \(silence)", - environment: nil - ) { _ in } - return status == 0 - } - - func createProject(named name: String, organizationIdentifier: String) async throws -> Project { - if !vm.isBooted { try await vm.boot() } - let validatedName = try Project.validatedName(name) - let path = Project.path(forValidatedName: validatedName) - let status = try await vm.run( - "mkdir -p \(GuestShell.quote(path)) && cd \(GuestShell.quote(path)) " - + "&& XTOOL_ORG=\(GuestShell.quote(organizationIdentifier)) " - + "xtool new --name \(GuestShell.quote(validatedName))", - environment: nil - ) { _ in } - guard status == 0 else { throw BuildError.stepFailed("xtool new", status) } - return Project(name: validatedName, organizationIdentifier: organizationIdentifier, rootPath: path) - } - - func resolve(_ project: Project) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - Task { - do { - if !vm.isBooted { try await vm.boot() } - continuation.yield(.plan("Resolving dependencies for \(project.name)…")) - guard project.hasSafeRootPath else { - throw ProjectValidationError.unsafePath - } - let status = try await vm.run( - "cd \(GuestShell.quote(project.rootPath)) && swift package resolve", - environment: nil - ) { line in - continuation.yield(.output(line)) - } - if status != 0 { - continuation.yield(.failed("Dependency resolution failed (exit \(status)).")) - continuation.finish() - return - } - continuation.yield(.finished) - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - } - } - - func build(_ project: Project, configuration: BuildConfiguration) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - Task { - do { - if !vm.isBooted { try await vm.boot() } - - var flags = "-s -i" // sign + output .ipa - if configuration == .release { flags = "-c release -s -i" } - - continuation.yield(.plan("Building \(project.name) (\(configuration.rawValue))…")) - guard project.hasSafeRootPath else { - throw ProjectValidationError.unsafePath - } - let code = try await vm.run( - "cd \(GuestShell.quote(project.rootPath)) && xtool dev build \(flags)", - environment: nil - ) { line in - continuation.yield(.output(line)) - } - - // Report the build's own failure before trying to collect an - // artifact, otherwise the user sees a misleading copy error. - guard code == 0 else { - continuation.yield(.failed("Build failed (exit \(code)).")) - continuation.finish() - return - } - - guard let guestIPA = try await newestIPA(in: project) else { - continuation.yield(.failed("The build finished but produced no .ipa in .build.")) - continuation.finish() - return - } - - let hostURL = stagingDir.appendingPathComponent((guestIPA as NSString).lastPathComponent) - try await vm.copyOut(guestPath: guestIPA, to: hostURL) - - stagedOutputs = [hostURL] - continuation.yield(.artifact(hostURL)) - continuation.yield(.finished) - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - } - } - - /// Newest `*.ipa` under the project's `.build`, rather than assuming a name - /// (xtool's output filename is not guaranteed). - private func newestIPA(in project: Project) async throws -> String? { - let box = ExecOutputBox() - let status = try await vm.run( - "find \(GuestShell.quote("\(project.rootPath)/.build")) -maxdepth 1 -type f " - + "-name '*.ipa' -print0 2>/dev/null | xargs -0 ls -t 2>/dev/null | head -1", - environment: nil - ) { box.append($0) } - guard status == 0 else { return nil } - let path = box.value.trimmingCharacters(in: .whitespacesAndNewlines) - return path.isEmpty ? nil : path - } -} diff --git a/App/Build/NativeBuildExecutor.swift b/App/Build/NativeBuildExecutor.swift new file mode 100644 index 0000000..08682de --- /dev/null +++ b/App/Build/NativeBuildExecutor.swift @@ -0,0 +1,451 @@ +import Foundation + +/// Builds a project with the toolchain linked into this process — no guest, no +/// subprocess. +/// +/// The stages mirror the Alpine executor it replaces, because the pipeline's +/// shape (provision → SDK → configure → resolve → compile → package) is what the +/// Build screen presents; what changed is where the work happens. Everything here +/// is a library call in XForge's own process: +/// +/// - `swift-frontend` through `swift::performFrontend`, +/// - `clang` through `CompilerInvocation` + `EmitObjAction`, +/// - `ld64.lld` through `lld::lldMain`, +/// - the Darwin SDK read straight out of the app sandbox. +/// +/// What it deliberately does *not* do is invent capability. Resolving SwiftPM +/// dependencies and compiling asset catalogs both need a package manager and +/// `actool`, neither of which can run on iOS, so the plan factory refuses those +/// projects by name instead of failing halfway through a build. +@MainActor +final class NativeBuildExecutor: BuildExecutor { + private let stagingDirectory: URL + private let fileManager: FileManager + + init( + stagingDirectory: URL = XForgeEnvironment.stagingDirectory, + fileManager: FileManager = .default + ) { + self.stagingDirectory = stagingDirectory + self.fileManager = fileManager + } + + // MARK: - Bootstrap + + func bootstrap() async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + Task { @MainActor in + let capabilities = NativeToolchainCapabilities.current + continuation.yield(.plan("Checking the native toolchain…")) + for line in capabilities.report { + continuation.yield(.output(line)) + } + + guard capabilities.canCompile else { + continuation.yield(.failed(capabilities.unavailableReason ?? "The native compiler is unavailable.")) + continuation.finish() + return + } + + continuation.yield(.plan("Checking the Darwin SDK…")) + do { + let layout = try NativeSDK.layout() + continuation.yield(.output("✓ SDK \(layout.sdkRoot.lastPathComponent)")) + if !capabilities.hasSwiftFrontend { + continuation.yield(.output( + "! The Swift frontend is not linked into this build: " + + "C and Objective-C sources compile, Swift does not." + )) + } + } catch { + continuation.yield(.failed( + "\(error.localizedDescription) Install it from the Toolchain screen." + )) + continuation.finish() + return + } + + continuation.yield(.finished) + continuation.finish() + } + } + } + + // MARK: - SDK + + func installSDK(from source: SDKSource) async throws { + switch source { + case .hostedRemote(let url): + try await NativeSDK.install(fromRemote: url) + } + } + + // MARK: - Project creation + + /// Write a new package into the app's own container. + /// + /// The template is the smallest thing that both SwiftPM and this driver + /// understand: a `Package.swift` with one executable product, one SwiftUI + /// entry point, and the app metadata XForge keeps in `xtool.yml`. + func createProject(named name: String, organizationIdentifier: String) async throws -> Project { + let validated = try Project.validatedName(name) + let project = Project( + name: validated, + organizationIdentifier: organizationIdentifier, + rootPath: Project.path(forValidatedName: validated) + ) + let root = project.rootURL + let sources = root.appendingPathComponent("Sources/\(validated)", isDirectory: true) + + guard !fileManager.fileExists(atPath: root.path) else { + throw NativeBuildError.projectAlreadyExists(root) + } + try fileManager.createDirectory(at: sources, withIntermediateDirectories: true) + + let manifest = """ + // swift-tools-version: 6.0 + import PackageDescription + + let package = Package( + name: "\(validated)", + platforms: [.iOS(.v17)], + products: [ + .executable(name: "\(validated)", targets: ["\(validated)"]) + ], + targets: [ + .executableTarget( + name: "\(validated)", + path: "Sources/\(validated)" + ) + ] + ) + """ + try manifest.write( + to: root.appendingPathComponent("Package.swift"), + atomically: true, + encoding: .utf8 + ) + + let entry = """ + import SwiftUI + + @main + struct \(validated)App: App { + var body: some Scene { + WindowGroup { + ContentView() + } + } + } + + struct ContentView: View { + var body: some View { + VStack(spacing: 12) { + Image(systemName: "hammer.fill").font(.largeTitle) + Text("\(validated)").font(.title2.bold()) + Text("Built by XForge on device.").foregroundStyle(.secondary) + } + .padding() + } + } + """ + try entry.write( + to: sources.appendingPathComponent("\(validated)App.swift"), + atomically: true, + encoding: .utf8 + ) + + let metadata = """ + name: \(validated) + bundleIdentifier: \(organizationIdentifier).\(validated) + deploymentTarget: "17.0" + """ + try metadata.write( + to: root.appendingPathComponent("xtool.yml"), + atomically: true, + encoding: .utf8 + ) + + return project + } + + // MARK: - Resolve + + func resolve(_ project: Project) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + Task { @MainActor in + do { + let manifest = project.rootURL.appendingPathComponent("Package.swift") + let dependencies = try NativeBuildPlanFactory.swiftPackageDependencies( + manifestURL: manifest + ) + guard dependencies.isEmpty else { + continuation.yield(.plan("Resolving dependencies…")) + throw NativeBuildPlanError.unsupportedSwiftPackageDependencies(dependencies) + } + continuation.yield(.plan("Resolving dependencies…")) + continuation.yield(.output("No package dependencies: nothing to resolve.")) + continuation.yield(.finished) + continuation.finish() + } catch { + continuation.yield(.failed(error.localizedDescription)) + continuation.finish() + } + } + } + } + + // MARK: - Build + + func build( + _ project: Project, + configuration: BuildConfiguration + ) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + Task { @MainActor in + do { + try await self.build(project, configuration: configuration) { event in + continuation.yield(event) + } + continuation.finish() + } catch { + continuation.yield(.failed(error.localizedDescription)) + continuation.finish() + } + } + } + } + + /// The pipeline proper. Kept off the stream so failures can use `throw` and + /// carry their diagnostics, rather than being flattened into a string. + private func build( + _ project: Project, + configuration: BuildConfiguration, + emit: @MainActor (BuildEvent) -> Void + ) async throws { + let capabilities = NativeToolchainCapabilities.current + guard capabilities.canCompile else { + throw NativeBuildError.toolchainUnavailable(capabilities.unavailableReason) + } + + let sdk = try NativeSDK.layout() + let appInfo = project.appInfo ?? .default(for: project) + let plan = try NativeBuildPlanFactory.makePlan( + root: project.rootURL, + appInfo: appInfo, + configuration: configuration, + sdk: sdk + ) + + emit(.plan("Compiling \(plan.moduleName) for arm64-apple-ios\(plan.minimumIOSVersion)…")) + emit(.output("\(plan.swiftSources.count) Swift, \(plan.clangSources.count) C, \(plan.resources.count) resource files")) + + if !plan.swiftSources.isEmpty, !capabilities.hasSwiftFrontend { + throw NativeBuildError.swiftFrontendMissing(plan.swiftSources.count) + } + + let buildDirectory = project.rootURL.appendingPathComponent(".xforge-build", isDirectory: true) + try? fileManager.removeItem(at: buildDirectory) + try fileManager.createDirectory(at: buildDirectory, withIntermediateDirectories: true) + + var objects: [URL] = [] + + for source in plan.swiftSources { + let object = buildDirectory + .appendingPathComponent(objectName(for: source)) + let arguments = try NativeToolchainInvocation.swiftFrontendArguments( + source: source, + object: object, + plan: plan, + sdk: sdk + ) + emit(.output("swift-frontend \(source.lastPathComponent)")) + let result = try await runDetached { + try NativeToolchain.runSwiftFrontend(arguments: arguments) + } + try check(result, tool: "swift-frontend", source: source) + objects.append(object) + } + + for source in plan.clangSources { + let object = buildDirectory + .appendingPathComponent(objectName(for: source.url)) + emit(.output("clang \(source.url.lastPathComponent)")) + let result = try await runDetached { + try NativeToolchain.compileC( + source: source.url, + object: object, + sdk: sdk.sdkRoot, + target: NativeToolchainInvocation.targetTriple( + minimumIOSVersion: plan.minimumIOSVersion + ), + language: source.language + ) + } + try check(result, tool: "clang", source: source.url) + objects.append(object) + } + + let executable = buildDirectory.appendingPathComponent(plan.executableName) + emit(.plan("Linking \(plan.executableName)…")) + let linkerArguments = NativeToolchainInvocation.linkerArguments( + objects: objects, + plan: plan, + sdk: sdk, + executable: executable + ) + let linkResult = try await runDetached { + try NativeToolchain.linkMachO(arguments: linkerArguments) + } + guard linkResult.succeeded else { + throw NativeBuildError.linkFailed(linkResult.diagnostics) + } + emit(.output("ld64.lld → \(plan.executableName)")) + + let bundle = try assembleBundle(plan: plan, executable: executable, emit: emit) + let ipa = try IPABuilder.buildIPA( + appBundle: bundle, + appInfo: appInfo, + outputDir: buildDirectory + ) + + try fileManager.createDirectory(at: stagingDirectory, withIntermediateDirectories: true) + let staged = stagingDirectory.appendingPathComponent("\(project.name)-\(appInfo.version).ipa") + try? fileManager.removeItem(at: staged) + try fileManager.copyItem(at: ipa, to: staged) + + emit(.output("Packaged \(staged.lastPathComponent) (unsigned — sign it on the Signing screen).")) + emit(.artifact(staged)) + emit(.finished) + } + + // MARK: - Bundle assembly + + /// `.app` with the executable, an Info.plist and the resources. + private func assembleBundle( + plan: NativeBuildPlan, + executable: URL, + emit: @MainActor (BuildEvent) -> Void + ) throws -> URL { + let bundle = plan.root + .appendingPathComponent(".xforge-build", isDirectory: true) + .appendingPathComponent("\(plan.appName).app", isDirectory: true) + if fileManager.fileExists(atPath: bundle.path) { + try fileManager.removeItem(at: bundle) + } + try fileManager.createDirectory(at: bundle, withIntermediateDirectories: true) + + let destination = bundle.appendingPathComponent(plan.executableName) + try fileManager.copyItem(at: executable, to: destination) + try fileManager.setAttributes([.posixPermissions: 0o755], ofItemAtPath: destination.path) + + var info: [String: Any] = [ + "CFBundleIdentifier": plan.bundleIdentifier, + "CFBundleName": plan.moduleName, + "CFBundleDisplayName": plan.appName, + "CFBundleExecutable": plan.executableName, + "CFBundlePackageType": "APPL", + "CFBundleShortVersionString": plan.version, + "CFBundleVersion": plan.buildNumber, + "MinimumOSVersion": plan.minimumIOSVersion, + "LSRequiresIPhoneOS": true, + "UIDeviceFamily": [1], + "UILaunchScreen": [String: Any](), + "UIRequiredDeviceCapabilities": ["arm64"], + ] + // A project's own Info.plist wins over the generated keys, which is what + // makes hand-written scene manifests and orientations survive a build. + if let template = plan.infoPlistTemplate, + let data = try? Data(contentsOf: template), + let decoded = try? PropertyListSerialization.propertyList(from: data, format: nil) + as? [String: Any] { + info.merge(decoded) { _, project in project } + } + + let plist = try PropertyListSerialization.data( + fromPropertyList: info, + format: .xml, + options: 0 + ) + try plist.write(to: bundle.appendingPathComponent("Info.plist"), options: .atomic) + + for resource in plan.resources { + if resource.pathExtension.lowercased() == "xcassets" { + // There is no actool on iOS: asset catalogs are compiled by a + // macOS-only tool. Copy the catalog so the project round-trips + // and say plainly what is missing rather than shipping an app + // whose icon silently vanishes. + emit(.output( + "! \(resource.lastPathComponent) was copied uncompiled: " + + "asset catalogs need actool, which cannot run on iOS." + )) + } + let target = bundle.appendingPathComponent(resource.lastPathComponent) + try? fileManager.removeItem(at: target) + try fileManager.copyItem(at: resource, to: target) + } + + return bundle + } + + // MARK: - Helpers + + private func objectName(for source: URL) -> String { + // Path separators cannot appear in a file name, and two sources with the + // same last path component would otherwise collide. + source.path + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "..", with: "_") + + ".o" + } + + private func check(_ result: NativeToolchainResult, tool: String, source: URL) throws { + guard result.succeeded else { + throw NativeBuildError.compileFailed( + tool: tool, + source: source, + diagnostics: result.diagnostics + ) + } + } + + /// Run a blocking compiler call off the main actor. The frontend and linker + /// are synchronous C++ entry points: a large module can take seconds, and the + /// UI must keep drawing while it does. + private func runDetached(_ work: @escaping @Sendable () throws -> T) async throws -> T { + try await Task.detached(priority: .userInitiated, operation: work).value + } +} + +enum NativeBuildError: LocalizedError { + case toolchainUnavailable(String?) + case swiftFrontendMissing(Int) + case projectAlreadyExists(URL) + case missingProjectDirectory(URL) + case compileFailed(tool: String, source: URL, diagnostics: String) + case linkFailed(String) + + var errorDescription: String? { + switch self { + case .toolchainUnavailable(let reason): + return reason ?? "The native compiler toolchain is not linked into this build." + case .swiftFrontendMissing(let count): + return """ + \(count) Swift source file(s) must be compiled, but the Swift frontend \ + is not linked into this build of XForge. Only the C/C++/Objective-C \ + half of the toolchain is present. + """ + case .projectAlreadyExists(let url): + return "A project already exists at \(url.path)." + case .missingProjectDirectory(let url): + return "The project directory is missing: \(url.path)." + case .compileFailed(let tool, let source, let diagnostics): + let text = diagnostics.trimmingCharacters(in: .whitespacesAndNewlines) + let detail = text.isEmpty ? "\(tool) reported a failure without diagnostics." : text + return "\(source.lastPathComponent): \(detail)" + case .linkFailed(let diagnostics): + let text = diagnostics.trimmingCharacters(in: .whitespacesAndNewlines) + return text.isEmpty ? "The linker failed without diagnostics." : text + } + } +} diff --git a/App/Build/NativeBuildPlan.swift b/App/Build/NativeBuildPlan.swift new file mode 100644 index 0000000..f434628 --- /dev/null +++ b/App/Build/NativeBuildPlan.swift @@ -0,0 +1,279 @@ +import Foundation + +/// A resolved, self-contained description of what to compile and link. +/// +/// The plan is the boundary between "reading a project" and "running a compiler". +/// It exists so the on-device driver is *not* a second build system: every source +/// file, framework and search path is decided here, and the executor only turns +/// this value into `swift-frontend` / `clang` / `ld64.lld` argument lists. That +/// split is what makes the argument construction testable on a simulator, where +/// no compiler libraries exist. +struct NativeBuildPlan: Sendable, Equatable { + /// Directory holding `Package.swift` / the sources. + var root: URL + /// The product being built, e.g. `Sources/MyApp` → `MyApp`. + var moduleName: String + /// Display name of the produced app bundle (`.app`). + var appName: String + /// Mach-O executable name inside the bundle. + var executableName: String + var bundleIdentifier: String + var minimumIOSVersion: String + var version: String + var buildNumber: String + var configuration: BuildConfiguration + + /// Swift translation units, in a stable order. + var swiftSources: [URL] + /// C / Objective-C / Objective-C++ translation units. + var clangSources: [NativeClangSource] + /// Files copied into the bundle unchanged. + var resources: [URL] + /// A hand-written Info.plist in the project, merged over the generated one. + var infoPlistTemplate: URL? + /// Frameworks every app needs; the plan adds the project's own on top. + var frameworks: [String] + /// Extra `-L` paths, e.g. the Swift runtime inside the Darwin SDK. + var librarySearchPaths: [URL] + /// Extra `-F` paths. + var frameworkSearchPaths: [URL] + + var isEmpty: Bool { + swiftSources.isEmpty && clangSources.isEmpty + } + + var hasSwift: Bool { !swiftSources.isEmpty } +} + +/// One non-Swift translation unit plus the language clang must be told to use. +struct NativeClangSource: Sendable, Equatable, Hashable { + var url: URL + /// `c`, `objective-c`, `c++`, `objective-c++`. + var language: String + + static func language(for url: URL) -> String? { + switch url.pathExtension.lowercased() { + case "c": return "c" + case "m": return "objective-c" + case "mm": return "objective-c++" + case "cc", "cpp", "cxx": return "c++" + default: return nil + } + } +} + +enum NativeBuildPlanError: LocalizedError { + case noSources(URL) + case missingTargetsDirectory(URL) + case unsupportedSwiftPackageDependencies([String]) + case missingPackageManifest(URL) + + var errorDescription: String? { + switch self { + case .noSources(let root): + return "No compilable sources were found under \(root.path)." + case .missingTargetsDirectory(let root): + return "This project has no Sources/ directory, so XForge cannot tell what to build (\(root.path))." + case .unsupportedSwiftPackageDependencies(let names): + let list = names.prefix(5).joined(separator: ", ") + let more = names.count > 5 ? " and \(names.count - 5) more" : "" + return """ + This project declares SwiftPM dependencies (\(list)\(more)). Resolving \ + them needs a package manager, which cannot run inside an app that may \ + not spawn processes. Remove the dependencies, or build a project whose \ + sources are self-contained. + """ + case .missingPackageManifest(let root): + return "No Package.swift was found at \(root.path)." + } + } +} + +/// Turns a project directory into a `NativeBuildPlan`. +/// +/// Deliberately conservative: it reads the layout SwiftPM and `xtool new` produce +/// (`Package.swift`, `Sources//**`) and refuses anything it cannot resolve +/// exactly, rather than guessing and producing a binary that links the wrong code. +enum NativeBuildPlanFactory { + static func makePlan( + root: URL, + appInfo: AppInfo, + configuration: BuildConfiguration, + sdk: NativeSDKLayout, + fileManager: FileManager = .default + ) throws -> NativeBuildPlan { + let manifest = root.appendingPathComponent("Package.swift") + guard fileManager.fileExists(atPath: manifest.path) else { + throw NativeBuildPlanError.missingPackageManifest(root) + } + + // Dependencies are the one thing an in-process driver cannot resolve: + // SwiftPM resolves them by fetching and *running* package manifests. + // Refuse them by name instead of failing somewhere further down. + let dependencies = try swiftPackageDependencies(manifestURL: manifest) + if !dependencies.isEmpty { + throw NativeBuildPlanError.unsupportedSwiftPackageDependencies(dependencies) + } + + let sourcesRoot = root.appendingPathComponent("Sources", isDirectory: true) + var isDirectory: ObjCBool = false + guard fileManager.fileExists(atPath: sourcesRoot.path, isDirectory: &isDirectory), + isDirectory.boolValue else { + throw NativeBuildPlanError.missingTargetsDirectory(root) + } + + let (moduleName, sourcesDirectory) = try target(in: sourcesRoot, fileManager: fileManager) + let swiftSources = try files(in: sourcesDirectory, extensions: ["swift"], fileManager: fileManager) + let clangSources = try files( + in: sourcesDirectory, + extensions: ["c", "m", "mm", "cc", "cpp", "cxx"], + fileManager: fileManager + ).compactMap { url -> NativeClangSource? in + guard let language = NativeClangSource.language(for: url) else { return nil } + return NativeClangSource(url: url, language: language) + } + + guard !swiftSources.isEmpty || !clangSources.isEmpty else { + throw NativeBuildPlanError.noSources(sourcesDirectory) + } + + // Resources live next to the sources in the layouts XForge creates and in + // the ones `xtool new` produces. + let resources = try files( + in: sourcesDirectory, + extensions: ["xcassets", "json", "plist", "strings", "storyboard", "xib", "png", "jpg"], + fileManager: fileManager + ).filter { !$0.lastPathComponent.hasPrefix("Info.plist") } + + var searchPaths: [URL] = [] + var frameworkSearchPaths: [URL] = [] + var frameworks = ["Foundation"] + if !swiftSources.isEmpty { + // A Swift executable links the Swift runtime statically: the Darwin + // SDK bundle carries it, and there is no dyld-shared runtime on a + // sideloaded device build. + searchPaths.append(contentsOf: sdk.swiftRuntimeLibraryPaths) + frameworks.append("UIKit") + } + + let template = root.appendingPathComponent("Support/Info.plist") + let hasTemplate = fileManager.fileExists(atPath: template.path) + + return NativeBuildPlan( + root: root, + moduleName: moduleName, + appName: appInfo.displayName, + executableName: moduleName, + bundleIdentifier: appInfo.bundleIdentifier, + minimumIOSVersion: appInfo.minimumOSVersion, + version: appInfo.version, + buildNumber: appInfo.buildNumber, + configuration: configuration, + swiftSources: swiftSources, + clangSources: clangSources, + resources: resources, + infoPlistTemplate: hasTemplate ? template : nil, + frameworks: frameworks, + librarySearchPaths: searchPaths, + frameworkSearchPaths: frameworkSearchPaths + ) + } + + /// The package's declared `.package(url: "…")` dependencies, by name. + /// + /// A text scan on purpose: a full manifest evaluation means compiling and + /// *running* `Package.swift`, which is exactly what this path cannot do. The + /// scan only needs to answer "are there any", which it does reliably; the + /// names are for the error message. + static func swiftPackageDependencies(manifestURL: URL) throws -> [String] { + guard let text = try? String(contentsOf: manifestURL, encoding: .utf8) else { + return [] + } + var names: [String] = [] + for line in text.split(separator: "\n") { + let trimmed = line.trimmingCharacters(in: .whitespaces) + guard trimmed.hasPrefix(".package(") else { continue } + if let name = quotedName(in: trimmed) { + names.append(name) + } else { + names.append("an unnamed package") + } + } + return names + } + + /// The last quoted path-ish component of a `.package(...)` line, e.g. + /// `…/swift-argument-parser.git` → `swift-argument-parser`. + private static func quotedName(in line: String) -> String? { + guard let firstQuote = line.firstIndex(of: "\"") else { return nil } + let rest = line[line.index(after: firstQuote)...] + guard let lastQuote = rest.firstIndex(of: "\"") else { return nil } + let value = String(rest[..` when there is exactly one, the + /// project's own name when several exist, and the only directory otherwise. + private static func target( + in sourcesRoot: URL, + fileManager: FileManager + ) throws -> (name: String, directory: URL) { + let entries = (try? fileManager.contentsOfDirectory( + at: sourcesRoot, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + )) ?? [] + let directories = entries.filter { + (try? $0.resourceValues(forKeys: [.isDirectoryKey]))?.isDirectory == true + } + guard !directories.isEmpty else { + throw NativeBuildPlanError.noSources(sourcesRoot) + } + guard directories.count > 1 else { + return (directories[0].lastPathComponent, directories[0]) + } + // Several targets: the app target is the one named after the package, + // which is the layout `xtool new` and XForge both produce. + let packageName = sourcesRoot.deletingLastPathComponent().lastPathComponent + if let match = directories.first(where: { $0.lastPathComponent == packageName }) { + return (match.lastPathComponent, match) + } + throw NativeBuildPlanError.noSources(sourcesRoot) + } + + /// Regular files with one of `extensions`, recursively, sorted by path. + private static func files( + in directory: URL, + extensions: Set, + fileManager: FileManager + ) throws -> [URL] { + guard let enumerator = fileManager.enumerator( + at: directory, + includingPropertiesForKeys: [.isRegularFileKey], + options: [.skipsHiddenFiles, .skipsPackageDescendants] + ) else { + return [] + } + var found: [URL] = [] + for case let url as URL in enumerator { + guard extensions.contains(url.pathExtension.lowercased()) else { continue } + // Asset catalogs are directories, and they are copied whole. + if extensions.contains("xcassets") { + if url.pathExtension.lowercased() == "xcassets" { + found.append(url) + enumerator.skipDescendants() + } else if (try? url.resourceValues(forKeys: [.isRegularFileKey]))?.isRegularFile == true { + found.append(url) + } + continue + } + if (try? url.resourceValues(forKeys: [.isRegularFileKey]))?.isRegularFile == true { + found.append(url) + } + } + return found.sorted { $0.path < $1.path } + } +} diff --git a/App/Build/NativeToolchainCapabilities.swift b/App/Build/NativeToolchainCapabilities.swift new file mode 100644 index 0000000..2a1f4bc --- /dev/null +++ b/App/Build/NativeToolchainCapabilities.swift @@ -0,0 +1,55 @@ +import Foundation + +/// What this build of XForge can actually compile with. +/// +/// The toolchain is linked in at *link* time through +/// `Support/NativeToolchain.generated.xcconfig`, so a plain clone — and every CI +/// build — has no compiler at all, while a build that installed the toolchain +/// bundle has some or all of it (Clang and LLD for a long time, the Swift +/// frontend only once its libraries exist for iOS). Reporting that honestly is +/// the difference between "this project needs a toolchain" and a build that fails +/// somewhere in the middle with a linker error. +struct NativeToolchainCapabilities: Sendable, Equatable { + let hasClang: Bool + let hasLLDMachO: Bool + let hasSwiftFrontend: Bool + let backendDescription: String + let sdkInstalled: Bool + + static var current: NativeToolchainCapabilities { + NativeToolchainCapabilities( + hasClang: NativeToolchain.isAvailable, + hasLLDMachO: NativeToolchain.isAvailable, + hasSwiftFrontend: NativeToolchain.isSwiftAvailable, + backendDescription: NativeToolchain.isAvailable + ? NativeToolchain.version + : "not linked", + sdkInstalled: NativeSDK.isInstalled + ) + } + + /// Whether anything can be compiled at all. The Swift frontend is deliberately + /// not required: a C/Objective-C target links fine without it, and refusing to + /// start would be wrong about why the build fails. + var canCompile: Bool { hasClang && hasLLDMachO } + + var unavailableReason: String? { + guard !canCompile else { return nil } + return """ + The native compiler is not linked into this build of XForge, so it cannot \ + compile anything. Install the toolchain bundle (NativeToolchain/install-bundle.sh) \ + and rebuild, or use a release that ships it. + """ + } + + /// Lines for the build console: one per component, present or missing. + var report: [String] { + [ + "clang: \(hasClang ? "linked" : "missing")", + "ld64.lld: \(hasLLDMachO ? "linked" : "missing")", + "swift-frontend: \(hasSwiftFrontend ? "linked" : "missing")", + "darwin SDK: \(sdkInstalled ? "installed" : "missing")", + "backend: \(backendDescription)", + ] + } +} diff --git a/App/Build/NativeToolchainInvocation.swift b/App/Build/NativeToolchainInvocation.swift new file mode 100644 index 0000000..14d7e82 --- /dev/null +++ b/App/Build/NativeToolchainInvocation.swift @@ -0,0 +1,117 @@ +import Foundation + +/// Argument construction for the in-process toolchain. +/// +/// Pure functions, on purpose: the compiler itself only exists inside a device +/// build (the LLVM/Swift libraries are linked in by `NativeToolchain.generated.xcconfig`, +/// which is disabled on a plain clone), so a mis-built argument list could only be +/// discovered on a phone with a real SDK. Splitting it out means the unit tests +/// exercise the exact strings the frontend and linker receive, on a simulator, +/// with no compiler present. +enum NativeToolchainInvocation { + /// `arm64-apple-ios17.0` — the triple every iOS arm64 tool in the stack uses. + static func targetTriple(minimumIOSVersion: String) -> String { + "arm64-apple-ios\(minimumIOSVersion)" + } + + /// `swift-frontend` arguments, exactly as `swift::performFrontend` expects + /// them. + /// + /// The convention is not obvious and is worth stating: the driver's own main + /// calls `performFrontend(argv[2…], argv[0], …)` for a `swift-frontend -frontend …` + /// command line, so the array must contain neither the program name nor + /// `-frontend`. Passing `-frontend` makes `CompilerInvocation::parseArgs` + /// reject the whole invocation. + static func swiftFrontendArguments( + source: URL, + object: URL, + plan: NativeBuildPlan, + sdk: NativeSDKLayout + ) throws -> [String] { + guard let swiftResources = sdk.swiftResources else { + throw NativeToolchainError.io( + "The Darwin SDK does not declare swiftResourcesPath, so the Swift standard library cannot be found." + ) + } + + var arguments = [ + // `-parse-as-library` unless the file *is* main.swift: an app's entry + // point is `@main`, and without this the frontend treats top-level + // code as the entry point and rejects the attribute. + "-parse-as-library", + "-c", source.path, + "-target", targetTriple(minimumIOSVersion: plan.minimumIOSVersion), + "-sdk", sdk.sdkRoot.path, + "-resource-dir", swiftResources.path, + "-module-name", plan.moduleName, + "-o", object.path, + ] + if source.lastPathComponent == "main.swift" { + arguments.removeFirst() + } + arguments.append(plan.configuration == .release ? "-O" : "-Onone") + // DebugInfo makes a crash in a built app diagnosable; it costs object size + // only, and the linker strips what the app does not use. + arguments.append(contentsOf: ["-g", "-debug-info-format=dwarf"]) + return arguments + } + + /// `ld64.lld` arguments for an arm64 iOS executable. + /// + /// `fileExists` is injected so the Swift runtime libraries are only named when + /// the SDK bundle actually carries them: naming `-lswift_Concurrency` for a + /// bundle that predates the split would fail the link with "library not + /// found", and a Swift 5.7+ runtime has it inside `libswiftCore` anyway. + static func linkerArguments( + objects: [URL], + plan: NativeBuildPlan, + sdk: NativeSDKLayout, + executable: URL, + fileExists: (URL) -> Bool = { FileManager.default.fileExists(atPath: $0.path) } + ) -> [String] { + let deploymentTarget = plan.minimumIOSVersion + var arguments = [ + "-arch", "arm64", + // lld needs the platform *and* the version pair explicitly; without + // it the Mach-O records no LC_BUILD_VERSION and iOS refuses to load + // the binary. + "-platform_version", "ios", deploymentTarget, deploymentTarget, + "-syslibroot", sdk.sdkRoot.path, + "-o", executable.path, + ] + + for path in plan.librarySearchPaths where fileExists(path) { + arguments.append(contentsOf: ["-L", path.path]) + } + for path in plan.frameworkSearchPaths where fileExists(path) { + arguments.append(contentsOf: ["-F", path.path]) + } + + if plan.hasSwift { + arguments.append(contentsOf: ["-lswiftCore", "-lswift_Concurrency", "-lswift_StringProcessing"] + .filter { library in + // -lswiftCore is unconditional: a Swift binary cannot run + // without it. The other two merged into it over time. + library == "-lswiftCore" || swiftLibraryExists(library, sdk: sdk, fileExists: fileExists) + }) + } + + arguments.append(contentsOf: ["-lSystem"]) + for framework in plan.frameworks { + arguments.append(contentsOf: ["-framework", framework]) + } + arguments.append(contentsOf: objects.map(\.path)) + return arguments + } + + private static func swiftLibraryExists( + _ flag: String, + sdk: NativeSDKLayout, + fileExists: (URL) -> Bool + ) -> Bool { + let name = String(flag.dropFirst(2)) // "-lswift_Concurrency" → "swift_Concurrency" + return sdk.swiftRuntimeLibraryPaths.contains { directory in + fileExists(directory.appendingPathComponent("lib\(name).a")) + } + } +} diff --git a/App/Build/SDKInstaller.swift b/App/Build/SDKInstaller.swift deleted file mode 100644 index a05d45e..0000000 --- a/App/Build/SDKInstaller.swift +++ /dev/null @@ -1,134 +0,0 @@ -import Foundation - -/// Installs the Darwin Swift SDK entirely inside the embedded Alpine rootfs. -/// -/// iOS resolves release metadata and displays progress, but the archive download, -/// extraction, validation, and Swift SDK installation are Linux commands. Temporary -/// files live under /root/.cache and the installed SDK lives in SwiftPM's guest-side -/// SDK directory; /host is never used as an installation location. -@MainActor -enum SDKInstaller { - static let guestCacheDirectory = "/root/.cache/xforge-sdk" - static let guestBundlePath = guestCacheDirectory + "/darwin.artifactbundle" - - /// Conservative free-space requirement for the archive, expanded bundle, and - /// installed copy. The fakefs root lives in the app container, so host capacity - /// is also the capacity available to Alpine. - static let requiredFreeBytes: Int64 = 2_000_000_000 - - static func install( - vm: LinuxVM, - remoteURL: URL? = nil, - advance: (Double, String) -> Void - ) async throws { - advance(0.02, "Checking the Alpine Swift SDK installation…") - try await vm.boot() - - let alreadyInstalled = try await vm.run( - "swift sdk list 2>&1 | grep -qi darwin", - environment: nil - ) { _ in } - if alreadyInstalled == 0 { - advance(1.0, "Darwin SDK is already installed in Alpine") - XForgeLog.note("sdk: darwin is already installed in the guest rootfs") - return - } - - let free = XForgeEnvironment.availableBytes - if free > 0, free < requiredFreeBytes { - throw ToolchainError.notEnoughSpace(needed: requiredFreeBytes, free: free) - } - - advance(0.05, "Resolving the latest Darwin SDK release…") - let url: URL - if let remoteURL { - url = remoteURL - } else { - url = try await XForgeReleases.darwinSDKURL() - } - XForgeLog.note("sdk: Alpine will download \(url.absoluteString)") - - let archiveName = url.lastPathComponent.isEmpty ? "darwin-sdk.zip" : url.lastPathComponent - let archivePath = guestCacheDirectory + "/" + archiveName - let script = """ - set -eu - cache=\(GuestShell.quote(guestCacheDirectory)) - archive=\(GuestShell.quote(archivePath)) - bundle=\(GuestShell.quote(guestBundlePath)) - - command -v curl >/dev/null 2>&1 - command -v unzip >/dev/null 2>&1 - command -v swift >/dev/null 2>&1 - - mkdir -p "$cache" - rm -f "$archive" - rm -rf "$bundle" - - echo "Downloading Darwin SDK inside Alpine..." - curl -fL --retry 3 --progress-bar \ - \(GuestShell.quote(url.absoluteString)) -o "$archive" - - echo "Extracting Darwin SDK inside the rootfs..." - unzip -q "$archive" -d "$cache" - test -f "$bundle/info.json" - - echo "Installing Darwin SDK into the Alpine SwiftPM store..." - swift sdk install "$bundle" - - rm -f "$archive" - rm -rf "$bundle" - swift sdk list - """ - - advance(0.1, "Downloading and installing the SDK inside Alpine…") - let output = OutputCollector() - let status = try await vm.run(script, environment: nil) { chunk in - output.append(chunk) - let text = chunk.trimmingCharacters(in: .whitespacesAndNewlines) - if !text.isEmpty { - XForgeLog.note("guest: " + text) - } - } - guard status == 0 else { - let tail = output.tail - if tail.contains("info.json") { - throw ToolchainError.sdkLayoutUnexpected - } - throw ToolchainError.sdkInstallFailed(status, output.tail) - } - - let verified = try await vm.run( - "swift sdk list 2>&1 | grep -qi darwin", - environment: nil - ) { _ in } - guard verified == 0 else { - throw ToolchainError.sdkInstallFailed(verified, "") - } - - advance(1.0, "Darwin SDK installed in Alpine") - XForgeLog.note("sdk: installed and verified in the guest rootfs") - } -} - -/// Thread-safe string accumulator for output delivered from a `@Sendable` callback. -final class OutputCollector: @unchecked Sendable { - private let lock = NSLock() - private var text = "" - - func append(_ chunk: String) { - lock.lock() - text += chunk - lock.unlock() - } - - var value: String { - lock.lock() - defer { lock.unlock() } - return text - } - - /// Last few lines — what an error message should show. - var tail: String { - value.split(separator: "\n").suffix(6).joined(separator: "\n") - } -} diff --git a/App/Build/XForgeEnvironment.swift b/App/Build/XForgeEnvironment.swift index 39a0ee2..c04788b 100644 --- a/App/Build/XForgeEnvironment.swift +++ b/App/Build/XForgeEnvironment.swift @@ -1,7 +1,7 @@ import Foundation -/// Application-level wiring: where the embedded Linux lives, how a build executor is -/// constructed for a project, and where staged artifacts land. +/// Application-level wiring: where the app's directories are, how a build executor +/// is constructed for a project, and where staged artifacts land. @MainActor enum XForgeEnvironment { /// App sandbox root. @@ -13,30 +13,6 @@ enum XForgeEnvironment { FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] } - /// App sandbox subdirectory holding the embedded Linux userspace. - static var embeddedRoot: URL { - documentDirectory.appendingPathComponent("embedded-linux", isDirectory: true) - } - - /// Installed ish-arm64 `fakefs` root filesystems. The bundled Alpine rootfs is - /// imported here at launch and reused afterwards (the first boot imports it - /// as a fallback). - static var rootsDirectory: URL { - embeddedRoot.appendingPathComponent("roots", isDirectory: true) - } - - /// Whether the bundled Alpine rootfs has already been imported. - static var isRootfsInstalled: Bool { - RootfsInstaller.isInstalled(in: rootsDirectory) - } - - /// Directory shared into the guest at `/host` (read-write, realfs). Large - /// artifacts are staged here by the host instead of being pushed through the - /// guest command pipe. - static var hostShareDirectory: URL { - embeddedRoot.appendingPathComponent("host", isDirectory: true) - } - /// Host-side downloads (SDK archives, toolchain bundles). static var downloadsDirectory: URL { documentDirectory.appendingPathComponent("downloads", isDirectory: true) @@ -47,6 +23,16 @@ enum XForgeEnvironment { documentDirectory.appendingPathComponent("native-sdk", isDirectory: true) } + /// Projects live in the app's own container. + /// + /// They used to live inside the embedded Linux filesystem (`/root/projects`), + /// which is why `Project.rootPath` still carries a guest-shaped string. Now + /// that the toolchain runs in-process, a project is a directory of files this + /// process can open, and this is the one place that decides where. + nonisolated static var projectsDirectory: URL { + documentDirectory.appendingPathComponent("projects", isDirectory: true) + } + /// Diagnostics (`XForgeLog` writes the engine log here). static var logsDirectory: URL { XForgeLog.directory } @@ -70,26 +56,21 @@ enum XForgeEnvironment { /// iOS data storage guidelines forbid — it bloats every backup and can get an /// app rejected — so: /// - /// - the downloads, staged artifacts and engine log are excluded: each is - /// produced again by downloading, rebuilding or running; - /// - **the guest filesystem is excluded too.** It is the bundled rootfs - /// imported into fakefs, and with `XFORGE_PROVISION=all` that is several - /// gigabytes of Alpine, Swift, xtool and the SDK. The user's projects live - /// *inside* it — nothing here can separate one from the other — so this is - /// a real trade-off: projects are no longer in device backups. The way out - /// is `ProjectExporter` ("Export project" on a project's screen, and the - /// archive appears in `Documents/exports`, which *is* backed up), plus the - /// `/host` share and the Terminal for anything else. + /// - the downloads, the staged artifacts and the log are excluded: each is + /// produced again by downloading, rebuilding or running. /// - /// See Docs/DESIGN.md, "What is backed up", for why the two cannot be split. + /// Projects are *not* excluded. They used to be inside the guest filesystem, + /// which had to be kept out of backup as a whole and took the user's work with + /// it; now that they are plain directories in the container, they are backed up + /// like any other document. static func prepareStorage() { let fm = FileManager.default - let excluded = [embeddedRoot, downloadsDirectory, nativeSDKDirectory, stagingDirectory, logsDirectory] + let excluded = [downloadsDirectory, nativeSDKDirectory, stagingDirectory, logsDirectory] for directory in excluded { try? fm.createDirectory(at: directory, withIntermediateDirectories: true) excludeFromBackup(directory) } - // Exports are the user's work leaving the guest, so they are left alone: + // Exports are the user's work leaving the app, so they are left alone: // backed up, and visible in the Files app. try? fm.createDirectory(at: exportsDirectory, withIntermediateDirectories: true) } @@ -137,34 +118,12 @@ enum XForgeEnvironment { .sorted { $0.date > $1.date } } - /// The single embedded Linux VM for the whole app. + /// Construct the build executor: the toolchain linked into this process. /// - /// ish-arm64 can only boot one guest per process, so every screen (Terminal, - /// Toolchain, Build) must share this instance rather than creating its own. - private static var sharedVM: LinuxVM? - - static func makeVM() -> LinuxVM { - if let sharedVM { return sharedVM } - for dir in [embeddedRoot, rootsDirectory, hostShareDirectory, downloadsDirectory] { - try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - } - let vm = EmbeddedLinuxVM(root: embeddedRoot, - hostShare: hostShareDirectory, - emulator: makeEmulator()) - sharedVM = vm - return vm - } - - /// Construct the build executor. `Local` uses the embedded Linux VM. + /// There is exactly one implementation now. It used to choose between the + /// embedded Linux and a "remote" backend that never existed; both the choice + /// and the guest it offered are gone. static func makeExecutor(for project: Project? = nil) -> BuildExecutor { - EmbeddedLinuxExecutor(vm: makeVM(), stagingDir: stagingDirectory) - } - - /// The in-process Linux emulator that runs the embedded Alpine guest. - /// ish-arm64 runs a real aarch64 Linux guest in-process; its threaded-code - /// interpreter dispatches guest instructions to pre-compiled "gadgets", so it - /// needs no JIT entitlement and works in a sideloaded app. - static func makeEmulator() -> LinuxEmulator { - ISHEmulator(rootsDirectory: rootsDirectory, hostDirectory: hostShareDirectory) + NativeBuildExecutor(stagingDirectory: stagingDirectory) } } diff --git a/App/EmbeddedVM/ConsoleShellSession.swift b/App/EmbeddedVM/ConsoleShellSession.swift deleted file mode 100644 index 905f0d8..0000000 --- a/App/EmbeddedVM/ConsoleShellSession.swift +++ /dev/null @@ -1,50 +0,0 @@ -import Foundation - -/// The terminal's shell, as it actually is: the guest's console. -/// -/// There is nothing to attach and nothing to wait for — the console is up as soon -/// as the guest's init has opened it, and it stays up. Typing goes into the tty; -/// the guest's own line discipline does the rest. See `InteractiveShellSession`. -@MainActor -final class ConsoleShellSession: InteractiveShellSession { - private let console: any GuestConsole - - /// init owns the console, and it respawns the session the user types into, so - /// init is the process this session speaks for. - let pid: Int32 = 1 - private(set) var isRunning = true - - /// Ctrl-C, the byte the guest's tty turns into SIGINT for the foreground - /// group. - private static let interruptByte: UInt8 = 0x03 - - init(console: any GuestConsole) { - self.console = console - } - - @discardableResult - func send(_ text: String) -> Bool { - guard isRunning else { return false } - return console.write(text) - } - - @discardableResult - func sendControl(_ scalar: UInt8) -> Bool { - guard isRunning else { return false } - return console.write(String(UnicodeScalar(scalar))) - } - - func interruptForeground() async { - _ = sendControl(Self.interruptByte) - } - - func resize(cols: Int, rows: Int) { - guard isRunning else { return } - console.resize(cols: cols, rows: rows) - } - - func stop() { - isRunning = false - console.stop() - } -} diff --git a/App/EmbeddedVM/EmbeddedLinuxVM.swift b/App/EmbeddedVM/EmbeddedLinuxVM.swift deleted file mode 100644 index c09df0f..0000000 --- a/App/EmbeddedVM/EmbeddedLinuxVM.swift +++ /dev/null @@ -1,527 +0,0 @@ -import Foundation - -/// Thread-safe string accumulator for output captured from a `@Sendable` callback. -private final class OutputBox: @unchecked Sendable { - private let lock = NSLock() - private var text = "" - - func append(_ chunk: String) { - lock.lock() - text += chunk - lock.unlock() - } - - var value: String { - lock.lock() - defer { lock.unlock() } - return text - } -} - -/// Concrete `LinuxVM` that drives an in-process `LinuxEmulator` (ish-arm64). -/// -/// iOS cannot spawn subprocesses, so the guest runs inside the app and this type -/// provides the command/file bridge to it. -/// -/// File transfer goes through the directory shared with the guest at `/host` -/// (`hostShare`), *not* through the command pipe: an `.ipa` is easily tens of -/// megabytes, and piping that as base64 through a shell would both truncate and -/// exhaust memory. The pipe is only a fallback for when no share is configured. -@MainActor -final class EmbeddedLinuxVM: LinuxVM { - let root: URL - private let hostShare: URL? - private let emulator: LinuxEmulator - private(set) var isBooted = false - private var bootTask: Task? - - /// No timeout: builds run in the foreground and may legitimately take minutes. - private static let noTimeout: TimeInterval = 0 - /// Capture cap for the base64 fallback path (the share path is uncapped). - private static let outputCap = 64 * 1024 * 1024 - /// Guest path of the shared host directory. - private static let guestShare = "/host" - /// Staging subdirectory inside the share used for transfers. - private static let transferDir = ".xforge-transfer" - /// Launch shell for probes and one-shot commands. Not the guest's boot: pid 1 - /// is `/sbin/init` (see `initCommand`), which is what starts a shell on the - /// console. - static let launchCommand = "/bin/sh" - /// The program XForge boots pid 1 with. It reads `/etc/inittab`, which runs - /// `/sbin/xforge-login root` on the console — that script execs root's login - /// shell from `/etc/passwd`, which is the shell the Terminal tab shows. - static let initCommand = "/sbin/init" - - /// Why the guest's init did not start, if it did not. Kept rather than thrown - /// from `boot()` because the rest of the app — builds, file transfers — runs - /// as children of pid 1 whatever it is, and works either way; it is the - /// terminal that cannot exist without a console, so the terminal reports it. - private var initProblem: String? - - init(root: URL, hostShare: URL? = nil, emulator: LinuxEmulator) { - self.root = root - self.hostShare = hostShare - self.emulator = emulator - } - - func boot() async throws { - guard !isBooted else { return } - if let bootTask { - try await bootTask.value - return - } - - let task = Task { - if !emulator.isRunning { - try await emulator.boot() - } - guard emulator.isRunning else { - throw LinuxVMError.guestDidNotStart - } - // `xf_ish_boot` has mounted the fakefs, but that alone does not mean - // the guest can execute a shell or reach its /host share. The latter - // is the transport used for every large SDK import and for streamed - // terminal output, so do not report Alpine as ready until both paths - // have completed a bounded round trip. - try await verifyRootfs() - try await verifyCommandBridge() - await configureGuestResolver() - await startInit() - isBooted = true - XForgeLog.note("boot: verified Alpine guest is ready") - } - bootTask = task - do { - try await task.value - bootTask = nil - } catch { - isBooted = false - bootTask = nil - throw error - } - } - - /// Import the bundled rootfs before anything needs the guest, so the first - /// build or terminal session is not the one that pays for it. Best effort: - /// `boot()` imports it anyway when this has not run or has failed. - func prepareRootfs() async { - do { - try await emulator.prepareRootfs() - } catch { - XForgeLog.note("rootfs: pre-install failed: \(error.localizedDescription)") - } - } - - /// Bring up the guest's own init, which is what turns the mounted root into a - /// booted system: `/sbin/init` reads `/etc/inittab` and respawns - /// `/sbin/xforge-login root` on the console. - /// - /// A failure is recorded, not thrown. Everything except the terminal works - /// without it — the command runner makes its own children of pid 1, who that - /// is makes no difference — so failing the whole boot would take the build - /// pipeline down with the terminal. - private func startInit() async { - do { - try await emulator.startInit(Self.initCommand) - initProblem = nil - XForgeLog.note("boot: \(Self.initCommand) is pid 1") - } catch { - initProblem = error.localizedDescription - XForgeLog.note("boot: \(Self.initCommand) did not start: \(error.localizedDescription)") - } - } - - /// Confirm that the mounted filesystem is a usable Alpine guest before any - /// app feature is allowed to issue commands. This deliberately calls the - /// emulator directly because the normal command path waits for boot. - private func verifyRootfs() async throws { - let probe = """ - set -eu - test -x /bin/sh - test -r /etc/alpine-release - test -r /proc/version - test -e /dev/null - test -d /host - __xf_probe=/root/.xforge-health-$$ - printf x > "$__xf_probe" - test "$(cat "$__xf_probe")" = x - rm -f "$__xf_probe" - printf 'Alpine %s; kernel %s; shell %s\\n' \ - "$(cat /etc/alpine-release)" "$(uname -r)" "/bin/sh" - """ - let result = try await emulator.runCommand( - probe, - shell: Self.launchCommand, - timeout: 15, - maxOutput: 64 * 1024 - ) - let detail = result.output.trimmingCharacters(in: .whitespacesAndNewlines) - guard result.status == 0 else { - throw LinuxVMError.guestHealthCheckFailed( - detail.isEmpty ? "health check exited \(result.status)" : detail) - } - XForgeLog.note("rootfs: \(detail); /proc, /dev, /host, and writable root verified") - } - - /// Exercise the same guest-to-host streaming path used by Terminal and every - /// app command. A mounted rootfs is not enough if command output cannot cross - /// the /host realfs bridge back into Swift. - private func verifyCommandBridge() async throws { - let token = "__XFORGE_BRIDGE_OK__" - let output = OutputBox() - let status = try await runLoginStreamingAfterBoot( - "printf '%s\\n' \(GuestShell.quote(token))", - environment: nil, - onOutput: output.append - ) - let received = output.value - guard status == 0, received.contains(token) else { - let detail = received.trimmingCharacters(in: .whitespacesAndNewlines) - throw LinuxVMError.commandBridgeFailed( - detail.isEmpty ? "no output returned from the guest" : detail) - } - XForgeLog.note("bridge: interactive /host command round trip verified") - } - - /// Write the guest's `/etc/resolv.conf` from the device's own DNS. - /// - /// Resolution happens in the guest, and the bundled minirootfs ships no - /// nameservers, so without this `apk add` — the first thing provisioning - /// runs — cannot reach its repositories. Failure is not fatal: the guest - /// still boots, and a user can see what happened in the engine log. - private func configureGuestResolver() async { - guard let share = hostShare else { return } - let (text, source) = GuestNetwork.resolvConfForDevice() - - let staging = share.appendingPathComponent(Self.transferDir, isDirectory: true) - let file = staging.appendingPathComponent("resolv.conf") - do { - try FileManager.default.createDirectory(at: staging, withIntermediateDirectories: true) - try text.write(to: file, atomically: true, encoding: .utf8) - } catch { - XForgeLog.note("dns: could not stage resolv.conf: \(error.localizedDescription)") - return - } - - let status = (try? await runLoginStreamingAfterBoot( - // The file can be a dangling symlink into /run in newer roots, so - // replace it rather than writing through it (same reason the engine's - // app unlinks first). - "rm -f /etc/resolv.conf && cp -f /host/\(Self.transferDir)/resolv.conf /etc/resolv.conf && cat /etc/resolv.conf", - environment: nil - ) { _ in }) ?? -1 - - XForgeLog.note("dns: resolv.conf from \(source) servers (exit \(status)): " - + text.split(separator: "\n").joined(separator: " ")) - - // Do not make Alpine startup wait on a network query. The resolver is - // now installed, and the download command that needs it reports any - // real DNS/network error in its own output. - } - - /// Run every app command through the same interactive login-shell transport - /// used by Terminal, forwarding output while the command is running. - func run( - _ command: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - try await runLoginStreaming( - command, - environment: environment, - onOutput: onOutput - ) - } - - /// Low-level one-shot primitive used only to launch the login-shell transport. - /// App features must call `run` so their commands use the interactive terminal. - private func runCaptured( - _ command: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - try await boot() - return try await runCapturedAfterBoot( - command, - environment: environment, - onOutput: onOutput - ) - } - - private func runCapturedAfterBoot( - _ command: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - let env = GuestShell.environment(environment) - - // Always execute through Alpine's non-login shell. A login wrapper is - // inappropriate for the headless bridge (no controlling TTY/PAM - // session) and causes otherwise valid commands such as cp, nslookup and - // terminal probes to exit 1 before the script body runs. - let result = try await emulator.runCommand( - env + command, - shell: "/bin/sh", - timeout: Self.noTimeout, - maxOutput: Self.outputCap - ) - if !result.output.isEmpty { - onOutput(result.output) - } - if result.truncated { - onOutput("\n[output truncated at \(Self.outputCap / (1024 * 1024)) MB]\n") - } - return result.status - } - - /// Run a script through XForge's default Alpine launch shell, streaming its - /// merged stdout+stderr to `onOutput` **as the guest writes it**. - /// - /// The engine's command primitive returns a command's output only once it - /// has finished, so live output cannot come from there. Instead the guest - /// redirects the session's output into the shared folder (`/host`), which is - /// realfs, and the host tails that file while the (blocking) command is still - /// running. The command's exit status still comes back from the engine. - func runLoginStreaming( - _ script: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - try await boot() - return try await runLoginStreamingAfterBoot( - script, - environment: environment, - onOutput: onOutput - ) - } - - /// Run the shared-folder terminal transport after the caller has completed - /// boot. Keeping this separate prevents the boot health check from awaiting - /// its own in-flight boot task. - private func runLoginStreamingAfterBoot( - _ script: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - guard let share = hostShare else { - // No shared folder to tail; fall back to the captured output. - return try await runCapturedAfterBoot(script, environment: environment, onOutput: onOutput) - } - - let transfer = share.appendingPathComponent(Self.transferDir, isDirectory: true) - try FileManager.default.createDirectory(at: transfer, withIntermediateDirectories: true) - let name = "term-\(UUID().uuidString).log" - let logURL = transfer.appendingPathComponent(name) - FileManager.default.createFile(atPath: logURL.path, contents: nil) - let guestLog = "\(Self.guestShare)/\(Self.transferDir)/\(name)" - - // The C bridge already runs this text as `/bin/sh -c`. A login - // program requires a TTY and exits 1 in this headless command runner. - let wrapped = """ - { - \(script) - } > \(GuestShell.quote(guestLog)) 2>&1 - """ - - let tailer = FileTailer(url: logURL, onChunk: onOutput) - tailer.start() - do { - let status = try await runCapturedAfterBoot(wrapped, environment: environment) { _ in } - tailer.stop() - try? FileManager.default.removeItem(at: logURL) - return status - } catch { - tailer.stop() - try? FileManager.default.removeItem(at: logURL) - throw error - } - } - - // MARK: - Interactive shell - - /// Start the terminal's shell. - /// - /// The shell is not a process this code launches: it is the guest's own - /// console session, started by `/sbin/init` from `/etc/inittab`, which - /// respawns `/sbin/xforge-login root` on the console. So all this does is - /// attach the screen to the console the guest is already driving. - func startInteractiveShell( - onOutput: @escaping @Sendable (String) -> Void - ) async throws -> any InteractiveShellSession { - try await boot() - if let initProblem { - throw LinuxVMError.initFailed(initProblem) - } - let console = try await emulator.openConsole(onOutput: onOutput) - XForgeLog.note("terminal: attached to the guest console") - return ConsoleShellSession(console: console) - } - - // MARK: - File transfer - - /// Copy a file out of the guest into a host URL. - func copyOut(guestPath: String, to hostURL: URL) async throws { - if let share = hostShare { - let name = "out-\(UUID().uuidString)" - let status = try await run( - "mkdir -p \(Self.guestShare)/\(Self.transferDir) && " - + "cp -f \(GuestShell.quote(guestPath)) " - + "\(GuestShell.quote("\(Self.guestShare)/\(Self.transferDir)/\(name)"))", - environment: nil - ) { _ in } - guard status == 0 else { throw LinuxVMError.fileCopyFailed } - - let staged = share - .appendingPathComponent(Self.transferDir, isDirectory: true) - .appendingPathComponent(name) - // Whatever happens next, the staged copy is ours to clean up: a - // *failed* copy used to leave it in the share folder for good, and the - // things copied through here are large (an Xcode.xip is gigabytes, and - // the container has no room to keep two of them). - defer { try? FileManager.default.removeItem(at: staged) } - guard FileManager.default.fileExists(atPath: staged.path) else { - throw LinuxVMError.fileCopyFailed - } - try? FileManager.default.removeItem(at: hostURL) - try FileManager.default.createDirectory( - at: hostURL.deletingLastPathComponent(), withIntermediateDirectories: true) - try FileManager.default.moveItem(at: staged, to: hostURL) - return - } - - // Fallback: base64 over the shell. - let box = OutputBox() - let code = try await run( - "base64 -w0 \(GuestShell.quote(guestPath)) 2>/dev/null || echo __XF_COPY_ERR__", - environment: nil - ) { box.append($0) } - - let trimmed = box.value.trimmingCharacters(in: .whitespacesAndNewlines) - guard code == 0, !trimmed.contains("__XF_COPY_ERR__"), - let data = Data(base64Encoded: trimmed) else { - throw LinuxVMError.fileCopyFailed - } - try FileManager.default.createDirectory( - at: hostURL.deletingLastPathComponent(), withIntermediateDirectories: true) - try data.write(to: hostURL) - } - - /// Copy a host file into the guest. - func copyIn(hostURL: URL, to guestPath: String) async throws { - if let share = hostShare { - let name = "in-\(UUID().uuidString)" - let staging = share - .appendingPathComponent(Self.transferDir, isDirectory: true) - .appendingPathComponent(name) - try FileManager.default.createDirectory( - at: staging.deletingLastPathComponent(), withIntermediateDirectories: true) - try? FileManager.default.removeItem(at: staging) - try FileManager.default.copyItem(at: hostURL, to: staging) - // The guest deletes the shared copy when it has taken it, and this - // covers the case where it does not get that far. - defer { try? FileManager.default.removeItem(at: staging) } - - let dir = (guestPath as NSString).deletingLastPathComponent - let status = try await run( - "mkdir -p \(GuestShell.quote(dir)) && " - + "cp -f \(GuestShell.quote("\(Self.guestShare)/\(Self.transferDir)/\(name)")) " - + "\(GuestShell.quote(guestPath)) && " - + "rm -f \(GuestShell.quote("\(Self.guestShare)/\(Self.transferDir)/\(name)"))", - environment: nil - ) { _ in } - guard status == 0 else { throw LinuxVMError.fileCopyFailed } - return - } - - let data = try Data(contentsOf: hostURL) - let b64 = data.base64EncodedString() - let dir = (guestPath as NSString).deletingLastPathComponent - let status = try await run( - "mkdir -p \(GuestShell.quote(dir)) && base64 -d > \(GuestShell.quote(guestPath)) " - + "<<'__XF_B64__'\n\(b64)\n__XF_B64__\n", - environment: nil - ) { _ in } - guard status == 0 else { throw LinuxVMError.fileCopyFailed } - } -} - -enum LinuxVMError: LocalizedError { - case notImplemented(String) - case fileCopyFailed - case guestDidNotStart - case guestHealthCheckFailed(String) - case commandBridgeFailed(String) - case initFailed(String) - - var errorDescription: String? { - switch self { - case .notImplemented(let m): return m - case .fileCopyFailed: - return "Could not copy the file to/from the embedded Linux." - case .guestDidNotStart: - return "The embedded Linux engine did not start." - case .guestHealthCheckFailed(let detail): - return "The Alpine system started but is not usable: \(detail)" - case .commandBridgeFailed(let detail): - return "Linux started, but XForge could not communicate with it: \(detail)" - case .initFailed(let detail): - return "The guest's init did not start, so there is no console to type " - + "into: \(detail)\nThe bundled rootfs has to provide /sbin/init and " - + "/etc/inittab — an older rootfs is replaced on next launch." - } - } -} - -/// Polls a file as the guest appends to it and hands each new chunk to `onChunk`. -/// -/// This is what makes the terminal live despite the engine's one-shot command -/// primitive: the guest writes its output into the shared folder, and this reads -/// it back while the command is still running. `onChunk` runs on a private queue -/// and must not block (hop to the main actor asynchronously instead). -final class FileTailer: @unchecked Sendable { - private let url: URL - private let onChunk: @Sendable (String) -> Void - private let queue = DispatchQueue(label: "org.xforge.terminal.tail") - private var offset: UInt64 = 0 - private var running = false - - init(url: URL, onChunk: @escaping @Sendable (String) -> Void) { - self.url = url - self.onChunk = onChunk - } - - func start() { - queue.async { [self] in - guard !running else { return } - running = true - poll() - } - } - - func stop() { - queue.sync { [self] in - guard running else { return } - running = false - drain() - } - } - - private func poll() { - guard running else { return } - drain() - queue.asyncAfter(deadline: .now() + .milliseconds(80)) { [self] in poll() } - } - - private func drain() { - guard let handle = try? FileHandle(forReadingFrom: url) else { return } - defer { try? handle.close() } - do { - try handle.seek(toOffset: offset) - guard let data = try handle.readToEnd(), !data.isEmpty else { return } - offset += UInt64(data.count) - onChunk(String(decoding: data, as: UTF8.self)) - } catch { - // The file may not exist (or have new bytes) yet; the next tick retries. - } - } -} diff --git a/App/EmbeddedVM/ISHBridge.c b/App/EmbeddedVM/ISHBridge.c deleted file mode 100644 index 75d7b6d..0000000 --- a/App/EmbeddedVM/ISHBridge.c +++ /dev/null @@ -1,1039 +0,0 @@ -// -// ISHBridge.c -// XForge -// -// C shim over the embedded Linux engine, ish-arm64. See ISHBridge.h. -// -// The boot sequence mirrors what the engine's own iOS app does in -// -[AppDelegate boot] (mount the root, create init, mount the virtual -// filesystems and create device nodes) and how its ISHShellExecutor spawns a -// process (become_new_init_child, wire fds, do_execve, task_start), minus the -// UIKit/terminal machinery XForge does not need. -// -// The engine is built for the iOS *device* only (see -// EmbeddedLinux/build-ish-core.sh), so simulator builds — which the unit tests -// use — compile the stub below instead of linking the engine. -// - -#include "ISHBridge.h" -#include - -#if TARGET_OS_SIMULATOR - -// --- simulator stub --------------------------------------------------------- -// No engine is linked in simulator builds. Every entry point reports clearly. - -#include - -static char s_sim_error[256]; - -const char *xf_ish_last_error(void) { return s_sim_error; } - -static int xf_sim_unsupported(void) { - strncpy(s_sim_error, - "The ish-arm64 Linux engine runs on device only (simulator builds do not link it).", - sizeof(s_sim_error) - 1); - return -1; -} - -int xf_ish_set_log_file(const char *path) { - (void) path; - return 0; -} - -int xf_ish_log(const char *text) { - (void) text; - return 0; -} - -int xf_ish_boot(const char *root_dir, const char *host_dir) { - (void) root_dir; (void) host_dir; - return xf_sim_unsupported(); -} -int xf_ish_is_booted(void) { return 0; } - -int xf_ish_run(const char *command, const char *shell, int timeout_ms, - size_t max_output, xf_guest_result *result) { - (void) command; (void) shell; (void) timeout_ms; (void) max_output; - if (result != NULL) memset(result, 0, sizeof(*result)); - return xf_sim_unsupported(); -} - -int xf_ish_run_detached(const char *command, const char *shell, const char *stdin_path) { - (void) command; (void) shell; (void) stdin_path; - return xf_sim_unsupported(); -} - -int xf_ish_process_alive(int pid) { - // No guest in the simulator, so nothing is running — which is also the right - // answer for a caller polling a pid that does not exist. - (void) pid; - return 0; -} - -int xf_ish_kill_process(int pid, int signal) { - (void) pid; (void) signal; - return xf_sim_unsupported(); -} - -void xf_guest_result_free(xf_guest_result *result) { - if (result != NULL) memset(result, 0, sizeof(*result)); -} - -int xf_ish_start_init(const char *program) { - (void) program; - return xf_sim_unsupported(); -} - -int xf_ish_console_ready(void) { return 0; } - -ssize_t xf_ish_console_read(char *buf, size_t len, int timeout_ms) { - (void) buf; (void) len; (void) timeout_ms; - return xf_sim_unsupported(); -} - -ssize_t xf_ish_console_write(const char *buf, size_t len) { - (void) buf; (void) len; - return xf_sim_unsupported(); -} - -int xf_ish_console_resize(int cols, int rows) { - (void) cols; (void) rows; - return xf_sim_unsupported(); -} - -void xf_ish_shutdown(void) {} - -#else - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// --- ish-arm64 core headers ------------------------------------------------- -#include "kernel/init.h" // mount_root, become_first_process, become_new_init_child -#include "kernel/fs.h" // struct fs_ops, do_mount, fs_register, fakefs/procfs/... -#include "kernel/task.h" // struct task, current, task_start, exit_hook -#include "kernel/calls.h" // do_execve -#include "kernel/signal.h" // send_signal, SIGINFO_NIL -#include "fs/real.h" // realfs -#include "fs/fd.h" // adhoc_fd_create, realfs_fdops -#include "fs/devices.h" // MEM_MAJOR, DEV_NULL_MINOR, TTY_ALTERNATE_MAJOR, ... -#include "fs/tty.h" // struct tty, tty_drivers, tty_input, tty_set_winsize -#include "fs/path.h" // AT_PWD -#include "util/sync.h" // lock, unlock - -// --- state ------------------------------------------------------------------ - -// The engine's `current` is thread-local, so the thread that boots must be the -// one that runs commands. Swift funnels every call through one permanent OS -// thread. -static __thread char s_error[512]; -static bool s_global_inited = false; -static bool s_booted = false; - -static void xf_fail(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - vsnprintf(s_error, sizeof(s_error), fmt, ap); - va_end(ap); -} - -const char *xf_ish_last_error(void) { - return s_error[0] ? s_error : ""; -} - -// --- engine log -------------------------------------------------------------- -// -// The engine's own printk goes through the log handler it was built with -// (-Dlog_handler=nslog here, so it reaches the device console). This file is for -// XForge's own breadcrumbs, which is what a bug report actually needs: the boot -// stage it reached, and the output of the command that failed. - -// Hard ceiling on what the log may grow to. A provisioning run that compiles or -// downloads for an hour would otherwise fill the app container. -#define XF_LOG_MAX_BYTES (4 * 1024 * 1024) - -static int s_log_fd = -1; -static long long s_log_bytes = 0; -static bool s_log_capped = false; - -int xf_ish_log(const char *text) { - if (text == NULL) - return -EINVAL; - if (s_log_fd < 0) - return -ENODEV; - if (s_log_capped) - return -EFBIG; - - struct timeval tv; - gettimeofday(&tv, NULL); - struct tm tm; - time_t secs = tv.tv_sec; - localtime_r(&secs, &tm); - - char line[2048]; - int head = snprintf(line, sizeof(line), "[%02d:%02d:%02d.%03d] ", - tm.tm_hour, tm.tm_min, tm.tm_sec, (int) (tv.tv_usec / 1000)); - if (head < 0) - head = 0; - // Truncate rather than split: a partial line is worse than a clipped one. - size_t room = sizeof(line) - (size_t) head - 2; - size_t len = strlen(text); - if (len > room) - len = room; - memcpy(line + head, text, len); - line[head + (int) len] = '\n'; - ssize_t written = write(s_log_fd, line, (size_t) head + len + 1); - if (written > 0) { - s_log_bytes += written; - if (s_log_bytes > XF_LOG_MAX_BYTES) { - s_log_capped = true; - static const char full[] = - "[xforge] log reached its size limit; further lines are dropped\n"; - write(s_log_fd, full, sizeof(full) - 1); - } - } - return written < 0 ? -errno : 0; -} - -// printf-style breadcrumb into the log file. -static void xf_logf(const char *fmt, ...) { - char buf[1024]; - va_list ap; - va_start(ap, fmt); - vsnprintf(buf, sizeof(buf), fmt, ap); - va_end(ap); - xf_ish_log(buf); -} - -int xf_ish_set_log_file(const char *path) { - if (s_log_fd >= 0) { - close(s_log_fd); - s_log_fd = -1; - } - s_log_bytes = 0; - s_log_capped = false; - if (path == NULL || path[0] == '\0') - return 0; - - int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0644); - if (fd < 0) - return -errno; - // Continue counting from whatever is already in the file, so the ceiling - // holds across the session rather than per call. - struct stat st; - if (fstat(fd, &st) == 0 && st.st_size > 0) - s_log_bytes = st.st_size; - s_log_fd = fd; - return 0; -} - -// --- command exit status ----------------------------------------------------- -// -// The engine reports a process's exit through its global `exit_hook`, which is -// the only place the status of a guest process is observable from the host -// (there is no host-side waitpid for guest pids). XForge runs one command at a -// time on one thread, so the hook records the status of the pid it is waiting -// for and ignores everything else — the guest starts several tasks of its own -// during boot, and those exits must not be mistaken for the command's. -static pthread_mutex_t s_exit_lock = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t s_exit_cond = PTHREAD_COND_INITIALIZER; -static pid_t_ s_wait_pid = 0; -static int s_wait_status = 0; -static bool s_wait_done = false; -/// Pids observed to exit, so a detached process can be distinguished from one -/// still running. A small fixed ring: the app starts a handful of detached -/// processes at most (one terminal shell, occasionally an install), and an -/// over-full ring only means an old entry is forgotten — which degrades to -/// asking the engine, not to a wrong answer. -#define XF_EXITED_MAX 16 -static pid_t_ s_exited[XF_EXITED_MAX]; -static int s_exited_count = 0; - -static bool xf_exit_was_seen(pid_t_ pid) { - for (int i = 0; i < s_exited_count; i++) { - if (s_exited[i] == pid) return true; - } - return false; -} - -static void xf_exit_hook(struct task *task, int code) { - pthread_mutex_lock(&s_exit_lock); - if (task != NULL) { - if (!xf_exit_was_seen(task->pid)) { - if (s_exited_count < XF_EXITED_MAX) { - s_exited[s_exited_count++] = task->pid; - } else { - // Drop the oldest to make room; see the comment above. - for (int i = 1; i < XF_EXITED_MAX; i++) - s_exited[i - 1] = s_exited[i]; - s_exited[XF_EXITED_MAX - 1] = task->pid; - } - } - if (s_wait_pid != 0 && task->pid == s_wait_pid) { - s_wait_status = code; - s_wait_done = true; - pthread_cond_broadcast(&s_exit_cond); - } - } - pthread_mutex_unlock(&s_exit_lock); -} - -// --- guest console ----------------------------------------------------------- -// -// --- pid 1's stdio ---------------------------------------------------------- -// -// The engine's `create_stdio` opens the file it is given with plain O_RDWR, and -// for the console that has a consequence the guest can see: -// * pid 1 is a session leader when this runs (the engine's `construct_task` -// calls `task_setsid`), so the open makes the console *pid 1's* controlling -// terminal, pinning the tty's session to pid 1 (`tty_open` in fs/tty.c -// hands a tty over only while `tty->session == 0`); -// * the login session init later starts does `setsid()` and opens /dev/tty1 — -// and by then the tty has a session, so it never gets it. The shell runs with -// no controlling terminal, which bash reports on the console as -// "cannot set terminal process group (-1): Not a tty" followed by -// "no job control in this shell": no Ctrl-C to the foreground program, no -// Ctrl-Z, no SIGWINCH to the shell on resize. -// -// Linux's init has no controlling terminal either — the console belongs to the -// login session that opens it — and the only difference is O_NOCTTY. So this is -// `create_stdio` with that flag, kept here rather than patching the vendored -// engine: the engine's version is right for every other caller. -static int xf_create_console_stdio(void) { - struct fd *fd = generic_open("/dev/console", O_RDWR_ | O_NOCTTY_, 0); - if (!IS_ERR(fd) && !S_ISCHR(fd->stat.mode)) { - // Opened a regular file instead of a character device (realfs cannot - // create device nodes), so close it and take the same fallback the engine - // does. - fd_close(fd); - fd = ERR_PTR(_ENOENT); - } - if (IS_ERR(fd)) { - fd = adhoc_fd_create(NULL); - if (fd == NULL) - return _ENOMEM; - fd->stat.rdev = dev_make(TTY_CONSOLE_MAJOR, 1); - fd->stat.mode = S_IFCHR | S_IRUSR; - // O_NOCTTY again: the fallback opens the tty directly, and it must not - // claim the session either. - fd->flags = O_RDWR_ | O_NOCTTY_; - int dev_err = dev_open(TTY_CONSOLE_MAJOR, 1, DEV_CHAR, fd); - if (dev_err < 0) - return dev_err; - } - - fd->refcount = 0; - current->files->files[0] = fd_retain(fd); - current->files->files[1] = fd_retain(fd); - current->files->files[2] = fd_retain(fd); - xf_logf("boot: pid 1's stdio is the console (O_NOCTTY — the login session owns it)"); - return 0; -} - -// The guest's console is a tty the *host* implements. Everything above the -// driver — the line discipline, echo, signal generation, job control — is the -// guest kernel's, so the app gets a real terminal rather than a pipe: Ctrl-C -// becomes SIGINT for the foreground program, the shell does its own line -// editing and history, and a full-screen program can put the tty in raw mode. -// -// The engine calls the driver's write op from whichever guest thread is doing -// the writing, so this side is only a byte buffer plus a condition variable. The -// host reader blocks in xf_ish_console_read on a thread of its own, which is -// safe: that call touches no per-thread engine state. - -#define XF_CONSOLE_BUFFER_BYTES (256 * 1024) - -static pthread_mutex_t s_console_lock = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t s_console_cond = PTHREAD_COND_INITIALIZER; -static char s_console_buffer[XF_CONSOLE_BUFFER_BYTES]; -static size_t s_console_head = 0; // index of the oldest unread byte -static size_t s_console_length = 0; // unread bytes -static long long s_console_dropped = 0; -static bool s_console_stopping = false; -static struct tty *s_console_tty = NULL; - -/// Append what the guest wrote, waking the host reader. -static void xf_console_push(const char *data, size_t len) { - pthread_mutex_lock(&s_console_lock); - bool first_drop = false; - for (size_t i = 0; i < len; i++) { - if (s_console_length == sizeof(s_console_buffer)) { - // The reader has fallen a quarter of a megabyte behind. Drop the - // oldest byte rather than block: a writer blocked inside the kernel - // stalls the whole guest, which is much worse than a gap in the - // scrollback. Only report the first loss, to keep this cheap. - s_console_head = (s_console_head + 1) % sizeof(s_console_buffer); - s_console_length--; - s_console_dropped++; - first_drop = first_drop || s_console_dropped == 1; - } - s_console_buffer[(s_console_head + s_console_length) % sizeof(s_console_buffer)] = data[i]; - s_console_length++; - } - pthread_cond_broadcast(&s_console_cond); - pthread_mutex_unlock(&s_console_lock); - if (first_drop) - xf_logf("console: output is arriving faster than it is read; dropping bytes"); -} - -static int xf_console_tty_init(struct tty *tty) { - // Called from tty_get with ttys_lock held, so record the tty and nothing - // else — no behaviour that could take another lock or block. - if (tty->num == 1) { - pthread_mutex_lock(&s_console_lock); - s_console_tty = tty; - pthread_mutex_unlock(&s_console_lock); - xf_logf("console: tty1 attached"); - } - return 0; -} - -static int xf_console_tty_write(struct tty *tty, const void *buf, size_t len, bool blocking) { - (void) blocking; - if (tty->num == 1 && len > 0) - xf_console_push((const char *) buf, len); - // Claim every byte. Under-reporting would leave the kernel waiting for room - // that only this side could free. - return (int) len; -} - -static void xf_console_tty_cleanup(struct tty *tty) { - if (tty->num != 1) - return; - pthread_mutex_lock(&s_console_lock); - if (s_console_tty == tty) - s_console_tty = NULL; - pthread_cond_broadcast(&s_console_cond); - pthread_mutex_unlock(&s_console_lock); -} - -static struct tty_driver_ops xf_console_ops = { - .init = xf_console_tty_init, - .write = xf_console_tty_write, - .cleanup = xf_console_tty_cleanup, -}; - -// Major 4 (TTY_CONSOLE_MAJOR), minors 1..7. Only minor 1 has a screen; the rest -// accept and discard, so a program writing to /dev/tty2 cannot wedge. -DEFINE_TTY_DRIVER(xf_console_driver, &xf_console_ops, TTY_CONSOLE_MAJOR, 8); - -static struct tty *xf_console_tty(void) { - pthread_mutex_lock(&s_console_lock); - struct tty *tty = s_console_tty; - pthread_mutex_unlock(&s_console_lock); - return tty; -} - -int xf_ish_console_ready(void) { - return xf_console_tty() != NULL ? 1 : 0; -} - -ssize_t xf_ish_console_read(char *buf, size_t len, int timeout_ms) { - if (buf == NULL || len == 0) - return -EINVAL; - - pthread_mutex_lock(&s_console_lock); - if (timeout_ms != 0) { - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - if (timeout_ms > 0) { - deadline.tv_sec += timeout_ms / 1000; - deadline.tv_nsec += (long) (timeout_ms % 1000) * 1000000L; - if (deadline.tv_nsec >= 1000000000L) { - deadline.tv_sec++; - deadline.tv_nsec -= 1000000000L; - } - } - while (s_console_length == 0 && !s_console_stopping) { - if (timeout_ms < 0) { - pthread_cond_wait(&s_console_cond, &s_console_lock); - } else if (pthread_cond_timedwait(&s_console_cond, &s_console_lock, &deadline) == ETIMEDOUT) { - break; - } - } - } - - size_t take = s_console_length < len ? s_console_length : len; - for (size_t i = 0; i < take; i++) - buf[i] = s_console_buffer[(s_console_head + i) % sizeof(s_console_buffer)]; - s_console_head = (s_console_head + take) % sizeof(s_console_buffer); - s_console_length -= take; - pthread_mutex_unlock(&s_console_lock); - return (ssize_t) take; -} - -ssize_t xf_ish_console_write(const char *buf, size_t len) { - if (buf == NULL) - return -EINVAL; - if (len == 0) - return 0; - - struct tty *tty = xf_console_tty(); - if (tty == NULL) - return -ENODEV; - // tty_input takes the tty's own lock and reads no per-thread engine state, - // so input can be pushed from the UI thread while the guest is running — - // which is the whole point of a terminal you can type into. - return tty_input(tty, buf, len, false); -} - -int xf_ish_console_resize(int cols, int rows) { - if (cols <= 0 || rows <= 0) - return -EINVAL; - - struct tty *tty = xf_console_tty(); - if (tty == NULL) - return -ENODEV; - // tty_set_winsize does not lock; it also raises SIGWINCH for the foreground - // group, which is what makes a full-screen program redraw itself. - lock(&tty->lock); - tty_set_winsize(tty, (struct winsize_) { - .row = (word_t) rows, - .col = (word_t) cols, - }); - unlock(&tty->lock); - return 0; -} - -// --- command output ---------------------------------------------------------- - -/// Read chunk for the blocking command runner's pipe. -#define XF_READ_CHUNK 8192 - -// --- API -------------------------------------------------------------------- - -int xf_ish_is_booted(void) { - return s_booted ? 1 : 0; -} - -// One-time process-global emulator init. Mirrors what the engine's app does -// before its first guest process: arm the memory governor, then seed the -// emulator. Kept to what a headless host needs. -static void xf_global_init(void) { - if (s_global_inited) return; - s_global_inited = true; - xf_logf("engine: global init"); -} - -// The bundled minirootfs ships an empty /dev, and the guest userland needs the -// standard nodes before anything else runs. Mirrors the device nodes the -// engine's app creates in -[AppDelegate boot]. -static void xf_ensure_devices(void) { - generic_mkdirat(AT_PWD, "/dev", 0755); - generic_mkdirat(AT_PWD, "/dev/pts", 0755); - generic_mkdirat(AT_PWD, "/dev/shm", 01777); - generic_mkdirat(AT_PWD, "/run", 0755); - - static const struct { const char *path; mode_t_ mode; int major; int minor; } nodes[] = { - {"/dev/null", S_IFCHR | 0666, MEM_MAJOR, DEV_NULL_MINOR}, - {"/dev/zero", S_IFCHR | 0666, MEM_MAJOR, DEV_ZERO_MINOR}, - {"/dev/full", S_IFCHR | 0666, MEM_MAJOR, DEV_FULL_MINOR}, - {"/dev/random", S_IFCHR | 0666, MEM_MAJOR, DEV_RANDOM_MINOR}, - {"/dev/urandom", S_IFCHR | 0666, MEM_MAJOR, DEV_URANDOM_MINOR}, - {"/dev/tty", S_IFCHR | 0666, TTY_ALTERNATE_MAJOR, DEV_TTY_MINOR}, - {"/dev/console", S_IFCHR | 0666, TTY_ALTERNATE_MAJOR, DEV_CONSOLE_MINOR}, - {"/dev/ptmx", S_IFCHR | 0666, TTY_ALTERNATE_MAJOR, DEV_PTMX_MINOR}, - }; - for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) { - // EEXIST is expected on every boot after the first. - generic_mknodat(AT_PWD, nodes[i].path, nodes[i].mode, - dev_make(nodes[i].major, nodes[i].minor)); - } - - // Linux guarantees these modes; a tarball can leave them stricter. - generic_setattrat(AT_PWD, "/tmp", (struct attr) { .type = attr_mode, .mode = S_IFDIR | 01777 }, false); - generic_setattrat(AT_PWD, "/", (struct attr) { .type = attr_mode, .mode = S_IFDIR | 0755 }, false); -} - -int xf_ish_boot(const char *root_dir, const char *host_dir) { - if (root_dir == NULL) return -EINVAL; - if (s_booted) return 0; - - xf_global_init(); - xf_logf("boot: mounting %s/data as /", root_dir); - - // The fakefs filesystem is the root's `data` directory; `meta.db` sits - // beside it. This is the same source the engine's app passes to mount_root. - char data_dir[4096]; - int n = snprintf(data_dir, sizeof(data_dir), "%s/data", root_dir); - if (n <= 0 || (size_t) n >= sizeof(data_dir)) { - xf_fail("root path too long"); - return -ENAMETOOLONG; - } - - intptr_t err = mount_root(&fakefs, data_dir); - if (err < 0) { - xf_logf("boot: mount_root failed: %s (%ld)", strerror((int) -err), (long) err); - xf_fail("mount_root failed: %s (%ld)", strerror((int) -err), (long) err); - return (int) err; - } - xf_logf("boot: root mounted"); - - // NOTE: nothing is registered here on purpose. The engine's fs/mount.c - // already carries a static table of the filesystems it uses (realfs, - // procfs, devptsfs, tmpfs) and MAX_FILESYSTEMS is 10. A registration path - // that overflows it hits `assert(!"reached filesystem limit")`, which is an - // abort() — and in an iOS app stderr goes nowhere, so that presents as the - // app simply dying during boot with no message. Registration is only for - // filesystems that are *not* in that table, so there is nothing to add. - - // The engine reports guest exits through this global hook; it is how - // xf_ish_run learns a command's exit status. - exit_hook = xf_exit_hook; - - // Creates pid 1. After this the guest has an init task, which is all the - // headless command runner needs — XForge does not run /sbin/init, it runs - // build commands as fresh children of init. - xf_logf("boot: become_first_process"); - err = become_first_process(); - if (err < 0) { - xf_logf("boot: become_first_process failed: %s (%ld)", strerror((int) -err), (long) err); - xf_fail("become_first_process failed: %s (%ld)", strerror((int) -err), (long) err); - return (int) err; - } - xf_logf("boot: pid 1 created"); - - // Virtual filesystems and device nodes the guest userland expects. Failures - // are not fatal: the rootfs may already provide some of these. - xf_logf("boot: device nodes + /proc /dev/pts"); - xf_ensure_devices(); - do_mount(&procfs, "proc", "/proc", "", 0); - do_mount(&devptsfs, "devpts", "/dev/pts", "", 0); - - // The console. Registering the host's tty driver *before* pid 1's stdio is - // opened is what makes /dev/console and /dev/tty1 the same terminal, and that - // terminal is the app's screen — so init, the console session it respawns, and - // the shell the user types into all appear there. - xf_logf("boot: console tty driver"); - pthread_mutex_lock(&s_console_lock); - s_console_stopping = false; - pthread_mutex_unlock(&s_console_lock); - tty_drivers[TTY_CONSOLE_MAJOR] = &xf_console_driver; - generic_mknodat(AT_PWD, "/dev/tty1", S_IFCHR | 0666, dev_make(TTY_CONSOLE_MAJOR, 1)); - set_console_device(TTY_CONSOLE_MAJOR, 1); - - // Share the app's own container into the guest at /host (realfs) so large - // artifacts — the darwin Swift SDK is hundreds of megabytes — can be staged - // by the host instead of being pushed through the command pipe. Not fatal. - if (host_dir != NULL && host_dir[0] != '\0') { - generic_mkdirat(AT_PWD, "/host", 0777); - int merr = do_mount(&realfs, host_dir, "/host", "", 0); - if (merr < 0) { - xf_logf("boot: could not share %s at /host: %s", host_dir, strerror(-merr)); - } else { - xf_logf("boot: %s shared at /host", host_dir); - } - } - - // pid 1's stdin/stdout/stderr are the console. Nothing inherits them by - // accident — the headless command runner gives every child its own fds — but - // init and everything init starts do, which is how the guest's own boot - // appears on the screen. - // - // Opened with O_NOCTTY, which is the difference between a console that works - // and one whose shell has no job control — see xf_create_console_stdio. - int stdio_err = xf_create_console_stdio(); - if (stdio_err < 0) { - // Not fatal for the command runner, which does not use pid 1's stdio. - // `xf_ish_start_init` will report the real problem if the console is - // missing, and the terminal shows that to the user. - xf_logf("boot: could not wire pid 1's stdio to /dev/console: %s", - strerror((int) -stdio_err)); - } - - s_booted = true; - xf_logf("boot: guest is up (pid 1 created; %s)", - xf_ish_console_ready() ? "console attached" : "no console"); - return 0; -} - -int xf_ish_start_init(const char *program) { - if (!s_booted) { - xf_fail("guest is not booted"); - return -ENODEV; - } - - const char *path = (program != NULL && program[0] != '\0') ? program : "/sbin/init"; - xf_logf("init: exec %s as pid 1", path); - - // argv, as do_execve wants it: NUL-separated entries and a final NUL. - size_t path_len = strlen(path); - char *argv = malloc(path_len + 2); - if (argv == NULL) { - xf_fail("out of memory building init's argv"); - return -ENOMEM; - } - memcpy(argv, path, path_len + 1); - argv[path_len + 1] = '\0'; - - // init gets the environment a console session expects; /sbin/xforge-login - // replaces TERM/HOME/PATH with its own anyway, and /etc/profile re-exports - // them for the shell it execs. - static const char *const envp = - "TERM=xterm-256color\0" - "HOME=/root\0" - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\0"; - - int err = do_execve(path, 1, argv, envp); - free(argv); - if (err < 0) { - xf_logf("init: do_execve(%s) failed: %s (%d)", path, strerror(-err), err); - xf_fail("do_execve(%s) failed: %s (%d)", path, strerror(-err), err); - return err; - } - // init runs on a thread of its own, like every other guest task, so the - // engine thread is free again as soon as this returns. - int start_err = task_start(current); - if (start_err != 0) { - // task_start reports a guest errno (negative), like the rest of the engine. - xf_logf("init: could not start pid 1: %s (%d)", strerror(-start_err), start_err); - xf_fail("could not start pid 1: %s (%d)", strerror(-start_err), start_err); - return start_err; - } - return 0; -} -// --- spawning a guest child -------------------------------------------------- -// -// Shared by the blocking and the detached entry points below: everything up to -// and including `task_start`, which is the point where the two diverge (one waits -// for the child, the other returns its pid and lets it run). -// -// `stdout_fd` is where the child's stdout and stderr go; pass -1 for none (a -// detached command redirects its own output in its command line). -// `stdin_path` is opened for the child's stdin; NULL means /dev/null. -// On success the child's pid is written to `*pid_out` and 0 is returned; on -// failure a negative errno is returned and nothing has been started. -static int xf_spawn_child(const char *command, const char *shell, - int stdout_fd, const char *stdin_path, - pid_t_ *pid_out) { - struct task *saved_current = current; - - int err = become_new_init_child(); - if (err < 0) { - current = saved_current; - xf_fail("become_new_init_child failed: %s (%d)", strerror(-err), err); - return err; - } - struct task *task = current; - - // stdin. A detached interactive shell is given a real file here: that file is - // its standard input for as long as it lives, which is what lets the host - // feed it a byte stream. - struct fd *stdin_fd = adhoc_fd_create(&realfs_fdops); - if (stdin_fd != NULL) { - const char *path = (stdin_path != NULL && stdin_path[0] != '\0') - ? stdin_path : "/dev/null"; - int fd = open(path, O_RDONLY); - if (fd < 0 && stdin_path != NULL) { - // Falling back to /dev/null would leave a shell reading EOF and - // exiting immediately, so report it instead of starting something - // that cannot work. - int open_err = errno; - current = saved_current; - xf_fail("could not open stdin %s: %s", path, strerror(open_err)); - return -open_err; - } - stdin_fd->real_fd = fd; - task->files->files[0] = stdin_fd; - } - - for (int idx = 1; idx <= 2; idx++) { - if (stdout_fd < 0) - break; - struct fd *out_fd = adhoc_fd_create(&realfs_fdops); - if (out_fd != NULL) { - out_fd->real_fd = dup(stdout_fd); - task->files->files[idx] = out_fd; - } - } - - // argv, as the NUL-separated, double-NUL-terminated block do_execve wants. - // `-c` is what turns the command into something /bin/sh runs and reports the - // exit status of. - static const char *const argv0 = "sh"; - static const char *const flag = "-c"; - size_t argv_len = strlen(argv0) + 1 + strlen(flag) + 1 + strlen(command) + 1 + 1; - char *argv_buf = calloc(1, argv_len); - if (argv_buf == NULL) { - current = saved_current; - xf_fail("out of memory building argv"); - return -ENOMEM; - } - size_t pos = 0; - memcpy(argv_buf + pos, argv0, strlen(argv0) + 1); pos += strlen(argv0) + 1; - memcpy(argv_buf + pos, flag, strlen(flag) + 1); pos += strlen(flag) + 1; - memcpy(argv_buf + pos, command, strlen(command) + 1); pos += strlen(command) + 1; - argv_buf[pos] = '\0'; - - // A build environment needs a real PATH, and HOME for the toolchains. TERM is - // set so an interactive shell does not refuse to prompt. - static const char *const envp = - "TERM=xterm-256color\0" - "HOME=/root\0" - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\0"; - - err = do_execve(shell, 3, argv_buf, envp); - if (err < 0) { - free(argv_buf); - current = saved_current; - xf_fail("do_execve(%s) failed: %s (%d)", shell, strerror(-err), err); - return err; - } - free(argv_buf); - - *pid_out = task->pid; - task_start(task); - current = saved_current; - return 0; -} - -int xf_ish_run(const char *command, const char *shell, int timeout_ms, - size_t max_output, xf_guest_result *result) { - if (command == NULL || result == NULL) return -EINVAL; - memset(result, 0, sizeof(*result)); - - if (!s_booted) { - xf_fail("guest is not booted"); - return -ENODEV; - } - - const char *exec_path = (shell != NULL && shell[0] != '\0') ? shell : "/bin/sh"; - xf_logf("run: %.200s", command); - - int pipe_fds[2]; - if (pipe(pipe_fds) != 0) { - xf_fail("pipe failed: %s", strerror(errno)); - return -errno; - } - - // stdout+stderr to the pipe, merged — XForge presents one combined - // transcript, and separating them would need a second reader thread for no - // gain here. - pid_t_ pid = 0; - int err = xf_spawn_child(command, exec_path, pipe_fds[1], NULL, &pid); - close(pipe_fds[1]); - if (err < 0) { - close(pipe_fds[0]); - return err; - } - - result->launched = 1; - - // Register interest in this pid BEFORE reading, so a command that finishes - // immediately cannot report its exit before anyone is listening. - pthread_mutex_lock(&s_exit_lock); - s_wait_pid = pid; - s_wait_status = 0; - s_wait_done = false; - pthread_mutex_unlock(&s_exit_lock); - - // Read until the child closes the pipe (it does so when it exits), which - // both collects the output and waits for the command. The timeout is - // enforced with poll() so a hung guest command cannot wedge XForge. - char *buffer = malloc(XF_READ_CHUNK * 4); - if (buffer == NULL) { - close(pipe_fds[0]); - pthread_mutex_lock(&s_exit_lock); - s_wait_pid = 0; - pthread_mutex_unlock(&s_exit_lock); - xf_fail("out of memory reading command output"); - return -ENOMEM; - } - size_t length = 0, capacity = XF_READ_CHUNK * 4; - int truncated = 0; - struct timeval start; - gettimeofday(&start, NULL); - bool pipe_eof = false; - - while (!pipe_eof) { - if (timeout_ms > 0) { - struct timeval now; - gettimeofday(&now, NULL); - long long elapsed = (long long) (now.tv_sec - start.tv_sec) * 1000 - + (now.tv_usec - start.tv_usec) / 1000; - if (elapsed >= timeout_ms) { - result->timed_out = 1; - xf_logf("run: timeout after %d ms (pid %d)", timeout_ms, (int) pid); - break; - } - struct pollfd pfd = { .fd = pipe_fds[0], .events = POLLIN }; - int pr = poll(&pfd, 1, (int) (timeout_ms - elapsed)); - if (pr < 0 && errno != EINTR) { - pipe_eof = true; - break; - } - if (pr == 0) { - result->timed_out = 1; - xf_logf("run: timeout waiting for pid %d", (int) pid); - break; - } - } - // poll() said there is data (or the write end closed, which shows up as - // POLLHUP). Drain what is available; EOF ends the loop. - char chunk[XF_READ_CHUNK]; - ssize_t n = read(pipe_fds[0], chunk, sizeof(chunk)); - if (n < 0) { - if (errno == EINTR) - continue; - break; - } - if (n == 0) { - pipe_eof = true; - break; - } - if (max_output == 0 || length < max_output) { - size_t room = (max_output == 0 ? (size_t) n : max_output - length); - size_t take = (size_t) n < room ? (size_t) n : room; - if (length + take + 1 > capacity) { - size_t want = capacity * 2; - while (want < length + take + 1) - want *= 2; - char *grown = realloc(buffer, want); - if (grown == NULL) - break; - buffer = grown; - capacity = want; - } - memcpy(buffer + length, chunk, take); - length += take; - if (take < (size_t) n) - truncated = 1; - } else { - truncated = 1; - } - } - - close(pipe_fds[0]); - buffer[length] = '\0'; - - // Wait for the exit status, briefly: the pipe closes when the process exits, - // but the hook that records the status may run a moment later. - pthread_mutex_lock(&s_exit_lock); - if (!s_wait_done) { - struct timeval then; - gettimeofday(&then, NULL); - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += 5; - while (!s_wait_done) { - if (pthread_cond_timedwait(&s_exit_cond, &s_exit_lock, &deadline) != 0) - break; - } - (void) then; - } - bool have_status = s_wait_done; - int status = s_wait_status; - s_wait_pid = 0; - s_wait_done = false; - pthread_mutex_unlock(&s_exit_lock); - - result->truncated = truncated; - result->output = buffer; - result->output_len = length; - if (have_status) { - result->exited = 1; - // The guest reports a wait(2)-style status; expose the plain exit code. - result->exit_code = (status & 0x7f) == 0 ? ((status >> 8) & 0xff) : 0; - result->term_signal = (status & 0x7f) == 0 ? 0 : (status & 0x7f); - } else if (!result->timed_out) { - // Output stopped but no exit was observed: the command was killed or the - // engine never reported it. Say so rather than reporting success. - result->exited = 1; - result->exit_code = -1; - } - - xf_logf("run: collected %zu bytes (truncated=%d timed_out=%d exited=%d " - "code=%d signal=%d)", - length, truncated, result->timed_out, result->exited, - result->exit_code, result->term_signal); - return 0; -} - -// --- detached processes ------------------------------------------------------ - -int xf_ish_run_detached(const char *command, const char *shell, const char *stdin_path) { - if (command == NULL) return -EINVAL; - if (!s_booted) { - xf_fail("guest is not booted"); - return -ENODEV; - } - - const char *exec_path = (shell != NULL && shell[0] != '\0') ? shell : "/bin/sh"; - xf_logf("run-detached: %.200s", command); - - // No stdout fd: a detached command redirects its own output, because there is - // no reader here to drain a pipe. That matters — a child writing into a pipe - // nobody reads fills the pipe buffer and blocks forever. - pid_t_ pid = 0; - int err = xf_spawn_child(command, exec_path, -1, stdin_path, &pid); - if (err < 0) - return err; - - xf_logf("run-detached: started pid %d", (int) pid); - return (int) pid; -} - -int xf_ish_process_alive(int pid) { - if (pid <= 0) return -EINVAL; - - pthread_mutex_lock(&s_exit_lock); - bool known_exited = xf_exit_was_seen((pid_t_) pid); - pthread_mutex_unlock(&s_exit_lock); - if (known_exited) return 0; - - // The engine gives no waitpid for guest pids, so ask the task table directly. - // A pid that has exited but not been reaped still appears here as a zombie, - // which is why the flag above is checked first: the exit hook is the only - // place that observes an exit reliably. - struct task *task = pid_get_task_zombie((dword_t) pid); - if (task == NULL) return 0; - return task->zombie ? 0 : 1; -} - -int xf_ish_kill_process(int pid, int signal) { - if (pid <= 0) return -EINVAL; - - struct task *task = pid_get_task_zombie((dword_t) pid); - if (task == NULL) { - xf_fail("no such process: %d", pid); - return -ESRCH; - } - // A real guest signal, not a byte on stdin. Sending the Ctrl-C *character* - // would need a tty to interpret it — the terminal's transport is a plain - // pipe, so the byte would simply be read as input. This reaches the process - // regardless of what its stdin is. - xf_logf("kill: pid %d signal %d", pid, signal); - send_signal(task, signal, SIGINFO_NIL); - return 0; -} - -void xf_guest_result_free(xf_guest_result *result) { - if (result == NULL) return; - free(result->output); - memset(result, 0, sizeof(*result)); -} - -void xf_ish_shutdown(void) { - // The engine cannot boot a second machine inside the same process, so there - // is nothing to tear down beyond forgetting that we booted — but a reader - // blocked in xf_ish_console_read has to be let go, or it would outlive the - // session it was reading for. - pthread_mutex_lock(&s_console_lock); - s_console_stopping = true; - pthread_cond_broadcast(&s_console_cond); - pthread_mutex_unlock(&s_console_lock); - s_booted = false; -} - -#endif /* TARGET_OS_SIMULATOR */ diff --git a/App/EmbeddedVM/ISHBridge.h b/App/EmbeddedVM/ISHBridge.h deleted file mode 100644 index 4aee34b..0000000 --- a/App/EmbeddedVM/ISHBridge.h +++ /dev/null @@ -1,178 +0,0 @@ -// -// ISHBridge.h -// XForge -// -// Thin C shim over the embedded Linux engine, **ish-arm64** -// (github.com/OpenMinis/ish-arm64). -// -// ish-arm64 is a fork of ish-app/ish that adds a native AArch64 guest backend -// to the threaded-code interpreter. That matters for XForge: the bundled root -// filesystem is an Alpine *aarch64* minirootfs, so with this engine it runs as -// a same-architecture guest (a handful of host instructions per guest -// instruction) instead of being cross-translated from x86. It emulates -// instructions by dispatching to pre-compiled "gadget" functions — it does not -// emit machine code at runtime and needs no executable memory, so it works in -// a sideloaded app with no JIT entitlement. -// -// XForge links the engine's core static libraries (`libish`, `libish_emu`, -// `libfakefs`) plus `tools/fakefs.c` (for `fakefs_import`) and this shim, then -// boots the root filesystem bundled in the app. -// -// Everything here is a plain C ABI so Swift can call it through the bridging -// header. All calls must be made from one dedicated permanent OS thread: -// the engine's `current` task pointer is thread-local, and a serial dispatch -// queue does not guarantee pthread affinity. -// -#ifndef XFORGE_ISH_BRIDGE_H -#define XFORGE_ISH_BRIDGE_H - -#include -#include - -/// Outcome of one headless guest command. -typedef struct xf_guest_result { - int launched; ///< the command started in the guest - int exited; ///< exit_code is valid - int exit_code; ///< exit status when exited != 0 - int term_signal; ///< terminating signal when exited == 0 - int timed_out; ///< killed because timeout_ms elapsed - int truncated; ///< output hit max_output - char *output; ///< merged stdout+stderr, malloc'd (free with xf_guest_result_free) - size_t output_len; -} xf_guest_result; - -/// Point the engine's own log at `path` (append mode) and remember it for -/// `xf_ish_log`. Pass NULL to stop logging. -/// -/// The engine's `printk` goes through its `log_handler`, chosen at build time. -/// XForge builds with `-Dlog_handler=nslog` (see EmbeddedLinux/build-ish-core.sh) -/// so kernel messages reach the device console. This file is for the messages -/// XForge itself produces around them — boot stages, import progress, and the -/// output of a command that failed — which is what makes a bug report readable. -/// -/// Returns 0 on success or a negative errno. -int xf_ish_set_log_file(const char *path); - -/// Append one timestamped line to the log set by `xf_ish_set_log_file`. -/// No-op (returning -ENODEV) when no log file is set. -int xf_ish_log(const char *text); - -/// Console — the guest's own terminal. -/// -/// XForge implements the guest console as a tty driver, which is what makes the -/// terminal a *terminal* rather than a pipe: the line discipline lives in the -/// guest's kernel, so echo, line editing, Ctrl-C (ISIG), Ctrl-D and job control -/// all behave exactly as they do on a console. The engine calls the driver when -/// the guest writes to `/dev/tty1` or `/dev/console`; the host pushes input back -/// with `xf_ish_console_write`. -/// -/// `/dev/tty1` and `/dev/console` are the same terminal (major `TTY_CONSOLE_MAJOR`, -/// minor 1), and it is the one pid 1's stdio is wired to — so `/sbin/init` and -/// everything it starts (the console session and the shell it execs) talk to the -/// screen in the app. -/// -/// Read and write are safe from any thread: the write callback only appends to a -/// host buffer, `xf_ish_console_write` takes the tty's own lock inside the engine, -/// and neither touches the engine's per-thread state. - -/// Start the guest's init as pid 1, giving it the console as its stdio. -/// -/// This is what turns the mounted root into a *booted system*: `/sbin/init` -/// reads `/etc/inittab`, which respawns `/sbin/xforge-login root` on the console. -/// Must be called on the engine thread, after `xf_ish_boot`. Returns 0, or a -/// negative errno. -int xf_ish_start_init(const char *program); - -/// 1 once the guest console exists (something has opened `/dev/console`). -int xf_ish_console_ready(void); - -/// Copy output produced by the guest console into `buf`, waiting up to -/// `timeout_ms` for at least one byte (-1 waits indefinitely, 0 returns at once). -/// -/// Returns the number of bytes copied, 0 if the wait timed out, or a negative -/// errno. Callers poll this in a loop; it does not touch the engine's thread- -/// local state, so it may block on a thread of its own. -ssize_t xf_ish_console_read(char *buf, size_t len, int timeout_ms); - -/// Feed host input into the guest console (keystrokes, pastes, control bytes). -/// Returns the number of bytes accepted, or a negative errno. -ssize_t xf_ish_console_write(const char *buf, size_t len); - -/// Tell the guest console how big the screen is, so the guest's programs wrap -/// and lay out correctly. Returns 0, or a negative errno. -int xf_ish_console_resize(int cols, int rows); - -/// Boot the guest from an installed fakefs root directory. -/// -/// The root is *already* fakefs — a `data/` tree plus `meta.db`, converted on the -/// build machine by `tools/fakefsify` (see EmbeddedLinux/build-rootfs.sh). The app -/// only unzips it, so there is no import step here: `fakefs_import` is a build-time -/// tool in this design, not a runtime one. -/// -/// `host_dir` (may be NULL) is mounted read-write into the guest at `/host`, so -/// the app's own container is reachable from inside Linux. That is how large -/// artifacts get in without pushing megabytes through the shell pipe. Returns 0 -/// on success or a negative errno. -/// -/// This creates pid 1, mounts the virtual filesystems, wires pid 1's stdio to the -/// console tty and brings up the host's end of that terminal — but it does not -/// exec anything: the caller decides when the guest's own `init` takes over, with -/// `xf_ish_start_init`. Keeping the two apart is what lets XForge verify the -/// mounted root before handing the machine to `/sbin/init`. -int xf_ish_boot(const char *root_dir, const char *host_dir); - -/// 1 once the guest is booted and commands can be run. -int xf_ish_is_booted(void); - -/// Run one command headlessly as a fresh child of init and capture its merged -/// stdout+stderr. `shell` is an absolute guest path (e.g. "/bin/sh") or NULL for -/// `/bin/sh`. `timeout_ms` of 0 means no timeout. Returns 0 if the command was -/// launched (inspect `result`), or a negative errno if it could not start. -/// -/// This call blocks until the command exits. -int xf_ish_run(const char *command, const char *shell, int timeout_ms, - size_t max_output, xf_guest_result *result); - -/// Start a command and return its guest pid immediately, without waiting for it. -/// -/// This exists because `xf_ish_run` blocks until the child exits, and the guest -/// engine may only be driven from one thread at a time — so a long-lived process -/// (an interactive shell) started through `xf_ish_run` would hold that thread -/// forever and nothing else could ever run. Starting it detached leaves the -/// engine free, and the caller polls the pid instead. -/// -/// Returns the guest pid (> 0) on success, or a negative errno. -/// -/// The process reads its stdin from `stdin_path` when that is not NULL: it is -/// opened once, when the child is created. Output is *not* captured — a detached -/// command is expected to redirect its own stdout/stderr (that is how the -/// terminal's transport works), so pass a path in the command itself. -int xf_ish_run_detached(const char *command, const char *shell, const char *stdin_path); - -/// Whether a detached process is still running. Returns 1 while it is alive, -/// 0 once it has exited, or a negative errno for an unknown pid. -int xf_ish_process_alive(int pid); - -// There is deliberately no blocking "wait for a detached process": the engine -// may only be driven from one thread, and a wait would hold it for the process's -// whole lifetime — the starvation that detached runs exist to avoid. Callers poll -// `xf_ish_process_alive` instead, which answers without blocking. - -/// Ask a detached process to stop, by delivering `signal` to it. Returns 0 if the -/// signal was delivered or a negative errno. -/// -/// This is a real guest signal, so Ctrl-C on an interactive shell reaches the -/// foreground program rather than merely detaching the screen. -int xf_ish_kill_process(int pid, int signal); - -/// Free `result->output` and zero the struct. -void xf_guest_result_free(xf_guest_result *result); - -/// Shut the guest down for this launch. The engine cannot boot a second machine -/// in the same process, so this only tears down local state. -void xf_ish_shutdown(void); - -/// Human-readable description of the last failure on this thread. -const char *xf_ish_last_error(void); - -#endif /* XFORGE_ISH_BRIDGE_H */ diff --git a/App/EmbeddedVM/ISHEmulator.swift b/App/EmbeddedVM/ISHEmulator.swift deleted file mode 100644 index 7c350f4..0000000 --- a/App/EmbeddedVM/ISHEmulator.swift +++ /dev/null @@ -1,427 +0,0 @@ -import Foundation - -/// A serial executor backed by one permanent OS thread. -/// -/// A DispatchQueue is not sufficient here: it preserves ordering but may move -/// work between pthreads. ish-arm64 stores its active guest process in thread-local -/// storage, so importing, booting, and every command must run on the exact same -/// pthread for the lifetime of the guest. -private final class GuestThreadExecutor: @unchecked Sendable { - private let condition = NSCondition() - private var jobs: [@Sendable () -> Void] = [] - private var thread: Thread? - - init() { - let thread = Thread { [weak self] in - self?.run() - } - thread.name = "org.xforge.ish.guest" - thread.qualityOfService = .userInitiated - self.thread = thread - thread.start() - } - - func submit(_ job: @escaping @Sendable () -> Void) { - condition.lock() - jobs.append(job) - condition.signal() - condition.unlock() - } - - private func run() { - while true { - condition.lock() - while jobs.isEmpty { - condition.wait() - } - let job = jobs.removeFirst() - condition.unlock() - job() - } - } -} - -/// `LinuxEmulator` backed by the embedded ish-arm64 engine. -/// -/// ish-arm64 runs a real Linux guest in-process: its aarch64 backend dispatches -/// guest instructions to pre-compiled "gadget" functions rather than emitting -/// machine code, so it needs no JIT entitlement and works in a sideloaded app. -/// The root filesystem is the Alpine aarch64 minirootfs bundled in the app; on -/// first boot it is imported into the engine's `fakefs` format, then mounted as `/`. -/// -/// Threading: ish-arm64 keeps its current guest task in a thread-local, so boot and -/// every command run serialized on one dedicated queue. -@MainActor -final class ISHEmulator: LinuxEmulator { - let name = "ish-arm64 (arm64 guest)" - private(set) var isRunning = false - - private let rootsDirectory: URL - private let hostDirectory: URL - private let guestThread = GuestThreadExecutor() - - init(rootsDirectory: URL, hostDirectory: URL) { - self.rootsDirectory = rootsDirectory - self.hostDirectory = hostDirectory - } - - func boot() async throws { - guard !isRunning else { return } - let roots = rootsDirectory - let host = hostDirectory - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - do { - // Capture XForge's own breadcrumbs before the engine can - // write anything. The engine's kernel messages go through - // its build-time log handler (nslog, so they reach the - // device console); this file is the boot/import/command - // trace that makes a bug report readable. - XForgeLog.prepare() - XForgeLog.note("emulator: boot requested (rootfs \(roots.lastPathComponent))") - - // Import the bundled Alpine rootfs into fakefs the first - // time; subsequent launches reuse it. - let root = try RootfsInstaller.installIfNeeded(into: roots) - XForgeLog.note("emulator: rootfs ready at \(root.lastPathComponent)") - let rc = root.path.withCString { rootPath in - host.path.withCString { hostPath in - xf_ish_boot(rootPath, hostPath) - } - } - guard rc == 0 else { - throw LinuxVMError.notImplemented(ishLastError(fallback: "Could not boot the guest (errno \(rc)).")) - } - XForgeLog.note("emulator: boot complete") - continuation.resume() - } catch { - XForgeLog.note("emulator: boot failed: \(error.localizedDescription)") - continuation.resume(throwing: error) - } - } - } - isRunning = true - } - - /// Import the bundled rootfs on the guest's permanent serial thread, - /// without booting the emulator. The importer itself is host-only; the - /// engine's one-time initialization happens later in `boot()`. - func prepareRootfs() async throws { - let roots = rootsDirectory - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - do { - XForgeLog.prepare() - XForgeLog.note("emulator: rootfs pre-install requested") - let root = try RootfsInstaller.installIfNeeded(into: roots) - XForgeLog.note("emulator: rootfs pre-installed at \(root.lastPathComponent)") - continuation.resume() - } catch { - XForgeLog.note("emulator: rootfs pre-install failed: \(error.localizedDescription)") - continuation.resume(throwing: error) - } - } - } - } - - func runCommand( - _ command: String, - shell: String?, - timeout: TimeInterval, - maxOutput: Int - ) async throws -> GuestCommandResult { - if !isRunning { try await boot() } - let timeoutMs = timeout <= 0 ? 0 : Int32(min(timeout * 1000, Double(Int32.max))) - let maxOut = maxOutput > 0 ? maxOutput : 256 * 1024 - - return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - var raw = xf_guest_result() - let rc: Int32 = command.withCString { cCommand in - if let shell { - return shell.withCString { cShell in - xf_ish_run(cCommand, cShell, timeoutMs, maxOut, &raw) - } - } - return xf_ish_run(cCommand, nil, timeoutMs, maxOut, &raw) - } - guard rc == 0 else { - continuation.resume(throwing: LinuxVMError.notImplemented( - ishLastError(fallback: "The command could not start (errno \(rc))."))) - return - } - - let output = raw.output.map { String(cString: $0) } ?? "" - let result = GuestCommandResult( - launched: raw.launched != 0, - exited: raw.exited != 0, - exitCode: raw.exit_code, - termSignal: raw.term_signal, - timedOut: raw.timed_out != 0, - truncated: raw.truncated != 0, - output: output - ) - var mutable = raw - xf_guest_result_free(&mutable) - continuation.resume(returning: result) - } - } - } - - func startInit(_ program: String) async throws { - if !isRunning { try await boot() } - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - let rc = program.withCString { xf_ish_start_init($0) } - guard rc == 0 else { - continuation.resume(throwing: LinuxVMError.notImplemented( - ishLastError(fallback: "The guest's init could not start (errno \(rc))."))) - return - } - XForgeLog.note("emulator: \(program) is pid 1") - continuation.resume() - } - } - } - - func openConsole( - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> any GuestConsole { - if !isRunning { try await boot() } - let console = BridgeConsole(onOutput: onOutput) - console.startReading() - XForgeLog.note("emulator: console reader started") - return console - } - - func startDetached( - _ command: String, - shell: String?, - stdinPath: String? - ) async throws -> DetachedProcess { - if !isRunning { try await boot() } - return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - let rc: Int32 = command.withCString { cCommand in - let run: (UnsafePointer?) -> Int32 = { cShell in - if let stdinPath { - return stdinPath.withCString { cStdin in - xf_ish_run_detached(cCommand, cShell, cStdin) - } - } - return xf_ish_run_detached(cCommand, cShell, nil) - } - if let shell { return shell.withCString { run($0) } } - return run(nil) - } - guard rc > 0 else { - // Negative: the process never started. Report it rather than - // handing back a handle to something that does not exist. - continuation.resume(throwing: LinuxVMError.notImplemented( - ishLastError(fallback: "The process could not start (errno \(rc))."))) - return - } - continuation.resume(returning: GuestProcess(pid: rc, thread: self.guestThread)) - } - } - } - - func shutdown() async { - guard isRunning else { return } - await withCheckedContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - xf_ish_shutdown() - continuation.resume() - } - } - isRunning = false - } -} - -/// A guest process started by `ISHEmulator.startDetached`. -/// -/// Every call hops onto the one guest thread, because the engine may only be -/// driven from there. The thread is free precisely because the process was -/// *started* detached rather than run to completion — which is the whole reason -/// this exists: a blocking run would have held that thread for the process's -/// lifetime and starved everything else in the app. -/// -/// Deliberately NOT `@MainActor`: it is a handle to something that lives on the -/// guest thread, and it is constructed from inside that thread's queue. Marking -/// it main-actor isolated made its initialiser unreachable from where it is -/// created, and it has no main-actor state to protect. -private final class GuestProcess: DetachedProcess, @unchecked Sendable { - let pid: Int32 - private let thread: GuestThreadExecutor - - init(pid: Int32, thread: GuestThreadExecutor) { - self.pid = pid - self.thread = thread - } - - var isRunning: Bool { - // Asks the guest, which means hopping onto its thread — so this is - // `async` rather than a plain property, and it does not block: the hop is - // a queued job that completes immediately, not a wait on the process. - get async { await checkAlive() } - } - - private func checkAlive() async -> Bool { - let pid = self.pid - return await withCheckedContinuation { (continuation: CheckedContinuation) in - thread.submit { - continuation.resume(returning: xf_ish_process_alive(pid) == 1) - } - } - } - - func signal(_ number: Int32) async { - let pid = self.pid - await withCheckedContinuation { (continuation: CheckedContinuation) in - thread.submit { - _ = xf_ish_kill_process(pid, number) - continuation.resume() - } - } - } - - /// Poll for exit rather than blocking the guest thread. - /// - /// A blocking wait would occupy the one thread the engine allows, for as long - /// as the process runs — the very starvation that detached processes exist to - /// avoid. Polling asks a cheap question repeatedly instead, leaving the thread - /// free between asks. - @discardableResult - func waitForExit(timeout: TimeInterval) async -> Bool { - let deadline = timeout > 0 ? Date().addingTimeInterval(timeout) : nil - while true { - if !(await checkAlive()) { return true } - if let deadline, Date() >= deadline { return false } - try? await Task.sleep(for: .milliseconds(200)) - } - } -} - -/// The host end of the guest's console tty. -/// -/// Two directions, two threads, because that is what a terminal is: -/// - **out**: a thread of its own blocks in `xf_ish_console_read` and hands each -/// chunk to `onOutput` as the guest writes it. It is not the engine's thread: -/// reading the console touches none of the engine's per-thread guest state, and -/// the engine thread has to stay free to run the guest (and any command -/// XForge has started) while output streams out. -/// - **in**: writes are handed to a serial queue. A tty's input buffer is small, -/// so a paste larger than it has to be pushed in pieces, waiting for the guest -/// to drain each one — and that waiting must not happen on the thread that is -/// handling a keystroke. -final class BridgeConsole: GuestConsole, @unchecked Sendable { - /// Big enough that a screenful arrives in a single read, small enough that a - /// burst of output does not allocate. - private static let readChunk = 32 * 1024 - /// How long a single read waits before the loop checks whether it should stop. - private static let readTimeoutMs: Int32 = 500 - /// Give up on an input burst the guest has not drained — a tty nobody is - /// reading from must not wedge the keyboard. - private static let inputDeadline: TimeInterval = 5 - - private let onOutput: @Sendable (String) -> Void - private let inputQueue = DispatchQueue(label: "org.xforge.console.input") - private let state = NSLock() - private var stopped = false - - init(onOutput: @escaping @Sendable (String) -> Void) { - self.onOutput = onOutput - } - - deinit { - stop() - } - - var isReady: Bool { xf_ish_console_ready() == 1 } - - /// Begin delivering the console's output. Called once, by the emulator. - func startReading() { - let thread = Thread { [weak self] in self?.readLoop() } - thread.name = "org.xforge.console.reader" - thread.qualityOfService = .userInitiated - thread.stackSize = 256 * 1024 - thread.start() - } - - @discardableResult - func write(_ text: String) -> Bool { - guard !text.isEmpty, let data = text.data(using: .utf8), !data.isEmpty else { - return false - } - // `GuestConsole` promises `false` for "there is no console to write to", - // and the terminal reports that to the user. Returning `true` here - // unconditionally meant input was accepted, queued against a console that - // was stopped or not up yet, and silently dropped — the one failure mode a - // terminal must not have. - guard !isStopped, isReady else { return false } - inputQueue.async { [weak self] in self?.push(data) } - return true - } - - func resize(cols: Int, rows: Int) { - guard cols > 0, rows > 0 else { return } - let result = xf_ish_console_resize(Int32(cols), Int32(rows)) - if result != 0 { - XForgeLog.note("console: could not set \(cols)x\(rows) (error \(result))") - } - } - - func stop() { - state.lock() - stopped = true - state.unlock() - } - - private var isStopped: Bool { - state.lock() - defer { state.unlock() } - return stopped - } - - /// Read until stopped. `xf_ish_console_read` returns 0 when its short wait - /// expires, which is what gives this loop the chance to notice `stop()`. - private func readLoop() { - var buffer = [CChar](repeating: 0, count: Self.readChunk) - while !isStopped { - let count = buffer.withUnsafeMutableBufferPointer { pointer -> Int in - guard let base = pointer.baseAddress else { return 0 } - return Int(xf_ish_console_read(base, pointer.count, Self.readTimeoutMs)) - } - guard count > 0 else { continue } - let bytes = buffer.prefix(count).map { UInt8(bitPattern: $0) } - onOutput(String(decoding: bytes, as: UTF8.self)) - } - } - - /// Push input in pieces, waiting for the guest to drain the tty between them. - private func push(_ data: Data) { - var offset = 0 - let deadline = Date().addingTimeInterval(Self.inputDeadline) - while offset < data.count, !isStopped, Date() < deadline { - let accepted = data.withUnsafeBytes { raw -> Int in - guard let base = raw.bindMemory(to: CChar.self).baseAddress else { return -1 } - return Int(xf_ish_console_write(base + offset, data.count - offset)) - } - if accepted > 0 { - offset += accepted - continue - } - // No console yet, or its buffer is full while the guest reads. - Thread.sleep(forTimeInterval: 0.01) - } - if offset < data.count { - XForgeLog.note("console: \(data.count - offset) byte(s) of input not delivered") - } - } -} - -/// Reads the bridge's thread-local last error. Must be called on the guest thread. -private func ishLastError(fallback: String) -> String { - let message = String(cString: xf_ish_last_error()) - return message.isEmpty ? fallback : message -} diff --git a/App/EmbeddedVM/InteractiveShellSession.swift b/App/EmbeddedVM/InteractiveShellSession.swift deleted file mode 100644 index 0585b06..0000000 --- a/App/EmbeddedVM/InteractiveShellSession.swift +++ /dev/null @@ -1,53 +0,0 @@ -import Foundation - -/// A live shell in the embedded Linux: what the terminal types into. -/// -/// The shell runs on the guest's **console**, which the host implements as a tty -/// (`GuestConsole`). That is the whole design, and it is what makes the terminal a -/// terminal rather than a pipe: -/// - the host writes keystrokes into the guest's tty, so the guest kernel's line -/// discipline supplies echo, backspace, line editing, `Ctrl-D` and the rest; -/// - `Ctrl-C` is a *byte* again, turned into `SIGINT` for the foreground program -/// by the tty — so an interrupt reaches whatever is running, including -/// `Ctrl-Z`'s job control; -/// - output appears as the guest writes it, including prompts that do not end in -/// a newline, which is what makes a password prompt or a progress bar usable. -/// -/// The guest's `/sbin/init` owns that console and respawns `/sbin/xforge-login -/// root` on it, so the shell is that script's child — it execs root's login shell -/// from `/etc/passwd`: logging out gives a fresh session rather than a dead -/// screen. -@MainActor -protocol InteractiveShellSession: AnyObject { - /// Guest pid the session speaks for. For a console login this is init, which - /// owns the console; the shell itself is init's child and may be replaced. - var pid: Int32 { get } - /// False once the session has stopped; `send` reports failure rather than - /// appearing to accept input nobody will read. - var isRunning: Bool { get } - - /// Send text to the console. Returns false when the session is gone or there - /// is no console to write to, so a caller can say so instead of silently - /// dropping it. - @discardableResult - func send(_ text: String) -> Bool - - /// Send a single control byte — `0x03` for Ctrl-C, `0x04` for Ctrl-D — or any - /// character the key bar cannot produce. - @discardableResult - func sendControl(_ scalar: UInt8) -> Bool - - /// Interrupt whatever is running in the foreground, with a real `SIGINT`. - /// - /// This is the Ctrl-C *character* written to the console: the guest's tty - /// turns it into the signal for the foreground process group, which is exactly - /// what a terminal is supposed to do. - func interruptForeground() async - - /// Tell the guest how big the screen is, so its programs wrap and lay out - /// correctly. - func resize(cols: Int, rows: Int) - - /// Stop the session and its console reader. - func stop() -} diff --git a/App/EmbeddedVM/LinuxEmulator.swift b/App/EmbeddedVM/LinuxEmulator.swift deleted file mode 100644 index 2f4fd12..0000000 --- a/App/EmbeddedVM/LinuxEmulator.swift +++ /dev/null @@ -1,125 +0,0 @@ -import Foundation - -/// Result of one headless command run in the guest. -struct GuestCommandResult: Sendable { - var launched: Bool - var exited: Bool - var exitCode: Int32 - var termSignal: Int32 - var timedOut: Bool - var truncated: Bool - var output: String - - /// Exit status the caller should branch on (non-zero also for timeouts/signals). - var status: Int32 { - if timedOut { return 124 } // like coreutils `timeout` - if !exited && termSignal != 0 { return 128 + termSignal } - return exitCode - } -} - -/// The host end of the guest's console — a real tty, so what the host writes goes -/// through the guest kernel's line discipline: it echoes, it edits, it turns -/// Ctrl-C into a signal for the foreground program, and it lets a full-screen -/// program put the terminal in raw mode. -/// -/// Deliberately not main-actor isolated: reading blocks until the guest writes, -/// so it happens on a thread of its own, and the calls it makes touch no -/// per-thread engine state. -protocol GuestConsole: AnyObject, Sendable { - /// Whether the guest has opened its console yet. - var isReady: Bool { get } - /// Push host input into the guest's console. Returns false only when there is - /// no console to write to. - @discardableResult func write(_ text: String) -> Bool - /// Tell the guest how big the screen is, so its programs wrap and lay out - /// correctly (and redraw when the size changes). - func resize(cols: Int, rows: Int) - /// Stop reading. Idempotent. - func stop() -} - -/// An in-process Linux execution engine. -/// -/// iOS cannot spawn subprocesses, so the embedded Linux runs as a **library** -/// inside the app. XForge uses ish-arm64 (github.com/OpenMinis/ish-arm64): a real Linux -/// kernel + userspace emulator whose threaded-code interpreter dispatches aarch64 -/// guest instructions to pre-compiled functions ("gadgets"). It emits no machine -/// code and needs no executable memory, so no JIT entitlement is required and it -/// works in a sideloaded app. -/// -/// The engine's primitive is "run this command line, give me its merged output -/// and exit status" — the engine's guest command runner — not a byte -/// pipe. Conformers boot a guest and then execute commands through that. The one -/// exception is the guest console, which *is* a byte pipe in both directions (see -/// `GuestConsole`), because a terminal has to be a tty to behave like one. -@MainActor -protocol LinuxEmulator: AnyObject { - var name: String { get } - var isRunning: Bool { get } - /// Boot the guest; returns once commands can be run. - func boot() async throws - /// Import the bundled root filesystem into the engine's on-disk format - /// ahead of first use, without booting the guest. Idempotent: a root that - /// is already installed is left untouched. - func prepareRootfs() async throws - /// Run one command headlessly in the guest. - func runCommand( - _ command: String, - shell: String?, - timeout: TimeInterval, - maxOutput: Int - ) async throws -> GuestCommandResult - - /// Start the guest's own init as pid 1, giving it the console as its stdio. - /// This is what boots the guest as a *system*: init reads /etc/inittab, which - /// puts a root shell on the console. Safe to call once per boot. - func startInit(_ program: String) async throws - - /// Attach to the guest console, delivering its output as the guest produces - /// it. Returns the host end, which is what the terminal types into. - func openConsole(onOutput: @Sendable @escaping (String) -> Void) async throws -> any GuestConsole - - /// Start a command and return immediately, without waiting for it to exit. - /// - /// `stdinPath` is a *guest* path the child reads its stdin from (NULL for - /// `/dev/null`). A detached command redirects its own stdout/stderr, so the - /// caller supplies a complete command line. - /// - /// Returns a handle for waiting on and signalling the process. - func startDetached( - _ command: String, - shell: String?, - stdinPath: String? - ) async throws -> DetachedProcess - - func shutdown() async -} - -/// A guest process started with `startDetached`. -/// -/// It is deliberately thin: the engine reports no exit *status* for a process -/// nobody waits on, so a detached command communicates its result through -/// whatever its command line redirects to. This handle answers the two questions -/// the host actually needs — is it still running, and how do I stop it. -/// Not `@MainActor`: the handle itself is thread-agnostic and does its own -/// hopping onto the guest thread, so callers may hold and use it from anywhere. -/// `Sendable` is what carries that guarantee. -protocol DetachedProcess: AnyObject, Sendable { - /// Guest pid, for logging and for `kill`. - var pid: Int32 { get } - /// Whether it is still running. - /// - /// `async`, because answering it means asking the guest — every engine call - /// has to hop onto the one thread the engine allows, so this cannot be a - /// synchronous property even though it reads like one. - var isRunning: Bool { get async } - /// Send a real guest signal. `SIGINT` (2) interrupts the foreground program, - /// `SIGTERM` (15) asks it to stop, `SIGKILL` (9) cannot be caught. - func signal(_ number: Int32) async - /// Wait for exit, up to `timeout` seconds (0 waits indefinitely). - /// Returns true if it exited. - @discardableResult - func waitForExit(timeout: TimeInterval) async -> Bool -} - diff --git a/App/EmbeddedVM/LinuxVM.swift b/App/EmbeddedVM/LinuxVM.swift deleted file mode 100644 index 81f4c5d..0000000 --- a/App/EmbeddedVM/LinuxVM.swift +++ /dev/null @@ -1,61 +0,0 @@ -import Foundation - -enum GuestShell { - static func quote(_ value: String) -> String { - "'" + value.replacingOccurrences(of: "'", with: "'\\''") + "'" - } - - static func environment(_ values: [String: String]?) -> String { - guard let values else { return "" } - return values.sorted { $0.key < $1.key } - .map { "\($0.key)=\(quote($0.value))" } - .joined(separator: " ") + " " - } -} - -/// Interface to the embedded Linux VM. The concrete implementation boots a -/// bundled iSH-style userspace and gives us a shell to run commands in plus -/// access to the guest filesystem (read/write files, stage artifacts). -@MainActor -protocol LinuxVM: AnyObject { - /// Boot the embedded Linux (blocking until a shell is ready). - func boot() async throws - /// Install the bundled rootfs ahead of first use, so the first build does - /// not pay for the import. Best effort; a no-op once it is installed. - func prepareRootfs() async - /// Run a command through the guest's interactive login terminal, streaming output. - func run( - _ command: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 - /// Run a shell script through the terminal's login launch command - /// (`/bin/sh`), delivering output as the guest writes it. - func runLoginStreaming( - _ script: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 - /// Read a file out of the guest filesystem into a host URL. - func copyOut(guestPath: String, to hostURL: URL) async throws - /// Stage a host file into the guest filesystem. - func copyIn(hostURL: URL, to guestPath: String) async throws - /// Attach the screen to the guest's console shell: the session `/sbin/init` - /// respawns there. Output is delivered as the guest writes it. - func startInteractiveShell( - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> any InteractiveShellSession - var isBooted: Bool { get } -} - -extension LinuxVM { - /// Default for VMs with no shared folder to tail: fall back to the one-shot - /// capture, which delivers the output when the command finishes. - func runLoginStreaming( - _ script: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - try await run(script, environment: environment, onOutput: onOutput) - } -} diff --git a/App/EmbeddedVM/RootfsInstaller.swift b/App/EmbeddedVM/RootfsInstaller.swift deleted file mode 100644 index 048ec9e..0000000 --- a/App/EmbeddedVM/RootfsInstaller.swift +++ /dev/null @@ -1,316 +0,0 @@ -import Foundation -import ZIPFoundation - -/// Installs the bundled Alpine aarch64 root filesystem into the app container -/// ahead of first use — at launch, with the first boot as the fallback. -/// -/// The archive ships *inside the app*, so nothing is downloaded after install. -/// -/// **The archive is already a fakefs**, not a plain tarball: it contains a -/// `data/` tree and a `meta.db` SQLite database, converted on the build machine -/// by the engine's own `tools/fakefsify` (see `EmbeddedLinux/build-rootfs.sh`). -/// That is deliberate, and it is what the engine's reference implementation does: -/// the conversion touches thousands of files, and doing it here means first -/// launch is a plain unzip instead of a multi-minute import on a phone. -/// -/// The root is a full Alpine userspace. The published root also carries the build -/// toolchain — xtool, swiftly, the Swift toolchain and the Darwin SDK — installed -/// at build time by `EmbeddedLinux/install-toolchain.sh`, run in a chroot of the -/// root before the fakefs conversion (see `EmbeddedLinux/build-rootfs.sh`). A root -/// built without it (`XFORGE_PROVISION=none`) is provisioned *by the guest* -/// instead, on demand, with the same script — see the Toolkit screen. -enum RootfsInstaller { - /// The ZIP of the pre-converted fakefs, built by `EmbeddedLinux/build-rootfs.sh`. - static let archiveName = "alpine-rootfs" - static let archiveExtension = "zip" - - /// Directory name of the installed root inside `/embedded-linux/roots`. - /// The Alpine version is in the name so a distribution bump starts clean. - static let rootName = "alpine-3.21" - - /// Roots from earlier layouts that must not be reused. XForge used to import a - /// tarball at runtime (`alpine-3.24.2`) and, before that, a different root - /// entirely (`alpine`). They are removed only once the replacement is valid. - static let legacyRootNames = ["alpine", "alpine-3.24.2", "alpine-3.23.3"] - - /// The ZIP holds `alpine-rootfs/` at its top level (the builder packs the - /// directory, so the paths are relative and unzip into Documents directly). - private static let archiveEntryPrefix = "alpine-rootfs" - - /// The manifest the builder writes into every root, relative to `data/`. - /// Its `stamp:` line names the revision — see `EmbeddedLinux/build-rootfs.sh`. - private static let manifestPath = "usr/local/share/xforge/rootfs-manifest.txt" - - static func bundledArchiveURL() -> URL? { - Bundle.main.url(forResource: archiveName, withExtension: archiveExtension) - } - - /// Whether the installed root is a different revision from the one bundled in - /// this build of the app, and therefore has to be replaced. - /// - /// Only a root that carries a manifest — which is every root XForge has ever - /// installed itself — can be compared, so anything unidentified is left in - /// place rather than replaced by something that might be worse. - private static func installedRootIsStale(_ root: URL) -> Bool { - let manifest = root.appendingPathComponent("data/\(manifestPath)") - guard let installed = stamp(inManifestAt: manifest) else { return false } - guard let bundled = bundledStamp() else { return false } - return installed != bundled - } - - /// The revision named by the manifest inside the bundled ZIP, read straight - /// out of the archive so asking the question costs no unpacking. - private static func bundledStamp() -> String? { - guard let archiveURL = bundledArchiveURL(), - let zip = try? Archive(url: archiveURL, accessMode: .read), - let entry = zip["\(archiveEntryPrefix)/\(manifestPath)"] else { - return nil - } - var contents = Data() - guard (try? zip.extract(entry, consumer: { contents.append($0) })) != nil else { - return nil - } - guard let text = String(data: contents, encoding: .utf8) else { return nil } - return stamp(inManifestText: text) - } - - private static func stamp(inManifestAt url: URL) -> String? { - guard let data = try? Data(contentsOf: url), - let text = String(data: data, encoding: .utf8) else { return nil } - return stamp(inManifestText: text) - } - - private static func stamp(inManifestText text: String) -> String? { - for line in text.split(separator: "\n") where line.hasPrefix("stamp:") { - let value = line.dropFirst("stamp:".count) - .trimmingCharacters(in: .whitespaces) - if !value.isEmpty { return value } - } - return nil - } - - static func installedRoot(in rootsDirectory: URL) -> URL { - rootsDirectory.appendingPathComponent(rootName, isDirectory: true) - } - - /// A root is usable when both halves of the fakefs are present and non-empty: - /// the `data/` tree and the `meta.db` database that indexes it. - static func isInstalled(in rootsDirectory: URL) -> Bool { - let fm = FileManager.default - let root = installedRoot(in: rootsDirectory) - let metadata = root.appendingPathComponent("meta.db") - let data = root.appendingPathComponent("data", isDirectory: true) - - var isDirectory: ObjCBool = false - guard fm.fileExists(atPath: data.path, isDirectory: &isDirectory), - isDirectory.boolValue, - fm.fileExists(atPath: metadata.path), - let attributes = try? fm.attributesOfItem(atPath: metadata.path), - let size = attributes[.size] as? NSNumber, - size.intValue > 0 else { - return false - } - return true - } - - /// Returns the ready-to-boot fakefs root, unpacking the bundled archive the - /// first time — or when the bundled archive is a newer revision than the one - /// already installed. - @discardableResult - static func installIfNeeded(into rootsDirectory: URL, - progress: RootfsImportProgress? = nil) throws -> URL { - let root = installedRoot(in: rootsDirectory) - if isInstalled(in: rootsDirectory) { - if installedRootIsStale(root) { - // The guest's own setup lives in the root — its /etc/inittab, its - // default shell, its profile. Without this check, a device that - // had booted once would keep the first root it ever installed and - // never see any of that change. - XForgeLog.note("rootfs: the installed root is not the bundled revision; reinstalling") - } else { - removeLegacyRoots(in: rootsDirectory, fileManager: .default) - return root - } - } - - guard let archive = bundledArchiveURL() else { - throw RootfsError.archiveMissing( - "No Alpine rootfs is bundled in the app (looked for " - + "\(archiveName).\(archiveExtension)). Run EmbeddedLinux/build-rootfs.sh " - + "before building, or rebuild via CI.") - } - - XForgeLog.note("rootfs: unpacking the bundled \(archive.lastPathComponent)") - return try install(archive: archive, into: rootsDirectory, progress: progress) - } - - /// Unpack a rootfs ZIP. The existing root is retained until the new archive - /// has completely unpacked and passed fakefs validation. - /// - /// `progress` is called from the unpacking thread with a 0..1 fraction. - @discardableResult - static func install(archive: URL, into rootsDirectory: URL, - progress: RootfsImportProgress? = nil) throws -> URL { - let fm = FileManager.default - let root = installedRoot(in: rootsDirectory) - guard archive.pathExtension.lowercased() == archiveExtension else { - throw RootfsError.importFailed( - "Choose an \(archiveName).\(archiveExtension) rootfs archive.") - } - - try fm.createDirectory(at: rootsDirectory, withIntermediateDirectories: true) - - // Unpack into a staging directory and move it into place afterwards, so a - // failure part-way cannot leave a half-populated root that later looks - // installed. (The old import path needed the same care; an interrupted - // staging directory is as large as the whole root filesystem.) - let staging = rootsDirectory.appendingPathComponent( - ".import-\(UUID().uuidString)", isDirectory: true) - try? fm.removeItem(at: staging) - try fm.createDirectory(at: staging, withIntermediateDirectories: true) - - progress?.report(fraction: 0, message: "Opening \(archive.lastPathComponent)") - - guard let zip = try? Archive(url: archive, accessMode: .read) else { - try? fm.removeItem(at: staging) - throw RootfsError.importFailed( - "\(archive.lastPathComponent) could not be opened — it is not a readable ZIP.") - } - - // ZIPFoundation walks entries in order; progress is entry count, which is - // the honest denominator here (entries are small and uniform enough). - let entries = Array(zip) - let total = max(entries.count, 1) - var index = 0 - - for entry in entries { - index += 1 - // Strip the leading `alpine-rootfs/` so the ZIP unpacks to - // `staging/data`, `staging/meta.db` — the layout `mount_root` wants. - let relative = stripArchivePrefix(entry.path) - if relative.isEmpty { continue } - let destination = staging.appendingPathComponent(relative) - - do { - if entry.type == .directory { - try fm.createDirectory(at: destination, withIntermediateDirectories: true) - continue - } - try fm.createDirectory(at: destination.deletingLastPathComponent(), - withIntermediateDirectories: true) - // `.overwrite` keeps a re-unpack into a dirty staging dir honest. - _ = try zip.extract(entry, to: destination, skipCRC32: false) - } catch { - try? fm.removeItem(at: staging) - throw RootfsError.importFailed( - "unpacking \(relative) failed: \(error.localizedDescription)") - } - - if index % 64 == 0 || index == total { - progress?.report(fraction: Double(index) / Double(total), message: relative) - } - } - - guard isValidImportedRoot(staging, fileManager: fm) else { - try? fm.removeItem(at: staging) - throw RootfsError.importFailed( - "the unpacked rootfs is missing its data directory or metadata database") - } - - // Safe to remove the old root only now: `staging` has already passed - // validation, so a failure cannot leave the app with no root at all. - if fm.fileExists(atPath: root.path) { - try fm.removeItem(at: root) - } - try fm.moveItem(at: staging, to: root) - removeLegacyRoots(in: rootsDirectory, fileManager: fm) - progress?.report(fraction: 1, message: nil) - XForgeLog.note("rootfs: ready at \(root.lastPathComponent)") - return root - } - - /// Drops the builder's top-level directory from an archive entry path. - private static func stripArchivePrefix(_ path: String) -> String { - var relative = path - let prefix = archiveEntryPrefix + "/" - if relative == archiveEntryPrefix { return "" } - if relative.hasPrefix(prefix) { - relative.removeFirst(prefix.count) - } - return relative.trimmingCharacters(in: CharacterSet(charactersIn: "/")) - } - - private static func removeLegacyRoots(in rootsDirectory: URL, fileManager fm: FileManager) { - for name in legacyRootNames where name != rootName { - let stale = rootsDirectory.appendingPathComponent(name, isDirectory: true) - if fm.fileExists(atPath: stale.path) { - XForgeLog.note("rootfs: removing the superseded \(name) root") - try? fm.removeItem(at: stale) - } - } - } - - /// Validates an unpacked fakefs: the `data/` tree plus a non-empty `meta.db`. - private static func isValidImportedRoot(_ directory: URL, fileManager fm: FileManager) -> Bool { - var isDirectory: ObjCBool = false - guard fm.fileExists(atPath: directory.appendingPathComponent("data").path, - isDirectory: &isDirectory), - isDirectory.boolValue else { - return false - } - let metadata = directory.appendingPathComponent("meta.db") - guard fm.fileExists(atPath: metadata.path), - let attributes = try? fm.attributesOfItem(atPath: metadata.path), - let size = attributes[.size] as? NSNumber, - size.intValue > 0 else { - return false - } - return true - } - - enum RootfsError: LocalizedError { - case archiveMissing(String) - case importFailed(String) - - var errorDescription: String? { - switch self { - case .archiveMissing(let detail): return detail - case .importFailed(let detail): return detail - } - } - } -} - -/// Reports the progress of a rootfs unpack. -/// -/// Deliberately a plain `@unchecked Sendable` type and NOT a `@MainActor` one: -/// it is called from the unpacking thread, and a main-actor type cannot be -/// referenced from a non-isolated closure under Swift 6's strict concurrency. -/// `report` does its own hop if the caller needs one. -final class RootfsImportProgress: @unchecked Sendable { - /// One update per this much progress. - private static let step = 0.005 - - private let lock = NSLock() - private var lastPublished: Double = -1 - private let onUpdate: @Sendable (Double, String?) -> Void - - init(onUpdate: @escaping @Sendable (Double, String?) -> Void) { - self.onUpdate = onUpdate - } - - /// Called from the unpacking thread. `fraction` is clamped to 0...1. - func report(fraction: Double, message: String?) { - let clamped = fraction.isFinite ? min(max(fraction, 0), 1) : 0 - lock.lock() - // Always let the final update through, so the row can reach 100%. - let isLast = clamped >= 1 - if !isLast && clamped - lastPublished < Self.step { - lock.unlock() - return - } - lastPublished = clamped - lock.unlock() - onUpdate(clamped, message) - } -} diff --git a/App/Models/BuildPipeline.swift b/App/Models/BuildPipeline.swift index 8f36b5d..97f91ab 100644 --- a/App/Models/BuildPipeline.swift +++ b/App/Models/BuildPipeline.swift @@ -1,6 +1,10 @@ import Foundation /// The stages of the on-device IPA build pipeline. +/// +/// The last stage packages; it does not sign. Signing needs an Apple Developer +/// identity, which arrives separately (the Signing screen), so a stage that claimed +/// to sign would be reporting work nobody did. enum BuildStage: String, CaseIterable, Identifiable { case provision case sdk @@ -19,7 +23,7 @@ enum BuildStage: String, CaseIterable, Identifiable { case .configure: return "Configure app" case .resolve: return "Resolve dependencies" case .compile: return "Compile (arm64-apple-ios)" - case .package: return "Package & sign .ipa" + case .package: return "Package .ipa" case .artifact: return "Stage artifact" } } diff --git a/App/Models/Project.swift b/App/Models/Project.swift index 0cc9641..d15819d 100644 --- a/App/Models/Project.swift +++ b/App/Models/Project.swift @@ -2,11 +2,14 @@ import Foundation /// A SwiftPM package that can be built into an iOS app. struct Project: Identifiable, Hashable, Codable { - static let projectsRoot = "/root/projects" + /// Project directories live under `/projects`; this is the stored + /// prefix a project's `rootPath` is built from, kept so a project record and + /// its directory can be checked against each other. + static let projectsRoot = "projects" var id: UUID = UUID() var name: String var organizationIdentifier: String = "com.example" - /// Path of the package root inside the embedded Linux filesystem. + /// The project's stored path, relative to the app container (`projects/`). var rootPath: String var createdAt: Date = Date() /// Configured Info.plist settings for the produced app (editable in the GUI). @@ -15,6 +18,16 @@ struct Project: Identifiable, Hashable, Codable { var packageManifestPath: String { "\(rootPath)/Package.swift" } var ipaOutputPath: String { "\(rootPath)/.build/xforge-\(name).ipa" } + /// The project directory in XForge's own container. + /// + /// Derived from `name` rather than stored: `rootPath` is the string this model + /// carries around for display and for the safety check, and two stored + /// locations for one project is how a build ends up reading a directory nobody + /// created. + var rootURL: URL { + XForgeEnvironment.projectsDirectory.appendingPathComponent(name, isDirectory: true) + } + static func validatedName(_ value: String) throws -> String { let name = value.trimmingCharacters(in: .whitespacesAndNewlines) let allowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "-_")) @@ -63,7 +76,9 @@ enum BuildEvent: Sendable { case failed(String) } -/// Pluggable build backend. `Local` = embedded Linux VM, `Remote` = future build server. +/// Pluggable build backend. There is one implementation — the in-process +/// toolchain (`NativeBuildExecutor`) — and the seam is kept so a future backend +/// (a remote build server, say) does not have to rewrite the pipeline. @MainActor protocol BuildExecutor { /// Verify the user-installed base toolchain + xtool without installing it. @@ -80,7 +95,8 @@ protocol BuildExecutor { enum SDKSource { /// A prebuilt `darwin.artifactbundle` we host (built in CI from Xcode). + /// + /// It used to have a `.bundled` case for an SDK already inside the guest + /// filesystem. There is no guest, so there is one way to get an SDK: fetch it. case hostedRemote(URL) - /// Already inside the embedded Linux filesystem. - case bundled(String) } diff --git a/App/Models/SystemInfo.swift b/App/Models/SystemInfo.swift index aa9b686..1707d00 100644 --- a/App/Models/SystemInfo.swift +++ b/App/Models/SystemInfo.swift @@ -21,8 +21,8 @@ enum SystemInfo { /// Free and total space. /// - /// Free is the *important usage* figure — the same one the app decides with - /// (`XForgeEnvironment.availableBytes`, which guards the Xcode.xip import) — so + /// Free is the *important usage* figure — the same one an import decides with + /// (`DarwinSDKBuilder.availableBytes`, which guards the `Xcode.xip` build) — so /// this screen cannot report space that an import then refuses. The two come /// from different APIs and disagree: `systemFreeSize` does not count the space /// iOS would free up for you. diff --git a/App/Models/XcodeProject.swift b/App/Models/XcodeProject.swift index 3d1a021..8c4867b 100644 --- a/App/Models/XcodeProject.swift +++ b/App/Models/XcodeProject.swift @@ -20,7 +20,7 @@ import Foundation /// a parser instead of fighting the graph. See Docs/XCODE-ALTERNATIVE.md. /// /// Deliberately *not* here yet: -/// - anything that compiles (see `NativeToolchain`/`EmbeddedLinuxExecutor`); +/// - anything that compiles (see `NativeToolchain`/`NativeBuildExecutor`); /// - `$(...)` evaluation, `[config=…]` conditions and Xcode's huge default /// settings table: values are reported as written; /// - schemes, asset catalogs and storyboards. `actool`/`ibtool` are Xcode's own diff --git a/App/NativeToolchain/DarwinSDKBuilder.swift b/App/NativeToolchain/DarwinSDKBuilder.swift new file mode 100644 index 0000000..d85f92f --- /dev/null +++ b/App/NativeToolchain/DarwinSDKBuilder.swift @@ -0,0 +1,368 @@ +import Foundation + +/// Builds a `darwin.artifactbundle` — the Darwin SDK the native compiler reads — +/// out of an `Xcode.xip`, on the device. +/// +/// This is what `xtool sdk build ` does on a Mac, and it has to exist +/// here because that is exactly what a phone cannot do: xtool is a macOS program +/// that shells out to `xar`, and there is no guest to run it in any more. The +/// layout it produces is reproduced rather than invented, so a bundle this writes +/// is the same shape as the one the app downloads: +/// +/// ``` +/// darwin.artifactbundle/ +/// swift-sdk.json the metadata NativeSDK reads +/// darwin-sdk-version.txt +/// Developer/ +/// Platforms/iPhoneOS.platform/ +/// Info.plist +/// Developer/SDKs/iPhoneOS27.0.sdk/… +/// Developer/usr/lib/… (the tbd stubs' link paths) +/// Developer/Library/Frameworks/… (XCTest and friends, linked from the SDK) +/// Toolchains/XcodeDefault.xctoolchain/usr/lib/ +/// swift/… the Swift modules and resource dir +/// swift_static/iphoneos/… the static runtime a sideloaded app links +/// clang/… the clang resource dir +/// ``` +/// +/// Which paths are kept is not a guess: `SDKBuilder.wanted` in xtool enumerates +/// them, and this follows it. One deliberate difference — xtool builds a bundle for +/// three platforms (device, simulator, macOS) because a Mac builds for all of them; +/// XForge only ever compiles for the device, so only `iPhoneOS.platform` is kept and +/// `swift-sdk.json` declares only `arm64-apple-ios`. That is roughly a third of the +/// extraction work and a third of the disk for exactly the same capability. +enum DarwinSDKBuilder { + /// What the caller gets back. `bundle` is inside the staging directory; the + /// caller installs it (`NativeSDK.install(preparedBundle:)`). + struct Result { + var bundle: URL + var sdkRoot: String + var files: Int + var bytes: Int64 + var skipped: Int + /// False when the archive carries no `swift_static/iphoneos`. That is a + /// warning rather than a failure: the SDK's headers and tbd stubs are what C, + /// Objective-C and Objective-C++ targets need, and refusing the whole import + /// over a missing Swift runtime would take those away too. + var hasSwiftStaticRuntime: Bool + } + + struct Progress: Sendable { + /// 0…1 over the whole job (extraction dominates). + var fraction: Double + var message: String + } + + enum Error: LocalizedError { + case noXcodeInside + case noiPhoneOSSDK + case notEnoughSpace(needed: Int64, available: Int64) + var errorDescription: String? { + switch self { + case .noXcodeInside: + return "The .xip does not contain an Xcode.app." + case .noiPhoneOSSDK: + return "Xcode was found, but it has no iPhoneOS SDK — the xip is probably " + + "a Command Line Tools or a simulator-only archive." + case .notEnoughSpace(let needed, let available): + return "Not enough space: building the SDK needs about \(Self.size(needed)) " + + "free, and there is \(Self.size(available)). Delete some projects or " + + "downloads and try again." + } + } + + private static func size(_ bytes: Int64) -> String { + ByteCountFormatter.string(fromByteCount: bytes, countStyle: .file) + } + } + + /// Roughly what the iPhoneOS-only bundle occupies. Used for the space guard + /// before the expensive part starts, not as a hard limit. + static let estimatedBytes: Int64 = 1_500_000_000 + + // MARK: - Entry point + + /// Extract an `Xcode.xip` into `destination/`. + /// + /// - Parameters: + /// - xip: the archive, wherever the document picker put it. + /// - destination: the directory the bundle is written into (its parent, not + /// the bundle itself) — `NativeSDK`'s staging directory, in practice. + /// - progress: called often, from the calling thread. + static func build(fromXip xip: URL, + into destination: URL, + progress: (@Sendable (Progress) -> Void)? = nil) throws -> Result { + let available = availableBytes(at: destination) + if available > 0, available < estimatedBytes { + throw Error.notEnoughSpace(needed: estimatedBytes, available: available) + } + + let staging = destination.appendingPathComponent( + "darwin.artifactbundle.building-\(UUID().uuidString)", isDirectory: true) + do { + let content = try XipArchive.content(of: xip) + let writer = Writer(staging: staging) + XForgeLog.note("xip: Content at byte \(content.offset), \(content.length) bytes") + + // Extraction is essentially all of the wall clock, so the fraction of the + // compressed member that has been consumed is the honest fraction of the + // whole job; the metadata write at the end gets the last few percent. + try XipArchive.forEachEntry(in: xip, content: content, progress: { fraction in + progress?(Progress(fraction: fraction * 0.95, + message: "Extracting Xcode — \(Int(fraction * 100))%")) + }) { entry, payload in + try writer.accept(entry, payload) + } + + let result = try writer.finish(progress: progress) + + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + return result + } catch { + try? FileManager.default.removeItem(at: staging) + throw error + } + } + + // MARK: - Which paths become the bundle + + /// The subset of `Xcode.app/Contents/Developer` a Darwin SDK needs. + /// + /// Mirrors `SDKBuilder.SDKEntry.wanted` in xtool, restricted to the device + /// platform, plus its one exclusion (`swift/prebuilt-modules`, which is + /// per-configuration build output measured in gigabytes). + /// + /// Written as explicit list comparisons rather than the nested matcher xtool + /// uses: there are five leaves, and a reader can check this against xtool's table + /// line by line. + static func isWanted(_ path: String) -> Bool { + var components = path.split(separator: "/").map(String.init) + if components.first == "." { components.removeFirst() } + // The archive's own root is `Xcode.app/…`; stripping it lets the same filter + // read both that and a bare `Contents/Developer/…`. + if components.first?.hasSuffix(".app") == true { components.removeFirst() } + + guard components.starts(with: ["Contents", "Developer"]) else { return false } + let rest = Array(components.dropFirst(2)) + + // The toolchain: the Swift modules, the static runtime, the clang resource + // directory. These are the parts a compiler needs that the SDK does not carry. + if rest.starts(with: ["Toolchains", "XcodeDefault.xctoolchain", "usr", "lib"]) { + let tail = Array(rest.dropFirst(4)) + guard let name = tail.first, + ["swift", "swift_static", "clang"].contains(name) else { return false } + // Prebuilt modules are generated per platform/configuration and are + // enormous. The published darwin bundle proves the intended layout: + // it has no prebuilt-modules entries for iphoneos (or any other Swift + // platform). Check the path by component, not only for + // `swift/prebuilt-modules`: real Xcodes use + // `swift/iphoneos/prebuilt-modules/...`. + if name == "swift", tail.dropFirst().contains("prebuilt-modules") { return false } + return true + } + + guard rest.starts(with: ["Platforms", "iPhoneOS.platform"]) else { return false } + let tail = Array(rest.dropFirst(2)) + + // `Info.plist` describes the platform, and the SDK's own tooling reads it. + if tail == ["Info.plist"] { return true } + + guard tail.first == "Developer" else { return false } + let developer = Array(tail.dropFirst()) + if developer.first == "SDKs" || developer.first == "usr" { return true } + // XCTest and Testing live in the platform's Library rather than in the SDK, + // and the SDK links to them from there (see `swift-sdk.json`'s search paths). + return developer.starts(with: ["Library", "Frameworks"]) + || developer.starts(with: ["Library", "PrivateFrameworks"]) + } + + // MARK: - Writing the bundle + + /// Turns entries into the bundle tree, one at a time. + /// + /// Stateful because the archive is: a hard link is only recognisable once its + /// first occurrence has been seen, and a directory entry is the only thing that + /// says a directory exists at all. + private final class Writer { + private let staging: URL + private var hardLinks: [UInt64: URL] = [:] + private var files = 0 + private var bytes: Int64 = 0 + private var skipped = 0 + private var sdkRoot: String? + private var sdkVersion: String? + private var sawSwiftStaticRuntime = false + private var sawDeveloperTree = false + + init(staging: URL) { + self.staging = staging + } + + func accept(_ entry: XipArchive.Entry, _ payload: XipArchive.Payload) throws { + // Noted before the filter: an archive with no `Contents/Developer` at all is + // not an Xcode, and saying that is better than "it has no iPhoneOS SDK". + if entry.name.contains("Contents/Developer") { + sawDeveloperTree = true + } + // `Self` here is `Writer`, so the outer type is named in full. + guard DarwinSDKBuilder.isWanted(entry.name) else { + skipped += 1 + return + } + // `Xcode.app/` prefixes everything; the bundle re-roots at `Developer/`, + // which is what `swift-sdk.json`'s paths are relative to. + let relative = Self.bundlePath(for: entry.name) + let target = staging.appendingPathComponent(relative) + + if entry.isDirectory { + try FileManager.default.createDirectory(at: target, withIntermediateDirectories: true) + if relative.contains("Developer/SDKs/"), relative.hasSuffix(".sdk") { + // `iPhoneOS.sdk` is a symlink to `iPhoneOS27.0.sdk` in recent + // Xcodes; the versioned name is the one worth declaring, so it + // wins whenever it turns up. + let name = target.lastPathComponent + if sdkRoot == nil || name != "iPhoneOS.sdk" { + sdkRoot = relative + sdkVersion = Self.platformVersion(fromSDKName: name) + } + } + } else if entry.isSymlink { + let link = String(decoding: try payload.readAll(), as: UTF8.self) + try FileManager.default.createDirectory( + at: target.deletingLastPathComponent(), withIntermediateDirectories: true) + try? FileManager.default.removeItem(at: target) + try FileManager.default.createSymbolicLink(atPath: target.path, + withDestinationPath: link) + files += 1 + } else if entry.isRegular { + try FileManager.default.createDirectory( + at: target.deletingLastPathComponent(), withIntermediateDirectories: true) + + // A second occurrence of the same inode is a hard link, and Xcode's + // trees are full of them: writing the bytes again would multiply the + // bundle by the link count. When the first occurrence was filtered out + // there is nothing to link to, and the data is written instead. + if entry.linkCount > 1 { + let key = UInt64(entry.device) << 32 | UInt64(entry.inode) + if let original = hardLinks[key] { + try? FileManager.default.removeItem(at: target) + try FileManager.default.linkItem(at: original, to: target) + files += 1 + return + } + hardLinks[key] = target + } + + if relative.contains("swift_static/iphoneos/") { + sawSwiftStaticRuntime = true + } + try payload.write(to: target) + files += 1 + bytes += entry.size + } else { + // Device nodes, fifos and sockets: nothing an SDK is built from. + skipped += 1 + } + } + + /// Write the metadata, having seen the whole tree. + func finish(progress: (@Sendable (Progress) -> Void)?) throws -> Result { + guard sawDeveloperTree else { throw Error.noXcodeInside } + guard let sdkRoot else { throw Error.noiPhoneOSSDK } + progress?(Progress(fraction: 0.97, message: "Writing swift-sdk.json…")) + + let json = SDKDefinition( + schemaVersion: "4.0", + // One triple, unlike xtool's five: XForge compiles for the device. + targetTriples: [ + "arm64-apple-ios": SDKDefinition.Triple( + sdkRootPath: sdkRoot, + includeSearchPaths: ["Developer/Platforms/iPhoneOS.platform/Developer/usr/lib"], + librarySearchPaths: ["Developer/Platforms/iPhoneOS.platform/Developer/usr/lib"], + swiftResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift", + swiftStaticResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static" + ), + ] + ) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(json).write(to: staging.appendingPathComponent("swift-sdk.json")) + + // xtool's marker, with the platform version rather than an SDK-build + // counter: this bundle was not built by xtool, and claiming otherwise + // would make `xtool sdk` try to update it. + try Data("\(sdkVersion ?? "unknown")\n".utf8) + .write(to: staging.appendingPathComponent("darwin-sdk-version.txt")) + + progress?(Progress(fraction: 1, message: "Done")) + let result = Result(bundle: staging, sdkRoot: sdkRoot, files: files, + bytes: bytes, skipped: skipped, + hasSwiftStaticRuntime: sawSwiftStaticRuntime) + XForgeLog.note("xip: \(result.files) files (\(result.bytes) bytes), " + + "skipped \(result.skipped), sdkRoot \(result.sdkRoot), " + + "swift_static=\(result.hasSwiftStaticRuntime)") + return result + } + + /// `Xcode.app/Contents/Developer/…` → `Developer/…`. + /// + /// Only `Contents` is dropped: `Developer` is kept because it *is* the + /// bundle's root — `swift-sdk.json` names `Developer/Platforms/…`, and + /// dropping it too would put the SDK where the metadata does not look for it. + static func bundlePath(for entry: String) -> String { + var components = entry.split(separator: "/").map(String.init) + if components.first == "." { components.removeFirst() } + if components.first?.hasSuffix(".app") == true { components.removeFirst() } + if components.first == "Contents" { components.removeFirst() } + return components.joined(separator: "/") + } + + /// `iPhoneOS27.0.sdk` → `27.0`; a bare `iPhoneOS.sdk` has no version to give. + static func platformVersion(fromSDKName name: String) -> String? { + let trimmed = name.hasSuffix(".sdk") ? String(name.dropLast(4)) : name + let digits = trimmed.drop { !$0.isNumber } + return digits.isEmpty ? nil : String(digits) + } + } + + // MARK: - Metadata + + private struct SDKDefinition: Encodable { + struct Triple: Encodable { + var sdkRootPath: String + var includeSearchPaths: [String] + var librarySearchPaths: [String] + var swiftResourcesPath: String + var swiftStaticResourcesPath: String + // xtool also writes `toolsetPaths`, pointing at a `toolset.json` it + // generates for macro plugins. XForge never runs a plugin (a macro is a + // separate process, and iOS has none), so listing a file that is not + // there would be a worse lie than omitting the key. + } + + var schemaVersion: String + var targetTriples: [String: Triple] + } + + /// Build from the pbzx `Content` member extracted from Apple's ZIP wrapper. + static func buildPBZX(from content: URL, into destination: URL, + progress: (@Sendable (Progress) -> Void)? = nil) throws -> Result { + let staging = destination + try FileManager.default.createDirectory(at: staging, withIntermediateDirectories: true) + let writer = Writer(staging: staging) + try XipArchive.forEachPBZX(at: content, progress: { fraction in + progress?(Progress(fraction: fraction, message: "Extracting Xcode Content — \(Int(fraction * 100))%")) + }) { entry, payload in + try writer.accept(entry, payload) + } + return try writer.finish(progress: progress) + } + + /// Free space on the volume the bundle will be written to. + static func availableBytes(at url: URL) -> Int64 { + let probe = url.deletingLastPathComponent().path + let values = try? URL(fileURLWithPath: probe) + .resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + return Int64(values?.volumeAvailableCapacityForImportantUsage ?? 0) + } +} diff --git a/App/NativeToolchain/NativeSDK.swift b/App/NativeToolchain/NativeSDK.swift index b54b33a..fa11d68 100644 --- a/App/NativeToolchain/NativeSDK.swift +++ b/App/NativeToolchain/NativeSDK.swift @@ -9,8 +9,33 @@ struct NativeSDKLayout: Sendable { let platformLibrarySearchPaths: [URL] } +extension NativeSDKLayout { + /// `-L` paths that make `-lswiftCore` and friends resolvable when linking + /// statically. + /// + /// A sideloaded app links the Swift runtime into the binary — there is no + /// system-wide `/usr/lib/swift` on iOS to load it from at runtime — so the + /// linker has to be pointed at the static runtime the Darwin SDK carries. + /// `swift-sdk.json` names it as `swiftStaticResourcesPath`, and the compiled + /// libraries for the device live one directory below it (`…/iphoneos`). + /// `librarySearchPaths` from the same file is the fallback for bundles that + /// lay it out differently, and both are merged rather than chosen between: + /// a duplicate `-L` costs nothing, a missing one fails the link. + var swiftRuntimeLibraryPaths: [URL] { + var paths: [URL] = [] + if let swiftStaticResources { + paths.append(swiftStaticResources) + paths.append(swiftStaticResources.appendingPathComponent("iphoneos", isDirectory: true)) + } + paths.append(contentsOf: platformLibrarySearchPaths) + var seen = Set() + return paths.filter { seen.insert($0.path).inserted } + } +} + enum NativeSDKError: LocalizedError { case missingBundle + case notABundle(String) case invalidMetadata case missingTarget case missingSDK(String) @@ -19,6 +44,10 @@ enum NativeSDKError: LocalizedError { switch self { case .missingBundle: return "No native Darwin SDK is installed." + case .notABundle(let name): + return "\(name) is not a Darwin SDK bundle and not an Xcode.xip. A bundle is a " + + "folder (or a zip of one) containing darwin.artifactbundle or swift-sdk.json; " + + "an Xcode archive starts with 'xar!'." case .invalidMetadata: return "The native Darwin SDK has an invalid swift-sdk.json." case .missingTarget: @@ -29,9 +58,10 @@ enum NativeSDKError: LocalizedError { } } -/// Reads the same swift-sdk.json layout that xtool's Darwin SDK builder emits, -/// but resolves it directly from the iOS app sandbox rather than through SwiftPM -/// in the Alpine guest. +/// Reads the same `swift-sdk.json` layout that xtool's Darwin SDK builder emits, +/// but resolves it directly from the app's container: SwiftPM's SDK store is a +/// directory inside a toolchain installation, and there is no installation here — +/// the bundle is a folder the app downloaded and unpacked itself. enum NativeSDK { static var installedBundle: URL { XForgeEnvironment.nativeSDKDirectory @@ -107,11 +137,23 @@ enum NativeSDK { static func installLatestPrebuilt() async throws { let remote = try await XForgeReleases.darwinSDKURL() + try await install(fromRemote: remote) + } + + /// Download and install a Darwin SDK archive from an explicit URL. + /// + /// The URL-parameterised form is the one the build pipeline uses: the release + /// lookup happens in the caller, so a caller that already resolved an asset + /// (or pinned one) does not pay for a second lookup. + static func install(fromRemote remote: URL) async throws { let (downloaded, response) = try await URLSession.shared.download(from: remote) if let http = response as? HTTPURLResponse, !(200..<300).contains(http.statusCode) { throw DownloadError.http(status: http.statusCode, url: remote) } + try installDownloadedArchive(at: downloaded) + } + private static func installDownloadedArchive(at downloaded: URL) throws { let fm = FileManager.default let extractionRoot = XForgeEnvironment.nativeSDKDirectory .appendingPathComponent("download-\(UUID().uuidString)", isDirectory: true) @@ -142,11 +184,172 @@ enum NativeSDK { return } - throw NativeSDKError.missingBundle + throw NativeSDKError.notABundle(downloaded.lastPathComponent) } static func remove() throws { guard FileManager.default.fileExists(atPath: installedBundle.path) else { return } try FileManager.default.removeItem(at: installedBundle) } + + // MARK: - Importing + + /// Install a bundle the app built itself. + /// + /// A *move* where `install(from:)` copies: this bundle is already inside the + /// app's container, so copying a gigabyte only to delete the original costs a + /// gigabyte of writes and a second gigabyte of free space for nothing. + static func install(preparedBundle: URL) throws { + let fm = FileManager.default + let destination = installedBundle + try fm.createDirectory(at: XForgeEnvironment.nativeSDKDirectory, + withIntermediateDirectories: true) + _ = try layout(at: preparedBundle) // validate before replacing what is there + if fm.fileExists(atPath: destination.path) { + try fm.removeItem(at: destination) + } + try fm.moveItem(at: preparedBundle, to: destination) + } + + /// Install from whatever the document picker handed over. + /// + /// Three shapes arrive this way, and they are told apart by what they are rather + /// than by what they are called: + /// + /// What an import did, so the screen can say more than "done". + struct ImportReport: Sendable { + var files: Int = 0 + var bytes: Int64 = 0 + /// Things that did not stop the install but will bite later. + var warnings: [String] = [] + } + + /// - a `darwin.artifactbundle` **folder** — already built, perhaps by + /// `xtool sdk build` on a Mac; + /// - a **zip** of one, which is how the hosted bundle is published; + /// - an Apple **`Xcode.xip`**, which `DarwinSDKBuilder` turns into one here. + /// That is the case that needs a Mac otherwise, and the reason the picker no + /// longer filters for folders only. + /// + /// Which one it is decided by what the file *is*, not what it is called: the + /// picker hands over whatever the user chose, and "that is a zip, not a xip" is a + /// better answer than a bundle search that fails for the wrong reason. + @discardableResult + static func installImported(from source: URL, + progress: (@Sendable (DarwinSDKBuilder.Progress) -> Void)? = nil) throws -> ImportReport { + let accessed = source.startAccessingSecurityScopedResource() + defer { if accessed { source.stopAccessingSecurityScopedResource() } } + + var isDirectory: ObjCBool = false + let exists = FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) + let kind = source.pathExtension.lowercased() + let header = firstBytes(of: source) + let size = (try? source.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0 + + // The log is the only way to see this on a device: what was picked, whether + // the sandbox let us read it, how big it is and what it actually is. + XForgeLog.note("sdk import: \(source.lastPathComponent) " + + "(\(ByteCountFormatter.string(fromByteCount: Int64(size), countStyle: .file))), " + + "kind=\(kind.isEmpty ? "none" : kind), " + + "readable=\(accessed), header=\(hex(header))") + + if exists, isDirectory.boolValue { + try install(from: source) + return ImportReport() + } + if header == Data("xar!".utf8) || kind == "xip" { + let built = try DarwinSDKBuilder.build(fromXip: source, + into: XForgeEnvironment.nativeSDKDirectory, + progress: progress) + try install(preparedBundle: built.bundle) + return ImportReport(files: built.files, bytes: built.bytes, + warnings: built.hasSwiftStaticRuntime ? [] : [ + "the archive has no Swift static runtime " + + "(Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static/iphoneos), " + + "so linking Swift needs one from elsewhere", + ]) + } + if header.starts(with: Data("PK".utf8)) || kind == "zip" { + if try installAppleXcodeZipIfPresent(at: source, progress: progress) { + return ImportReport(files: 0, bytes: 0, + warnings: ["imported Apple Xcode Content stream"]) + } + try installDownloadedArchive(at: source) + return ImportReport() + } + throw NativeSDKError.notABundle(source.lastPathComponent) + } + + /// Apple Developer's Xcode download can reach Files as a ZIP wrapper containing + /// `Xcode_27/Content` and `Metadata`, rather than as a file named `.xip`. Do not + /// unzip it: `Content` declares a ~10 GB cpio and the ZIP itself can already be + /// 2 GB. Copy just that one compressed member to a temporary file, then use the + /// same streaming pbzx/cpio builder as a xar xip. + private static func installAppleXcodeZipIfPresent( + at source: URL, + progress: (@Sendable (DarwinSDKBuilder.Progress) -> Void)? + ) throws -> Bool { + let archive = try Archive(url: source, accessMode: .read) + guard let contentEntry = archive.first(where: { + $0.path == "Xcode_27/Content" || + ($0.path.hasSuffix("/Content") && $0.path.split(separator: "/").count == 2) + }) else { return false } + + let contentURL = XForgeEnvironment.nativeSDKDirectory + .appendingPathComponent("xcode-content-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: XForgeEnvironment.nativeSDKDirectory, + withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: contentURL) } + + var completed = Int64(0) + FileManager.default.createFile(atPath: contentURL.path, contents: nil) + let output = try FileHandle(forWritingTo: contentURL) + defer { try? output.close() } + _ = try archive.extract(contentEntry) { data in + try output.write(contentsOf: data) + completed += Int64(data.count) + if let progress, contentEntry.uncompressedSize > 0 { + let fraction = min(0.95, Double(completed) / Double(contentEntry.uncompressedSize)) + progress(DarwinSDKBuilder.Progress(fraction: fraction, + message: "Copying Xcode Content — \(Int(fraction * 100))%")) + } + } + + let staging = XForgeEnvironment.nativeSDKDirectory + .appendingPathComponent("darwin.artifactbundle.building-\(UUID().uuidString)", + isDirectory: true) + defer { try? FileManager.default.removeItem(at: staging) } + let built = try DarwinSDKBuilder.buildPBZX(from: contentURL, into: staging, + progress: progress) + try install(preparedBundle: built.bundle) + return true + } + + /// The first four bytes, read without mapping a multi-gigabyte file into memory. + private static func firstBytes(of url: URL) -> Data { + guard let handle = try? FileHandle(forReadingFrom: url) else { return Data() } + defer { try? handle.close() } + return (try? handle.read(upToCount: 4)) ?? Data() + } + + private static func hex(_ data: Data) -> String { + data.isEmpty ? "none" : data.map { String(format: "%02x", $0) }.joined() + } + + /// Delete the copy the document picker made of an imported archive. + /// + /// `fileImporter` hands over a copy inside the app's own container, and for an + /// `Xcode.xip` that copy is ~10 GB: leaving it behind would fill the device on + /// the first import. Only a file that really is one of ours is removed — the + /// picker's import directories are under `tmp/`, and a URL the picker returned + /// in place (or a folder the user owns) is left alone. + static func discardImportCopy(at source: URL) { + let fm = FileManager.default + let path = source.standardizedFileURL.path + let temporary = fm.temporaryDirectory.standardizedFileURL.path + let inbox = XForgeEnvironment.documentDirectory + .appendingPathComponent("Inbox", isDirectory: true).standardizedFileURL.path + guard path.hasPrefix(temporary + "/") || path.hasPrefix(inbox + "/") else { return } + try? fm.removeItem(at: source) + } } diff --git a/App/NativeToolchain/NativeToolchain.swift b/App/NativeToolchain/NativeToolchain.swift index 46f1e50..a2ee9ad 100644 --- a/App/NativeToolchain/NativeToolchain.swift +++ b/App/NativeToolchain/NativeToolchain.swift @@ -139,22 +139,59 @@ enum NativeToolchain { throw NativeToolchainError.link("No LLD arguments were supplied.") } + return try marshal(arguments, tool: "ld64.lld", failure: { + NativeToolchainError.link($0) + }) { argc, argv, diagnostics, capacity in + xf_native_lld_link(argc, argv, diagnostics, capacity) + } + } + + /// Compile through `swift::performFrontend` with a caller-supplied argument + /// list. + /// + /// The argument list is built by `NativeToolchainInvocation`, not here: it is + /// pure, so it can be unit-tested on a simulator, where this call cannot even + /// be linked. The convention `performFrontend` expects is documented there. + static func runSwiftFrontend(arguments: [String]) throws -> NativeToolchainResult { + guard isAvailable, isSwiftAvailable else { + throw NativeToolchainError.unavailable + } + guard !arguments.isEmpty else { + throw NativeToolchainError.compile("No swift-frontend arguments were supplied.") + } + return try marshal(arguments, tool: "swift-frontend", failure: { + NativeToolchainError.compile($0) + }) { argc, argv, diagnostics, capacity in + xf_native_swift_frontend(argc, argv, diagnostics, capacity) + } + } + + /// Marshal a Swift `[String]` into the C ABI's `char *const *` and collect the + /// diagnostics buffer both entry points share. + private static func marshal( + _ arguments: [String], + tool: String, + failure: (String) -> Error, + body: (Int32, UnsafePointer?>?, UnsafeMutablePointer, Int) -> Int32 + ) throws -> NativeToolchainResult { let duplicated: [UnsafeMutablePointer] = arguments.compactMap { strdup($0) } defer { duplicated.forEach { free($0) } } guard duplicated.count == arguments.count else { - throw NativeToolchainError.io("Could not allocate linker arguments.") + throw failure("Could not allocate \(tool) arguments.") } let argv: [UnsafePointer?] = duplicated.map { UnsafePointer($0) } var diagnostics = [CChar](repeating: 0, count: 64 * 1024) let diagnosticsCapacity = diagnostics.count let code = argv.withUnsafeBufferPointer { buffer -> Int32 in - Int32(xf_native_lld_link( - Int32(arguments.count), - buffer.baseAddress, - &diagnostics, - diagnosticsCapacity - )) + diagnostics.withUnsafeMutableBufferPointer { storage in + body( + Int32(arguments.count), + buffer.baseAddress, + storage.baseAddress!, + diagnosticsCapacity + ) + } } return NativeToolchainResult( exitCode: code, @@ -171,7 +208,10 @@ enum NativeToolchain { let source = dir.appendingPathComponent("main.c") let object = dir.appendingPathComponent("main.o") - try "int xforge_native_smoke(void) { return 42; }\n" + // A real entry point, so the object can also be *linked*: a smoke test + // that only compiles cannot tell a working toolchain from one that can + // produce an object the linker rejects. + try "int main(void) { return 0; }\nint xforge_native_smoke(void) { return 42; }\n" .write(to: source, atomically: true, encoding: .utf8) let result = try compileC(source: source, object: object, sdk: sdk) diff --git a/App/NativeToolchain/NativeToolchainBridge.mm b/App/NativeToolchain/NativeToolchainBridge.mm index 79fc1cb..d9d5488 100644 --- a/App/NativeToolchain/NativeToolchainBridge.mm +++ b/App/NativeToolchain/NativeToolchainBridge.mm @@ -16,17 +16,37 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity __has_include() && \ __has_include() && \ __has_include() && \ + __has_include() && \ __has_include() +// Apple's SDK predefines IBAction and IBOutlet as macros (`#define IBOutlet +// __attribute__((iboutlet))`). Clang's generated AttrList.inc expands +// `INHERITABLE_ATTR(IBAction)` into `ATTR(IBAction)` → `class IBAction##Attr;`, so +// the predefined macro is pasted into the token paste and the header does not +// compile: "pasting formed ')Attr', an invalid preprocessing token". Clang's own +// headers undefine them for this reason on some revisions and not others, so it is +// done here, before any clang header is included. +#ifdef IBAction +#undef IBAction +#endif +#ifdef IBOutlet +#undef IBOutlet +#endif + #include +#include #include #include #include #include +#include #include #include +#include #include +#include + #if defined(XFORGE_HAS_SWIFT_FRONTEND) && __has_include() #include #define XFORGE_SWIFT_FRONTEND_READY 1 @@ -65,7 +85,17 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity xf_copy_diag("native swift: empty frontend argument list", diagnostics, diagnostics_capacity); return 64; } - llvm::ArrayRef args(argv, static_cast(argc)); + // `swift-frontend -frontend …` is how the driver invokes the frontend, and it + // hands performFrontend everything *after* `-frontend`. Accept both shapes, so + // a caller that mirrors the command line does not hand parseArgs a flag it + // rejects ("error: unknown argument: '-frontend'"). + const char *const *argsBegin = argv; + size_t argsCount = static_cast(argc); + if (argsCount > 0 && std::strcmp(argsBegin[0], "-frontend") == 0) { + argsBegin += 1; + argsCount -= 1; + } + llvm::ArrayRef args(argsBegin, argsCount); const int rc = swift::performFrontend( args, "swift-frontend", @@ -101,11 +131,17 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity std::string diagText; llvm::raw_string_ostream diagOS(diagText); - // DiagnosticOptions is a plain value and is taken by reference by both the - // printer and the engine; it is no longer refcounted (LLVM 19+), so it must - // not be wrapped in an IntrusiveRefCntPtr. - clang::DiagnosticOptions diagOpts; - auto diagPrinter = std::make_unique(diagOS, diagOpts); + // One clang generation, and it is the older one on purpose. Every bundle + // XForge builds comes from Swift's LLVM fork (`swiftlang/llvm-project`), which + // is where this API lives: `DiagnosticOptions` is reference-counted, + // `TextDiagnosticPrinter` takes a `DiagnosticOptions *`, and a + // `CompilerInstance` is handed its invocation rather than constructed with it. + // Supporting upstream's newer API as well was tried and abandoned: the two + // differ in three places, and a shared function cannot hold both, because a + // discarded `if constexpr` branch is still type-checked when its code does not + // depend on the template parameter. + auto diagOpts = llvm::makeIntrusiveRefCnt(); + auto diagPrinter = std::make_unique(diagOS, diagOpts.get()); auto diagIDs = llvm::IntrusiveRefCntPtr(new clang::DiagnosticIDs()); clang::DiagnosticsEngine diags(diagIDs, diagOpts, diagPrinter.get(), false); @@ -141,9 +177,16 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity return 65; } - // CompilerInstance takes its invocation through the constructor now; - // setInvocation() no longer exists. - clang::CompilerInstance compiler(invocation); + // The three signatures that differ between clang revisions, as this one + // declares them (swiftlang/llvm-project swift/release/6.2): + // DiagnosticsEngine(IntrusiveRefCntPtr, + // IntrusiveRefCntPtr, DiagnosticConsumer *, + // bool) ← options are refcounted here + // TextDiagnosticPrinter(raw_ostream &, DiagnosticOptions *) ← pointer to them + // CompilerInstance(shared_ptr, ModuleCache *) + // and the invocation goes in through setInvocation(). + clang::CompilerInstance compiler(std::make_shared()); + compiler.setInvocation(invocation); compiler.createDiagnostics(diagPrinter.release(), true); if (!compiler.hasDiagnostics()) { xf_copy_diag("native clang: failed to create diagnostics engine", diagnostics, diagnostics_capacity); diff --git a/App/NativeToolchain/XipArchive.swift b/App/NativeToolchain/XipArchive.swift new file mode 100644 index 0000000..a38df04 --- /dev/null +++ b/App/NativeToolchain/XipArchive.swift @@ -0,0 +1,587 @@ +import Compression +import Foundation + +/// Reads an Apple `.xip` — the container Xcode is distributed in — far enough to +/// pull out the paths a Darwin SDK bundle needs. +/// +/// The format is not documented, but it is stable and public in the sense that +/// tools read it (`xar`, `pbzx`, `unxip`): +/// +/// **xip** is a xar archive: a 28-byte header (`xar!`, header size, version, the +/// compressed and uncompressed size of the table of contents, a checksum +/// algorithm), a zlib-compressed XML TOC listing the members, then the members' +/// data. An Xcode xip has exactly one member worth reading: `Content`, which +/// holds `Xcode.app`. +/// +/// **Content** is a pbzx stream: the bytes `pbzx`, an 8-byte big-endian chunk +/// size, then blocks of (decompressed size, compressed size, bytes) — repeated +/// while a block's decompressed size equals the chunk size. A block is either +/// stored or an **xz** stream, which the two sizes reveal (`pbzx.c` checks for +/// the `\xFD7zXZ` magic, and Apple's `COMPRESSION_LZMA` accepts it). +/// +/// **decompressed** it is an `odc` cpio archive (magic `070707`, six-digit octal +/// fields, no alignment padding) of the `Xcode.app` tree, ending at a +/// `TRAILER!!!` entry. `newc` (`070701`, eight-digit hex, four-byte padding) is +/// accepted too: it is what other tools produce, and the cost of accepting both +/// is a branch. +/// +/// Two properties shape the API: +/// +/// - **It streams.** A full Xcode is ~15 GB and the part that belongs in a build +/// SDK is a small fraction of it, so nothing is ever materialised whole: the +/// walk hands each entry to a closure and only what that closure writes reaches +/// the disk. Peak memory is one decompressed chunk (16 MB for the xips Apple +/// ships) plus the caller's own buffers. +/// - **It is synchronous.** The work is blocking file I/O over a local file, so +/// there is no async surface to get wrong; a caller that does not want to block +/// runs it inside a detached task. +/// +/// LZMA and zlib are Apple's own (`Compression`), the same decoders `unxip` uses on +/// Apple platforms, so this needs no vendored library. +enum XipArchive { + /// A byte range inside the xip, as the archive's TOC records it. + struct Member: Sendable { + var offset: Int64 + var length: Int64 + } + + /// One cpio entry. `mode` decides what it is: `0o040000` directory, + /// `0o120000` symbolic link, `0o100000` regular file. + struct Entry: Sendable { + var name: String + var mode: UInt32 + var size: Int64 + var inode: UInt32 + var device: UInt32 + var linkCount: UInt32 + + var isDirectory: Bool { mode & 0o170000 == 0o040000 } + var isSymlink: Bool { mode & 0o170000 == 0o120000 } + var isRegular: Bool { mode & 0o170000 == 0o100000 } + } + + enum Error: LocalizedError { + case notAXip + case unsupportedArchive(String) + case noContentMember + case unsupportedPayload(String) + case truncated(String) + case decompressionFailed(Int64) + + var errorDescription: String? { + switch self { + case .notAXip: + return "That file is not an Apple .xip: it does not start with 'xar!'." + case .unsupportedArchive(let detail): + return "Unsupported .xip structure: \(detail)." + case .noContentMember: + return "The .xip has no 'Content' member, so there is no Xcode inside it." + case .unsupportedPayload(let detail): + return "Unrecognised .xip payload: \(detail)." + case .truncated(let what): + return "The .xip ended in the middle of \(what). It is probably incomplete." + case .decompressionFailed(let offset): + return "Could not decompress the .xip payload at byte \(offset). The file may be corrupt." + } + } + } + + // MARK: - The xar header and TOC + + /// The `Content` member's byte range, read from the xar table of contents. + static func content(of url: URL) throws -> Member { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + + /// `read(upToCount:)` is allowed to return short, and for a table of contents + /// or a header that would surface as "the .xip is truncated" on a file that + /// is perfectly fine. + func read(_ count: Int) throws -> Data { + var data = Data() + while data.count < count { + guard let piece = try handle.read(upToCount: count - data.count), + !piece.isEmpty else { + throw Error.truncated("the archive header") + } + data += piece + } + return data + } + + guard try read(4).elementsEqual(Array("xar!".utf8)) else { throw Error.notAXip } + + let headerSize = Int(try read(2).bigEndian(as: UInt16.self)) + let version = try read(2).bigEndian(as: UInt16.self) + guard version == 1 else { throw Error.unsupportedArchive("xar version \(version)") } + let tocCompressedSize = Int(try read(8).bigEndian(as: UInt64.self)) + let tocUncompressedSize = Int(try read(8).bigEndian(as: UInt64.self)) + _ = try read(4) // checksum algorithm + + // The header is padded to `headerSize`; skip whatever is left of it. + let consumed = 28 + if headerSize > consumed { + _ = try read(headerSize - consumed) + } + + let compressedTOC = try read(tocCompressedSize) + let toc = try inflate(compressedTOC, to: tocUncompressedSize, what: "the table of contents") + + // A member's `` is relative to the *heap*, which starts immediately + // after the compressed TOC — not to the start of the file. Reading it as an + // absolute offset lands somewhere inside the header and fails on the `pbzx` + // magic, which is at least a loud way to be wrong. + let member = try contentMember(inTOC: toc) + return Member(offset: Int64(headerSize + tocCompressedSize) + member.offset, + length: member.length) + } + + /// The `Content` member of a xar TOC. + /// + /// Parsed with `XMLParser` rather than a string scan: the TOC is the one part of + /// the format whose shape is not fixed (a xip can carry signatures and + /// metadata), and fishing offsets out of XML by hand is how a format reader + /// starts to fail on the next release. + private static func contentMember(inTOC toc: Data) throws -> Member { + let parser = TOCParser() + let xml = XMLParser(data: toc) + xml.delegate = parser + guard xml.parse() else { + throw Error.unsupportedArchive(xml.parserError?.localizedDescription ?? "unreadable TOC") + } + guard let content = parser.members["Content"] else { throw Error.noContentMember } + return content + } + + /// Collects `………`. + /// + /// The offset and length are only read *inside* ``: a member's entry can + /// carry a `` with an offset of its own, and taking the last + /// offset seen would point the payload at the checksum instead of the data. + private final class TOCParser: NSObject, XMLParserDelegate { + private(set) var members: [String: Member] = [:] + private var name: String? + private var offset: Int64? + private var length: Int64? + private var inData = false + private var text = "" + + func parser(_ parser: XMLParser, didStartElement element: String, + namespaceURI: String?, qualifiedName: String?, + attributes: [String: String] = [:]) { + text = "" + switch element { + case "data": inData = true + case "name": name = nil + case "offset" where inData: offset = nil + case "length" where inData: length = nil + default: break + } + } + + func parser(_ parser: XMLParser, foundCharacters string: String) { text += string } + + func parser(_ parser: XMLParser, didEndElement element: String, + namespaceURI: String?, qualifiedName: String?) { + let value = text.trimmingCharacters(in: .whitespacesAndNewlines) + switch element { + case "name": name = value + case "offset" where inData: offset = Int64(value) + case "length" where inData: length = Int64(value) + case "data": inData = false + case "file": + // A `` closes each member, and its `` comes before it, so + // by now the name/offset/length that belong to it are the last ones + // seen. + if let name, let offset, let length { + members[name] = Member(offset: offset, length: length) + } + name = nil; offset = nil; length = nil; inData = false + default: + break + } + text = "" + } + } + + // MARK: - The pbzx stream + + /// Walk a xar `Content` member. + static func forEachEntry( + in url: URL, + content: Member, + progress: (@Sendable (Double) -> Void)? = nil, + _ body: (Entry, Payload) throws -> Void + ) throws { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + try handle.seek(toOffset: UInt64(content.offset)) + let end = content.offset + content.length + let stream = try BlockStream(handle: handle, end: end, contentLength: content.length, + progress: progress) + try readCPIO(from: stream, body) + } + + /// Walk the pbzx `Content` member directly, without a xar wrapper. + static func forEachPBZX( + at url: URL, + progress: (@Sendable (Double) -> Void)? = nil, + _ body: (Entry, Payload) throws -> Void + ) throws { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + let end = Int64(try handle.seekToEnd()) + try handle.seek(toOffset: 0) + let stream = try BlockStream(handle: handle, end: end, contentLength: end, + progress: progress) + try readCPIO(from: stream, body) + } + + /// Decompressed bytes of the payload, one pbzx block at a time. + /// + /// A cursor over a sequence of decompressed blocks rather than one buffer: the + /// payload is ~15 GB decompressed, and a cpio entry — let alone a whole block — + /// does not respect block boundaries, so reads have to be able to span them. + fileprivate final class BlockStream { + private let handle: FileHandle + private let end: Int64 + private let contentLength: Int64 + private let progress: (@Sendable (Double) -> Void)? + private var buffer: [UInt8] = [] + private var position = 0 + private var chunkSize: Int64 = 0 + private var finished = false + private var consumed: Int64 = 0 + + init(handle: FileHandle, end: Int64, contentLength: Int64, + progress: (@Sendable (Double) -> Void)?) throws { + self.handle = handle + self.end = end + self.contentLength = contentLength + self.progress = progress + + guard try readExact(4).elementsEqual(Array("pbzx".utf8)) else { + throw Error.unsupportedPayload("the payload does not start with 'pbzx'") + } + chunkSize = Int64(try readExact(8).bigEndian(as: UInt64.self)) + guard chunkSize > 0, chunkSize <= 1 << 30 else { + throw Error.unsupportedPayload("implausible chunk size \(chunkSize)") + } + } + + /// The next decompressed block, or `nil` at the end of the payload. + private func nextBlock() throws -> [UInt8]? { + if finished { return nil } + let offset = Int64(handle.offsetInFile) + if offset >= end { return nil } + let header = try readUpTo(16) + guard header.count == 16 else { return nil } + let decompressedSize = Int64(header[0..<8].bigEndian(as: UInt64.self)) + let compressedSize = Int64(header[8..<16].bigEndian(as: UInt64.self)) + guard decompressedSize > 0, compressedSize > 0 else { return nil } + + // Never read past the member: the archive's own signature and metadata + // follow it, and swallowing those as payload would be reported as a + // corrupt xip rather than as the end of the stream. + let available = end - offset + let block = try readExact(Int(min(compressedSize, available))) + // The block is stored when it did not shrink; otherwise it is xz. + if decompressedSize < chunkSize { + finished = true + } + if Int64(block.count) == decompressedSize { + return [UInt8](block) + } + return try XipArchive.inflateLZMA(block, to: Int(decompressedSize)) + } + + private func loadNext() throws -> Bool { + guard let block = try nextBlock(), !block.isEmpty else { return false } + buffer = block + position = 0 + consumed = Int64(handle.offsetInFile) + if let progress, contentLength > 0 { + progress(min(1, Double(consumed) / Double(contentLength))) + } + return true + } + + /// Read up to `count` bytes, across block boundaries. Empty at the end. + func read(upTo count: Int) throws -> [UInt8] { + var result: [UInt8] = [] + result.reserveCapacity(count) + while result.count < count { + if position >= buffer.count { + buffer = [] + position = 0 + guard try loadNext() else { break } + } + let take = min(count - result.count, buffer.count - position) + result.append(contentsOf: buffer[position..<(position + take)]) + position += take + } + return result + } + + /// Discard `count` bytes. Same failure mode as `read`, minus the copying. + func skip(_ count: Int64) throws { + var remaining = count + while remaining > 0 { + if position >= buffer.count { + buffer = [] + position = 0 + guard try loadNext() else { + throw Error.truncated("a cpio entry") + } + } + let take = min(Int64(buffer.count - position), remaining) + position += Int(take) + remaining -= take + } + } + + private func readExact(_ count: Int) throws -> Data { + var data = Data() + while data.count < count { + guard let piece = try handle.read(upToCount: count - data.count), + !piece.isEmpty else { + throw Error.truncated("the payload") + } + data += piece + } + consumed = Int64(handle.offsetInFile) + return data + } + + private func readUpTo(_ count: Int) throws -> Data { + let data = try handle.read(upToCount: count) ?? Data() + consumed = Int64(handle.offsetInFile) + return data + } + } + + /// A cursor over one entry's data, valid only for the duration of the callback. + final class Payload { + private let stream: BlockStream + private var remaining: Int64 + + /// `fileprivate` because it takes a `BlockStream`, which is: only the walk + /// in this file constructs payloads. `private` would be too narrow — Swift's + /// `private` does not reach the *enclosing* type, so the walk could not call it. + fileprivate init(stream: BlockStream, size: Int64) { + self.stream = stream + self.remaining = size + } + + var bytesRemaining: Int64 { remaining } + + /// Up to `count` bytes. Shorter than requested only at the end of the entry. + func read(upTo count: Int) throws -> [UInt8] { + guard remaining > 0 else { return [] } + let chunk = try stream.read(upTo: Int(min(Int64(count), remaining))) + remaining -= Int64(chunk.count) + return chunk + } + + /// Everything left of this entry. + func readAll() throws -> [UInt8] { + var result: [UInt8] = [] + while remaining > 0 { + result.append(contentsOf: try read(upTo: 1 << 20)) + } + return result + } + + /// Write the rest of this entry to `url` in `bufferSize` pieces. + func write(to url: URL, bufferSize: Int = 1 << 20) throws { + try? FileManager.default.removeItem(at: url) + FileManager.default.createFile(atPath: url.path, contents: nil) + let output = try FileHandle(forWritingTo: url) + defer { try? output.close() } + while remaining > 0 { + try Task.checkCancellation() + let chunk = try read(upTo: min(bufferSize, Int(remaining))) + guard !chunk.isEmpty else { break } + try output.write(contentsOf: chunk) + } + } + + /// Skip to the end of this entry. + func discard() throws { + guard remaining > 0 else { return } + try stream.skip(remaining) + remaining = 0 + } + } + + // MARK: - cpio + + /// The numeric part of a cpio entry header. + /// + /// Two dialects, and they differ in every field: `odc` (`070707`, which Apple's + /// xips use) is six-digit octal and has no alignment padding, `newc` (`070701`) + /// is eight-digit hex and pads the name to four bytes. Only the fields a Darwin + /// SDK needs are kept — ownership, times and device numbers are dropped rather + /// than carried around unread. + private struct RawHeader { + var mode: UInt32 + var inode: UInt32 + var device: UInt32 + var linkCount: UInt32 + var size: Int64 + var nameLength: Int + var padded: Bool + } + + private static func readCPIO(from stream: BlockStream, + _ body: (Entry, Payload) throws -> Void) throws { + while true { + try Task.checkCancellation() + let magic = try stream.read(upTo: 6) + guard magic.count == 6 else { throw Error.truncated("a cpio entry") } + + let header: RawHeader + switch String(decoding: magic, as: UTF8.self) { + case "070707": header = try odcHeader(from: stream) + case "070701", "070702": header = try newcHeader(from: stream) + case let other: throw Error.unsupportedPayload("cpio magic '\(other)'") + } + + let raw = try stream.read(upTo: header.nameLength) + guard raw.count == header.nameLength, raw.last == 0 else { + throw Error.unsupportedPayload("unnamed cpio entry") + } + let name = String(decoding: raw.dropLast(), as: UTF8.self) + if header.padded { + // `newc` aligns the whole header-plus-name to four bytes; the header + // is 110 bytes, so only the name can push it out of alignment. + let padding = (4 - ((110 + header.nameLength) % 4)) % 4 + if padding > 0 { try stream.skip(Int64(padding)) } + } + + // The end marker: there is nothing after it, and its size is zero. + if name == "TRAILER!!!" { return } + + let payload = Payload(stream: stream, size: header.size) + let entry = Entry(name: name, mode: header.mode, size: header.size, + inode: header.inode, device: header.device, + linkCount: header.linkCount) + try body(entry, payload) + // Whatever the caller did not read still has to come off the stream: + // cpio is sequential, so an entry cannot be seeked past. + try payload.discard() + } + } + + /// `odc`: magic consumed, then six-digit octal fields. + private static func odcHeader(from stream: BlockStream) throws -> RawHeader { + func octal(_ digits: Int) throws -> UInt32 { + let bytes = try stream.read(upTo: digits) + guard bytes.count == digits, + let value = UInt32(String(decoding: bytes, as: UTF8.self), radix: 8) else { + throw Error.unsupportedPayload("unreadable cpio field") + } + return value + } + // Sizes get their own parse: `odc` gives them eleven octal digits, which is + // 33 bits, and reading that into a `UInt32` would fail on exactly the large + // files a compiler's SDK is most likely to contain. + func sizeField(_ digits: Int) throws -> Int64 { + let bytes = try stream.read(upTo: digits) + guard bytes.count == digits, + let value = Int64(String(decoding: bytes, as: UTF8.self), radix: 8) else { + throw Error.unsupportedPayload("unreadable cpio file size") + } + return value + } + let device = try octal(6) + let inode = try octal(6) + let mode = try octal(6) + _ = try octal(6) // uid + _ = try octal(6) // gid + let linkCount = try octal(6) + _ = try octal(6) // rdev + _ = try stream.read(upTo: 11) // mtime + let nameLength = try octal(6) + let size = try sizeField(11) + + return RawHeader(mode: mode, inode: inode, device: device, linkCount: linkCount, + size: size, nameLength: Int(nameLength), padded: false) + } + + /// `newc`: magic consumed, then eight-digit hex fields. + private static func newcHeader(from stream: BlockStream) throws -> RawHeader { + func hexField(_ digits: Int, as type: T.Type) throws -> T { + let bytes = try stream.read(upTo: digits) + guard bytes.count == digits, + let value = T(String(decoding: bytes, as: UTF8.self), radix: 16) else { + throw Error.unsupportedPayload("unreadable cpio field") + } + return value + } + let inode = try hexField(8, as: UInt32.self) + let mode = try hexField(8, as: UInt32.self) + _ = try hexField(8, as: UInt32.self) // uid + _ = try hexField(8, as: UInt32.self) // gid + let linkCount = try hexField(8, as: UInt32.self) + _ = try hexField(8, as: UInt32.self) // mtime + let size = try hexField(8, as: Int64.self) + let deviceMajor = try hexField(8, as: UInt32.self) + let deviceMinor = try hexField(8, as: UInt32.self) + _ = try hexField(8, as: UInt32.self) // rdev major + _ = try hexField(8, as: UInt32.self) // rdev minor + let nameLength = try hexField(8, as: UInt32.self) + _ = try hexField(8, as: UInt32.self) // check + + return RawHeader(mode: mode, inode: inode, + device: (deviceMajor << 16) | deviceMinor, + linkCount: linkCount, size: size, + nameLength: Int(nameLength), padded: true) + } + + // MARK: - Decompression + + /// zlib inflate into a buffer of the declared size. + static func inflate(_ data: Data, to size: Int, what: String) throws -> Data { + guard size > 0 else { throw Error.unsupportedPayload("\(what) declares no size") } + var output = [UInt8](repeating: 0, count: size) + let written = decode(data, into: &output, algorithm: COMPRESSION_ZLIB) + guard written == size else { throw Error.unsupportedArchive("could not inflate \(what)") } + return Data(output) + } + + /// An xz stream as pbzx stores it, decoded by Apple's own implementation. + static func inflateLZMA(_ data: Data, to size: Int) throws -> [UInt8] { + var output = [UInt8](repeating: 0, count: size) + let written = decode(data, into: &output, algorithm: COMPRESSION_LZMA) + guard written == size else { throw Error.decompressionFailed(Int64(size)) } + return output + } + + /// `compression_decode_buffer`, with the input handed over as a pointer rather + /// than relying on the implicit array conversion at the call site. + private static func decode(_ data: Data, into output: inout [UInt8], + algorithm: compression_algorithm) -> Int { + data.withUnsafeBytes { raw in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return 0 } + return compression_decode_buffer(&output, output.count, base, data.count, + nil, algorithm) + } + } +} + +private extension Data { + /// Big-endian integer from the leading bytes of this data. + /// + /// Assembled by shifting rather than by reinterpreting a byte buffer: the + /// pointer form reads as `withUnsafeMutableBytes` gymnastics whose element type + /// inference depends on the enclosing generic, and gets none of that right by + /// being clever. + func bigEndian(as type: T.Type) -> T { + let width = MemoryLayout.size + var value: T = 0 + for (index, byte) in prefix(width).enumerated() { + value |= T(byte) << (8 * (width - 1 - index)) + } + return value + } +} diff --git a/App/Services/AppBundleSigner.swift b/App/Services/AppBundleSigner.swift new file mode 100644 index 0000000..23582bc --- /dev/null +++ b/App/Services/AppBundleSigner.swift @@ -0,0 +1,165 @@ +import Foundation +import Security +import XKit + +/// Signing an app bundle with XKit, in this process. +/// +/// This is the replacement for running `zsign` inside the embedded Linux guest. +/// XKit is xtool's own library and the app already depends on it; its signer is +/// the bundled C implementation, so this stays in-process, needs no subprocess, +/// and never has to copy a private key anywhere the app cannot see it. +/// +/// What it does *not* replace: the Apple ID session. Obtaining a certificate +/// without one means anisette + GrandSlam + 2FA (XKit's `SigningContext` / +/// `DeveloperServices` path), which is a separate, device-only piece of work. This +/// signs with an identity the user already has (a `.p12` and a profile). +enum AppBundleSigner { + /// Everything needed to sign, as `Sendable` values. + /// + /// The entitlements travel as property-list *bytes* rather than a dictionary + /// so this stays `Sendable`: `[String: Any]` is not, and passing one into an + /// async signer is a data-race error under strict concurrency. Decoding inside + /// the signer keeps the non-`Sendable` value in the scope that consumes it. + struct Identity: Sendable { + /// DER bytes of the signing certificate. + var certificateDER: Data + /// The private key, in the format `PrivateKey(data:)` accepts (PEM). + var privateKey: Data + /// `embedded.mobileprovision` contents, when the identity has one. + var provisioningProfile: Data? + /// A property list of entitlements to seal into the signature. Nil means + /// none beyond the profile's own. + var entitlementsPlist: Data? + } + + enum SigningError: LocalizedError { + case noSignerAvailable + case entitlementsUnreadable(String) + + var errorDescription: String? { + switch self { + case .noSignerAvailable: + return "XKit reported no usable codesigning backend in this build." + case .entitlementsUnreadable(let detail): + return "The entitlements could not be read: \(detail)" + } + } + } + + /// Sign `appBundleURL` in place. + /// + /// The profile has to be inside the bundle *before* signing: the signer reads + /// it to decide which entitlements the signature may carry, and a bundle whose + /// signature does not match its profile is rejected at install time rather than + /// at signing time. + static func sign(appBundleURL: URL, identity: Identity) async throws { + let certificate = try Certificate(data: identity.certificateDER) + let privateKey = try PrivateKey(data: identity.privateKey) + + if let profile = identity.provisioningProfile { + try profile.write( + to: appBundleURL.appendingPathComponent("embedded.mobileprovision"), + options: .atomic + ) + } + + let sealedEntitlements = try entitlements(from: identity.entitlementsPlist) + + let signer = try Signer.first() + try await signer.sign( + app: appBundleURL, + identity: .real(certificate, privateKey), + entitlementMapping: [appBundleURL: sealedEntitlements], + progress: { _ in } + ) + } + + /// `Entitlements` decodes from a property list. Building the type directly is + /// not public API, so the bytes are decoded here. + private static func entitlements(from plist: Data?) throws -> Entitlements { + guard let plist, !plist.isEmpty else { + return try Entitlements(entitlements: []) + } + do { + return try PropertyListDecoder().decode(Entitlements.self, from: plist) + } catch { + throw SigningError.entitlementsUnreadable(error.localizedDescription) + } + } +} + +/// Loading a `.p12` into an `AppBundleSigner.Identity` using the Security +/// framework, which is the only PKCS#12 reader available on iOS. +enum PKCS12Identity { + enum LoadError: LocalizedError { + case badPassword + case noIdentity + case missingCertificate(Int32) + case missingKey(Int32) + + var errorDescription: String? { + switch self { + case .badPassword: + return "The PKCS#12 password is wrong, or the file is not a .p12." + case .noIdentity: + return "That file holds no signing identity (certificate + private key)." + case .missingCertificate(let status): + return "Could not export the certificate (OSStatus \(status))." + case .missingKey(let status): + return "Could not export the private key (OSStatus \(status))." + } + } + } + + /// Import a `.p12` and return the certificate and key bytes. + /// + /// The key is exported as PKCS#1 DER and then wrapped in a PEM envelope, + /// which is the form the signer's parser reads. A key that cannot be exported + /// this way (a hardware-backed one, say) fails here with the Security status + /// rather than somewhere inside the signer. + static func load(data: Data, password: String) throws -> (certificate: Data, key: Data) { + var items: CFArray? + let options = [kSecImportExportPassphrase as String: password] as CFDictionary + let status = SecPKCS12Import(data as CFData, options, &items) + guard status == errSecSuccess else { throw LoadError.badPassword } + + guard let entries = items as? [[String: Any]] else { throw LoadError.noIdentity } + for entry in entries { + guard let identityRef = entry[kSecImportItemIdentity as String] else { continue } + let identity = identityRef as! SecIdentity + + var certificateRef: SecCertificate? + let certStatus = SecIdentityCopyCertificate(identity, &certificateRef) + guard certStatus == errSecSuccess, let certificate = certificateRef else { + throw LoadError.missingCertificate(certStatus) + } + + var keyRef: SecKey? + let keyStatus = SecIdentityCopyPrivateKey(identity, &keyRef) + guard keyStatus == errSecSuccess, let key = keyRef else { + throw LoadError.missingKey(keyStatus) + } + + let certificateData = SecCertificateCopyData(certificate) as Data + var error: Unmanaged? + guard let keyData = SecKeyCopyExternalRepresentation(key, &error) as Data? else { + let code = (error?.takeRetainedValue()).map { CFErrorGetCode($0) } ?? -1 + throw LoadError.missingKey(Int32(code)) + } + return (certificateData, pem(keyData)) + } + throw LoadError.noIdentity + } + + /// Wrap raw private-key bytes in a PEM envelope. + /// + /// `SecKeyCopyExternalRepresentation` returns the key in its *natural* form: + /// PKCS#1 for RSA. The label therefore says RSA rather than the PKCS#8 + /// "PRIVATE KEY" — a parser that trusts the label would read PKCS#1 bytes as + /// PKCS#8 and fail on the wrapper it expects to find there. + private static func pem(_ der: Data) -> Data { + let base64 = der.base64EncodedString(options: [.lineLength64Characters, .endLineWithLineFeed]) + let text = "-----BEGIN RSA PRIVATE KEY-----\n\(base64)\n-----END RSA PRIVATE KEY-----\n" + return Data(text.utf8) + } +} diff --git a/App/Services/DirectorySize.swift b/App/Services/DirectorySize.swift deleted file mode 100644 index 8030ba2..0000000 --- a/App/Services/DirectorySize.swift +++ /dev/null @@ -1,41 +0,0 @@ -import Foundation - -/// Recursive on-disk size of a directory. -/// -/// Deliberately a plain `nonisolated` function so callers can run it off the main -/// actor: walking the imported rootfs is tens of thousands of file stats, and doing -/// that during a SwiftUI `body` evaluation can freeze — or watchdog-kill — the app. -enum DirectorySize { - static func bytes(at url: URL, limit: Int = 400_000) -> Int64 { - let fm = FileManager.default - var isDirectory: ObjCBool = false - guard fm.fileExists(atPath: url.path, isDirectory: &isDirectory) else { return 0 } - guard isDirectory.boolValue else { - let size = (try? url.resourceValues(forKeys: [.fileSizeKey]))?.fileSize ?? 0 - return Int64(size) - } - // No `.skipsHiddenFiles`: in the guest rootfs the interesting things *are* - // hidden — the Swift toolchain is /root/.local/share/swiftly, the SDK is - // /root/.swiftpm/swift-sdks, the caches are /root/.cache — so skipping - // hidden entries reported a few MB for a download that is several GB, and - // reported almost none of the rootfs's own size. (Symlinks are not - // followed by this enumerator, so counting them cannot loop.) - guard let enumerator = fm.enumerator( - at: url, - includingPropertiesForKeys: [.fileSizeKey, .isRegularFileKey], - options: [] - ) else { return 0 } - - var total: Int64 = 0 - var visited = 0 - for case let fileURL as URL in enumerator { - visited += 1 - if visited > limit { break } - guard let values = try? fileURL.resourceValues(forKeys: [.fileSizeKey, .isRegularFileKey]), - values.isRegularFile == true, - let size = values.fileSize else { continue } - total += Int64(size) - } - return total - } -} diff --git a/App/Services/DownloadError.swift b/App/Services/DownloadError.swift new file mode 100644 index 0000000..205faa4 --- /dev/null +++ b/App/Services/DownloadError.swift @@ -0,0 +1,26 @@ +import Foundation + +/// Failures of a network download. +/// +/// This used to live in `DownloadManager`, which served the Downloads screen and +/// the Xcode.xip staging flow — both of which went with the guest. The error type +/// stayed because two things still download: `XForgeReleases` (release metadata and +/// asset URLs) and `NativeSDK` (the Darwin SDK bundle). Its cases are the ones those +/// two raise; the `.notDownloaded` case went with the download manager that could +/// report a partial file. +enum DownloadError: LocalizedError { + case http(status: Int, url: URL) + case invalidResponse(URL) + case assetNotFound(String) + + var errorDescription: String? { + switch self { + case .http(let status, let url): + return "The server returned HTTP \(status) for \(url.lastPathComponent)." + case .invalidResponse(let url): + return "Unexpected response from \(url.host ?? "the server")." + case .assetNotFound(let what): + return "No published release asset found: \(what)." + } + } +} diff --git a/App/Services/DownloadManager.swift b/App/Services/DownloadManager.swift deleted file mode 100644 index 13dd0d1..0000000 --- a/App/Services/DownloadManager.swift +++ /dev/null @@ -1,237 +0,0 @@ -import Foundation -import Combine - -/// One downloadable item with live state. -struct DownloadItem: Identifiable, Equatable { - let id = UUID() - var name: String - var url: URL - var state: State = .idle - var progress: Double = 0 - var destination: URL? - var error: String? - - enum State: String, Equatable { - case idle, downloading, done, failed - } -} - -/// Real host-side downloads with state tracking. Files land in `folder` (default the -/// app's Documents/downloads), which is shared into the guest Linux at `/host/downloads`. -@MainActor -final class DownloadManager: ObservableObject { - @Published private(set) var items: [DownloadItem] = [] - let folder: URL - - init(folder: URL? = nil) { - let base = folder ?? XForgeEnvironment.downloadsDirectory - self.folder = base - try? FileManager.default.createDirectory(at: self.folder, withIntermediateDirectories: true) - } - - @discardableResult - func enqueue(name: String, url: URL) -> UUID { - if let existing = items.firstIndex(where: { $0.name == name }) { - items[existing].url = url - items[existing].state = .idle - items[existing].progress = 0 - items[existing].error = nil - return items[existing].id - } - let item = DownloadItem(name: name, url: url) - items.append(item) - return item.id - } - - func start(_ id: UUID) async { - guard let idx = items.firstIndex(where: { $0.id == id }) else { return } - items[idx].state = .downloading - items[idx].progress = 0 - items[idx].error = nil - let name = items[idx].name - let url = items[idx].url - do { - let dest = folder.appendingPathComponent(name) - let saved = try await DownloadManager.download(url, to: dest) { fraction in - // Called from the session's delegate queue. - Task { @MainActor [weak self] in - guard let self, let i = self.items.firstIndex(where: { $0.id == id }) else { return } - self.items[i].progress = fraction - } - } - if let i = items.firstIndex(where: { $0.id == id }) { - items[i].state = .done - items[i].progress = 1 - items[i].destination = saved - } - } catch { - if let i = items.firstIndex(where: { $0.id == id }) { - items[i].state = .failed - items[i].error = error.localizedDescription - } - } - } - - func retry(_ id: UUID) async { - await start(id) - } - - func remove(_ id: UUID) { - guard let idx = items.firstIndex(where: { $0.id == id }) else { return } - if let dest = items[idx].destination { - try? FileManager.default.removeItem(at: dest) - } - items.remove(at: idx) - } - - /// Download `url` to `destination`, replacing anything already there and - /// reporting byte progress. - /// - /// Retries on failure: a 456 MB download dropped its connection 22 seconds - /// in on a real device, and one dropped connection is not a reason to make - /// the user start over. A server-side answer (404/403) is not retried -- - /// it will not change. - @discardableResult - nonisolated static func download( - _ url: URL, - to destination: URL, - attempts: Int = 3, - progress: @escaping @Sendable (Double) -> Void - ) async throws -> URL { - try FileManager.default.createDirectory( - at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) - - var lastError: Error = DownloadError.invalidResponse(url) - for attempt in 1...max(1, attempts) { - do { - let temp = try await ProgressDownload(url: url, progress: progress).start() - - try? FileManager.default.removeItem(at: destination) - try FileManager.default.moveItem(at: temp, to: destination) - progress(1.0) - return destination - } catch let error as DownloadError { - throw error - } catch { - lastError = error - XForgeLog.note("download: attempt \(attempt)/\(attempts) of " - + "\(url.lastPathComponent) failed: \(error.localizedDescription)") - if attempt < attempts { - try? await Task.sleep(nanoseconds: UInt64(attempt) * 2_000_000_000) - } - } - } - throw lastError - } -} - -/// A single download that reports progress. -/// -/// `URLSession.download(from:)` cannot report anything until it is finished, and -/// the files here are hundreds of megabytes — so a delegate does the work and -/// this object keeps the session, the delegate and the continuation together -/// for as long as the transfer lasts. -private final class ProgressDownload: NSObject, URLSessionDownloadDelegate, @unchecked Sendable { - private let url: URL - private let progress: @Sendable (Double) -> Void - - private let lock = NSLock() - private var continuation: CheckedContinuation? - private var session: URLSession? - private var finished = false - - init(url: URL, progress: @escaping @Sendable (Double) -> Void) { - self.url = url - self.progress = progress - } - - /// Returns the finished download's temporary file, which the caller owns. - func start() async throws -> URL { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - lock.lock() - self.continuation = continuation - let session = URLSession(configuration: .default, delegate: self, delegateQueue: nil) - self.session = session - lock.unlock() - session.downloadTask(with: url).resume() - } - } - - private func finish(_ result: Result) { - lock.lock() - defer { lock.unlock() } - guard !finished, let continuation else { return } - finished = true - self.continuation = nil - session?.finishTasksAndInvalidate() - session = nil - continuation.resume(with: result) - } - - func urlSession( - _ session: URLSession, - downloadTask: URLSessionDownloadTask, - didWriteData bytesWritten: Int64, - totalBytesWritten: Int64, - totalBytesExpectedToWrite: Int64 - ) { - guard totalBytesExpectedToWrite > 0 else { return } - progress(min(1.0, Double(totalBytesWritten) / Double(totalBytesExpectedToWrite))) - } - - func urlSession( - _ session: URLSession, - downloadTask: URLSessionDownloadTask, - didFinishDownloadingTo location: URL - ) { - // The file at `location` is deleted when this returns, so take it now. - if let http = downloadTask.response as? HTTPURLResponse, - !(200..<300).contains(http.statusCode) { - finish(.failure(DownloadError.http(status: http.statusCode, url: url))) - return - } - let kept = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-\(UUID().uuidString).download") - do { - try? FileManager.default.removeItem(at: kept) - try FileManager.default.moveItem(at: location, to: kept) - finish(.success(kept)) - } catch { - finish(.failure(error)) - } - } - - func urlSession( - _ session: URLSession, - task: URLSessionTask, - didCompleteWithError error: Error? - ) { - // Success already went through didFinishDownloadingTo; a non-nil error - // here is the transfer giving up (connection lost, timeout, ...). - if let error { - finish(.failure(error)) - } else { - finish(.failure(DownloadError.invalidResponse(url))) - } - } -} - -enum DownloadError: LocalizedError { - case http(status: Int, url: URL) - case invalidResponse(URL) - case notDownloaded - case assetNotFound(String) - - var errorDescription: String? { - switch self { - case .http(let status, let url): - return "The server returned HTTP \(status) for \(url.lastPathComponent)." - case .invalidResponse(let url): - return "Unexpected response from \(url.host ?? "the server")." - case .notDownloaded: - return "The file has not finished downloading." - case .assetNotFound(let what): - return "No published release asset found: \(what)." - } - } -} diff --git a/App/Services/GuestNetwork.swift b/App/Services/GuestNetwork.swift deleted file mode 100644 index 5a6cc57..0000000 --- a/App/Services/GuestNetwork.swift +++ /dev/null @@ -1,67 +0,0 @@ -import Foundation - -/// The guest's `nameserver` configuration. -/// -/// The engine hands the guest the host's sockets, but **name resolution happens -/// inside the guest**: musl (and busybox's `wget`, and `apk`) read -/// `/etc/resolv.conf`, and the Alpine minirootfs XForge bundles ships no -/// nameservers at all. ish-arm64's own app writes that file from the device's DNS -/// on every boot for exactly this reason; without the same step the guest -/// resolves nothing — `apk add`, and therefore all of -/// `install-toolchain.sh`, fails with `DNS: transient error`. -/// -/// **Order matters, and it is not the obvious one.** A home network hands out -/// its router (`192.168.x.1`), which is a *local* address: iOS refuses -/// connections to those unless the user has granted Local Network access, and -/// the guest's DNS queries are made by this very process, so the refusal lands -/// as an in-guest "DNS: transient error" with a correct-looking resolv.conf. -/// Observed exactly that on a device whose resolver was `192.168.4.1`: the file -/// was right, `apk` still failed. -/// -/// So the public resolvers go first — they are reachable without any -/// permission — and the device's own servers are kept as the fallback for -/// networks where public DNS is blocked (captive portals, some VPNs). -enum GuestNetwork { - /// The device's DNS servers, in preference order. Empty when the system - /// will not say (no network, or the dnsinfo SPI is unavailable) — see - /// HostDNS.c for why this is not SystemConfiguration. - static func systemDNSServers() -> [String] { - var buffer = [CChar](repeating: 0, count: 512) - let count = buffer.withUnsafeMutableBufferPointer { pointer -> Int32 in - guard let base = pointer.baseAddress else { return 0 } - return xf_host_dns_servers(base, pointer.count) - } - guard count > 0 else { return [] } - return String(cString: buffer) - .split(separator: "\n") - .map(String.init) - .filter { !$0.isEmpty } - } - - /// Reachable without Local Network permission, so they are tried first. - static let publicServers = ["1.1.1.1", "8.8.8.8"] - - /// musl reads every `nameserver` line and tries them in order, so the list - /// is short on purpose: a dead entry costs a timeout before the next one. - static let maxServers = 3 - - static func resolvConf(servers: [String]) -> String { - var ordered: [String] = [] - for server in publicServers + servers where !ordered.contains(server) { - ordered.append(server) - if ordered.count == maxServers { break } - } - return ordered.map { "nameserver \($0)" }.joined(separator: "\n") + "\n" - } - - /// The file the guest should have. `source` says which servers were - /// actually used, for the engine log. - static func resolvConfForDevice() -> (text: String, source: String) { - let deviceServers = systemDNSServers() - let text = resolvConf(servers: deviceServers) - let source = deviceServers.isEmpty - ? "public (the device published none)" - : "public + device (\(deviceServers.joined(separator: ", ")))" - return (text, source) - } -} diff --git a/App/Services/HostDNS.c b/App/Services/HostDNS.c deleted file mode 100644 index dfeb313..0000000 --- a/App/Services/HostDNS.c +++ /dev/null @@ -1,158 +0,0 @@ -// -// HostDNS.c -// XForge -// -// The device's DNS servers, for the guest's /etc/resolv.conf. -// -// Resolution happens *inside* the guest, and the Alpine minirootfs XForge -// bundles ships no nameservers, so the guest cannot resolve anything until -// something writes that file — `apk add`, and therefore all of -// install-toolchain.sh, fails with DNS errors without it. -// -// Where the servers come from, on iOS: -// -// - SystemConfiguration's dynamic-store API (SCDynamicStoreCopyValue on -// State:/Network/Global/DNS) is the obvious answer and is what the first -// version of this used. It is macOS-only: on iOS those symbols are marked -// unavailable at compile time. -// - libresolv's res_ninit() would be next, but it is not usable from an app. -// - What *is* available is libSystem's dnsinfo SPI: -// dns_configuration_copy(), which is exactly what ish-arm64's own app uses -// for this same job. It is resolved with dlsym rather than linked, so the -// app carries no reference to a private symbol; if it ever disappears, -// this returns 0 and the caller falls back to public resolvers. -// -// The struct layouts below must match the SPI's. They are the ones ish-arm64 -// ships (app/AppDelegate.m), including the #pragma pack(4). -// -#include "HostDNS.h" - -#include -#include -#include -#include -#include -#include -#include - -#pragma pack(push, 4) -typedef struct { - struct in_addr address; - struct in_addr mask; -} xf_dns_sortaddr_t; - -typedef struct { - char *domain; - int32_t n_nameserver; - struct sockaddr **nameserver; - uint16_t port; - int32_t n_search; - char **search; - int32_t n_sortaddr; - xf_dns_sortaddr_t **sortaddr; - char *options; - uint32_t timeout; - uint32_t search_order; - uint32_t if_index; - uint32_t flags; - uint32_t reach_flags; - uint32_t reserved[5]; -} xf_dns_resolver_t; - -typedef struct { - int32_t n_resolver; - xf_dns_resolver_t **resolver; - int32_t n_scoped_resolver; - xf_dns_resolver_t **scoped_resolver; - uint32_t reserved[5]; -} xf_dns_config_t; -#pragma pack(pop) - -typedef xf_dns_config_t *(*xf_dns_copy_fn)(void); -typedef void (*xf_dns_free_fn)(xf_dns_config_t *); - -// Copies rather than casts: the resolver and nameserver arrays are arrays of -// pointers that the SPI does not promise are aligned for the destination type. -static xf_dns_resolver_t *xf_resolver_at(xf_dns_resolver_t **resolvers, int index) { - xf_dns_resolver_t *resolver = NULL; - if (resolvers == NULL || index < 0) - return NULL; - memcpy(&resolver, resolvers + index, sizeof(resolver)); - return resolver; -} - -static struct sockaddr *xf_nameserver_at(struct sockaddr **nameservers, int index) { - struct sockaddr *address = NULL; - if (nameservers == NULL || index < 0) - return NULL; - memcpy(&address, nameservers + index, sizeof(address)); - return address; -} - -int xf_host_dns_servers(char *out, size_t cap) { - static xf_dns_copy_fn copy_fn; - static xf_dns_free_fn free_fn; - static int looked_up = 0; - if (!looked_up) { - looked_up = 1; - copy_fn = (xf_dns_copy_fn) dlsym(RTLD_DEFAULT, "dns_configuration_copy"); - free_fn = (xf_dns_free_fn) dlsym(RTLD_DEFAULT, "dns_configuration_free"); - } - if (copy_fn == NULL || free_fn == NULL || out == NULL || cap == 0) - return 0; - - xf_dns_config_t *config = copy_fn(); - if (config == NULL) - return 0; - - size_t used = 0; - int written = 0; - out[0] = '\0'; - - for (int r = 0; r < config->n_resolver; r++) { - xf_dns_resolver_t *resolver = xf_resolver_at(config->resolver, r); - if (resolver == NULL || resolver->n_nameserver <= 0) - continue; - // mDNS resolvers (.local) list 224.0.0.251, which is not a nameserver - // to hand to musl. - if (resolver->options != NULL && strcmp(resolver->options, "mdns") == 0) - continue; - - for (int i = 0; i < resolver->n_nameserver; i++) { - struct sockaddr *address = xf_nameserver_at(resolver->nameserver, i); - if (address == NULL) - continue; - char text[INET6_ADDRSTRLEN] = {0}; - void *source = NULL; - if (address->sa_family == AF_INET) - source = &((struct sockaddr_in *) address)->sin_addr; - else if (address->sa_family == AF_INET6) - source = &((struct sockaddr_in6 *) address)->sin6_addr; - else - continue; - if (inet_ntop(address->sa_family, source, text, sizeof(text)) == NULL) - continue; - if (text[0] == '\0') - continue; - - // Skip duplicates: the SPI lists one resolver per interface. - if (strstr(out, text) != NULL) - continue; - - size_t len = strlen(text); - if (used + len + 2 > cap) - break; - memcpy(out + used, text, len); - used += len; - out[used++] = '\n'; - out[used] = '\0'; - written++; - if (written >= 4) - goto done; - } - } - -done: - free_fn(config); - return written; -} diff --git a/App/Services/HostDNS.h b/App/Services/HostDNS.h deleted file mode 100644 index 8c11723..0000000 --- a/App/Services/HostDNS.h +++ /dev/null @@ -1,20 +0,0 @@ -// -// HostDNS.h -// XForge -// -// The device's DNS servers, for the guest's /etc/resolv.conf. -// -// See HostDNS.c for why this is not SystemConfiguration. -// -#ifndef XFORGE_HOST_DNS_H -#define XFORGE_HOST_DNS_H - -#include - -/// Write up to `cap` bytes of the device's DNS servers into `out`, one IPv4 or -/// IPv6 address per line, and return how many were written. Returns 0 when the -/// system does not publish any (no network, or the SPI is unavailable), in -/// which case the caller should fall back to public resolvers. -int xf_host_dns_servers(char *out, size_t cap); - -#endif /* XFORGE_HOST_DNS_H */ diff --git a/App/Services/IPAConfigureSignService.swift b/App/Services/IPAConfigureSignService.swift deleted file mode 100644 index f95105f..0000000 --- a/App/Services/IPAConfigureSignService.swift +++ /dev/null @@ -1,186 +0,0 @@ -import Foundation -import SwiftUI - -/// Host-side state for the zsign workflow. zsign is a Linux C++ command-line -/// signer, so it runs inside the embedded Alpine guest; the host UI only stages -/// files, invokes it, and copies the signed IPA back. -@MainActor -final class IPAConfigureSignService: ObservableObject { - @Published var inputIPA: URL? - @Published var provisioningProfile: URL? - @Published var p12: URL? - @Published var entitlements: URL? - /// Never persisted. Cleared immediately after the signing attempt. - @Published var password = "" - @Published var bundleIdentifier = "" - @Published var displayName = "" - @Published var version = "" - @Published var status = "Ready to sign." - @Published var signedIPA: URL? - @Published var isWorking = false - @Published var error: String? - - private let makeVM: @MainActor () -> any LinuxVM - - init(makeVM: @escaping @MainActor () -> any LinuxVM = { XForgeEnvironment.makeVM() }) { - self.makeVM = makeVM - } - - var canSign: Bool { - inputIPA != nil && provisioningProfile != nil && p12 != nil && !password.isEmpty && !isWorking - } - - func sign() async { - guard let inputIPA, let provisioningProfile, let p12 else { return } - let oneShotPassword = password - guard !oneShotPassword.contains("\n"), !oneShotPassword.contains("\r") else { - password = "" - error = "The PKCS#12 password cannot contain a newline." - return - } - let ipaScoped = inputIPA.startAccessingSecurityScopedResource() - let profileScoped = provisioningProfile.startAccessingSecurityScopedResource() - let p12Scoped = p12.startAccessingSecurityScopedResource() - let entitlementsScoped = entitlements?.startAccessingSecurityScopedResource() ?? false - defer { - if ipaScoped { inputIPA.stopAccessingSecurityScopedResource() } - if profileScoped { provisioningProfile.stopAccessingSecurityScopedResource() } - if p12Scoped { p12.stopAccessingSecurityScopedResource() } - if entitlementsScoped { entitlements?.stopAccessingSecurityScopedResource() } - } - - isWorking = true - error = nil - signedIPA = nil - defer { - // Do not leave the signing secret in the view model after the attempt. - password = "" - isWorking = false - } - - let id = UUID().uuidString - let dir = "/root/xforge/zsign-jobs/\(id)" - let ipaPath = "\(dir)/input.ipa" - let p12Path = "\(dir)/signing.p12" - let profilePath = "\(dir)/profile.mobileprovision" - let passwordPath = "\(dir)/.p12-password" - let scriptPath = "\(dir)/run-sign.sh" - let outputPath = "\(dir)/signed.ipa" - var vm: (any LinuxVM)? - - do { - status = "Starting Alpine…" - let guest = makeVM() - vm = guest - await guest.prepareRootfs() - try await guest.boot() - try await run(guest, "mkdir -p \(GuestShell.quote(dir)) && chmod 700 \(GuestShell.quote(dir))") - - status = "Copying signing inputs into the guest…" - try await guest.copyIn(hostURL: inputIPA, to: ipaPath) - try await guest.copyIn(hostURL: p12, to: p12Path) - try await guest.copyIn(hostURL: provisioningProfile, to: profilePath) - try await SystemComponents.ensureZsignInstaller(in: guest) - - // zsign's upstream -p accepts a password as a process argument, which - // a same-user guest process could read through /proc//cmdline. - // A short-lived file is narrower exposure: random name inside a 700 - // job directory, chmod 600, consumed by the pinned -Q password-file - // patch and removed with the rest of this per-signing directory. - let passwordTemp = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-p12-\(UUID().uuidString)") - try Data(oneShotPassword.utf8).write(to: passwordTemp, options: .atomic) - try FileManager.default.setAttributes([.posixPermissions: 0o600], - ofItemAtPath: passwordTemp.path) - defer { try? FileManager.default.removeItem(at: passwordTemp) } - try await guest.copyIn(hostURL: passwordTemp, to: passwordPath) - try await run(guest, "chmod 600 \(GuestShell.quote(passwordPath))") - - var args = "-f -k \(GuestShell.quote(p12Path)) -m \(GuestShell.quote(profilePath))" - if let entitlements { - let path = "\(dir)/entitlements.plist" - try await guest.copyIn(hostURL: entitlements, to: path) - args += " -e \(GuestShell.quote(path))" - } - if !bundleIdentifier.isEmpty { args += " -b \(GuestShell.quote(bundleIdentifier))" } - if !displayName.isEmpty { args += " -n \(GuestShell.quote(displayName))" } - if !version.isEmpty { args += " -r \(GuestShell.quote(version))" } - - // The patched zsign reads its password from this mode-600 file rather - // than argv, so the signing secret is not exposed in the guest process - // table. The per-job directory is deleted after the sign on success - // or failure. - let script = """ - #!/bin/sh - set -eu - umask 077 - exec zsign \(args) -Q \(GuestShell.quote(passwordPath)) -o \(GuestShell.quote(outputPath)) \(GuestShell.quote(ipaPath)) - """ - let hostScript = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-zsign-\(UUID().uuidString).sh") - try Data(script.utf8).write(to: hostScript, options: .atomic) - defer { try? FileManager.default.removeItem(at: hostScript) } - try await guest.copyIn(hostURL: hostScript, to: scriptPath) - try await run(guest, "chmod 700 \(GuestShell.quote(scriptPath))") - - status = "Installing the pinned zsign build in Alpine (first use only)…" - try await run(guest, "sh /root/xforge/install-zsign.sh") - status = "Signing in the guest…" - try await run(guest, "sh \(GuestShell.quote(scriptPath))") - - status = "Copying the signed IPA back…" - let destination = XForgeEnvironment.documentDirectory - .appendingPathComponent("Signed-\(inputIPA.deletingPathExtension().lastPathComponent).ipa") - try await guest.copyOut(guestPath: outputPath, to: destination) - signedIPA = destination - status = "Signed IPA ready: \(destination.lastPathComponent)" - } catch { - self.error = error.localizedDescription - status = "Signing failed." - } - - // Remove the private key, provisioning profile and password-bearing script - // from the guest even when signing failed. Keep the output IPA (if any). - if let vm { - _ = try? await vm.run("rm -rf \(GuestShell.quote(dir))", environment: nil) { _ in } - } - } - - private func run(_ vm: any LinuxVM, _ command: String) async throws { - let box = OutputBox() - let status = try await vm.run(command, environment: nil) { [weak self] line in - box.append(line) - Task { @MainActor in - // The status panel is a live short line, not a transcript of - // secrets or command text. Only show guest output that does not - // contain arguments from the signing script. - self?.status = line - } - } - guard status == 0 else { - throw SignError.commandFailed(status, detail: box.value) - } - } - - enum SignError: LocalizedError { - case commandFailed(Int32, detail: String) - var errorDescription: String? { - switch self { - case .commandFailed(let code, let detail): - let safe = detail.trimmingCharacters(in: .whitespacesAndNewlines) - return safe.isEmpty - ? "Guest signing command failed (exit \(code))." - : "Guest signing command failed (exit \(code)): \(safe)" - } - } - } -} - -/// Thread-safe output accumulator used while a guest command streams from its -/// background tailer. -private final class OutputBox: @unchecked Sendable { - private let lock = NSLock() - private var stored = "" - func append(_ value: String) { lock.lock(); stored += value; lock.unlock() } - var value: String { lock.lock(); defer { lock.unlock() }; return stored } -} diff --git a/App/Services/IPASigningJob.swift b/App/Services/IPASigningJob.swift new file mode 100644 index 0000000..f71dd18 --- /dev/null +++ b/App/Services/IPASigningJob.swift @@ -0,0 +1,167 @@ +import Foundation +import Combine + +/// Sign an `.ipa` in this process with XKit. +/// +/// This replaces the `zsign` job that used to run inside the embedded Linux. The +/// shape is deliberately the same — IPA + `.p12` + profile + entitlements in, a +/// signed IPA out — because that is the workflow a user has: they have a built +/// (unsigned) IPA from the Build screen and an Apple Developer certificate. +/// +/// Two things are narrower than before, both on purpose: +/// +/// - the private key never leaves the app. It is imported through the Security +/// framework and handed to the signer in memory; nothing is written to disk, +/// which is why the `zsign` password-file workaround is gone with `zsign`; +/// - the signature is produced by XKit's own signer, so a signed app is one the +/// same code path would produce on a Mac (zsign was a copy of it). +@MainActor +final class IPASigningJob: ObservableObject { + @Published var inputIPA: URL? + @Published var p12: URL? + @Published var provisioningProfile: URL? + @Published var entitlements: URL? + /// Never persisted. Cleared immediately after the signing attempt. + @Published var password = "" + @Published var bundleIdentifier = "" + @Published var displayName = "" + @Published var version = "" + @Published var status = "Ready to sign." + @Published var signedIPA: URL? + @Published var isWorking = false + @Published var error: String? + + var canSign: Bool { + inputIPA != nil && p12 != nil && !password.isEmpty && !isWorking + } + + func sign() async { + guard let inputIPA, let p12 else { return } + let oneShotPassword = password + + let scoped = [inputIPA, p12, provisioningProfile, entitlements].compactMap { $0 } + .map { ($0, $0.startAccessingSecurityScopedResource()) } + defer { + for (url, accessed) in scoped where accessed { + url.stopAccessingSecurityScopedResource() + } + } + + isWorking = true + error = nil + signedIPA = nil + defer { + // Do not leave the signing secret in the view model after the attempt. + password = "" + isWorking = false + } + + do { + status = "Reading the signing identity…" + let keyMaterial = try PKCS12Identity.load( + data: try Data(contentsOf: p12), + password: oneShotPassword + ) + + let profileData = try provisioningProfile.map { try Data(contentsOf: $0) } + let entitlementsPlist = try entitlements.map { try Data(contentsOf: $0) } + + status = "Unpacking \(inputIPA.lastPathComponent)…" + let work = FileManager.default.temporaryDirectory + .appendingPathComponent("xforge-sign-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: work) } + try FileManager.default.createDirectory(at: work, withIntermediateDirectories: true) + try FileManager.default.unzipItem(at: inputIPA, to: work) + + let payload = work.appendingPathComponent("Payload", isDirectory: true) + let apps = try apps(in: payload) + guard apps.count == 1, let app = apps.first else { + throw SigningJobError.expectedOneApp(apps.map(\.lastPathComponent)) + } + + try applyIdentityOverrides(to: app) + + status = "Signing \(app.lastPathComponent)…" + try await AppBundleSigner.sign( + appBundleURL: app, + identity: .init( + certificateDER: keyMaterial.certificate, + privateKey: keyMaterial.key, + provisioningProfile: profileData, + entitlementsPlist: entitlementsPlist + ) + ) + + status = "Repackaging…" + let destination = XForgeEnvironment.documentDirectory.appendingPathComponent( + "Signed-" + inputIPA.deletingPathExtension().lastPathComponent + ".ipa" + ) + try? FileManager.default.removeItem(at: destination) + try FileManager.default.zipItem( + at: payload, + to: destination, + shouldKeepParent: true + ) + signedIPA = destination + status = "Signed IPA ready: \(destination.lastPathComponent)" + } catch { + self.error = error.localizedDescription + status = "Signing failed." + } + } + + // MARK: - Helpers + + private func apps(in payload: URL) throws -> [URL] { + let entries = (try? FileManager.default.contentsOfDirectory( + at: payload, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + )) ?? [] + return entries.filter { $0.pathExtension == "app" } + } + + /// Apply the identity fields the user typed over the bundle's own plist. + /// + /// Only these three: replacing the plist wholesale would drop + /// `CFBundleExecutable` and produce an app that cannot launch. + private func applyIdentityOverrides(to app: URL) throws { + var overrides: [String: Any] = [:] + if !bundleIdentifier.isEmpty { overrides["CFBundleIdentifier"] = bundleIdentifier } + if !displayName.isEmpty { + overrides["CFBundleDisplayName"] = displayName + overrides["CFBundleName"] = displayName + } + if !version.isEmpty { overrides["CFBundleShortVersionString"] = version } + guard !overrides.isEmpty else { return } + + let plistURL = app.appendingPathComponent("Info.plist") + guard let data = try? Data(contentsOf: plistURL), + var plist = try? PropertyListSerialization.propertyList(from: data, format: nil) + as? [String: Any] else { + throw SigningJobError.unreadableInfoPlist + } + for (key, value) in overrides { plist[key] = value } + let updated = try PropertyListSerialization.data( + fromPropertyList: plist, + format: .xml, + options: 0 + ) + try updated.write(to: plistURL, options: .atomic) + } +} + +enum SigningJobError: LocalizedError { + case expectedOneApp([String]) + case unreadableInfoPlist + + var errorDescription: String? { + switch self { + case .expectedOneApp(let found): + let list = found.isEmpty ? "none" : found.joined(separator: ", ") + return "The IPA must contain exactly one .app in Payload/; found \(list)." + case .unreadableInfoPlist: + return "The app's Info.plist could not be read." + } + } +} diff --git a/App/Services/ProjectExporter.swift b/App/Services/ProjectExporter.swift index d2ab589..382c0bd 100644 --- a/App/Services/ProjectExporter.swift +++ b/App/Services/ProjectExporter.swift @@ -1,68 +1,82 @@ import Foundation +import ZIPFoundation -/// Getting a project out of the guest. +/// Getting a project out of the app. /// -/// Projects live inside the guest filesystem's `fakefs`, and that filesystem is -/// deliberately kept *out* of iCloud backup: it is several gigabytes of toolchain -/// wrapped around a few kilobytes of source, and nothing in the app can mark one -/// without the other (see Docs/DESIGN.md, "What is backed up"). That makes an -/// export the way work leaves the device, so it is a first-class action rather -/// than something to be assembled from `tar` in the Terminal. +/// Projects are directories in the app's container, so an export is a ZIP of one +/// directory — no guest, no `tar` in a shell. The archive lands in +/// `/exports`, which the Files app shows and the share sheet can hand on. /// -/// The archive is built *inside* the guest — that is where the files are — and -/// copied out into the app's own `Documents/exports`, where the Files app shows it -/// and the share sheet can hand it on. +/// The walk skips build products: `.xforge-build` and `.build` are output, and an +/// export that carries a stale binary is worse than one that does not. enum ProjectExporter { - /// Where exported archives land: `/exports`, which is user data and - /// is backed up (unlike the guest filesystem it came from). Defined by + /// Where exported archives land: `/exports`. Defined by /// `XForgeEnvironment`, which owns the storage rules. static var exportsDirectory: URL { XForgeEnvironment.exportsDirectory } - /// Archive `project` inside the guest and copy the archive out. - /// - /// Returns the host URL of the archive, ready to share. - /// - /// `@MainActor` because the VM it is handed is (`LinuxVM` is main-actor isolated - /// and not `Sendable`, so a nonisolated function taking one is a Swift 6 error). - /// Only the unit-test build reports it: the Release build compiles the module - /// whole and does not type-check every body. - @MainActor - static func export(_ project: Project, via vm: LinuxVM) async throws -> URL { - guard project.hasSafeRootPath else { throw ProjectValidationError.unsafePath } + /// Archive `project` and return the host URL of the `.zip`. + static func export(_ project: Project) throws -> URL { let name = try Project.validatedName(project.name) - try await vm.boot() - - let guestArchive = "/tmp/\(name).tar.gz" - let parent = (project.rootPath as NSString).deletingLastPathComponent - // `-C parent name` keeps the archive to one top-level directory, so - // unpacking it cannot scatter files over wherever it is opened. - let status = try await vm.run( - "rm -f \(GuestShell.quote(guestArchive)) && " - + "tar -czf \(GuestShell.quote(guestArchive)) " - + "-C \(GuestShell.quote(parent)) \(GuestShell.quote(name))", - environment: nil - ) { _ in } - guard status == 0 else { throw ExportError.archiveFailed } + let root = project.rootURL + guard FileManager.default.fileExists(atPath: root.path) else { + throw ExportError.projectMissing(root) + } let directory = exportsDirectory try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - let destination = directory.appendingPathComponent("\(name).tar.gz") - try await vm.copyOut(guestPath: guestArchive, to: destination) + let destination = directory.appendingPathComponent("\(name).zip") + try? FileManager.default.removeItem(at: destination) + + let staging = FileManager.default.temporaryDirectory + .appendingPathComponent("xforge-export-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: staging) } - // The guest copy is a second copy of the project; the host has it now. - _ = try? await vm.run("rm -f \(GuestShell.quote(guestArchive))", environment: nil) { _ in } + let copy = staging.appendingPathComponent(name, isDirectory: true) + try FileManager.default.createDirectory(at: copy, withIntermediateDirectories: true) + try copyContents(of: root, to: copy) + + // `shouldKeepParent` on the *staging* directory keeps one top-level folder, + // so unpacking the archive cannot scatter files wherever it is opened. + try FileManager.default.zipItem( + at: copy, + to: destination, + shouldKeepParent: true + ) XForgeLog.note("export: \(name) archived to \(destination.lastPathComponent)") return destination } + /// Copy a project directory, skipping build output. + private static func copyContents(of source: URL, to destination: URL) throws { + let fileManager = FileManager.default + let excluded: Set = [".xforge-build", ".build", ".swiftpm", ".git"] + + let entries = try fileManager.contentsOfDirectory( + at: source, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + ) + for entry in entries { + let name = entry.lastPathComponent + guard !excluded.contains(name) else { continue } + let values = try entry.resourceValues(forKeys: [.isDirectoryKey]) + let target = destination.appendingPathComponent(name) + if values.isDirectory == true { + try fileManager.createDirectory(at: target, withIntermediateDirectories: true) + try copyContents(of: entry, to: target) + } else { + try fileManager.copyItem(at: entry, to: target) + } + } + } + enum ExportError: LocalizedError { - case archiveFailed + case projectMissing(URL) var errorDescription: String? { switch self { - case .archiveFailed: - return "The project could not be archived inside the embedded Linux. " - + "The Terminal tab has the guest's own output." + case .projectMissing(let url): + return "The project directory is missing: \(url.path)" } } } diff --git a/App/Services/ProjectFiles.swift b/App/Services/ProjectFiles.swift new file mode 100644 index 0000000..91930b4 --- /dev/null +++ b/App/Services/ProjectFiles.swift @@ -0,0 +1,152 @@ +import Foundation + +/// Reading and writing a project's files. +/// +/// This replaces `GuestProjectFiles`, which listed and copied files through the +/// embedded Linux. A project is now an ordinary directory in the app's container, +/// so the whole thing is `FileManager` — but the *relative-path* discipline stays: +/// every path is resolved against the project root and rejected if it escapes, +/// because these paths come from editable text in the UI. +@MainActor +enum ProjectFiles { + struct File: Identifiable, Hashable { + let id: String + let name: String + let relativePath: String + var contents: String + } + + /// Files that are never project content: build products and editor state. + private static let excludedDirectories: Set = [".xforge-build", ".git", ".build", ".swiftpm"] + + static func load(project: Project, sourcesOnly: Bool) async throws -> [File] { + let root = sourcesOnly + ? project.rootURL.appendingPathComponent("Sources", isDirectory: true) + : project.rootURL + return loadFiles(under: root, project: project) + } + + static func load(relativePath: String, project: Project) async throws -> File { + let url = try resolve(relativePath: relativePath, in: project) + return try read(url, relativePath: relativePath) + } + + static func save(_ contents: String, file: File, project: Project) async throws { + let url = try resolve(relativePath: file.relativePath, in: project) + try contents.write(to: url, atomically: true, encoding: .utf8) + } + + /// Write a file that may not exist yet, creating intermediate directories. + static func write(_ contents: String, relativePath: String, project: Project) throws { + let url = try resolve(relativePath: relativePath, in: project) + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try contents.write(to: url, atomically: true, encoding: .utf8) + } + + static func createDirectory(relativePath: String, project: Project) throws { + let url = try resolve(relativePath: relativePath, in: project) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + } + + static func delete(relativePath: String, project: Project) throws { + let url = try resolve(relativePath: relativePath, in: project) + try FileManager.default.removeItem(at: url) + } + + // MARK: - Internals + + /// Synchronous on purpose. + /// + /// `FileManager.DirectoryEnumerator` cannot be iterated from an asynchronous + /// context — Swift 6 refuses the `for … in` outright ("makeIterator is + /// unavailable from asynchronous contexts") — so the walk lives here and the + /// async entry points simply call it. + private static func loadFiles(under root: URL, project: Project) -> [File] { + let fileManager = FileManager.default + guard fileManager.fileExists(atPath: root.path) else { return [] } + + guard let enumerator = fileManager.enumerator( + at: root, + includingPropertiesForKeys: [.isRegularFileKey, .isDirectoryKey], + options: [.skipsHiddenFiles, .skipsPackageDescendants] + ) else { return [] } + + var files: [File] = [] + for case let url as URL in enumerator { + let values = try? url.resourceValues(forKeys: [.isRegularFileKey, .isDirectoryKey]) + if values?.isDirectory == true { + if excludedDirectories.contains(url.lastPathComponent) { + enumerator.skipDescendants() + } + continue + } + guard values?.isRegularFile == true else { continue } + guard let relative = projectRelativePath(of: url, in: project) else { continue } + // Text only: the editor cannot show a binary, and a project full of + // build products should not be read into memory to find that out. + guard let contents = try? String(contentsOf: url, encoding: .utf8) else { continue } + files.append(File( + id: relative, + name: url.lastPathComponent, + relativePath: relative, + contents: contents + )) + } + return files.sorted { $0.relativePath < $1.relativePath } + } + + private static func read(_ url: URL, relativePath: String) throws -> File { + let contents = try String(contentsOf: url, encoding: .utf8) + return File( + id: relativePath, + name: (relativePath as NSString).lastPathComponent, + relativePath: relativePath, + contents: contents + ) + } + + /// A project-relative path for `url`, or nil when it is outside the project. + private static func projectRelativePath(of url: URL, in project: Project) -> String? { + let root = project.rootURL.standardizedFileURL.path + let path = url.standardizedFileURL.path + guard path.hasPrefix(root + "/") else { return nil } + return String(path.dropFirst(root.count + 1)) + } + + /// Resolve a user-supplied relative path inside the project. + /// + /// `..` is rejected rather than merely normalized: an editor that can write + /// outside its own project is a sandbox escape with a nice UI, and the + /// standardized-path check catches what the textual one cannot. + private static func resolve(relativePath: String, in project: Project) throws -> URL { + guard isSafe(relativePath: relativePath) else { + throw ProjectFileError.unsafeRelativePath + } + let url = project.rootURL.appendingPathComponent(relativePath) + guard projectRelativePath(of: url, in: project) != nil else { + throw ProjectFileError.unsafeRelativePath + } + return url + } + + static func isSafe(relativePath: String) -> Bool { + !relativePath.isEmpty + && !relativePath.hasPrefix("/") + && !relativePath.split(separator: "/").contains("..") + && !relativePath.contains("\n") + } +} + +enum ProjectFileError: LocalizedError { + case unsafeRelativePath + + var errorDescription: String? { + switch self { + case .unsafeRelativePath: + return "That path is outside the project." + } + } +} diff --git a/App/Services/SystemComponents.swift b/App/Services/SystemComponents.swift deleted file mode 100644 index d9b0434..0000000 --- a/App/Services/SystemComponents.swift +++ /dev/null @@ -1,247 +0,0 @@ -import Foundation - -/// The commands that install the embedded system's components, as commands. -/// -/// Everything in the guest is managed *from the terminal*. That is deliberate: -/// the guest is a real Alpine system, and the tools that provision it are -/// ordinary Linux programs. XForge's job is to know the right commands, get the -/// user's files into the guest's own storage, and run them where the output can -/// be watched — not to reimplement them behind a button. -/// -/// These strings are shared by the Terminal screen (its Components menu) and the -/// Toolchain screen, so both offer exactly the same thing. -@MainActor -enum SystemComponents { - /// Where the guest's own copy of the provisioning script lives. - static let guestInstallerScript = "/root/install-toolchain.sh" - - /// Where a user-supplied Xcode.xip is stored inside the guest filesystem. - static let guestXIPDirectory = "/root/xforge/xip" - - /// xtool's Darwin-SDK cache inside the guest. - static let guestSDKCache = "/root/.cache/xforge-sdk" - - enum Component: String, CaseIterable, Identifiable { - case glibc - case xtool - case swift - case darwinSDK - - var id: String { rawValue } - - var title: String { - switch self { - case .glibc: return "glibc compatibility layer" - case .xtool: return "xtool" - case .swift: return "Swift toolchain" - case .darwinSDK: return "Darwin SDK" - } - } - - var detail: String { - switch self { - case .glibc: - return "The glibc runtime xtool and the Swift toolchain link against (Alpine is musl)." - case .xtool: - return "xtool builds iOS apps from Swift packages on Linux." - case .swift: - return "Swift 6 for Linux, installed with swiftly from swift.org." - case .darwinSDK: - return "The arm64-apple-ios Swift SDK, built from your own Xcode.xip with xtool." - } - } - } - - /// The one-line install for a component that is just a step of the bundled - /// provisioning script. - static func scriptCommand(_ component: Component) -> String { - "sh \(guestInstallerScript) \(component.rawValue)" - } - - /// The Swift toolchain, installed by the guest's own provisioning script. - /// - /// This used to spell out swift.org's documented steps inline — `curl` the - /// swiftly tarball, `tar zxf`, `./swiftly init`, `env.sh`, `hash -r` — which - /// had two problems that only showed up on a device: - /// - /// - it was *unconditional*, so on a root that already carried the toolchain - /// (every published root does now: `EmbeddedLinux/build-rootfs.sh` with - /// `XFORGE_PROVISION=all`) it downloaded 28 MB and ran `swiftly init` over a - /// working installation; - /// - `swiftly init` launches a child process, and this engine does not let it: - /// it fails with "Failed to launch the new process. Underlying error: - /// Invalid argument" before downloading anything. - /// - /// The script's steps are the same work with the guards that belong on them — - /// `swiftly` is a no-op when `/usr/local/bin/swiftly` is already there, and - /// `swift` installs a toolchain only when the toolchains directory is empty — - /// so a provisioned guest answers instantly and an empty one still gets a - /// toolchain. It is also the same code the rootfs is built with, which is the - /// point: one implementation of provisioning, not two that drift. - static var swiftInstallCommand: String { - """ - sh \(guestInstallerScript) glibc && \ - sh \(guestInstallerScript) swiftly && \ - sh \(guestInstallerScript) swift && \ - swift --version - """ - } - - /// xtool installs with its own step (it needs the glibc layer, which the same - /// script installs), then verifies itself. - static var xtoolInstallCommand: String { - """ - sh \(guestInstallerScript) glibc && \ - sh \(guestInstallerScript) xtool && \ - sh \(guestInstallerScript) verify - """ - } - - /// The command that turns an Xcode.xip the user picked into an installed - /// Darwin Swift SDK. - /// - /// The `.xip` is copied into the guest's *own* storage first (`/root/xforge`), - /// and xtool is pointed at that path: xtool does the extraction and the SDK - /// post-processing itself, so nothing has to be unpacked on the host. - /// - /// The removal first is not tidiness. SwiftPM refuses to install a bundle - /// whose artifact ID is already present (`swiftSDKArtifactAlreadyInstalled`, - /// which tells the user to remove one of them) — and the bundled rootfs ships - /// the darwin artifact, so without this the whole point of the import, using - /// *your* Xcode, was unreachable: the install failed with nothing in the UI to - /// explain it. `sdk-remove` deletes SwiftPM's store entry, which is what it - /// consults before installing, and it is the guest's own script so there is - /// one implementation of this rather than one here and one in the rootfs - /// build. - static func darwinSDKInstallCommand(guestXIPPath: String) -> String { - """ - sh \(guestInstallerScript) sdk-remove && \ - xtool sdk install \(GuestShell.quote(guestXIPPath)) && \ - swift sdk list - """ - } - - /// The prebuilt-bundle alternative, for when the user has no Xcode.xip to - /// hand (it downloads XForge's own darwin.artifactbundle inside the guest). - /// - /// The same removal as the `.xip` path, for the same reason: SwiftPM will not - /// install a bundle whose artifact ID is already there, and the bundled rootfs - /// ships one — so "install the prebuilt SDK" over a provisioned guest failed - /// exactly like the import did. - static func darwinSDKDownloadCommand(from url: URL) -> String { - let archive = guestSDKCache + "/darwin-sdk.zip" - let bundle = guestSDKCache + "/darwin.artifactbundle" - return """ - set -eu - sh \(guestInstallerScript) sdk-remove - mkdir -p \(GuestShell.quote(guestSDKCache)) - curl -fL --retry 3 \(GuestShell.quote(url.absoluteString)) -o \(GuestShell.quote(archive)) - rm -rf \(GuestShell.quote(bundle)) - unzip -q \(GuestShell.quote(archive)) -d \(GuestShell.quote(guestSDKCache)) - test -f \(GuestShell.quote(bundle))/info.json - swift sdk install \(GuestShell.quote(bundle)) - rm -f \(GuestShell.quote(archive)) - swift sdk list - """ - } - - // MARK: - Host → guest files - - /// Copy the bundled provisioning script into the guest, where the terminal's - /// commands expect it. - static func ensureInstallerScript(in vm: LinuxVM) async throws { - guard let script = Bundle.main.url(forResource: "install-toolchain", - withExtension: "sh") else { - throw SystemComponentsError.installerScriptMissing - } - try await vm.copyIn(hostURL: script, to: guestInstallerScript) - } - - /// Copy the bundled zsign installer into the guest. - static func ensureZsignInstaller(in vm: LinuxVM) async throws { - guard let script = Bundle.main.url(forResource: "install-zsign", - withExtension: "sh") else { - throw SystemComponentsError.zsignInstallerMissing - } - try await vm.run("mkdir -p /root/xforge", environment: nil) { _ in } - try await vm.copyIn(hostURL: script, to: "/root/xforge/install-zsign.sh") - guard let patch = Bundle.main.url(forResource: "patch-zsign-password-file", - withExtension: "py") else { - throw SystemComponentsError.zsignInstallerMissing - } - try await vm.copyIn(hostURL: patch, to: "/root/xforge/patch-zsign-password-file.py") - let status = try await vm.run( - "chmod 700 /root/xforge/install-zsign.sh && chmod 600 /root/xforge/patch-zsign-password-file.py", - environment: nil - ) { _ in } - guard status == 0 else { throw SystemComponentsError.zsignInstallerMissing } - } - - /// Copy the chosen `.xip` into the guest's own filesystem and return the path the guest should install from. - static func stageXIP(_ source: URL, in vm: LinuxVM, report: @escaping (String) -> Void) async throws -> String { - let scoped = source.startAccessingSecurityScopedResource() - defer { if scoped { source.stopAccessingSecurityScopedResource() } } - - let size = (try? FileManager.default.attributesOfItem(atPath: source.path))?[.size] as? Int64 ?? 0 - if size > 0 { - // The xip is copied *and* xtool expands it inside the guest, so the - // guest needs room for both plus the installed SDK. - let needed = size * 2 + 1_000_000_000 - let free = XForgeEnvironment.availableBytes - if free > 0, free < needed { - throw SystemComponentsError.notEnoughSpace(needed: needed, free: free) - } - } - - let guestPath = "\(guestXIPDirectory)/\(safeName(source.lastPathComponent))" - report("creating \(guestXIPDirectory) in the guest filesystem") - _ = try await vm.run("mkdir -p \(GuestShell.quote(guestXIPDirectory))", environment: nil) { _ in } - - report("copying \(source.lastPathComponent) into the guest (\(byteString(size)))") - XForgeLog.note("components: copying \(source.lastPathComponent) to \(guestPath)") - try await vm.copyIn(hostURL: source, to: guestPath) - - let guestSize = try await vm.run( - "wc -c < \(GuestShell.quote(guestPath))", - environment: nil - ) { _ in } - guard guestSize == 0 else { - throw SystemComponentsError.copyFailed(guestPath) - } - return guestPath - } - - private static func safeName(_ name: String) -> String { - let allowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: ".-_")) - let cleaned = name.unicodeScalars.map { allowed.contains($0) ? Character($0) : "-" } - let result = String(cleaned) - return result.isEmpty ? "Xcode.xip" : result - } - - private static func byteString(_ size: Int64) -> String { - size > 0 ? ByteCountFormatter.string(fromByteCount: size, countStyle: .file) : "unknown size" - } -} - -enum SystemComponentsError: LocalizedError { - case installerScriptMissing - case zsignInstallerMissing - case notEnoughSpace(needed: Int64, free: Int64) - case copyFailed(String) - - var errorDescription: String? { - switch self { - case .installerScriptMissing: - return "install-toolchain.sh is not bundled in the app." - case .zsignInstallerMissing: - return "The guest zsign installer is not bundled in the app." - case .notEnoughSpace(let needed, let free): - let formatter = ByteCountFormatter() - return "Not enough free space: this needs about " - + "\(formatter.string(fromByteCount: needed)) and " - + "\(formatter.string(fromByteCount: free)) is free." - case .copyFailed(let path): - return "The file did not arrive in the guest at \(path)." - } - } -} diff --git a/App/Services/ToolchainManager.swift b/App/Services/ToolchainManager.swift deleted file mode 100644 index 9344609..0000000 --- a/App/Services/ToolchainManager.swift +++ /dev/null @@ -1,261 +0,0 @@ -import Foundation -import Combine - -/// Manages the on-device build toolchain. -/// -/// The pieces live in two places, and this type is careful to check the right one: -/// -/// - **Alpine rootfs** — bundled inside the app and imported into the engine's fakefs -/// on first boot. Nothing is downloaded; "installing" it just performs the import. -/// - **Swift / xtool / darwin SDK** — inside the guest Linux, reached over the VM -/// bridge. Installing them means running commands *in the guest*; this type -/// checks what is there and stages the files those commands need, while the -/// commands themselves run in the Terminal (`SystemComponents` holds them). -@MainActor -final class ToolchainManager: ObservableObject { - enum Component: String, CaseIterable, Identifiable { - case rootfs = "Alpine Rootfs" - case swift = "Swift Toolchain" - case xtool = "xtool" - case sdk = "Darwin SDK" - - var id: String { rawValue } - - var icon: String { - switch self { - case .rootfs: return "shippingbox" - case .swift: return "swift" - case .xtool: return "hammer" - case .sdk: return "externaldrive.connected.to.line.below" - } - } - - var blurb: String { - switch self { - case .rootfs: return "Alpine aarch64 userspace · bundled in the app" - case .swift: return "Swift toolchain, installed with swiftly" - case .xtool: return "xtool aarch64 (glibc, via the compatibility layer)" - case .sdk: return "arm64-apple-ios Swift SDK" - } - } - - /// Every component is stored in the embedded Alpine guest. The rootfs is - /// imported into ish-arm64 fakefs; Swift, xtool, and SDKs install below it. - var livesInGuest: Bool { true } - } - - @Published private(set) var installed: Set = [] - @Published private(set) var isInstalling: Component? - @Published private(set) var guestChecked = false - @Published private(set) var activity: String? - @Published var message: String? - - /// 0…1 while a host-side preparation (staging a file into the guest) runs. - @Published private(set) var progress: Double = 0 - @Published private(set) var progressLabel: String? - - private let vm: LinuxVM - - init(vm: LinuxVM? = nil) { - self.vm = vm ?? XForgeEnvironment.makeVM() - } - - func isInstalled(_ component: Component) -> Bool { installed.contains(component) } - - /// Exposed for the import UI: ish-arm64 cannot replace a mounted rootfs. - var isGuestBooted: Bool { vm.isBooted } - - // MARK: - Status - - /// Refresh component status. - /// - /// Host-side facts are always checked. Guest-side components are only probed - /// when `probeGuest` is true (or the guest is already running), because probing - /// boots the embedded Linux — which imports the rootfs the first time. - func refresh(probeGuest: Bool = false) async { - var found: Set = [] - - // RootTabView starts this at launch, but Settings can be presented before - // that task has completed. Await the same idempotent preparation here so - // the row reflects the actual bundled rootfs rather than a stale snapshot. - if !RootfsInstaller.isInstalled(in: XForgeEnvironment.rootsDirectory) { - activity = "Preparing the bundled Alpine rootfs…" - await vm.prepareRootfs() - activity = nil - } - if RootfsInstaller.isInstalled(in: XForgeEnvironment.rootsDirectory) { - found.insert(.rootfs) - } - - if probeGuest || vm.isBooted { - activity = vm.isBooted ? "Checking the embedded Linux…" : "Starting the embedded Linux…" - defer { activity = nil } - do { - XForgeLog.prepare() - XForgeLog.note("refresh: probing the guest") - try await vm.boot() - for component in [Component.swift, .xtool, .sdk] { - if await guestHas(component) { found.insert(component) } - } - guestChecked = true - } catch { - XForgeLog.note("refresh: guest probe failed: \(error.localizedDescription)") - message = "Could not check the embedded Linux: \(error.localizedDescription)" - // Keep whatever we already knew rather than reporting everything missing. - found.formUnion(guestKnown) - } - } else { - found.formUnion(guestKnown) - } - - guestKnown = found.subtracting([.rootfs]) - installed = found - } - - /// Last known guest-side state, so an un-probed refresh doesn't lose it. - private var guestKnown: Set = [] - - /// True when the guest reports the component present (exit status 0). - private func guestHas(_ component: Component) async -> Bool { - let command: String - switch component { - case .rootfs: - return RootfsInstaller.isInstalled(in: XForgeEnvironment.rootsDirectory) - case .swift: - command = "command -v swift >/dev/null 2>&1" - case .xtool: - command = "command -v xtool >/dev/null 2>&1" - case .sdk: - // `swift sdk list` is a glibc binary being asked a question, so its - // stderr is piped rather than sent to /dev/null: this engine kills a - // forked guest program whose output points at /dev/null, and the - // answer here decides whether the SDK row reads "installed". - command = "command -v swift >/dev/null 2>&1 && swift sdk list 2>&1 | grep -qi darwin" - } - do { - let status = try await vm.run(command, environment: nil) { _ in } - return status == 0 - } catch { - return false - } - } - - // MARK: - Getting files into the guest - - /// Copy the user's `Xcode.xip` into the guest's own storage and return the - /// command that installs it there. - /// - /// The `.xip` is never unpacked on the host. The guest keeps the file in its - /// own filesystem (`/root/xforge/xip`), and `xtool sdk install ` — run - /// in the Terminal, where the user can watch it — does the extraction and the - /// SDK post-processing inside Alpine. The returned string is the command to - /// hand to `TerminalSession`. - func installSDKFromXcode(xip: URL) async throws -> String { - isInstalling = .sdk - defer { isInstalling = nil } - XForgeLog.prepare() - let keepAwake = InstallAssertion.begin(reason: "stage Xcode.xip for the guest") - defer { keepAwake.end() } - beginProgress(.sdk) - defer { endProgress() } - - try await vm.boot() - activity = "Copying \(xip.lastPathComponent) into the guest…" - let guestPath = try await SystemComponents.stageXIP(xip, in: vm) { [weak self] line in - self?.advanceProgress(0.5, line) - XForgeLog.note("sdk: \(line)") - } - activity = nil - advanceProgress(1.0, "Ready to install inside the guest") - message = "\(xip.lastPathComponent) is in the guest at \(guestPath). " - + "Its install is running in the Terminal." - return SystemComponents.darwinSDKInstallCommand(guestXIPPath: guestPath) - } - - /// Replace the bundled rootfs with an Alpine `.tar.gz` chosen from Files. - /// ish-arm64 cannot switch roots after it has booted, so this is intentionally - /// limited to a fresh app launch. - func importRootfs(from archive: URL) async { - guard !vm.isBooted else { - message = "Quit and reopen XForge before replacing the Alpine rootfs. The running Linux guest cannot switch roots." - return - } - - isInstalling = .rootfs - defer { isInstalling = nil } - beginProgress(.rootfs) - let scoped = archive.startAccessingSecurityScopedResource() - defer { if scoped { archive.stopAccessingSecurityScopedResource() } } - do { - activity = "Importing \(archive.lastPathComponent)…" - advanceProgress(0.2, "Validating and importing the selected Alpine rootfs") - _ = try RootfsInstaller.install( - archive: archive, - into: XForgeEnvironment.rootsDirectory - ) - advanceProgress(1.0, "Done") - message = "Alpine rootfs imported from \(archive.lastPathComponent)." - installed.insert(.rootfs) - } catch { - message = error.localizedDescription - } - activity = nil - endProgress() - } - - // MARK: - Progress - - private func beginProgress(_ component: Component) { - progress = 0 - progressLabel = "Starting \(component.rawValue)" - } - - private func advanceProgress(_ fraction: Double, _ label: String) { - progress = max(0, min(1, fraction)) - progressLabel = label - XForgeLog.note("progress: \(Int(progress * 100))% — \(label)") - } - - private func endProgress() { - progress = 0 - progressLabel = nil - } - - // MARK: - Reset - - /// Remove the imported rootfs, the staged SDK and downloaded archives. - func reset() async { - isInstalling = nil - activity = nil - endProgress() - let fm = FileManager.default - try? fm.removeItem(at: XForgeEnvironment.rootsDirectory) - try? fm.removeItem(at: XForgeEnvironment.downloadsDirectory) - installed = [] - guestKnown = [] - guestChecked = false - message = "Removed the imported rootfs, the staged SDK and downloads. " - + "Quit and reopen XForge to boot a fresh guest." - } -} - -enum ToolchainError: LocalizedError { - case sdkLayoutUnexpected - case sdkInstallFailed(Int32, String) - case notEnoughSpace(needed: Int64, free: Int64) - - var errorDescription: String? { - switch self { - case .sdkLayoutUnexpected: - return "The downloaded Darwin SDK archive did not contain darwin.artifactbundle/info.json." - case .sdkInstallFailed(let status, let output): - return "`swift sdk install` failed inside the guest (exit \(status)). " - + (output.isEmpty ? "See Settings → Diagnostics → Engine log." : String(output.suffix(400))) - case .notEnoughSpace(let needed, let free): - let formatter = ByteCountFormatter() - return "Not enough free space: this needs about " - + "\(formatter.string(fromByteCount: needed)) and " - + "\(formatter.string(fromByteCount: free)) is free." - } - } -} diff --git a/App/Services/XForgeLog.swift b/App/Services/XForgeLog.swift index cc2299d..41fd4a3 100644 --- a/App/Services/XForgeLog.swift +++ b/App/Services/XForgeLog.swift @@ -1,21 +1,13 @@ import Foundation -/// On-device diagnostics for the embedded Linux. +/// On-device diagnostics. /// -/// Two writers, one file: +/// Plain file writes, no `os_log`: the point is a file the user can share from the +/// app, and `os_log` messages are only readable with a console attached. /// -/// - **The engine.** Its `printk` — every kernel message, including the one -/// `die()` prints immediately before it calls `abort()` — goes to file -/// descriptor 555 (`kernel/log.c`'s dprintf handler). Nothing in an iOS app -/// opens that descriptor, so without `prepare()` a guest crash kills the app -/// and leaves no trace anywhere. -/// - **This app.** `note(_:)` writes a breadcrumb through the same bridge call, -/// so the last line of the file says how far an install got before the app -/// disappeared — which is the difference between "it crashes" and a bug -/// report. -/// -/// Plain file writes, no `os_log`: the point is a file the user can share from -/// the app, and `os_log` messages are only readable with a console attached. +/// The engine that used to share this file — the embedded Linux, whose `printk` +/// went to file descriptor 555 through the ish bridge — is gone, so this is now +/// only XForge's own breadcrumb trail. enum XForgeLog { /// `/logs` — the same container the Files app shows. static var directory: URL { @@ -23,17 +15,24 @@ enum XForgeLog { .appendingPathComponent("logs", isDirectory: true) } - static var url: URL { directory.appendingPathComponent("engine.log") } + static var url: URL { directory.appendingPathComponent("xforge.log") } /// Rotate past this size so the file stays shareable. static let maxBytes = 512 * 1024 - /// Point the bridge (and therefore the engine) at the log file. - /// Safe to call repeatedly; also safe to call after the guest has booted. + /// Create the log directory and rotate an oversized log. + /// + /// Kept as a separate step from `note(_:)` so the directory exists before the + /// first breadcrumb, and so a caller that wants the log path can ask for it + /// without writing anything. @discardableResult static func prepare() -> Bool { let fm = FileManager.default - try? fm.createDirectory(at: directory, withIntermediateDirectories: true) + do { + try fm.createDirectory(at: directory, withIntermediateDirectories: true) + } catch { + return false + } if let attributes = try? fm.attributesOfItem(atPath: url.path), let size = attributes[.size] as? Int, @@ -45,13 +44,37 @@ enum XForgeLog { try? String(tail).write(to: url, atomically: true, encoding: .utf8) } } + return true + } - return url.path.withCString { xf_ish_set_log_file($0) } == 0 + /// ISO-8601 UTC, built per call. + /// + /// `ISO8601DateFormatter` is not `Sendable`, so it cannot live in a static + /// that `note(_:)` — a nonisolated function — reaches: Swift 6 rejects that + /// outright. Constructing one per line costs a few microseconds on a call that + /// always writes a file anyway. + private static func timestamp() -> String { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime] + return formatter.string(from: Date()) } - /// Append one breadcrumb line, timestamped by the bridge. + /// Append one timestamped breadcrumb line. + /// + /// Best effort by design: a diagnostic that can fail a build is worse than a + /// diagnostic that is missing a line. static func note(_ line: String) { - _ = line.withCString { xf_ish_log($0) } + let entry = "\(Self.timestamp()) \(line)\n" + let fm = FileManager.default + if !fm.fileExists(atPath: url.path) { + _ = prepare() + try? entry.write(to: url, atomically: true, encoding: .utf8) + return + } + guard let handle = try? FileHandle(forWritingTo: url) else { return } + defer { try? handle.close() } + _ = try? handle.seekToEnd() + try? handle.write(contentsOf: Data(entry.utf8)) #if DEBUG print("[XForge] \(line)") #endif diff --git a/App/Views/Build/SigningView.swift b/App/Views/Build/SigningView.swift index bc7b3ca..242fb91 100644 --- a/App/Views/Build/SigningView.swift +++ b/App/Views/Build/SigningView.swift @@ -1,17 +1,21 @@ import SwiftUI import UniformTypeIdentifiers -/// Full IPA configure, sign and install surface. +/// Sign a built IPA with an Apple Developer identity. /// -/// Signing is intentionally performed by zsign inside the Alpine guest. zsign is -/// a cross-platform C++ signer, not an iOS framework; running it in the guest -/// keeps the private key on the device, gives the user live command output, and -/// avoids pretending XKit's currently-unwired Apple-ID service can sign an IPA. +/// The signing itself is XKit — xtool's own signer — running in this process, so +/// the private key is read through the Security framework, used, and never written +/// anywhere. That used to be a `zsign` binary inside the embedded Linux with a +/// password file on disk; both are gone. +/// +/// The Apple ID half of xtool (obtaining a certificate, 2FA, provisioning) is a +/// separate piece of work and is not faked here: this signs with an identity the +/// user already has. struct SigningView: View { @ObservedObject var signing: XKitSigningService @ObservedObject var device: XKitDeviceService - @StateObject private var job = IPAConfigureSignService() + @StateObject private var job = IPASigningJob() @State private var importer: ImportTarget? @State private var showPassword = false @@ -35,24 +39,25 @@ struct SigningView: View { fileRow(title: "IPA to sign", url: job.inputIPA, icon: "app.badge") { importer = .ipa } - if job.inputIPA != nil { - Text("The original IPA is copied into the guest and never modified in place.") + if let ipa = job.inputIPA { + Text("The original is never modified: the signed IPA is written next to it in Documents.") .font(.caption).foregroundStyle(.secondary) + Text(ipa.lastPathComponent).font(.caption).foregroundStyle(.secondary) } } Section("Signing files") { - fileRow(title: "Provisioning profile", url: job.provisioningProfile, - icon: "checkmark.seal") { importer = .mobileprovision } fileRow(title: "Certificate / private key (.p12)", url: job.p12, icon: "key.fill") { importer = .p12 } SecureField("PKCS#12 password", text: $job.password) .textInputAutocapitalization(.never) .autocorrectionDisabled() + fileRow(title: "Provisioning profile (optional)", url: job.provisioningProfile, + icon: "checkmark.seal") { importer = .mobileprovision } fileRow(title: "Entitlements (optional)", url: job.entitlements, icon: "list.bullet.rectangle") { importer = .entitlements } - Text("Keep the .p12 password private. It is held in memory for this signing job and is never saved to the project or build history.") + Text("The password and key stay in memory for this signing job and are never written to disk.") .font(.caption).foregroundStyle(.secondary) } @@ -64,7 +69,7 @@ struct SigningView: View { TextField("Version (optional)", text: $job.version) .textInputAutocapitalization(.never) .autocorrectionDisabled() - Text("Leave a field blank to preserve the IPA's existing value. If you change the bundle identifier, the provisioning profile must authorize it.") + Text("Leave a field blank to keep the IPA's existing value. If you change the bundle identifier, the provisioning profile must authorize it.") .font(.caption).foregroundStyle(.secondary) } @@ -74,8 +79,7 @@ struct SigningView: View { } label: { HStack { if job.isWorking { ProgressView().controlSize(.small) } - Label(job.isWorking ? "Signing in Alpine…" : "Configure & Sign IPA", - systemImage: "signature") + Label(job.isWorking ? "Signing…" : "Sign IPA", systemImage: "signature") Spacer() } } @@ -90,73 +94,65 @@ struct SigningView: View { } header: { Text("Action") } footer: { - Text("zsign is installed into the embedded Linux guest on first use. It supports .p12/.pfx keys, .mobileprovision profiles, entitlements, bundle ID/name/version changes, and IPA output.") + Text(job.status) } - Section("Status") { - Text(job.status).font(.system(.footnote, design: .monospaced)) - if let error = job.error { - Text(error).foregroundStyle(.red) + if let error = job.error { + Section { + Label(error, systemImage: "exclamationmark.triangle.fill") + .font(.footnote).foregroundStyle(.red) } } - } - .navigationTitle("Configure & Sign") - .fileImporter(isPresented: Binding( - get: { importer != nil }, - set: { if !$0 { importer = nil } } - ), allowedContentTypes: importer?.types ?? [.data], allowsMultipleSelection: false) { result in - guard case .success(let urls) = result, let url = urls.first, - let target = importer else { return } - do { - let localURL = try persistImportedFile(url) - switch target { - case .ipa: job.inputIPA = localURL - case .p12: job.p12 = localURL - case .mobileprovision: job.provisioningProfile = localURL - case .entitlements: job.entitlements = localURL + + Section { + NavigationLink { + InstallView(device: device) + } label: { + Label("Install on this device", systemImage: "iphone.and.arrow.forward") } - } catch { - job.error = "Import failed: \(error.localizedDescription)" + } footer: { + Text("Installing a signed build needs a device link that an app cannot open by itself; " + + "the Install screen explains the ways that do work on device.") } - importer = nil } - } - - /// File-provider URLs returned by UIDocumentPicker are security-scoped and can - /// become unreadable as soon as the picker callback returns. Copy the selected - /// item into XForge's sandbox while access is active so signing can reliably - /// stage it into Alpine later. - private func persistImportedFile(_ source: URL) throws -> URL { - let scoped = source.startAccessingSecurityScopedResource() - defer { if scoped { source.stopAccessingSecurityScopedResource() } } - - let directory = XForgeEnvironment.documentDirectory - .appendingPathComponent("Signing Imports", isDirectory: true) - try FileManager.default.createDirectory(at: directory, - withIntermediateDirectories: true) - let destination = directory.appendingPathComponent(source.lastPathComponent) - if FileManager.default.fileExists(atPath: destination.path) { - try FileManager.default.removeItem(at: destination) + .navigationTitle("Signing") + .fileImporter( + isPresented: Binding( + get: { importer != nil }, + set: { if !$0 { importer = nil } } + ), + allowedContentTypes: importer?.types ?? [.data], + allowsMultipleSelection: false + ) { result in + guard case .success(let urls) = result, let url = urls.first else { return } + switch importer { + case .ipa: job.inputIPA = url + case .p12: job.p12 = url + case .mobileprovision: job.provisioningProfile = url + case .entitlements: job.entitlements = url + case nil: break + } + importer = nil } - try FileManager.default.copyItem(at: source, to: destination) - return destination } @ViewBuilder - private func fileRow(title: String, url: URL?, icon: String, - choose: @escaping () -> Void) -> some View { - Button(action: choose) { - HStack(spacing: 12) { - Image(systemName: icon).foregroundStyle(.tint).frame(width: 22) - VStack(alignment: .leading, spacing: 2) { - Text(title).foregroundStyle(.primary) - Text(url?.lastPathComponent ?? "Choose from Files…") - .font(.caption).foregroundStyle(url == nil ? .secondary : .primary) - .lineLimit(1) - } + private func fileRow( + title: String, + url: URL?, + icon: String, + action: @escaping () -> Void + ) -> some View { + Button(action: action) { + HStack { + Label(title, systemImage: icon) Spacer() - Image(systemName: "chevron.right").font(.caption.bold()).foregroundStyle(.tertiary) + Text(url?.lastPathComponent ?? "Choose…") + .font(.caption) + .foregroundStyle(url == nil ? .secondary : .primary) + .lineLimit(1) } } + .foregroundStyle(.primary) } } diff --git a/App/Views/Projects/DependenciesView.swift b/App/Views/Projects/DependenciesView.swift deleted file mode 100644 index 70079eb..0000000 --- a/App/Views/Projects/DependenciesView.swift +++ /dev/null @@ -1,154 +0,0 @@ -import SwiftUI - -/// Add SwiftPM dependencies directly to the project's real Package.swift. -struct DependenciesView: View { - let project: Project - - @State private var enabled: Set = [] - @State private var customPackages: [CatalogPackage] = [] - @State private var customURL = "" - @State private var customVersion = "1.0.0" - @State private var applying = false - @State private var message: String? - - struct CatalogPackage: Identifiable, Hashable { - let id: String - let name: String - let from: String - } - - private static let catalog: [CatalogPackage] = [ - CatalogPackage(id: "https://github.com/xtool-org/xtool", name: "XKit (xtool)", from: "1.17.0"), - CatalogPackage(id: "https://github.com/apple/swift-nio", name: "SwiftNIO", from: "2.77.0"), - CatalogPackage(id: "https://github.com/apple/swift-crypto", name: "swift-crypto", from: "4.5.0"), - CatalogPackage(id: "https://github.com/apple/swift-argument-parser", name: "swift-argument-parser", from: "1.5.0"), - CatalogPackage(id: "https://github.com/pointfreeco/swift-dependencies", name: "swift-dependencies", from: "1.6.2"), - CatalogPackage(id: "https://github.com/Alamofire/Alamofire", name: "Alamofire", from: "5.10.0"), - CatalogPackage(id: "https://github.com/SDWebImage/SDWebImage", name: "SDWebImage", from: "5.20.0"), - CatalogPackage(id: "https://github.com/SwiftyJSON/SwiftyJSON", name: "SwiftyJSON", from: "5.0.2"), - CatalogPackage(id: "https://github.com/krzyzanowskim/OpenSSL", name: "OpenSSL", from: "3.3.2000"), - ] - - var body: some View { - Form { - Section("Catalog") { - ForEach(Self.catalog + customPackages) { package in - Button { - toggle(package) - } label: { - HStack { - VStack(alignment: .leading) { - Text(package.name) - Text(package.id) - .font(.caption) - .foregroundStyle(.secondary) - } - Spacer() - Image(systemName: enabled.contains(package.id) - ? "checkmark.circle.fill" : "circle") - .foregroundStyle(enabled.contains(package.id) - ? .green : .secondary) - } - } - .foregroundStyle(.primary) - } - } - - Section("Custom Package") { - TextField("https://github.com/user/repo", text: $customURL) - .keyboardType(.URL) - .autocorrectionDisabled() - .textInputAutocapitalization(.never) - TextField("Version (from)", text: $customVersion) - .keyboardType(.numbersAndPunctuation) - Button("Add Custom") { addCustom() } - .disabled(customURL.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) - } - - Section { - Button { - Task { await apply() } - } label: { - Label(applying ? "Applying…" : "Apply to Package.swift", - systemImage: "shippingbox.and.arrow.backward") - } - .disabled(enabled.isEmpty || applying) - - if let message { - Text(message) - .font(.footnote) - .foregroundStyle(message.hasPrefix("Added") ? .green : .red) - } - } - } - .navigationTitle("Dependencies") - } - - private var selectedPackages: [CatalogPackage] { - (Self.catalog + customPackages).filter { enabled.contains($0.id) } - } - - private func toggle(_ package: CatalogPackage) { - if enabled.contains(package.id) { - enabled.remove(package.id) - } else { - enabled.insert(package.id) - } - } - - private func addCustom() { - let url = customURL.trimmingCharacters(in: .whitespacesAndNewlines) - let version = customVersion.trimmingCharacters(in: .whitespacesAndNewlines) - guard URL(string: url)?.scheme == "https", !version.isEmpty else { - message = "Enter an HTTPS repository URL and a version." - return - } - - let name = URL(string: url)?.deletingPathExtension().lastPathComponent ?? "Package" - let package = CatalogPackage(id: url, name: name, from: version) - if !customPackages.contains(where: { $0.id == url }) { - customPackages.append(package) - } - enabled.insert(url) - customURL = "" - } - - private func apply() async { - guard project.hasSafeRootPath else { - message = ProjectValidationError.unsafePath.localizedDescription - return - } - - applying = true - message = nil - defer { applying = false } - - do { - let vm = XForgeEnvironment.makeVM() - for package in selectedPackages { - let command = "cd \(GuestShell.quote(project.rootPath)) && " - + "swift package add-dependency \(GuestShell.quote(package.id)) " - + "--from \(GuestShell.quote(package.from))" - let status = try await vm.run(command, environment: nil) { _ in } - guard status == 0 else { - throw DependencyError.addFailed(package.name, status) - } - } - message = "Added \(selectedPackages.count) package(s) to Package.swift." - enabled.removeAll() - } catch { - message = error.localizedDescription - } - } -} - -enum DependencyError: LocalizedError { - case addFailed(String, Int32) - - var errorDescription: String? { - switch self { - case .addFailed(let name, let status): - return "Could not add \(name) (exit \(status))." - } - } -} diff --git a/App/Views/Projects/FileBrowserView.swift b/App/Views/Projects/FileBrowserView.swift index 16bf149..9b01aea 100644 --- a/App/Views/Projects/FileBrowserView.swift +++ b/App/Views/Projects/FileBrowserView.swift @@ -3,7 +3,7 @@ import SwiftUI /// Browse every readable file in the project's real guest filesystem. struct FileBrowserView: View { let project: Project - @State private var files: [GuestProjectFiles.File] = [] + @State private var files: [ProjectFiles.File] = [] @State private var error: String? @State private var loading = true @@ -34,7 +34,7 @@ struct FileBrowserView: View { } } .navigationTitle("Files") - .navigationDestination(for: GuestProjectFiles.File.self) { file in + .navigationDestination(for: ProjectFiles.File.self) { file in SourceEditorView(file: file, project: project) { updated in if let index = files.firstIndex(where: { $0.id == file.id }) { files[index].contents = updated @@ -48,13 +48,13 @@ struct FileBrowserView: View { loading = true defer { loading = false } do { - files = try await GuestProjectFiles.load(project: project, sourcesOnly: false) + files = try await ProjectFiles.load(project: project, sourcesOnly: false) } catch { self.error = error.localizedDescription } } - private func symbol(for file: GuestProjectFiles.File) -> String { + private func symbol(for file: ProjectFiles.File) -> String { file.name.hasSuffix(".swift") ? "swift" : "doc" } } diff --git a/App/Views/Projects/ImportProjectView.swift b/App/Views/Projects/ImportProjectView.swift index 63c962d..4c918ef 100644 --- a/App/Views/Projects/ImportProjectView.swift +++ b/App/Views/Projects/ImportProjectView.swift @@ -1,10 +1,17 @@ import SwiftUI +import UniformTypeIdentifiers -/// Sheet for importing an existing SwiftPM package from a git URL. +/// Import an existing package from the Files app. +/// +/// This used to `git clone` into the embedded Linux. There is no `git` in an iOS +/// process and no process to run it in, so the import is now what an app can +/// actually do: copy a folder the user picked into the projects directory. A +/// repository can still be fetched with an app that does have git — Working Copy, +/// or Files' own "Download" on a zip — and imported from there. struct ImportProjectView: View { @EnvironmentObject private var preferences: AppPreferences @Environment(\.dismiss) private var dismiss - @State private var gitURL = "" + @State private var picked: URL? @State private var name = "" @State private var orgId = "" @State private var isImporting = false @@ -13,32 +20,49 @@ struct ImportProjectView: View { let onImported: (Project) -> Void private var derivedName: String { - name.isEmpty ? (gitURL.split(separator: "/").last?.split(separator: ".").first.map(String.init) ?? "Imported") : name + if !name.isEmpty { return name } + guard let picked else { return "" } + return picked.lastPathComponent } var body: some View { NavigationStack { Form { - Section("Repository") { - TextField("https://github.com/you/repo.git", text: $gitURL) - .keyboardType(.URL).autocorrectionDisabled().textInputAutocapitalization(.never) + Section("Folder") { + Button { + showingPicker = true + } label: { + Label(picked?.lastPathComponent ?? "Choose a project folder…", + systemImage: "folder") + } + if let picked { + Text(picked.path) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(2) + } } + Section("Project") { TextField("Name", text: $name) TextField("Organization Identifier", text: $orgId) .keyboardType(.alphabet).autocorrectionDisabled() } + Section { if isImporting { - HStack { ProgressView(); Text("Cloning into embedded Linux…") } + HStack { ProgressView(); Text("Copying into XForge…") } } if let error { Label(error, systemImage: "exclamationmark.triangle.fill") .font(.footnote).foregroundStyle(.red) } + } footer: { + Text("The folder must contain a Package.swift with a Sources/ directory. " + + "It is copied, not moved: the original stays where it is.") } } - .navigationTitle("Import from Git") + .navigationTitle("Import a project") .onAppear { if orgId.isEmpty { orgId = preferences.defaultOrgId } } @@ -48,39 +72,59 @@ struct ImportProjectView: View { } ToolbarItem(placement: .confirmationAction) { Button("Import") { startImport() } - .disabled(gitURL.isEmpty || isImporting) + .disabled(picked == nil || isImporting) } } } .interactiveDismissDisabled(isImporting) + .fileImporter( + isPresented: $showingPicker, + allowedContentTypes: [.folder], + allowsMultipleSelection: false + ) { result in + guard case .success(let urls) = result, let url = urls.first else { return } + picked = url + if name.isEmpty { name = url.lastPathComponent } + } } + @State private var showingPicker = false + private func startImport() { + guard let source = picked else { return } isImporting = true error = nil - let url = gitURL.trimmingCharacters(in: .whitespacesAndNewlines) + let projectName = derivedName let org = orgId + Task { defer { isImporting = false } do { - let projectName = try Project.validatedName(derivedName) - // Clone into the guest; without this the project path does not - // exist and every later build step fails. - let vm = XForgeEnvironment.makeVM() - if !vm.isBooted { try await vm.boot() } - let path = Project.path(forValidatedName: projectName) - let status = try await vm.run( - "mkdir -p \(GuestShell.quote(Project.projectsRoot)) && " - + "rm -rf \(GuestShell.quote(path)) && " - + "git clone --depth 1 -- \(GuestShell.quote(url)) \(GuestShell.quote(path))", - environment: nil - ) { _ in } - guard status == 0 else { - throw ImportError.cloneFailed(status, url) + let validated = try Project.validatedName(projectName) + let accessed = source.startAccessingSecurityScopedResource() + defer { if accessed { source.stopAccessingSecurityScopedResource() } } + + let manifest = source.appendingPathComponent("Package.swift") + guard FileManager.default.fileExists(atPath: manifest.path) else { + throw ImportError.notAPackage(source.lastPathComponent) } - onImported(Project(name: projectName, - organizationIdentifier: org, - rootPath: path)) + + let project = Project( + name: validated, + organizationIdentifier: org, + rootPath: Project.path(forValidatedName: validated) + ) + let destination = project.rootURL + try FileManager.default.createDirectory( + at: destination.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + if FileManager.default.fileExists(atPath: destination.path) { + throw ImportError.alreadyExists(validated) + } + try FileManager.default.copyItem(at: source, to: destination) + + onImported(project) dismiss() } catch { self.error = error.localizedDescription @@ -90,13 +134,15 @@ struct ImportProjectView: View { } enum ImportError: LocalizedError { - case cloneFailed(Int32, String) + case notAPackage(String) + case alreadyExists(String) var errorDescription: String? { switch self { - case .cloneFailed(let status, let url): - return "git clone failed (exit \(status)) for \(url). Check the URL, and that the " - + "embedded Linux has network access." + case .notAPackage(let name): + return "\(name) has no Package.swift, so it is not a Swift package." + case .alreadyExists(let name): + return "A project named \(name) already exists. Rename it, or remove the existing one first." } } } diff --git a/App/Views/Projects/ManifestEditorView.swift b/App/Views/Projects/ManifestEditorView.swift index 7dffaf5..a88ad0f 100644 --- a/App/Views/Projects/ManifestEditorView.swift +++ b/App/Views/Projects/ManifestEditorView.swift @@ -1,6 +1,6 @@ import SwiftUI -/// Edit the real Package.swift stored in the embedded Alpine guest. +/// Edit the project's Package.swift. struct ManifestEditorView: View { let project: Project @Environment(\.dismiss) private var dismiss @@ -78,7 +78,7 @@ struct ManifestEditorView: View { loading = true defer { loading = false } do { - text = try await GuestProjectFiles.load( + text = try await ProjectFiles.load( relativePath: "Package.swift", project: project ).contents @@ -98,13 +98,13 @@ struct ManifestEditorView: View { error = nil defer { saving = false } do { - let file = GuestProjectFiles.File( + let file = ProjectFiles.File( id: "Package.swift", name: "Package.swift", relativePath: "Package.swift", contents: text ) - try await GuestProjectFiles.save(text, file: file, project: project) + try await ProjectFiles.save(text, file: file, project: project) onSave(text) dismiss() } catch { diff --git a/App/Views/Projects/ProjectDetailView.swift b/App/Views/Projects/ProjectDetailView.swift index bc3814a..b4edfc8 100644 --- a/App/Views/Projects/ProjectDetailView.swift +++ b/App/Views/Projects/ProjectDetailView.swift @@ -39,7 +39,6 @@ private struct OverviewSection: View { let project: Project @State private var showingFiles = false @State private var showingInfo = false - @State private var showingDeps = false @State private var exporting = false @State private var exportedArchive: URL? @State private var exportError: String? @@ -74,16 +73,12 @@ private struct OverviewSection: View { store.update(p) } } - .sheet(isPresented: $showingDeps) { - NavigationStack { DependenciesView(project: project) } - } } private var toolsRow: some View { HStack(spacing: 12) { toolButton("Files", icon: "folder", action: { showingFiles = true }) toolButton("App Info", icon: "doc.badge.gearshape", action: { showingInfo = true }) - toolButton("Dependencies", icon: "shippingbox", action: { showingDeps = true }) Spacer() } .padding(.horizontal).padding(.bottom, 10) @@ -187,7 +182,7 @@ private struct OverviewSection: View { .buttonStyle(.bordered) } else { Button { - Task { await exportProject() } + exportProject() } label: { Label(exporting ? "Exporting…" : "Export project", systemImage: "archivebox") @@ -207,14 +202,12 @@ private struct OverviewSection: View { } } - /// Archive the project inside the guest and stage it for sharing. - private func exportProject() async { + /// Archive the project's directory and stage it for sharing. + private func exportProject() { exporting = true defer { exporting = false } do { - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - exportedArchive = try await ProjectExporter.export(project, via: vm) + exportedArchive = try ProjectExporter.export(project) } catch { exportError = error.localizedDescription } diff --git a/App/Views/Projects/SourceBrowserView.swift b/App/Views/Projects/SourceBrowserView.swift index 4188a34..bdd2a08 100644 --- a/App/Views/Projects/SourceBrowserView.swift +++ b/App/Views/Projects/SourceBrowserView.swift @@ -1,132 +1,12 @@ import SwiftUI -@MainActor -enum GuestProjectFiles { - struct File: Identifiable, Hashable { - let id: String - let name: String - let relativePath: String - var contents: String - } - - static func load(project: Project, sourcesOnly: Bool) async throws -> [File] { - guard project.hasSafeRootPath else { throw ProjectValidationError.unsafePath } - - let vm = XForgeEnvironment.makeVM() - let root = sourcesOnly ? "\(project.rootPath)/Sources" : project.rootPath - let output = GuestFileOutputCollector() - let status = try await vm.run( - "find \(GuestShell.quote(root)) -type f -print 2>/dev/null | sort", - environment: nil - ) { output.append($0) } - guard status == 0 else { throw GuestProjectFileError.listFailed(status) } - - let paths = output.value - .split(separator: "\n") - .map(String.init) - .filter { $0.hasPrefix(project.rootPath + "/") } - - var files: [File] = [] - for path in paths { - let relativePath = String(path.dropFirst(project.rootPath.count + 1)) - guard !relativePath.contains("\n") else { continue } - let temporaryURL = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-source-\(UUID().uuidString)") - defer { try? FileManager.default.removeItem(at: temporaryURL) } - try await vm.copyOut(guestPath: path, to: temporaryURL) - guard let contents = try? String(contentsOf: temporaryURL, encoding: .utf8) else { - continue - } - files.append(File( - id: relativePath, - name: (relativePath as NSString).lastPathComponent, - relativePath: relativePath, - contents: contents - )) - } - return files - } - - static func load(relativePath: String, project: Project) async throws -> File { - guard project.hasSafeRootPath else { throw ProjectValidationError.unsafePath } - guard isSafe(relativePath: relativePath) else { - throw GuestProjectFileError.unsafeRelativePath - } - - let temporaryURL = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-source-\(UUID().uuidString)") - defer { try? FileManager.default.removeItem(at: temporaryURL) } - try await XForgeEnvironment.makeVM().copyOut( - guestPath: "\(project.rootPath)/\(relativePath)", - to: temporaryURL - ) - let contents = try String(contentsOf: temporaryURL, encoding: .utf8) - return File( - id: relativePath, - name: (relativePath as NSString).lastPathComponent, - relativePath: relativePath, - contents: contents - ) - } - - static func save(_ contents: String, file: File, project: Project) async throws { - guard project.hasSafeRootPath else { throw ProjectValidationError.unsafePath } - guard isSafe(relativePath: file.relativePath) else { - throw GuestProjectFileError.unsafeRelativePath - } - - let temporaryURL = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-source-\(UUID().uuidString)") - defer { try? FileManager.default.removeItem(at: temporaryURL) } - try contents.write(to: temporaryURL, atomically: true, encoding: .utf8) - try await XForgeEnvironment.makeVM().copyIn( - hostURL: temporaryURL, - to: "\(project.rootPath)/\(file.relativePath)" - ) - } - - private static func isSafe(relativePath: String) -> Bool { - !relativePath.isEmpty - && !relativePath.hasPrefix("/") - && !relativePath.split(separator: "/").contains("..") - && !relativePath.contains("\n") - } -} - -private final class GuestFileOutputCollector: @unchecked Sendable { - private let lock = NSLock() - private var storage = "" - - func append(_ text: String) { - lock.lock() - storage += text - lock.unlock() - } - - var value: String { - lock.lock() - defer { lock.unlock() } - return storage - } -} - -enum GuestProjectFileError: LocalizedError { - case listFailed(Int32) - case unsafeRelativePath - - var errorDescription: String? { - switch self { - case .listFailed(let status): - return "Could not list project files (exit \(status))." - case .unsafeRelativePath: - return "The selected file is outside the project." - } - } -} - +/// Browse and edit the project's Swift sources. +/// +/// The files are ordinary files in the app container, read through +/// `ProjectFiles`: the guest filesystem they used to live in is gone. struct SourceBrowserView: View { let project: Project - @State private var files: [GuestProjectFiles.File] = [] + @State private var files: [ProjectFiles.File] = [] @State private var error: String? @State private var loading = true @@ -157,7 +37,7 @@ struct SourceBrowserView: View { } } .navigationTitle("Sources") - .navigationDestination(for: GuestProjectFiles.File.self) { file in + .navigationDestination(for: ProjectFiles.File.self) { file in SourceEditorView(file: file, project: project) { updated in if let index = files.firstIndex(where: { $0.id == file.id }) { files[index].contents = updated @@ -171,7 +51,7 @@ struct SourceBrowserView: View { loading = true defer { loading = false } do { - files = try await GuestProjectFiles.load(project: project, sourcesOnly: true) + files = try await ProjectFiles.load(project: project, sourcesOnly: true) } catch { self.error = error.localizedDescription } @@ -184,12 +64,12 @@ struct SourceEditorView: View { @State private var saving = false @State private var error: String? - let file: GuestProjectFiles.File + let file: ProjectFiles.File let project: Project let onSave: (String) -> Void init( - file: GuestProjectFiles.File, + file: ProjectFiles.File, project: Project, onSave: @escaping (String) -> Void ) { @@ -230,7 +110,7 @@ struct SourceEditorView: View { error = nil defer { saving = false } do { - try await GuestProjectFiles.save(text, file: file, project: project) + try await ProjectFiles.save(text, file: file, project: project) onSave(text) dismiss() } catch { diff --git a/App/Views/RootTabView.swift b/App/Views/RootTabView.swift index 10a4543..bdd7a2a 100644 --- a/App/Views/RootTabView.swift +++ b/App/Views/RootTabView.swift @@ -5,16 +5,17 @@ struct RootTabView: View { @StateObject private var signing = XKitSigningService() @StateObject private var device = XKitDeviceService() @StateObject private var preferences = AppPreferences() - /// One terminal for the whole app: the Toolchain screen and the terminal - /// itself hand commands to the same session, so what a component install is - /// doing is visible where it runs. - @StateObject private var terminal = TerminalSession() @State private var selection: AppTab = .projects + /// Three tabs, not four. + /// + /// The Terminal tab existed to type into the embedded Linux. With the guest + /// gone there is nothing behind it — the toolchain runs in this process and + /// reports into the Build screen's console — so the tab is gone rather than + /// left opening an empty screen. enum AppTab: Hashable { case projects case build - case terminal case settings } @@ -31,26 +32,16 @@ struct RootTabView: View { .tabItem { Label("Build", systemImage: "hammer") } .tag(AppTab.build) - TerminalTab() - .environmentObject(terminal) - .tabItem { Label("Terminal", systemImage: "terminal.fill") } - .tag(AppTab.terminal) - SettingsTab(preferences: preferences) - .environmentObject(terminal) .tabItem { Label("Settings", systemImage: "gearshape") } .tag(AppTab.settings) } .task { // Create the app's own directories and keep the regenerable ones out - // of iCloud backup — before anything writes into them. + // of iCloud backup — before anything writes into them. Projects are + // user data and are backed up. XForgeEnvironment.prepareStorage() - // Install the bundled rootfs early, but do not make app presentation - // depend on booting the guest. The guest command bridge can take time - // to initialize on a physical device; Terminal and Toolchain perform - // the same idempotent boot and surface its result when actually used. - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() + XForgeLog.prepare() } } } diff --git a/App/Views/Settings/DownloadsView.swift b/App/Views/Settings/DownloadsView.swift deleted file mode 100644 index 7a56d72..0000000 --- a/App/Views/Settings/DownloadsView.swift +++ /dev/null @@ -1,138 +0,0 @@ -import SwiftUI - -/// Guest-backed provisioning controls. -/// -/// The app never downloads build tooling into a host-side cache: every component -/// is installed by a command inside the embedded Alpine filesystem, and those -/// commands run in the Terminal tab so their output is visible. This screen -/// shows what is present and hands the command over. -struct DownloadsView: View { - @EnvironmentObject private var terminal: TerminalSession - @StateObject private var toolchain = ToolchainManager() - - var body: some View { - List { - Section { - LinuxProvisioningRow( - component: .swift, - installed: toolchain.isInstalled(.swift), - isBusy: toolchain.isInstalling != nil - ) { - install(SystemComponents.swiftInstallCommand, "Swift toolchain") - } - - LinuxProvisioningRow( - component: .xtool, - installed: toolchain.isInstalled(.xtool), - isBusy: toolchain.isInstalling != nil - ) { - install(SystemComponents.xtoolInstallCommand, "xtool") - } - - LinuxProvisioningRow( - component: .sdk, - installed: toolchain.isInstalled(.sdk), - isBusy: toolchain.isInstalling != nil - ) { - installPrebuiltSDK() - } - } header: { - Text("Alpine Toolchain") - } footer: { - Text("Each install is an ordinary Linux command run inside the " - + "embedded Alpine filesystem, shown in the Terminal tab.") - } - - if let message = toolchain.message { - Section { - Text(message) - .font(.footnote) - .foregroundStyle(.secondary) - } - } - - Section { - Text("The Alpine terminal is the Terminal tab, where these installs " - + "run: use it to inspect them with swift --version, " - + "xtool --version and swift sdk list.") - .font(.footnote) - .foregroundStyle(.secondary) - } - } - .navigationTitle("Linux Toolchain") - .task { - await toolchain.refresh(probeGuest: true) - } - } - - /// Boot the guest, make sure the provisioning script is in it, and hand the - /// command to the shared terminal. - private func install(_ command: String, _ what: String) { - Task { - do { - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(command, label: "Settings") - toolchain.message = "\(what): running in the Terminal tab." - } catch { - toolchain.message = error.localizedDescription - } - } - } - - private func installPrebuiltSDK() { - Task { - do { - let url = try await XForgeReleases.darwinSDKURL() - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - // The command runs the guest's script first, to remove the SDK the - // rootfs already carries: same preparation as the other installs. - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(SystemComponents.darwinSDKDownloadCommand(from: url), - label: "Settings") - toolchain.message = "Darwin SDK: downloading and installing in the Terminal tab." - } catch { - toolchain.message = error.localizedDescription - } - } - } -} - -private struct LinuxProvisioningRow: View { - let component: ToolchainManager.Component - let installed: Bool - let isBusy: Bool - let install: () -> Void - - var body: some View { - HStack(alignment: .top) { - Image(systemName: component.icon) - .foregroundStyle(installed ? .green : .secondary) - .frame(width: 24) - - VStack(alignment: .leading, spacing: 2) { - Text(component.rawValue) - Text(component.blurb) - .font(.caption) - .foregroundStyle(.secondary) - if installed { - Label("Installed in Alpine", systemImage: "checkmark") - .font(.caption) - .foregroundStyle(.green) - } - } - - Spacer() - - if !installed { - Button("Install", action: install) - .buttonStyle(.bordered) - .disabled(isBusy) - } - } - } -} diff --git a/App/Views/Settings/EngineLogView.swift b/App/Views/Settings/EngineLogView.swift deleted file mode 100644 index a9a7c1c..0000000 --- a/App/Views/Settings/EngineLogView.swift +++ /dev/null @@ -1,81 +0,0 @@ -import SwiftUI -import UIKit - -/// The engine log: the engine's own kernel messages plus XForge's install -/// breadcrumbs, in one file, oldest first. -/// -/// This is the thing to share when the app dies without saying why. The engine's -/// `die()` prints its reason and then calls `abort()` — killing the whole app — -/// and that message is the last line of this file. The breadcrumbs above it say -/// how far the install got first. -@MainActor -struct EngineLogView: View { - @State private var text = "" - @State private var bytes: Int64 = 0 - @State private var copied = false - - private var url: URL { XForgeLog.url } - - var body: some View { - Group { - if text.isEmpty { - ContentUnavailableViewCompat( - title: "Nothing logged yet", - systemImage: "doc.text", - message: "Boot the embedded Linux, or install a toolchain component, " - + "and the engine's messages land here." - ) - } else { - ScrollView { - Text(text) - .font(.system(.caption, design: .monospaced)) - .textSelection(.enabled) - .frame(maxWidth: .infinity, alignment: .leading) - .padding(12) - } - } - } - .navigationTitle("Engine log") - .navigationBarTitleDisplayMode(.inline) - .toolbar { - ToolbarItem(placement: .primaryAction) { - ShareLink(item: url) { - Label("Share", systemImage: "square.and.arrow.up") - } - .disabled(text.isEmpty) - } - ToolbarItem(placement: .secondaryAction) { - Button { - UIPasteboard.general.string = text - copied = true - } label: { - Label(copied ? "Copied" : "Copy", systemImage: "doc.on.doc") - } - .disabled(text.isEmpty) - } - ToolbarItem(placement: .secondaryAction) { - Button(role: .destructive) { - XForgeLog.clear() - reload() - } label: { - Label("Clear", systemImage: "trash") - } - .disabled(text.isEmpty) - } - } - .safeAreaInset(edge: .bottom) { - if !text.isEmpty { - Text("\(ByteCountFormatter.string(fromByteCount: bytes, countStyle: .file)) · \(url.lastPathComponent)") - .font(.caption2) - .foregroundStyle(.secondary) - .padding(.vertical, 4) - } - } - .task { reload() } - } - - private func reload() { - text = XForgeLog.text() - bytes = XForgeLog.byteCount() - } -} diff --git a/App/Views/Settings/SettingsTab.swift b/App/Views/Settings/SettingsTab.swift index f2a2387..f5ee26b 100644 --- a/App/Views/Settings/SettingsTab.swift +++ b/App/Views/Settings/SettingsTab.swift @@ -1,6 +1,6 @@ import SwiftUI -/// Tab 5 — settings, storage, diagnostics, about. +/// Tab 3 — toolchain, app files, build defaults, diagnostics, about. struct SettingsTab: View { @ObservedObject var preferences: AppPreferences diff --git a/App/Views/Settings/SettingsView.swift b/App/Views/Settings/SettingsView.swift index 4fda823..a7a3971 100644 --- a/App/Views/Settings/SettingsView.swift +++ b/App/Views/Settings/SettingsView.swift @@ -1,25 +1,19 @@ import SwiftUI +/// Settings: the toolchain, the app's files, build defaults and diagnostics. +/// +/// Everything here used to be a *guest* concern — install Alpine, install Swift in +/// it, watch the engine log, size the fakefs. The toolchain is now linked into the +/// app and the SDK is a folder, so what is left is what the user can actually act on. @MainActor struct SettingsView: View { @ObservedObject var preferences: AppPreferences - @EnvironmentObject private var terminal: TerminalSession - @StateObject private var toolchain = ToolchainManager() - /// On-disk sizes, filled off the main actor — never computed during `body`. - @State private var sizes: [ToolchainManager.Component: Int64] = [:] - @State private var artifactsSize: Int64 = 0 var body: some View { List { Section { NavigationLink { ToolchainView() } label: { - Label("Toolchain & Alpine", systemImage: "wrench.and.screwdriver") - } - NavigationLink { NativeToolchainView() } label: { - Label("Native Toolchain", systemImage: "cpu") - } - NavigationLink { DownloadsView() } label: { - Label("Downloads", systemImage: "arrow.down.circle") + Label("Toolchain & SDK", systemImage: "wrench.and.screwdriver") } NavigationLink { SandboxBrowserView(root: XForgeEnvironment.documentDirectory) @@ -29,79 +23,20 @@ struct SettingsView: View { NavigationLink { HistoryView() } label: { Label("Build History", systemImage: "clock.arrow.circlepath") } + NavigationLink { XForgeLogView() } label: { + Label("Log", systemImage: "doc.text.magnifyingglass") + } } header: { Text("System & Files") } footer: { - Text("Alpine, the compiler toolchain, SDKs, downloads and app files are managed here.") + Text("Projects, downloads, built IPAs and the log all live in the app's " + + "container, visible in the Files app.") } preferencesSection - storageSection diagnosticsSection aboutSection } - .task { - await toolchain.refresh() - await loadSizes() - } - } - - /// Compute the storage sizes once, off the main actor. - private func loadSizes() async { - let components = ToolchainManager.Component.allCases - let paths: [(ToolchainManager.Component, URL)] = components.map { - ($0, storageURL(for: $0)) - } - let staging = XForgeEnvironment.stagingDirectory - let result = await Task.detached(priority: .utility) { () -> ([ToolchainManager.Component: Int64], Int64) in - var map: [ToolchainManager.Component: Int64] = [:] - for (component, url) in paths { - map[component] = DirectorySize.bytes(at: url) - } - return (map, DirectorySize.bytes(at: staging)) - }.value - sizes = result.0 - artifactsSize = result.1 - } - - /// Where a component's bytes live on the host. - /// - /// The guest's `/` is the installed fakefs's `data/` directory — the files are - /// there, their modes are in `meta.db` — so this maps a guest path onto the - /// app container. Naming the *component's* directory rather than the whole - /// fakefs is not a detail: every component used to report the size of the - /// entire root (the same five gigabytes four times over), and measuring it four - /// times meant four walks of a thirty-thousand-file tree on every settings - /// visit. - private func storageURL(for component: ToolchainManager.Component) -> URL { - let guest = RootfsInstaller.installedRoot(in: XForgeEnvironment.rootsDirectory) - .appendingPathComponent("data", isDirectory: true) - switch component { - case .rootfs: - return XForgeEnvironment.rootsDirectory - case .swift: - return guest.appendingPathComponent("root/.local/share/swiftly", isDirectory: true) - case .xtool: - return guest.appendingPathComponent("opt/xtool", isDirectory: true) - case .sdk: - // Whatever the install recorded, which is the only reliable answer: - // SwiftPM's store has moved between releases. - return recordedSDKPath(under: guest) ?? guest.appendingPathComponent( - "root/.swiftpm/swift-sdks", isDirectory: true) - } - } - - /// The Darwin SDK's installed path, from the record the installer writes next - /// to the manifest (`darwin-sdk-path:`), mapped into the app container. - private func recordedSDKPath(under guest: URL) -> URL? { - let record = guest.appendingPathComponent("usr/local/share/xforge/darwin-sdk.txt") - guard let text = try? String(contentsOf: record, encoding: .utf8) else { return nil } - for line in text.split(separator: "\n") where line.hasPrefix("path:") { - let value = line.dropFirst("path:".count).trimmingCharacters(in: .whitespaces) - guard value.hasPrefix("/") else { return nil } - return guest.appendingPathComponent(String(value.dropFirst()), isDirectory: true) - } - return nil } private var preferencesSection: some View { @@ -119,90 +54,6 @@ struct SettingsView: View { } } - private var storageSection: some View { - Section { - ForEach(ToolchainManager.Component.allCases) { component in - StorageRow( - title: component.rawValue, - detail: storageDetail(for: component), - installed: toolchain.isInstalled(component), - installing: toolchain.isInstalling == component, - progress: toolchain.isInstalling == component ? toolchain.progress : nil, - progressLabel: toolchain.isInstalling == component ? toolchain.progressLabel : nil - ) { - install(component) - } - } - StorageRow(title: "Build artifacts", - detail: ByteCountFormatter.string(fromByteCount: artifactsSize, countStyle: .file), - installed: true) {} - } header: { - Text("Storage & Toolchain") - } footer: { - Text("The Alpine rootfs is bundled and installs offline. Swift, xtool and " - + "the darwin SDK are installed by commands inside the embedded Linux: " - + "Install hands the command to the Terminal tab, where you can watch it.") - } - } - - /// Hand a component's install command to the Terminal, after making sure the - /// guest is running and the provisioning script is in it. - private func install(_ component: ToolchainManager.Component) { - Task { - do { - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - switch component { - case .rootfs: - toolchain.message = "The Alpine rootfs is bundled in the app and " - + "already installed in the guest filesystem." - case .swift: - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(SystemComponents.swiftInstallCommand, label: "Settings") - toolchain.message = "Swift toolchain: installing in the Terminal tab." - case .xtool: - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(SystemComponents.xtoolInstallCommand, label: "Settings") - toolchain.message = "xtool: installing in the Terminal tab." - case .sdk: - let url = try await XForgeReleases.darwinSDKURL() - // The command removes the installed SDK first (SwiftPM will not - // install a second bundle with the same artifact ID), so the - // script it calls has to be in the guest. - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(SystemComponents.darwinSDKDownloadCommand(from: url), - label: "Settings") - toolchain.message = "Darwin SDK: downloading and installing in the Terminal tab." - } - } catch { - toolchain.message = error.localizedDescription - } - } - } - - private func storageDetail(for component: ToolchainManager.Component) -> String { - let size = ByteCountFormatter.string(fromByteCount: sizes[component] ?? 0, countStyle: .file) - switch (component, toolchain.isInstalled(component)) { - case (.rootfs, true): - return "Installed · \(size)" - case (.rootfs, false): - return "Bundled in the app · installs offline" - case (.sdk, true): - return "Installed in the guest · \(size)" - case (.swift, true), (.xtool, true): - return "Installed in the embedded Linux · \(size)" - case (.sdk, false): - return "Downloads on demand (457 MB)" - case (_, true): - return "Installed in the embedded Linux" - case (_, false): - return component.livesInGuest && !toolchain.guestChecked - ? "Not checked — tap Check on the Toolchain screen" - : "Not installed" - } - } - private var diagnosticsSection: some View { Section { LabeledContent("Device", value: "\(SystemInfo.deviceName) (\(SystemInfo.deviceModel))") @@ -213,17 +64,9 @@ struct SettingsView: View { LabeledContent("CPU", value: "\(SystemInfo.processorCount) cores") LabeledContent("Storage free", value: SystemInfo.storage.free + " of " + SystemInfo.storage.total) LabeledContent("Low Power Mode", value: SystemInfo.isLowPowerMode ? "On" : "Off") - NavigationLink { - EngineLogView() - } label: { - Label("Engine log", systemImage: "doc.text.magnifyingglass") - } + LabeledContent("Compiler", value: NativeToolchainCapabilities.current.backendDescription) } header: { Text("Diagnostics") - } footer: { - Text("The engine log holds the embedded Linux's own messages and " - + "XForge's install breadcrumbs — share it when something dies " - + "without an explanation.") } } @@ -235,61 +78,61 @@ struct SettingsView: View { } header: { Text("About") } footer: { - Text("XForge builds iOS apps on-device with xtool — a cross-platform Xcode replacement.") + Text("XForge builds iOS apps on-device with xtool's libraries — no Mac, no Linux guest.") } } } -struct StorageRow: View { - let title: String - let detail: String - var installed = true - var installing = false - /// 0…1 while this component installs, and what that fraction is measuring. - var progress: Double? - var progressLabel: String? - let onInstall: () -> Void +/// XForge's own log, with the share sheet. +/// +/// It used to be the *engine's* log — the embedded Linux's kernel messages, which +/// was where a crash left its last trace. The compiler runs in this process now, so +/// what is left is XForge's own breadcrumbs plus whatever the build console showed. +struct XForgeLogView: View { + @State private var text = "" + @State private var size: Int64 = 0 var body: some View { - VStack(alignment: .leading, spacing: 6) { - HStack { - Image(systemName: installed ? "checkmark.circle.fill" : "circle.dashed") - .foregroundStyle(installed ? .green : .secondary) - VStack(alignment: .leading, spacing: 1) { - Text(title) - Text(detail).font(.caption).foregroundStyle(.secondary) - } - Spacer() - if !installed { - Button { - onInstall() - } label: { - if installing { - ProgressView().controlSize(.small) - } else { - Label("Install", systemImage: "arrow.down.circle") - } - } - .buttonStyle(.bordered) - .disabled(installing) + Group { + if text.isEmpty { + ContentUnavailableViewCompat( + title: "No Log", + systemImage: "doc.text", + message: "Nothing has been written yet." + ) + } else { + ScrollView { + Text(text) + .font(.system(.caption, design: .monospaced)) + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(8) } } - - // Under the row it belongs to, with what is happening right now: the - // engine returns a guest command's output only when it finishes, so - // this line is the only live feedback there is. - if let progress, let progressLabel { - VStack(alignment: .leading, spacing: 3) { - ProgressView(value: progress).progressViewStyle(.linear) - HStack(spacing: 4) { - Text("\(Int(progress * 100))%") - Text(progressLabel) + } + .navigationTitle("Log") + .toolbar { + ToolbarItem(placement: .primaryAction) { + Menu { + ShareLink(item: XForgeLog.url) { + Label("Share log", systemImage: "square.and.arrow.up") } - .font(.caption2) - .foregroundStyle(.secondary) + Button(role: .destructive) { + XForgeLog.clear() + load() + } label: { + Label("Clear", systemImage: "trash") + } + } label: { + Label("Actions", systemImage: "ellipsis.circle") } - .padding(.leading, 28) } } + .task { load() } + } + + private func load() { + text = XForgeLog.text() + size = XForgeLog.byteCount() } } diff --git a/App/Views/Terminal/GuestFileBrowserView.swift b/App/Views/Terminal/GuestFileBrowserView.swift deleted file mode 100644 index d1d993c..0000000 --- a/App/Views/Terminal/GuestFileBrowserView.swift +++ /dev/null @@ -1,216 +0,0 @@ -import SwiftUI -import UIKit - -private final class GuestListingBox: @unchecked Sendable { - private let lock = NSLock() - private var value = "" - func append(_ chunk: String) { lock.lock(); value += chunk; lock.unlock() } - var text: String { lock.lock(); defer { lock.unlock() }; return value } -} - -/// File explorer for the actual Alpine filesystem. -struct GuestFileBrowserView: View { - @State private var path: String - @State private var entries: [Entry] = [] - @State private var loading = false - @State private var error: String? - @State private var vm: (any LinuxVM)? - @State private var showHidden = false - @State private var pendingAction: FileAction? - @State private var destination = "" - @State private var deleteTarget: Entry? - @Environment(\.dismiss) private var dismiss - - struct Entry: Identifiable, Hashable { - let name: String - let path: String - let isDirectory: Bool - let size: Int64 - var id: String { path } - } - - enum FileAction: Identifiable { - case copy(Entry), move(Entry) - var id: String { - switch self { case .copy(let e): return "copy:" + e.path; case .move(let e): return "move:" + e.path } - } - var title: String { switch self { case .copy: return "Copy"; case .move: return "Move" } } - var entry: Entry { switch self { case .copy(let e), .move(let e): return e } } - } - - init(path: String = "/") { _path = State(initialValue: path) } - - var body: some View { - NavigationStack { - List { - if path != "/" { - Button { navigate(to: parent(path)) } label: { - Label("..", systemImage: "arrow.turn.up.left") - } - } - if loading { ProgressView("Reading \(path)…") } - if let error { - ContentUnavailableViewCompat(title: "Could Not Read Folder", - systemImage: "exclamationmark.triangle", - message: error) - } - ForEach(entries) { entry in row(entry) } - } - .navigationTitle(path) - .navigationBarTitleDisplayMode(.inline) - .refreshable { await load() } - .toolbar { - ToolbarItem(placement: .navigationBarLeading) { - Menu { - Toggle("Show Hidden Files", isOn: $showHidden) - Button("Copy Current Path", systemImage: "doc.on.doc") { - UIPasteboard.general.string = path - } - Button("Refresh", systemImage: "arrow.clockwise") { Task { await load() } } - } label: { Image(systemName: "ellipsis.circle") } - } - ToolbarItem(placement: .confirmationAction) { Button("Done") { dismiss() } } - } - .onChange(of: showHidden) { _ in Task { await load() } } - .task { await load() } - .alert(pendingAction?.title ?? "File Action", - isPresented: Binding(get: { pendingAction != nil }, set: { if !$0 { pendingAction = nil } })) { - TextField("Destination path", text: $destination) - .textInputAutocapitalization(.never) - .autocorrectionDisabled() - Button("Cancel", role: .cancel) { pendingAction = nil } - Button(pendingAction?.title ?? "Apply") { - guard let action = pendingAction else { return } - Task { await perform(action) } - } - } message: { - Text("Enter an absolute Alpine path or a destination directory.") - } - .confirmationDialog("Delete \(deleteTarget?.name ?? "item")?", - isPresented: Binding(get: { deleteTarget != nil }, set: { if !$0 { deleteTarget = nil } }), - titleVisibility: .visible) { - Button("Delete", role: .destructive) { - guard let target = deleteTarget else { return } - Task { await remove(target) } - } - Button("Cancel", role: .cancel) { deleteTarget = nil } - } - } - } - - @ViewBuilder private func row(_ entry: Entry) -> some View { - Button { - if entry.isDirectory { navigate(to: entry.path) } - } label: { - HStack(spacing: 10) { - Image(systemName: entry.isDirectory ? "folder.fill" : icon(entry.name)) - .foregroundStyle(entry.isDirectory ? Color.accentColor : Color.secondary) - VStack(alignment: .leading) { - Text(entry.name).lineLimit(1) - if !entry.isDirectory { - Text(ByteCountFormatter.string(fromByteCount: entry.size, countStyle: .file)) - .font(.caption2).foregroundStyle(.secondary) - } - } - Spacer() - if entry.isDirectory { Image(systemName: "chevron.right").foregroundStyle(.tertiary) } - } - .contentShape(Rectangle()) - } - .buttonStyle(.plain) - .contextMenu { - Button("Copy Path", systemImage: "doc.on.doc") { UIPasteboard.general.string = entry.path } - Button("Copy…", systemImage: "plus.square.on.square") { begin(.copy(entry)) } - Button("Move…", systemImage: "folder") { begin(.move(entry)) } - Divider() - Button(role: .destructive) { deleteTarget = entry } label: { Label("Delete", systemImage: "trash") } - } - .swipeActions(edge: .trailing) { - Button(role: .destructive) { deleteTarget = entry } label: { Label("Delete", systemImage: "trash") } - } - } - - private func begin(_ action: FileAction) { - pendingAction = action - destination = path - } - - private func navigate(to value: String) { - path = value.hasPrefix("/") ? value : "/" + value - Task { await load() } - } - - private func load() async { - loading = true; error = nil - do { - let guest = vm ?? XForgeEnvironment.makeVM() - vm = guest - try await guest.boot() - let box = GuestListingBox() - let hiddenGlob = showHidden ? " \(GuestShell.quote(path))/.[!.]*" : "" - let listing = """ - for item in \(GuestShell.quote(path))/*\(hiddenGlob); do - [ -e "$item" ] || [ -L "$item" ] || continue - if [ -d "$item" ]; then kind=d; else kind=f; fi - size=$(stat -c %s "$item" 2>/dev/null || echo 0) - printf '%s\\t%s\\t%s\\n' "$kind" "$item" "$size" - done | sort -k2 - """ - let status = try await guest.run(listing, environment: nil) { box.append($0) } - guard status == 0 else { throw BrowserError.operationFailed("The guest could not list this folder.") } - entries = box.text.split(separator: "\n").compactMap { line in - let parts = line.split(separator: "\t", maxSplits: 2).map(String.init) - guard parts.count == 3 else { return nil } - let full = parts[1] - return Entry(name: URL(fileURLWithPath: full).lastPathComponent, - path: full, isDirectory: parts[0] == "d", size: Int64(parts[2]) ?? 0) - } - } catch { self.error = error.localizedDescription } - loading = false - } - - private func perform(_ action: FileAction) async { - let target = destination.trimmingCharacters(in: .whitespacesAndNewlines) - guard target.hasPrefix("/") else { - error = "Destination must be an absolute Alpine path."; pendingAction = nil; return - } - do { - let guest = vm ?? XForgeEnvironment.makeVM() - vm = guest - let verb = action.title == "Copy" ? "cp -R" : "mv" - let command = "\(verb) -- \(GuestShell.quote(action.entry.path)) \(GuestShell.quote(target))" - let status = try await guest.run(command, environment: nil) { _ in } - guard status == 0 else { throw BrowserError.operationFailed("\(action.title) failed.") } - pendingAction = nil - await load() - } catch { self.error = error.localizedDescription; pendingAction = nil } - } - - private func remove(_ entry: Entry) async { - do { - let guest = vm ?? XForgeEnvironment.makeVM() - vm = guest - let status = try await guest.run("rm -rf -- \(GuestShell.quote(entry.path))", environment: nil) { _ in } - guard status == 0 else { throw BrowserError.operationFailed("Delete failed.") } - deleteTarget = nil - await load() - } catch { self.error = error.localizedDescription; deleteTarget = nil } - } - - private func parent(_ value: String) -> String { - let p = (value as NSString).deletingLastPathComponent - return p.isEmpty ? "/" : p - } - - private func icon(_ name: String) -> String { - if name.hasSuffix(".ipa") { return "app.badge" } - if name.hasSuffix(".zip") || name.hasSuffix(".tar.gz") { return "archivebox" } - if name.hasSuffix(".dylib") { return "shippingbox" } - return "doc" - } - - enum BrowserError: LocalizedError { - case operationFailed(String) - var errorDescription: String? { if case .operationFailed(let message) = self { return message }; return nil } - } -} diff --git a/App/Views/Terminal/TerminalKeys.swift b/App/Views/Terminal/TerminalKeys.swift deleted file mode 100644 index 9d0d4c7..0000000 --- a/App/Views/Terminal/TerminalKeys.swift +++ /dev/null @@ -1,152 +0,0 @@ -import Foundation -import SwiftUI - -/// One key on the terminal's extra-keys bar. -/// -/// The bar exists to supply the characters and actions a phone keyboard cannot -/// produce, so which keys are useful depends on what the user is doing — someone -/// running `apk` wants different keys from someone editing a script. Each key is -/// therefore a named, individually hideable item rather than a hard-coded row. -enum TerminalKey: String, CaseIterable, Identifiable, Codable { - case interrupt - case escape - case tab - case previous - case next - case dash - case dot - case slash - case colon - case bang - case pipe - case components - case files - case paste - case hideKeyboard - - var id: String { rawValue } - - /// The title shown in the configuration list. - var title: String { - switch self { - case .interrupt: return "Interrupt (Ctrl-C)" - case .escape: return "Escape" - case .tab: return "Tab" - case .previous: return "Previous command" - case .next: return "Next command" - case .dash: return "Minus" - case .dot: return "Period" - case .slash: return "Slash" - case .colon: return "Colon" - case .bang: return "Exclamation mark" - case .pipe: return "Pipe" - case .components: return "Components" - case .files: return "List files" - case .paste: return "Paste" - case .hideKeyboard: return "Hide keyboard" - } - } - - /// What the key shows. Punctuation and the history arrows are their own glyph; - /// the rest are SF Symbols. - var label: String { - switch self { - case .previous: return "↑" - case .next: return "↓" - case .dash: return "-" - case .dot: return "." - case .slash: return "/" - case .colon: return ":" - case .bang: return "!" - case .pipe: return "|" - default: return "" - } - } - - var symbol: String? { - switch self { - case .interrupt: return "control" - case .escape: return "escape" - case .tab: return "arrow.right.to.line.alt" - case .components: return "wrench.and.screwdriver" - case .files: return "folder" - case .paste: return "doc.on.clipboard" - case .hideKeyboard: return "keyboard.chevron.compact.down" - default: return nil - } - } - - /// A key that cannot be turned off. - /// - /// Hide Keyboard is pinned: without it there is no way back to the terminal's - /// screen once the keyboard covers it, so a mis-tap in the configuration list - /// could leave the terminal unusable. Every other key is a preference. - var isPinned: Bool { self == .hideKeyboard } - - /// Grouping for the configuration list, so related keys read together. - static let groups: [(title: String, keys: [TerminalKey])] = [ - ("Control", [.interrupt, .escape, .tab]), - ("History", [.previous, .next]), - ("Punctuation", [.dash, .dot, .slash, .colon, .bang, .pipe]), - ("Actions", [.components, .files, .paste]), - ("Always shown", [.hideKeyboard]), - ] -} - -/// Which keys the terminal shows, persisted between launches. -/// -/// Stored as the set of *hidden* keys rather than the visible ones, so a key added -/// in a future version shows up by default instead of being silently absent from -/// an existing user's bar. -@MainActor -final class TerminalKeyConfiguration: ObservableObject { - @Published private(set) var hidden: Set = [] - @Published private(set) var order: [TerminalKey] = [] - - private static let defaultsKey = "org.xforge.terminal.hiddenKeys" - private static let orderKey = "org.xforge.terminal.keyOrder" - - init() { - let raw = UserDefaults.standard.stringArray(forKey: Self.defaultsKey) ?? [] - hidden = Set(raw.compactMap(TerminalKey.init(rawValue:))) - let saved = (UserDefaults.standard.stringArray(forKey: Self.orderKey) ?? []) - .compactMap(TerminalKey.init(rawValue:)) - let defaults = TerminalKey.groups.flatMap(\.keys) - order = saved + defaults.filter { !saved.contains($0) } - } - - /// Whether a key is shown. Pinned keys ignore the stored preference. - func isVisible(_ key: TerminalKey) -> Bool { - if key.isPinned { return true } - return !hidden.contains(key) - } - - func setVisible(_ visible: Bool, for key: TerminalKey) { - guard !key.isPinned else { return } - if visible { hidden.remove(key) } else { hidden.insert(key) } - save() - } - - func toggle(_ key: TerminalKey) { - setVisible(!isVisible(key), for: key) - } - - /// Restore every key. - func resetToDefaults() { - hidden = [] - order = TerminalKey.groups.flatMap(\.keys) - save() - } - - func move(from source: IndexSet, to destination: Int) { - order.move(fromOffsets: source, toOffset: destination) - save() - } - - var visibleKeys: [TerminalKey] { order.filter(isVisible) } - - private func save() { - UserDefaults.standard.set(hidden.map(\.rawValue).sorted(), forKey: Self.defaultsKey) - UserDefaults.standard.set(order.map(\.rawValue), forKey: Self.orderKey) - } -} diff --git a/App/Views/Terminal/TerminalSession.swift b/App/Views/Terminal/TerminalSession.swift deleted file mode 100644 index d3008ad..0000000 --- a/App/Views/Terminal/TerminalSession.swift +++ /dev/null @@ -1,296 +0,0 @@ -import Foundation -import SwiftUI -import UIKit - -/// The app's one terminal into the embedded Alpine Linux. -/// -/// This is the guest's own console: `/sbin/init` owns it, and it respawns -/// `/sbin/xforge-login root` on it, so the shell you type into is a real login -/// session with a real tty — the login shell root is configured with in -/// `/etc/passwd`. See `InteractiveShellSession` for what that buys. -/// -/// Consequences of it being a real console, which are the point of the change: -/// - the working directory is the shell's own, so `cd` persists; -/// - the *guest's* line discipline supplies echo, backspace, `Ctrl-D` and -/// editing — the host does not synthesise any of it; -/// - `Ctrl-C` is a byte again, so the tty turns it into `SIGINT` for whatever is -/// running in the foreground, and `Ctrl-Z` suspends it; -/// - logging out (or killing the shell) gives a fresh login, because init -/// respawns it rather than leaving a dead screen. -@MainActor -final class TerminalSession: ObservableObject { - /// The screen model. Views render `buffer.lines` and re-render when - /// `revision` changes. - let buffer = TerminalBuffer() - - /// Bumped whenever the screen changes. - @Published private(set) var revision = 0 - /// True while the guest console is attached and input is being forwarded to it. - @Published private(set) var running = false - @Published private(set) var booting = false - /// Commands handed over by other screens, run through the same console. - @Published private(set) var pending: [QueuedLine] = [] - @Published private(set) var activeLabel: String? - @Published private(set) var isBooted = false - @Published private(set) var problem: String? - - /// A line queued for the shell, optionally with the label of the screen that - /// asked for it. - struct QueuedLine: Identifiable, Equatable { - let id = UUID() - let text: String - var label: String? - } - - private var shell: (any InteractiveShellSession)? - private var bootTask: Task? - private var didAttemptBoot = false - - /// How the session obtains a guest. Injectable so the terminal's logic can be - /// tested against a stub instead of a real emulator. - private let makeVM: @MainActor () -> any LinuxVM - - init(makeVM: @escaping @MainActor () -> any LinuxVM = { XForgeEnvironment.makeVM() }) { - self.makeVM = makeVM - let saved = Self.read(Self.transcriptURL) - if let saved, !saved.isEmpty { - if !saved.hasSuffix("\n") { buffer.feed(saved + "\n") } - else { buffer.feed(saved) } - } - if buffer.isAtStart { banner() } - } - - /// Whether the exit key bar should send to a live shell. - var acceptsInput: Bool { !booting && shell?.isRunning == true } - - // MARK: - Boot - - /// Start the shell, booting the guest first. Safe to call from several - /// screens; only the first call does work. - func boot() async { - guard !didAttemptBoot else { - await bootTask?.value - return - } - didAttemptBoot = true - booting = true - problem = nil - buffer.appendLine("[starting the embedded Linux…]", - style: TerminalStyle(foreground: .index(8))) - revision += 1 - - let task = Task { [weak self] in - guard let self else { return } - do { - let vm = self.makeVM() - await vm.prepareRootfs() - try await vm.boot() - let session = try await vm.startInteractiveShell( - onOutput: { [weak self] chunk in - Task { @MainActor in self?.consume(chunk) } - } - ) - self.shell = session - self.isBooted = true - self.running = true - self.buffer.appendLine("[guest is up — Alpine aarch64 Linux]", - style: TerminalStyle(foreground: .index(2))) - // The console's size is the screen's, but layout may have run - // before the shell existed. - self.reportSize() - } catch { - self.problem = error.localizedDescription - self.buffer.appendLine("[error] \(error.localizedDescription)", - style: TerminalStyle(foreground: .index(1))) - } - self.booting = false - self.revision += 1 - self.save() - self.drainQueue() - } - bootTask = task - await task.value - } - - /// Stop the shell. The console belongs to the guest's init, which keeps - /// running; this only stops this screen from reading and writing to it. - func shutdown() { - shell?.stop() - shell = nil - running = false - revision += 1 - } - - // MARK: - Screen size - - /// The size the terminal is drawn at, in characters. The guest needs it or its - /// programs wrap to nothing — `ls` prints one name per line at width 0. - private var screenCols = 0 - private var screenRows = 0 - - /// Called by the terminal surface whenever its geometry changes. - func consoleResized(cols: Int, rows: Int) { - guard cols > 0, rows > 0 else { return } - guard cols != screenCols || rows != screenRows else { return } - screenCols = cols - screenRows = rows - shell?.resize(cols: cols, rows: rows) - } - - private func reportSize() { - guard screenCols > 0, screenRows > 0 else { return } - shell?.resize(cols: screenCols, rows: screenRows) - } - - // MARK: - Input - - /// Send raw keyboard/terminal input straight to the console. - /// - /// Nothing is interpreted here: the console is a tty, so the guest's line - /// discipline and the shell's own line editor are what echo, erase, complete - /// and recall. That is why there is no command field and no host-side prompt. - func sendRaw(_ text: String) { - guard !text.isEmpty else { return } - sendToShell(text, label: nil) - } - - /// Hand a line to the console from another screen. It is written like anything - /// else, so it runs in the same session — with the same environment, the same - /// directory, and after whatever is already queued. - /// - /// Booting this session is part of the job. A screen that queues a command - /// boots the *guest* first, and that is not the same as this console being - /// attached to it: only the Terminal tab called `boot()`. So a command queued - /// before the Terminal tab had ever been opened — which is every install from - /// the Toolchain, Downloads and Settings screens on a fresh launch, since - /// SwiftUI does not build an unselected tab — was reported as undeliverable to - /// a buffer nobody was looking at, while the screen that queued it said it was - /// running in the Terminal. - func enqueue(_ line: String, label: String? = nil) { - enqueue(QueuedLine(text: line, label: label)) - } - - func enqueue(_ line: QueuedLine) { - pending.append(line) - guard shell?.isRunning != true else { - drainQueue() - return - } - Task { [weak self] in - guard let self else { return } - await self.boot() - // If the boot failed the queue is still drained, which reports the - // line as undelivered — the honest outcome, and the reason the drop - // path stays. - self.drainQueue() - } - } - - private func drainQueue() { - guard !pending.isEmpty else { return } - guard running, shell?.isRunning == true else { - // No shell to write to. Take the line out of the queue and report it, - // rather than leaving it pending forever and looking like it will run: - // a command that silently never executes is the worst of the options. - let dropped = pending.removeFirst() - sendToShell(dropped.text, label: dropped.label) - return - } - let next = pending.removeFirst() - // A command from another screen is announced, because the user did not - // type it and needs to know where it came from. The echo of the command - // itself comes from the shell. - if let label = next.label, !label.isEmpty { - buffer.appendLine("↑ requested by \(label)", - style: TerminalStyle(foreground: .index(8))) - revision += 1 - } - sendToShell(next.text + "\n", label: next.label) - } - - private func sendToShell(_ text: String, label: String?) { - guard let shell, shell.isRunning else { - // Nothing to write to. Report it rather than dropping the line, since - // a queued install that silently never runs is worse than an error. - buffer.appendLine("[error] the shell is not running; \(text.count) character(s) not sent", - style: TerminalStyle(foreground: .index(1))) - revision += 1 - return - } - activeLabel = label - if !shell.send(text) { - buffer.appendLine("[error] the guest did not accept input", - style: TerminalStyle(foreground: .index(1))) - revision += 1 - } - } - - /// Interrupt the foreground program with Ctrl-C. - /// - /// The byte, not a host-side `kill`: the console is a real tty, so the guest's - /// line discipline turns `0x03` into `SIGINT` for the foreground process group - /// — which reaches whatever is running, and leaves the shell itself alone. - func interrupt() { - guard let shell, shell.isRunning else { return } - Task { [weak self] in - await shell.interruptForeground() - self?.activeLabel = nil - self?.revision += 1 - } - } - - func pasteFromClipboard() { - guard let text = UIPasteboard.general.string, !text.isEmpty else { return } - // Hand it straight to the shell: it may be multi-line, or contain control - // characters the key bar cannot produce. - sendToShell(text, label: nil) - } - - // MARK: - Output - - private func consume(_ chunk: String) { - // Straight to the screen: the shell prints its own prompt, echoes what it - // reads, and emits its own escape sequences — and the buffer understands - // carriage returns, backspaces and SGR colour, so there is nothing to - // interpret here. - buffer.feed(chunk) - revision += 1 - } - - // MARK: - Screen - - func clear() { - buffer.clear() - banner() - revision += 1 - save() - } - - private func banner() { - buffer.appendLine("XForge terminal — the guest's own console.", - style: TerminalStyle(foreground: .index(11))) - buffer.appendLine("Alpine boots /sbin/init as pid 1, which starts root's login shell on this", - style: TerminalStyle(foreground: .index(8))) - buffer.appendLine("terminal — the shell /etc/passwd names for root. Ctrl-C interrupts the", - style: TerminalStyle(foreground: .index(8))) - buffer.appendLine("foreground program; `exit` ends the session and init starts a fresh one.", - style: TerminalStyle(foreground: .index(8))) - buffer.appendLine("") - revision += 1 - } - - // MARK: - Persistence - - private static var transcriptURL: URL { - XForgeEnvironment.documentDirectory.appendingPathComponent("terminal-transcript.txt") - } - - private static func read(_ url: URL) -> String? { - try? String(contentsOf: url, encoding: .utf8) - } - - private func save() { - try? String(buffer.plainText.suffix(200_000)) - .write(to: Self.transcriptURL, atomically: true, encoding: .utf8) - } -} diff --git a/App/Views/Terminal/TerminalText.swift b/App/Views/Terminal/TerminalText.swift deleted file mode 100644 index bb3f558..0000000 --- a/App/Views/Terminal/TerminalText.swift +++ /dev/null @@ -1,446 +0,0 @@ -import SwiftUI - -// MARK: - Colour - -/// One terminal colour: an indexed palette entry (0–255) or a direct RGB value. -/// -/// The guest is a real Linux system, so its programs emit whatever SGR codes -/// they like — `xtool`, `swift` and `apk` all colourise their output. Parsing -/// them here is what lets the terminal show a build the way it looks in a -/// desktop shell instead of a wall of escape sequences. -enum TerminalColor: Equatable, Hashable { - case index(Int) - case rgb(Int, Int, Int) - - /// The 16 ANSI colours, tuned for a dark terminal background. - private static let palette: [Color] = [ - Color(red: 0.13, green: 0.13, blue: 0.15), // 0 black - Color(red: 0.87, green: 0.29, blue: 0.29), // 1 red - Color(red: 0.35, green: 0.82, blue: 0.44), // 2 green - Color(red: 0.93, green: 0.78, blue: 0.32), // 3 yellow - Color(red: 0.33, green: 0.58, blue: 0.96), // 4 blue - Color(red: 0.78, green: 0.47, blue: 0.96), // 5 magenta - Color(red: 0.32, green: 0.82, blue: 0.86), // 6 cyan - Color(red: 0.86, green: 0.88, blue: 0.91), // 7 white - Color(red: 0.38, green: 0.39, blue: 0.44), // 8 bright black - Color(red: 0.96, green: 0.42, blue: 0.42), // 9 bright red - Color(red: 0.47, green: 0.92, blue: 0.56), // 10 bright green - Color(red: 0.99, green: 0.86, blue: 0.42), // 11 bright yellow - Color(red: 0.47, green: 0.69, blue: 1.00), // 12 bright blue - Color(red: 0.87, green: 0.62, blue: 1.00), // 13 bright magenta - Color(red: 0.47, green: 0.92, blue: 0.96), // 14 bright cyan - Color(red: 0.99, green: 0.99, blue: 0.99), // 15 bright white - ] - - var color: Color { - switch self { - case .rgb(let r, let g, let b): - return Color(red: Self.clamp(r), green: Self.clamp(g), blue: Self.clamp(b)) - case .index(let value): - let n = max(0, min(value, 255)) - if n < 16 { return Self.palette[n] } - if n < 232 { - // The 6×6×6 colour cube. - let i = n - 16 - func level(_ v: Int) -> Double { - v == 0 ? 0 : Double(55 + v * 40) / 255 - } - return Color(red: level(i / 36), - green: level((i / 6) % 6), - blue: level(i % 6)) - } - // The 24-step grey ramp. - let v = Double(8 + (n - 232) * 10) / 255 - return Color(red: v, green: v, blue: v) - } - } - - private static func clamp(_ value: Int) -> Double { - Double(max(0, min(value, 255))) / 255 - } -} - -/// The attributes an SGR sequence can turn on. -struct TerminalStyle: Equatable { - var foreground: TerminalColor? - var background: TerminalColor? - var bold = false - var dim = false - var italic = false - var underline = false - var inverse = false - - static let plain = TerminalStyle() - - /// The colour text is drawn in, with `inverse` resolved. - var textColor: Color { - if inverse { return (background ?? .index(0)).color } - return (foreground ?? Self.defaultForeground).color - } - - /// The colour behind the run, or nil for "the terminal's own background". - var backgroundColor: Color? { - if inverse { return (foreground ?? .index(15)).color } - guard let background else { return nil } - return background.color - } - - /// Slightly warm off-white — the same default the iSH terminal uses. - static let defaultForeground = TerminalColor.rgb(0xE6, 0xE9, 0xE6) -} - -// MARK: - Cells, lines, and the buffer - -/// One screen cell: a character and the style it was written with. -struct TerminalCell: Equatable { - var character: Character - var style: TerminalStyle -} - -/// One completed line of the scrollback. -struct TerminalLine: Equatable { - var cells: [TerminalCell] = [] - - var isEmpty: Bool { cells.isEmpty } - - var text: String { String(cells.map(\.character)) } - - /// The line as an `AttributedString`, with runs of equal style merged. - func attributedString(fontSize: CGFloat) -> AttributedString { - guard !cells.isEmpty else { return AttributedString(" ") } - - var result = AttributedString() - var index = 0 - while index < cells.count { - let style = cells[index].style - var text = "" - while index < cells.count, cells[index].style == style { - text.append(cells[index].character) - index += 1 - } - var run = AttributedString(text) - run.foregroundColor = style.textColor - if let background = style.backgroundColor { - run.backgroundColor = background - } - if style.underline { - run.underlineStyle = .single - } - if style.bold || style.italic || style.dim { - var font = Font.system(size: fontSize, design: .monospaced) - if style.bold { font = font.weight(.bold) } - if style.italic { font = font.italic() } - run.font = font - if style.dim, let foreground = style.foreground { - // "Dim" has no monospaced-font equivalent; approximate it by - // blending the colour toward the background. - run.foregroundColor = foreground.color.opacity(0.65) - } else if style.dim { - run.foregroundColor = style.textColor.opacity(0.65) - } - } - result.append(run) - } - return result - } -} - -/// A very small terminal screen model. -/// -/// It exists because the guest *is* a terminal program: a command that only -/// returns its output when it finishes still returns carriage-return rewrites -/// (curl's progress bar), backspaces and SGR colour. Feeding those through a -/// screen model instead of appending raw bytes is the difference between a log -/// view and a terminal. -/// -/// It is a line/screen model, not a full emulator: cursor addressing, alternate -/// screens and scrolling regions are recognised and either honoured (`clear`, -/// `\r`, `\b`) or dropped, because XForge's bridge runs one command at a time -/// rather than hosting a live pty. -@MainActor -final class TerminalBuffer { - /// Completed lines, oldest first. - private(set) var lines: [TerminalLine] = [] - - /// The line still being written to. - private(set) var current = TerminalLine() - private var cursor = 0 - - /// Scrollback cap. The transcript is also persisted, so this is about the - /// cost of keeping it on screen, not about losing history. - var maxLines = 2000 - private let maxColumns = 512 - - private var style = TerminalStyle() - - private enum EscapeState { - case ground - case escape - case csi - case osc - case oscEscape - } - - private var state: EscapeState = .ground - private var csiParameters = "" - private var pendingCarriageReturn = false - - var isAtStart: Bool { lines.isEmpty && current.isEmpty } - - /// Everything on screen plus the scrollback, as plain text. - var plainText: String { - (lines.map(\.text) + [current.text]).joined(separator: "\n") - } - - func clear() { - lines.removeAll() - current = TerminalLine() - cursor = 0 - style = .plain - state = .ground - csiParameters = "" - pendingCarriageReturn = false - } - - /// Feed a chunk of guest output through the screen model. - func feed(_ text: String) { - for character in text { - switch state { - case .ground: - handleGround(character) - case .escape: - handleEscape(character) - case .csi: - handleCSI(character) - case .osc: - // OSC payload: skipped until BEL or ST. - if character == "\u{07}" { - state = .ground - } else if character == "\u{1b}" { - state = .oscEscape - } - case .oscEscape: - state = character == "\\" ? .ground : .osc - } - } - } - - /// Put a line on screen immediately (used for the terminal's own prompt and - /// status lines, which are not guest output). - func appendLine(_ text: String, style: TerminalStyle = .plain) { - let saved = self.style - self.style = style - for character in text { write(character) } - newline() - self.style = saved - } - - // MARK: - Ground state - - private func handleGround(_ character: Character) { - switch character { - case "\u{1b}": - pendingCarriageReturn = false - state = .escape - case "\n", "\r\n": - // Swift treats CR+LF as a *single* grapheme cluster, so a guest - // program that emits Windows-style line endings delivers "\r\n" as - // ONE Character — not as "\r" followed by "\n". Matching only "\n" - // here would write that character into a cell and the line would - // never break, turning a whole command's output into one line. - pendingCarriageReturn = false - newline() - case "\r": - // Held until the next character: "\r\n" is a newline, a bare "\r" - // rewinds to column 0 so a progress bar can redraw its line. - pendingCarriageReturn = true - case "\u{08}": - backspace() - case "\t": - applyPendingCarriageReturn() - tab() - default: - // Remaining control characters (BEL, NUL, …) are not printable. A - // grapheme can hold more than one scalar, so test all of them rather - // than `asciiValue`, which is nil for anything multi-scalar. - if character.unicodeScalars.allSatisfy({ $0.value < 0x20 }) { return } - applyPendingCarriageReturn() - write(character) - } - } - - private func applyPendingCarriageReturn() { - guard pendingCarriageReturn else { return } - pendingCarriageReturn = false - cursor = 0 - } - - private func handleEscape(_ character: Character) { - switch character { - case "[": - csiParameters = "" - state = .csi - case "]": - state = .osc - case "(", ")", "#", "%": - // Two-byte sequences with a designator byte; stay in escape state - // for one more character so the designator is swallowed too. - state = .escape - default: - state = .ground - } - } - - private func handleCSI(_ character: Character) { - // Parameter bytes, then a final byte in the range 0x40-0x7E. - guard let ascii = character.asciiValue, ascii >= 0x40, ascii <= 0x7E else { - if csiParameters.count < 32 { csiParameters.append(character) } - return - } - state = .ground - applyCSI(final: character, parameters: csiParameters) - csiParameters = "" - } - - private func applyCSI(final: Character, parameters: String) { - switch final { - case "m": - applySGR(parameters) - case "J": - // Erase in display. Only the "whole screen" form is meaningful here - // (the bridge has no cursor addressing), so `clear` and - // `printf '\033[2J'` wipe the scrollback while the partial forms just - // clear the line being written. - if parameters.isEmpty || parameters == "2" { - lines.removeAll() - current = TerminalLine() - cursor = 0 - } else if cursor < current.cells.count { - current.cells.removeSubrange(cursor.. Int? { - guard index + offset < parts.count else { return nil } - return Int(parts[index + offset]) - } - while index < parts.count { - let value = Int(parts[index]) ?? 0 - switch value { - case 0: - style = .plain - case 1: - style.bold = true - case 2: - style.dim = true - case 3: - style.italic = true - case 4: - style.underline = true - case 7: - style.inverse = true - case 21, 22: - style.bold = false - style.dim = false - case 23: - style.italic = false - case 24: - style.underline = false - case 27: - style.inverse = false - case 30...37: - style.foreground = .index(value - 30) - case 39: - style.foreground = nil - case 40...47: - style.background = .index(value - 40) - case 49: - style.background = nil - case 90...97: - style.foreground = .index(value - 90 + 8) - case 100...107: - style.background = .index(value - 100 + 8) - case 38, 48: - let isForeground = value == 38 - if let mode = parameter(1), mode == 5, let n = parameter(2) { - let colour = TerminalColor.index(n) - if isForeground { style.foreground = colour } else { style.background = colour } - index += 2 - } else if let mode = parameter(1), mode == 2, - let r = parameter(2), let g = parameter(3), let b = parameter(4) { - let colour = TerminalColor.rgb(r, g, b) - if isForeground { style.foreground = colour } else { style.background = colour } - index += 4 - } - default: - break - } - index += 1 - } - } - - // MARK: - Writing - - private func write(_ character: Character) { - guard cellsAllowMore() else { return } - let cell = TerminalCell(character: character, style: style) - if cursor < current.cells.count { - current.cells[cursor] = cell - } else { - current.cells.append(cell) - } - cursor += 1 - } - - private func cellsAllowMore() -> Bool { - cursor < maxColumns || cursor < current.cells.count - } - - private func tab() { - let next = (cursor / 8 + 1) * 8 - while cursor < next { write(" ") } - } - - private func backspace() { - pendingCarriageReturn = false - guard cursor > 0 else { return } - cursor -= 1 - if cursor < current.cells.count { - current.cells.remove(at: cursor) - } - } - - private func newline() { - lines.append(current) - if lines.count > maxLines { - lines.removeFirst(lines.count - maxLines) - } - current = TerminalLine() - cursor = 0 - } -} diff --git a/App/Views/Terminal/TerminalView.swift b/App/Views/Terminal/TerminalView.swift deleted file mode 100644 index a780048..0000000 --- a/App/Views/Terminal/TerminalView.swift +++ /dev/null @@ -1,533 +0,0 @@ -import SwiftUI -import UIKit -import UniformTypeIdentifiers - -/// The Terminal tab: the guest's console. -/// -/// The screen *is* the terminal. Alpine boots `/sbin/init` as pid 1, which starts -/// root's login shell on the console, and the console is this view: what you type goes -/// into the guest's tty, and what the tty produces is drawn here. The shell prints -/// its own prompt and echoes its own input, so there is nothing host-side to draw -/// or interpret. -struct TerminalView: View { - @EnvironmentObject private var session: TerminalSession - - @State private var fontSize: Double = 12 - @State private var showGuestFiles = false - @State private var showEngineLog = false - @State private var importingXIP = false - /// Which keys the extra-keys bar shows. Shared with the configuration sheet. - @StateObject private var keyConfiguration = TerminalKeyConfiguration() - @State private var showKeyConfiguration = false - - var body: some View { - VStack(spacing: 0) { - canvas - if !session.pending.isEmpty || session.activeLabel != nil { - activityStrip - } - TerminalKeyBar(session: session, - configuration: keyConfiguration, - onFiles: { showGuestFiles = true }, - onComponents: { component in install(component) }, - onHideKeyboard: { UIApplication.shared.sendAction(#selector(UIResponder.resignFirstResponder), to: nil, from: nil, for: nil) }, - onConfigure: { showKeyConfiguration = true }) - } - .background(Color.black) - .navigationTitle("Alpine Linux") - .navigationBarTitleDisplayMode(.inline) - .toolbar { toolbarContent } - .task { - await session.boot() - } - .fileImporter( - isPresented: $importingXIP, - allowedContentTypes: [UTType(filenameExtension: "xip") ?? .data], - allowsMultipleSelection: false - ) { result in - guard case .success(let urls) = result, let url = urls.first else { return } - stageXIP(url) - } - .sheet(isPresented: $showEngineLog) { - NavigationStack { EngineLogView() } - } - .sheet(isPresented: $showGuestFiles) { - GuestFileBrowserView() - } - .sheet(isPresented: $showKeyConfiguration) { - TerminalKeyConfigurationView() - .environmentObject(keyConfiguration) - } - } - - // MARK: - Screen - - private var canvas: some View { - InteractiveTerminalSurface(session: session, fontSize: fontSize) - .frame(maxWidth: .infinity, maxHeight: .infinity) - .background(Color.black) - } - - private static let bottomID = "org.xforge.terminal.bottom" - - // MARK: - Status strip - - /// Shown while something is outstanding. There is no "running" indicator for - /// an ordinary command any more: the shell is always there, so the absence of - /// a prompt is the indication that a program is still working. - private var activityStrip: some View { - HStack(spacing: 8) { - if !session.pending.isEmpty { - ProgressView().controlSize(.mini).tint(.white) - Text("\(session.pending.count) command(s) queued") - .font(.caption2) - .foregroundStyle(.white.opacity(0.85)) - } else if let label = session.activeLabel { - Text("requested by \(label)") - .font(.caption2) - .foregroundStyle(.white.opacity(0.85)) - } - Spacer() - } - .padding(.horizontal, 12) - .padding(.vertical, 5) - .background(Color(white: 0.13)) - } - - // MARK: - Toolbar - - @ToolbarContentBuilder - private var toolbarContent: some ToolbarContent { - ToolbarItem(placement: .principal) { - // No working-directory line: the shell owns its cwd and nothing here - // can see it, so a hardcoded "/root" was wrong the moment anyone ran - // `cd`. (Asking the guest for it would mean a command per keystroke; - // if it is wanted, that is what it costs.) - Text("Alpine Linux").font(.footnote.bold()) - } - ToolbarItem(placement: .primaryAction) { - Menu { - Section("Components") { - ForEach(SystemComponents.Component.allCases) { component in - Button { - install(component) - } label: { - Label("Install the \(component.title)", systemImage: icon(for: component)) - } - } - Button { - importingXIP = true - } label: { - Label("Install the Darwin SDK from an Xcode.xip…", - systemImage: "doc.badge.plus") - } - } - - Section("Session") { - Button { UIPasteboard.general.string = session.buffer.plainText } label: { - Label("Copy transcript", systemImage: "doc.on.doc") - } - Button { session.clear() } label: { - Label("Clear", systemImage: "eraser") - } - Button { session.sendRaw("\u{1b}[A") } label: { - Label("Previous command", systemImage: "chevron.up") - } - } - - Section("Keys") { - Button { showKeyConfiguration = true } label: { - Label("Configure terminal keys…", systemImage: "slider.horizontal.3") - } - } - - Section("Font size") { - Button { fontSize = max(9, fontSize - 1) } label: { - Label("Smaller", systemImage: "textformat.size.smaller") - } - Button { fontSize = min(20, fontSize + 1) } label: { - Label("Larger", systemImage: "textformat.size.larger") - } - } - - Section { - Button { showEngineLog = true } label: { - Label("Engine log", systemImage: "doc.text.magnifyingglass") - } - } - } label: { - Label("Terminal options", systemImage: "ellipsis.circle") - } - } - } - - private func icon(for component: SystemComponents.Component) -> String { - switch component { - case .glibc: return "shippingbox" - case .xtool: return "hammer" - case .swift: return "swift" - case .darwinSDK: return "externaldrive.connected.to.line.below" - } - } - - // MARK: - Actions - - /// Queue a component install in the terminal so its output is visible here. - private func install(_ component: SystemComponents.Component) { - switch component { - case .glibc: - session.enqueue(SystemComponents.scriptCommand(.glibc), label: "Components") - case .xtool: - Task { await prepareInstallerScript(); session.enqueue(SystemComponents.xtoolInstallCommand, label: "Components") } - case .swift: - Task { await prepareInstallerScript(); session.enqueue(SystemComponents.swiftInstallCommand, label: "Components") } - case .darwinSDK: - importingXIP = true - } - } - - private func prepareInstallerScript() async { - do { - let vm = XForgeEnvironment.makeVM() - try await vm.boot() - try await SystemComponents.ensureInstallerScript(in: vm) - } catch { - session.enqueue("echo \(GuestShell.quote("install-toolchain.sh: \(error.localizedDescription)"))", - label: "Components") - } - } - - /// Copy the chosen `.xip` into the guest's own filesystem, then hand xtool - /// the install command. - private func stageXIP(_ url: URL) { - session.enqueue("echo \(GuestShell.quote("staging \(url.lastPathComponent) into the guest…"))", - label: "Components") - Task { - do { - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - let guestPath = try await SystemComponents.stageXIP(url, in: vm) { line in - XForgeLog.note("components: \(line)") - } - session.enqueue(SystemComponents.darwinSDKInstallCommand(guestXIPPath: guestPath), - label: "Components") - } catch { - session.enqueue("echo \(GuestShell.quote("could not stage the xip: \(error.localizedDescription)"))", - label: "Components") - } - } - } -} - -/// A native iOS terminal surface. The view itself becomes first responder, so -/// software/hardware keyboard events go directly to Alpine instead of through a -/// separate command field. Touch controls scrolling and selects/copies terminal -/// text; tapping the terminal summons the iOS keyboard. -private struct InteractiveTerminalSurface: UIViewRepresentable { - @ObservedObject var session: TerminalSession - let fontSize: Double - - func makeUIView(context: Context) -> TerminalTextView { - let view = TerminalTextView() - view.session = session - view.font = .monospacedSystemFont(ofSize: fontSize, weight: .regular) - view.refresh(from: session) - DispatchQueue.main.async { view.becomeFirstResponder() } - return view - } - - func updateUIView(_ view: TerminalTextView, context: Context) { - view.session = session - view.font = .monospacedSystemFont(ofSize: fontSize, weight: .regular) - view.refresh(from: session) - } -} - -private final class TerminalTextView: UITextView { - weak var session: TerminalSession? - private var lastRevision = -1 - - override var canBecomeFirstResponder: Bool { true } - override init(frame: CGRect, textContainer: NSTextContainer?) { - super.init(frame: frame, textContainer: textContainer) - backgroundColor = .black - textColor = UIColor(white: 0.92, alpha: 1) - tintColor = .systemGreen - isEditable = true - isSelectable = true - keyboardType = .asciiCapable - autocorrectionType = .no - autocapitalizationType = .none - spellCheckingType = .no - smartQuotesType = .no - smartDashesType = .no - smartInsertDeleteType = .no - alwaysBounceVertical = true - keyboardDismissMode = .interactive - textContainerInset = UIEdgeInsets(top: 6, left: 8, bottom: 8, right: 8) - // `self.` is required: inside this initializer the parameter named - // `textContainer` shadows the property of the same name, and the - // parameter is optional — `textContainer.lineFragmentPadding` reads as - // an optional member access and does not compile. - self.textContainer.lineFragmentPadding = 0 - addGestureRecognizer(UITapGestureRecognizer(target: self, action: #selector(focusTerminal))) - } - - required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } - - @objc private func focusTerminal() { becomeFirstResponder() } - - /// Report the terminal's size in characters to the guest. - /// - /// Without this the guest believes its terminal is 0×0 and every program that - /// lays out by width breaks: `ls` prints one name per line, and anything - /// full-screen draws in a corner. The cell size comes from the same - /// monospaced font the text is drawn in. - override func layoutSubviews() { - super.layoutSubviews() - reportSize() - } - - private func reportSize() { - guard let session, let font, - bounds.width > 0, bounds.height > 0 else { return } - let advance = ("W" as NSString).size(withAttributes: [.font: font]).width - guard advance > 0 else { return } - let usableWidth = bounds.width - textContainerInset.left - textContainerInset.right - let usableHeight = bounds.height - textContainerInset.top - textContainerInset.bottom - let cols = Int(usableWidth / advance) - let rows = Int(usableHeight / font.lineHeight) - guard cols > 0, rows > 0 else { return } - session.consoleResized(cols: cols, rows: rows) - } - - func refresh(from session: TerminalSession) { - guard lastRevision != session.revision else { return } - lastRevision = session.revision - let wasNearBottom = contentOffset.y + bounds.height >= contentSize.height - 44 - let rendered = session.buffer.plainText - if text != rendered { text = rendered } - selectedRange = NSRange(location: (text as NSString).length, length: 0) - if wasNearBottom || !isTracking { - let end = NSRange(location: (text as NSString).length, length: 0) - scrollRangeToVisible(end) - } - } - - // UIKeyInput is intentionally implemented even though the UITextView is not - // editable: UIKit still presents the keyboard, while every keystroke is sent - // to the guest shell instead of mutating host-side text. - override var hasText: Bool { true } - override func insertText(_ text: String) { - session?.sendRaw(text) - } - override func deleteBackward() { - session?.sendRaw("\u{7f}") - } - - override var keyCommands: [UIKeyCommand]? { - [ - UIKeyCommand(input: UIKeyCommand.inputUpArrow, modifierFlags: [], action: #selector(upArrow)), - UIKeyCommand(input: UIKeyCommand.inputDownArrow, modifierFlags: [], action: #selector(downArrow)), - UIKeyCommand(input: UIKeyCommand.inputLeftArrow, modifierFlags: [], action: #selector(leftArrow)), - UIKeyCommand(input: UIKeyCommand.inputRightArrow, modifierFlags: [], action: #selector(rightArrow)), - UIKeyCommand(input: "c", modifierFlags: .control, action: #selector(controlC)), - UIKeyCommand(input: "\t", modifierFlags: [], action: #selector(tabKey)), - UIKeyCommand(input: UIKeyCommand.inputEscape, modifierFlags: [], action: #selector(escapeKey)) - ] - } - - @objc private func upArrow() { session?.sendRaw("\u{1b}[A") } - @objc private func downArrow() { session?.sendRaw("\u{1b}[B") } - @objc private func leftArrow() { session?.sendRaw("\u{1b}[D") } - @objc private func rightArrow() { session?.sendRaw("\u{1b}[C") } - @objc private func controlC() { session?.interrupt() } - @objc private func tabKey() { session?.sendRaw("\t") } - @objc private func escapeKey() { session?.sendRaw("\u{1b}") } -} - -/// the iSH terminal's extra-keys bar: the characters a phone keyboard cannot produce, -/// between the terminal and the keyboard. -private struct TerminalKeyBar: View { - @ObservedObject var session: TerminalSession - @ObservedObject var configuration: TerminalKeyConfiguration - var onFiles: () -> Void - var onComponents: (SystemComponents.Component) -> Void - var onHideKeyboard: () -> Void - var onConfigure: () -> Void - - /// Key metrics, defined once. The keys are deliberately small — they sit - /// under the terminal and exist to supply characters a phone keyboard cannot - /// produce, not to be a primary control surface. - private static let keyWidth: CGFloat = 15 - private static let keyHeight: CGFloat = 14 - private static let keySpacing: CGFloat = 3 - /// The glyph is inset from the key so a symbol never touches the border. - private static var glyphSize: CGFloat { 9 } - - var body: some View { - ScrollView(.horizontal, showsIndicators: false) { - HStack(spacing: Self.keySpacing) { - ForEach(configuration.visibleKeys) { key in - self.key(key) - } - } - .padding(.horizontal, 6) - .padding(.vertical, 4) - } - .background(Color(white: 0.16)) - .accessibilityLabel("Terminal keyboard") - } - - /// One key, rendered from its declaration rather than from a hard-coded row, - /// so the configuration and the bar cannot disagree about what exists. - @ViewBuilder - private func key(_ key: TerminalKey) -> some View { - switch key { - case .components: - componentsMenu - default: - Button { - activate(key) - } label: { - if let symbol = key.symbol { - Image(systemName: symbol) - .font(.system(size: Self.glyphSize)) - .frame(width: Self.keyWidth, height: Self.keyHeight) - } else { - Text(key.label) - .font(.system(size: Self.glyphSize + 1, design: .monospaced)) - .frame(width: Self.keyWidth, height: Self.keyHeight) - } - } - .buttonStyle(.bordered) - .tint(.white) - .accessibilityLabel(key.title) - .contextMenu { - // Turning a key off from the key itself is the fastest way to - // trim the bar, but the configuration sheet is where keys that - // are already hidden can be brought back. - if !key.isPinned { - Button("Hide “\(key.title)”", systemImage: "eye.slash") { - configuration.setVisible(false, for: key) - } - } - Button("Configure keys…", systemImage: "slider.horizontal.3") { - onConfigure() - } - } - } - } - - private func activate(_ key: TerminalKey) { - switch key { - case .interrupt: session.interrupt() - // Everything else is a raw byte sequence for the console. The guest owns - // the editing — its line discipline and the shell's own line editor — so - // the key bar only has to produce the characters a phone keyboard cannot, - // and the shell's history is reached with the arrow keys it understands. - case .escape: session.sendRaw("\u{1b}") - case .tab: session.sendRaw("\t") - case .previous: session.sendRaw("\u{1b}[A") - case .next: session.sendRaw("\u{1b}[B") - case .dash: session.sendRaw("-") - case .dot: session.sendRaw(".") - case .slash: session.sendRaw("/") - case .colon: session.sendRaw(":") - case .bang: session.sendRaw("!") - case .pipe: session.sendRaw("|") - case .files: onFiles() - case .paste: session.pasteFromClipboard() - case .hideKeyboard: onHideKeyboard() - case .components: break // rendered as a menu, never reaches here - } - } - - /// The gear/wrench key: the system components, installed by command. - private var componentsMenu: some View { - Menu { - ForEach(SystemComponents.Component.allCases) { component in - Button(component.title) { onComponents(component) } - } - Divider() - Button("Configure keys…", systemImage: "slider.horizontal.3") { - onConfigure() - } - } label: { - Image(systemName: "wrench.and.screwdriver") - .font(.system(size: Self.glyphSize)) - .frame(width: Self.keyWidth, height: Self.keyHeight) - } - .buttonStyle(.bordered) - .tint(.white) - .accessibilityLabel("Components") - } -} - -/// Chooses which keys the extra-keys bar shows. -/// -/// Hide Keyboard is listed but not switchable, with the reason stated: without it -/// there is no way back to the screen once the keyboard covers it. -private struct TerminalKeyConfigurationView: View { - @EnvironmentObject private var configuration: TerminalKeyConfiguration - @Environment(\.dismiss) private var dismiss - - var body: some View { - NavigationStack { - List { - Section { - ForEach(configuration.order) { key in - row(for: key) - } - .onMove(perform: configuration.move) - } header: { - Text("Command bar") - } footer: { - Text("Drag to reorganize buttons. Hidden buttons remain here so they can be restored.") - } - - Section { - Button("Restore default buttons and order", systemImage: "arrow.counterclockwise") { - configuration.resetToDefaults() - } - } - } - .navigationTitle("Terminal keys") - .navigationBarTitleDisplayMode(.inline) - .toolbar { - ToolbarItem(placement: .navigationBarLeading) { EditButton() } - ToolbarItem(placement: .confirmationAction) { - Button("Done") { dismiss() } - } - } - } - } - - @ViewBuilder - private func row(for key: TerminalKey) -> some View { - if key.isPinned { - HStack { - Text(key.title) - Spacer() - Text("Always shown") - .font(.footnote) - .foregroundStyle(.secondary) - } - } else { - Toggle(key.title, isOn: Binding( - get: { configuration.isVisible(key) }, - set: { configuration.setVisible($0, for: key) } - )) - } - } -} - -/// Tab wrapper, so the root view can hand the terminal its session. -struct TerminalTab: View { - var body: some View { - NavigationStack { - TerminalView() - } - } -} diff --git a/App/Views/Toolchain/NativeToolchainView.swift b/App/Views/Toolchain/NativeToolchainView.swift deleted file mode 100644 index 41ad56e..0000000 --- a/App/Views/Toolchain/NativeToolchainView.swift +++ /dev/null @@ -1,150 +0,0 @@ -import SwiftUI - -@MainActor -struct NativeToolchainView: View { - @State private var status = "" - @State private var busy = false - @State private var sdkSummary = "Not installed" - - var body: some View { - List { - Section("Compiler") { - LabeledContent( - "Backend", - value: NativeToolchain.isAvailable ? "Native LLVM" : "Not linked" - ) - LabeledContent("Version", value: NativeToolchain.version) - LabeledContent( - "Swift frontend", - value: NativeToolchain.isSwiftAvailable ? "Linked" : "Not linked" - ) - - if NativeToolchain.isAvailable { - Label("Clang and LLD run in-process", systemImage: "checkmark.circle.fill") - .foregroundStyle(.green) - } else { - Text("Install the native toolchain bundle before generating the Xcode project.") - .font(.caption) - .foregroundStyle(.secondary) - } - } - - Section("Darwin SDK") { - LabeledContent("Status", value: sdkSummary) - - Button { - installSDK() - } label: { - Label("Download Native Darwin SDK", systemImage: "arrow.down.circle") - } - .disabled(busy) - - if NativeSDK.isInstalled { - Button(role: .destructive) { - removeSDK() - } label: { - Label("Remove Native SDK", systemImage: "trash") - } - .disabled(busy) - } - } - - Section("Validation") { - Button { - runSmokeTest() - } label: { - Label("Compile Native C Smoke Test", systemImage: "hammer") - } - .disabled(busy || !NativeToolchain.isAvailable || !NativeSDK.isInstalled) - - Button { - runSwiftSmokeTest() - } label: { - Label("Compile Native Swift Smoke Test", systemImage: "swift") - } - .disabled(busy || !NativeToolchain.isSwiftAvailable || !NativeSDK.isInstalled) - - if !status.isEmpty { - Text(status) - .font(.caption.monospaced()) - .textSelection(.enabled) - } - } - } - .navigationTitle("Native Toolchain") - .task { refreshSDKStatus() } - } - - private func refreshSDKStatus() { - guard NativeSDK.isInstalled else { - sdkSummary = "Not installed" - return - } - do { - let layout = try NativeSDK.layout() - sdkSummary = layout.sdkRoot.lastPathComponent - } catch { - sdkSummary = "Invalid" - status = error.localizedDescription - } - } - - private func installSDK() { - busy = true - status = "Resolving Darwin SDK release…" - Task { - defer { busy = false } - do { - try await NativeSDK.installLatestPrebuilt() - refreshSDKStatus() - status = "Native Darwin SDK installed." - } catch { - status = error.localizedDescription - } - } - } - - private func removeSDK() { - do { - try NativeSDK.remove() - refreshSDKStatus() - status = "Native Darwin SDK removed." - } catch { - status = error.localizedDescription - } - } - - private func runSmokeTest() { - busy = true - status = "Compiling C in-process…" - Task { - defer { busy = false } - do { - let layout = try NativeSDK.layout() - let object = try NativeToolchain.smokeCompile(sdk: layout.sdkRoot) - let attributes = try FileManager.default.attributesOfItem(atPath: object.path) - let size = (attributes[.size] as? NSNumber)?.int64Value ?? 0 - status = "C success: \(object.lastPathComponent) (\(size) bytes)" - } catch { - status = error.localizedDescription - } - } - } - - private func runSwiftSmokeTest() { - busy = true - status = "Compiling Swift in-process…" - Task { - defer { busy = false } - do { - let layout = try NativeSDK.layout() - let object = try NativeToolchain.smokeCompileSwift(sdk: layout) - let attributes = try FileManager.default.attributesOfItem(atPath: object.path) - let size = (attributes[.size] as? NSNumber)?.int64Value ?? 0 - status = "Swift success: \(object.lastPathComponent) (\(size) bytes)" - } catch { - status = error.localizedDescription - } - } - } -} diff --git a/App/Views/Toolchain/ToolchainView.swift b/App/Views/Toolchain/ToolchainView.swift index 218ba2c..4d65dcf 100644 --- a/App/Views/Toolchain/ToolchainView.swift +++ b/App/Views/Toolchain/ToolchainView.swift @@ -1,291 +1,397 @@ import SwiftUI import UniformTypeIdentifiers -/// Manage the on-device build infrastructure. +/// Manage the build toolchain. /// -/// Every component lives in the embedded Alpine system, and every one of them is -/// installed *by running a command inside it*. That is what this screen sets up: -/// it puts the user's files where the guest can reach them and hands the right -/// command to the Terminal, where the install runs and reports what it is doing. +/// Two halves, and the screen is honest about which one is missing: /// -/// - **Alpine rootfs** — bundled in the app; "installing" it is a local import. -/// - **Swift / xtool** — the commands swift.org and xtool document, run in the -/// guest (see `SystemComponents`). -/// - **Darwin SDK** — either your own `Xcode.xip`, copied into the guest's -/// storage and installed there with `xtool sdk install`, or XForge's prebuilt -/// bundle, downloaded and installed inside the guest. +/// - **the compiler**, linked into this build of the app +/// (`Support/NativeToolchain.generated.xcconfig`, written by +/// `NativeToolchain/prepare-xcode.sh`). It cannot be installed at runtime — +/// either the libraries are in the binary or they are not — so this reports +/// what is there; +/// - **the Darwin SDK**, a bundle in the app's container, downloaded or imported +/// here. +/// +/// The smoke test compiles a real file through the linked compiler and links it, +/// which is the only way to know the toolchain works before a build depends on it. struct ToolchainView: View { - @EnvironmentObject private var terminal: TerminalSession - @StateObject private var toolchain = ToolchainManager() - @State private var importingXIP = false - @State private var importingRootfs = false + @State private var sdkPath: String? + @State private var sdkError: String? + @State private var importing = false + @State private var working = false + @State private var message: String? + @State private var smokeResult: String? @State private var confirmingDownload = false - @State private var preparing: String? + @StateObject private var importState = SDKImportState() + + /// What the picker offers. Deliberately permissive. + /// + /// A `.xip` has no UTI the app can rely on: if the system declares one, the type + /// its extension resolves to is right; if it does not, the file carries a + /// *dynamic* type, and a differently-derived dynamic type need not match it. A + /// file that matches none of the allowed types is greyed out in Files — which + /// looks like "the import does nothing" with no error to read. So `.data` is + /// allowed too, and `XipArchive` says exactly what is wrong with an unrelated + /// file (`xar!` missing, no `Content` member, no `pbzx` payload, unreadable cpio). + private static var importableTypes: [UTType] { + var types: [UTType] = [.folder, .zip, .data] + if let xip = UTType("com.apple.xip-archive") { + types.append(xip) + } + return types + } + + private var capabilities: NativeToolchainCapabilities { .current } var body: some View { Form { Section { - ForEach(ToolchainManager.Component.allCases) { component in - row(component) - } + row("clang", present: capabilities.hasClang) + row("ld64.lld (Mach-O linker)", present: capabilities.hasLLDMachO) + row("swift-frontend", present: capabilities.hasSwiftFrontend) } header: { - Text("Components") + Text("Compiler") } footer: { - Text("Installs run in the Terminal tab, so you can watch them — and " - + "answer anything they ask — while they work.") + Text(capabilities.hasClang + ? "Linked into this build: \(capabilities.backendDescription)" + : "Not linked into this build. The native compiler has to be in the binary, " + + "so install the toolchain bundle and rebuild (NativeToolchain/install-bundle.sh).") } Section { - Button { - Task { await toolchain.refresh(probeGuest: true) } - } label: { - Label("Check the embedded Linux", systemImage: "arrow.clockwise") + if sdkPath != nil { + LabeledContent("SDK", value: "installed") + LabeledContent("Path", value: sdkPath ?? "") + } else { + Text("Not installed. A build cannot compile anything without it.") + .foregroundStyle(.secondary) + } + if let sdkError { + Text(sdkError).font(.footnote).foregroundStyle(.red) } - .disabled(toolchain.activity != nil || preparing != nil) + } header: { + Text("Darwin SDK") } footer: { - Text("Green = present. Guest components are verified inside the " - + "embedded Linux, which this boots if it is not running.") + Text("The SDK is xtool's darwin.artifactbundle: the iPhoneOS headers, the " + + "tbd stubs and the static Swift runtime, read straight out of the " + + "app's container. About 460 MB downloaded, or built here from an " + + "Xcode.xip — which the picker copies into the container first, so " + + "an 11 GB xip needs roughly 12 GB free, plus about 1.5 GB for the " + + "bundle. Extraction takes a few minutes.") } Section { - Button { - importingXIP = true - } label: { - Label("Install the Darwin SDK from an Xcode.xip…", - systemImage: "doc.badge.plus") - } Button { confirmingDownload = true } label: { - Label("Install the prebuilt Darwin SDK…", systemImage: "arrow.down.circle") + Label(working ? "Working…" : "Download the Darwin SDK", systemImage: "arrow.down.circle") } - } header: { - Text("Darwin SDK") - } footer: { - Text("With an Xcode.xip: the file is copied into the Alpine system's own " - + "storage and then installed there with " - + "`xtool sdk install \"path/to/xip\"`. Without one, XForge's " - + "prebuilt darwin.artifactbundle is fetched and installed inside " - + "the guest instead. Either way the SDK already in the guest is " - + "removed first — SwiftPM will not install a second bundle " - + "carrying the same artifact ID, and the bundled rootfs ships one.") - } + .disabled(working) - Section { Button { - importingRootfs = true + importing = true } label: { - Label("Import an Alpine rootfs archive…", - systemImage: "shippingbox.and.arrow.backward") + Label("Install from an Xcode.xip or a bundle…", systemImage: "doc.badge.plus") } - .disabled(toolchain.isInstalling != nil || toolchain.isGuestBooted) - } header: { - Text("Offline Imports") - } footer: { - Text("Choose a rootfs archive to replace the bundled one. A plain Alpine " - + ".tar.gz is imported on the host, so it will not have XForge's " - + "root configuration. Quit and reopen XForge first if Linux is running.") - } + .disabled(working) - if let activity = toolchain.activity ?? preparing { - Section { - HStack(spacing: 8) { - ProgressView().controlSize(.small) - Text(activity).font(.footnote).foregroundStyle(.secondary) + if importState.isRunning { + VStack(alignment: .leading, spacing: 4) { + ProgressView(value: importState.fraction) + Text(importState.status) + .font(.caption) + .foregroundStyle(.secondary) } } - } - if let message = toolchain.message { - Section { - Label(message, systemImage: "info.circle") - .font(.footnote).foregroundStyle(.secondary) + if sdkPath != nil { + Button(role: .destructive) { + removeSDK() + } label: { + Label("Remove the SDK", systemImage: "trash") + } + .disabled(working) } + } header: { + Text("Actions") } - Section(footer: Text("Reset removes the imported rootfs, the staged darwin SDK " - + "and any downloads. The app re-imports the bundled rootfs " - + "on the next boot.")) { - Button(role: .destructive) { - Task { await toolchain.reset() } + Section { + Button { + Task { await runSmokeTest() } } label: { - Label("Reset Toolchain", systemImage: "trash") + Label("Compile and link a test file", systemImage: "checkmark.seal") + } + .disabled(working || !capabilities.canCompile || sdkPath == nil) + + if let smokeResult { + Text(smokeResult) + .font(.footnote) + .foregroundStyle(smokeResult.hasPrefix("✓") ? .green : .red) } - .disabled(toolchain.isInstalling != nil) + } header: { + Text("Verify") + } footer: { + Text("Compiles a C file with clang, links it into an arm64 executable with " + + "ld64.lld, and — when the Swift frontend is linked — compiles a Swift " + + "file too. All in this process, against the installed SDK.") } - Section(footer: Text("If something dies without an explanation, share the log: " - + "the engine's own messages and XForge's install " - + "breadcrumbs are both in it.")) { - NavigationLink { - EngineLogView() - } label: { - Label("Engine log", systemImage: "doc.text.magnifyingglass") + if let message { + Section { + Label(message, systemImage: "info.circle") + .font(.footnote) + .foregroundStyle(.secondary) } } } .navigationTitle("Toolchain") - .task { await toolchain.refresh() } + .task { refresh() } .fileImporter( - isPresented: $importingXIP, - allowedContentTypes: [UTType(filenameExtension: "xip") ?? .data], + isPresented: $importing, + allowedContentTypes: Self.importableTypes, allowsMultipleSelection: false ) { result in - guard case .success(let urls) = result, let url = urls.first else { return } - Task { await installFromXIP(url) } - } - .fileImporter( - isPresented: $importingRootfs, - allowedContentTypes: [.archive, .gzip, .data], - allowsMultipleSelection: false - ) { result in - guard case .success(let urls) = result, let url = urls.first else { return } - Task { await toolchain.importRootfs(from: url) } - } - .confirmationDialog("Install the prebuilt Darwin SDK?", - isPresented: $confirmingDownload) { - Button("Download and install in the Terminal") { - Task { await installPrebuiltSDK() } + switch result { + case .success(let urls): + guard let url = urls.first else { return } + Task { await install(from: url) } + case .failure(let error): + // This used to `return` on anything that was not a success: an iCloud + // file the picker cannot download, a provider that refuses to copy it — + // nothing on screen, nothing in the log, and the user left believing + // the import did nothing. + XForgeLog.note("sdk import: the picker failed: \(error)") + Task { @MainActor in sdkError = error.localizedDescription } } + } + .confirmationDialog( + "Download the Darwin SDK?", + isPresented: $confirmingDownload + ) { + Button("Download") { Task { await downloadSDK() } } } message: { - Text("About 457 MB is downloaded inside the guest and installed there. " - + "It runs in the Terminal tab.") + Text("About 460 MB, downloaded into the app's container.") } } // MARK: - Rows - @ViewBuilder - private func row(_ component: ToolchainManager.Component) -> some View { - VStack(alignment: .leading, spacing: 6) { - HStack(spacing: 10) { - Image(systemName: component.icon) - .foregroundStyle(.tint) - .frame(width: 22) - VStack(alignment: .leading, spacing: 2) { - Text(component.rawValue).font(.subheadline.weight(.semibold)) - Text(component.blurb).font(.caption).foregroundStyle(.secondary) - } - Spacer() - statusBadge(component) - } - - HStack(spacing: 12) { - ForEach(actions(for: component), id: \.title) { action in - Button(action.title) { action.run() } - .font(.footnote) - .disabled(toolchain.isInstalling != nil || preparing != nil) - } - } - .padding(.leading, 32) + private func row(_ title: String, present: Bool) -> some View { + HStack { + Image(systemName: present ? "checkmark.circle.fill" : "circle.dashed") + .foregroundStyle(present ? .green : .secondary) + Text(title) + Spacer() + Text(present ? "linked" : "missing") + .font(.caption) + .foregroundStyle(present ? .green : .secondary) } - .padding(.vertical, 2) } - private struct RowAction { - let title: String - let run: () -> Void - } + // MARK: - Actions - private func actions(for component: ToolchainManager.Component) -> [RowAction] { - switch component { - case .rootfs: - // The bundled rootfs installs itself; a custom archive is the - // "Offline Imports" section below. - return [] - case .swift: - return [RowAction(title: "Install in Terminal") { - installInTerminal(.swift) - }] - case .xtool: - return [RowAction(title: "Install in Terminal") { - installInTerminal(.xtool) - }] - case .sdk: - return [ - RowAction(title: "From an Xcode.xip…") { importingXIP = true }, - RowAction(title: "Prebuilt download…") { confirmingDownload = true }, - ] + private func refresh() { + do { + let layout = try NativeSDK.layout() + sdkPath = layout.sdkRoot.path + sdkError = nil + } catch { + sdkPath = nil + sdkError = NativeSDK.isInstalled ? error.localizedDescription : nil } } - @ViewBuilder - private func statusBadge(_ component: ToolchainManager.Component) -> some View { - if toolchain.isInstalled(component) { - Label("Installed", systemImage: "checkmark") - .labelStyle(.iconOnly) - .font(.footnote.bold()) - .foregroundStyle(.green) - } else if component.livesInGuest && !toolchain.guestChecked { - Text("?").font(.footnote.bold()).foregroundStyle(.secondary) - } else { - Image(systemName: "circle.dashed") - .font(.footnote.bold()) - .foregroundStyle(.orange) + private func downloadSDK() async { + working = true + message = nil + defer { working = false } + do { + try await NativeSDK.installLatestPrebuilt() + refresh() + message = "Darwin SDK installed." + } catch { + sdkError = error.localizedDescription } } - // MARK: - Actions + /// Install whatever was picked, off the main actor, with progress. + /// + /// The work is minutes of blocking decompression, so it runs in a detached task. + /// Its progress comes back through a stream rather than a closure that writes to + /// this view's state: the closure handed to the detached task is `@Sendable`, and + /// capturing the view in one is the sort of thing that compiles until it does not + /// — the receiver is a `@MainActor` type, which is `Sendable`, so it can be + /// captured freely. Same shape as the build pipeline's stage streams. + @MainActor + private func install(from url: URL) async { + working = true + message = nil + sdkError = nil + + // The object, not the property wrapper: this is what the consumer task + // captures, so that no closure here has to capture the view. + let state = importState + state.begin("Reading \(url.lastPathComponent)…") - /// Prepare the guest and hand the component's install command to the Terminal. - private func installInTerminal(_ component: SystemComponents.Component) { - Task { - preparing = "Preparing the guest for \(component.title)…" - defer { preparing = nil } - do { - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(command(for: component), label: "Toolchain") - toolchain.message = "\(component.title): running in the Terminal tab." - } catch { - toolchain.message = error.localizedDescription + // Minutes of CPU, and iOS suspends an app whose screen locks — taking the + // extraction with it until the user comes back. Same guard the guest-era + // provisioning used for the same reason (`InstallAssertion`: idle timer plus + // a background task assertion). + let assertion = InstallAssertion.begin(reason: "sdk-import") + defer { assertion.end() } + + let (updates, continuation) = AsyncStream.makeStream() + let consumer = Task { @MainActor in + // A progress trail, not just a bar: "started and then nothing" is what a + // stall looks like in a shared log, and the last milestone says how far it + // got. Four lines per import, at most. + var milestone = -1 + for await update in updates { + state.update(fraction: update.fraction, status: update.message) + let reached = Int(update.fraction * 4) + if reached > milestone { + milestone = reached + XForgeLog.note("sdk import: \(Int(update.fraction * 100))%") + } } + state.finish() } + + do { + let report = try await Task.detached(priority: .userInitiated) { + defer { continuation.finish() } + return try NativeSDK.installImported(from: url) { continuation.yield($0) } + }.value + refresh() + message = Self.summary(for: report, named: url.lastPathComponent) + XForgeLog.note("sdk import: installed from \(url.lastPathComponent), " + + "files=\(report.files), warnings=\(report.warnings.count)") + } catch is CancellationError { + message = "Import cancelled." + } catch { + // The log is the only place a device failure can be read back from: the + // screen shows one line, the log gets the error with its case and values. + XForgeLog.note("sdk import failed: \(error)") + sdkError = error.localizedDescription + } + + continuation.finish() + await consumer.value // the consumer clears `isRunning` when the stream ends + working = false + + // The picker's copy of the archive is the app's to delete, and for an + // `Xcode.xip` it is most of the free space on the device. + await Task.detached(priority: .utility) { + NativeSDK.discardImportCopy(at: url) + }.value } - private func command(for component: SystemComponents.Component) -> String { - switch component { - case .glibc: - return SystemComponents.scriptCommand(.glibc) - case .xtool: - return SystemComponents.xtoolInstallCommand - case .swift: - return SystemComponents.swiftInstallCommand - case .darwinSDK: - // Always driven by a file or a download, never by a bare tap. - return SystemComponents.scriptCommand(.xtool) + /// What to say after an import: what landed, and anything that will bite later. + /// + /// An `.xip` import is minutes long, so "installed" on its own is not an answer + /// the user can check — the file count and size are, and they are also what makes + /// a report of a failure specific. + private static func summary(for report: NativeSDK.ImportReport, named name: String) -> String { + var lines = ["Darwin SDK installed from \(name)."] + if report.files > 0 { + let size = ByteCountFormatter.string(fromByteCount: report.bytes, countStyle: .file) + lines = ["Darwin SDK installed from \(name): \(report.files) files, \(size)."] } + lines.append(contentsOf: report.warnings) + return lines.joined(separator: " ") } - /// Copy the user's Xcode.xip into the guest's storage, then install it there. - private func installFromXIP(_ url: URL) async { - preparing = "Copying \(url.lastPathComponent) into the guest…" - defer { preparing = nil } + private func removeSDK() { do { - let command = try await toolchain.installSDKFromXcode(xip: url) - terminal.enqueue(command, label: "Toolchain") + try NativeSDK.remove() + refresh() + message = "Darwin SDK removed." } catch { - toolchain.message = error.localizedDescription + sdkError = error.localizedDescription } } - /// Download XForge's prebuilt Darwin SDK bundle inside the guest and install it. - private func installPrebuiltSDK() async { - preparing = "Resolving the latest Darwin SDK release…" - defer { preparing = nil } + /// Compile *and* link, in this process, against the installed SDK. + /// + /// A smoke test that only compiles cannot tell a working toolchain from one + /// whose objects the linker rejects, and the linker is the half that carries + /// the platform version and the entry point. The Swift frontend is exercised + /// too when it is linked, so the moment the frontend libraries arrive this + /// screen proves the whole chain rather than half of it. + private func runSmokeTest() async { + working = true + smokeResult = nil + defer { working = false } do { - let url = try await XForgeReleases.darwinSDKURL() - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(SystemComponents.darwinSDKDownloadCommand(from: url), - label: "Toolchain") - toolchain.message = "Downloading and installing the Darwin SDK in the Terminal tab." + let layout = try NativeSDK.layout() + smokeResult = try await Task.detached(priority: .userInitiated) { () -> String in + // Matches the default target of `NativeToolchain.compileC` + // (`arm64-apple-ios17.0.0`), which is also XForge's own floor. + let minimumIOSVersion = "17.0" + var lines: [String] = [] + + let object = try NativeToolchain.smokeCompile(sdk: layout.sdkRoot) + lines.append("✓ clang: \(object.lastPathComponent)") + + let executable = object + .deletingLastPathComponent() + .appendingPathComponent("xforge-smoke") + let linked = try NativeToolchain.linkMachO(arguments: [ + "-arch", "arm64", + "-platform_version", "ios", minimumIOSVersion, minimumIOSVersion, + "-syslibroot", layout.sdkRoot.path, + "-o", executable.path, + "-lSystem", + object.path + ]) + guard linked.succeeded else { + throw NativeToolchainError.link(linked.diagnostics) + } + let size = (try? executable.resourceValues(forKeys: [.fileSizeKey]))?.fileSize ?? 0 + lines.append("✓ ld64.lld: \(executable.lastPathComponent) (\(size) bytes)") + + if NativeToolchain.isSwiftAvailable { + let swiftObject = try NativeToolchain.smokeCompileSwift(sdk: layout) + lines.append("✓ swift-frontend: \(swiftObject.lastPathComponent)") + } else { + lines.append("· swift-frontend: not linked, so not tested") + } + return lines.joined(separator: "\n") + }.value } catch { - toolchain.message = error.localizedDescription + smokeResult = error.localizedDescription } } } + +/// Progress of a Darwin SDK import, as the screen renders it. +/// +/// A reference type and a plain `ObservableObject` — the shape the rest of the app +/// uses for state a long job updates — rather than two `@State` fields, because the +/// consumer of the progress stream has to hold *something* across an actor boundary: +/// this object is `@MainActor`, and therefore `Sendable`, while the view is not. +/// Holding the view in a task's closure is the kind of capture that compiles until +/// it does not. +@MainActor +final class SDKImportState: ObservableObject { + @Published private(set) var fraction: Double = 0 + @Published private(set) var status = "" + @Published private(set) var isRunning = false + + func begin(_ status: String) { + fraction = 0 + self.status = status + isRunning = true + } + + func update(fraction: Double, status: String) { + self.fraction = fraction + self.status = status + } + + func finish() { + isRunning = false + } +} diff --git a/AppTests/NativeBuildTests.swift b/AppTests/NativeBuildTests.swift new file mode 100644 index 0000000..10a12da --- /dev/null +++ b/AppTests/NativeBuildTests.swift @@ -0,0 +1,366 @@ +import XCTest +@testable import XForge + +/// Tests for the native build path's pure half: reading a project into a plan and +/// turning that plan into compiler/linker argument lists. +/// +/// This is the only part of the native pipeline a simulator can exercise — the +/// compiler itself is linked into device builds only — and it is deliberately the +/// largest part, because a wrong argument is invisible until a build runs on a +/// phone with a real SDK. +final class NativeBuildTests: XCTestCase { + private var root: URL! + + override func setUpWithError() throws { + root = FileManager.default.temporaryDirectory + .appendingPathComponent("xforge-native-tests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: root) + } + + // MARK: - Fixtures + + private func sdkFixture() -> NativeSDKLayout { + let bundle = root.appendingPathComponent("darwin.artifactbundle", isDirectory: true) + return NativeSDKLayout( + bundle: bundle, + sdkRoot: bundle.appendingPathComponent("iPhoneOS.sdk", isDirectory: true), + swiftResources: bundle.appendingPathComponent("usr/lib/swift", isDirectory: true), + swiftStaticResources: bundle.appendingPathComponent("usr/lib/swift_static", isDirectory: true), + platformLibrarySearchPaths: [bundle.appendingPathComponent("usr/lib", isDirectory: true)] + ) + } + + private func appInfo() -> AppInfo { + AppInfo( + bundleIdentifier: "com.example.Demo", + displayName: "Demo", + version: "2.1", + buildNumber: "7", + minimumOSVersion: "17.0" + ) + } + + /// A package laid out the way `xtool new` and `XForge` both produce one. + @discardableResult + private func writePackage( + name: String = "Demo", + packageSwift: String? = nil, + files: [String] = ["Demo.swift"] + ) throws -> URL { + let project = root.appendingPathComponent(name, isDirectory: true) + let sources = project.appendingPathComponent("Sources/\(name)", isDirectory: true) + try FileManager.default.createDirectory(at: sources, withIntermediateDirectories: true) + + let manifest = packageSwift ?? """ + // swift-tools-version: 6.0 + import PackageDescription + + let package = Package( + name: "\(name)", + products: [.executable(name: "\(name)", targets: ["\(name)"])], + targets: [.executableTarget(name: "\(name)", path: "Sources/\(name)")] + ) + """ + try manifest.write( + to: project.appendingPathComponent("Package.swift"), + atomically: true, + encoding: .utf8 + ) + for file in files { + try "// \(file)\n".write( + to: sources.appendingPathComponent(file), + atomically: true, + encoding: .utf8 + ) + } + return project + } + + // MARK: - Plan factory + + func testPlanFindsSourcesAndModule() throws { + let project = try writePackage(files: ["Demo.swift", "README.md", "helper.c"]) + + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdkFixture() + ) + + XCTAssertEqual(plan.moduleName, "Demo") + XCTAssertEqual(plan.executableName, "Demo") + XCTAssertEqual(plan.appName, "Demo") + XCTAssertEqual(plan.swiftSources.count, 1) + XCTAssertEqual(plan.clangSources.map(\.url.lastPathComponent), ["helper.c"]) + XCTAssertEqual(plan.clangSources.first?.language, "c") + // A Swift app links UIKit; a C-only one does not, which is why the plan + // decides this rather than the executor. + XCTAssertTrue(plan.frameworks.contains("UIKit")) + XCTAssertTrue(plan.frameworks.contains("Foundation")) + XCTAssertTrue(plan.hasSwift) + } + + func testPlanRejectsPackageWithoutSourcesDirectory() throws { + let project = root.appendingPathComponent("Empty", isDirectory: true) + try FileManager.default.createDirectory(at: project, withIntermediateDirectories: true) + try "// swift-tools-version: 6.0\n" + .write(to: project.appendingPathComponent("Package.swift"), + atomically: true, encoding: .utf8) + + XCTAssertThrowsError(try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdkFixture() + )) { error in + guard case NativeBuildPlanError.missingTargetsDirectory = error else { + return XCTFail("unexpected error: \(error)") + } + } + } + + func testPlanRejectsMissingManifest() throws { + XCTAssertThrowsError(try NativeBuildPlanFactory.makePlan( + root: root, + appInfo: appInfo(), + configuration: .debug, + sdk: sdkFixture() + )) { error in + guard case NativeBuildPlanError.missingPackageManifest = error else { + return XCTFail("unexpected error: \(error)") + } + } + } + + /// The load-bearing refusal: a package manager cannot run inside an app that + /// may not spawn processes, so a project with dependencies fails by name. + func testPlanRefusesSwiftPackageDependencies() throws { + let manifest = """ + // swift-tools-version: 6.0 + import PackageDescription + + let package = Package( + name: "Demo", + dependencies: [ + .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.0.0") + ], + targets: [.executableTarget(name: "Demo", path: "Sources/Demo")] + ) + """ + let project = try writePackage(packageSwift: manifest) + + XCTAssertThrowsError(try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdkFixture() + )) { error in + guard case NativeBuildPlanError.unsupportedSwiftPackageDependencies(let names) = error else { + return XCTFail("unexpected error: \(error)") + } + XCTAssertEqual(names, ["swift-argument-parser"]) + XCTAssertTrue(error.localizedDescription.contains("swift-argument-parser")) + } + } + + func testDependencyScanIgnoresLocalTargets() throws { + let manifest = """ + // swift-tools-version: 6.0 + import PackageDescription + + let package = Package( + name: "Demo", + targets: [.executableTarget(name: "Demo", path: "Sources/Demo")] + ) + """ + let url = root.appendingPathComponent("Package.swift") + try manifest.write(to: url, atomically: true, encoding: .utf8) + XCTAssertEqual(try NativeBuildPlanFactory.swiftPackageDependencies(manifestURL: url), []) + } + + func testClangLanguageMapping() { + XCTAssertEqual(NativeClangSource.language(for: URL(fileURLWithPath: "/a/b.c")), "c") + XCTAssertEqual(NativeClangSource.language(for: URL(fileURLWithPath: "/a/b.m")), "objective-c") + XCTAssertEqual(NativeClangSource.language(for: URL(fileURLWithPath: "/a/b.mm")), "objective-c++") + XCTAssertEqual(NativeClangSource.language(for: URL(fileURLWithPath: "/a/b.cpp")), "c++") + XCTAssertNil(NativeClangSource.language(for: URL(fileURLWithPath: "/a/b.swift"))) + } + + // MARK: - swift-frontend arguments + + func testSwiftFrontendArgumentsMatchTheEmbeddedFrontendContract() throws { + let project = try writePackage() + let sdk = sdkFixture() + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdk + ) + let source = try XCTUnwrap(plan.swiftSources.first) + let object = root.appendingPathComponent("Demo.o") + + let arguments = try NativeToolchainInvocation.swiftFrontendArguments( + source: source, + object: object, + plan: plan, + sdk: sdk + ) + + let target = "arm64-apple-ios\(plan.minimumIOSVersion)" + XCTAssertEqual(NativeToolchainInvocation.targetTriple(minimumIOSVersion: "17.0"), + "arm64-apple-ios17.0") + XCTAssertTrue(arguments.contains(target)) + XCTAssertTrue(arguments.contains("-sdk")) + XCTAssertTrue(arguments.contains(sdk.sdkRoot.path)) + XCTAssertTrue(arguments.contains("-resource-dir")) + XCTAssertTrue(arguments.contains("-module-name")) + XCTAssertTrue(arguments.contains(plan.moduleName)) + XCTAssertTrue(arguments.contains("-o")) + XCTAssertTrue(arguments.contains(object.path)) + // -parse-as-library for a file that is not main.swift, and never the + // driver's own `-frontend`: performFrontend receives argv[2…] and would + // reject it. + XCTAssertEqual(arguments.first, "-parse-as-library") + XCTAssertFalse(arguments.contains("-frontend")) + XCTAssertTrue(arguments.contains("-Onone")) + } + + func testMainSwiftDoesNotGetParseAsLibrary() throws { + let project = try writePackage(files: ["main.swift"]) + let sdk = sdkFixture() + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .release, + sdk: sdk + ) + let source = try XCTUnwrap(plan.swiftSources.first) + + let arguments = try NativeToolchainInvocation.swiftFrontendArguments( + source: source, + object: root.appendingPathComponent("main.o"), + plan: plan, + sdk: sdk + ) + + XCTAssertFalse(arguments.contains("-parse-as-library")) + XCTAssertTrue(arguments.contains("-O")) + XCTAssertFalse(arguments.contains("-Onone")) + } + + func testSwiftFrontendArgumentsRequireSwiftResourceDirectory() throws { + let project = try writePackage() + var sdk = sdkFixture() + sdk = NativeSDKLayout( + bundle: sdk.bundle, + sdkRoot: sdk.sdkRoot, + swiftResources: nil, + swiftStaticResources: sdk.swiftStaticResources, + platformLibrarySearchPaths: sdk.platformLibrarySearchPaths + ) + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdk + ) + + XCTAssertThrowsError(try NativeToolchainInvocation.swiftFrontendArguments( + source: try XCTUnwrap(plan.swiftSources.first), + object: root.appendingPathComponent("Demo.o"), + plan: plan, + sdk: sdk + )) + } + + // MARK: - Linker arguments + + func testLinkerArgumentsDescribeAnIOSExecutable() throws { + let project = try writePackage() + let sdk = sdkFixture() + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdk + ) + let object = root.appendingPathComponent("Demo.o") + let executable = root.appendingPathComponent("Demo") + + let arguments = NativeToolchainInvocation.linkerArguments( + objects: [object], + plan: plan, + sdk: sdk, + executable: executable, + fileExists: { _ in true } + ) + + XCTAssertEqual(arguments[0], "-arch") + XCTAssertEqual(arguments[1], "arm64") + XCTAssertTrue(arguments.contains("-platform_version")) + let platformIndex = try XCTUnwrap(arguments.firstIndex(of: "-platform_version")) + XCTAssertEqual(arguments[platformIndex + 1], "ios") + XCTAssertEqual(arguments[platformIndex + 2], plan.minimumIOSVersion) + XCTAssertTrue(arguments.contains("-syslibroot")) + XCTAssertTrue(arguments.contains(sdk.sdkRoot.path)) + XCTAssertTrue(arguments.contains("-lswiftCore")) + XCTAssertTrue(arguments.contains("-lSystem")) + XCTAssertTrue(arguments.contains("-framework")) + XCTAssertTrue(arguments.contains(object.path)) + XCTAssertTrue(arguments.contains(executable.path)) + XCTAssertEqual(arguments[arguments.count - 1], object.path) + } + + func testLinkerSkipsSearchPathsThatDoNotExist() throws { + let project = try writePackage() + let sdk = sdkFixture() + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdk + ) + + let arguments = NativeToolchainInvocation.linkerArguments( + objects: [root.appendingPathComponent("Demo.o")], + plan: plan, + sdk: sdk, + executable: root.appendingPathComponent("Demo"), + fileExists: { _ in false } + ) + + // -lswiftCore stays (a Swift binary cannot run without it) but nothing + // that names a file that is not there. + XCTAssertTrue(arguments.contains("-lswiftCore")) + XCTAssertFalse(arguments.contains("-L")) + XCTAssertFalse(arguments.contains("-lswift_Concurrency")) + } + + func testCLanguageLibraryPathsComeFromThePlan() throws { + let project = try writePackage(files: ["main.c"]) + var sdk = sdkFixture() + sdk = NativeSDKLayout( + bundle: sdk.bundle, + sdkRoot: sdk.sdkRoot, + swiftResources: sdk.swiftResources, + swiftStaticResources: sdk.swiftStaticResources, + platformLibrarySearchPaths: sdk.platformLibrarySearchPaths + ) + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdk + ) + + XCTAssertFalse(plan.hasSwift) + XCTAssertFalse(plan.frameworks.contains("UIKit")) + XCTAssertTrue(plan.librarySearchPaths.isEmpty) + } +} diff --git a/AppTests/ProjectTests.swift b/AppTests/ProjectTests.swift index e4b22bc..fdeaa5a 100644 --- a/AppTests/ProjectTests.swift +++ b/AppTests/ProjectTests.swift @@ -3,10 +3,10 @@ import XCTest final class ProjectTests: XCTestCase { func testProjectDefaults() { - let p = Project(name: "Demo", rootPath: "/root/projects/Demo") + let p = Project(name: "Demo", rootPath: "projects/Demo") XCTAssertEqual(p.organizationIdentifier, "com.example") - XCTAssertEqual(p.packageManifestPath, "/root/projects/Demo/Package.swift") - XCTAssertEqual(p.ipaOutputPath, "/root/projects/Demo/.build/xforge-Demo.ipa") + XCTAssertEqual(p.packageManifestPath, "projects/Demo/Package.swift") + XCTAssertEqual(p.ipaOutputPath, "projects/Demo/.build/xforge-Demo.ipa") } func testBuildConfigurationRawValues() { @@ -14,11 +14,6 @@ final class ProjectTests: XCTestCase { XCTAssertEqual(BuildConfiguration.release.rawValue, "release") } - func testGuestShellQuoteEscapesSingleQuotes() { - XCTAssertEqual(GuestShell.quote("one'two"), "'one'\\''two'") - XCTAssertEqual(GuestShell.environment(["TOKEN": "a b'c"]), "TOKEN='a b'\\''c' ") - } - func testProjectNameValidationAndSafePath() throws { let name = try Project.validatedName("Demo-App_2") XCTAssertEqual(name, "Demo-App_2") diff --git a/AppTests/ReleaseResolutionTests.swift b/AppTests/ReleaseResolutionTests.swift index 1984eab..878247e 100644 --- a/AppTests/ReleaseResolutionTests.swift +++ b/AppTests/ReleaseResolutionTests.swift @@ -6,6 +6,9 @@ import XCTest /// The darwin SDK is published under its own `darwin-sdk-*` series, so /// `releases/latest/download/` 404s whenever the newest release is an /// XForge IPA. These tests pin the behaviour that replaced it. +/// +/// The tests that lived beside these — the guest rootfs naming, the in-guest +/// toolchain manager, the embedded Linux executor — went with the guest. @MainActor final class ReleaseResolutionTests: XCTestCase { @@ -65,245 +68,10 @@ final class ReleaseResolutionTests: XCTestCase { in: fixtures, tagPrefix: "alpine-rootfs-", assetName: "alpine-rootfs.tar.xz") - XCTAssertNil(url, "the rootfs is bundled in the app, not published as an asset") + XCTAssertNil(url, "the guest rootfs is gone; nothing should resolve it") } func testUnwrapsTheCorrectXForgeRepo() { XCTAssertEqual(XForgeReleases.repository, "R0GUEEE/XForge") } - - func testOnlyTheBundledAlpineRootfsIsEligibleForBoot() { - // The bundled root is a ZIP of an already-converted fakefs, not a tarball - // the app imports: the conversion happens at build time - // (EmbeddedLinux/build-rootfs.sh), so first launch is an unzip. - XCTAssertEqual(RootfsInstaller.archiveName, "alpine-rootfs") - XCTAssertEqual(RootfsInstaller.archiveExtension, "zip") - XCTAssertEqual(RootfsInstaller.rootName, "alpine-3.21") - // Roots from the earlier layouts must not be reused. - XCTAssertTrue(RootfsInstaller.legacyRootNames.contains("alpine")) - XCTAssertTrue(RootfsInstaller.legacyRootNames.contains("alpine-3.24.2")) - XCTAssertFalse(RootfsInstaller.legacyRootNames.contains(RootfsInstaller.rootName)) - } - - func testCurrentRootRemovesTheLegacyAlpineInstall() throws { - let roots = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-roots-\(UUID().uuidString)", isDirectory: true) - defer { try? FileManager.default.removeItem(at: roots) } - - let current = RootfsInstaller.installedRoot(in: roots) - try FileManager.default.createDirectory( - at: current.appendingPathComponent("data", isDirectory: true), - withIntermediateDirectories: true - ) - try Data([1]).write(to: current.appendingPathComponent("meta.db")) - - let legacy = roots.appendingPathComponent("alpine", isDirectory: true) - try FileManager.default.createDirectory(at: legacy, withIntermediateDirectories: true) - - XCTAssertEqual(try RootfsInstaller.installIfNeeded(into: roots), current) - XCTAssertFalse(FileManager.default.fileExists(atPath: legacy.path)) - } -} - -/// A `LinuxVM` that answers from a script, so the toolchain probes can be tested -/// without booting the real guest. -@MainActor -final class StubLinuxVM: LinuxVM { - var isBooted = true - var bootCount = 0 - /// Commands that should report success (exit 0). - var succeeding: [String] = [] - private(set) var ranCommands: [String] = [] - - func boot() async throws { bootCount += 1; isBooted = true } - - func prepareRootfs() async {} - - func run( - _ command: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - ranCommands.append(command) - return succeeding.contains(where: { command.contains($0) }) ? 0 : 1 - } - - func copyOut(guestPath: String, to hostURL: URL) async throws {} - func copyIn(hostURL: URL, to guestPath: String) async throws {} - - // MARK: - Interactive shell - - /// A fake console session that records what is sent to it and can be made to - /// emit output, so the terminal's behaviour is testable without a guest. - func startInteractiveShell( - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> any InteractiveShellSession { - let shell = StubShellSession(onOutput: onOutput) - startedShells.append(shell) - return shell - } - - private(set) var startedShells: [StubShellSession] = [] -} - -/// Records what the terminal writes to the console, and can emit output back. -@MainActor -final class StubShellSession: InteractiveShellSession { - /// init owns the console the fake session speaks for. - let pid: Int32 = 4242 - private(set) var isRunning = false - /// Everything written to the console, in order. - private(set) var received: [String] = [] - private(set) var interrupts = 0 - private(set) var stopped = false - private(set) var sizes: [(cols: Int, rows: Int)] = [] - - private let onOutput: @Sendable (String) -> Void - - init(onOutput: @escaping @Sendable (String) -> Void) { - self.onOutput = onOutput - self.isRunning = true - } - - @discardableResult - func send(_ text: String) -> Bool { - guard isRunning else { return false } - received.append(text) - return true - } - - /// Ctrl-C is the byte again: the guest's tty is what turns it into SIGINT, so - /// the session records it as a byte and counts the intent. - func interruptForeground() async { - interrupts += 1 - received.append(String(UnicodeScalar(0x03))) - } - - @discardableResult - func sendControl(_ scalar: UInt8) -> Bool { - guard isRunning else { return false } - received.append(String(UnicodeScalar(scalar))) - return true - } - - func resize(cols: Int, rows: Int) { - sizes.append((cols, rows)) - } - - func stop() { - isRunning = false - stopped = true - } - - /// Simulate the guest producing output. - func emit(_ text: String) { onOutput(text) } -} - -@MainActor -final class ToolchainManagerTests: XCTestCase { - - func testProbeFindsInstalledComponentsInTheGuest() async { - let vm = StubLinuxVM() - vm.succeeding = ["command -v swift", "command -v xtool", "swift sdk list"] - let manager = ToolchainManager(vm: vm) - - await manager.refresh(probeGuest: true) - - XCTAssertTrue(manager.guestChecked) - XCTAssertTrue(manager.isInstalled(.swift)) - XCTAssertTrue(manager.isInstalled(.xtool)) - XCTAssertTrue(manager.isInstalled(.sdk)) - XCTAssertEqual(vm.bootCount, 1, "probing should boot the guest exactly once") - } - - func testProbeReportsMissingComponents() async { - let vm = StubLinuxVM() - vm.succeeding = [] - let manager = ToolchainManager(vm: vm) - - await manager.refresh(probeGuest: true) - - XCTAssertTrue(manager.guestChecked) - XCTAssertFalse(manager.isInstalled(.swift)) - XCTAssertFalse(manager.isInstalled(.xtool)) - XCTAssertFalse(manager.isInstalled(.sdk)) - } - - func testRefreshWithoutProbeDoesNotBootOrInventGuestState() async { - let vm = StubLinuxVM() - vm.isBooted = false - let manager = ToolchainManager(vm: vm) - - await manager.refresh() - - XCTAssertEqual(vm.bootCount, 0, "a plain refresh must not boot the guest") - XCTAssertFalse(manager.guestChecked) - XCTAssertFalse(manager.isInstalled(.swift)) - } - - func testEveryComponentIsAccountedFor() { - // A component that is neither bundled nor guest-side would silently never - // be checked; make sure the enum and the split stay in step. - for component in ToolchainManager.Component.allCases { - XCTAssertFalse(component.rawValue.isEmpty) - XCTAssertFalse(component.blurb.isEmpty) - XCTAssertFalse(component.icon.isEmpty) - } - XCTAssertTrue(ToolchainManager.Component.rootfs.livesInGuest) - XCTAssertTrue(ToolchainManager.Component.swift.livesInGuest) - XCTAssertTrue(ToolchainManager.Component.sdk.livesInGuest) - } - - func testDownloadErrorsDescribeTheFailure() { - let http = DownloadError.http(status: 404, url: URL(string: "https://example.com/a.zip")!) - XCTAssertTrue(http.localizedDescription.contains("404")) - XCTAssertTrue(http.localizedDescription.contains("a.zip")) - - let missing = DownloadError.assetNotFound("darwin.artifactbundle.zip in any darwin-sdk-* release") - XCTAssertTrue(missing.localizedDescription.contains("darwin.artifactbundle.zip")) - } - - func testXcodeImportStagesTheXIPInTheGuestAndHandsBackTheInstallCommand() async throws { - let xip = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-sdk-test-\(UUID().uuidString).xip") - try Data().write(to: xip) - defer { try? FileManager.default.removeItem(at: xip) } - - let vm = StubLinuxVM() - vm.succeeding = ["mkdir -p", "wc -c"] - let manager = ToolchainManager(vm: vm) - - let command = try await manager.installSDKFromXcode(xip: xip) - - // The xip goes into the guest's own storage, and xtool is pointed at that - // path: current xtool installs an Xcode.xip directly, so there is no - // host-side unpack step and no obsolete artifactbundle hand-off. - XCTAssertTrue(vm.ranCommands.contains { $0.contains("mkdir -p '/root/xforge/xip'") }, - "the guest storage directory should be created: \(vm.ranCommands)") - XCTAssertTrue(command.contains("xtool sdk install"), - "the guest should install the xip with xtool: \(command)") - XCTAssertFalse(command.contains("xtool sdk build")) - XCTAssertTrue(command.contains("/root/xforge/xip/"), - "the command must point at the guest's own copy: \(command)") - XCTAssertTrue(command.contains("swift sdk list"), - "the install should verify itself: \(command)") - } - - func testBuildBootstrapDoesNotProvisionAMissingToolchain() async throws { - let vm = StubLinuxVM() - let executor = EmbeddedLinuxExecutor( - vm: vm, - stagingDir: FileManager.default.temporaryDirectory - ) - let stream = try await executor.bootstrap() - - do { - for try await _ in stream {} - } catch { - // The current implementation throws after it tries the installer; - // the replacement reports a missing toolchain as a failed event. - } - - XCTAssertFalse(vm.ranCommands.contains { $0.contains("install-toolchain.sh") }) - } } diff --git a/AppTests/TerminalBufferTests.swift b/AppTests/TerminalBufferTests.swift deleted file mode 100644 index f964f63..0000000 --- a/AppTests/TerminalBufferTests.swift +++ /dev/null @@ -1,139 +0,0 @@ -import XCTest -@testable import XForge - -/// The terminal's screen model is the part of the Terminal tab that cannot be -/// checked by looking at the app: the guest is a real Linux system, so its output -/// arrives as carriage returns, backspaces and SGR colour that have to be turned -/// into a screen. These tests feed it what the guest actually sends. -@MainActor -final class TerminalBufferTests: XCTestCase { - - func testPlainOutputIsSplitIntoScreenLines() { - let buffer = TerminalBuffer() - buffer.feed("hello\nworld") - - XCTAssertEqual(buffer.lines.map(\.text), ["hello"]) - XCTAssertEqual(buffer.current.text, "world") - } - - func testCarriageReturnOverwritesTheCurrentLine() { - let buffer = TerminalBuffer() - buffer.feed("100%\r50%") - - // What a terminal shows: the new text written over the old one, so the - // column the old text occupied but the new one does not keeps its - // character. - XCTAssertEqual(buffer.current.text, "50%%") - XCTAssertTrue(buffer.lines.isEmpty) - } - - func testCarriageReturnNewlineIsOneNewline() { - let buffer = TerminalBuffer() - buffer.feed("abc\r\n") - - XCTAssertEqual(buffer.lines.map(\.text), ["abc"]) - XCTAssertTrue(buffer.current.isEmpty) - } - - func testCarriageReturnAndNewlineSplitAcrossChunksIsStillOneNewline() { - // The other half of the same problem: CR and LF can also arrive in - // separate chunks, where CR is held until the next character arrives. - let buffer = TerminalBuffer() - buffer.feed("abc\r") - buffer.feed("\nnext") - - XCTAssertEqual(buffer.lines.map(\.text), ["abc"]) - XCTAssertEqual(buffer.current.text, "next") - } - - func testBackspaceErasesThePreviousCharacter() { - let buffer = TerminalBuffer() - buffer.feed("ab\u{08}c") - - XCTAssertEqual(buffer.current.text, "ac") - } - - func testTabAdvancesToTheNextTabStop() { - let buffer = TerminalBuffer() - buffer.feed("a\tb") - - XCTAssertEqual(buffer.current.text, "a b") - } - - func testColourIsKeptAsStyleAndNotAsText() { - let buffer = TerminalBuffer() - buffer.feed("\u{1b}[1;31mred\u{1b}[0m plain") - - XCTAssertEqual(buffer.current.text, "red plain") - XCTAssertEqual(buffer.current.cells.first?.style.foreground, .index(1)) - XCTAssertEqual(buffer.current.cells.first?.style.bold, true) - // The reset returns the rest of the line to the default style. - XCTAssertEqual(buffer.current.cells.last?.style, .plain) - } - - func testTrueColourAndExtendedPalette() { - let buffer = TerminalBuffer() - buffer.feed("\u{1b}[38;2;10;20;30mtruecolour") - XCTAssertEqual(buffer.current.cells.first?.style.foreground, .rgb(10, 20, 30)) - - buffer.feed("\u{1b}[0m\u{1b}[38;5;208mindexed") - XCTAssertEqual(buffer.current.cells.last?.style.foreground, .index(208)) - } - - func testClearScreenRemovesTheScrollback() { - let buffer = TerminalBuffer() - buffer.feed("one\ntwo\n") - buffer.feed("\u{1b}[2J") - - XCTAssertTrue(buffer.lines.isEmpty) - XCTAssertTrue(buffer.current.isEmpty) - XCTAssertTrue(buffer.isAtStart) - } - - func testEraseInLineLeavesTheScrollbackAlone() { - let buffer = TerminalBuffer() - buffer.feed("kept\npartly") - buffer.feed("\u{1b}[K") - - XCTAssertEqual(buffer.lines.map(\.text), ["kept"]) - // The cursor sits at the end of the text, so there is nothing after it to - // erase — the line is unchanged and the scrollback is untouched. - XCTAssertEqual(buffer.current.text, "partly") - } - - func testAnEscapeSequenceSplitAcrossChunksStillApplies() { - // Output arrives in whatever size the transport hands over; a sequence - // must survive being cut in half. - let buffer = TerminalBuffer() - buffer.feed("\u{1b}[3") - buffer.feed("1mgreen") - - XCTAssertEqual(buffer.current.text, "green") - XCTAssertEqual(buffer.current.cells.first?.style.foreground, .index(1)) - } - - func testUnsupportedSequencesAreSwallowedWithoutLosingText() { - let buffer = TerminalBuffer() - buffer.feed("\u{1b}[1;1Hmove\u{1b}]0;window title\u{07}after") - - XCTAssertEqual(buffer.current.text, "moveafter") - } - - func testScrollbackIsCapped() { - let buffer = TerminalBuffer() - buffer.maxLines = 10 - for index in 0..<50 { buffer.feed("line \(index)\n") } - - XCTAssertEqual(buffer.lines.count, 10) - XCTAssertEqual(buffer.lines.last?.text, "line 49") - } - - func testHostLinesAreStyledWithoutLeakingIntoGuestOutput() { - let buffer = TerminalBuffer() - buffer.appendLine("$ ls", style: TerminalStyle(foreground: .index(10))) - buffer.feed("guest output") - - XCTAssertEqual(buffer.lines.first?.text, "$ ls") - XCTAssertEqual(buffer.current.cells.first?.style, .plain) - } -} diff --git a/AppTests/TerminalKeyConfigurationTests.swift b/AppTests/TerminalKeyConfigurationTests.swift deleted file mode 100644 index 205c1d8..0000000 --- a/AppTests/TerminalKeyConfigurationTests.swift +++ /dev/null @@ -1,91 +0,0 @@ -import XCTest -@testable import XForge - -/// The extra-keys bar is configurable, with one key that deliberately is not. -@MainActor -final class TerminalKeyConfigurationTests: XCTestCase { - - private func makeConfiguration() -> TerminalKeyConfiguration { - // Start from a clean slate: the configuration persists in UserDefaults, - // so a test that ran before this one would otherwise leak into it. - UserDefaults.standard.removeObject(forKey: "org.xforge.terminal.hiddenKeys") - return TerminalKeyConfiguration() - } - - func testHideKeyboardIsAlwaysVisible() { - let configuration = makeConfiguration() - XCTAssertTrue(configuration.isVisible(.hideKeyboard)) - - // Even if something tries to switch it off: without it there is no way - // back to the screen once the keyboard covers it. - configuration.setVisible(false, for: .hideKeyboard) - XCTAssertTrue(configuration.isVisible(.hideKeyboard), - "the hide-keyboard key is pinned on purpose") - XCTAssertTrue(configuration.visibleKeys.contains(.hideKeyboard)) - } - - func testKeysStartVisible() { - let configuration = makeConfiguration() - for key in TerminalKey.allCases { - XCTAssertTrue(configuration.isVisible(key), "\(key.rawValue) should start shown") - } - } - - func testHidingAKeyRemovesItFromTheBar() { - let configuration = makeConfiguration() - configuration.setVisible(false, for: .pipe) - - XCTAssertFalse(configuration.isVisible(.pipe)) - XCTAssertFalse(configuration.visibleKeys.contains(.pipe)) - XCTAssertTrue(configuration.visibleKeys.contains(.slash), "others are unaffected") - } - - func testHidingPersistsBetweenInstances() { - let configuration = makeConfiguration() - configuration.setVisible(false, for: .colon) - - // A new instance reads what was stored — otherwise the choice would be - // lost every time the terminal is reopened. - let reloaded = TerminalKeyConfiguration() - XCTAssertFalse(reloaded.isVisible(.colon)) - } - - func testResetShowsEverything() { - let configuration = makeConfiguration() - configuration.setVisible(false, for: .bang) - configuration.setVisible(false, for: .previous) - configuration.resetToDefaults() - - XCTAssertEqual(configuration.visibleKeys.count, TerminalKey.allCases.count) - } - - func testBarOrderIsStable() { - // The keys keep their declared order regardless of what is hidden, so - // hiding one does not shuffle the rest. - let configuration = makeConfiguration() - let before = configuration.visibleKeys - configuration.setVisible(false, for: .tab) - let after = configuration.visibleKeys - - XCTAssertEqual(after, before.filter { $0 != .tab }) - } - - func testEveryKeyHasALabelOrASymbol() { - // A key that renders as neither would be an invisible button. - for key in TerminalKey.allCases { - let described = !key.label.isEmpty || key.symbol != nil - XCTAssertTrue(described, "\(key.rawValue) has nothing to draw") - } - } - - func testPunctuationKeysInsertTheirOwnCharacter() { - // The label and the inserted text are declared separately, so they can - // drift; this pins the ones where drift would be a real bug. - for key in [TerminalKey.dash, .dot, .slash, .colon, .bang, .pipe] { - XCTAssertEqual(key.label.count, 1, "\(key.rawValue) should be one character") - } - XCTAssertEqual(TerminalKey.pipe.label, "|") - XCTAssertEqual(TerminalKey.bang.label, "!") - XCTAssertEqual(TerminalKey.dash.label, "-") - } -} diff --git a/AppTests/TerminalSessionTests.swift b/AppTests/TerminalSessionTests.swift deleted file mode 100644 index 1ce3f2d..0000000 --- a/AppTests/TerminalSessionTests.swift +++ /dev/null @@ -1,165 +0,0 @@ -import XCTest -@testable import XForge - -/// The terminal is the guest's console: what is typed goes into the guest's tty, -/// the guest's own line discipline and shell do the editing and echoing, and what -/// the tty produces is the screen. -/// -/// These tests pin the parts that are easy to get subtly wrong — that a typed -/// line reaches the console verbatim (no host-side shell wrapping, no host-side -/// echoing), that an interrupt is the Ctrl-C *byte* for the tty to turn into a -/// signal, and that the guest is told how big the screen is. -@MainActor -final class TerminalSessionTests: XCTestCase { - - /// A session wired to a stub guest, with its transcript kept out of the way. - private func makeSession() async -> (TerminalSession, StubLinuxVM) { - let vm = StubLinuxVM() - let session = TerminalSession(makeVM: { vm }) - await session.boot() - return (session, vm) - } - - private func shell(of vm: StubLinuxVM) throws -> StubShellSession { - try XCTUnwrap(vm.startedShells.first, "the terminal should have attached to a console") - } - - func testBootingAttachesToTheConsoleOnce() async throws { - let (session, vm) = await makeSession() - await session.boot() - - XCTAssertEqual(vm.startedShells.count, 1, "booting twice should reuse the session") - XCTAssertTrue(session.isBooted) - XCTAssertTrue(try shell(of: vm).isRunning) - } - - func testTypedLineGoesToTheConsoleVerbatim() async throws { - let (session, vm) = await makeSession() - session.sendRaw("echo hello\n") - - // Exactly what was typed, newline included: the tty and the shell are what - // interpret it. - XCTAssertEqual(try shell(of: vm).received, ["echo hello\n"]) - } - - func testTheLineIsNotWrappedInAShellInvocation() async throws { - // The old design composed `sh -c ` per line. Nothing is wrapped - // now: the console receives exactly what was typed, which is what lets - // shell state (cd, variables) persist and lets a program read stdin. - let (session, vm) = await makeSession() - session.sendRaw("cd /tmp && pwd\n") - - let sent = try XCTUnwrap(try shell(of: vm).received.first) - XCTAssertEqual(sent, "cd /tmp && pwd\n") - XCTAssertFalse(sent.contains("/bin/sh"), "the host must not wrap the line in a shell call") - XCTAssertFalse(sent.contains("__XFORGE_PWD__"), - "the host no longer needs to ask the shell where it is") - } - - func testKeystrokesReachTheConsoleOneChunkAtATime() async throws { - // A terminal sends what was typed, not whole lines: the guest has to see - // each keystroke so its line discipline can echo it, and so `read` in a - // running program can consume it character by character. - let (session, vm) = await makeSession() - for text in ["l", "s", "\n"] { - session.sendRaw(text) - } - - XCTAssertEqual(try shell(of: vm).received, ["l", "s", "\n"]) - } - - func testMultipleLinesAccumulateInOneSession() async throws { - // The console is persistent: each line joins the same shell session, so the - // second command could depend on the first. - let (session, vm) = await makeSession() - for line in ["cd /etc", "cat hostname"] { - session.sendRaw(line + "\n") - } - - XCTAssertEqual(try shell(of: vm).received, ["cd /etc\n", "cat hostname\n"]) - } - - func testOutputFromTheConsoleReachesTheScreen() async throws { - let (session, vm) = await makeSession() - let shell = try shell(of: vm) - - // The guest writes from its own thread, and TerminalSession hops that onto - // the main actor before touching the screen — so the assertion has to let - // that hop run. That asynchrony is deliberate: the reader must never block - // on the UI. - shell.emit("total 8\ndrwxr-xr-x\n") - try await settle() - XCTAssertTrue(session.buffer.plainText.contains("total 8")) - - let before = session.revision - shell.emit("more\n") - try await settle() - XCTAssertGreaterThan(session.revision, before, "the view must be told to redraw") - } - - func testOutputThatDoesNotEndInANewlineIsShown() async throws { - // A prompt, or a program waiting for input, writes without a trailing - // newline. Dropping that would leave the screen looking dead while the - // guest waits for an answer. - let (session, vm) = await makeSession() - try shell(of: vm).emit("Password: ") - try await settle() - - XCTAssertTrue(session.buffer.plainText.contains("Password: ")) - } - - /// Let queued main-actor hops from the guest run. - private func settle() async { - for _ in 0..<5 { await Task.yield() } - try? await Task.sleep(for: .milliseconds(20)) - } - - func testInterruptIsTheCtrlCByte() async throws { - // Ctrl-C is a byte again, because the console is a real tty: the guest's - // line discipline is what turns 0x03 into SIGINT for the foreground - // process group. - let (session, vm) = await makeSession() - session.sendRaw("sleep 100\n") - let shell = try shell(of: vm) - let before = shell.received.count - - session.interrupt() - try await settle() - - XCTAssertEqual(shell.interrupts, 1) - XCTAssertEqual(shell.received.count, before + 1, "the interrupt is one byte") - XCTAssertEqual(shell.received.last, "\u{3}", "and it is Ctrl-C") - } - - func testCommandsFromOtherScreensJoinTheSameConsole() async throws { - let (session, vm) = await makeSession() - session.enqueue("apk add --no-cache git", label: "Toolchain") - try await settle() - - XCTAssertEqual(try shell(of: vm).received, ["apk add --no-cache git\n"]) - XCTAssertEqual(try shell(of: vm).pid, 4242, "it runs on the console the terminal is on") - } - - func testTheGuestIsToldHowBigTheScreenIs() async throws { - // A guest that believes its terminal is 0×0 wraps everything to one column, - // so the size has to reach it — and only when it actually changes, since - // each report raises SIGWINCH in the guest. - let (session, vm) = await makeSession() - session.consoleResized(cols: 80, rows: 24) - session.consoleResized(cols: 80, rows: 24) - session.consoleResized(cols: 120, rows: 40) - - XCTAssertEqual(try shell(of: vm).sizes.map { "\($0.cols)x\($0.rows)" }, - ["80x24", "120x40"]) - } - - func testSendingWithNoConsoleReportsRatherThanDroppingTheLine() async throws { - let (session, _) = await makeSession() - session.shutdown() - session.enqueue("echo lost") - try await settle() - - XCTAssertTrue(session.buffer.plainText.contains("the shell is not running"), - "a queued line that cannot run must be visible") - } -} diff --git a/AppTests/XipArchiveTests.swift b/AppTests/XipArchiveTests.swift new file mode 100644 index 0000000..9518dae --- /dev/null +++ b/AppTests/XipArchiveTests.swift @@ -0,0 +1,503 @@ +import Compression +import Foundation +import XCTest +@testable import XForge + +/// The `.xip` reader and the Darwin SDK builder. +/// +/// There is no real Xcode xip to test against — they are 11 GB and need an Apple ID — +/// so these tests build one, layer by layer, the way the format notes in `XipArchive` +/// describe it: `odc` cpio inside pbzx blocks inside a xar. Two things that would +/// otherwise go untested are covered anyway: the pbzx framing is exercised across +/// block boundaries and with both a stored and a compressed block, and the LZMA +/// decode is checked against a fixture produced by **liblzma** (Python's `lzma` +/// module), so the decoder is validated against a different implementation than its +/// own encoder. +final class XipArchiveTests: XCTestCase { + + // MARK: - Fixture scaffolding + + private struct Spec { + var path: String + var mode: UInt32 + var data: Data = Data() + var inode: UInt32 + var links: UInt32 = 1 + } + + private static let directory: UInt32 = 0o040755 + private static let regular: UInt32 = 0o100644 + private static let symlink: UInt32 = 0o120777 + + /// The paths of the fake Xcode tree, rooted at `Xcode.app` as the archive's are. + /// + /// Bare `developer` rather than `Self.developer` in the two below: a stored + /// property's initializer cannot reference `Self` at all — the type is still + /// being built — and inside a *static* property there is no ambiguity to resolve. + private static let developer = "Xcode.app/Contents/Developer" + private static let platform = "\(developer)/Platforms/iPhoneOS.platform" + private static let toolchain = "\(developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib" + + private static func fakeXcode() -> [Spec] { + var inode: UInt32 = 0 + func next() -> UInt32 { inode += 1; return inode } + + let directories = [ + "Xcode.app", "Xcode.app/Contents", developer, "\(Self.developer)/Platforms", + platform, "\(Self.platform)/Developer", + "\(Self.platform)/Developer/SDKs", + "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk", + "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr", + "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include", + "\(Self.developer)/Toolchains", "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain", + "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr", + "\(Self.toolchain)", "\(Self.toolchain)/swift_static", "\(Self.toolchain)/swift_static/iphoneos", + "\(Self.toolchain)/swift", "\(Self.toolchain)/swift/prebuilt-modules", + "Xcode.app/Contents/Resources", + "\(Self.developer)/Platforms/MacOSX.platform", + ] + var specs = directories.map { Spec(path: $0, mode: directory, inode: next()) } + + // Kept: the platform manifest, a header from the SDK, the Swift static runtime. + specs.append(Spec(path: "\(Self.platform)/Info.plist", mode: regular, + data: Data("".utf8), inode: next())) + specs.append(Spec(path: "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h", + mode: regular, data: Data("#pragma once\n".utf8), inode: next())) + let runtimeInode = next() + specs.append(Spec(path: "\(Self.toolchain)/swift_static/iphoneos/libswiftCore.a", + mode: regular, data: Data(repeating: 0xAB, count: 5_000), + inode: runtimeInode, links: 2)) + + // Dropped: Xcode's own build output, another platform, anything under + // Contents that is not the Developer directory. + specs.append(Spec(path: "\(Self.toolchain)/swift/prebuilt-modules/Foundation.swiftmodule", + mode: regular, data: Data(repeating: 0xFF, count: 64), inode: next())) + specs.append(Spec(path: "\(Self.developer)/Platforms/MacOSX.platform/Info.plist", + mode: regular, data: Data("".utf8), inode: next())) + specs.append(Spec(path: "Xcode.app/Contents/Resources/Dropped.bin", + mode: regular, data: Data(repeating: 0x11, count: 32), inode: next())) + specs.append(Spec(path: "Xcode.app/Contents/version.plist", mode: regular, + data: Data("".utf8), inode: next())) + + // The second link to the runtime: same inode, and both are inside the filter, + // so the bundle should carry one file and one hard link, not two copies. + specs.append(Spec(path: "\(Self.toolchain)/swift/libswiftCore.a", mode: regular, + data: Data(), inode: runtimeInode, links: 2)) + + // `iPhoneOS.sdk` is a symlink to the versioned directory in recent Xcodes. + specs.append(Spec(path: "\(Self.platform)/Developer/SDKs/iPhoneOS.sdk", mode: symlink, + data: Data("iPhoneOS27.0.sdk".utf8), inode: next())) + return specs + } + + private static func cpio(_ specs: [Spec]) -> Data { + var out = Data() + for spec in specs { out += odcEntry(spec) } + out += odcEntry(Spec(path: "TRAILER!!!", mode: 0, inode: 0)) + return out + } + + /// `odc`: magic, six-digit octal fields, an 11-digit size, then the name and data. + private static func odcEntry(_ spec: Spec) -> Data { + func octal(_ value: UInt32, _ width: Int) -> Data { + Data(String(format: "%0\(width)o", value).utf8) + } + var out = Data("070707".utf8) + out += octal(0, 6) // device + out += octal(spec.inode, 6) // inode + out += octal(spec.mode, 6) // mode + out += octal(0, 6) // uid + out += octal(0, 6) // gid + out += octal(spec.links, 6) // link count + out += octal(0, 6) // rdev + out += octal(0, 11) // mtime + out += octal(UInt32(spec.path.utf8.count + 1), 6) // name length + out += octal(UInt32(spec.data.count), 11) // file size + out += Data(spec.path.utf8) + out += Data([0]) + out += spec.data + return out + } + + /// pbzx-wrap a payload. The first block is LZMA-compressed when the platform can + /// encode it (Apple's LZMA encoder is decode-mostly), the rest are stored — the + /// same mix a real xip has. + private static func pbzx(_ payload: Data, chunkSize: Int = 2_048) -> Data { + var out = Data("pbzx".utf8) + out += bigEndian(UInt64(chunkSize), 8) + var index = payload.startIndex + var block = 0 + while index < payload.endIndex { + let end = payload.index(index, offsetBy: chunkSize, limitedBy: payload.endIndex) + ?? payload.endIndex + let uncompressed = Data(payload[index..` is heap-relative. + /// + /// The `` is not decoration: it carries an `` of its + /// own, and a reader that takes the last offset it sees points the payload at the + /// checksum. A real xip's TOC has one, so this fixture does too. + private static func xar(content: Data) -> Data { + let toc = """ + + 0Contentfile\ + 0\(content.count)\(content.count)\ + \(content.count)\ + 20 + """ + let tocData = Data(toc.utf8) + let compressed = zlibEncode(tocData) ?? tocData + var out = Data("xar!".utf8) + out += bigEndian(UInt16(28), 2) + out += bigEndian(UInt16(1), 2) + out += bigEndian(UInt64(compressed.count), 8) + out += bigEndian(UInt64(tocData.count), 8) + out += bigEndian(UInt32(0), 4) + out += compressed + out += content + return out + } + + private static func bigEndian(_ value: T, _ width: Int) -> Data { + var big = value.bigEndian + return withUnsafeBytes(of: &big) { Data($0.suffix(width)) } + } + + private static func zlibEncode(_ data: Data) -> Data? { + var buffer = [UInt8](repeating: 0, count: data.count + 4_096) + return data.withUnsafeBytes { raw -> Data? in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return nil } + let written = compression_encode_buffer(&buffer, buffer.count, base, data.count, + nil, COMPRESSION_ZLIB) + return written > 0 ? Data(buffer[0.. Data? { + var buffer = [UInt8](repeating: 0, count: data.count * 2 + 4_096) + return data.withUnsafeBytes { raw -> Data? in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return nil } + let written = compression_encode_buffer(&buffer, buffer.count, base, data.count, + nil, COMPRESSION_LZMA) + return written > 0 ? Data(buffer[0.. URL { + let cpio = payload ?? Self.cpio(Self.fakeXcode()) + return try writeXip(Self.xar(content: Self.pbzx(cpio))) + } + + /// Write xar bytes into a fresh temporary directory. + private func writeXip(_ archive: Data) throws -> URL { + let directory = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("xip-tests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + addTeardownBlock { try? FileManager.default.removeItem(at: directory) } + + let url = directory.appendingPathComponent("Xcode_27.xip") + try archive.write(to: url) + return url + } + + // MARK: - The xar layer + + /// The `Content` member's byte range. The TOC records a *heap-relative* offset, so + /// a reader that treats it as absolute lands inside the header and finds no pbzx + /// magic — which is what this pins down. + func testContentMemberIsLocatedAfterTheTableOfContents() throws { + let url = try makeXip() + let content = try XipArchive.content(of: url) + + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + try handle.seek(toOffset: UInt64(content.offset)) + let magic = try XCTUnwrap(try handle.read(upToCount: 4)) + XCTAssertEqual(magic, Data("pbzx".utf8), + "the Content member should start at the pbzx stream") + XCTAssertGreaterThan(content.length, 0) + } + + func testRejectsAFileThatIsNotAXip() throws { + let url = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("not-a-xip-\(UUID().uuidString).xip") + try Data("this is just a file".utf8).write(to: url) + addTeardownBlock { try? FileManager.default.removeItem(at: url) } + + XCTAssertThrowsError(try XipArchive.content(of: url)) { error in + guard let xipError = error as? XipArchive.Error, case .notAXip = xipError else { + return XCTFail("expected notAXip, got \(error)") + } + } + } + + // MARK: - The pbzx and cpio layers + + /// Every entry, in order, with the shapes intact — and across block boundaries: + /// the fixture's chunk size is 2 KB and the fake tree is larger than that. + func testWalkYieldsEveryEntry() throws { + let url = try makeXip() + let content = try XipArchive.content(of: url) + + var entries: [XipArchive.Entry] = [] + var payloads: [String: Int] = [:] + try XipArchive.forEachEntry(in: url, content: content) { entry, payload in + entries.append(entry) + payloads[entry.name] = try payload.readAll().count + } + + let names = entries.map(\.name) + XCTAssertTrue(names.contains("\(Self.platform)/Info.plist")) + XCTAssertFalse(names.contains("TRAILER!!!"), "the trailer ends the walk") + XCTAssertEqual(entries.filter(\.isDirectory).count, + Self.fakeXcode().filter { $0.mode == Self.directory }.count) + + // A file's data has to survive whole even when it straddles blocks. + XCTAssertEqual(payloads["\(Self.toolchain)/swift_static/iphoneos/libswiftCore.a"], 5_000) + XCTAssertEqual(payloads["\(Self.platform)/Info.plist"], Data("".utf8).count) + + // A symlink's "data" is its target, which is how the bundle recreates it. + let link = entries.first { $0.name.hasSuffix("/SDKs/iPhoneOS.sdk") } + XCTAssertEqual(link?.isSymlink, true) + XCTAssertEqual(payloads["\(Self.platform)/Developer/SDKs/iPhoneOS.sdk"], 16) + } + + /// The compressed path, validated against an implementation other than this one. + /// + /// A pbzx block is an **xz** stream — `pbzx.c` checks for the `\xFD7zXZ` magic and + /// refuses anything else — and Apple's `COMPRESSION_LZMA` is what decodes it. + /// This fixture is 787 bytes of odc cpio compressed to 256 bytes by **liblzma** + /// (Python's `lzma` module, xz container), wrapped in a single pbzx block inside a + /// xar. Walking it exercises the whole compressed path — the framing decision + /// (compressed size differs from decompressed size), the xz decode and the cpio + /// parse — against a stream this process did not produce. An encoder round trip + /// would have proved nothing: two matching bugs look exactly like correctness. + func testWalksAnXZPayloadProducedByLiblzma() throws { + let base64 = """ + /Td6WFoAAATm1rRGAgAhARYAAAB0L+Wj4AMSAL1dABgN3wegMRkMF/SCaXaknqoWiVsR\ + ntc2yBB5DloIwewVgm0EJoUwmh+RvTVwtHxlYFJ1cZxGEPK8IgJOaAlEu44Mfu1l8oOZ\ + B6IWQLeOhCnnHNhdx7piLXn/q9nM4+Ka84R4dbk6jozpre+YlFurXBoqU2Bn3zhVpRu1\ + XnLHE0TGI/caB2jNykEyKiHElx5ZcXjGvi+cpoApZ4ZX3NbQxDozsrusWS7CR2dkq2yL\ + Lc5s0cid+htDMzoOxwaAAAAAAAAh8cN2Rcr+YAAB2QGTBgAAcoJ8YLHEZ/sCAAAAAARZ\ + Wg== + """ + let compressed = try XCTUnwrap(Data(base64Encoded: base64)) + XCTAssertEqual(compressed.prefix(6), Data([0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00]), + "the fixture has to be an xz stream, which is what pbzx holds") + + // One block, compressed: the uncompressed size is in the block header, and the + // two sizes differing is what tells the reader to decompress it. + let uncompressedSize = 787 + var stream = Data("pbzx".utf8) + stream += Self.bigEndian(UInt64(4_096), 8) + stream += Self.bigEndian(UInt64(uncompressedSize), 8) + stream += Self.bigEndian(UInt64(compressed.count), 8) + stream += compressed + + let url = try writeXip(Self.xar(content: stream)) + let content = try XipArchive.content(of: url) + + var names: [String] = [] + var stdio: [UInt8] = [] + try XipArchive.forEachEntry(in: url, content: content) { entry, payload in + names.append(entry.name) + if entry.name.hasSuffix("/usr/include/stdio.h") { + stdio = try payload.readAll() + } + } + + XCTAssertEqual(names, [ + "Fixture.app", + "Fixture.app/Developer", + "Fixture.app/Developer/Info.plist", + "Fixture.app/Developer/SDKs/iPhoneOS99.0.sdk", + "Fixture.app/Developer/SDKs/iPhoneOS99.0.sdk/usr", + "Fixture.app/Developer/SDKs/iPhoneOS99.0.sdk/usr/include/stdio.h", + ]) + XCTAssertEqual(Data(stdio), Data("#pragma once\n".utf8)) + } + + // MARK: - What becomes the bundle + + /// The filter, checked against the list xtool keeps (`SDKBuilder.wanted`), with + /// XForge's one deliberate narrowing: the device platform only. + func testWantedPaths() { + let kept = [ + "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphoneos/Swift.swiftmodule", + "\(Self.toolchain)/swift_static/iphoneos/libswiftCore.a", + // The directory itself is kept: it holds the static runtime, and it is what + // `swiftStaticResourcesPath` names. + "\(Self.toolchain)/swift_static/iphoneos", + "\(Self.toolchain)/swift_static", + "\(Self.toolchain)/clang/include/stdarg.h", + "\(Self.platform)/Info.plist", + "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h", + "\(Self.platform)/Developer/usr/lib/libSystem.tbd", + "\(Self.platform)/Developer/Library/Frameworks/XCTest.framework/XCTest", + ] + for path in kept { + XCTAssertTrue(DarwinSDKBuilder.isWanted(path), "should be kept: \(path)") + } + + let dropped = [ + "\(Self.toolchain)/swift/prebuilt-modules/Foundation.swiftmodule", + "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphoneos/prebuilt-modules/Foundation.swiftmodule", + "\(Self.developer)/Platforms/MacOSX.platform/Info.plist", + "\(Self.developer)/Applications/Whatever.app/Whatever", + "Contents/Resources/English.lproj/InfoPlist.strings", + "Xcode.app/Contents/version.plist", + "\(Self.developer)", + "\(Self.developer)/Platforms", + "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/bin/swiftc", + "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-ide-test", + ] + for path in dropped { + XCTAssertFalse(DarwinSDKBuilder.isWanted(path), "should be dropped: \(path)") + } + } + + // MARK: - End to end + + /// An `.xip` in, a `darwin.artifactbundle` out — one the app's own reader accepts. + func testBuildsAnInstallableBundleFromAXip() throws { + let url = try makeXip() + let destination = url.deletingLastPathComponent() + .appendingPathComponent("out", isDirectory: true) + + let result = try DarwinSDKBuilder.build(fromXip: url, into: destination) + // Five: the platform manifest, a header, the static runtime, the archive's hard + // link to it, and the symlink. Directories are not counted. + XCTAssertGreaterThanOrEqual(result.files, 5) + XCTAssertGreaterThan(result.skipped, 0, "the unwanted paths should be counted as skipped") + XCTAssertEqual(result.sdkRoot, + "Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS27.0.sdk") + + let bundle = result.bundle + let fileManager = FileManager.default + func exists(_ relative: String) -> Bool { + fileManager.fileExists(atPath: bundle.appendingPathComponent(relative).path) + } + + XCTAssertTrue(exists("Developer/Platforms/iPhoneOS.platform/Info.plist")) + XCTAssertTrue(exists("Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h")) + XCTAssertTrue(exists("Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static/iphoneos/libswiftCore.a")) + + // The tree is re-rooted at Developer/, not copied as Xcode.app. + XCTAssertFalse(exists("Xcode.app")) + + XCTAssertFalse(exists("Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/prebuilt-modules/Foundation.swiftmodule")) + XCTAssertFalse(exists("Developer/Platforms/MacOSX.platform/Info.plist")) + XCTAssertFalse(exists("Contents/Resources/Dropped.bin")) + + // The symlink is a symlink, pointing at the versioned SDK directory. + let sdkLink = bundle.appendingPathComponent( + "Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk") + XCTAssertEqual(try fileManager.destinationOfSymbolicLink(atPath: sdkLink.path), + "iPhoneOS27.0.sdk") + + // The two links to one inode stay one file plus a link. + func inode(_ relative: String) throws -> NSNumber? { + let attributes = try fileManager.attributesOfItem( + atPath: bundle.appendingPathComponent(relative).path) + return attributes[.systemFileNumber] as? NSNumber + } + let runtime = try inode("Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static/iphoneos/libswiftCore.a") + let other = try inode("Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/libswiftCore.a") + XCTAssertEqual(runtime, other, "the archive's hard link should stay a hard link") + + // The metadata, as `NativeSDK` reads it. + let json = try XCTUnwrap( + try JSONSerialization.jsonObject( + with: Data(contentsOf: bundle.appendingPathComponent("swift-sdk.json"))) + as? [String: Any]) + XCTAssertEqual(json["schemaVersion"] as? String, "4.0") + let triples = try XCTUnwrap(json["targetTriples"] as? [String: Any]) + let triple = try XCTUnwrap(triples["arm64-apple-ios"] as? [String: Any]) + XCTAssertEqual(triple["sdkRootPath"] as? String, result.sdkRoot) + XCTAssertEqual(triple["swiftStaticResourcesPath"] as? String, + "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static") + XCTAssertEqual(try String(contentsOf: bundle.appendingPathComponent("darwin-sdk-version.txt"), + encoding: .utf8) + .trimmingCharacters(in: .whitespacesAndNewlines), "27.0") + XCTAssertTrue(result.hasSwiftStaticRuntime, "the fake Xcode carries the static runtime") + + // And the app's own reader resolves it — the assertion that matters, because a + // bundle nothing can read is not an SDK. + let layout = try NativeSDK.layout(at: bundle) + XCTAssertEqual(layout.sdkRoot.lastPathComponent, "iPhoneOS27.0.sdk") + XCTAssertEqual(layout.swiftStaticResources?.lastPathComponent, "swift_static") + XCTAssertTrue(layout.swiftRuntimeLibraryPaths.contains { + $0.path.hasSuffix("/swift_static/iphoneos") + }) + } + + /// A xip without an iPhoneOS SDK is refused with a message that says so, rather + /// than installing a bundle that fails later. + func testRejectsAXipWithNoIPhoneOSSDK() throws { + let specs = [ + Spec(path: "Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Info.plist", + mode: Self.regular, data: Data("".utf8), inode: 1), + ] + let url = try makeXip(payload: Self.cpio(specs)) + let destination = url.deletingLastPathComponent() + .appendingPathComponent("out", isDirectory: true) + + XCTAssertThrowsError(try DarwinSDKBuilder.build(fromXip: url, into: destination)) { error in + guard let builderError = error as? DarwinSDKBuilder.Error, + case .noiPhoneOSSDK = builderError else { + return XCTFail("expected noiPhoneOSSDK, got \(error)") + } + } + XCTAssertFalse(FileManager.default.fileExists( + atPath: destination.appendingPathComponent("darwin.artifactbundle").path)) + } + + /// Something that is a valid cpio archive but not an Xcode says so, instead of + /// reporting the absence of an iOS SDK in a tree that has no Xcode at all. + func testRejectsAnArchiveThatIsNotAnXcode() throws { + let specs = [ + Spec(path: "Downloads", mode: Self.directory, inode: 1), + Spec(path: "Downloads/notes.txt", mode: Self.regular, + data: Data("hello\n".utf8), inode: 2), + ] + let url = try makeXip(payload: Self.cpio(specs)) + let destination = url.deletingLastPathComponent() + .appendingPathComponent("out", isDirectory: true) + + XCTAssertThrowsError(try DarwinSDKBuilder.build(fromXip: url, into: destination)) { error in + guard let builderError = error as? DarwinSDKBuilder.Error, + case .noXcodeInside = builderError else { + return XCTFail("expected noXcodeInside, got \(error)") + } + } + } + + /// A newer Xcode that keeps no static Swift runtime still produces an SDK: the + /// headers and tbd stubs are what C targets need, and refusing the import over a + /// Swift runtime would take those away too. The result says so instead. + func testImportsWithoutTheSwiftStaticRuntime() throws { + let specs = Self.fakeXcode().filter { !$0.path.contains("swift_static") } + let url = try makeXip(payload: Self.cpio(specs)) + let destination = url.deletingLastPathComponent() + .appendingPathComponent("out", isDirectory: true) + + let result = try DarwinSDKBuilder.build(fromXip: url, into: destination) + XCTAssertFalse(result.hasSwiftStaticRuntime) + XCTAssertTrue(FileManager.default.fileExists( + atPath: result.bundle.appendingPathComponent("swift-sdk.json").path)) + XCTAssertTrue(FileManager.default.fileExists( + atPath: result.bundle.appendingPathComponent( + "Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h").path)) + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 0643436..7f7b28a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,127 @@ All notable changes to **XForge** are documented here. +## [0.6.0] — 2026-09-24 — The embedded Linux is removed; the toolchain runs in the app + +**Breaking change.** XForge no longer boots a Linux guest. The emulator, the +root filesystem, the Terminal tab and the guest-side services that provisioned and +drove them are gone from the app; what builds an app now is the LLVM/Clang/LLD and +Swift libraries linked into the process, called through a C++ bridge. Anyone who +relied on the guest or on the Terminal has nothing to fall back to: the Terminal +existed to type into the guest, and with the guest gone there was nothing behind +it. Projects are no longer files inside a Linux filesystem you could `cd` into; +they are ordinary directories in the app's container. + +### Removed +- **`App/EmbeddedVM/**`** — the ish-arm64 bridge, the emulator, the rootfs + installer, the console and shell sessions. The app links no Linux kernel now. +- **`App/Build/EmbeddedLinuxExecutor.swift` and `App/Build/SDKInstaller.swift`** — + replaced by `NativeBuildExecutor` and `NativeSDK`. +- **`App/Services/{SystemComponents,GuestNetwork,ToolchainManager,IPAConfigureSignService}.swift`, + `HostDNS.{c,h}`, `DownloadManager.swift`, `DirectorySize.swift`** — guest + provisioning, guest DNS, the in-guest component installer and the `zsign` job. +- **`App/Views/Terminal/**`, `EngineLogView`, `DownloadsView`** — the Terminal, the + engine log screen, and the download screen that existed to drive `curl` inside + the guest. The app is now three tabs: Projects, Build, Settings. +- **The bundled Alpine rootfs and everything that produced it.** The artifact was + ~1.6 GB and arrived with a five-gigabyte provisioning story attached; there is no + rootfs, no `install-toolchain.sh` to run in a guest, and no pinned root asset to + download. `EmbeddedLinux/` is gone, and `project.yml` no longer links `-lish`, + `-lz` or `-liconv`, sets `GUEST_ARM64`, or requires `Vendor/ish-arm64*` to exist. +- **`.github/workflows/ios-share.yml` (30 KB) and `builder.json`.** A build-for-the- + simulator workflow whose own comments tell you to keep its `js toolchain` blocks + identical to `ios-build.yml` and `runner.sh` — neither of which is in the + repository — and which drives the Flutter / React Native / Expo / KMP branches of + a multi-framework builder. It has never run here (no `ios-share/*` tag has ever + been pushed) and this is a Swift app with an in-process toolchain. Both are in git + history if the MobAI simulator flow is ever wanted back. +- **`.github/workflows/bridge-api-check.yml`.** A "fast loop" for the bridge + compile-check, written when reaching that check cost 40–90 minutes. It now costs + eleven seconds inside a two-minute warm toolchain run, and the workflow carried + its own copy of the LLVM configure recipe — a second place to update, and a + silent divergence waiting to happen. + +### Added +- **`App/Build/NativeBuildPlan.swift`** — reads a project directory into a fully + resolved plan (module, sources, resources, frameworks, search paths) and refuses, + **by name**, the one thing an in-process driver cannot resolve: SwiftPM + dependencies. +- **`App/Build/NativeToolchainInvocation.swift`** — the `swift-frontend` and + `ld64.lld` argument lists as pure functions, so they are testable on a simulator + where no compiler libraries exist. +- **`App/Services/AppBundleSigner.swift` and `App/Services/IPASigningJob.swift`** — + in-process signing with XKit. The private key is read through the Security + framework and used in memory; the `.p12` password file that `zsign` needed on + disk is gone with it. +- **`App/Services/ProjectFiles.swift`** — replaces `GuestProjectFiles`, keeping the + same relative-path discipline (a `..` is rejected, not normalised) without a + guest underneath it. +- **`App/Views/Toolchain/ToolchainView.swift`** — the toolchain and SDK screen, now + native: it reports which libraries are linked in, installs or removes the Darwin + SDK in the app container, and runs a compile smoke test. +- **Importing an `Xcode.xip` on the device.** The Toolchain screen accepts Apple's + xip and builds the Darwin SDK out of it here — the job `xtool sdk build` does on a + Mac, and the reason a phone with no Mac could not use a newer SDK than the one we + publish. A xip is a xar archive holding a pbzx stream of xz blocks that + decompresses to a cpio archive of `Xcode.app`; `XipArchive` reads it in one + streaming pass (Apple's `Compression` decodes the xz, so nothing is vendored) + and `DarwinSDKBuilder` keeps xtool's own list of paths, restricted to + `iPhoneOS.platform` because XForge only compiles for the device. The picker also + takes a zip or a folder, for a bundle built elsewhere. Free space is checked + before extraction starts, progress is reported while it runs, and the picker's own + copy of the archive — gigabytes, for a xip — is deleted afterwards. +- **The toolchain bundle is published, not just uploaded.** `native-toolchain.yml` + attaches the finished bundle to a release tagged + `toolchain---ios-sdk` — the tag names what is + inside, because the tag is what a consumer asks for and a bundle that could not be + substituted for another must never be handed over as "the newest one". A dispatched + run is the only thing that publishes; a pull request or push run builds the + LLVM-only half and stays out of the way. `NativeToolchain/install-bundle.sh --release + [tag]` (and `make toolchain-release`) installs one without ever having run the + workflow, and `build-ipa.yml` takes `with_toolchain: true` plus an optional + `toolchain_tag` to link it into the app. Workflow artifacts expire and cannot be + fetched by tag; a release can. + +### Changed +- **Build path.** `NativeBuildExecutor` compiles each translation unit through the + bridge, links with `ld64.lld`, assembles the `.app` and packages the unsigned + `.ipa`, streaming every step into the Build screen's console. `BuildManager`'s + stages lost their guest commands: the SDK stage installs the bundle into the app + container, and "Configure" validates the project directory instead of `mkdir`-ing + inside a guest. +- **Projects move to the app's own container**, `/projects/`, and + `Project.rootURL` is derived from the project name. They used to live inside the + guest filesystem, which is why `Project.rootPath` still carries a guest-shaped + string (`projects/`) that is now only used to check a record against the + directory it names. +- **Export is a ZIP of a directory instead of a `tar` inside the guest** + (`ProjectExporter`, ZIPFoundation), skipping `.xforge-build` and `.build`. +- **The app's log is an ordinary file the user can share.** It used to be the + engine's kernel messages through the ish bridge. +- **Importing a project copies a folder picked in Files.** An iOS process cannot + run `git`, so a clone is not something the app can offer. +- **The dependencies editor is gone.** A project that declares SwiftPM dependencies + cannot be resolved without a package manager, so `NativeBuildPlanFactory` refuses + such a project with a message that names the packages — an editor that could only + produce unbuildable projects was worse than no editor. +- **The guest filesystem is no longer the reason a project is not backed up.** See + the reversal under "Projects can leave the device" below: what that entry + described as impossible is now the default, because a project is a plain + directory in the container rather than a file inside an opaque fakefs. + +### Known gap +- **Swift needs a bundle that carries the frontend.** `.github/workflows/native-toolchain.yml` + builds the Swift frontend libraries when it is dispatched with `with_swift=true`, + and that bundle says `swift_frontend=1` in its manifest; the default dispatch and + every pull-request run build the Clang/LLD half only. Against a bundle without the + frontend a project whose sources are Swift still fails at the Swift compile step + with `swiftFrontendMissing`, naming the count of files, instead of silently needing + a guest. C and Objective-C targets compile and link either way. The guest is not + coming back as a fallback: the honest replacement for it is a bundle that carries + the frontend, which is now a thing you can build and install. +- **Asset catalogs are copied uncompiled**, with a warning: `actool` is a + macOS-only tool with no open-source replacement. + ## Projects can leave the device, and the guest filesystem stays out of backups ### Added diff --git a/Docs/DESIGN.md b/Docs/DESIGN.md index 787d9db..c09da87 100644 --- a/Docs/DESIGN.md +++ b/Docs/DESIGN.md @@ -1,273 +1,256 @@ # XForge — build iOS apps on-device with xtool -Working title: **XForge**. An iOS app (sideload-only) that embeds a Linux userspace -running [xtool](https://github.com/xtool-org/xtool) + a Swift toolchain, so you can -author SwiftPM packages and compile them into real iOS apps entirely on your iPhone. +Working title: **XForge**. An iOS app (sideload-only) that compiles an authored +project into a real, signed iOS app entirely on the phone, using the compiler, +linker and signing libraries linked into its own process. -Status: Architecture A (fully on-device via embedded iSH-style Linux), built from -scratch, sideload-only. +Status: fully on-device, no remote build host. The old architecture — an embedded +iSH-style Linux — is gone; this document describes what replaced it and why the +replacement takes the shape it does. --- ## 1. How the build works (the mechanism) -`xtool dev build` is just an orchestration layer over **SwiftPM cross-compilation**: +XForge does not orchestrate a build by running other programs. **iOS does not let +an app spawn a process** — there is no `fork`/`exec`/`posix_spawn` in the sandbox — +so every step that a build tool would normally run as a child is instead a library +call in XForge's own process: + +1. **Read the project into a plan.** `NativeBuildPlanFactory` walks the project + directory (`Package.swift`, `Sources//`) and produces a + `NativeBuildPlan`: the module and executable names, the Swift and C-family + translation units, the resources, the frameworks, and the `-L`/`-F` search + paths. Nothing compiles until this value exists, and nothing in the executor + re-decides any of it. +2. **Compile.** Swift sources would go through `swift::performFrontend` — but the + frontend libraries are only in a bundle built with `with_swift`, so a plan with + Swift files is refused at this point when the installed bundle has none (§2, + third bullet). C, Objective-C and + Objective-C++ sources compile through clang's `CompilerInstance` with + `EmitObjAction`. Both take an argument list built by + `NativeToolchainInvocation`, against the iPhoneOS SDK in the Darwin SDK bundle. +3. **Link.** `ld64.lld` (through `lld::lldMain`) links the objects against the SDK's + `.tbd` stubs and the static Swift runtime into an `arm64-apple-ios` executable. +4. **Assemble.** The executor writes `.app`: the executable at mode 0755, an + `Info.plist` (generated keys, over which a hand-written project template is + merged), and the resources. +5. **Package.** `IPABuilder` lays the bundle out under `Payload/` and zips it. The + result is an **unsigned** `.ipa`. +6. **Sign.** `IPASigningJob` + `AppBundleSigner` sign with `XKit`, xtool's own + library and the same code path that signs on a Mac. Signing is a separate step + from packaging on purpose: the pipeline can hand an unsigned IPA to a signing + service, and it can hand the same IPA to XKit. + +### Why the plan is a separate value + +The compiler only exists in a *device* build — the libraries are linked in by +`Support/NativeToolchain.generated.xcconfig`, which is disabled on a plain clone. +A wrong argument list would therefore be undiscoverable until a build ran on a +phone with a real SDK. Splitting "what to build" (the plan) and "how to invoke +the tools" (the pure `NativeToolchainInvocation` functions) from "call the tools" +(the executor) means the parts that can be wrong silently are the parts that unit +tests can check, on a simulator, with no compiler present. `AppTests/NativeBuildTests.swift` +is exactly that. + +### Where the plan draws the line + +An in-process driver cannot do everything a build system does. Rather than fail +somewhere in the middle, the plan factory refuses by name where it cannot resolve +what is needed, and the executor states plainly where it can only copy: + +- **SwiftPM dependencies.** Resolving them means fetching packages and *running* + their manifests, which needs a package manager process. The factory scans + `Package.swift` textually for `.package(` lines — enough to answer "are there + any", which is all it needs — and refuses the project, naming the packages. +- **Projects with no `Sources/` directory**, or no compilable sources in it: there + is no useful build to produce and no way to guess one. +- **Asset catalogs are *not* refused**, but they are not compiled either: `.xcassets` + are copied into the bundle whole and the console says why. `actool` is a + macOS-only tool, so an app whose icon lives in a catalog ships without it. Copying + rather than dropping them keeps the project round-tripping. + +## 2. Where the toolchain comes from + +The Clang/LLD libraries are **cross-built for iPhoneOS in CI** on a macOS runner +(`.github/workflows/native-toolchain.yml`), merged into a single static archive and +published as a release asset (tagged `toolchain---ios…`). +`NativeToolchain/install-bundle.sh --release` fetches it, or unpacks a local archive, +into `Vendor/NativeToolchain`, and `NativeToolchain/prepare-xcode.sh` writes the +xcconfig the app target consumes. + +The consequences of linking the compiler in rather than shipping it as data: + +- **It cannot be installed at runtime.** Either the libraries are in the binary or + they are not; the Toolchain screen can only *report* them. That is why + `NativeToolchainCapabilities` exists and why the first build stage fails with an + explanation instead of a linker error. +- **A plain clone still builds.** `Support/NativeToolchain.generated.xcconfig` is + checked in with the backend disabled, so the bridge compiles to a "not available" + stub. Simulator builds and `make test` need nothing else. +- **The Swift frontend is a separate, larger port.** The workflow builds Clang and + LLD only, so `swift-frontend` is absent from the artifact and Swift sources + cannot be compiled yet. The capability is reported honestly rather than assumed: + `canCompile` requires clang and LLD, not the frontend, because refusing to start + a C build would be wrong about why it works. + +## 3. Where the `darwin` SDK comes from + +`xtool sdk build ` produces the `darwin` SDK from a real Xcode. That is a +macOS program shelling out to `xar`, so the `.xip` path used to stop at "impossible +on a phone" — and the default is still the cheap one: a `darwin.artifactbundle` is +**built once in CI** and published as a release asset under XForge's own +`darwin-sdk-` release series, and the app resolves the newest matching release at +build time. About 460 MB. + +The bundle is installed into the app container at +`/native-sdk/darwin.artifactbundle` and read directly: `NativeSDK` parses +the same `swift-sdk.json` layout xtool's builder emits, resolving the SDK root, the +Swift resource directory and the static runtime search paths. + +Three things can be installed from the Toolchain screen, told apart by what they are +rather than by what they are called: that hosted bundle (a zip), a +`darwin.artifactbundle` **folder** (perhaps built by `xtool sdk build` on a Mac), and +an Apple **`Xcode.xip`**, which the app now builds into a bundle itself +(`DarwinSDKBuilder` + `XipArchive`). The third is the one that needs no Mac: + +- a `.xip` is a xar archive whose `Content` member is a pbzx stream of LZMA2 blocks, + which decompresses to an `odc` cpio archive of `Xcode.app`. `XipArchive` reads it + in one pass, streaming, and hands only the wanted entries to disk; +- which entries those are is xtool's own list (`SDKBuilder.wanted`), not a guess: + `Contents/Developer/Platforms/iPhoneOS.platform/{Info.plist,Developer/{SDKs,usr/lib,Library/Frameworks,Library/PrivateFrameworks}}` + and `Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/{swift,swift_static,clang}`, + minus `swift/prebuilt-modules` (per-configuration build output measured in GB); +- one deliberate narrowing: xtool builds a bundle for three platforms because a Mac + builds for all of them, and XForge only ever compiles for the device, so only + `iPhoneOS.platform` is extracted and `swift-sdk.json` declares only + `arm64-apple-ios`. A third of the work and a third of the disk for the same + capability. + +Extraction is minutes of CPU and ~1.5 GB of disk on top of whatever the document +picker copied, and both are checked before it starts rather than discovered halfway +through. The install is staged and validated (`NativeSDK.layout`) before it replaces +what is there, so a bad bundle cannot leave the app without an SDK. -1. SwiftPM (from the Linux Swift toolchain) compiles the SwiftPM package to - `arm64-apple-ios` using a Swift SDK named **`darwin`** (installed via - `swift sdk install`, SwiftPM's native mechanism). -2. xtool's `Packer` turns the `.app` bundle into a signed `.ipa` (zsign) with the - right Info.plist, entitlements, and codesigning identity. -3. `XKit` (the library) talks to Apple Developer Services and devices natively on iOS. - -So the iOS app needs three heavyweight pieces to *compile* (all Linux-side, all -self-contained, all fetchable): -- **Swift aarch64 Linux toolchain** (swift.org Ubuntu build) — compiler/clang/lld. -- **`darwin` Swift SDK bundle** for `arm64-apple-ios` — contains the iOS SDK (headers, - .tbd stubs, module maps) + the iOS Swift stdlib. This is the big one (multi-GB). -- **`xtool` aarch64 binary** (the prebuilt `xtool-aarch64.AppImage`, 51 MB). - -## 1b. The userspace is **Alpine aarch64 with the build toolchain in it**, bundled as a fakefs ZIP +> Note: Apple also publishes official iOS Swift SDKs on swift.org, but they use the +> triple `aarch64-apple-ios` under a different bundle name. xtool hardcodes +> `darwin` + `arm64-apple-ios`, so we produce our own `darwin` bundle to match +> xtool exactly. -The embedded Linux boots the official **Alpine Linux aarch64 minirootfs**: +## 4. App architecture ``` -https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/aarch64/alpine-minirootfs-3.21.0-aarch64.tar.gz +XForge.app (nothing else — no guest, no subprocesses) +├─ Projects — author a SwiftPM package; ProjectFiles reads/writes the directory +│ ├─ project directory /projects/ +│ ├─ file browser / source editor / manifest editor +│ ├─ import copy a folder picked in Files (no git: an app cannot spawn it) +│ └─ export ProjectExporter → /exports/.zip +├─ Build +│ ├─ BuildManager — the stage state machine the Build screen draws +│ ├─ NativeBuildPlan — what to build, resolved and refused up front +│ ├─ NativeBuildExecutor — the BuildExecutor; calls the libraries +│ ├─ NativeToolchain* — the C++ bridge (clang, swift-frontend, ld64.lld) +│ └─ IPABuilder — Payload/ + zip → unsigned .ipa +├─ Signing — IPASigningJob: .p12 + profile → AppBundleSigner (XKit) → signed .ipa +├─ Toolchain — capabilities, Darwin SDK install/remove, smoke test +└─ Settings — app sandbox browser, build history, log, diagnostics ``` -- `EmbeddedLinux/build-rootfs.sh` downloads that archive, installs the guest's - own packages into it with the guest's own `apk` (the shell session's: see - *the console* below), **provisions the build toolchain into it by running the - guest's own `install-toolchain.sh` in a chroot of the tree** (glibc layer, apk - build dependencies, xtool, swiftly, the Swift toolchain, the Darwin SDK), - configures the root (mount points, `/etc/passwd`, `/etc/profile`, `/etc/motd`, - `/etc/apk/repositories`, a default `/etc/resolv.conf`), converts it to the - engine's `fakefs` format with the engine's own `tools/fakefsify`, and packs it - as `alpine-rootfs.zip`. -- **Provisioning is the same code path as the guest's, not a parallel one.** The - script the build runs in the chroot is the script the app ships - (`EmbeddedLinux/install-toolchain.sh`, staged at `/root/install-toolchain.sh`), - invoked step by step: `deps`, `glibc`, `xtool`, `swiftly`, `swift`, `sdk`, then - `verify`. A root provisioned on the build machine and one provisioned by hand - in the Terminal therefore differ in nothing but where and when the work - happened, and there is one implementation to keep correct. -- **The console is a root login session, and the root is what starts it.** - `/etc/inittab` respawns `/sbin/xforge-login root` on `tty1`; that script (from - `EmbeddedLinux/xforge-login`) reads root's shell out of `/etc/passwd` and execs - it as a login shell, so the Terminal tab opens in the shell the root names — - `/bin/bash` by default — and `apk add zsh` plus that field changes it. Busybox - `login -f root` is deliberately not in this path: it authenticates, needs utmp - and takes over the terminal, none of which this guest has, and none of which it - can report failing. -- **The root therefore carries the session's dependencies**: `bash`, - `coreutils`, `less` and `ncurses-terminfo` (`XFORGE_CONSOLE_PACKAGES`). The - terminfo package is not optional detail — `/etc/profile` exports - `TERM=xterm-256color`, and the *base* terminfo package does not contain the - xterm entries, so without it every curses program in the guest runs against an - unknown terminal. -- **`EmbeddedLinux/verify-rootfs.sh` runs the console program in a chroot of the - root** and asserts which shell comes up, as a login shell, from the field in - `/etc/passwd`. `build-rootfs.sh` runs it on the tree before conversion and - `build-rootfs.yml` runs it on the published ZIP, so "the terminal works" is - checked against the guest's own binaries rather than against the intent of the - configuration that produced them. -- **The conversion happens at build time, not in the app.** `fakefsify` runs on - the build machine, so the ZIP already *is* a fakefs — a `data/` tree plus - `meta.db` — and first launch costs an unzip rather than an import of thousands - of files into SQLite on a phone. This is the same layout OpenMinis ships, and - it is why `App/EmbeddedVM/RootfsInstaller.swift` uses `unzip` and then - `mount_root` instead of `fakefs_import`. -- **The root arrives ready to build.** Swift, xtool and the Darwin SDK *are* in - it, installed before the conversion, which is what the artifact's size buys: the - guest can run `xtool new` and build on first launch instead of provisioning - itself under emulation first. `XFORGE_PROVISION=none` builds the plain - few-MB root instead, where the guest installs the same things on demand with - the same script. The **glibc compatibility layer** is included either way: - every tool XForge builds with is a glibc binary (xtool is a Swift program built - on Ubuntu, and so is the toolchain), Alpine is musl, and `gcompat` is not - enough for them. Baking that layer in removes the most failure-prone step of an - on-device provision (a package renamed between Ubuntu releases yields a layer - that loads but cannot resolve a symbol, which surfaces much later inside a - tool). -- **Everything — the layer, the toolchain, the SDK — is installed before the - fakefs conversion, and that ordering is load-bearing.** `fakefsify` writes `meta.db` as an index of the tree as it - stands, and the engine resolves files through the database rather than by - scanning `data/`. Installing the layer *after* conversion therefore produces a - root where the files are on disk and completely invisible in the guest — 568 - files present, zero rows in `meta.db`. `build-rootfs.sh` asserts the layer is - indexed before it packs. -- **Slimming, then proof.** Before packing, the build drops what an iOS build - never loads — the static *Linux* stdlib, lldb, the editor tooling (sourcekit-lsp, - the index stores), and every download cache (the SDK archive, the Ubuntu `.deb` - pile the glibc layer was built from, the apk index) — and then runs - `install-toolchain.sh verify` with `XFORGE_VERIFY_COMPILE=1`, which *compiles - and runs a Swift program* and whose verdict fails the build. Printing a version - is not compiling; this is the check that makes slimming safe. -- **The root is a pinned release asset** (`rootfs-v5`) rather than rebuilt per IPA - run. `build-ipa.yml` downloads it and verifies its sha256, and fails if the - pinned root does not carry the toolchain its manifest claims; `build-rootfs.yml` - rebuilds and republishes it when the Alpine base, the toolchain or the root's - configuration changes. Committing it to git is not an option — GitHub rejects - any file over 100 MB in a push, and this one is ~1.6 GB. -- **The Darwin SDK comes from XForge's own release** (`darwin-sdk-`, built in CI - with xtool from an Xcode.xip) for the bundled root, and it is recorded — the - manifest names the tag and the sha256 of the bundle that went in, and the path - SwiftPM installed it to. A user with their own Xcode.xip can still put a - different one in: the Toolchain screen installs `xtool sdk install ` and - replaces whatever is there. - -## 2. Where the `darwin` SDK comes from - -`xtool sdk build ` produces the `darwin` SDK from a real Xcode — impossible -on a phone. Plan: **build the `darwin.artifactbundle` once in CI on a macOS runner** -(using xtool's own `SDKBuilder` from an Xcode install), then host it as a downloadable -artifact. That is what the bundled root contains: `EmbeddedLinux/build-rootfs.sh` -downloads the pinned `darwin-sdk-` asset and installs it into the root being built. -For a guest that has none — a plain root, or a swap to a newer SDK — the Toolchain -screen resolves the newest `darwin-sdk-*` release, fetches it into the guest and runs -`swift sdk install` there. +Two structural decisions are worth stating because they are not obvious: -> Note: Apple also publishes official iOS Swift SDKs on swift.org, but they use the -> triple `aarch64-apple-ios` under a different bundle name. xtool hardcodes `darwin` + -> `arm64-apple-ios`, so we build our own `darwin` bundle to match xtool exactly. +- **The build pipeline has one executor.** There used to be a choice between a + local guest and a "remote" backend that never existed. Both the choice and the + guest are gone; `XForgeEnvironment.makeExecutor` returns one implementation. +- **Signing is not part of the build stage.** The stage named "Package & sign + .ipa" packages; the signing happens on the Signing screen, where the user + supplies the identity. Keeping them apart is what lets the same unsigned IPA be + signed by XKit or handed to SideStore. -## 3. App architecture - -``` -XForge.app -├─ Native iOS (SwiftUI, fast path) ───────────────────────────────────────── -│ Project list / editor — author SwiftPM packages -│ Package manifest editor — Package.swift + Sources -│ Git integration — clone/push to GitHub -│ XKit signing — free Apple ID, Apple Developer Services -│ .ipa management — bundle, export, hand off to SideStore/AltStore -│ BuildExecutor protocol — pluggable: Local (embedded iSH) | Remote (future) -│ Embedded Linux VM — runs the compile sandbox -│ -└─ Embedded Linux userspace (aarch64, inside the VM) ──────────────────────── - Alpine/ish-arm64-style rootfs - ├─ glibc compatibility layer (Ubuntu's, for the glibc tools below) - ├─ Swift aarch64 Linux toolchain ┐ - ├─ darwin Swift SDK ├─ in the bundled root, installed at - └─ xtool (aarch64) ┘ build time by install-toolchain.sh - → `xtool new` / `xtool dev build -s -i` -``` +## 5. What is backed up, and what is not -## 3b. What is backed up, and what is not +Everything XForge generates lives in `Documents`, which iOS backs up to iCloud and +— because the app declares `UIFileSharingEnabled` — shows in the Files app. Two +kinds of things live there and they want opposite treatment: -Everything XForge generates lives in `Documents` — which iOS backs up to iCloud and, -because the app declares `UIFileSharingEnabled`, shows in the Files app. Two kinds of -things live there and they want opposite treatment: +- **Regenerable**: downloads, the Darwin SDK, staged build artifacts, the log. + Marked `isExcludedFromBackup` at launch (`XForgeEnvironment.prepareStorage`). + Backing them up bloats every device backup with data the app can produce again, + which the storage guidelines forbid. +- **User data**: projects and exports. Both are backed up, deliberately. -- **Regenerable**: host-side downloads, staged build artifacts, the engine log. Marked - `isExcludedFromBackup` at launch (`XForgeEnvironment.prepareStorage`). Backing them up - bloats every device backup with data the app can produce again, which the storage - guidelines forbid. -- **The guest filesystem** (`Documents/embedded-linux`): excluded as well. It is the - bundled rootfs imported into fakefs, and with `XFORGE_PROVISION=all` that is several - gigabytes of Alpine, Swift, xtool and the Darwin SDK — the least suited thing to - upload to iCloud once per device. +**Projects are ordinary directories in the app container and are backed up.** That +is a reversal of the old position, and the reason it changed is worth recording: a +project used to live *inside* the guest filesystem, which had to be excluded as a +whole — several gigabytes of Linux, toolchain and SDK — and nothing could mark the +megabyte of source without also marking the gigabytes around it. The only way work +survived a device restore was to export it. Now that the toolchain runs in-process, +a project is a directory of files this process can open, so it is excluded from +nothing and restored like any other document. Export remains, and is still the way +work leaves the app, but it is a convenience rather than the only escape hatch. -The price of the second one is that **the user's projects live inside it**, in the same -opaque fakefs, and nothing in the app can mark one without the other. That is why the -export exists: "Export project" on a project's screen tars it *inside the guest* and -copies the archive to `Documents/exports`, which is deliberately **not** excluded — it is -user data, it is small, and it is the thing that should survive a device restore. The -`/host` share (also visible in the Files app) and the Terminal remain the escape hatch -for anything else. +The rule the code follows is: exclude what can be downloaded or rebuilt, back up +what the user wrote. `XForgeEnvironment` is the single place that decides which +directories these are, so the storage rules cannot drift from `prepareStorage`. -If projects ever need to be backed up as projects rather than as archives, they have to -live outside the fakefs. Nothing about a flag on the directory can do it. - -## 4. BuildExecutor abstraction +## 6. BuildExecutor, and what is left of the abstraction ```swift +@MainActor protocol BuildExecutor { - func createProject(_ template: ProjectTemplate) async throws + func bootstrap() async throws -> AsyncThrowingStream + func installSDK(from source: SDKSource) async throws + func createProject(named:organizationIdentifier:) async throws -> Project + func resolve(_ project: Project) async throws -> AsyncThrowingStream func build(_ project: Project, configuration: BuildConfiguration) async throws -> AsyncThrowingStream - func installSDK(_ source: SDKSource) async throws - func fetchToolchain() async throws } ``` -- `EmbeddedLinuxExecutor`: drives the embedded VM via the `LinuxVM` bridge (below). -- `RemoteExecutor` (future): same interface over SSH/WebSocket to a build server. - -## 4b. The Linux engine is **ish-arm64** (in-process, no subprocesses) - -**iOS cannot spawn subprocesses** (no `fork`/`exec`/`posix_spawn` in the app sandbox), -so the embedded Linux cannot run as a child process. It runs **in-process** as a -library. XForge uses [ish-arm64](https://github.com/OpenMinis/ish-arm64) for this: -it is a real Linux kernel + aarch64 emulator whose threaded-code interpreter -dispatches each guest instruction to a pre-compiled "gadget" function. It emits -**no machine code** and needs no executable memory, so it requires **no JIT -entitlement** and works in a sideloaded app. ish-arm64 is vendored as the -`Vendor/ish-arm64` git submodule and built for iOS by -`EmbeddedLinux/build-ish-core.sh`. - -That build requires **clang**: the aarch64 gadget sources alias registers with -`.req` (`_cpu .req x1`, `_pc .req x28`, …) and then use those names as operands, -which only clang's integrated assembler accepts — GNU `as` rejects every such -instruction. This is why the engine is built on a macOS runner rather than on -Linux, and why `build-ish-core.sh` probes for that capability up front instead -of failing with a wall of assembler errors. - -``` -BuildExecutor (EmbeddedLinuxExecutor) - │ drives - ▼ -LinuxVM (EmbeddedLinuxVM) ← command/file bridge, runs on MainActor - │ run / copyIn / copyOut - ▼ -LinuxEmulator (protocol) ← in-process execution engine - └─ ISHEmulator ← drives the embedded ish-arm64 core - │ one dedicated serial queue (the engine's `current` is thread-local) - ▼ -ISHBridge.c ← plain-C shim (bridging header → Swift) - │ - ▼ -libish + libish_emu + libfakefs + fakefs_import (built from Vendor/ish-arm64) -``` -- the engine's primitive is one-shot command capture - (`run_guest_command_capture_shell`), so `LinuxVM.run` executes a command and - forwards its merged stdout+stderr; `copyIn`/`copyOut` still move files via - base64 over the guest shell. -- **Boot** (`ISHBridge.c`) mirrors ish-arm64's own app: mount the imported - rootfs with `mount_root`, create init with `become_first_process`, then mount - `/proc`, `/sys`, `/dev/pts`. It does *not* run `/sbin/init` — XForge runs build - commands as fresh children of init, which is all the headless runner needs. -- The engine is built for the iOS **device** (arm64) only. Simulator builds (used - by unit tests) compile a stub in `ISHBridge.c` instead, so `make test` needs - neither the submodule nor the core libraries. - -## 5. Delivery / sideload pipeline - -- **CI (GitHub Actions, macOS runner)** builds the unsigned IPA (our proven pattern from - SideStore / tctiSH), commits it to a release. -- **Device install**: user installs XForge via SideStore/AltStore (already built by user). -- **Built apps**: XForge produces `.ipa`s that are handed to SideStore to install on the +`NativeBuildExecutor` is the only implementation. The protocol survives for two +reasons: it is the seam a future `RemoteExecutor` would implement, and it keeps the +`BuildManager` state machine honest about what a stage is allowed to assume. + +Every method streams `BuildEvent`s — progress lines for the console, structured +stage transitions for the step UI — so long work reports while it happens instead +of at the end. + +## 7. Honest performance note + +Compilation is native code now, not emulation: the old note about a JIT emulator +being slow no longer applies to the compiler itself. What is still true is that the +executor compiles translation units **one at a time**, so it does not use the +device's cores the way `swift build` or `xcodebuild` do, and that a phone compiles +more slowly than a Mac. How much of either matters for a real project is +**unverified** — it has not been measured on a device against a non-trivial +project, and it cannot even be approached until Swift compiles at all. XForge is a +*demonstrator and authoring tool* for the on-device path; a remote executor is the +sane answer for large projects, and the protocol above is where it would attach. + +## 8. Delivery / sideload pipeline + +- **CI (GitHub Actions, macOS runner)** builds the unsigned IPA, verifies it, and + publishes it as a release asset. +- **Device install**: the user installs XForge via SideStore/AltStore. +- **Built apps**: XForge produces `.ipa`s that are signed with XKit (with an + identity the user supplies) or handed to SideStore to sign and install on the same device. -## 6. Honest performance note - -Swift compilation under JIT emulation is slow. Tiny SwiftUI apps (hello-world / toy -packages) build in minutes. Real-world apps are impractical on-device; that's why the -`RemoteExecutor` exists. XForge is a *demonstrator + authoring tool* for the on-device -path, with a fast remote path available later. - -## 7. Repo layout +## 9. Repo layout ``` -.github/workflows/build-ipa.yml # CI: build the ish-arm64 core + unsigned XForge.ipa -.github/workflows/build-rootfs.yml # CI: build and publish the pinned Alpine rootfs -project.yml # XcodeGen definition -App/ # SwiftUI app sources (native shell) -App/EmbeddedVM/ # LinuxVM bridge, ISHEmulator, C bridge, rootfs unpack -Support/ # Info.plist, entitlements, Resources/ (bundled rootfs) -Vendor/ish-arm64/ # git submodule: the embedded Linux engine -Vendor/ish-arm64-build/ # core static libs (built, gitignored) -EmbeddedLinux/ # build-rootfs.sh, build-ish-core.sh, install-toolchain.sh, - # verify-rootfs.sh, xforge-login (installed into the guest) -Docs/ # this design doc + tutorials -Docs/ISH-ARM64-INTEGRATION.md # engine integration vs. OpenMinis's reference +.github/workflows/ build-ipa.yml (the unsigned IPA), native-toolchain.yml (the + iOS LLVM cross-build, and the toolchain release) +project.yml XcodeGen definition +App/ SwiftUI app sources +App/Build/ the pipeline: plan, invocation, executor, IPABuilder, manager +App/NativeToolchain/ the in-process compiler bridge + the Darwin SDK store +App/Services/ signing, project files, export, releases, log +App/Models/ pipeline, history, project store, XcodeProject +App/Views/ Projects, Build, Signing, Toolchain, Settings +AppTests/ unit tests (plan and argument construction) +Vendor/NativeToolchain/ the installed toolchain bundle (absent by default) +NativeToolchain/ install-bundle.sh, prepare-xcode.sh +Support/ Info.plist spec, entitlements, assets, the generated xcconfig +Tools/ gen-appicon.py +Docs/ this design doc + the pipeline, toolchain and Xcode-alternative notes ``` diff --git a/Docs/IPA-BUILD.md b/Docs/IPA-BUILD.md index c3634de..e80c2c2 100644 --- a/Docs/IPA-BUILD.md +++ b/Docs/IPA-BUILD.md @@ -1,15 +1,20 @@ # XForge IPA Build Tool — design -The on-device engine that turns a SwiftPM project into a signed, sideloadable `.ipa`. -This is the "build tool" layer between the GUI and the embedded Linux. +The on-device engine that turns an authored project into a signed, sideloadable +`.ipa`. This is the "build tool" layer between the GUI and the compiler libraries +linked into the app. ## 1. Goal From a project the user has authored, produce a valid `.ipa`: ``` -SwiftPM project ──► arm64-apple-ios .app ──► signed .ipa ──► SideStore/device - (editable) (compiled in guest) (packaged+sign on host) +project directory ──► NativeBuildPlan ──► arm64-apple-ios executable + (editable) (resolved, refused (clang / swift-frontend, + up front) linked with ld64.lld, + in this process) + ──► .app ──► unsigned .ipa ──► signed .ipa + (Payload/ + zip) (XKit, on the Signing screen) ``` Everything is driven from one orchestrator (`BuildManager`) that exposes a strict @@ -18,36 +23,91 @@ state machine to the UI, so the GUI always knows exactly which stage is running. ## 2. Pipeline stages (state machine) ``` -1. provision ensure embedded Linux is booted + Swift toolchain + xtool present -2. sdk ensure the `darwin` Swift SDK (arm64-apple-ios) is installed -3. configure write xtool.yml + Info.plist from the user's AppInfo/settings -4. resolve `swift package resolve` in the project (pull SPM deps) -5. compile `xtool dev build` → cross-compile to arm64-apple-ios .app -6. package host-side IPABuilder: assemble Payload, write Info.plist/entitlements, - codesign (real identity or ad-hoc), zip → .ipa -7. artifact stage the .ipa, hand it to the UI for export / install +1. provision report the linked toolchain (clang, ld64.lld, swift-frontend) and + require the Darwin SDK; fail with an explanation if either is absent +2. sdk install the Darwin SDK into the app container, if it is not there +3. configure check the project directory and record the app identity +4. resolve report "nothing to resolve", or refuse a project that declares + SwiftPM dependencies +5. compile plan → per-file compile → link → assemble .app → package unsigned .ipa +6. package accept the packaged artifact +7. artifact confirm the .ipa exists, is non-empty, and report its size ``` Each stage is idempotent (skippable if already done) and reports progress. Any stage -can fail; failures carry a human message + a machine-readable reason. +can fail; failures carry a human message, and the pipeline stops at the **first** +one, because everything after it fails as a consequence and would bury the cause. -## 3. Split of responsibilities +There is no "boot a guest" stage, and that is the whole point of the current design: +the compiler is a library call, so there is nothing to start before compiling. -| Concern | Where | Why | -|---|---|---| -| Provision, SDK install, resolve, compile | **Embedded Linux** (VM) via `BuildExecutor` | needs Swift + xtool + SDK | -| Config injection (xtool.yml, Info.plist) | host, before compile | user edits live in the GUI | -| Packaging (Payload, zip), final signing | **host** via `IPABuilder` | testable, works with any .app | -| Certificates / Apple ID | host via `SigningService` (XKit) | native on iOS | +### What each stage actually does -This split means `IPABuilder` is fully testable without the VM: give it any compiled -`.app` + `AppInfo` + a signer and it produces a valid `.ipa`. +| Stage | Implementation | Notes | +|---|---|---| +| provision | `NativeBuildExecutor.bootstrap` + `NativeToolchainCapabilities` | Reports one line per component. Fails when clang or LLD is not linked; *warns* when the Swift frontend is not. | +| sdk | `NativeSDK.install(fromRemote:)` | Skips itself when the bundle is already in the container. | +| configure | `BuildManager.configure` | Writes `/.xforge-build/build.json` (bundle ID, version, build number, configuration) so a failed build is reproducible, and throws `missingProjectDirectory` if the directory is gone. | +| resolve | `NativeBuildExecutor.resolve` + `NativeBuildPlanFactory.swiftPackageDependencies` | A textual scan of `Package.swift`; the answer it needs is only "are there any". | +| compile | `NativeBuildExecutor.build` | The real work: plan, compile, link, assemble, package. Also the stage that refuses Swift sources (`swiftFrontendMissing`). | +| package | `BuildManager.package` | The executor packages the IPA as the last step of compiling, so this stage accepts the result rather than producing it. | +| artifact | `BuildManager.stageArtifact` | A final stage that checks the file rather than reporting success unconditionally. | + +### The Swift frontend limitation, stated where it bites + +`NativeBuildExecutor.build` throws `NativeBuildError.swiftFrontendMissing(n)` — with +`n` the number of Swift files — when the plan has Swift sources and the frontend is +not linked. In the shipped toolchain bundle the frontend is never linked: the CI +workflow that builds the bundle builds Clang and Mach-O LLD only. So **a Swift +project cannot be compiled today**, and the failure happens before any file is +compiled, not after a partial build. + +The consequence for this pipeline is structural, not cosmetic: the stages that +exist are correct and exercised for C and Objective-C targets, and the Swift- +specific stages (module boundaries, a dependency graph, asset catalogs) are the +work that remains. Until then the pipeline can honestly describe itself as "a build +tool that compiles C", and the app says so rather than letting a user discover it +three stages in. -## 4. Signing strategy +## 3. Split of responsibilities -The current build exports an unsigned `.ipa`, which must be signed by Xcode, -SideStore/AltStore, or another signing service before installation. XKit-based -Apple ID authentication and signing are planned but are not wired into this build. +| Concern | Where | Why | +|---|---|---| +| Reading a project into a plan | `NativeBuildPlanFactory` (host) | Pure directory reading; no compiler needed, so it is fully testable | +| Argument construction | `NativeToolchainInvocation` (host) | Pure functions; the compiler only exists on a device, so this is the only half that can be unit-tested | +| Compiling and linking | `NativeToolchain*` (in-process libraries) | iOS cannot spawn a compiler | +| Bundle assembly, `Info.plist` merge | `NativeBuildExecutor` | Needs the plan and the compiled executable | +| Packaging (Payload, zip) | `IPABuilder` (host) | Testable, works with any `.app`, has no opinion about what built it | +| Signing | `AppBundleSigner` / `IPASigningJob` (XKit, in-process) | A separate step, on its own screen, so the unsigned IPA stays a first-class artifact | + +This split means `IPABuilder` is fully testable without a compiler: give it any +compiled `.app` + `AppInfo` and it produces a valid `.ipa`. + +## 4. Signing strategy: unsigned first, then signed + +The build produces an **unsigned** `.ipa`, staged at +`/staging/-.ipa`. Signing is `IPASigningJob`: + +1. Unzip the `.ipa` into a temporary directory and find the single `.app` under + `Payload/` (more or fewer than one is an error, not a guess). +2. Read the `.p12` through the Security framework (`PKCS12Identity.load`), with the + password held only for the duration of the attempt and cleared afterwards. The + private key is used in memory and written nowhere — which is why the old + `zsign` password-file workaround is gone with `zsign`. +3. Apply the identity overrides the user typed (bundle ID, display name, version) + to the bundle's `Info.plist`. Only those three: replacing the plist wholesale + would drop `CFBundleExecutable` and produce an app that cannot launch. +4. Write `embedded.mobileprovision` into the bundle **before** signing — the signer + reads it to decide which entitlements the signature may carry, and a signature + that does not match its profile is rejected at install time. +5. Sign with XKit's `Signer`, and re-zip the `Payload/` as + `/Signed-.ipa`. + +What this does **not** do is obtain a certificate. An Apple ID session needs +anisette + GrandSlam + 2FA (XKit's `SigningContext`/`DeveloperServices` path), which +is a separate, device-only piece of work; `XKitSigningService.signIn` throws rather +than reporting a signed-in account it does not have. The signer takes an existing +`.p12` and profile. ## 5. The pipeline's own types @@ -69,48 +129,66 @@ struct PipelineSnapshot: Equatable { } ``` -Each stage is a method on `BuildManager` run in order (`provision` → `ensureSDK` → -`configure` → `resolve` → `compile` → `package` → `stageArtifact`), and each one stops -the pipeline on failure. `BuildExecutor` (`App/Models/Project.swift`) is the seam to the -VM: `bootstrap()`, `createProject`, `installSDK`, `resolve` and `build` all return an -`AsyncThrowingStream`, which `BuildManager.consume` folds into the -snapshot. Packaging is a single host-side call: +Each stage is a method on `BuildManager` run in order, and each one stops the +pipeline on failure. `BuildExecutor` is the seam to the toolchain: `bootstrap()`, +`createProject`, `installSDK`, `resolve` and `build` all return an +`AsyncThrowingStream`, which `BuildManager.consume` folds into +the snapshot. Packaging inside the executor is a single host-side call: `IPABuilder.buildIPA(appBundle:appInfo:outputDir:)`. +The stage **title** for `package` in `BuildPipeline.swift` reads "Package & sign +.ipa", which is a leftover from when the stage did both; the stage now packages and +the Signing screen signs. + ## 6. Failure handling -- Stage-scoped: a failure marks that stage `.failed` and stops the pipeline (unless the - stage is retryable and the user retries). -- `BuildEvent` stream carries both progress lines (for the console) and structured - stage transitions (for the step UI). -- Staged artifacts are written under `Documents/staging/` and listed by the Artifacts - screen. +- Stage-scoped: a failure marks that stage `.failed` and stops the pipeline (unless + the stage is retryable and the user retries). +- The `BuildEvent` stream carries both progress lines (for the console) and + structured stage transitions (for the step UI). +- Compile and link failures carry the tool's diagnostics text; an empty diagnostics + buffer is reported as "reported a failure without diagnostics" rather than + silently producing a bare exit code. +- Staged artifacts are written under `Documents/staging/` and listed by the + Artifacts screen. Intermediate objects and the `.app` live under + `/.xforge-build`, which is discarded at the start of the next build. ## 7. Testability -- `IPABuilder` unit tests build a fake `.app` fixture in the test bundle and assert the - produced `.ipa` contains `Payload/.app/Info.plist` with the right values and is - a valid zip. -- `XcodeProjectTests` covers the `.xcodeproj` reader against an embedded project file. -- `ReleaseResolutionTests` drives `ToolchainManager` and `XForgeReleases` through a - stubbed `LinuxVM`/`ShellSession`, so SDK resolution is testable without the VM. +- `NativeBuildTests` covers the half of the build that can be tested without a + compiler: plan acceptance and refusals (including the SwiftPM-dependency refusal + and the textual dependency scan ignoring local targets), the Swift frontend and + linker argument lists, the `main.swift` case, and search-path filtering. +- `IPABuilderTests` builds a fake `.app` fixture in the test bundle and asserts the + produced `.ipa` contains `Payload/.app/Info.plist` with the right values and + is a valid zip. +- `XcodeProjectTests` covers the `.xcodeproj` reader against an embedded project + file. +- Nothing tests the compiler itself: it does not exist in a simulator build. That + is the boundary this design accepts, and the reason argument construction was + extracted into pure functions. ## 8. Files ``` -Docs/IPA-BUILD.md this design -App/Models/BuildPipeline.swift BuildStage, BuildStageState, PipelineSnapshot -App/Models/Project.swift BuildExecutor protocol, BuildEvent -App/Build/BuildManager.swift orchestrator / state machine -App/Build/IPABuilder.swift host-side packaging (Payload, Info.plist, zip) -App/Build/EmbeddedLinuxExecutor.swift the VM-backed BuildExecutor -App/Models/BuildHistoryStore.swift run history for the History screen -App/Services/SigningService.swift signing abstraction (integration pending) -AppTests/IPABuilderTests.swift packaging unit tests -App/Views/Build/* UI driven by BuildManager.snapshot +Docs/IPA-BUILD.md this design +App/Models/BuildPipeline.swift BuildStage, BuildStageState, PipelineSnapshot +App/Models/Project.swift BuildExecutor protocol, BuildEvent, SDKSource +App/Build/BuildManager.swift orchestrator / state machine +App/Build/NativeBuildPlan.swift the plan and the factory that refuses +App/Build/NativeToolchainInvocation.swift argument construction (pure) +App/Build/NativeBuildExecutor.swift the BuildExecutor: compile, link, assemble, package +App/Build/IPABuilder.swift host-side packaging (Payload, Info.plist, zip) +App/Build/NativeToolchainCapabilities.swift what this build can compile with +App/Services/AppBundleSigner.swift XKit signing, in process +App/Services/IPASigningJob.swift the signing workflow (unzip, overrides, re-zip) +App/Models/BuildHistoryStore.swift run history for the History screen +AppTests/NativeBuildTests.swift plan + argument unit tests +AppTests/IPABuilderTests.swift packaging unit tests +App/Views/Build/* UI driven by BuildManager.snapshot ``` -Docs that go with this one: `Docs/DESIGN.md` (the whole app), -`Docs/XCODE-ALTERNATIVE.md` (building existing Xcode projects on-device), -`Docs/NATIVE-TOOLCHAIN.md` (the in-process LLVM path) and -`Docs/ISH-ARM64-INTEGRATION.md` (the embedded Linux engine). +Docs that go with this one: `Docs/DESIGN.md` (the whole app) and +`Docs/XCODE-ALTERNATIVE.md` (building existing Xcode projects on-device). The +toolchain bundle that makes any of this run is described in +`Docs/NATIVE-TOOLCHAIN.md`. diff --git a/Docs/ISH-ARM64-INTEGRATION.md b/Docs/ISH-ARM64-INTEGRATION.md deleted file mode 100644 index 198ced2..0000000 --- a/Docs/ISH-ARM64-INTEGRATION.md +++ /dev/null @@ -1,204 +0,0 @@ -# Embedding ish-arm64: what the reference requires, and where XForge stands - -XForge's embedded Linux is **ish-arm64** (`Vendor/ish-arm64`, a git submodule of -[OpenMinis/ish-arm64](https://github.com/OpenMinis/ish-arm64)) running a **plain -Alpine aarch64 rootfs** that is shipped as an already-converted **fakefs ZIP**. - -The reference for that integration is OpenMinis's own app — the engine's author — -and its guide `deps/ISH_INTEGRATION.md` ("iSH-ARM64 iOS 集成指南"), plus the two -scripts that build the engine (`deps/build_ish.sh`) and the root -(`deps/prepare_alpine_rootfs.sh`). This document records what the guide requires, -what XForge does instead, and *why* where the two differ. It exists because the -differences are the interesting part: several of them are deliberate, and the ones -that are not were bugs. - -## The guide's checklist, item by item - -| The guide says | XForge | Where | -|---|---|---| -| Link `deps/libs/libish.a`, `libish_emu.a`, `libfakefs.a` | exactly those three | `project.yml` (`OTHER_LDFLAGS[sdk=iphoneos*]`) | -| Add system library `libsqlite3.tbd` | `-lsqlite3` | same | -| Header search path `deps/include` | `$(ISH_ROOT)` and `$(ISH_ROOT)/deps/libarchive/libarchive` | `project.yml` (`HEADER_SEARCH_PATHS[sdk=iphoneos*]`) | -| Other linker flags `-ObjC`, `-all_load` | **not used** — see below | — | -| Other C flags `-DISH_INTERNAL` | **not used** — see below | — | -| Bundle `alpine-rootfs.zip` | bundled, and CI asserts the app contains exactly one rootfs | `.github/workflows/build-ipa.yml`, "Verify the built app" | -| Bundle `libvdso.so.elf` (*optional* per the guide) | not bundled: it is compiled and linked **into `libish.a`** | `EmbeddedLinux/build-ish-core.sh` | -| Unzip the root to `Documents/alpine-rootfs`, mount `/data` | same layout, unpacked into a staging directory and moved into place | `App/EmbeddedVM/RootfsInstaller.swift` | -| `mount_root(&fakefs, /data)` then `become_first_process()` | same order, then device nodes, `/proc`, `/dev/pts` | `App/EmbeddedVM/ISHBridge.c` (`xf_ish_boot`) | -| `current->thread = pthread_self()` after `become_first_process()` | not needed: every guest task is started with `task_start()`, which sets `task->thread` via `pthread_create` (`kernel/task.c:243`) | `ISHBridge.c` (`xf_ish_start_init`, `xf_spawn_child`) | -| Register a custom tty driver: `DEFINE_TTY_DRIVER(..., TTY_CONSOLE_MAJOR, 8)` with `init`/`write`/`cleanup` | same driver on the same major, plus `/dev/tty1` and `set_console_device(4,1)` | `ISHBridge.c` (`xf_console_driver`) | -| Feed keystrokes with `tty_get()` + `tty_input()` + `tty_release()` | equivalent: the tty is recorded in the driver's `init` and typed into with `tty_input` (which reads no per-thread engine state) | `ISHBridge.c` (`xf_ish_console_write`, `xf_ish_console_*`) | -| `do_execve` + `task_start(current)` to run the console program | same, for pid 1 **and** for every command the app runs | `ISHBridge.c` | -| Build options (`build_ish.sh`): `--buildtype=release`, `-Db_ndebug=true`, `-Dlog=""`, `-Dlog_handler=nslog`, `-Dkernel=ish`, `-Dengine=asbestos`, `-Dguest_arch=arm64` | identical set | `EmbeddedLinux/build-ish-core.sh` | - -### Why no `libarchive` (or `libfakefsify`) - -The reference links libarchive and builds the fakefs tools for the app. XForge did -too, until the app's own header said otherwise: the bundled root is *already* a -fakefs ZIP, unpacked with ZIPFoundation into `Documents`, and `fakefs_import` is -never called — "there is no import step here: `fakefs_import` is a build-time tool -in this design, not a runtime one" (`App/EmbeddedVM/ISHBridge.h`). The `fakefsify` -the build machine needs is built by the engine's meson project on Linux, in -`EmbeddedLinux/build-rootfs.sh`. - -So `-larchive`, `-lfakefsify`, the libarchive header path, the -`deps/libarchive` submodule fetch and the libarchive iOS build are all gone from -the app's build. It was not just dead weight: libarchive has no iOS build system of -its own here, so `build-ish-core.sh` drove its Xcode project with a *nested* -`xcodebuild` — and nested inside Xcode's own build phase, that crashed the Xcode -build system (`error: unexpected service error: The Xcode build system has crashed`) -after 90/90 engine targets had already succeeded. Removing the dependency removed -the failure. - -### Why not `-ObjC -all_load` - -The guide asks for both, and the reason is a static-library one: a linker drops any -object file nothing references, and Objective-C categories (and code reachable only -through the ObjC runtime or a constructor) can be dropped with it. Neither applies -here: - -* XForge's engine libraries are **C** — there is no category to preserve — and the - bridge references every subsystem it needs directly, so a wrongly-dropped object - would be a *link error* rather than silent misbehaviour. -* The objects that *do* rely on a constructor (`fs/fake.c`'s `init_fake_fdops`, - `kernel/memory.c`'s `get_real_page_size`, `kernel/task.c`'s `create_attr`) are all - in objects that are already pulled in by symbols the app calls (`fakefs`, - `mount_root`, `task_start`). The engine's own authors avoid constructors precisely - where that is not true — see the comment in `kernel/native_offload_sym.c:111` - ("not via a constructor, which a static-library linker dead-strips when the TU is - otherwise unreferenced"). -* `-all_load` would then force the whole of `libish.a` and `libarchive.a` into the - app, for no reachability benefit. - -### Why not `-DISH_INTERNAL` - -The guide needs it because its `ISHKernel.m` includes `ish/fs/fake.h`, which refuses -to compile otherwise: - -```c -#ifndef ISH_INTERNAL -#error "for internal use only" -#endif -``` - -`fs/fake.h` is the *only* header with that gate. XForge's bridge needs the fakefs -instance, and that is declared in a header that is not gated -(`kernel/fs.h:184: extern const struct fs_ops fakefs;`), which the bridge already -includes. If it ever needs `fs/fake.h` (the bind-mount API there, for instance), -`-DISH_INTERNAL` has to be added to `OTHER_CFLAGS[sdk=iphoneos*]` in `project.yml`. - -## What the console is, and why XForge's is not the reference's - -The guide's terminal does this: - -```swift -create_stdio("/dev/tty1", TTY_CONSOLE_MAJOR, 1) // pid 1's stdio is the console -ISHKernel.execute(command: ["/bin/sh", "-l"]) // …then pid 1 *becomes* the shell -``` - -So in the reference the shell **is pid 1**, and the shell's controlling terminal is -the one pid 1 claimed when its stdio was wired. XForge instead boots a real init -(`/sbin/init`) which respawns `/sbin/xforge-login root` on `tty1`; that script reads -root's login shell out of `/etc/passwd` and execs it. That buys a guest that behaves -like a system — a real init for the guest's own scripts, `exit` giving a fresh -session, a motd — and it is what the Terminal tab and the whole rootfs design assume. - -It also had a consequence the reference never sees, because the reference never -creates a second session: **`create_stdio` opens the console without `O_NOCTTY`**, -so wiring pid 1's stdio made the console *pid 1's* controlling terminal and pinned -`tty->session` to it (`fs/tty.c:126`, `tty_open` hands a tty over only while -`tty->session == 0`). The login session init then started could never take it, and -the shell ran with no controlling terminal: bash printed - -``` --bash: cannot set terminal process group (-1): Not a tty --bash: no job control in this shell -``` - -and no `Ctrl-C`, `Ctrl-Z` or `SIGWINCH`-on-resize reached the shell. XForge's fix is -to open pid 1's stdio with `O_NOCTTY` (`xf_create_console_stdio` in -`App/EmbeddedVM/ISHBridge.c`), which is what Linux does too: init has no controlling -terminal, the console belongs to the login session that opens it. Same flags, same -`create_stdio` shape, one bit different — and the guest keeps its init. - -## The rootfs: what the reference configures, and what XForge adds - -`deps/prepare_alpine_rootfs.sh` downloads the plain Alpine minirootfs, converts it -with the engine's own `tools/fakefsify`, configures it in place (mount points, -`/etc/passwd`, `/etc/profile`, `/etc/motd`, `/etc/inittab`, `/etc/resolv.conf`, -`/etc/apk/repositories`) and packs it with `zip -r alpine-rootfs.zip`, excluding the -SQLite `-wal`/`-shm` sidecars. XForge's `EmbeddedLinux/build-rootfs.sh` does the same -thing and keeps the same ordering rule, which is the one that is easy to get wrong: - -> **`fakefsify` writes `meta.db` as an index of the tree as it stands.** Anything -> added to the root after the conversion exists in `data/` and is invisible to the -> guest. XForge installs the glibc layer and the guest's packages *before* the -> conversion, and asserts afterwards that they are indexed. - -On top of the reference's list, XForge's root also carries: - -* the **glibc compatibility layer** (`/opt/glibc`, wired through `/lib` and - `/usr/lib`) — xtool and the Swift toolchains are glibc binaries and Alpine is musl; -* the **console packages** (`bash coreutils less ncurses-terminfo`) and root's login - shell in `/etc/passwd`; -* `/sbin/xforge-login` and XForge's `/etc/inittab` (the stock one starts openrc, - which is not in this root, and six gettys on terminals that do not exist); -* `EmbeddedLinux/verify-rootfs.sh`, which runs the console program in a chroot of the - root — for a plain tree — and checks the fakefs ZIP through `meta.db`, which is the - database the engine actually resolves through. - -## Building it: the engine and the rootfs are prerequisites of the app - -The guide treats the engine as something you build first and then link. That is fine -for a human following instructions and wrong for a pipeline: any build that does not -know to run `deps/build_ish.sh` / `EmbeddedLinux/build-ish-core.sh` first fails -part-way through compiling the bridge, with - -``` -App/EmbeddedVM/ISHBridge.c:123:10: error: 'kernel/init.h' file not found -``` - -which reads like a missing header rather than a missing engine (and a fresh clone -has the submodule empty, so the header really is missing). So the app target now -carries a **first build phase**, `EmbeddedLinux/build-engine-for-xcode.sh`, which: - -* skips itself for simulator builds (which compile the stub branch in `ISHBridge.c`) - and for `XFORGE_SKIP_ENGINE=1`; -* fetches the engine's submodules if the checkout is empty; -* installs meson/ninja/llvm/lld/libarchive with Homebrew when they are missing - (Xcode build phases get a bare `PATH`, so Homebrew is put on it first); -* runs `build-ish-core.sh`, which is a no-op unless the engine revision, the script - or `project.yml` changed (`XFORGE_REBUILD_ENGINE=1` forces a rebuild). - -The result is that `xcodebuild`, Xcode itself, the IPA workflow and any other -pipeline all produce the same app from the same checkout. - -The **rootfs** is a prerequisite in the same sense, but it cannot be fetched from a -build phase: it is a *resource*, so it has to be on disk before the project is -generated (XcodeGen fixes the resource list when it walks the directory). Hence -`EmbeddedLinux/fetch-rootfs.sh`, which reads the pinned tag out of -`build-ipa.yml`, downloads the asset, verifies its published sha256 and checks it is -a fakefs ZIP — run by the IPA workflow before XcodeGen, by the generic iOS pipeline, -and by hand before opening the project in Xcode. A device build whose resource is -missing fails on purpose (`Check the bundled guest rootfs`), because an app without a -root is an app that cannot boot Linux, and finding that out on a phone is worse than -a red build. - -## Verification, and what is still only verified on a device - -What CI checks, on every IPA build (`.github/workflows/build-ipa.yml`): - -* the pinned rootfs release's sha256, its contents, and that the glibc layer and the - console paths are indexed in `meta.db`; -* that the engine libraries exist and that `libish_emu.a` carries the arm64 backend; -* that the app is arm64, contains exactly one rootfs (the fakefs ZIP), has no build - leftovers, and that its entitlements are the four the engine needs - (`extended-virtual-addressing` and `increased-memory-limit` among them); -* for the rootfs itself, that `/sbin/xforge-login` starts the shell `/etc/passwd` - names, as a login shell, in a chroot of the root. - -What that cannot check is the guest *running on iOS*: the engine executes in-process, -under the app's entitlements, with the host's real networking and the app's tty. That -is what the Terminal tab and the smoke harness (`Tools/engine-smoke/`) are for, and -it is why a change to the console path is worth trying on a device before believing -it. diff --git a/Docs/NATIVE-TOOLCHAIN.md b/Docs/NATIVE-TOOLCHAIN.md index fcbb2c0..7739940 100644 --- a/Docs/NATIVE-TOOLCHAIN.md +++ b/Docs/NATIVE-TOOLCHAIN.md @@ -1,95 +1,275 @@ -# Native iPhone toolchain proof of concept +# The native iPhone toolchain -This branch starts the migration away from the embedded Linux runtime. - -## Goal - -Run the compiler and Mach-O linker directly inside `XForge.app` on a stock iPhone: +XForge compiles in its own process: there is no Linux guest and no subprocess. The +compiler and the Mach-O linker are LLVM libraries **linked into the app binary**, +called through a small Objective-C++ bridge. ``` -source -> Clang/Swift frontend -> arm64 object -> LLD Mach-O -> .app -> sign -> IPA +source -> Clang / swift-frontend -> arm64 object -> ld64.lld -> .app -> .ipa + (in-process, no fork/exec/posix_spawn, no remote build host) ``` -No Linux rootfs, `fork`, `exec`, `posix_spawn`, or remote build host is required at runtime. +## Goal + +Run the compiler and Mach-O linker directly inside `XForge.app` on a stock iPhone. +iOS cannot spawn a process, so "run clang" means "call clang's libraries"; the only +alternative would be a remote build host, which is a different product. ## What is implemented -- `NativeToolchainBridge.mm`: in-process Objective-C++ bridge for Clang codegen and - the LLD Darwin driver. -- `NativeToolchain.swift`: safe Swift wrapper plus an end-to-end C -> arm64 iOS - object smoke test. -- Stub behavior when the LLVM bundle is not linked, so normal XForge builds remain - buildable while the native toolchain artifact is being produced. -- `.github/workflows/native-toolchain.yml`: builds the iOS-hosted LLVM/Clang/LLD - libraries on a macOS GitHub runner and uploads them as - `XForgeNativeToolchain-arm64-ios.tar.gz`. -- `NativeToolchain/install-bundle.sh `: unpacks that artifact into - `Vendor/NativeToolchain` and runs `NativeToolchain/prepare-xcode.sh`, which writes - `Support/NativeToolchain.generated.xcconfig` — the file the app target consumes - (checked in with the backend disabled, so an ordinary clone builds without it). - `build-ipa.yml` runs `prepare-xcode.sh` too, so CI never depends on the bundle - being installed locally. - -The existing Alpine executor is intentionally retained as a fallback until the native -backend can compile Swift and package a complete app. - -## Runtime contract +- `App/NativeToolchain/NativeToolchainBridge.mm` — the in-process bridge: Clang + codegen (`CompilerInvocation` + `EmitObjAction`), the `swift::performFrontend` + entry point, and the LLD Darwin driver. +- `App/NativeToolchain/NativeToolchain.swift` — the Swift wrapper, plus an + end-to-end C compile smoke test. +- `App/Build/NativeToolchainInvocation.swift` — the argument lists as pure + functions. The convention is not obvious and is worth stating: the frontend entry + point is handed everything *after* `-frontend`, so neither the program name nor + `-frontend` itself may appear in the array; passing it makes + `CompilerInvocation::parseArgs` reject the whole invocation. +- **Stub behaviour when the bundle is not linked**, so an ordinary clone (and every + simulator build) compiles and runs with no toolchain present. +- `.github/workflows/native-toolchain.yml` — builds the iOS-hosted LLVM/Clang/LLD + libraries and publishes the bundle. See below. +- `NativeToolchain/install-bundle.sh | --release [tag]` — installs that + bundle, from a local file or from its release. +- `App/NativeToolchain/XipArchive.swift` — reads Apple's `.xip` container far enough + to take an SDK out of it. xar header and table of contents, the pbzx stream inside + the `Content` member, and the `odc` cpio archive that decompresses to — streamed, + one 16 MB block at a time, with Apple's own `Compression` decoding the LZMA2 blocks + so the app carries no decompressor of its own. +- `App/NativeToolchain/DarwinSDKBuilder.swift` — turns an `Xcode.xip` into a + `darwin.artifactbundle` on the device: xtool's path list, its bundle layout, and + the `swift-sdk.json` `NativeSDK` reads. +- `NativeToolchain/prepare-xcode.sh` — writes the xcconfig the app target consumes. + `build-ipa.yml` runs it too, so CI never depends on a bundle being installed + locally; on a checkout with no bundle it writes the disabled configuration and CI + builds the stub. -The native implementation is enabled only when the app target defines -`XFORGE_HAS_LLVM=1` and can see the LLVM/Clang/LLD headers. Otherwise the bridge -returns `Native LLVM backend not linked`. +## What a bundle contains — and which half it has + +The workflow cross-builds, for iPhoneOS arm64: + +- **Clang**, as the library set the bridge calls (`clangCodeGen`, `clangFrontend`, + `clangFrontendTool`, `clangDriver`, `clangSerialization`, `clangSema`, `clangParse`, + `clangAST`, `clangLex`, `clangBasic`), plus `clangDependencyScanning`; +- **Mach-O LLD** (`lldMachO`, `lldCommon`); +- **`LLVMOrcJIT`**, which Swift's in-process JIT needs; +- and, in a `with_swift` build, **Swift's frontend libraries** (`swiftFrontendTool` + and the swiftAST / swiftSema / IRGen / ClangImporter set it pulls in), built from + `swiftlang/swift` against `swiftlang/llvm-project` — for iOS, as a library set. -The first validation milestone is: +The target list is not "everything the app might use" but the closure the bundle has +to satisfy, because the link check force-loads every archive: each member's undefined +symbols must resolve *inside* the bundle. `clangDependencyScanning` (used by +`ClangImporter` to scan a target's module dependencies) and `LLVMOrcJIT` (used by +`SwiftMaterializationUnit.cpp`) are the two that no clang library pulls in by itself, +and leaving them out produced a bundle that compiled and then failed to link — with +the missing symbols visible only as a truncated tail. The link check now prints all +of them. + +`manifest.txt` is how a consumer knows what it has: ``` -int xforge_native_smoke(void) { return 42; } - | - v -clang::CompilerInstance + EmitObjAction - | - v -arm64-apple-ios main.o +bundle_tag=toolchain---ios-sdk +target=arm64-apple-ios +deployment_target=17.0 +llvm_commit=… +swift_frontend=0|1 +swift_ref=swift-6.2.4-RELEASE|none +xcode=16.4 +ios_sdk=18.5 +built_at=… +archives=lib/XForgeToolchain.libraries.txt +header_roots=include-generated include ``` -The next milestone is linking that object with the Darwin LLD driver against a -minimal iPhoneOS SDK, followed by embedding the Swift frontend. +`prepare-xcode.sh` reads `swift_frontend` and defines `XFORGE_HAS_SWIFT_FRONTEND` +accordingly, so a bundle without the frontend still builds the whole app: the bridge +compiles to the stub for that entry point and the UI reports `swift-frontend: missing` +rather than claiming a compiler it does not have. +The two halves are deliberately independent. The Clang/LLD half is a complete +deliverable for C, Objective-C and Objective-C++ targets and can be validated on its +own while the considerably larger Swift port is brought up; the tag says which one a +bundle is (`…-swift--…` versus `…-noswift-…`), so they never have to be guessed +apart. -## Native Darwin SDK +## How the bundle is built (CI) -The app now has a host-side SDK store at: +`.github/workflows/native-toolchain.yml`, job `llvm-ios`, on a `macos-15` runner: -``` -Documents/native-sdk/darwin.artifactbundle -``` +1. Check out Swift's LLVM fork (`swiftlang/llvm-project`, the branch paired with the + Swift version) and, for a `with_swift` build, `swiftlang/swift`, `swift-cmark`, + `swift-syntax` and `swift-experimental-string-processing`. +2. Restore the build trees from the Actions cache. The key names the LLVM revision, + the deployment target and the runner's Xcode/SDK, so a hit means "the compiler for + exactly these inputs" — and the sources are then aged to a fixed past date, so the + restored tree is newer than every source. Without that ageing ninja treats the + whole tree as stale and rebuilds it: the restore paid seconds to avoid a + 2601-edge rebuild and got it anyway. +3. Build the host tools natively (`llvm-tblgen`, `clang-tblgen`, `llvm-ar`, + `llvm-nm`) and, with Swift, a native `cmark` plus the iOS cmark archives. +4. Configure LLVM for iPhoneOS arm64 with `LLVM_ENABLE_PROJECTS="clang;lld"`, + `LLVM_TARGETS_TO_BUILD=AArch64`, deployment target 17.0, tools and examples off. + Skipped when the restored tree came back already configured and built. +5. Build the library targets listed above (~37 minutes cold), then, with Swift, + the frontend libraries (~34 minutes cold). +6. **Compile-check the bridge against the headers just built** — `clang++ -arch arm64 + -isysroot -DXFORGE_HAS_LLVM=1 -c App/NativeToolchain/NativeToolchainBridge.mm`. + This is the check that keeps a header change in LLVM from silently breaking the + bridge in a device build nobody can run yet. +7. Stage the headers into two roots — sources into `include/`, the build tree into + `include-generated/`, because they disagree about `swift/bridging` — and merge + every static archive in the build into one `dist/lib/libXForgeNativeToolchain.a` + with `llvm-ar -M` and an MRI `addlib` script. `addlib` copies *every* member: + `libtool -static` de-duplicates them by name (`duplicate member name 'X86.cpp.o' + from libclangCodeGen.a and …`), and a dropped member is a silently missing object — + which is how the JIT symbols once came out undefined while their object was + demonstrably in the archive. The merge fails if it produced fewer members than it + consumed. +8. **Link-check**: link a program that calls `xf_native_toolchain_available()` against + the merged archive and the bridge object, with every archive force-loaded, and + assert the result is a real Mach-O. Force-loading makes this the strictest check + available: it fails unless every member resolves inside the bundle. On failure it + prints every undefined symbol, not a tail of them. +9. Write `manifest.txt` and tar the result as `XForgeNativeToolchain-arm64-ios.tar.gz` + — uploaded as the workflow artifact of the same name, and, when a hand-dispatched + run that built the Swift half succeeded, **published as a release asset** tagged + `toolchain--swift--ios-sdk`. A `noswift` bundle is a + complete deliverable for C and Objective-C targets but is not published: a + consumer asking for "the newest toolchain release" means the full one, and the + tag would otherwise be the only thing standing between them and a bundle with no + frontend in it. -It consumes xtool's existing `swift-sdk.json` metadata directly and resolves the -`arm64-apple-ios` SDK root, Swift resource directory and platform library search -paths without booting Alpine. +The cache is why none of this needs doing twice. A run whose inputs are unchanged +restores both trees and goes straight to the staging and link checks. Everything +unusual about the workflow — the key salts, the save gating (a tree is stored only if +the build that produced it finished), the "skip the configure when the tree is already +built" rule, the aged sources — exists for that one property, and each was added after +a run that spent an hour rebuilding what the cache already had. -The Native Toolchain settings screen can download the existing -`darwin.artifactbundle.zip` release directly into this store and run the C smoke test. +Publishing is what takes the cost off everyone else. `build-ipa.yml` installs the +bundle from the newest `toolchain-*` release, so building the app never waits for a +toolchain run and a toolchain run never has to be in the same repository state as the +app it serves. Only dispatched runs publish: a pull request or push run builds the +LLVM-only half, and publishing that as the newest bundle is exactly what a consumer +asking for "the newest one" must not get. -## Swift frontend boundary +## How it is installed + +```bash +make toolchain-release # the newest published bundle +make gen +``` -The bridge now also exposes an optional Swift frontend: +A specific release, a local tarball, or a bundle that was built but not published: +```bash +make toolchain-release TAG=toolchain-<…> +make toolchain ARCHIVE=XForgeNativeToolchain-arm64-ios.tar.gz +NativeToolchain/install-bundle.sh --release artifact ``` -Swift source - | - v -xf_native_swift_frontend - | - v -swift::performFrontend(...) - | - v -arm64-apple-ios .o + +`install-bundle.sh` fetches the asset (the GitHub CLI if it is installed, `curl` +otherwise) or unpacks the local archive into `Vendor/NativeToolchain`, and refuses an +archive that is missing `manifest.txt`, `include/`, `include-generated/` or `lib/` — a +partial bundle would otherwise surface as an inscrutable compile error much later. It +then runs `prepare-xcode.sh`, which writes `Support/NativeToolchain.generated.xcconfig`: + ``` +XFORGE_NATIVE_TOOLCHAIN_AVAILABLE = 1 +XFORGE_NATIVE_HEADER_SEARCH_PATHS = $(SRCROOT)/Vendor/NativeToolchain/include-generated $(SRCROOT)/Vendor/NativeToolchain/include +XFORGE_NATIVE_LIBRARY_SEARCH_PATHS = $(SRCROOT)/Vendor/NativeToolchain/lib +XFORGE_NATIVE_CFLAGS = -DXFORGE_HAS_LLVM=1 +XFORGE_NATIVE_LDFLAGS = -lc++ -lz -liconv -lsqlite3 -framework Foundation +``` + +The two header roots are searched in that order and are deliberately not merged. The +build tree *generates a file* at `swift/bridging` (the C++ interop header, included as +``) while Swift's sources have a *directory* of the same name +(`include/swift/Bridging/`). On a case-insensitive filesystem that is one path, so +staging both into a single root fails — `cp: .../swift/bridging: Is a directory` — and +the ordered pair is what a normal Swift/LLVM cross-build uses as well. + +`project.yml` consumes those variables and nothing else, so the app target never +names a path inside the bundle directly. + +When the bundle is **absent**, `prepare-xcode.sh` writes the same file with the +backend disabled. That file is **checked in, disabled**, so a plain clone builds: +the bridge compiles to a "not available" stub and the app says so. This is why +`make gen` alone is a complete build for someone who only wants to work on the UI. + +## The Darwin SDK + +The compiler is in the binary; what it compiles *against* is a bundle in the app's +container, `/native-sdk/darwin.artifactbundle`. `NativeSDK` reads it the +way xtool's builder writes it (`swift-sdk.json`: SDK root, Swift resource directory, +static runtime search paths), and the Toolchain screen installs one of three things: + +| what you pick | what it is | +| --- | --- | +| *(button)* | the hosted `darwin-sdk-` release asset — about 460 MB, no Mac needed | +| a **folder** or **zip** | a `darwin.artifactbundle`, perhaps built by `xtool sdk build` on a Mac | +| an **`Xcode.xip`** | Apple's Xcode; the app builds the bundle from it here | + +The third is the one that would otherwise send you to a Mac. A `.xip` is a xar archive +whose `Content` member is a pbzx stream of LZMA2 blocks decompressing to an `odc` cpio +archive of `Xcode.app`; `XipArchive` walks it in a single streaming pass and writes +only the paths a build SDK needs, and `DarwinSDKBuilder` assembles them into a bundle +— xtool's own list of paths, restricted to the device platform, because XForge only +ever compiles for the device. Extracting is minutes of CPU and about 1.5 GB of disk +on top of the copy the document picker makes, so the import checks free space before +it starts, reports progress while it runs, and deletes the picker's copy afterwards. + +`Docs/DESIGN.md` §3 has the format and layout details; the code states them where the +decisions are made. + +## The runtime contract + +The native implementation is enabled only when the app target defines +`XFORGE_HAS_LLVM=1` and can see the LLVM/Clang/LLD headers. Otherwise the bridge +answers `xf_native_toolchain_available() == 0` and every entry point fails +cleanly — no half-linked state, and no path where the app believes it has a +compiler it does not have. + +`NativeToolchainCapabilities.current` is the single source of that truth for the +UI. It reports clang, `ld64.lld`, `swift-frontend`, the SDK and the backend version +as separate lines, because "the toolchain is missing" is not one condition: + +- `canCompile` requires clang and LLD. It deliberately does **not** require the + Swift frontend: a C target links fine without it, and refusing to start would be + wrong about why the build fails. +- When clang or LLD is absent the app reports the missing toolchain and names the + way to fix it (`NativeToolchain/install-bundle.sh`, then rebuild), rather than + failing somewhere in the middle with a linker error. + +## The smoke test + +The Toolchain screen's **Verify** section compiles a C file +(`int xforge_native_smoke(void) { return 42; }`) through the linked clang, against +the installed SDK, in this process, and reports the object file it produced and its +size. The button needs both a linked compiler and an installed SDK, so it cannot +report success on a build that only has one of the two. + +This is what a real build does, at the smallest size that can fail: a compiler that +is present but mis-linked (wrong target triple, no SDK, missing resource directory) +fails here in a way the capability rows cannot detect. The screen's own footer +describes the check as compiling *and linking*; the button currently compiles only, +and the build-time link check lives in the toolchain workflow (step 6 above). + +## Which half you have -This is compile-time gated behind `XFORGE_HAS_SWIFT_FRONTEND`. Until a Swift -frontend library set has been cross-built for iPhoneOS, the UI reports -`Swift frontend: Not linked` and the Swift smoke test remains disabled. +The frontend is in the bundle or it is not, and `manifest.txt` says which — so the +honest statement is per bundle rather than per project: -This separation is intentional: the LLVM/Clang/LLD port can be validated and used -independently while the significantly larger Swift compiler port is brought up. +- A **`noswift`** bundle has no frontend. Swift projects cannot be built from it and + fail up front with `swiftFrontendMissing` and the file count; that is not a fallback + situation — there is no guest to fall back to — it is the other half of the port, + while C, Objective-C and Objective-C++ work in full. +- A **`swift-*`** bundle carries the frontend, so the Swift capability row reports it + like any other part and the Swift smoke test is reachable. Its Swift revision has to + match the app's: the frontend reads the deployment SDK's standard-library modules, + which is why the workflow pins the LLVM/Swift refs to the pair the Xcode in use + ships, and why `build-ipa.yml` warns when the installed bundle and the runner's + Xcode disagree. diff --git a/Docs/XCODE-ALTERNATIVE.md b/Docs/XCODE-ALTERNATIVE.md index c60d68b..0122c49 100644 --- a/Docs/XCODE-ALTERNATIVE.md +++ b/Docs/XCODE-ALTERNATIVE.md @@ -10,91 +10,123 @@ piece needs, and which parts cannot be done at all with the tools that exist. | Path | Status | |---|---| -| Authoring SwiftPM packages (project editor, manifest, files) | works | -| Building a **SwiftPM** package in the embedded Alpine guest via `xtool dev build` | works (slow: it is emulated) | +| Authoring a SwiftPM-shaped project (project editor, manifest, files) | works | +| Compiling and linking **C / Objective-C** sources in-process | works (`NativeBuildExecutor`) | +| Compiling **Swift** sources in-process | needs a `with_swift` bundle: the frontend ships in it and links into the app, but no Swift build has been run on a device | +| Resolving SwiftPM dependencies | **refused by name** — no package manager can run here | | Packaging an `.ipa` from a built `.app` | works (`IPABuilder`, host-side) | -| Signing (Apple ID / certificates / ad-hoc) | **stub** — `XKitSigningService` throws | +| Signing with an existing `.p12` + profile | works (`AppBundleSigner`, XKit, in-process) | +| Signing with an Apple ID (certificate issuance) | **not implemented** — `XKitSigningService.signIn` throws | | Installing to the device | **stub** — export to SideStore/AltStore is the working path | -| Building a **`.xcodeproj`** | not started; reading one now works (`XcodeProjectReader`) | -| Native (no-VM) compilation | POC branch: Clang + Mach-O LLD in-process, C only | +| Reading a `.xcodeproj` | works (`XcodeProjectReader`); nothing builds from it | +| Building a `.xcodeproj` | not started | +| Asset catalogs | **not compiled** — `.xcassets` are copied into the bundle with a warning | -Two things this table already implies: the guest can only build *SwiftPM* projects -(xtool is SwiftPM-only — its `XcodePacker` **generates** an Xcode project from a -SwiftPM package, the reverse of what we want), and nothing in the app has ever -looked at an `.xcodeproj`. +Three things that table already implies: + +- **The compiler runs in-process, not in a guest.** The embedded Linux is gone; the + Clang and LLD libraries are linked into the app. That removes the "emulated, so + slow" caveat, and it removes the escape hatch: there is no longer a guest that can + do the things the in-process path cannot (install packages, run shell, compile + Swift through a Linux toolchain). +- **Swift is the gate.** Everything interesting — which is to say almost every iOS + app — is Swift, and Swift does not compile on device yet. +- **The project model is read-only.** The reader parses an `.xcodeproj`; the import + flow still requires a `Package.swift` and rejects anything else, so there is no + path from a read `.xcodeproj` to a build. ## 2. What an Xcode project needs, and where each piece stands 1. **Project model** — targets, product types, sources, build settings. - **Readable on-device now** (`App/Models/XcodeProject.swift`). An `.xcodeproj` is a - property list; `tuist/XcodeProj` parses it in pure Swift on iOS 17. -2. **Build-setting *evaluation*** — the settings a build actually uses are computed, - not stored: `$(VAR)` expansion, `[sdk=…]`/`[config=…]` conditions, and the large - set of defaults Xcode supplies (`xcodebuild -showBuildSettings` output is not in - the pbxproj). The reader today reports values as written and merges the - inheritance layers in Xcode's order; full evaluation is its own piece of work and - is a prerequisite for the compiler command lines. -3. **A build driver** — dependency order between targets, Swift module boundaries - (`-emit-module` ordering, `.swiftmodule` search paths), bridging headers, + **Readable on-device now** (`App/Models/XcodeProject.swift`), with XForge's own + OpenStep parser (see the note in that file on why `tuist/XcodeProj` cannot be + used from this dependency graph). An `.xcodeproj` is a text property list. +2. **Build-setting *evaluation*** — **still open.** The settings a build actually + uses are computed, not stored: `$(VAR)` expansion, `[sdk=…]`/`[config=…]` + conditions, and the large set of defaults Xcode supplies. The reader today + merges the project's own inheritance layers in Xcode's order and reports values + as written; full evaluation is its own piece of work and is a prerequisite for + the compiler command lines. +3. **A build driver** — **still open, and now harder than it looked.** Dependency + order between targets, Swift module boundaries (`-emit-module` ordering, + `.swiftmodule` search paths), bridging headers, `SWIFT_OBJC_INTERFACE_HEADER_NAME`, entitlements, `Info.plist` variable - substitution, resource phases, and **script phases** (arbitrary shell, which the - guest can run — a useful hybrid). - Nothing here can be borrowed from SwiftPM: `swift-build`/SwiftPM spawn compilers - as processes, and iOS has no `fork`/`exec`. + substitution, resource phases. **Script phases are the part the platform takes + away**: they are arbitrary shell, and an iOS app cannot run shell, so a project + with a script phase cannot be built on device at all — not by XForge, and not by + any other on-device tool. Nothing here can be borrowed from SwiftPM either: + `swift-build` spawns compilers as processes, and iOS has no `fork`/`exec`. The + plan/executor split in `Docs/DESIGN.md` is the shape a driver would take, but + the driver itself (one target, one module) is what remains. 4. **Compilers and linkers.** - - C/Objective-C: `clang::CompilerInstance` driving `EmitObjAction` works in-process - (POC branch), given `-resource-dir` pointing at a staged `clang/lib/Headers`. - - Link: `ld64.lld` in-process (same branch). - - **Swift: the gate.** `swift-frontend` is not in `llvm-project`; it must be built - from `swiftlang/swift` against `swiftlang/llvm-project` **for iOS**, as a library - set, and it needs the same resource directory the guest's toolchain has. - Until that exists, on-device compilation is C/ObjC-only, and every Swift app - (i.e. almost all of them) still needs the emulated guest. + - C/Objective-C: clang in-process — **done**, not a POC any more. + - Link: `ld64.lld` in-process — **done**. + - **Swift: still the gate.** `swift-frontend` is not in `llvm-project`; it must + be built from `swiftlang/swift` against `swiftlang/llvm-project` **for iOS**, + as a library set, and it needs the same resource directory the SDK bundle + carries. The bridge already exposes the entry point behind + `XFORGE_HAS_SWIFT_FRONTEND`, and the CI workflow that produces the toolchain + artifact does not build it. Until it does, every Swift app is out of reach on + device — there is no guest to fall back to. - `dsymutil`/`strip` equivalents: LLVM ships `llvm-strip`, and debug maps can be skipped for a sideload build. -5. **Resources.** Copying files is trivial. **Compiling them is not:** - `actool` (asset catalogs → `Assets.car`) and `ibtool` (storyboards/XIB → `.nib`) - are Xcode's own macOS-only tools. They cannot run on iOS (wrong platform, and no - process spawning), and they cannot run in the Alpine guest either — which is why - **xtool has no asset-catalog support at all** (grep its sources: no `actool`, no - `xcassets`). There is no general open-source replacement to point at. - Practical consequence: an app whose UI is pure SwiftUI code and whose icon/drawables - are not in an asset catalog builds; one that needs `Assets.car` does not, on any - on-device toolchain. Shipping without an icon is an acceptable dev-build trade-off, - and a minimal `Assets.car` writer for the common cases (app icon + colours) is - conceivable but is its own project. -6. **Dependencies.** An Xcode project's SPM dependencies (`XCRemoteSwiftPackageReference`) - need a fetch (git over the network), a checkout, and then the same build treatment - as the project's own targets. CocoaPods projects are worse: their xcconfigs and - script phases assume a full Xcode/macOS toolchain. -7. **Signing and installation.** Signing is native-able today via XKit (in-process - zsign + Developer Services); installing to the device needs a usbmux transport — - the app sandbox cannot use the system `usbmuxd` socket, which is why SideStore - embeds a userspace one (Minimuxer) plus a pairing file. Exporting to SideStore - remains the pragmatic path. +5. **Resources.** Copying files is trivial and the executor does it. **Compiling + them is not:** `actool` (asset catalogs → `Assets.car`) and `ibtool` + (storyboards/XIB → `.nib`) are Xcode's own macOS-only tools. They cannot run on + iOS — wrong platform, and no process spawning — and no general open-source + replacement exists to point at. Practical consequence: an app whose UI is pure + SwiftUI code and whose icon and drawables are not in an asset catalog builds; + one that needs `Assets.car` does not, on any on-device toolchain. XForge copies + `.xcassets` into the bundle uncompiled and says so, so the failure is visible + rather than an app that ships without its icon and cannot say why. A minimal + `Assets.car` writer for the common cases (app icon + colours) is conceivable but + is its own project. +6. **Dependencies** — **still open, and unfetchable as things stand.** An Xcode + project's SPM dependencies (`XCRemoteSwiftPackageReference`) need a fetch (git + over the network), a checkout, and then the same build treatment as the + project's own targets. An iOS app cannot run `git`, so the fetch is not a matter + of writing a driver: it means reimplementing enough of git's wire protocol to + resolve and check out a package graph. For the same reason, XForge today refuses + a `Package.swift` that declares any dependency, by name. +7. **Signing and installation.** Signing **works** in-process via XKit with an + identity the user already has (`.p12` + profile); what is missing is *obtaining* + one — an Apple ID session needs anisette + GrandSlam + 2FA, which is a separate, + device-only piece. Installing to the device needs a usbmux transport: the app + sandbox cannot use the system `usbmuxd` socket, which is why SideStore embeds a + userspace one plus a pairing file. Exporting to SideStore remains the pragmatic + path. ## 3. Staged plan -- **M1 — Read the project.** `XcodeProjectReader` (done). Next: surface it in the - import flow so an `.xcodeproj` can be opened and its targets listed, instead of - being rejected as "not a SwiftPM package". -- **M2 — Build one app on-device.** A single app target, no asset catalog, SwiftUI in - one module: compile with the native path, link with `ld64.lld`, package with - `IPABuilder`, sign with XKit. This is the milestone that proves the whole concept - end to end, and it does **not** need the Swift frontend port if the target is - C/ObjC — but the interesting version of it does. +- **M1 — Read the project.** `XcodeProjectReader` (**done**). Next: surface it in + the import flow so an `.xcodeproj` can be opened and its targets listed, instead + of being rejected as "not a SwiftPM package". +- **M2 — Build one app on-device.** A single app target, no asset catalog, one + module: compile with the native path, link with `ld64.lld`, package with + `IPABuilder`, sign with XKit. The toolchain now does the C/ObjC half of this; + the interesting version needs Swift and therefore M4. This is the milestone that + proves the concept end to end. - **M3 — Build settings and the rest of the bundle.** Full `$(…)` evaluation, - `Info.plist` processing, resource copy phases, script phases delegated to the guest. -- **M4 — Swift.** Port `swift-frontend` for iOS (the long pole), then the build driver - for modules and dependencies. -- **M5 — SPM dependencies, then asset catalogs** (with the caveat in §2.5: this may - never cover the general case). + `Info.plist` processing, resource copy phases, and a decision about script phases + (which cannot be supported at all). +- **M4 — Swift.** Port `swift-frontend` for iOS (the long pole), then the build + driver for modules and dependencies. **Everything Swift waits on this.** +- **M5 — SPM dependencies, then asset catalogs** — with the caveats in §2.5 and + §2.6: the first needs a git implementation, and the second may never cover the + general case. ## 4. What this means for the goal -"An Xcode alternative on iOS" is achievable for a real class of projects — SwiftUI -apps that do not pull assets or storyboards out of a catalog — but the order matters: -the Swift frontend port is the gate on everything Swift, and asset catalogs and -interface files are walled off by tools that only exist on macOS. Until M4 lands, -the embedded guest (or a build server) remains the only way to compile Swift, and -that is a property of the platform, not of this app. +"An Xcode alternative on iOS" is achievable for a real class of projects — apps +whose sources are C/Objective-C, or SwiftUI apps once the frontend is ported, that +pull nothing from an asset catalog and run no script phase. The order matters: + +- the **Swift frontend port** is the gate on everything Swift, and not having it is + now a plain gap in the app rather than a reason to boot Linux instead; +- **asset catalogs and interface files** are walled off by tools that only exist on + macOS, and no on-device toolchain can change that; +- **dependency fetching** is a network-protocol problem, not a compiler problem; +- **signing** is done, and **install** is the part the sandbox will not allow. + +Until M4 lands, XForge is a working on-device C/Objective-C build tool, and a +Swift project cannot be built on device by anything at all. diff --git a/EmbeddedLinux/build-engine-for-xcode.sh b/EmbeddedLinux/build-engine-for-xcode.sh deleted file mode 100755 index d6479fa..0000000 --- a/EmbeddedLinux/build-engine-for-xcode.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# -# build-engine-for-xcode.sh — the XForge target's first build phase. -# -# The app cannot compile without the embedded Linux engine: ISHBridge.c includes -# the engine's headers, and a device build links libish.a / libish_emu.a / -# libfakefs.a out of Vendor/ish-arm64-build/lib. Both come from -# EmbeddedLinux/build-ish-core.sh, and until now *only* the IPA workflow ran it — -# so any other way of building the project (Xcode itself, another CI pipeline) -# failed part-way through compiling the bridge with -# -# App/EmbeddedVM/ISHBridge.c:123:10: error: 'kernel/init.h' file not found -# -# which says nothing about the engine being missing. This phase closes that gap: -# it fetches the engine, builds it once per engine revision, and skips itself for -# builds that do not link the engine at all. -# -# Builds that link the engine: iphoneos (`-lish` in OTHER_LDFLAGS[sdk=iphoneos*]) -# Builds that do not: the simulator, which compiles the stub branch -# in ISHBridge.c (the unit tests use it) -# -# Environment: -# XFORGE_REBUILD_ENGINE=1 force a rebuild even if the engine is up to date -# XFORGE_SKIP_ENGINE=1 skip the engine entirely (a fast check that the -# rest of the app compiles for the simulator) -# -set -euo pipefail - -if [ "${XFORGE_SKIP_ENGINE:-0}" = "1" ]; then - echo "note: XFORGE_SKIP_ENGINE=1 — not building the embedded Linux engine" - exit 0 -fi - -# The engine is device-only. A simulator build compiles the stub branch instead, -# so there is nothing to fetch or build, and doing it anyway would make every unit -# test build spend minutes on an engine it will not link. -if [ "${PLATFORM_NAME:-}" != "iphoneos" ]; then - echo "note: ${PLATFORM_NAME:-unknown} build — the engine is not linked, nothing to do" - exit 0 -fi - -cd "${SRCROOT:-$(cd "$(dirname "$0")/.." && pwd)}" - -# Xcode hands a script phase a bare PATH (`/usr/bin:/bin:/usr/sbin:/sbin`), which -# contains neither Homebrew nor anything Homebrew installs — so `command -v meson` -# and even `command -v brew` would come back empty on a machine that has both. Put -# the usual prefixes on PATH first. -for prefix in /opt/homebrew/bin /usr/local/bin /opt/local/bin; do - if [ -d "$prefix" ]; then - case ":$PATH:" in - *":$prefix:"*) ;; - *) PATH="$prefix:$PATH" ;; - esac - fi -done -export PATH - -# --------------------------------------------------------------------------- -# The engine's sources -# -# A fresh clone has the submodule empty (`git submodule update --init` is a -# separate step in every CI pipeline, and easy to forget in a new one). Fetching -# it here is cheap — one shallow clone — and turns "error: 'kernel/init.h' file -# not found" into a build that just works. -# --------------------------------------------------------------------------- -if [ ! -f Vendor/ish-arm64/meson.build ]; then - echo "note: Vendor/ish-arm64 is empty — fetching the engine submodule" - # Only the engine itself: its own submodules (libarchive, libapps, linux) are - # not read by this build — see EmbeddedLinux/build-ish-core.sh. - git submodule update --init --depth 1 Vendor/ish-arm64 -fi - -# --------------------------------------------------------------------------- -# The build tools -# -# meson/ninja build it, libarchive is what the engine links against, and llvm -# provides the clang+lld the arm64 VDSO needs (Apple's clang cannot link a Linux -# target). On a machine with Homebrew, install what is missing rather than failing -# a build the user cannot act on from inside Xcode. -# --------------------------------------------------------------------------- -missing="" -for tool in meson ninja; do - command -v "$tool" >/dev/null 2>&1 || missing="$missing $tool" -done - -if [ -n "$missing" ]; then - if command -v brew >/dev/null 2>&1; then - echo "note: installing the engine's build tools:$missing (plus llvm/lld/libarchive) with Homebrew" - for formula in meson ninja llvm lld libarchive; do - brew list "$formula" >/dev/null 2>&1 || brew install "$formula" - done - else - printf 'error: the embedded Linux engine needs meson and ninja, and this machine\n' >&2 - printf ' has neither (missing:%s) nor Homebrew to install them.\n' "$missing" >&2 - printf ' Install them and build the engine once:\n' >&2 - printf ' brew install meson ninja llvm lld libarchive\n' >&2 - printf ' EmbeddedLinux/build-ish-core.sh\n' >&2 - exit 1 - fi -fi -command -v meson >/dev/null 2>&1 || { - printf 'error: meson is installed but not on this build phase'"'"'s PATH.\n' >&2 - printf ' Add %s/bin to it (Xcode does not read your shell profile).\n' "$(brew --prefix 2>/dev/null)" >&2 - exit 1 -} - -# --------------------------------------------------------------------------- -# Build (the script is a no-op when the engine is already up to date) -# --------------------------------------------------------------------------- -exec bash EmbeddedLinux/build-ish-core.sh diff --git a/EmbeddedLinux/build-ish-core.sh b/EmbeddedLinux/build-ish-core.sh deleted file mode 100644 index c35a49a..0000000 --- a/EmbeddedLinux/build-ish-core.sh +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env bash -# -# build-ish-core.sh — build the embedded Linux engine (OpenMinis/ish-arm64) for -# iOS and stage the static libraries where XForge's linker finds them. -# -# XForge embeds **ish-arm64** (Vendor/ish-arm64, a git submodule) as its -# in-process Linux runtime. It is a fork of ish-app/ish that adds a native -# AArch64 guest backend to the threaded-code interpreter, so the Alpine aarch64 -# rootfs runs as a same-architecture guest instead of being cross-translated. -# -# This script produces, for the iOS device (arm64): -# -# libish.a the emulator + kernel -# libish_emu.a per-arch instruction translation -# libfakefs.a the SQLite-backed filesystem -# -# into Vendor/ish-arm64-build/lib. -# -# Nothing else is built here. In particular the app does *not* link -# tools/fakefs.c (`fakefs_import`) or libarchive: the rootfs it installs is already -# a fakefs ZIP, unpacked with ZIPFoundation, so there is no import step at runtime -# (see App/EmbeddedVM/ISHBridge.h). The `fakefsify` the *build machine* needs to -# make that ZIP is produced by the engine's own meson build in -# EmbeddedLinux/build-rootfs.sh, on a Linux host. -# -# Requires: macOS, Xcode command line tools, meson, ninja, python3. -# brew install meson ninja llvm lld -# -# Usage: EmbeddedLinux/build-ish-core.sh -# -set -euo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -REPO="$(cd "$HERE/.." && pwd)" - -ISH="${ISH_ROOT:-$REPO/Vendor/ish-arm64}" -BUILD="${ISH_BUILD_DIR:-$REPO/Vendor/ish-arm64-build}" -OUT="${ISH_LIB_DIR:-$BUILD/lib}" - -# The guest is aarch64 (the bundled Alpine rootfs), so only that backend is -# built. Must match the -DGUEST_ARM64=1 in project.yml: the guest-arch defines -# select struct layouts, so engine and app have to agree. -GUEST_ARCH="${ISH_GUEST_ARCH:-arm64}" -MIN_SDK="${IPHONEOS_DEPLOYMENT_TARGET:-17.0}" - -log() { printf '\033[1m==> %s\033[0m\n' "$*"; } -note() { printf ' %s\n' "$*"; } -die() { printf 'error: %s\n' "$*" >&2; exit 1; } - -[ -f "$ISH/meson.build" ] || die "ish-arm64 sources not found at $ISH - run: git submodule update --init --depth 1 Vendor/ish-arm64" - -# ish-arm64's own submodules are deliberately not required. deps/libarchive only -# feeds the fakefs tools (which this app does not link — see the note at the top), -# deps/libapps is for the native host shells XForge does not embed, and deps/linux -# is for -Dkernel=linux, which XForge does not build. Requiring them would mean -# fetching something the build never reads. - -for tool in meson ninja xcrun python3; do - command -v "$tool" >/dev/null 2>&1 || die "$tool not found in PATH" -done - -# --------------------------------------------------------------------------- -# Already built? -# -# The libraries are a function of the engine's sources and of the flags this -# script passes, so the engine revision (the submodule commit) plus a hash of this -# script and of project.yml — which carries -DGUEST_ARM64=1, and has to agree with -# the engine or struct layouts differ — is enough to decide. That matters because -# this script is run from two places now: the IPA workflow, and the app target's -# first build phase (EmbeddedLinux/build-engine-for-xcode.sh), which runs on every -# Xcode build. A cold engine build takes minutes; the stamp makes the common case -# a couple of file reads. -# -# XFORGE_REBUILD_ENGINE=1 forces the build. So does deleting the lib directory. -# --------------------------------------------------------------------------- -stamp_value() { - local revision script_hash project_hash - revision="$(git -C "$ISH" rev-parse HEAD 2>/dev/null \ - || git -C "$REPO" rev-parse HEAD:Vendor/ish-arm64 2>/dev/null \ - || echo unknown)" - script_hash="$(shasum -a 256 "$HERE/build-ish-core.sh" | cut -d' ' -f1)" - project_hash="$(shasum -a 256 "$REPO/project.yml" 2>/dev/null | cut -d' ' -f1)" - printf '%s %s %s %s %s' "$revision" "$script_hash" "${project_hash:-none}" "$GUEST_ARCH" "$MIN_SDK" -} - -STAMP="$OUT/.engine-stamp" -WANT_STAMP="$(stamp_value)" -if [ "${XFORGE_REBUILD_ENGINE:-0}" != "1" ] && [ -f "$STAMP" ] \ - && [ "$(cat "$STAMP")" = "$WANT_STAMP" ]; then - complete=1 - for lib in libish.a libish_emu.a libfakefs.a; do - [ -f "$OUT/$lib" ] || complete="" - done - if [ -n "$complete" ]; then - log "The engine is already built ($OUT) — nothing to do" - note "engine revision: $(printf '%s' "$WANT_STAMP" | cut -d' ' -f1)" - note "force a rebuild with XFORGE_REBUILD_ENGINE=1" - exit 0 - fi - note "the stamp matches but libraries are missing — rebuilding" -fi - -# The VDSO step compiles a 32-bit i386-linux object with `-fuse-ld=lld`, which -# Apple's clang cannot do; it needs an LLVM clang with lld. Put Homebrew's LLVM -# first and make sure ld.lld is reachable (newer `llvm` bottles ship clang but -# not ld.lld — the separate `lld` formula provides it). -if command -v brew >/dev/null 2>&1; then - LLVM_BIN="$(brew --prefix llvm 2>/dev/null)/bin" - [ -d "$LLVM_BIN" ] && { export PATH="$LLVM_BIN:$PATH"; log "Using Homebrew LLVM: $LLVM_BIN"; } - LLD_BIN="$(brew --prefix lld 2>/dev/null)/bin" - [ -d "$LLD_BIN" ] && export PATH="$LLD_BIN:$PATH" -fi -if ! clang -target i386-linux -fuse-ld=lld -shared -nostdlib -x c /dev/null -o /dev/null 2>/dev/null; then - printf 'warning: clang cannot build the VDSO (needs LLVM + lld). Install: brew install llvm lld\n' >&2 -fi - -SDK="$(xcrun --sdk iphoneos --show-sdk-path)" -TRIPLE="arm64-apple-ios${MIN_SDK}" -MESON_BUILD="$BUILD/meson" - -rm -rf "$MESON_BUILD" -mkdir -p "$MESON_BUILD" "$OUT" - -# --- meson cross file for iOS ------------------------------------------------ -CROSS="$MESON_BUILD/ios-arm64.txt" -cat > "$CROSS" < "$MESON_BUILD/req-probe.S" 2>/dev/null; then - die "could not write the assembler capability probe" -fi -if ! clang -c "$MESON_BUILD/req-probe.S" -o "$MESON_BUILD/req-probe.o" 2>/dev/null; then - rm -f "$MESON_BUILD/req-probe.S" "$MESON_BUILD/req-probe.o" - die "this clang cannot assemble the engine's aarch64 gadgets (no .req register - alias support). Build the engine on macOS with Xcode's clang — the IPA - workflow does exactly that — rather than with a GNU binutils toolchain." -fi -rm -f "$MESON_BUILD/req-probe.S" "$MESON_BUILD/req-probe.o" - -log "Building libish / libish_emu / libfakefs" -ninja -C "$MESON_BUILD" libish.a libish_emu.a libfakefs.a - -# The VDSO is not a separate artifact to ship: kernel/vdso.c `.incbin`s -# libvdso.so.elf straight into libish.a, so building libish already built it. -# But the build degrades silently — if meson cannot find an aarch64-linux -# cross-compiler it substitutes a target that just `touch`es an empty file, and -# the guest then gets a bogus VDSO with nothing in the log to say so. Verify the -# file is a real aarch64 ELF shared object rather than trusting the build. -VDSO="$MESON_BUILD/vdso/arm64/libvdso.so.elf" -if [ ! -s "$VDSO" ]; then - die "the arm64 VDSO was not built: $VDSO is missing or empty. - meson falls back to an EMPTY placeholder when it cannot find an - aarch64-linux cross-compiler, so this is a silent failure otherwise. - Install one (brew install llvm lld) — clang must support - '-target aarch64-linux-gnu -fuse-ld=lld'." -fi -if command -v file >/dev/null 2>&1; then - file "$VDSO" | grep -q 'ELF 64-bit.*aarch64' || die \ - "$VDSO is not an aarch64 ELF (got: $(file -b "$VDSO"))" -fi -note "VDSO: $(du -h "$VDSO" | cut -f1) (embedded into libish.a)" - -for lib in libish.a libish_emu.a libfakefs.a; do - [ -f "$MESON_BUILD/$lib" ] || die "$lib was not produced" - cp "$MESON_BUILD/$lib" "$OUT/$lib" -done - -log "Staged into $OUT" -ls -lh "$OUT" - -# Record what was built, so the next run (Xcode's build phase runs this on every -# build) can tell in a moment that there is nothing to do. -printf '%s\n' "$WANT_STAMP" > "$STAMP" diff --git a/EmbeddedLinux/build-rootfs.sh b/EmbeddedLinux/build-rootfs.sh deleted file mode 100644 index 6f37f4f..0000000 --- a/EmbeddedLinux/build-rootfs.sh +++ /dev/null @@ -1,973 +0,0 @@ -#!/usr/bin/env bash -# -# build-rootfs.sh — build the Alpine aarch64 root filesystem that XForge boots, -# and pack it as a ZIP of an already-converted fakefs root. -# -# This follows OpenMinis's own recipe (deps/prepare_alpine_rootfs.sh in -# github.com/OpenMinis/OpenMinis), which is the reference implementation for -# this engine: -# -# 1. download the *plain* Alpine aarch64 minirootfs -# 2. build the engine's own `tools/fakefsify` for the build machine -# (NOT cross-compiled: it runs here, on this host) -# 3. unpack it as a plain tree, install the glibc layer into it, and install the -# packages the console session needs (bash/coreutils/less/terminfo) -# 4. provision the build toolchain into it — xtool, swiftly, Swift and the -# Darwin Swift SDK — by running EmbeddedLinux/install-toolchain.sh in the -# chroot, so the guest arrives ready to build iOS apps -# 5. configure the root in place (passwd, profile, motd, inittab, DNS, apk repos) -# — including the console program, EmbeddedLinux/xforge-login -# 6. convert the configured tree to fakefs — a `data/` tree + `meta.db` -# 7. `zip -r` the result, excluding the SQLite WAL/SHM sidecars -# -# Why a ZIP of a fakefs and not a tarball the app imports at runtime: the -# conversion is the expensive part (thousands of files into SQLite), and doing it -# here means the app's first launch is a plain unzip — no multi-minute import on -# a phone. This is the same reason OpenMinis ships `alpine-rootfs.zip` and its -# app only calls unzip + mount_root. -# -# What is *in* the root is XFORGE_PROVISION's decision, and it is the whole -# difference between a small root and a complete one: -# -# XFORGE_PROVISION=all plain Alpine plus the whole build toolchain: -# xtool, swiftly, the Swift toolchain and the Darwin -# Swift SDK, all installed before the conversion and -# therefore visible to the engine. The app then boots -# into a guest that can build an iOS app immediately, -# at the cost of ~1.6 GB of artifact. -# XFORGE_PROVISION=none plain Alpine only. The guest installs the toolchain -# itself on demand (`sh /root/install-toolchain.sh -# all`), which is what a few-MB root buys. -# -# Both are the same script, the same installer and the same layout: this decides -# only *when* install-toolchain.sh runs, on the build machine or in the guest. The -# small root exists so the choice is available; the provisioned one is what the -# release ships. -# -# Output: -# dist/rootfs/alpine-rootfs.zip the root (data/ + meta.db) -# dist/rootfs/alpine-rootfs.sha256 checksum, for pinning -# -# Usage: -# EmbeddedLinux/build-rootfs.sh [output-dir] # default: dist/rootfs -# -# Environment: -# XFORGE_ALPINE_VERSION Alpine release (default: 3.21, matching the -# reference; the engine is built for this guest) -# XFORGE_ALPINE_MINOR minor version (default: 0) -# XFORGE_ISH_ROOT engine checkout (default: /Vendor/ish-arm64) -# XFORGE_WORK_DIR scratch dir (default: /.rootfs-work) -# XFORGE_KEEP_WORK 1 to keep the scratch dir -# XFORGE_SKIP_GLIBC 1 to build a root without the glibc layer -# XFORGE_PROVISION all (default) to bake the build toolchain into the -# root; none to ship plain Alpine and let the guest -# install it. See above. -# XFORGE_PROVISION_SDK 1 (default) to include the Darwin Swift SDK in a -# provisioned root; 0 to stop after the Swift -# toolchain. The SDK is ~400 MB of download and -# ~1.3 GB installed, so this is the size lever. -# XFORGE_DARWIN_SDK_TAG release tag the SDK comes from (default: -# darwin-sdk-7). Pinned, not resolved: the published -# root has to name the SDK it contains. -# XFORGE_DARWIN_SDK_URL the SDK asset itself, overriding repo/tag/asset -# XFORGE_DARWIN_SDK_SHA256 -# refuse to use a downloaded SDK with any other hash -# XFORGE_SLIM_TOOLCHAIN 1 (default) to drop the parts of the Swift -# toolchain an iOS build never loads (the static -# Linux stdlib, lldb, the editor tooling). The -# provisioning verification runs *after* this, so a -# toolchain that stopped working fails the build. -# XFORGE_DEFAULT_SHELL login shell root's console starts (default: -# /bin/bash). Must be a shell the package list below -# installs; if it is not, the build falls back to -# /bin/sh and says so. -# XFORGE_CONSOLE_PACKAGES Alpine packages baked into the root for the console -# session (default: bash coreutils less -# ncurses-terminfo). Space-separated. -# -set -euo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -REPO="$(cd "$HERE/.." && pwd)" - -# Resolve the output directory to an absolute path up front. Packing happens -# from inside the work directory (so the ZIP holds relative paths), and a -# relative OUT_DIR would then resolve against the wrong directory — which is -# exactly how a CI run failed with "Could not create output file" while the same -# script worked locally with an absolute path. -OUT_DIR="${1:-$REPO/dist/rootfs}" -mkdir -p "$OUT_DIR" -OUT_DIR="$(cd "$OUT_DIR" && pwd)" -ALPINE_VERSION="${XFORGE_ALPINE_VERSION:-3.21}" -ALPINE_MINOR="${XFORGE_ALPINE_MINOR:-0}" -ALPINE_ARCH="aarch64" -ALPINE_MIRROR="https://dl-cdn.alpinelinux.org/alpine" -ISH="${XFORGE_ISH_ROOT:-$REPO/Vendor/ish-arm64}" -WORK="${XFORGE_WORK_DIR:-$REPO/.rootfs-work}" - -# The guest's console session, and what it needs from the root. -# -# XFORGE_DEFAULT_SHELL is the shell root's console starts in: it is written into -# /etc/passwd, and /etc/inittab runs xforge-login, which reads that field — so -# this is the one place the shell is chosen, and the guest can change it at -# runtime (`apk add zsh` + the passwd field). -# -# The package list is the shell's own environment, and every entry earns its -# place: -# bash the default shell (pulls readline, ncurses-libs) -# coreutils the tools a shell session expects to exist -# less a pager for them -# ncurses-terminfo terminfo entries — xterm/xterm-256color are NOT in -# ncurses-terminfo-base, and /etc/profile exports -# TERM=xterm-256color, so without this every curses -# program (less, top, vim) runs against an unknown -# terminal -DEFAULT_SHELL="${XFORGE_DEFAULT_SHELL:-/bin/bash}" -CONSOLE_PACKAGES="${XFORGE_CONSOLE_PACKAGES:-bash coreutils less ncurses-terminfo}" - -# Whether the build toolchain is provisioned into the root (see the header). The -# guest's installer is the only implementation of that, and it is run here in a -# chroot of the tree, so a provisioned root and a hand-provisioned guest differ -# in nothing but when the work happened. -PROVISION="${XFORGE_PROVISION:-all}" -PROVISION_SDK="${XFORGE_PROVISION_SDK:-1}" -SLIM_TOOLCHAIN="${XFORGE_SLIM_TOOLCHAIN:-1}" - -# The Darwin Swift SDK: which release it comes from, and what the asset is called. -# Kept in step with the app's XForgeReleases (the Toolchain screen downloads the -# same asset) — the tag is pinned here, not resolved through the API, because the -# published root has to name the exact SDK a device is booting. -DARWIN_SDK_REPO="${XFORGE_DARWIN_SDK_REPO:-R0GUEEE/XForge}" -DARWIN_SDK_TAG="${XFORGE_DARWIN_SDK_TAG:-darwin-sdk-7}" -DARWIN_SDK_ASSET="${XFORGE_DARWIN_SDK_ASSET:-darwin.artifactbundle.zip}" -DARWIN_SDK_URL="${XFORGE_DARWIN_SDK_URL:-https://github.com/$DARWIN_SDK_REPO/releases/download/$DARWIN_SDK_TAG/$DARWIN_SDK_ASSET}" -DARWIN_SDK_SHA256="${XFORGE_DARWIN_SDK_SHA256:-}" - -ROOTFS_NAME="alpine-rootfs" -ZIP_NAME="$ROOTFS_NAME.zip" - -log() { printf '\n==> %s\n' "$*"; } -note() { printf ' %s\n' "$*"; } -die() { printf '\nerror: %s\n' "$*" >&2; exit 1; } - -# Size of a tree in KiB, as a number, always. -# -# Measurement must not be able to fail the build, and it must not be able to -# confuse itself either. `du` over a tree with /proc or /dev mounted inside it -# walks entries that vanish mid-traversal — the guest's own mounts are still up -# when this runs — so it can exit non-zero after printing a perfectly good number; -# and a `|| echo 0` on top of that hands the caller two lines, which arithmetic -# then reads as "6256892\n0: syntax error in expression". Taking the first field -# of whatever was printed is what makes both impossible. -size_kib() { - { du -sk "$1" 2>/dev/null || true; } | awk 'NF { print $1; exit }' -} - -# --------------------------------------------------------------------------- -# Running things *inside* the root being built -# -# Two steps need this: installing the glibc layer (step 4) and installing the -# guest's own packages (step 5). Both run the guest's own apk in a chroot of the -# tree, which needs the kernel's /proc and a /dev with a working /dev/null — apk -# opens it, and so does every shell — plus /etc/resolv.conf, because the -# minirootfs ships no nameservers at all and every one of those commands -# downloads something. -# -# The mounts are recorded so the exit trap can undo them: a failed build must not -# leave a mount behind, or the next run's `rm -rf $WORK` walks into a live /proc. -# --------------------------------------------------------------------------- -GUEST_MOUNTS="" - -guest_mount() { - [ "$(id -u)" -eq 0 ] || die "building the root needs root: the guest's own - installer and apk are run in a chroot of the tree being built, and a chroot - needs mounts. Re-run with sudo." - - mount -t proc none "$DATA/proc" || die "could not mount /proc in $DATA" - GUEST_MOUNTS="$DATA/proc" - mount --rbind /dev "$DATA/dev" 2>/dev/null || true - GUEST_MOUNTS="$DATA/dev $GUEST_MOUNTS" - mount --rbind /sys "$DATA/sys" 2>/dev/null || true - GUEST_MOUNTS="$DATA/sys $GUEST_MOUNTS" - - # Name resolution for anything the chroot runs: the minirootfs ships no - # nameservers at all, and both steps below download (apk indexes, packages). - # - # Written with awk rather than `grep … | sed …`, and with the fallback keyed on - # the *result* rather than on the host file's contents: a pipeline whose output - # is redirected leaves an empty file behind if it produces nothing, and a - # truncated /etc/resolv.conf inside the chroot turns a network failure into a - # name-resolution failure three steps away. (Seen for real: iSH wedged the - # pipeline and the root being built got a 0-byte resolv.conf.) - awk '/^nameserver/ { print; if (++n == 3) exit }' /etc/resolv.conf \ - > "$DATA/etc/resolv.conf" 2>/dev/null || true - if ! [ -s "$DATA/etc/resolv.conf" ]; then - printf 'nameserver 1.1.1.1\nnameserver 8.8.8.8\n' > "$DATA/etc/resolv.conf" - fi -} - -guest_umount() { - for m in $GUEST_MOUNTS; do - umount -l "$m" 2>/dev/null || true - done - GUEST_MOUNTS="" -} - -# The environment every chrooted step starts from. -# -# HOME is what makes swiftly and SwiftPM install into /root — the tree's own -# /root, inside the root being built — and the XDG directories are *unset* on top -# of it, because SwiftPM's SDK store follows them: with the runner's -# XDG_CONFIG_HOME inherited, `swift sdk install` put the SDK in -# /swiftpm/swift-sdks inside the root being built, where the -# guest's own SwiftPM (which has no XDG set) would never look for it. Both the -# provisioning steps and the verification use this, so they agree on where things -# are. -GUEST_ENV="unset XDG_CONFIG_HOME XDG_DATA_HOME XDG_CACHE_HOME XDG_STATE_HOME; \ - export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - HOME=/root SWIFTLY_HOME_DIR=/root/.local/share/swiftly" - -cleanup() { - local status=$? - guest_umount - if [ "$status" -ne 0 ]; then - printf '\nFailed (exit %d). Work directory kept at %s\n' "$status" "$WORK" >&2 - elif [ "${XFORGE_KEEP_WORK:-0}" != "1" ]; then - rm -rf "$WORK" - fi - return $status -} -trap cleanup EXIT - -# --------------------------------------------------------------------------- -# Preconditions -# --------------------------------------------------------------------------- -[ -f "$ISH/meson.build" ] || die "the engine is not checked out at $ISH - run: git submodule update --init --depth 1 Vendor/ish-arm64" - -# fakefsify needs libarchive; it is a submodule of the engine. -[ -d "$ISH/deps/libarchive" ] && [ -n "$(ls -A "$ISH/deps/libarchive" 2>/dev/null)" ] || die \ - "the engine's deps/libarchive submodule is missing (fakefsify needs it) - run: git -C Vendor/ish-arm64 submodule update --init --depth 1 deps/libarchive" - -case "$PROVISION" in - all|none) ;; - *) die "XFORGE_PROVISION must be 'all' or 'none', not '$PROVISION'" ;; -esac - -for tool in curl meson ninja python3 zip; do - command -v "$tool" >/dev/null 2>&1 || die "$tool is required" -done -if [ "$PROVISION" = "all" ] && [ "$PROVISION_SDK" = "1" ]; then - for tool in unzip sha256sum; do - command -v "$tool" >/dev/null 2>&1 \ - || die "$tool is required to stage the Darwin Swift SDK" - done -fi - -mkdir -p "$WORK" - -# --------------------------------------------------------------------------- -# 1. Download the plain minirootfs -# --------------------------------------------------------------------------- -ROOTFS_FILE="alpine-minirootfs-${ALPINE_VERSION}.${ALPINE_MINOR}-${ALPINE_ARCH}.tar.gz" -ROOTFS_PATH="$WORK/$ROOTFS_FILE" -ROOTFS_URL="$ALPINE_MIRROR/v${ALPINE_VERSION}/releases/${ALPINE_ARCH}/${ROOTFS_FILE}" - -log "Fetching $ROOTFS_FILE" -if [ -f "$ROOTFS_PATH" ] && tar -tzf "$ROOTFS_PATH" >/dev/null 2>&1; then - note "using the cached copy" -else - curl -fL --retry 3 --retry-delay 2 -o "$ROOTFS_PATH.partial" "$ROOTFS_URL" - mv "$ROOTFS_PATH.partial" "$ROOTFS_PATH" -fi -tar -tzf "$ROOTFS_PATH" >/dev/null || die "$ROOTFS_FILE is not a readable .tar.gz" -note "$ROOTFS_FILE: $(du -h "$ROOTFS_PATH" | cut -f1)" - -# --------------------------------------------------------------------------- -# 2. Build fakefsify for the build machine -# -# A *native* build, deliberately: this tool only ever runs here, never on a -# device, and it is what converts the tarball into fakefs. Note the absent -# `-Db_ndebug` — unlike the device build, asserts stay ENABLED, so a broken -# assumption fails loudly at build time instead of quietly producing a wrong -# rootfs. -# --------------------------------------------------------------------------- -FAKEFSIFY_BUILD="$WORK/build-native" -FAKEFSIFY="$FAKEFSIFY_BUILD/tools/fakefsify" - -log "Building tools/fakefsify for this machine" -if [ -x "$FAKEFSIFY" ]; then - note "already built" -else - if [ ! -f "$FAKEFSIFY_BUILD/build.ninja" ]; then - meson setup "$FAKEFSIFY_BUILD" "$ISH" \ - --buildtype=release \ - -Dlog='' \ - -Dkernel=ish \ - -Dengine=asbestos \ - -Dguest_arch=arm64 - fi - ninja -C "$FAKEFSIFY_BUILD" tools/fakefsify -fi -[ -x "$FAKEFSIFY" ] || die "fakefsify was not produced at $FAKEFSIFY" -note "$FAKEFSIFY" - -# --------------------------------------------------------------------------- -# 3. Unpack the plain rootfs we will shape -# -# Everything happens on this tree — configuration and the glibc layer both — -# and it is converted to fakefs exactly once, at the end. That ordering is not -# cosmetic: `fakefsify` builds `meta.db` as an index of the files present when -# it runs, and the engine reads *the database*, not the directory. Files copied -# into `data/` after conversion exist on disk and are invisible in the guest — -# a first attempt installed the glibc layer that way and the layer did not exist -# as far as the engine was concerned (568 files on disk, 0 rows in meta.db). -# --------------------------------------------------------------------------- -DATA="$WORK/rootfs-tree" - -log "Unpacking $ROOTFS_FILE into the root tree" -rm -rf "$DATA" -mkdir -p "$DATA" -tar -xzf "$ROOTFS_PATH" -C "$DATA" -[ -x "$DATA/bin/sh" ] || die "the unpacked root has no /bin/sh" - -# Mount points the guest expects. The engine mounts /proc and /dev/pts itself; -# these directories have to exist first. -for dir in dev proc sys tmp run root home; do - mkdir -p "$DATA/$dir" -done - -# --------------------------------------------------------------------------- -# 4. Install the glibc compatibility layer -# -# Alpine is musl-based, but every tool XForge builds with is a *glibc* binary: -# xtool is a Swift program built on Ubuntu, and the Swift toolchain is too. -# `gcompat` is not enough for them (they die on strptime_l, fts_*, fcntl64 …), so -# the layer takes Ubuntu's own glibc and its libraries and points the loader at -# them. -# -# Baking it in rather than letting the guest install it removes the most -# failure-prone step of an on-device provision: a package renamed between Ubuntu -# releases produces a layer that loads but cannot resolve a symbol, which -# surfaces much later as a tool dying on an undefined symbol. It costs ~95 MB -# compressed, so the root stays small. -# -# This runs the *same* `install-toolchain.sh glibc` step the guest would run, in -# a chroot of this very tree, so there is only ever one implementation of the -# layer. The chroot is what makes it work: the step unpacks Ubuntu packages with -# `ar`/`zstd` and writes to /lib, /usr/lib and /opt/glibc, and those paths have -# to mean the guest root. -# --------------------------------------------------------------------------- -if [ "${XFORGE_SKIP_GLIBC:-0}" = "1" ]; then - log "Skipping the glibc layer (XFORGE_SKIP_GLIBC=1)" -else - log "Installing the glibc compatibility layer (this is the slow part)" - - # The step runs the guest's installer in a chroot, which needs /proc, /dev and - # DNS. guest_mount() sets those up (and refuses to run unless we are root: - # otherwise this surfaces as "mount: must be superuser to use mount" from - # somewhere in the middle of a long install). - guest_mount - - # The Alpine-side prerequisites for the step itself (it fetches packages with - # curl and unpacks them with ar/zstd), and its copy of the installer. - install -m 0755 "$HERE/install-toolchain.sh" "$DATA/root/install-toolchain.sh" - chroot "$DATA" /bin/sh -c \ - "apk add --no-cache curl tar xz zstd binutils ca-certificates" \ - || die "could not install the glibc step's own prerequisites in the chroot" - - chroot "$DATA" /bin/sh -c \ - "export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root; \ - sh /root/install-toolchain.sh glibc" \ - || die "installing the glibc layer in the chroot failed" - - # The layer is only useful if the guest can resolve it, so check the entry - # points a Swift or xtool binary loads rather than trusting the step's status. - # - # `-L` rather than `-e` on the symlinks, deliberately: they are absolute and - # point into the *guest's* /opt/glibc, which does not exist on the build - # machine, so `-e` (which follows the link) reports them missing here even - # though they resolve perfectly inside the guest. What has to exist at build - # time is the link itself. - for required in \ - opt/glibc/usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1 \ - usr/local/share/xforge/glibc.env; do - [ -e "$DATA/$required" ] || die "the glibc layer is incomplete: $required is missing" - done - for link in lib/ld-linux-aarch64.so.1 lib/aarch64-linux-gnu usr/lib/aarch64-linux-gnu; do - [ -L "$DATA/$link" ] || die "the glibc layer is not wired in: $link is missing" - done - - guest_umount - - # The layer's own prerequisite packages are no longer needed: they were - # installed to run the step, not to run the guest. - chroot "$DATA" /bin/sh -c "apk del --no-cache zstd binutils" >/dev/null 2>&1 || true - - note "glibc layer installed at /opt/glibc" -fi - -# --------------------------------------------------------------------------- -# 5. Install the guest's own packages -# -# The shell session XForge's Terminal *is* has dependencies, and they belong in -# the root rather than in an on-device install: the console is the first thing a -# new install shows, and a shell whose pager is missing (or whose TERM is unknown -# to curses) is a broken-looking guest, not a hint to run `apk add`. -# -# Installed with the guest's own apk, in a chroot of this tree, so the packages -# land in the root's own database (/lib/apk/db) and `apk del`/`apk upgrade` in the -# guest stay consistent with them. This is also why it runs *before* the fakefs -# conversion below: the conversion indexes the tree, and anything installed -# afterwards would be on disk and invisible to the engine. -# --------------------------------------------------------------------------- -log "Installing the guest's console packages: $CONSOLE_PACKAGES" -guest_mount -chroot "$DATA" /bin/sh -c "apk add --no-cache $CONSOLE_PACKAGES" \ - || die "installing the guest's own packages in the chroot failed" -guest_umount - -for package in $CONSOLE_PACKAGES; do - chroot "$DATA" /bin/sh -c "apk info -e $package" >/dev/null 2>&1 \ - || die "$package is not installed in the built root" -done -note "$(chroot "$DATA" /bin/sh -c 'apk info 2>/dev/null | wc -l' | tr -d ' ') packages in the root" - -# --------------------------------------------------------------------------- -# 5b. Provision the build toolchain into the root -# -# This is the difference between a root that can build an iOS app and one that -# first has to provision itself. The work is the guest's own installer, run in a -# chroot of this very tree — the same steps, the same order, the same result a -# user would get from `sh /root/install-toolchain.sh all` in the Terminal, except -# that it happens on a build machine with a fast network and a real CPU, once, -# instead of on a phone under emulation. -# -# Order matters and is not the installer's own: -# * glibc before anything else (xtool and the Swift toolchain are glibc -# binaries), which the build already did in step 4 — the step is idempotent -# and returns immediately; -# * `deps` before the tools that need it (xtool unpacks with `tar`, the SDK -# installs with `unzip`); -# * the SDK *after* Swift, because `swift sdk install` is a Swift program; -# * the slimming pass after the SDK and *before* verify, so what the -# verification checks is what ships; -# * verify last, and its verdict is load-bearing (see below). -# -# Everything is installed before the fakefs conversion in step 7: the conversion -# indexes the tree, and anything added afterwards would be on disk and invisible -# to the engine. That is not a detail — it is how a previous attempt shipped a -# glibc layer the guest could not see. -# --------------------------------------------------------------------------- -SDK_SHA256="" -if [ "$PROVISION" = "none" ]; then - log "Skipping the build toolchain (XFORGE_PROVISION=none)" - note "the root will provision itself: sh /root/install-toolchain.sh all" -else - [ "${XFORGE_SKIP_GLIBC:-0}" != "1" ] || die \ - "XFORGE_PROVISION=all needs the glibc layer (xtool and the Swift - toolchain are glibc binaries), but XFORGE_SKIP_GLIBC=1. Drop one of the two." - - log "Provisioning the build toolchain into the root (long: ~2 GB of downloads)" - - install -m 0755 "$HERE/install-toolchain.sh" "$DATA/root/install-toolchain.sh" - - # The Darwin SDK is the largest download by far, and it is fetched *here* - # rather than inside the chroot: on the build machine a failure is a line in - # the log, the archive is cached between runs, and the hash of what went in - # can be written into the published root's manifest — which is otherwise the - # only way to answer "which SDK is in this root" from the artifact alone. - SDK_GUEST_PATH="" - if [ "$PROVISION_SDK" = "1" ]; then - SDK_ARCHIVE="$WORK/$DARWIN_SDK_ASSET" - log "Fetching the Darwin Swift SDK: $DARWIN_SDK_TAG/$DARWIN_SDK_ASSET" - if [ -s "$SDK_ARCHIVE" ]; then - note "using the cached copy" - else - rm -f "$SDK_ARCHIVE.partial" - curl -fL --retry 3 --retry-delay 2 --no-progress-meter \ - -o "$SDK_ARCHIVE.partial" "$DARWIN_SDK_URL" \ - || die "could not download $DARWIN_SDK_URL - (set XFORGE_DARWIN_SDK_URL, or XFORGE_PROVISION_SDK=0 to build a root - with the Swift toolchain but no Darwin SDK)" - mv "$SDK_ARCHIVE.partial" "$SDK_ARCHIVE" - fi - [ -s "$SDK_ARCHIVE" ] || die "$SDK_ARCHIVE is empty" - SDK_SHA256="$(sha256sum "$SDK_ARCHIVE" | cut -d' ' -f1)" - if [ -n "$DARWIN_SDK_SHA256" ] && [ "$DARWIN_SDK_SHA256" != "$SDK_SHA256" ]; then - die "the SDK archive does not match XFORGE_DARWIN_SDK_SHA256 - expected $DARWIN_SDK_SHA256 - got $SDK_SHA256 - (remove $SDK_ARCHIVE to download it again)" - fi - note "$DARWIN_SDK_ASSET: $(du -h "$SDK_ARCHIVE" | cut -f1), sha256 $SDK_SHA256" - # Staged inside the tree, where the chroot can see it. The sdk step - # removes it again once SwiftPM has the SDK. - SDK_GUEST_PATH="/root/.cache/xforge-sdk/$DARWIN_SDK_ASSET" - mkdir -p "$DATA/root/.cache/xforge-sdk" - install -m 0644 "$SDK_ARCHIVE" "$DATA$SDK_GUEST_PATH" - else - note "the Darwin SDK is not included (XFORGE_PROVISION_SDK=0)" - fi - - guest_mount - - # One guest process per provisioning step, through the guest's own installer. - # - # `HOME=/root` is what makes swiftly and SwiftPM install into /root (the - # tree's own /root, so it is inside the root being built) rather than into - # whatever the build machine's environment has in mind — and the XDG - # directories are *unset* on top of it, because SwiftPM's SDK store follows - # them: with the runner's XDG_CONFIG_HOME inherited, `swift sdk install` put - # the SDK in /swiftpm/swift-sdks inside the root being built, - # where the guest's own SwiftPM (which has no XDG set) would never look. - provision() { - local step="$1" - shift - note "install-toolchain.sh $step" - chroot "$DATA" /bin/sh -c \ - "$GUEST_ENV $*; sh /root/install-toolchain.sh $step" \ - || die "the '$step' provisioning step failed in the chroot" - } - - provision deps - provision glibc - provision xtool - provision swiftly - provision swift - if [ -n "$SDK_GUEST_PATH" ]; then - provision sdk \ - "XFORGE_DARWIN_SDK_ARCHIVE=$SDK_GUEST_PATH" \ - "XFORGE_DARWIN_SDK_SHA256=$SDK_SHA256" \ - "XFORGE_DARWIN_SDK_TAG=$DARWIN_SDK_TAG" \ - "XFORGE_DARWIN_SDK_ASSET=$DARWIN_SDK_ASSET" \ - "XFORGE_DARWIN_SDK_URL=$DARWIN_SDK_URL" - fi - - # ----------------------------------------------------------------------- - # Slimming. - # - # Everything removed here is either a download cache (the archive just - # installed, the Ubuntu .deb pile the glibc layer was built from, the apk - # index) or a part of the Swift toolchain an iOS build never loads: the - # *static Linux* stdlib, the debugger and the editor tooling. Nothing here is - # guesswork in the sense that matters — the verification below runs after it, - # so a toolchain this broke would fail the build rather than a user's first - # build on a device. - # ----------------------------------------------------------------------- - log "Slimming the root" - before_kib="$(size_kib "$DATA")" - before_kib="${before_kib:-0}" - - rm -rf "$DATA/var/cache/apk" "$DATA/tmp/xforge-glibc" "$DATA/tmp/xforge-silent."* \ - "$DATA/root/.cache/xforge-sdk" - # /tmp is scratch, but the directory itself stays: apk, the shells and the - # engine's own tools all expect it to exist. - find "$DATA/tmp" -mindepth 1 -maxdepth 1 -exec rm -rf {} + 2>/dev/null || true - - if [ "$SLIM_TOOLCHAIN" = "1" ]; then - removed=0 - for toolchain in "$DATA"/root/.local/share/swiftly/toolchains/*/; do - [ -d "$toolchain" ] || continue - note "slim: $(basename "$toolchain")" - # The static Linux stdlib: only linked when a *Linux* program asks to - # be statically linked against it. xtool builds iOS binaries, whose - # stdlib comes from the Darwin SDK. - rm -rf "${toolchain}usr/lib/swift_static" - # The debugger and the editor tooling: lldb, sourcekit-lsp and the - # index stores exist to serve an IDE, not to compile. - # NOT usr/lib/swift/host/plugins: those are the host-side macro - # plugin libraries, and a package that uses macros needs them. They - # are a few MB, and the compile probe below would not notice their - # absence — a build on a device would. - rm -rf "${toolchain}usr/lib/python3"* \ - "${toolchain}usr/share/doc" "${toolchain}usr/share/man" 2>/dev/null || true - for path in \ - usr/bin/lldb usr/bin/lldb-server usr/bin/sourcekit-lsp \ - usr/lib/liblldb.so usr/lib/libsourcekitdInProc.so \ - usr/lib/libIndexStore.so ; do - rm -rf "${toolchain}${path}"* - done - removed=1 - done - [ "$removed" = "1" ] || note "slim: no toolchain directory found to slim" - else - note "slim: keeping the whole Swift toolchain (XFORGE_SLIM_TOOLCHAIN=0)" - fi - - after_kib="$(size_kib "$DATA")" - after_kib="${after_kib:-0}" - note "tree: $((before_kib / 1024)) MB before slimming, $((after_kib / 1024)) MB after" - - # Verification, and this time the verdict is load-bearing. - # - # install-toolchain.sh verify reports rather than fails (its job in the - # Terminal is to explain), so its XFORGE-VERIFY lines are read here: a tool - # that is missing or not running means the root being packaged cannot do the - # thing it exists to do, and a published artifact is the wrong place to find - # that out. XFORGE_VERIFY_COMPILE adds the check that actually matters — that - # the toolchain can compile and run a program, not just print a version. - log "Verifying the provisioned toolchain" - verify_log="$WORK/verify-toolchain.log" - chroot "$DATA" /bin/sh -c \ - "$GUEST_ENV XFORGE_VERIFY_COMPILE=${XFORGE_VERIFY_COMPILE:-1}; \ - sh /root/install-toolchain.sh verify 2>&1" | tee "$verify_log" - if grep -qE 'XFORGE-VERIFY (swift|swiftly|xtool|swift-sdk|swift-compile) (missing|broken)' \ - "$verify_log"; then - die "the provisioned toolchain does not work in the root being built — - see the XFORGE-VERIFY lines above. Packaging it would ship a guest that - cannot build anything." - fi - grep -q 'XFORGE-VERIFY swift ok' "$verify_log" \ - || die "the Swift toolchain did not report ok — see $verify_log" - - guest_umount - - note "toolchain provisioned: xtool, swiftly, swift${SDK_SHA256:+, darwin SDK}" -fi - -# --------------------------------------------------------------------------- -# 6. Configure the root -# -# A bare Alpine minirootfs is not quite bootable as XForge's guest: it has no -# mount points, no DNS, no apk repositories, and root may not have a usable -# shell. These are the same adjustments the reference makes. -# --------------------------------------------------------------------------- -log "Configuring the root" - -# The shell the console starts is a property of /etc/passwd — /sbin/xforge-login -# reads this field and execs what it names, so this one line is the whole -# "default shell" setting. `chsh` is not needed and is not in the root; editing -# the field (or `apk add shadow` for chsh) is the way to change it in the guest. -if [ -f "$DATA/etc/passwd" ]; then - if [ -x "$DATA$DEFAULT_SHELL" ]; then - sed -i "s|^root:.*|root:x:0:0:root:/root:$DEFAULT_SHELL|" "$DATA/etc/passwd" - else - # Not a hard failure: a root without the shell it promised is worse than a - # root that boots into the one it certainly has. Say which it is, loudly. - printf 'warning: %s is not installed in the root (add its package to\n' "$DEFAULT_SHELL" >&2 - printf ' XFORGE_CONSOLE_PACKAGES, or set XFORGE_DEFAULT_SHELL)\n' >&2 - printf ' — falling back to /bin/sh\n' >&2 - DEFAULT_SHELL=/bin/sh - sed -i 's|^root:.*|root:x:0:0:root:/root:/bin/sh|' "$DATA/etc/passwd" - fi -fi -note "root's login shell: $DEFAULT_SHELL" - -# What the console says about the toolchain, which is the one thing a user -# notices first: whether the guest can build anything yet, or whether it has to -# provision itself. It follows XFORGE_PROVISION rather than describing one of the -# two states as if it were the only one. -if [ "$PROVISION" = "all" ]; then - MOTD_TOOLCHAIN=" The build toolchain is installed: xtool, the Swift toolchain - and the Darwin SDK are ready to use. \`xtool --version\` and - \`swift sdk list\` answer from this shell with no setup." -else - MOTD_TOOLCHAIN=" Swift and xtool are not installed yet. Install them in the guest with: - sh /root/install-toolchain.sh all" -fi - -# /etc/profile: the non-interactive environment is built by XForge's exec layer, -# but an interactive shell (the terminal tab) reads this. -cat >> "$DATA/etc/profile" <<'EOF' - -# XForge guest configuration -export PS1='\u@xforge:\w\$ ' -export TERM=xterm-256color -export HOME=/root -export LANG=C.UTF-8 -export CHARSET=UTF-8 -export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/bin - -alias ll='ls -la' -alias la='ls -A' -alias l='ls -CF' - -cd ~ -EOF - -cat > "$DATA/etc/motd" < "$DATA/etc/inittab" <<'EOF' -# /etc/inittab — XForge -# -# pid 1 is /sbin/init. There is one console: tty1, the same terminal as -# /dev/console, and the one the app shows. -::sysinit:/etc/init.d/rcS - -# Respawned, so logging out (or a crash) gives a fresh session rather than a dead -# screen. No password, no getty: xforge-login execs the login shell configured for -# root in /etc/passwd. -tty1::respawn:/sbin/xforge-login root - -::ctrlaltdel:/sbin/reboot -::shutdown:/bin/umount -a -r -EOF -note "inittab: pid 1 is /sbin/init, tty1 respawns /sbin/xforge-login root" - -mkdir -p "$DATA/etc/init.d" -cat > "$DATA/etc/init.d/rcS" <<'EOF' -#!/bin/sh -# XForge: the sysinit step, run by init before it starts the console login. -# -# /proc and /dev/pts are already mounted by the engine before pid 1 exists, and -# this root has no openrc and no services, so there is nothing to start — rcS -# only has to finish the runtime layout a login expects. -mkdir -p /run /tmp /dev/pts /dev/shm -chmod 1777 /tmp -mount -t proc proc /proc 2>/dev/null -mount -t devpts devpts /dev/pts 2>/dev/null -hostname xforge 2>/dev/null -exit 0 -EOF -chmod 0755 "$DATA/etc/init.d/rcS" - -# apk needs to know where packages come from; this is what makes the in-guest -# toolchain install (and any `apk add`) work at all. -mkdir -p "$DATA/etc/apk" -cat > "$DATA/etc/apk/repositories" < "$DATA/etc/resolv.conf" <<'EOF' -nameserver 1.1.1.1 -nameserver 8.8.8.8 -EOF - -# The provisioning script is what the guest runs to install Swift/xtool. It ships -# inside the app and is written to /host by the host side, but a copy in the root -# makes the command in the motd work without the app. -if [ -f "$HERE/install-toolchain.sh" ]; then - install -m 0755 "$HERE/install-toolchain.sh" "$DATA/root/install-toolchain.sh" - note "install-toolchain.sh staged at /root/install-toolchain.sh" -fi - -# A stamp the app can check to know which root this is. XForge's installer reads -# it to decide whether an existing install can be reused. -mkdir -p "$DATA/usr/local/share/xforge" -MANIFEST_PATH="$DATA/usr/local/share/xforge/rootfs-manifest.txt" -{ - echo "base: alpine-minirootfs-${ALPINE_VERSION}.${ALPINE_MINOR}-${ALPINE_ARCH}.tar.gz" - echo "rootfs: ${ALPINE_VERSION}.${ALPINE_MINOR}" - echo "engine: ish-arm64" - echo "format: fakefs-zip" - echo "shell: $DEFAULT_SHELL" - echo "console: /sbin/xforge-login root (tty1, respawned by init)" - # What the guest arrives with. This is the line that tells a user (and the - # verification, and the app) whether the root is the plain one or the - # provisioned one, without booting it. - if [ "$PROVISION" = "all" ]; then - echo "toolchain: xtool, swiftly, swift" - if [ -n "$SDK_SHA256" ]; then - echo "darwin-sdk: $DARWIN_SDK_TAG ($DARWIN_SDK_ASSET)" - echo "sdk-sha256: $SDK_SHA256" - # The SDK's own record of where SwiftPM put it, passed through rather - # than recomputed: verification checks this exact path, and a path - # that is written down where it is made cannot drift from one that is - # guessed at where it is checked. - sdk_path="$(awk '/^[[:space:]]*path:/ { print $2 }' \ - "$DATA/usr/local/share/xforge/darwin-sdk.txt" 2>/dev/null || true)" - [ -n "$sdk_path" ] && echo "darwin-sdk-path: $sdk_path" - else - echo "darwin-sdk: not included" - fi - else - echo "toolchain: not provisioned (the guest installs it on demand)" - fi - echo "built-at: $(date -u +%Y-%m-%dT%H:%M:%SZ)" - # Bump this whenever anything about the guest's own setup changes — the app - # compares it with the root it already installed and replaces the root when - # they differ (see RootfsInstaller.installedRootIsStale). It must match the - # release tag the root is published under. - echo "stamp: rootfs-v5" -} > "$MANIFEST_PATH" - -# --------------------------------------------------------------------------- -# The console, exercised. -# -# Everything above is configuration, and configuration that looks right reads -# exactly like configuration that works. So the root is handed to -# EmbeddedLinux/verify-rootfs.sh, which runs the guest's own login program in a -# chroot of this tree — the way init runs it — and checks which shell ends up -# running. The same script checks the ZIP that gets published, so a device and -# this build agree on what "correct" means. -# --------------------------------------------------------------------------- -log "Checking the finished root" -guest_mount -"$HERE/verify-rootfs.sh" "$DATA" -guest_umount - -# --------------------------------------------------------------------------- -# 7. Convert to fakefs -# -# Last, after everything that adds, removes or links anything — the conversion -# builds `meta.db` as an index of the tree as it stands, and the engine reads -# the database rather than scanning the directory. Anything added afterwards -# would be on disk and invisible in the guest. -# -# `fakefs_import` reads an *archive* (libarchive, gzip+tar), not a directory, so -# the configured tree is packed once here and converted from that. -# --------------------------------------------------------------------------- -OUT_ROOTFS="$WORK/$ROOTFS_NAME" - -# The manifest is the root's own description of itself, the summary below prints -# it, and the tree it lives in is deleted by the conversion below — so it is -# copied out first. -MANIFEST_COPY="$WORK/rootfs-manifest.txt" -cp "$MANIFEST_PATH" "$MANIFEST_COPY" - -log "Converting to fakefs" -rm -rf "$OUT_ROOTFS" -STAGED_TAR="$WORK/$ROOTFS_NAME.tar.gz" -# Report what is being converted. The conversion is where a mistake in the -# ordering shows up (the engine reads meta.db, not the directory), so the tree's -# own numbers are worth having in the log next to the result. -note "tree: $(du -sh "$DATA" | cut -f1), $(find "$DATA" -mindepth 1 | wc -l) entries" -tar -czf "$STAGED_TAR" -C "$DATA" . -note "staged: $(du -h "$STAGED_TAR" | cut -f1)" - -# The tree is deleted *here*, between packing and converting, and the ordering is -# load-bearing rather than tidy. A provisioned tree is ~5 GB and the fakefs root -# it converts to is about the same size again, so holding both at once — which is -# what this did while the rootfs was small — needs ~11 GB, and a GitHub runner -# has 14 GB in total. Spending the tar instead (which fakefsify is about to read -# anyway) halves the peak, and the tar is already a complete copy of the tree. -# The cost is that a failure inside fakefsify leaves no tree to inspect; the -# packaging is what needs the disk, and the tar is kept until the conversion -# succeeds. -rm -rf "$DATA" -"$FAKEFSIFY" "$STAGED_TAR" "$OUT_ROOTFS" -rm -f "$STAGED_TAR" - -[ -d "$OUT_ROOTFS/data" ] || die "fakefsify produced no data/ directory" -[ -f "$OUT_ROOTFS/meta.db" ] || die "fakefsify produced no meta.db" -note "data: $(du -sh "$OUT_ROOTFS/data" | cut -f1)" -note "meta: $(du -h "$OUT_ROOTFS/meta.db" | cut -f1)" - -# The glibc layer must be *indexed*, not merely present. This is the check that -# would have caught installing it after conversion: the files existed on disk and -# the guest could not see them at all. -# -# `CAST(path AS TEXT)` is not decoration. fakefs stores paths as BLOB, and a -# LIKE against a BLOB only works through SQLite's implicit coercion — which is -# version-dependent, so the same query that matched every row locally matched -# zero rows on the runner's older SQLite. Comparing as text removes the -# ambiguity instead of relying on it. -if [ "${XFORGE_SKIP_GLIBC:-0}" != "1" ]; then - if command -v sqlite3 >/dev/null 2>&1; then - indexed="$(sqlite3 "$OUT_ROOTFS/meta.db" \ - "SELECT COUNT(*) FROM paths WHERE CAST(path AS TEXT) LIKE '%aarch64-linux-gnu%';" \ - 2>/dev/null || echo 0)" - total="$(sqlite3 "$OUT_ROOTFS/meta.db" \ - "SELECT COUNT(*) FROM paths;" 2>/dev/null || echo 0)" - [ "${indexed:-0}" -gt 0 ] || die \ - "the glibc layer is not indexed in meta.db ($indexed of ${total:-0} paths) - — the guest would not see it. It must be installed before the fakefs - conversion, not copied in afterwards." - note "glibc paths indexed in meta.db: $indexed of $total" - else - note "sqlite3 not available — skipping the glibc indexing check" - fi -fi - -# --------------------------------------------------------------------------- -# 8. Pack -# --------------------------------------------------------------------------- -log "Packing $ZIP_NAME" -ZIP_PATH="$OUT_DIR/$ZIP_NAME" -rm -f "$ZIP_PATH" - -# Run from the work dir so the archive contains `alpine-rootfs/...` rather than -# absolute paths — the app unzips it straight into Documents. -# -# The SQLite sidecar files are excluded on purpose: `meta.db-wal`/`-shm` are -# transient and are not part of the root. Shipping them would be harmless at -# best and a corrupt-looking database at worst. -( - cd "$WORK" - zip -qr "$ZIP_PATH" "$ROOTFS_NAME" -x "*.db-shm" -x "*.db-wal" -) - -[ -f "$ZIP_PATH" ] || die "the ZIP was not produced" -# Read the listing once: `unzip -Z1 … | grep -q` is not safe under `set -o -# pipefail` (grep exits at the first match, unzip dies of SIGPIPE, and the -# pipeline then reports failure for a check that passed). -unzip -Z1 "$ZIP_PATH" > "$WORK/zip-contents.txt" -grep -qE "^$ROOTFS_NAME/data/$" "$WORK/zip-contents.txt" \ - || die "the ZIP has no $ROOTFS_NAME/data/ entry" -grep -qE "^$ROOTFS_NAME/meta\.db$" "$WORK/zip-contents.txt" \ - || die "the ZIP has no $ROOTFS_NAME/meta.db entry" -! grep -qE '\.db-(wal|shm)$' "$WORK/zip-contents.txt" \ - || die "the ZIP contains SQLite WAL/SHM sidecars" - -# Checksum, so a consumer can pin the exact root it expects. -( cd "$OUT_DIR" && sha256sum "$ZIP_NAME" > "$ZIP_NAME.sha256" ) - -log "Done" -note "rootfs: $ZIP_PATH" -note "size: $(du -h "$ZIP_PATH" | cut -f1)" -note "sha256: $(cut -d' ' -f1 < "$ZIP_PATH.sha256")" -note "entries: $(wc -l < "$WORK/zip-contents.txt")" -echo -cat "$MANIFEST_COPY" - -if [ -n "${GITHUB_OUTPUT:-}" ]; then - { - echo "rootfs=$ZIP_PATH" - echo "rootfs_name=$ZIP_NAME" - echo "rootfs_size=$(stat -c %s "$ZIP_PATH" 2>/dev/null || stat -f %z "$ZIP_PATH")" - echo "rootfs_sha256=$(cut -d' ' -f1 < "$ZIP_PATH.sha256")" - echo "rootfs_toolchain=$([ "$PROVISION" = all ] && echo provisioned || echo none)" - } >> "$GITHUB_OUTPUT" -fi - -if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - { - echo "### Alpine rootfs" - echo - echo '```' - cat "$MANIFEST_COPY" - echo '```' - echo - echo "\`$ZIP_NAME\` — $(du -h "$ZIP_PATH" | cut -f1)" - echo - echo "- toolchain: $([ "$PROVISION" = all ] && echo \ - "xtool, swiftly, Swift${SDK_SHA256:+ and $DARWIN_SDK_TAG}" || \ - 'not provisioned')" - echo "- entries: $(wc -l < "$WORK/zip-contents.txt")" - } >> "$GITHUB_STEP_SUMMARY" -fi diff --git a/EmbeddedLinux/fetch-rootfs.sh b/EmbeddedLinux/fetch-rootfs.sh deleted file mode 100755 index 15e5a49..0000000 --- a/EmbeddedLinux/fetch-rootfs.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -# -# fetch-rootfs.sh — put the pinned Alpine rootfs where the app target bundles it. -# -# The guest root is a release asset, not a committed file: `alpine-rootfs.zip` is -# an already-converted fakefs published by the rootfs workflow, and -# `.github/workflows/build-ipa.yml` pins the tag it was built from (`ROOTFS_TAG`). -# It is ~165 MB when the root is plain Alpine and ~1.6 GB when it also carries the -# build toolchain (xtool, Swift, the Darwin SDK) — which is what the current tag -# pins, and the reason the download here is worth resuming and checking. -# It has to be in place *before* the Xcode project is generated — the app target -# lists it as a resource, and a build phase cannot add a file to a resource list -# that was fixed when XcodeGen walked the directory. -# -# Anything that builds this repo therefore needs this step before XcodeGen (or -# before opening the project in Xcode): -# -# EmbeddedLinux/fetch-rootfs.sh -# -# The workflow that publishes the IPA uses the same script, so there is one -# implementation of "which rootfs, verified how". -# -# Environment: -# XFORGE_ROOTFS_TAG override the pinned tag (default: ROOTFS_TAG in -# .github/workflows/build-ipa.yml) -# XFORGE_ROOTFS_FORCE=1 re-download even if the file is already there -# XFORGE_ROOTFS_DIR where it lands (default: /Support/Resources) -# -set -euo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -REPO="$(cd "$HERE/.." && pwd)" -DEST="${XFORGE_ROOTFS_DIR:-$REPO/Support/Resources}" -ASSET="alpine-rootfs.zip" - -log() { printf '\n==> %s\n' "$*"; } -note() { printf ' %s\n' "$*"; } -die() { printf '\nerror: %s\n' "$*" >&2; exit 1; } - -# The pinned tag lives in the IPA workflow, which is where someone upgrading the -# root changes it. Reading it from there keeps this from becoming a second place -# that has to be edited in step. -TAG="${XFORGE_ROOTFS_TAG:-$(sed -n 's/^[[:space:]]*ROOTFS_TAG:[[:space:]]*//p' \ - "$REPO/.github/workflows/build-ipa.yml" | head -1)}" -[ -n "$TAG" ] || die "could not read ROOTFS_TAG from .github/workflows/build-ipa.yml - (set XFORGE_ROOTFS_TAG to name the release yourself)" -# Which repository publishes the rootfs: this one. Ask git, so a fork fetches its -# own — and *not* `.gitmodules`, whose first entry is the engine (that mistake -# resolved the engine's URL and 404ed). A tarball export has no remote, so fall back -# to the app's release repository; keep in step with XForgeReleases.repository. -SLUG="$(git -C "$REPO" config --get remote.origin.url 2>/dev/null \ - | sed -e 's|^.*github\.com[:/]||' -e 's|\.git$||')" -[ -n "$SLUG" ] || SLUG="R0GUEEE/XForge" - -mkdir -p "$DEST" -TARGET="$DEST/$ASSET" - -if [ -f "$TARGET" ] && [ "${XFORGE_ROOTFS_FORCE:-0}" != "1" ]; then - log "The pinned rootfs is already staged" - note "$TARGET ($(du -h "$TARGET" | cut -f1))" - exit 0 -fi - -log "Fetching the pinned Alpine rootfs: $TAG" -note "into $TARGET" - -# `gh` when it is installed *and* has a token, plain curl otherwise. These releases -# are public, so no credential is needed — and a `gh` with no token is not an error -# to report, it is a reason to use curl (which is how a pipeline that does not pass -# GH_TOKEN gets its rootfs; treating that as fatal is how this step failed once). -BASE="https://github.com/$SLUG/releases/download/$TAG" - -if command -v gh >/dev/null 2>&1 \ - && { [ -n "${GH_TOKEN:-}" ] || [ -n "${GITHUB_TOKEN:-}" ]; }; then - note "using gh (a token is in the environment)" - rm -f "$TARGET" - if ! gh release download "$TAG" --repo "$SLUG" \ - --pattern "$ASSET" --pattern "$ASSET.sha256" \ - --dir "$DEST" --clobber; then - note "gh could not fetch it — falling back to curl" - fi -fi - -if [ ! -f "$TARGET" ]; then - # A half-downloaded asset must not survive a failure: the next run (or a human) - # would find a plausible-looking file of the wrong size. - trap 'rm -f "$TARGET.partial"' EXIT - note "downloading $BASE/$ASSET" - curl -fL --retry 3 --retry-delay 2 -o "$TARGET.partial" "$BASE/$ASSET" \ - || die "could not download $BASE/$ASSET" - mv "$TARGET.partial" "$TARGET" - curl -fL --retry 3 -o "$DEST/$ASSET.sha256" "$BASE/$ASSET.sha256" 2>/dev/null || true -fi - -[ -f "$TARGET" ] || die "$ASSET was not downloaded" - -# The checksum is the point of pinning a tag: a re-uploaded or truncated asset has -# to fail here rather than inside the guest on someone's phone. -if [ -f "$TARGET.sha256" ]; then - ( cd "$DEST" && shasum -a 256 -c "$ASSET.sha256" >/dev/null 2>&1 ) \ - || die "the downloaded $ASSET does not match its published sha256" - note "sha256 verified" -else - note "warning: no .sha256 beside the asset — not verified" -fi - -# And it has to be the *shape* the app unpacks: a fakefs ZIP, not a tarball. -command -v unzip >/dev/null 2>&1 || { note "$ASSET ($(du -h "$TARGET" | cut -f1)), not inspected"; exit 0; } -unzip -Z1 "$TARGET" > /tmp/xforge-rootfs-entries.txt -for entry in "alpine-rootfs/data/" "alpine-rootfs/meta.db"; do - grep -qx "$entry" /tmp/xforge-rootfs-entries.txt \ - || die "$TARGET has no $entry — that is not a fakefs root" -done -rm -f /tmp/xforge-rootfs-entries.txt - -log "Rootfs staged" -note "$TARGET ($(du -h "$TARGET" | cut -f1))" diff --git a/EmbeddedLinux/install-toolchain.sh b/EmbeddedLinux/install-toolchain.sh deleted file mode 100644 index 27a85f0..0000000 --- a/EmbeddedLinux/install-toolchain.sh +++ /dev/null @@ -1,920 +0,0 @@ -#!/bin/sh -# -# install-toolchain.sh — provision the XForge embedded Linux (inside the guest). -# -# Runs *in the guest* (Alpine aarch64 / musl), so the same script works from -# XForge's Terminal and from the Toolchain screen, which drives the steps one at -# a time to show progress: -# -# sh /root/install-toolchain.sh # every step -# sh /root/install-toolchain.sh deps # one step -# -# Set XFORGE_INSTALL_XTOOL=0 for a base build root that leaves xtool for the -# app's on-device component installer. -# Steps: deps | glibc | xtool | swiftly | swift | sdk | sdk-remove | verify -# -# Every step is idempotent: re-running it is a no-op once it has succeeded. -# -# The tools are glibc binaries (xtool is a Swift program built on Ubuntu; the -# toolchains swiftly installs are Ubuntu builds too) and this guest is musl, so -# a real glibc runtime is the first thing to install — `gcompat` is not enough. -# `test-glibc` below exercises it. -# -set -eu - -ARCH="$(uname -m)" -# Ubuntu names the same machine two different ways and both are needed: its -# archive and pool use "arm64"/"amd64", while the directory the libraries land in -# is the GNU triple, "aarch64-linux-gnu". -case "$ARCH" in - aarch64) UBUNTU_ARCH=arm64 ;; - x86_64) UBUNTU_ARCH=amd64 ;; - *) UBUNTU_ARCH="$ARCH" ;; -esac -MULTIARCH="$ARCH-linux-gnu" -GLIBC_ROOT=/opt/glibc -GLIBC_LIB="$GLIBC_ROOT/usr/lib/$MULTIARCH" -GLIBC_LD="$GLIBC_LIB/ld-linux-aarch64.so.1" -SHARE=/usr/local/share/xforge -SWIFTLY_HOME_DIR="${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}" -export SWIFTLY_HOME_DIR -INSTALL_XTOOL="${XFORGE_INSTALL_XTOOL:-1}" - -# Where the Darwin Swift SDK comes from, and where it is unpacked on the way in. -# -# The SDK is the one component XForge cannot generate: it is Apple's, and either -# the user's own Xcode.xip or a bundle someone already built from one. XForge -# publishes its own (`darwin-sdk-`, built with xtool) so a guest with no .xip -# still gets a working SDK — the app's Downloads screen installs exactly this -# asset, and this step installs the same thing when the rootfs is built. -# -# The tag is pinned rather than resolved through the API: the rootfs is packaged -# once and shipped, so "which SDK is in this root" has to be answerable from the -# build log, not from whatever `releases/latest` answered that minute. Set -# XFORGE_DARWIN_SDK_URL to install a bundle from somewhere else entirely. -SDK_REPO="${XFORGE_DARWIN_SDK_REPO:-R0GUEEE/XForge}" -SDK_TAG="${XFORGE_DARWIN_SDK_TAG:-darwin-sdk-7}" -SDK_ASSET="${XFORGE_DARWIN_SDK_ASSET:-darwin.artifactbundle.zip}" -SDK_URL="${XFORGE_DARWIN_SDK_URL:-https://github.com/$SDK_REPO/releases/download/$SDK_TAG/$SDK_ASSET}" -SDK_CACHE="${XFORGE_SDK_CACHE:-/root/.cache/xforge-sdk}" - -# Ubuntu release whose glibc the Swift toolchains are built against. -UBUNTU_SUITE="${XFORGE_UBUNTU_SUITE:-noble}" - -log() { echo "==> $*"; } - -# --------------------------------------------------------------------------- -# Output sinks -# -# Never send a *guest process*'s output to /dev/null here. the engine's arm64 -# engine has been observed to SIGKILL a forked guest program whose stdout/stderr -# points at /dev/null — `swift --version >/dev/null 2>&1` died where the same -# command without the redirect ran fine, and `apk info -e … >/dev/null` died -# intermittently. Pipes and real files are safe, and a shell *builtin* -# (`command -v x >/dev/null`) is not affected, so silence goes to a file. -# --------------------------------------------------------------------------- -SILENT="${TMPDIR:-/tmp}/xforge-silent.$$" -: > "$SILENT" - -# --------------------------------------------------------------------------- -# glibc compatibility layer -# --------------------------------------------------------------------------- -# -# Binaries built on Ubuntu need Ubuntu's glibc. Rather than compiling one (an -# hour in the emulator), take the runtime out of the distribution's own packages -# and run them through *its* loader: -# -# $GLIBC_LD --library-path $GLIBC_LIB: -# -# The loaders mount the necessary packages under $GLIBC_ROOT. The package list -# is what a Swift/xtool binary actually needs beyond libc itself: the toolchains -# link against libxml2, curl, krb5, ldap and ICU (Foundation), which the Ubuntu -# AppImage does not bundle. - -# The development halves are needed to *link*, not just to run: glibc's -# crt1.o/crti.o/crtn.o/libc.so and libc_nonshared.a come from libc6-dev, and -# GCC's crtbeginS.o/crtendS.o/libgcc.a from libgcc--dev. Without them a link -# dies with -# /usr/bin/ld: cannot find crtbeginS.o / cannot find -lgcc -# which is what compiling a Swift program in this rootfs hit — `swift --version` -# was fine, because printing a version does not link anything. -# -# One name per line, NO comments: the caller reads this list with word splitting. -glibc_packages() { - cat <<'EOF' -libc6 -libc6-dev -libgcc-s1 -libgcc-13-dev -libgcc-14-dev -libstdc++6 -libstdc++-13-dev -libstdc++-14-dev -zlib1g -libcom-err2 -libkeyutils1 -libkrb5-3 -libk5crypto3 -libkrb5support0 -libgssapi-krb5-2 -libsasl2-2 -libldap2 -libgnutls30t64 -libp11-kit0 -libtasn1-6 -libunistring5 -libidn2-0 -libffi8 -libbrotli1 -libicu74 -libxml2 -libpng16-16t64 -libselinux1 -libpcre2-8-0 -libacl1 -libattr1 -libgmp10 -libnettle8t64 -libhogweed6t64 -libcurl4t64 -libnghttp2-14 -libpsl5t64 -libssh-4 -librtmp1 -libssl3t64 -libzstd1 -liblzma5 -libgcrypt20 -libgpg-error0 -libuuid1 -libblkid1 -libcap2 -libedit2 -libpython3.12t64 -libsqlite3-0 -libncurses6 -libncursesw6 -libtinfo6 -libz3-4 -EOF -} - -# Resolve package names to pool URLs with the distribution's own indexes, so the -# script does not carry pinned filenames that go stale. -glibc_package_url() { - for index in /tmp/xforge-glibc/Packages*; do - [ -f "$index" ] || continue - url="$(awk -v want="$1" ' - /^Package: / { pkg = $2 } - /^Filename: / { if (pkg == want) { print "https://ports.ubuntu.com/ubuntu-ports/" $2; exit } } - ' "$index")" - [ -n "$url" ] && { echo "$url"; return 0; } - done - return 1 -} - -glibc_fetch_indexes() { - for component in main universe; do - target="/tmp/xforge-glibc/Packages-$component" - [ -s "$target" ] && continue - url="https://ports.ubuntu.com/ubuntu-ports/dists/$UBUNTU_SUITE/$component/binary-$UBUNTU_ARCH/Packages.gz" - if curl -fL --retry 3 -o "$target.gz" "$url" 2>>"$SILENT"; then - gzip -dc "$target.gz" > "$target" - fi - done -} - -step_glibc() { - # "Already present" has to mean the layer is *usable*, not merely that the - # loader exists. The two halves can diverge: the loader lives under - # /opt/glibc, while the wiring that makes glibc binaries actually run is the - # symlink farm in /lib and /usr/lib. Checking only the loader reports a - # half-built layer as complete, and the failure then appears much later as a - # tool dying on an undefined symbol or an unloadable interpreter. - # - # This is why the check is on the wiring too: the loader a Swift binary is - # loaded by, and the unversioned link names a Swift link resolves against. - if [ -x "$GLIBC_LD" ] && [ -f "$GLIBC_LIB/libc.so.6" ] \ - && [ -e "/lib/ld-linux-$ARCH.so.1" ] && [ -e "/usr/lib/$MULTIARCH" ]; then - log "glibc runtime already present at $GLIBC_ROOT" - else - log "Installing the glibc runtime (Ubuntu $UBUNTU_SUITE, $UBUNTU_ARCH)" - mkdir -p "$GLIBC_LIB" "$GLIBC_ROOT/usr/lib" /tmp/xforge-glibc - glibc_fetch_indexes - - missing_packages="" - for pkg in $(glibc_packages); do - url="$(glibc_package_url "$pkg" || true)" - if [ -z "$url" ]; then - # Not something to skip quietly. A name the suite does not have - # means the layer will be missing a library, and that surfaces - # much later as a symbol lookup error inside one of the tools: - # `libgnutls30` became `libgnutls30t64` in noble, and the miss - # only showed up as "swift-sdk: undefined symbol - # nettle_rsa_oaep_sha512_decrypt, version HOGWEED_6" — while - # `swift --version` kept working, so nothing looked wrong. - echo " $pkg: NOT FOUND in the $UBUNTU_SUITE indexes" >&2 - missing_packages="$missing_packages $pkg" - continue - fi - deb="/tmp/xforge-glibc/$pkg.deb" - [ -s "$deb" ] || curl -fL --retry 3 -o "$deb" "$url" - ( - cd /tmp/xforge-glibc - rm -rf unpack && mkdir unpack && cd unpack - ar x "$deb" - # Ubuntu ships the payload as zstd; older packages use xz/gz. - if [ -f data.tar.zst ]; then - zstd -d -c data.tar.zst | tar -xf - - elif [ -f data.tar.xz ]; then - tar -xJf data.tar.xz - else - tar -xzf data.tar.gz - fi - # Merged-/usr layout: the libraries live under usr/lib//. - if [ -d "usr/lib/$MULTIARCH" ]; then - cp -a "usr/lib/$MULTIARCH/." "$GLIBC_LIB/" - fi - # GCC's own directory: crtbeginS.o/crtendS.o/libgcc.a live under - # usr/lib/gcc///, which is where clang looks for - # the GNU toolchain it links with. - if [ -d "usr/lib/gcc/$MULTIARCH" ]; then - mkdir -p "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH" - cp -a "usr/lib/gcc/$MULTIARCH/." "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH/" - fi - if [ -e "usr/lib/ld-linux-$ARCH.so.1" ]; then - cp -a "usr/lib/ld-linux-$ARCH.so.1" "$GLIBC_ROOT/usr/lib/" 2>>"$SILENT" || true - fi - if [ -d usr/share/icu ]; then - mkdir -p "$GLIBC_ROOT/usr/share" - cp -a usr/share/icu "$GLIBC_ROOT/usr/share/" 2>>"$SILENT" || true - fi - ) - echo " $pkg" - done - if [ -n "$missing_packages" ]; then - echo "the glibc layer cannot be built — no such package:$missing_packages" >&2 - echo "fix glibc_packages() for Ubuntu $UBUNTU_SUITE ($UBUNTU_ARCH)" >&2 - exit 1 - fi - - # A name that resolved is not the same as a library that arrived: the - # layer is only useful if the files a Swift or xtool binary loads are - # actually present, so the files are what gets checked. - glibc_have() { - for candidate in "$GLIBC_LIB/$1"*; do - [ -e "$candidate" ] && return 0 - done - return 1 - } - incomplete="" - for lib in libc.so.6 libm.so.6 libpthread.so.0 libdl.so.2 libstdc++.so libstdc++.so.6 \ - libgcc_s.so.1 libz.so.1 libzstd.so.1 liblzma.so.5 libxml2.so.2 libcurl.so.4 \ - libssl.so.3 libcrypto.so.3 libgnutls.so.30 libhogweed.so.6 \ - libnettle.so.8 libgmp.so.10 libicuuc.so.74 liblber.so.2 \ - libldap.so.2 libpng16.so.16 libpsl.so.5 libz3.so.4 libsqlite3.so.0; do - glibc_have "$lib" || incomplete="$incomplete $lib" - done - # Startup and compiler objects, which are files rather than shared - # libraries: glibc's crt1.o/crti.o/crtn.o/libc.so land in the multiarch - # directory, GCC's crtbeginS.o/crtendS.o/libgcc.a in its versioned one. - for obj in crt1.o crti.o crtn.o libc_nonshared.a; do - glibc_have "$obj" || incomplete="$incomplete $obj" - done - for obj in crtbeginS.o crtendS.o libgcc.a; do - found="" - for candidate in "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH"/*/"$obj"; do - [ -e "$candidate" ] && found=1 - done - [ -n "$found" ] || incomplete="$incomplete $obj" - done - if [ -n "$incomplete" ]; then - echo "the glibc layer is incomplete — nothing provides:$incomplete" >&2 - echo "a package in glibc_packages() was renamed in $UBUNTU_SUITE; the" >&2 - echo "names above are what the Swift/xtool binaries load and link against." >&2 - exit 1 - fi - log "glibc at $GLIBC_LD" - fi - - mkdir -p "$GLIBC_ROOT/usr/lib" "$SHARE" - # A compiler finds its GNU toolchain by path, not through a search path: - # clang looks in /usr/lib/gcc//, so the extracted copy has - # to be visible there (the same trick the multiarch directory below uses). - if [ -d "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH" ]; then - mkdir -p /usr/lib/gcc - ln -sfn "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH" "/usr/lib/gcc/$MULTIARCH" - fi - if [ ! -e "$GLIBC_ROOT/usr/lib/ld-linux-$ARCH.so.1" ]; then - ln -sf "$GLIBC_LD" "$GLIBC_ROOT/usr/lib/ld-linux-$ARCH.so.1" - fi - - # Wire this glibc in as the system's glibc, not just a directory to point a - # wrapper at. Running a tool through the loader only covers the *first* - # process: Swift's driver execs swift-frontend and swift-build as children, - # and a child starts from its own ELF interpreter. With gcompat's stub - # loader at /lib/ld-linux-.so.1 those children went back to musl and - # died on glibc symbols (__isoc23_strtol, mallinfo2, pthread_cond_clockwait) - # and on libraries musl does not have (libncurses.so.6, libuuid.so.1). - # - # Alpine's own programs are unaffected: they use /lib/ld-musl-.so.1 - # and libc.musl-.so.1, never these names. - rm -f "/lib/ld-linux-$ARCH.so.1" - ln -s "$GLIBC_LD" "/lib/ld-linux-$ARCH.so.1" - ln -sfn "$GLIBC_LIB" "/usr/lib/$MULTIARCH" - ln -sfn "$GLIBC_LIB" "/lib/$MULTIARCH" - for lib in libc.so.6 libm.so.6 libpthread.so.0 librt.so.1 libdl.so.2 \ - libresolv.so.2 libutil.so.1 libcrypt.so.1; do - if [ -e "$GLIBC_LIB/$lib" ]; then - rm -f "/lib/$lib" - ln -s "$GLIBC_LIB/$lib" "/lib/$lib" - fi - done - - # What a link searches by default is the guest's /usr/lib, where Alpine's musl - # copies of the *unversioned* names (libc.so, libstdc++.so, libm.so …) come - # first. A Swift link asks for -lstdc++ and gets the musl build, which carries - # no symbol versions, so it fails with - # undefined reference to `std::__throw_logic_error(char const*)@GLIBCXX_3.4' - # Passing -L/-Xlinker -L from the wrapper did not change that (the driver does - # not always hand those through), so the link names themselves are pointed at - # the layer. Only the unversioned files are touched: the runtime sonames - # (libstdc++.so.6, libc.so.6) are left to Alpine's for Alpine's own binaries, - # and glibc binaries resolve those through /lib/aarch64-linux-gnu as before. - for link_name in libc.so libm.so libpthread.so librt.so libdl.so libutil.so \ - libresolv.so libcrypt.so libstdc++.so libgcc_s.so; do - source="" - [ -e "$GLIBC_LIB/$link_name" ] && source="$GLIBC_LIB/$link_name" - if [ -z "$source" ]; then - # libstdc++.so and libgcc_s.so live in GCC's versioned directory, not - # in the multiarch one — the first version of this loop looked only in - # the multiarch directory and silently skipped exactly the name that - # matters most. - for candidate in "$GLIBC_ROOT"/usr/lib/gcc/$MULTIARCH/*/"$link_name"; do - [ -e "$candidate" ] && source="$candidate" - done - fi - [ -n "$source" ] || continue - rm -f "/usr/lib/$link_name" - ln -s "$source" "/usr/lib/$link_name" - done - - cat > "$SHARE/glibc.env" <"$SILENT" 2>&1 || missing="$missing $package" - done - - if [ -n "$missing" ]; then - total=$(printf '%s\n' "$missing" | xargs -n1 | sed '/^[[:space:]]*$/d' | wc -l | tr -d ' ') - current=0 - for package in $missing; do - current=$((current + 1)) - log "Installing missing Alpine build dependency $current/$total: $package" - apk add --no-cache "$package" - done - update-ca-certificates >/dev/null 2>&1 || true - log "Alpine build dependencies installed" - else - log "Alpine build dependencies already installed" - fi -} -xtool_wrapper() { - cat > /usr/local/bin/xtool <<'EOF' -#!/bin/sh -# xtool is an Ubuntu glibc binary; run it through the glibc loader installed by -# install-toolchain.sh, with the libraries it ships alongside it. -. /usr/local/share/xforge/glibc.env -# The AppImage's own libraries come first: Ubuntu's libcurl (which Swift needs) -# is newer than the nghttp2 the AppImage bundles, and mixing them the other way -# round gives "undefined symbol: nghttp2_option_set_no_rfc9113_...". -exec "$XFORGE_GLIBC_LD" --library-path "/opt/xtool/usr/lib:$XFORGE_GLIBC_LIB" \ - /opt/xtool/usr/bin/xtool "$@" -EOF - chmod +x /usr/local/bin/xtool -} - -step_xtool() { - log "Installing xtool ($ARCH)" - if [ -x /opt/xtool/usr/bin/xtool ]; then - log "xtool already unpacked at /opt/xtool" - else - xtool_asset="" - for candidate in "$ARCH" "$UBUNTU_ARCH"; do - [ -n "$candidate" ] || continue - [ "$candidate" = "$xtool_asset" ] && continue - if curl -fsIL --retry 3 \ - "https://github.com/xtool-org/xtool/releases/latest/download/xtool-$candidate.AppImage" \ - >"$SILENT" 2>&1; then - xtool_asset="$candidate" - break - fi - done - [ -n "$xtool_asset" ] || { - echo "could not resolve an xtool AppImage for architecture '$ARCH' (also tried '$UBUNTU_ARCH')" >&2 - exit 1 - } - # Exactly the release asset xtool documents, unpacked once instead of - # every run (APPIMAGE_EXTRACT_AND_RUN extracts ~50 MB per invocation). - curl -fL --retry 3 \ - "https://github.com/xtool-org/xtool/releases/latest/download/xtool-$xtool_asset.AppImage" \ - -o /tmp/xtool.AppImage - chmod +x /tmp/xtool.AppImage - ( cd /opt && rm -rf squashfs-root && /tmp/xtool.AppImage --appimage-extract >"$SILENT" 2>&1 ) - rm -rf /opt/xtool && mv /opt/squashfs-root /opt/xtool - rm -f /tmp/xtool.AppImage - fi - xtool_wrapper - log "xtool installed at /usr/local/bin/xtool (unpacked in /opt/xtool)" -} - -step_swiftly() { - log "Installing swiftly (the Swift installer)" - if [ -x /usr/local/bin/swiftly ]; then - log "swiftly already present" - return 0 - fi - cd /tmp - curl -fL --retry 3 -O "https://download.swift.org/swiftly/linux/swiftly-$ARCH.tar.gz" - tar zxf "swiftly-$ARCH.tar.gz" - rm -f "swiftly-$ARCH.tar.gz" - # --assume-yes/--quiet-shell-followup: there is no terminal to answer its - # prompts from, and its profile edits are pointless (XForge runs - # non-interactive /bin/sh -c). - ./swiftly init --assume-yes --quiet-shell-followup --skip-install --platform "ubuntu24.04" - rm -f ./swiftly - ln -sf "$SWIFTLY_HOME_DIR/bin/swiftly" /usr/local/bin/swiftly - log "swiftly installed at /usr/local/bin/swiftly" -} - -# swift and swiftc are glibc binaries too, and they live under the swiftly home -# rather than on any PATH a `sh -c` command would search. The toolchain is looked -# up directly first: swiftly's own shims only work in a shell that has sourced its -# environment file, which is exactly what XForge's command runner is not. -swift_wrapper() { - name="$1" - # The linker's own search path is the guest's (musl) one, where Alpine's - # libstdc++ — which carries no symbol versions — shadows Ubuntu's, and - # linking anything against libswiftCore fails with - # undefined reference to \`std::__throw_logic_error(char const*)@GLIBCXX_3.4' - # So the layer's directories go on the link line explicitly, ahead of - # anything the driver adds. (-L only reorders the search; it changes no - # runtime path, so Alpine's own binaries are untouched.) - # -rpath-link is the part that actually matters, and it is not the same thing - # as -L: when the linker pulls in a *shared* library, that library's own - # NEEDED entries (libswiftCore.so wants libstdc++.so.6 and libc.so.6) are - # resolved through the rpath-link path, not the -L path. Without it the - # linker falls back to the guest's /lib and /usr/lib, where Alpine's - # libc6-compat stubs and musl's libstdc++ sit — which is why every attempt - # with -L alone still failed: - # ld: /lib/libc.musl-aarch64.so.1: warning: the `gets' ... - # undefined reference to `std::__throw_logic_error(char const*)@GLIBCXX_3.4' - # Verified before shipping: with -rpath-link pointing at the layer, a link - # against Ubuntu's libxml2.so.2 is clean and the binary runs in a musl guest. - link_dirs="" - for dir in "$GLIBC_LIB" "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH" "$GLIBC_ROOT"/usr/lib/gcc/$MULTIARCH/*/; do - [ -d "$dir" ] || continue - link_dirs="$link_dirs -Xlinker -L${dir%/} -Xlinker -rpath-link -Xlinker ${dir%/}" - done - - cat > "/usr/local/bin/$name" <:0: error: error opening input file 'sdk' (No such file or directory) - # That is exactly how a rootfs build failed after installing a toolchain that - # worked, and it is why the list below exists rather than a blanket exec: the - # SwiftPM subcommands reach the linker through their own \`swiftc\` (which is - # this same script), so they are not the invocations that need the flags. - # A driver subcommand this list misses fails loudly on its first run rather - # than quietly linking the wrong libraries. - case "\${1:-}" in - sdk|experimental-sdk|package|build|test|run|demangle) - exec "\$candidate" "\$@" ;; - esac - exec "\$candidate" $link_dirs "\$@" -done - -. "\$home/env.sh" >/dev/null 2>&1 || true # sourcing, not a forked program: the only safe /dev/null here -for dir in \${PATH}; do - case "\$dir" in /usr/local/bin) continue;; esac - if [ -x "\$dir/$name" ]; then - exec "\$dir/$name" "\$@" - fi -done - -echo "xforge: $name is not installed yet (run: swiftly install latest --use)" >&2 -echo " looked in \$home/toolchains/*/usr/bin and \$home/bin" >&2 -exit 127 -EOF - chmod +x "/usr/local/bin/$name" -} - -step_swift() { - log "Installing the Swift toolchain" - [ -e "$SHARE/glibc.env" ] || { echo "run the glibc step first" >&2; exit 1; } - . "$SHARE/glibc.env" - - # The wrappers go in first, so `swift --version` always answers something - # useful — a version, or exactly why there is none yet. - swift_wrapper swift - swift_wrapper swiftc - - . "$SWIFTLY_HOME_DIR/env.sh" - # `swiftly list` exits 0 and prints a separator even with nothing installed, - # so ask the directory that actually holds toolchains. - if [ -n "$(ls -A "$SWIFTLY_HOME_DIR/toolchains" 2>>"$SILENT")" ]; then - log "a toolchain is already installed: $(ls "$SWIFTLY_HOME_DIR/toolchains" | tail -1)" - return 0 - fi - - # swiftly verifies the download's signature with gpg, and refuses without it - # ("gpg is not installed ... To skip signature verification, specify - # --no-verify"). Try to verify; fall back to skipping it, loudly. - # swiftly exits non-zero for its Ubuntu "these dependencies should be - # installed" advice even when the install worked, so the toolchain directory - # is what decides whether to retry -- not the exit status. - if command -v gpg >/dev/null 2>&1; then - swiftly install latest --use --assume-yes || true - else - echo " gpg is not installed in the guest: skipping signature verification" - swiftly install latest --use --assume-yes --no-verify || true - fi - if [ -z "$(ls -A "$SWIFTLY_HOME_DIR/toolchains" 2>>"$SILENT")" ]; then - echo " no toolchain after the first attempt; retrying without signature verification" - swiftly install latest --use --assume-yes --no-verify - fi - - # `swiftly init --skip-install` leaves it unlinked; make it manage the - # toolchain we just installed. - swiftly link >"$SILENT" 2>&1 || true - - # Say what actually resolved. A guest whose swift cannot run is much easier to - # diagnose from these four lines than from a wrapper's refusal. - log "swift wrapper installed at /usr/local/bin/swift" - echo " toolchains: $(ls "$SWIFTLY_HOME_DIR/toolchains" 2>>"$SILENT" | tr '\n' ' ')" - echo " swiftly bin: $(ls "$SWIFTLY_HOME_DIR/bin" 2>>"$SILENT" | tr '\n' ' ')" - if out="$(swift --version 2>&1)"; then - echo " swift --version: $(printf '%s' "$out" | head -1)" - else - echo " swift --version FAILED: $(printf '%s' "$out" | head -3 | tr '\n' ' ')" - fi -} - -# --------------------------------------------------------------------------- -# The Darwin SDK -# -# The last piece of the toolchain, and the only one that is not on swift.org: -# the arm64-apple-ios Swift SDK xtool compiles iOS apps against. XForge publishes -# its own bundle (`darwin-sdk-`), built with xtool from an Xcode.xip, and this -# installs that bundle with SwiftPM's own `swift sdk install` — the same command -# the Toolchain screen runs, so a root provisioned here and a guest provisioned by -# hand end up with the SDK in the same place (`~/.swiftpm/swift-sdks`). -# -# This step is deliberately NOT part of `all`: it is a 400 MB download and a -# 1.3 GB install, and a user with their own Xcode.xip is better served by -# `xtool sdk install `. The rootfs build runs it explicitly. -# --------------------------------------------------------------------------- -step_sdk() { - log "Installing the Darwin Swift SDK ($SDK_TAG)" - if ! command -v swift >/dev/null 2>&1; then - echo "the Swift toolchain is required first: sh $0 swift" >&2 - exit 1 - fi - - # An explicit replacement removes what is installed first — SwiftPM will not - # install a second bundle carrying the same artifact ID, and the bundled rootfs - # already carries one. The default is the idempotent path: a root that has an - # SDK keeps it. - if [ "${XFORGE_DARWIN_SDK_REPLACE:-0}" = "1" ]; then - step_sdk_remove - fi - - # Idempotence, answered by SwiftPM rather than by a marker file: an SDK that - # was installed and then removed must be reinstalled, and only SwiftPM knows. - sdk_listing="/tmp/xforge-sdk-list.$$" - if timeout 180 swift sdk list >"$sdk_listing" 2>&1 \ - && grep -qi 'darwin' "$sdk_listing"; then - log "a Darwin SDK is already installed: $(grep -i darwin "$sdk_listing" | head -1)" - rm -f "$sdk_listing" - return 0 - fi - rm -f "$sdk_listing" - - command -v unzip >/dev/null 2>&1 || { - echo "unzip is required to install the SDK (run: sh $0 deps)" >&2 - exit 1 - } - - mkdir -p "$SDK_CACHE" - archive="$SDK_CACHE/$SDK_ASSET" - # A caller that already has the archive (the rootfs build downloads it on the - # host, where it can be checked) stages it and points this at it, so the - # chroot never needs the network for the largest download in the chain. - if [ -n "${XFORGE_DARWIN_SDK_ARCHIVE:-}" ] && [ -f "$XFORGE_DARWIN_SDK_ARCHIVE" ]; then - log "Using the staged SDK archive at $XFORGE_DARWIN_SDK_ARCHIVE" - # The caller may well have staged it *at* this path — the rootfs build - # does, because that is where this step looks for its cache — and `cp a a` - # is an error ("are the same file"), which is how a build failed after - # every download and install in it had succeeded. - if [ "$XFORGE_DARWIN_SDK_ARCHIVE" != "$archive" ]; then - cp -f "$XFORGE_DARWIN_SDK_ARCHIVE" "$archive" - fi - else - log "Downloading $SDK_ASSET ($SDK_URL)" - rm -f "$archive.partial" - curl -fL --retry 3 --retry-delay 2 --no-progress-meter \ - -o "$archive.partial" "$SDK_URL" \ - || { echo "could not download $SDK_URL" >&2; exit 1; } - mv "$archive.partial" "$archive" - fi - [ -s "$archive" ] || { echo "the SDK archive is empty: $archive" >&2; exit 1; } - - # Recorded here, written into the root's manifest by the caller, and worth - # having: "which SDK is in this root" is otherwise unanswerable from a - # published artifact. - sdk_sha="$(sha256sum "$archive" | cut -d' ' -f1)" - if [ -n "${XFORGE_DARWIN_SDK_SHA256:-}" ] && [ "$XFORGE_DARWIN_SDK_SHA256" != "$sdk_sha" ]; then - echo "the SDK archive does not match XFORGE_DARWIN_SDK_SHA256" >&2 - printf ' expected %s\n got %s\n' "$XFORGE_DARWIN_SDK_SHA256" "$sdk_sha" >&2 - exit 1 - fi - - bundle="$SDK_CACHE/darwin.artifactbundle" - rm -rf "$bundle" "$SDK_CACHE/__MACOSX" - unzip -q "$archive" -d "$SDK_CACHE" - [ -f "$bundle/info.json" ] || { - echo "$archive did not unpack to $bundle/info.json" >&2 - exit 1 - } - - log "Installing the SDK with swift sdk install" - swift sdk install "$bundle" - - # 400 MB of archive, 1.3 GB of unpacked bundle and whatever `unzip` left - # beside them are no longer needed once SwiftPM has the SDK, and in a packaged - # rootfs they would be dead weight in the app bundle. (The zip carries a macOS - # `__MACOSX` sidecar, which unpacks into a directory that *contains* a - # `.artifactbundle` — that is what a build once recorded as the - # installed SDK, so it is removed here and excluded from the search below.) - rm -f "$archive" - rm -rf "$bundle" "$SDK_CACHE/__MACOSX" - - if ! swift sdk list 2>&1 | grep -qi darwin; then - echo "swift sdk install reported success but swift sdk list names no darwin SDK" >&2 - exit 1 - fi - - # Where SwiftPM put it, found rather than assumed. - # - # The documented location is `~/.swiftpm/swift-sdks`, and Swift 6.4's - # `swift sdk install` puts it elsewhere — this step originally asserted that - # path and failed a build whose install had succeeded. What is looked for is - # the bundle directory itself, with the store's own name as the fallback, and - # the answer is what gets recorded and verified. The download cache is - # excluded: the copy that was just installed *from* is a different bundle, and - # `__MACOSX` shadows it in a name search. - sdk_find() { - # The whole filesystem, pruned rather than bounded: SwiftPM's SDK store - # lives under the home directory in theory, and a `-maxdepth` guess at its - # depth searched every plausible place and found nothing — a search that - # reports "nowhere" for something `swift sdk list` can see is worse than a - # slow one. /proc, /sys, /dev and the download cache are the only places - # worth skipping, and -xdev keeps the virtual filesystems out anyway. - find / -xdev \ - \( -path /proc -o -path /sys -o -path /dev -o -path "$SDK_CACHE" \ - -o -name '__MACOSX' \) -prune -o \ - -type d -name "$1" -print 2>"$SILENT" | head -1 || true - } - sdk_install_dir="$(sdk_find '*.artifactbundle')" - [ -n "$sdk_install_dir" ] || sdk_install_dir="$(sdk_find 'swift-sdks')" - if [ -z "$sdk_install_dir" ]; then - echo "swift sdk list names a darwin SDK but no bundle is anywhere on this filesystem:" >&2 - echo " HOME=$HOME SWIFTLY_HOME_DIR=$SWIFTLY_HOME_DIR" >&2 - swift sdk list 2>&1 | sed 's/^/ /' >&2 - exit 1 - fi - - mkdir -p "$SHARE" - cat > "$SHARE/darwin-sdk.txt" </dev/null 2>&1; then - echo " $tool: not installed" - printf 'XFORGE-VERIFY\t%s\tmissing\tnot installed\n' "$tool" - failed=1 - continue - fi - # The exit status decides, not the output: a wrapper that reports "not - # installed" prints a line and exits 127, and counting that as success is - # how "installed" and "works" got confused the first time. - version_file="/tmp/xforge-version.$$" - if timeout 120 "$tool" --version >"$version_file" 2>&1; then - out="$(head -1 "$version_file")" - if [ -n "$out" ]; then - echo " $tool: ok — $out" - printf 'XFORGE-VERIFY\t%s\tok\t%s\n' "$tool" "$out" - else - echo " $tool: INSTALLED BUT SILENT (no version output)" - printf 'XFORGE-VERIFY\t%s\tbroken\tinstalled, but it prints nothing\n' "$tool" - failed=1 - fi - else - out="$(head -1 "$version_file")" - echo " $tool: INSTALLED BUT NOT RUNNING — ${out:-no output}" - printf 'XFORGE-VERIFY\t%s\tbroken\t%s\n' "$tool" "${out:-no output}" - failed=1 - fi - rm -f "$version_file" - done - # `swift sdk list` runs a *second* Swift binary — swift-sdk — with its own - # library needs, and it is what installs the darwin SDK. It fails differently - # from `swift --version` when the glibc layer is incomplete (a symbol lookup - # error naming a library rather than the tool), so it is checked on its own. - if command -v swift >/dev/null 2>&1; then - sdk_probe="/tmp/xforge-sdk-probe.$$" - if timeout 180 swift sdk list >"$sdk_probe" 2>&1; then - line="$(head -1 "$sdk_probe" 2>>"$SILENT" || true)" - echo " swift-sdk: ok${line:+ — $line}" - printf 'XFORGE-VERIFY\tswift-sdk\tok\t%s\n' "${line:-swift sdk list}" - else - line="$(head -1 "$sdk_probe" 2>>"$SILENT" || true)" - last="$(tail -1 "$sdk_probe" 2>>"$SILENT" || true)" - echo " swift-sdk: INSTALLED BUT NOT RUNNING — ${last:-${line:-no output}}" - echo " what the probe said:" - head -6 "$sdk_probe" 2>>"$SILENT" | sed 's/^/ /' - # A load failure names a library but not why, so ask the loader which - # libraries it picks and which it cannot find at all. "not found" - # here means a package is missing from the glibc layer. - if [ -f "$SHARE/glibc.env" ]; then - . "$SHARE/glibc.env" - sdk_bin="$(ls "$SWIFTLY_HOME_DIR"/toolchains/*/usr/bin/swift-sdk 2>>"$SILENT" | head -1 || true)" - if [ -n "$sdk_bin" ]; then - echo " loader resolution for $(basename "$sdk_bin"):" - "$XFORGE_GLIBC_LD" --list "$sdk_bin" 2>&1 | grep '=>' | sed 's/^/ /' - fi - fi - printf 'XFORGE-VERIFY\tswift-sdk\tbroken\t%s\n' "${last:-${line:-no output}}" - failed=1 - fi - rm -f "$sdk_probe" - fi - - if [ "$failed" -eq 0 ]; then - log "All tools are ready." - else - log "Finished. Tools marked 'not running' are installed in the rootfs but" - log "the guest's emulation faults on them — see the Engine log." - fi - return 0 -} - -case "${1:-all}" in - deps) step_deps ;; - glibc) step_glibc ;; - xtool) step_xtool ;; - swiftly) step_swiftly ;; - swift) step_swift ;; - sdk) step_sdk ;; - sdk-remove) step_sdk_remove ;; - verify) step_verify ;; - all) - step_deps - step_glibc - if [ "$INSTALL_XTOOL" = "0" ]; then - log "Skipping xtool (XFORGE_INSTALL_XTOOL=0; the app installs it on demand)" - else - step_xtool - fi - step_swiftly - step_swift - step_verify - # The build executor checks this stamp alongside the actual commands and - # package database. Bump its suffix whenever the required rootfs setup - # changes so existing guests are provisioned again exactly once. - mkdir -p "$SHARE" - touch "$SHARE/build-environment-v2" - ;; - *) - echo "usage: $0 [deps|glibc|xtool|swiftly|swift|sdk|sdk-remove|verify|all]" >&2 - exit 2 - ;; -esac diff --git a/EmbeddedLinux/install-zsign.sh b/EmbeddedLinux/install-zsign.sh deleted file mode 100644 index fd93eee..0000000 --- a/EmbeddedLinux/install-zsign.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -set -eu - -# Build zsign inside the embedded Alpine guest on first use. The upstream project -# is a C++ command-line signer (MIT), so it belongs in Linux where it can run, -# not as an iOS subprocess/framework. Pin the source revision so the installed -# signer is reproducible and cannot silently change as upstream master moves. -ZSIGN_COMMIT=614caa8d1ca949e260e5746144aa52d27a4b08d6 -ZSIGN_SHA256=a373dc5ddbf81ba5c435a48c14b4a2857506c3dde02ec62e0136b6968797384b -MARK=/usr/local/share/xforge/zsign-installed -BIN=/usr/local/bin/zsign -if [ -x "$BIN" ] && [ -f "$MARK" ]; then - exit 0 -fi -apk add --no-cache curl tar git make g++ pkgconf python3 openssl-dev >/tmp/xforge-zsign-apk.log 2>&1 -rm -rf /tmp/zsign-src -mkdir -p /tmp/zsign-src -cd /tmp/zsign-src -curl -fL --retry 3 "https://github.com/zhlynn/zsign/archive/$ZSIGN_COMMIT.tar.gz" -o zsign.tar.gz -printf '%s %s\n' "$ZSIGN_SHA256" zsign.tar.gz | sha256sum -c - -tar -xzf zsign.tar.gz --strip-components=1 -# Keep the PKCS#12 password out of zsign's argv/process listing. The upstream -# CLI accepts -p ; patch this pinned source with -Q , -# which reads the secret into memory and trims its newline. The job deletes the -# file immediately after signing. -python3 /root/xforge/patch-zsign-password-file.py -make -C build/linux clean all -install -m 0755 bin/zsign "$BIN" -mkdir -p "$(dirname "$MARK")" -printf 'source=https://github.com/zhlynn/zsign\ncommit=%s\ninstalled-at=%s\n' \ - "$ZSIGN_COMMIT" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$MARK" -rm -rf /tmp/zsign-src -zsign -v diff --git a/EmbeddedLinux/patch-zsign-password-file.py b/EmbeddedLinux/patch-zsign-password-file.py deleted file mode 100644 index da8bf5e..0000000 --- a/EmbeddedLinux/patch-zsign-password-file.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -"""Add a password-file option to the pinned zsign CLI. - -Upstream's -p takes the password as argv, visible in process listings while -signing. XForge adds -Q/--password_file: zsign reads it into its private string, -then the caller can unlink the file. This patch is applied only to the verified, -pinned upstream source tree during the one-time guest build. -""" -from pathlib import Path - -p = Path("src/zsign.cpp") -s = p.read_text() - -old_opt = '\t{"password", required_argument, NULL, \'p\'},' -new_opt = old_opt + '\n\t{"password_file", required_argument, NULL, \'Q\'},' -assert old_opt in s, "pinned zsign option table changed; review the patch" -s = s.replace(old_opt, new_opt, 1) - -old_flags = '"dfva2LhiqwCRSEWUPc:k:m:o:p:e:b:n:z:l:D:t:r:x:M:I:"' -new_flags = '"dfva2LhiqwCRSEWUPc:k:m:o:p:Q:e:b:n:z:l:D:t:r:x:M:I:"' -assert old_flags in s, "pinned zsign getopt string changed; review the patch" -s = s.replace(old_flags, new_flags, 1) - -old_case = "\t\tcase 'p':\n\t\t\tstrPassword = optarg;\n\t\t\tbreak;" -new_case = old_case + "\n\t\tcase 'Q':\n\t\t\tif (!ZFile::ReadFile(optarg, strPassword)) {\n\t\t\t\tZLog::ErrorV(\">>> Failed to read password file! %s\\n\", optarg);\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t\twhile (!strPassword.empty() && (strPassword.back() == '\\n' || strPassword.back() == '\\r')) strPassword.pop_back();\n\t\t\tbreak;" -assert old_case in s, "pinned zsign password switch changed; review the patch" -s = s.replace(old_case, new_case, 1) -p.write_text(s) - -# This pinned upstream snapshot's Linux Makefile exposes a separate header bug: -# json.h uses time_t but never includes the header that defines it. Some host -# toolchains happen to include it transitively; musl/Alpine's does not. Make the -# dependency explicit so the guest build is deterministic. -p = Path("src/common/json.h") -s = p.read_text() -include = "#include " -assert include in s, "pinned zsign JSON header layout changed; review the patch" -if "#include " not in s and "#include " not in s: - s = s.replace(include, "#include \n" + include, 1) -p.write_text(s) diff --git a/EmbeddedLinux/verify-rootfs.sh b/EmbeddedLinux/verify-rootfs.sh deleted file mode 100755 index 83a0b8f..0000000 --- a/EmbeddedLinux/verify-rootfs.sh +++ /dev/null @@ -1,522 +0,0 @@ -#!/usr/bin/env bash -# -# verify-rootfs.sh — check that a built Alpine root actually boots the console -# XForge expects, and fails loudly if it does not. -# -# The rootfs is the one thing in this repo the app's tests cannot cover: a root -# that is wrong looks exactly like a root that is right until `init` runs on a -# device, where the failure is a blank terminal. So the checks live here, and both -# callers use them: -# -# * EmbeddedLinux/build-rootfs.sh runs it on the configured *tree*, before the -# fakefs conversion — where the guest's own binaries can be executed; -# * .github/workflows/build-rootfs.yml runs it on the packed *ZIP*, so the -# artifact that is actually published is the thing that was checked. -# -# It accepts three shapes of input and says which checks apply to each: -# -# plain tree the configured root before conversion. Everything runs, -# including starting the console in a chroot of it. -# fakefs ZIP what gets published: `alpine-rootfs/data/` plus `meta.db`. The -# fakefs dir ZIP is read entry by entry (nothing is unpacked). -# -# A fakefs root is a directory tree *and* a database, and the engine reads the -# database: a file can sit in data/ and be invisible to the guest. Two consequences -# shape this script, and both cost a failed build to learn: -# -# * every path is also looked up in `paths`, and the mode that matters (is the -# console program executable?) is the one in `stats` — fakefs writes its data -# files 0666 and keeps the real mode in the database; -# * a fakefs root cannot be chrooted at all, so the console cannot be *run* -# from it. fakefs does not store symlinks as symlinks: an `AE_IFLNK` entry -# becomes a regular file whose contents are the link target, and the engine -# turns it back into a symlink from `meta.db` (see tools/fakefs.c). So -# `data/bin/sh` is a text file reading "/bin/busybox" and -# `data/lib/ld-musl-aarch64.so.1` is one reading "/lib/libc.musl-aarch64.so.1" -# — a chroot into that tree has no loader and no shell. The execution check -# therefore belongs to the tree, and the ZIP is checked through the same -# database the guest will use. -# -# Usage: -# EmbeddedLinux/verify-rootfs.sh -# -# A root built with XFORGE_PROVISION=all also carries the build toolchain (xtool, -# the Swift toolchain and the Darwin SDK), and its own manifest says so — that -# claim is checked here too, because a half-provisioned root looks exactly like a -# working one until someone tries to build with it. -# -set -euo pipefail - -TARGET="${1:-dist/rootfs/alpine-rootfs.zip}" -ROOTFS_NAME="alpine-rootfs" -MANIFEST="usr/local/share/xforge/rootfs-manifest.txt" - -log() { printf '\n==> %s\n' "$*"; } -note() { printf ' %s\n' "$*"; } -die() { printf '\nerror: %s\n' "$*" >&2; exit 1; } - -[ -e "$TARGET" ] || die "$TARGET does not exist" - -# --------------------------------------------------------------------------- -# How to read the root. -# -# Everything below asks only two questions — "is this path there?" and "what does -# it say?" — and answers them per shape, so the shape is settled once, here. -# --------------------------------------------------------------------------- -MODE="" -TREE="" -ZIP="" -BASE="" -ENTRIES="" -META_DB="" # what a path is relative to: "" for a tree, "/data/" in a ZIP - -if [ -d "$TARGET" ]; then - if [ -f "$TARGET/meta.db" ] && [ -d "$TARGET/data" ]; then - MODE="fakefs-dir" - TREE="$TARGET/data" - note "input: a fakefs root directory" - elif [ -d "$TARGET/data" ] && [ -e "$TARGET/data/etc/passwd" ]; then - MODE="tree" - TREE="$TARGET/data" - note "input: a plain root tree" - else - MODE="tree" - TREE="$TARGET" - note "input: a plain root tree" - fi -elif [ -f "$TARGET" ]; then - command -v unzip >/dev/null 2>&1 || die "unzip is required to read the ZIP" - MODE="zip" - ZIP="$TARGET" - BASE="$ROOTFS_NAME/data/" - note "input: a packed fakefs ZIP (read entry by entry, nothing unpacked)" -else - die "$TARGET is neither a directory nor a file" -fi - -# path_exists -path_exists() { - if [ "$MODE" = "zip" ]; then - grep -qx "$BASE$1" "$ENTRIES" - else - # `-e` OR `-L`, deliberately. A root tree is full of symlinks whose - # targets are absolute *guest* paths — `/usr/local/bin/swiftly` → - # `/root/.local/share/swiftly/bin/swiftly`, `/bin/sh` → `/bin/busybox` — - # and on the build machine those resolve against the build machine's `/`, - # so `-e` alone reports a symlink that is perfectly present as missing. - # This is the same trap as the glibc layer's `/lib` entry points: test the - # link itself, not where it points. - [ -e "$TREE/$1" ] || [ -L "$TREE/$1" ] - fi -} - -# path_contents -path_contents() { - if [ "$MODE" = "zip" ]; then - unzip -p "$ZIP" "$BASE$1" - else - cat "$TREE/$1" - fi -} - -# path_matches -# -# For the paths that cannot be named exactly — a Swift toolchain lives under a -# versioned directory that changes with every release. The listing is written to a -# file before it is searched on purpose: `find … | grep -q` closes the pipe at the -# first match, the producer dies of SIGPIPE, and under `set -o pipefail` the -# pipeline then reports failure for a check that passed. -path_matches() { - local listing rc - listing="$(mktemp)" - if [ "$MODE" = "zip" ]; then - grep -E "^$BASE$1" "$ENTRIES" > "$listing" || true - else - # Matching, not just listing: the first version of this wrote the listing - # out and then only asked whether it was non-empty, so in tree mode every - # pattern "matched" and a root missing its toolchain passed the check. The - # test that drives those failures is Tools/test-verify-toolchain.sh. - find "$TREE" -mindepth 1 -print 2>/dev/null | sed "s|^$TREE/||" \ - | grep -E "^$1" > "$listing" || true - fi - if [ -s "$listing" ]; then rc=0; else rc=1; fi - rm -f "$listing" - return "$rc" -} - -# path_link_target -# -# In a tree this is a symlink and `readlink` is the answer; in fakefs the target is -# the file's contents. (Reading it as a file in tree mode would dump the target's -# binary, which is why this is not just `path_contents`.) -path_link_target() { - if [ "$MODE" = "zip" ] || [ "$MODE" = "fakefs-dir" ]; then - path_contents "$1" - else - readlink "$TREE/$1" || true - fi -} - -if [ "$MODE" = "zip" ]; then - ENTRIES="$(mktemp)" # the ZIP's entry list, read once - META_DB="$(mktemp)" - trap 'rm -f "$ENTRIES" "$META_DB"' EXIT - unzip -Z1 "$ZIP" > "$ENTRIES" - for entry in "$ROOTFS_NAME/data/" "$ROOTFS_NAME/meta.db"; do - grep -qx "$entry" "$ENTRIES" || die "the ZIP has no $entry" - done - # SQLite's WAL/SHM sidecars are transient and must not be captured: a stale - # `-wal` next to a fresh meta.db is a database that looks corrupt to the guest. - ! grep -qE '\.db-(wal|shm)$' "$ENTRIES" \ - || die "the ZIP contains SQLite WAL/SHM sidecars" - unzip -p "$ZIP" "$ROOTFS_NAME/meta.db" > "$META_DB" - note "entries: $(wc -l < "$ENTRIES" | tr -d ' ')" -elif [ "$MODE" = "fakefs-dir" ]; then - META_DB="$TARGET/meta.db" -fi - -# --------------------------------------------------------------------------- -# 1. What init needs -# --------------------------------------------------------------------------- -log "Files" -for path in \ - sbin/init \ - bin/sh \ - sbin/xforge-login \ - etc/inittab \ - etc/init.d/rcS \ - etc/passwd \ - etc/shadow \ - etc/profile \ - etc/apk/repositories \ - "$MANIFEST" \ - root/install-toolchain.sh ; do - path_exists "$path" || die "the root has no /$path" -done -note "11 required paths present" - -if [ "$MODE" = "tree" ]; then - # In a tree the modes are the files' own. In a fakefs root they are not (see - # the header): that is what the meta.db section below checks. - [ -x "$TREE/sbin/xforge-login" ] || die "/sbin/xforge-login is not executable" - [ -x "$TREE/bin/sh" ] || die "/bin/sh is not executable" - note "/sbin/xforge-login and /bin/sh are executable" -fi - -# /bin/sh has to lead somewhere that exists, or nothing can start at all. It is a -# symlink to busybox in every Alpine root, and fakefs will have stored the target -# as text — either way, the target has to be in the root. -SH_TARGET="$(path_link_target bin/sh)" -case "$SH_TARGET" in - /*) path_exists "${SH_TARGET#/}" \ - || die "/bin/sh points at $SH_TARGET, which is not in the root" ;; - *) die "/bin/sh is not where it should be: expected a symlink, got '${SH_TARGET:0:40}'" ;; -esac -note "/bin/sh → $SH_TARGET" - -# The login shell the root promises, from the root's own manifest — not from this -# script, so a root built with a different XFORGE_DEFAULT_SHELL is checked against -# what it was built to be. -EXPECTED_SHELL="$(path_contents "$MANIFEST" | awk '/^[[:space:]]*shell:/ { print $2 }')" -[ -n "$EXPECTED_SHELL" ] || die "the manifest does not name a login shell" -path_exists "${EXPECTED_SHELL#/}" \ - || die "the manifest promises $EXPECTED_SHELL, which is not in the root" -note "login shell in the manifest: $EXPECTED_SHELL" - -# The session's terminal has to be one curses knows about. /etc/profile is where -# the root names it, and an unknown TERM is an error nowhere — it just makes `less`, -# `top` and every full-screen program behave as if there were no terminal, which -# reads as a broken console rather than as a missing package. -TERM_VALUE="$(path_contents etc/profile | sed -n 's/^export TERM=//p' | tail -1 | tr -d "'\"")" -if [ -n "$TERM_VALUE" ]; then - ENTRY="${TERM_VALUE%${TERM_VALUE#?}}/$TERM_VALUE" - FOUND_TERMINFO="" - for dir in usr/share/terminfo etc/terminfo; do - if path_exists "$dir/$ENTRY"; then FOUND_TERMINFO="$dir/$ENTRY"; fi - done - if [ -z "$FOUND_TERMINFO" ]; then - die "TERM=$TERM_VALUE is exported but the root has no terminfo entry for it - (looked for $ENTRY). Every curses program in the guest would run against an - unknown terminal." - fi - note "TERM=$TERM_VALUE → $FOUND_TERMINFO" -fi - -# --------------------------------------------------------------------------- -# 2. The console -# -# One session, on the one terminal that exists, started by the guest's own program. -# The stock Alpine inittab fails all three: it starts openrc (not in this root) and -# respawns six gettys on terminals the engine does not provide. -# --------------------------------------------------------------------------- -log "inittab" -INITTAB="$(path_contents etc/inittab)" -grep_q() { printf '%s\n' "$INITTAB" | grep -q "$@"; } -grep_q -x '::sysinit:/etc/init.d/rcS' \ - || die "inittab does not run the sysinit step" -grep_q -x 'tty1::respawn:/sbin/xforge-login root' \ - || die "inittab does not respawn /sbin/xforge-login root on tty1" -if grep_q 'openrc'; then - die "inittab starts openrc, which this root does not contain" -fi -if grep_q -E '^tty[2-6]::'; then - die "inittab respawns gettys on terminals this guest does not have" -fi -note "pid 1 is /sbin/init; tty1 respawns /sbin/xforge-login root" - -# --------------------------------------------------------------------------- -# 3. Root's shell, in the file the console program reads -# --------------------------------------------------------------------------- -log "Root's shell" -PASSWD_ROOT="$(path_contents etc/passwd | awk -F: '$1 == "root" { print; exit }')" -[ -n "$PASSWD_ROOT" ] || die "there is no root in /etc/passwd" -PASSWD_SHELL="$(printf '%s\n' "$PASSWD_ROOT" | cut -d: -f7)" -[ "$PASSWD_SHELL" = "$EXPECTED_SHELL" ] || die \ - "root's shell in /etc/passwd is $PASSWD_SHELL, not the $EXPECTED_SHELL the manifest promises" -note "$PASSWD_ROOT" - -# --------------------------------------------------------------------------- -# 4. What the engine will resolve -# -# The tree is only half of a fakefs root; the other half is the database, and the -# paths above existing in data/ is not the same as the guest being able to see -# them. A root once shipped with 568 glibc files on disk and zero rows in `paths` -# for them, which the guest reads as "not there at all" — so everything the console -# needs is looked up here, along with the mode it will have. -# --------------------------------------------------------------------------- -if [ -n "$META_DB" ]; then - log "meta.db" - command -v sqlite3 >/dev/null 2>&1 || die "sqlite3 is required to check a fakefs root" - - # `CAST(path AS TEXT)` is not decoration: fakefs stores paths as BLOB, and - # comparing a BLOB against a string literal is version-dependent — the same - # query that matched every row locally matched zero rows on an older SQLite, so - # a correct root failed its own check. - # fakefs stores paths in its normalized form, `( '/' path-component )*`: - # "/bin/sh", not "bin/sh" and not "/bin/sh/" (see `path_normalize` in - # tools/fakefs.c). The lookup is on that form, so this takes a root-relative - # path and looks up its absolute spelling. - indexed() { - sqlite3 "$META_DB" \ - "SELECT COUNT(*) FROM paths WHERE CAST(path AS TEXT) = '/$1';" - } - for path in \ - bin/sh \ - sbin/xforge-login \ - etc/inittab \ - etc/passwd \ - etc/profile \ - "${EXPECTED_SHELL#/}" ; do - [ "$(indexed "$path")" -gt 0 ] || die "/$path is in the tree but not indexed - in meta.db — the guest would not see it. Anything the root needs has to be in - place *before* the fakefs conversion." - done - note "6 console paths indexed in meta.db" - - # The mode the engine will apply, which is the one fakefs kept (its data files - # are all written 0666). `stats.stat` is a binary `struct ish_stat` — mode, uid, - # gid, rdev as four little-endian uint32s — so the mode is the first two bytes of - # hex(), low byte first. - mode_of() { - local hex lo hi - hex="$(sqlite3 "$META_DB" "SELECT hex(substr(stats.stat, 1, 2)) FROM paths - JOIN stats ON paths.inode = stats.inode - WHERE CAST(paths.path AS TEXT) = '/$1';")" - [ -n "$hex" ] || return 1 - lo="${hex%??}" - hi="${hex#??}" - printf '%s' "$(( 0x${hi}${lo} ))" - } - for path in sbin/xforge-login "${EXPECTED_SHELL#/}"; do - mode="$(mode_of "$path")" || die "/$path has no stats row in meta.db" - # 8#111 is 0o111: any execute bit. - [ "$(( mode & 8#111 ))" -ne 0 ] || die "/$path is not executable in - meta.db (mode $(printf '%o' "$mode")) — the console would not start." - done - note "/sbin/xforge-login and $EXPECTED_SHELL are executable in meta.db" - - total="$(sqlite3 "$META_DB" 'SELECT COUNT(*) FROM paths;')" - note "meta.db: $total paths" -fi - -# --------------------------------------------------------------------------- -# 4b. The provisioned toolchain, when the manifest says there is one -# -# A root built with XFORGE_PROVISION=all carries xtool, the Swift toolchain and -# the Darwin SDK, and its manifest says so. That claim is checked here rather than -# trusted, for the same reason as everything else in this script: the failure mode -# is quiet. A root whose toolchain is half-installed boots perfectly, gives a -# working console, reports the toolchain as "installed" in the app, and fails on -# the user's first build — which is the most expensive place to find out. -# -# What cannot be done here is *run* the tools: a fakefs root cannot be chrooted -# (its symlinks are files, and only the engine resolves them), and the tree the -# verification ran on during the build no longer exists by then. So the checks are -# structural — the paths exist, and for a fakefs root the engine can see them — -# and the "it actually compiles" check happened on the tree, in the build, where -# install-toolchain.sh verify could run a real compile. -# --------------------------------------------------------------------------- -TOOLCHAIN_CLAIM="$(path_contents "$MANIFEST" | awk '/^[[:space:]]*toolchain:/ { sub(/^[[:space:]]*toolchain:[[:space:]]*/, ""); print; exit }')" -case "$TOOLCHAIN_CLAIM" in - ""|"not provisioned"*) - note "no toolchain in this root (the guest installs it on demand)" ;; - *) - log "Toolchain: $TOOLCHAIN_CLAIM" - - for path in \ - usr/local/bin/xtool \ - usr/local/bin/swift \ - usr/local/bin/swiftc \ - usr/local/bin/swiftly \ - opt/xtool/usr/bin/xtool \ - usr/local/share/xforge/glibc.env ; do - path_exists "$path" \ - || die "the manifest promises a toolchain but /$path is missing" - done - note "xtool, swift, swiftly and the glibc layer are present" - - # The toolchain itself lives in a versioned directory - # (swiftly/toolchains//usr/bin/...), so it is looked for by shape - # rather than by a name that changes with every Swift release. - path_matches 'root/\.local/share/swiftly/toolchains/[^/]+/usr/bin/swift-frontend' \ - || die "there is no Swift toolchain under /root/.local/share/swiftly/toolchains" - # Its stdlib, wherever this release keeps it: `linux`, `linux-musl`, or a - # per-architecture directory under either, which is why the pattern has a - # `.*` where the first version of this check named one exact path and - # failed a perfectly good toolchain. The interface files (.swiftmodule) are - # the half a too-thorough slimming would take, and the shared library is - # the half that would still look present. - path_matches 'root/\.local/share/swiftly/toolchains/[^/]+/usr/lib/swift/.*/libswiftCore\.so' \ - || die "the Swift toolchain has no Linux stdlib under - /root/.local/share/swiftly/toolchains/*/usr/lib/swift" - path_matches 'root/\.local/share/swiftly/toolchains/[^/]+/usr/lib/swift/.*/Swift\.swiftmodule' \ - || die "the Swift toolchain has no Linux stdlib interface under - /root/.local/share/swiftly/toolchains/*/usr/lib/swift — without it nothing - can be compiled against the stdlib" - note "a Swift toolchain with its stdlib is installed" - - if path_exists "usr/local/share/xforge/darwin-sdk.txt"; then - SDK_PATH="$(path_contents "usr/local/share/xforge/darwin-sdk.txt" \ - | awk '/^[[:space:]]*path:/ { print $2; exit }')" - SDK_TAG_LINE="$(path_contents "usr/local/share/xforge/darwin-sdk.txt" \ - | awk '/^[[:space:]]*tag:/ { print $2; exit }')" - [ -n "$SDK_PATH" ] \ - || die "/usr/local/share/xforge/darwin-sdk.txt does not name where the SDK went" - # The record names the *directory* SwiftPM installed the bundle into, - # and a ZIP listing writes directories with a trailing slash — an - # exact-line test for the bare name finds nothing and fails a root - # that has the SDK. Ask whether anything is under it instead, which is - # the question that matters either way. - sdk_rel="${SDK_PATH#/}" - if [ "$MODE" = "zip" ]; then - sdk_rel_pattern="$(echo "$sdk_rel" | sed 's/\./\\./g')" - grep -qE "^$BASE$sdk_rel_pattern/" "$ENTRIES" \ - || die "the Darwin SDK records itself at $SDK_PATH, which is not in this root - — the guest's first build would fail on a missing SDK" - else - [ -d "$TREE/$sdk_rel" ] \ - || die "the Darwin SDK records itself at $SDK_PATH, which is not in this root - — the guest's first build would fail on a missing SDK" - fi - note "Darwin SDK $SDK_TAG_LINE at $SDK_PATH" - - # And the engine has to be able to see it: the whole reason the - # toolchain is installed before the conversion is that a file in data/ - # with no row in `paths` does not exist as far as the guest is concerned. - if [ -n "$META_DB" ]; then - sdk_indexed="$(sqlite3 "$META_DB" \ - "SELECT COUNT(*) FROM paths WHERE CAST(path AS TEXT) LIKE '${SDK_PATH%/}/%';")" - [ "${sdk_indexed:-0}" -gt 0 ] \ - || die "the Darwin SDK is on disk but not indexed in meta.db - ($sdk_indexed paths under $SDK_PATH) — the guest would not see it. It has to - be installed before the fakefs conversion." - note "Darwin SDK paths indexed in meta.db: $sdk_indexed" - fi - else - note "no Darwin SDK in this root (XFORGE_PROVISION_SDK=0)" - fi - ;; -esac - -# --------------------------------------------------------------------------- -# 5. Run it — a tree only, and say plainly why it cannot be done otherwise -# --------------------------------------------------------------------------- -log "The console path" -if [ "$MODE" != "tree" ]; then - note "not run here: a fakefs root cannot be chrooted (its symlinks are files" - note "whose contents are the link target, and only the engine resolves them)," - note "so this root's console was run for real on the tree by build-rootfs.sh," - note "before the conversion. The checks above are the database the guest reads." - log "Done" - note "$TARGET checks out (console run: on the tree, before conversion)" - exit 0 -fi - -# An explicit skip is answered before the requirements of the thing being skipped: -# "set XFORGE_VERIFY_SKIP_CONSOLE=1" is the advice this script gives a caller who -# cannot chroot (not root, or not on aarch64), and it has to work for them — -# otherwise the advice is a dead end and a check that was meant to be skipped -# fails the caller anyway. -if [ "${XFORGE_VERIFY_SKIP_CONSOLE:-0}" = "1" ]; then - note "the console is not run (XFORGE_VERIFY_SKIP_CONSOLE=1): the file checks above are all" - note "that was asked for" - log "Done" - note "$TARGET checks out" - exit 0 -fi - -[ "$(id -u)" -eq 0 ] || die "checking the console needs root (it chroots into the - root). Re-run with sudo, or set XFORGE_VERIFY_SKIP_CONSOLE=1." -case "$(uname -m)" in - aarch64|arm64) ;; - *) die "this host is $(uname -m) and the root is aarch64, so its binaries cannot - run here. Run this on an aarch64 machine, or set XFORGE_VERIFY_SKIP_CONSOLE=1 - to check only the files." ;; -esac - -# A working /dev/null is not optional: every shell opens it, and a chroot without -# /dev is a chroot where commands fail for reasons that have nothing to do with what -# is being checked. -BOUND_DEV="" -MOUNTED_PROC="" -if ! grep -qs " $TREE/dev " /proc/mounts; then - mount --bind /dev "$TREE/dev" 2>/dev/null && BOUND_DEV="$TREE/dev" -fi -if ! grep -qs " $TREE/proc " /proc/mounts; then - mount -t proc none "$TREE/proc" 2>/dev/null && MOUNTED_PROC="$TREE/proc" -fi -cleanup_mounts() { - [ -n "$MOUNTED_PROC" ] && umount -l "$MOUNTED_PROC" 2>/dev/null || true - [ -n "$BOUND_DEV" ] && umount -l "$BOUND_DEV" 2>/dev/null || true - return 0 -} -trap 'cleanup_mounts' EXIT - -# Fed from a pipe, not a terminal: the shell is non-interactive and exits, so the -# session ends on its own. What is being read is the part that can silently be -# wrong — which shell runs, as a login shell (the leading dash in $0), in which -# directory, and with which environment. -CONSOLE_OUTPUT="$(printf 'echo XFORGE-CONSOLE-OK; echo "0=$0"; echo "SHELL=$SHELL"; pwd\n' \ - | chroot "$TREE" /sbin/xforge-login root 2>&1)" || true -printf '%s\n' "$CONSOLE_OUTPUT" | sed 's/^/ /' - -case "$CONSOLE_OUTPUT" in - *XFORGE-CONSOLE-OK*) ;; - *) die "/sbin/xforge-login did not start a shell — the console would be blank" ;; -esac -case "$CONSOLE_OUTPUT" in - *"0=-${EXPECTED_SHELL##*/}"*) ;; - *) die "/sbin/xforge-login did not start $EXPECTED_SHELL as a login shell" ;; -esac -case "$CONSOLE_OUTPUT" in - *"SHELL=$EXPECTED_SHELL"*) ;; - *) die "the console session's SHELL is not $EXPECTED_SHELL" ;; -esac -case "$CONSOLE_OUTPUT" in - *"not executable"*) die "/sbin/xforge-login fell back instead of running $EXPECTED_SHELL" ;; -esac -note "ok: /sbin/xforge-login root → $EXPECTED_SHELL, as a login shell" -cleanup_mounts - -log "Done" -note "$TARGET checks out" diff --git a/EmbeddedLinux/xforge-login b/EmbeddedLinux/xforge-login deleted file mode 100755 index 5e304c1..0000000 --- a/EmbeddedLinux/xforge-login +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/sh -# -# xforge-login — the program XForge's console runs, one instance per session. -# -# This file is not a build script: it is *installed into the guest* by -# EmbeddedLinux/build-rootfs.sh, at /sbin/xforge-login, and run from /etc/inittab -# (which respawns it on tty1, the guest's only console and the terminal the app -# shows). It lives here, next to the builder that installs it, because it is the -# one guest file with logic in it worth reading on its own. -# -# It used to be `/bin/login -f root`, and not needing login is the point: -# -# * login authenticates, allocates a utmp slot and takes over a controlling -# terminal. None of that applies to this guest: it has one user, whose -# password is locked (`root:*`), and init has already given the session the -# console tty. -# * what login did *not* do is what the guest actually wants — start the shell -# configured for the user. That shell is a property of /etc/passwd, so this -# script reads it: `chsh -s /bin/zsh root` (or editing the field) changes what -# the console starts, and the console still starts *something* if that shell -# was uninstalled underneath it. -# -# The session is a login session: the shell is exec'd with a leading dash in -# argv[0] — the POSIX convention that makes a shell read /etc/profile — with the -# user's home as the working directory and the environment `login` would have set. -# -# Usage: xforge-login [user] (default: root) -# -# The leading dash is done with `exec -a`, which is busybox ash's (this script is -# /bin/sh, and /bin/sh is busybox ash here). Passing `-l`/`--login` instead would -# mean knowing which flag each shell spells it with. -# -set -u - -user="${1:-root}" -fallback=/bin/sh - -# field : one colon-separated field of /etc/passwd, or nothing. -field() { - awk -F: -v u="$user" -v n="$1" '$1 == u { print $n; exit }' /etc/passwd 2>/dev/null -} - -wanted="$(field 7)" -home="$(field 6)" -[ -n "$home" ] || home=/ -[ -n "$wanted" ] || wanted="$fallback" - -# The configured shell may have been removed since it was set. Say so and run the -# fallback: init respawns this forever, and a console that dies silently on every -# respawn leaves a blank screen with nothing to read. -if [ ! -x "$wanted" ]; then - printf 'xforge-login: %s is not executable, falling back to %s\n' "$wanted" "$fallback" >&2 - wanted="$fallback" -fi - -# What `login` would export. TERM is the terminal the app emulates; /etc/profile -# sets it too, for shells that read it. -TERM="${TERM:-xterm-256color}" -HOME="$home" -SHELL="$wanted" -USER="$user" -LOGNAME="$user" -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/bin -export TERM HOME SHELL USER LOGNAME PATH - -cd "$home" 2>/dev/null || cd / - -# The motd `login` used to print, and only when there is a terminal to print it to. -if [ -t 0 ] && [ -s /etc/motd ]; then - cat /etc/motd -fi - -name="${wanted##*/}" -exec -a "-$name" "$wanted" diff --git a/Makefile b/Makefile index 9c86aa7..43058e5 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,29 @@ -.PHONY: bootstrap submodule rootfs rootfs-publish ish-core icon gen build test ipa init clean +.PHONY: toolchain toolchain-release icon gen build test ipa clean XCODE := xcodebuild SCHEME := XForge -## Everything needed to build locally, in order. -bootstrap: submodule rootfs ish-core - -## Pull the ish-arm64 engine sources (git submodule). -submodule: - git submodule update --init --depth 1 Vendor/ish-arm64 - git -C Vendor/ish-arm64 submodule update --init --depth 1 deps/libarchive - -## Build the bundled Alpine fakefs rootfs into Support/Resources. -## Needs root (it chroots into the tree) and room for the toolchain: the default -## is the *provisioned* root — xtool, Swift and the Darwin SDK installed, ~1.6 GB. -## XFORGE_PROVISION=none builds the few-MB plain one instead, where the guest -## installs the toolchain itself. -## build-ipa.yml downloads the published copy rather than rebuilding it. -rootfs: - @bash EmbeddedLinux/build-rootfs.sh Support/Resources - -## Build the rootfs and publish it as a pinned release asset for CI. -## Usage: make rootfs-publish TAG=rootfs-v5 [ALPINE=3.21] [PROVISION=none] -rootfs-publish: - @test -n "$(TAG)" || { echo "usage: make rootfs-publish TAG=rootfs-v5"; exit 1; } - @command -v gh >/dev/null 2>&1 || { \ - echo "GitHub CLI is required: https://cli.github.com/"; exit 1; \ - } - gh workflow run build-rootfs.yml -f tag=$(TAG) -f alpine_version=$(or $(ALPINE),3.21) \ - -f provision=$(or $(PROVISION),all) - @echo "Dispatched build-rootfs.yml for $(TAG); then set ROOTFS_TAG in build-ipa.yml." - -## Build the embedded ish-arm64 Linux engine into Vendor/ish-arm64-build/lib. -ish-core: - @bash EmbeddedLinux/build-ish-core.sh +## Install the native toolchain bundle into Vendor/NativeToolchain. +## +## The bundle is built by the `Native iOS Toolchain` workflow and published as a +## release asset whose tag names its contents, so the newest one is a well-defined +## thing to ask for. Without a bundle the app still builds and runs: the C bridge +## compiles to a "not available" stub and the Toolchain screen says so. +## +## make toolchain-release # newest published bundle +## make toolchain-release TAG=... # a specific one +## make toolchain ARCHIVE=... # a local tarball +toolchain: + @if [ -z "$(ARCHIVE)" ]; then \ + echo "usage: make toolchain ARCHIVE=XForgeNativeToolchain-arm64-ios.tar.gz"; \ + echo " make toolchain-release [TAG=...]"; \ + exit 64; \ + fi + @bash NativeToolchain/install-bundle.sh "$(ARCHIVE)" + +## Install the newest published toolchain bundle (see `toolchain` above) +toolchain-release: + @bash NativeToolchain/install-bundle.sh --release $(TAG) ## Regenerate the app icon + accent colour asset catalog (needs Pillow) icon: @@ -61,13 +51,5 @@ ipa: gh workflow run build-ipa.yml @echo "Dispatched build-ipa.yml; monitor it with: gh run watch" -## Install Swift + xtool into the guest, on the device. The published rootfs -## already carries them, so this is a no-op there; it is what a plain root -## (XFORGE_PROVISION=none) needs. Run it inside the XForge terminal, not the host: -## sh /root/install-toolchain.sh all -init: - @echo "Run this in the XForge terminal (embedded Linux), not on the host:" - @echo " sh /root/install-toolchain.sh all" - clean: - rm -rf build dist XForge.xcodeproj Vendor/ish-arm64-build .rootfs-work + rm -rf build dist XForge.xcodeproj diff --git a/NativeToolchain/install-bundle.sh b/NativeToolchain/install-bundle.sh index 3f32fb1..842e904 100644 --- a/NativeToolchain/install-bundle.sh +++ b/NativeToolchain/install-bundle.sh @@ -1,22 +1,136 @@ #!/usr/bin/env bash set -euo pipefail -if [ "$#" -ne 1 ]; then - echo "usage: $0 XForgeNativeToolchain-arm64-ios.tar.gz" >&2 - exit 64 -fi +# usage: install-bundle.sh +# install-bundle.sh --release [tag] +# install-bundle.sh --release artifact # newest workflow artifact +# +# The bundle is built by the `Native iOS Toolchain` workflow, which publishes it as +# a release asset whose tag names its contents (`toolchain--swift--ios…`). +# `--release` fetches that asset, so a machine that has never run the workflow — or +# came to it after the workflow artifact expired — can still install the compiler. +# With no tag, the newest published `toolchain-*` release is installed. + +REPO="${XFORGE_REPO:-R0GUEEE/XForge}" +ASSET="XForgeNativeToolchain-arm64-ios.tar.gz" +ARTIFACT_NAME="XForgeNativeToolchain-arm64-ios" ROOT="$(cd "$(dirname "$0")/.." && pwd)" -ARCHIVE="$1" DEST="$ROOT/Vendor/NativeToolchain" +TMP="" +# `if` rather than `[ … ] && …`: a false test would leave the trap returning +# non-zero, which is not something to have running at exit. +cleanup() { if [ -n "$TMP" ]; then rm -rf "$TMP"; fi; } +trap cleanup EXIT + +usage() { + cat >&2 < + $0 --release [tag] + $0 --release artifact + + --release [tag] fetch a published toolchain release asset (default: newest) + artifact the newest workflow artifact instead (needs the GitHub CLI) +EOF +} + +have_gh() { command -v gh >/dev/null 2>&1; } + +# The newest `toolchain-*` release. `gh release list` is newest-first; without the +# CLI this asks the API directly, which is rate-limited for anonymous callers but +# fine for the one-off case this is for. +api() { + # A token is only used when one happens to be in the environment: it is what makes + # this work against a private fork, and it raises the API's rate limit otherwise. + local token="${GH_TOKEN:-${GITHUB_TOKEN:-}}" + if [ -n "$token" ]; then + curl -fsSL -H "Authorization: Bearer $token" "$1" + else + curl -fsSL "$1" + fi +} + +newest_tag() { + if have_gh; then + gh release list --repo "$REPO" --limit 100 --json tagName \ + --jq '[.[] | select(.tagName | startswith("toolchain-"))][0].tagName' + return + fi + command -v python3 >/dev/null 2>&1 || { + echo "error: need either the GitHub CLI or python3 to find the newest release" >&2 + exit 69 + } + api "https://api.github.com/repos/$REPO/releases?per_page=100" \ + | python3 -c 'import json,sys +try: + releases = json.load(sys.stdin) +except ValueError: + releases = [] +if not isinstance(releases, list): + releases = [] +print(next((r["tag_name"] for r in releases + if isinstance(r, dict) and r.get("tag_name", "").startswith("toolchain-")), ""))' +} + +case "${1:-}" in + "") + usage + exit 64 + ;; + -h|--help) + usage + exit 0 + ;; + --release) + TAG="${2:-}" + TMP="$(mktemp -d)" + + if [ "$TAG" = "artifact" ]; then + have_gh || { echo "error: --release artifact needs the GitHub CLI" >&2; exit 69; } + gh run download --repo "$REPO" --name "$ARTIFACT_NAME" --dir "$TMP" + else + if [ -z "$TAG" ]; then + TAG="$(newest_tag)" + fi + [ -n "$TAG" ] && [ "$TAG" != "null" ] || { + echo "error: no published toolchain release in $REPO" >&2 + echo " run the 'Native iOS Toolchain' workflow by hand, with_swift=true" >&2 + exit 66 + } + echo "installing toolchain bundle from release $TAG" + if have_gh; then + gh release download "$TAG" --repo "$REPO" \ + --pattern "XForgeNativeToolchain-*.tar.gz" --dir "$TMP" || { + echo "error: no $ASSET asset in release $TAG of $REPO" >&2 + exit 66 + } + else + curl -fSL --retry 3 -o "$TMP/$ASSET" \ + "https://github.com/$REPO/releases/download/$TAG/$ASSET" || { + echo "error: no $ASSET asset in release $TAG of $REPO" >&2 + exit 66 + } + fi + fi + ARCHIVE="$TMP/$ASSET" + ;; + -*) + usage + exit 64 + ;; + *) + ARCHIVE="$1" + ;; +esac + [ -f "$ARCHIVE" ] || { echo "error: archive not found: $ARCHIVE" >&2; exit 66; } rm -rf "$DEST" mkdir -p "$DEST" tar -xzf "$ARCHIVE" -C "$DEST" -for required in manifest.txt include lib; do +for required in manifest.txt include include-generated lib; do [ -e "$DEST/$required" ] || { echo "error: native toolchain bundle is missing $required" >&2 rm -rf "$DEST" diff --git a/NativeToolchain/prepare-xcode.sh b/NativeToolchain/prepare-xcode.sh index 004d8c1..f235f97 100644 --- a/NativeToolchain/prepare-xcode.sh +++ b/NativeToolchain/prepare-xcode.sh @@ -1,13 +1,23 @@ #!/usr/bin/env bash set -euo pipefail +# Turns Vendor/NativeToolchain — the bundle the `Native iOS Toolchain` workflow +# uploads — into the xcconfig the app target is built with. Nothing here runs on +# iOS: this is a build-time step, and what it really decides is whether the C +# bridge compiles the compiler in or the "not available" stub. +# +# A missing bundle is not an error. A clone without one still builds, and the +# Toolchain screen says the compiler is not linked; that is a better default than +# refusing to build. + ROOT="$(cd "$(dirname "$0")/.." && pwd)" BUNDLE="$ROOT/Vendor/NativeToolchain" OUT="$ROOT/Support/NativeToolchain.generated.xcconfig" +MANIFEST="$BUNDLE/manifest.txt" mkdir -p "$(dirname "$OUT")" -if [ ! -f "$BUNDLE/manifest.txt" ] || [ ! -d "$BUNDLE/include" ] || [ ! -d "$BUNDLE/lib" ]; then +if [ ! -f "$MANIFEST" ] || [ ! -d "$BUNDLE/include" ] || [ ! -d "$BUNDLE/include-generated" ] || [ ! -d "$BUNDLE/lib" ]; then cat >"$OUT" <"$ARCHIVE_LIST" +# Only `$(SRCROOT)` is escaped: that one is meant for Xcode to expand. The +# parameter expansion is not — it has to happen here, in this shell. Escaping it +# too (`\${lib#…}`) put the literal text into the xcconfig, where Xcode read it as +# a variable named `lib#/Users/… `, resolved it to nothing, and handed the linker +# `$(SRCROOT)/` as a file name: `ld: file cannot be mmap()ed, errno=22 +# path=/Users/runner/work/XForge/XForge/`. Every build with the compiler linked in +# failed like that, and the guard at the end of this script now catches it here. +while IFS= read -r lib; do + [ -n "$lib" ] || continue + ARCHIVES+=("\$(SRCROOT)/${lib#"$ROOT/"}") +done <"$ARCHIVE_LIST" -if [ "\${#ARCHIVES[@]}" -eq 0 ]; then +if [ "${#ARCHIVES[@]}" -eq 0 ]; then echo "error: Native toolchain bundle has no static libraries in $BUNDLE/lib" >&2 exit 1 fi +# The bundle declares what it contains, and the bridge needs the matching defines: +# XFORGE_HAS_LLVM gates the Clang/LLD entry points, XFORGE_HAS_SWIFT_FRONTEND the +# one that calls swift::performFrontend. Without the second, a bundle that carries +# the Swift libraries still reports "swift-frontend: missing". +CFLAGS="-DXFORGE_HAS_LLVM=1" +if grep -q '^swift_frontend=1' "$MANIFEST" 2>/dev/null; then + CFLAGS="$CFLAGS -DXFORGE_HAS_SWIFT_FRONTEND=1" + SWIFT_REF="$(sed -n 's/^swift_ref=//p' "$MANIFEST" 2>/dev/null)" + echo "Native toolchain: Swift frontend included (${SWIFT_REF:-unknown revision})" +else + echo "Native toolchain: Clang and LLD only (no Swift frontend in this bundle)" +fi + { echo "XFORGE_NATIVE_TOOLCHAIN_AVAILABLE = 1" - echo "XFORGE_NATIVE_HEADER_SEARCH_PATHS = \$(SRCROOT)/Vendor/NativeToolchain/include" + # Two roots, generated first: `swift/bridging` is a generated *file* while the + # sources have a *directory* of the same name, so they cannot share a root (see + # the staging step in .github/workflows/native-toolchain.yml). + echo "XFORGE_NATIVE_HEADER_SEARCH_PATHS = \$(SRCROOT)/Vendor/NativeToolchain/include-generated \$(SRCROOT)/Vendor/NativeToolchain/include" echo "XFORGE_NATIVE_LIBRARY_SEARCH_PATHS = \$(SRCROOT)/Vendor/NativeToolchain/lib" - echo "XFORGE_NATIVE_CFLAGS = -DXFORGE_HAS_LLVM=1" + echo "XFORGE_NATIVE_CFLAGS = $CFLAGS" printf "XFORGE_NATIVE_LDFLAGS =" - for archive in "\${ARCHIVES[@]}"; do + for archive in "${ARCHIVES[@]}"; do printf " %s" "$archive" done echo " -lc++ -lz -liconv -lsqlite3 -framework Foundation" } >"$OUT" -echo "Native toolchain: enabled (\${#ARCHIVES[@]} archives)" +# A `$(…)` is for Xcode; a `${…}` is this script failing to expand something, and +# Xcode will resolve it to the empty string. That is worth failing for: it took a +# full archive of a 20-minute app build to find the last one. Written as `case` +# rather than `grep -q`: no pipeline, no dependence on which grep is installed. +while IFS= read -r line; do + case "$line" in + *'${'*) + echo "error: $OUT contains an unexpanded shell expansion:" >&2 + echo " $line" >&2 + exit 65 + ;; + esac +done <"$OUT" + +for archive in "${ARCHIVES[@]}"; do + case "$archive" in + *'$(SRCROOT)/'|*'$(SRCROOT)/ ') + echo "error: an archive path came out empty: $archive" >&2 + exit 65 + ;; + esac +done + +echo "Native toolchain: enabled (${#ARCHIVES[@]} archives)" diff --git a/README.md b/README.md index 2933ca9..d1d1769 100644 --- a/README.md +++ b/README.md @@ -1,77 +1,180 @@ # XForge -**Build iOS apps on your iPhone — powered by [xtool](https://github.com/xtool-org/xtool).** - -XForge embeds a real Linux userspace (running xtool + a Swift toolchain) inside an iOS -app. You author SwiftPM packages in the SwiftUI shell, and compile them into real, -signed iOS `.ipa` files entirely on-device. - -The Linux engine is **[ish-arm64](https://github.com/OpenMinis/ish-arm64)** — a real Linux -kernel + aarch64 emulator running **in-process** on iOS. Its aarch64 backend dispatches -guest instructions to pre-compiled "gadget" functions instead of emitting machine code, -so it needs no JIT entitlement and works in a sideloaded app. The Alpine aarch64 root filesystem ships -**inside the app**, so there is nothing to download after install. - -> **Status: active.** The SwiftUI shell, build pipeline, and CI are in place, and the -> embedded Linux is ish-arm64 with a bundled Alpine rootfs that already carries the -> build toolchain — xtool, the Swift toolchain and the `darwin` Swift SDK are in the -> root the app unpacks, so there is nothing to provision before the first build. The -> remaining work is wiring signing — see [Docs/DESIGN.md](Docs/DESIGN.md). - -## Why this works - -`xtool dev build` is SwiftPM cross-compilation to `arm64-apple-ios` using a Swift SDK -named `darwin`. All three heavyweight pieces are self-contained Linux artifacts: - -| Piece | Source | Notes | -|---|---|---| -| Linux engine | ish-arm64 (`Vendor/ish-arm64` submodule), built for iOS | runs in-process, no JIT entitlement | -| Alpine aarch64 rootfs | `alpine-rootfs.zip` (Alpine 3.21, engine `fakefs`) | **bundled in the app**, unpacked on first boot; pinned by release tag | -| Swift aarch64 Linux toolchain | swift.org, via `swiftly` | **in the bundled root**, installed at build time by `install-toolchain.sh` | -| `xtool` aarch64 binary | prebuilt `xtool-aarch64.AppImage` | **in the bundled root** | -| `darwin` Swift SDK (arm64-apple-ios) | XForge's `darwin-sdk-` release | **in the bundled root**; the Toolchain screen can also install one from your own `Xcode.xip` | - -The bundled root is **provisioned**: it is built by running the guest's own -installer (`EmbeddedLinux/install-toolchain.sh`) in a chroot of the root being -built, so the app unpacks a guest that already has xtool, the Swift toolchain and -the Darwin SDK — the whole chain is exercised on a build machine with a real CPU -and a fast network, once, instead of on a phone under emulation. Everything is -installed *before* the fakefs conversion, because the conversion indexes the tree -and anything added afterwards is on disk and invisible to the guest. The archive -is then converted to the engine's `fakefs` format, so the app only unzips. - -That makes the artifact ~1.6 GB, which is the price of arriving ready to build. -`EmbeddedLinux/build-rootfs.sh` takes `XFORGE_PROVISION=none` for a plain few-MB -Alpine root where the guest provisions itself on demand — the same script, the -same installer, run in the guest instead of at build time. The build also drops -what an iOS build never loads (the static Linux stdlib, lldb, the editor tooling, -and every download cache) and then *verifies the result compiles a Swift -program*, so a slimmed toolchain fails the build rather than a user's first build. +**Build iOS apps on your iPhone, with no Mac in the loop.** + +XForge is an on-device alternative to Xcode's build step. It is built on +[xtool](https://github.com/xtool-org/xtool)'s libraries — `XKit` for in-process +codesigning, and xtool's `darwin.artifactbundle` for the iPhoneOS SDK — and it +runs no helper process: the compiler and the Mach-O linker are linked into the app +binary and called in-process through a small C++ bridge. + +> **Status: active, and narrower than it looks.** A project authored in the SwiftUI +> shell is read into a build plan, compiled and linked in-process, packaged into an +> unsigned `.ipa`, and — with a certificate the user already has — signed with XKit. +> **C, Objective-C and Objective-C++ targets build today.** Swift needs a toolchain +> bundle built with `with_swift=true`, which carries the frontend libraries; the app +> links them, and running an actual Swift build on a device is the one step nobody +> has taken yet. See *What works today* and *What does not*. + +## How it is put together + +``` +project (a directory) → build plan → compile + link in-process → .app + → unsigned .ipa → XKit signing → signed .ipa +``` + +1. **Project.** A project is an ordinary directory in the app's container, + `/projects/`, holding a `Package.swift`, `Sources//`, + an optional `Support/Info.plist` and the `xtool.yml` metadata XForge and xtool + both read. `ProjectFiles` reads and writes it directly, keeping the + relative-path discipline the editor needs (a `..` is rejected rather than + normalised, because the paths come from editable text in the UI). A folder + picked in the Files app is copied in: an iOS process cannot run `git`, so a + clone is not something XForge can do. +2. **Build plan.** `NativeBuildPlanFactory` turns that directory into a fully + resolved `NativeBuildPlan` — module, sources, resources, frameworks, search + paths. It is deliberately conservative: it refuses, *by name*, the one thing an + in-process driver cannot resolve (SwiftPM dependencies, which need a package + manager that fetches and runs manifests) instead of failing halfway through a + build. +3. **Compile and link.** `NativeBuildExecutor` turns the plan into argument lists + (`NativeToolchainInvocation`, pure functions, so they can be unit-tested on a + simulator where no compiler exists) and calls the entry points behind the + bridge: clang through `CompilerInvocation` + `EmitObjAction`, `swift-frontend` + through `swift::performFrontend`, `ld64.lld` through `lld::lldMain`. The + blocking calls are pushed off the main actor so the Build screen keeps drawing. + The result is assembled into `.app` with an `Info.plist` and the + project's resources. +4. **Package.** `IPABuilder` puts the bundle under `Payload/` and zips it into an + unsigned `.ipa`, staged as `/staging/-.ipa`. +5. **Sign.** `IPASigningJob` + `AppBundleSigner` do the signing in this process + with XKit: the `.p12` is read through the Security framework, used in memory, + and never written anywhere. The result lands as `/Signed-.ipa`. + +Every screen reports from the same values, so the app cannot claim a capability it +does not have: the Build screen's stages, the Toolchain screen (capabilities, SDK, +smoke test) and the Settings screen (log, project files, diagnostics) all read the +in-process toolchain and the SDK in the container. + +## What works today + +- **Building a project on device.** C, Objective-C and Objective-C++ sources are + compiled by clang and linked by `ld64.lld` against the Darwin SDK, in-process, + into a `.app` and then an unsigned `.ipa`. +- **A project's own `Info.plist`.** A hand-written `Support/Info.plist` is merged + over the generated keys, so scene manifests and orientations survive a build. +- **Signing an app with an identity you already have** — a `.p12` and a + provisioning profile, with entitlements if you have them. +- **Exporting.** `ProjectExporter` zips a project into `/exports`, and + the file is visible in the Files app and shareable from there. Build output + (`.xforge-build`, `.build`) is left out of the archive. +- **Reading an `.xcodeproj`.** `App/Models/XcodeProject.swift` parses the project + model on device, with its own OpenStep parser. Nothing builds from it yet. +- **Building and testing XForge itself without any toolchain bundle.** A plain + clone compiles the bridge to a "not available" stub; `make test` runs on the + simulator. + +## What does not work (yet) + +- **Swift, end to end on a device.** The frontend libraries (`swiftFrontendTool` and + the swiftAST / swiftSema / IRGen / ClangImporter set it pulls in) are built for + iPhoneOS by a `with_swift=true` toolchain run, published in that bundle, and linked + into the app — the Toolchain screen reports `swift-frontend` as linked. What has + not happened is a Swift *build* on a device: every part of the path is exercised + except that one, and a bundle without the frontend still refuses a Swift plan at + the compile stage with `swiftFrontendMissing` and the file count. +- **SwiftPM dependencies.** Refused by name at plan time. Fetching and resolving + packages means running a package manager, which cannot happen here — and a + dependency editor that could only produce unbuildable projects was removed + rather than kept as a trap. +- **Asset catalogs.** `.xcassets` are copied into the bundle uncompiled, with a + warning: `actool` is a macOS-only tool and there is no open-source replacement + to point at, so an app whose icon lives in a catalog ships without it. +- **Apple ID provisioning.** Obtaining a certificate without one already in hand + means anisette + GrandSlam + 2FA, which is a separate, device-only piece of + work. The signer takes an existing identity; `XKitSigningService` throws rather + than reporting a session it does not have. +- **Installing to the device.** Exporting to SideStore/AltStore remains the + working path; in-app install needs a usbmux transport the sandbox cannot reach. +- **A remote build server.** `RemoteExecutor` is still a plan, not code. + +## Build the app + +Requires macOS + Xcode + [XcodeGen](https://github.com/yonaskolb/XcodeGen). + +```bash +brew install xcodegen + +make gen && open XForge.xcodeproj +``` + +That is the whole of it for a plain clone: there is no submodule to initialise and +no second build system to drive. The app builds with the compiler bridge compiled +to a stub and says so on the Toolchain screen. + +To build with the compiler *linked in*, install the toolchain bundle first: + +```bash +make toolchain-release # the newest published bundle +make gen +``` + +`install-bundle.sh --release` fetches the bundle from its release (a specific tag, +or a local tarball, also work — see `make toolchain`), unpacks it into +`Vendor/NativeToolchain` and runs `prepare-xcode.sh`, which writes +`Support/NativeToolchain.generated.xcconfig` — the file the app target consumes. +That xcconfig is checked in *with the backend disabled*, so a clone without the +bundle is buildable; see +[Docs/NATIVE-TOOLCHAIN.md](Docs/NATIVE-TOOLCHAIN.md). + +The only build prerequisite beyond XcodeGen is the toolchain bundle, and only if +you want a working compiler. + +Or build the unsigned IPA for sideloading via GitHub Actions +(`.github/workflows/build-ipa.yml`) and install it with SideStore/AltStore. + +## On-device build pipeline + +1. **Check build toolchain** — `NativeToolchainCapabilities` reports clang, + `ld64.lld`, `swift-frontend` and the SDK; the stage fails with an explanation + when nothing is linked, rather than letting a build die in the middle. +2. **Install Darwin SDK** — the SDK is a folder in the app container. The stage + skips itself when it is already there, otherwise it downloads the newest + `darwin-sdk-*` release asset. +3. **Configure app** — records the app identity (bundle ID, version, build number, + configuration) into `/.xforge-build/build.json` so a failed build is + reproducible, and checks that the project directory exists. +4. **Resolve dependencies** — reports "nothing to resolve" for a project with no + dependencies, and refuses a project that declares any. +5. **Compile (arm64-apple-ios)** — the work described above; also assembles the + `.app` and packages the unsigned `.ipa`. +6. **Package & sign .ipa** — accepts the packaged artifact. (The signing itself is + on the Signing screen, not in this stage.) +7. **Stage artifact** — confirms the `.ipa` exists and is non-empty, and reports + its size. ## Repo layout ``` App/ SwiftUI app — project editing, build pipeline, signing -App/EmbeddedVM/ ish-arm64 bridge: ISHEmulator, C shim, rootfs unpack -App/Build/ BuildExecutor protocol + EmbeddedLinuxExecutor, IPABuilder +App/Build/ BuildExecutor protocol, the NativeBuild* pipeline, IPABuilder App/Models/ pipeline, history, project store, XcodeProject (reads .xcodeproj) -App/NativeToolchain/ in-process Clang/LLD bridge + host-side Darwin SDK store -App/Services/ auth/signing/device seams, downloads, toolchain components -App/Views/ screens: Projects, Build, Toolchain, Terminal, Settings -Vendor/ish-arm64/ git submodule: the embedded Linux engine -Vendor/NativeToolchain/ where an installed LLVM/Clang bundle goes (empty by default) -EmbeddedLinux/ build-rootfs.sh, build-ish-core.sh, install-toolchain.sh -NativeToolchain/ prepare-xcode.sh (wires a vendor bundle into the project) -Support/ entitlements, assets, Resources/ (bundled rootfs) -Tools/ gen-appicon.py, the engine smoke harness, rootfs test +App/NativeToolchain/ in-process Clang/LLD bridge + the Darwin SDK store +App/Services/ signing, project files, export, device seams, the log +App/Views/ screens: Projects, Build, Signing, Settings (Toolchain, …) +AppTests/ unit tests (plan and argument construction, which run without a compiler) +Vendor/NativeToolchain/ where an installed toolchain bundle goes (absent by default) +NativeToolchain/ install-bundle.sh, prepare-xcode.sh +Support/ entitlements, assets, the generated toolchain xcconfig +Tools/ gen-appicon.py (writes the app icon into the asset catalog) project.yml XcodeGen definition -.github/workflows/ build-ipa.yml (the IPA), build-rootfs.yml (the pinned root), +Makefile gen, build, test, ipa, toolchain +.github/workflows/ build-ipa.yml (the unsigned IPA), native-toolchain.yml (the iOS LLVM cross-build) Docs/DESIGN.md full architecture write-up -Docs/ISH-ARM64-INTEGRATION.md the engine integration vs. its reference implementation Docs/IPA-BUILD.md the build pipeline and its stages +Docs/NATIVE-TOOLCHAIN.md the toolchain bundle, how it is built and linked Docs/XCODE-ALTERNATIVE.md what building existing Xcode projects on-device takes -Docs/NATIVE-TOOLCHAIN.md the in-process LLVM/Clang/LLD path CHANGELOG.md what changed, release by release ``` @@ -84,7 +187,7 @@ Everything about the app's identity lives in the project-level `settings` block |---|---| | Bundle identifier | `com.r0gueee.xforge` | | Display name | `XForge` | -| Apple team | set in `XFORGE_DEVELOPMENT_TEAM` (a wildcard `TEAMID.*` profile, so any bundle ID works) | +| Apple team | set in `XFORGE_DEVELOPMENT_TEAM` | | Marketing version | `XFORGE_MARKETING_VERSION` | | Build number | `XFORGE_BUILD_NUMBER` | | App icon | `Support/Assets.xcassets/AppIcon.appiconset` | @@ -111,73 +214,47 @@ the forge palette) and written into the asset catalog: make icon # rewrites Support/Assets.xcassets ``` -## Build the app - -Requires macOS + Xcode + [XcodeGen](https://github.com/yonaskolb/XcodeGen), meson and -ninja (for the ish-arm64 core). - -```bash -brew install xcodegen meson ninja llvm lld libarchive - -# One-time: engine sources, bundled rootfs, then the iOS engine libraries. -make bootstrap - -make gen && open XForge.xcodeproj -``` - -`make bootstrap` runs three steps: - -1. `git submodule update --init --depth 1 Vendor/ish-arm64` — the engine sources - (plus its `deps/libarchive` submodule, which the fakefs tools link against). -2. `EmbeddedLinux/build-rootfs.sh` — builds the Alpine aarch64 rootfs into - `Support/Resources/` so it is bundled into `XForge.app`. It shapes a plain - Alpine minirootfs, installs the glibc layer and the build toolchain into a - chroot of it (needs root: a chroot needs mounts), and converts the result to - the engine's fakefs format *on this machine*, so the app only has to unzip it. - `build-ipa.yml` downloads the published copy instead of rebuilding it — see - `EmbeddedLinux/build-rootfs.sh` and `build-rootfs.yml`. A local build without - root or without room for the toolchain can use `XFORGE_PROVISION=none`. -3. `EmbeddedLinux/build-ish-core.sh` — builds the engine's static libraries into - `Vendor/ish-arm64-build/lib` for the linker. This step needs macOS: the - aarch64 gadgets use `.req` register aliases that only clang's assembler - accepts. - -The engine is device-only; simulator builds (and `make test`) compile a stub instead -and need none of the above beyond a plain `make gen`. - -Or build the unsigned IPA for sideloading via GitHub Actions -(`.github/workflows/build-ipa.yml`) and install it with SideStore/AltStore. - -## On-device build pipeline - -1. **Embedded Linux** — ish-arm64 boots the bundled Alpine aarch64 rootfs, which is - already in the engine's `fakefs` format and is unpacked on first use. -2. **Toolchain** — nothing to do: the bundled root already carries the apk build - dependencies, the glibc compatibility layer, xtool, the Swift toolchain and the - Darwin SDK, all installed at build time. The Toolchain screen verifies them and - can install any of them into a guest that lacks one - (`sh /root/install-toolchain.sh all`, `sh /root/install-toolchain.sh sdk`), and - can replace the Darwin SDK with one built from your own `Xcode.xip`. -3. **Build** — `xtool dev build -s -i` runs in the guest; the `.ipa` is copied back out. -4. **Signing** — export the unsigned `.ipa` to SideStore/AltStore or another signing - service. Direct free-Apple-ID signing through XKit remains planned. +## Continuous integration + +- **`build-ipa.yml`** — runs the unit tests, archives the app unsigned + (`CODE_SIGNING_ALLOWED=NO`), verifies the built app and the final IPA, and + publishes the unsigned IPA for sideloading. It runs `prepare-xcode.sh` too, so CI + never depends on a toolchain bundle being installed in the checkout. +- **`native-toolchain.yml`** — cross-builds LLVM's Clang and Mach-O LLD for + iPhoneOS (and, with `with_swift=true`, Swift's frontend libraries) on a macOS + runner, merges the static archives into one, compile-checks and link-checks + `NativeToolchainBridge.mm` against the LLVM headers it just built, and publishes + `XForgeNativeToolchain-arm64-ios.tar.gz` as a release asset tagged + `toolchain---ios-sdk` (and as a workflow + artifact). It is dispatched by hand or by a pull request that touches + `App/NativeToolchain/**`; only a dispatched run publishes, because a run that + never built the Swift half must not become "the newest bundle". Its build trees + are cached, so a run whose inputs are unchanged finishes in minutes rather than + the ~80 minutes a cold build takes. + See [Docs/NATIVE-TOOLCHAIN.md](Docs/NATIVE-TOOLCHAIN.md). ## Roadmap -- [x] Embedded Linux engine: ish-arm64 built for iOS, running in-process -- [x] Alpine aarch64 rootfs bundled in the app and imported on first boot -- [x] Provisioned Alpine rootfs — bundled with the build dependencies, the glibc - layer, xtool, the Swift toolchain and the Darwin SDK, all installed at build - time; the Toolchain screen can still install or replace any of them in-guest -- [ ] XKit signing (free Apple ID) wired into the export flow +- [x] In-process compilation: clang + `ld64.lld` linked into the app, called + through a C++ bridge +- [x] Projects as ordinary directories in the app container +- [x] XKit signing wired into the export flow, for an identity the user has +- [x] Native toolchain bundle, built in CI, published as a release, installed by + `make toolchain-release` +- [x] **The Swift frontend libraries for iPhoneOS** — built by a `with_swift=true` + toolchain run, linked into the app, reported by the Toolchain screen. The + remaining unknown is a Swift *build* on a device, which is what the screen's + smoke test is for +- [ ] Build-setting evaluation and a build driver for a dependency graph +- [ ] `actool`/`ibtool` replacements (asset catalogs, storyboards) +- [ ] Apple ID provisioning (certificate issuance, 2FA) in-app - [ ] Hand-off of built `.ipa` to SideStore/AltStore for install - [ ] `RemoteExecutor` (build server) for fast compilation of real apps ## Licence note -XForge links the engine's core. That core is GPLv2/GPLv3 depending on which guest -architectures and native programs are compiled in. XForge builds it with native -bash/zsh/dash/helix **disabled** and only the arm64 guest, which keeps the linked -subset to the engine's own GPLv2 kernel code. If you redistribute XForge you must comply -with those terms; see [the engine's README](https://github.com/OpenMinis/ish-arm64) for the -full breakdown. +XForge links no emulator and no kernel code; the app target is the SwiftUI sources +plus two SwiftPM dependencies, xtool's `XKit` and +[ZIPFoundation](https://github.com/weichsel/ZIPFoundation) (MIT). xtool's own terms +apply to `XKit` — see its repository. The GPL note that used to be here concerned a +bundled emulator that is no longer part of the app. diff --git a/Support/Resources/.gitkeep b/Support/Resources/.gitkeep deleted file mode 100644 index 88d9b83..0000000 --- a/Support/Resources/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# Keep the Resources directory in git; the rootfs archive is fetched at build time. diff --git a/Support/XForge-Bridging-Header.h b/Support/XForge-Bridging-Header.h index 7fdd522..e5ba439 100644 --- a/Support/XForge-Bridging-Header.h +++ b/Support/XForge-Bridging-Header.h @@ -2,9 +2,8 @@ // XForge-Bridging-Header.h // XForge // -// Exposes the C shims to Swift: the embedded Linux engine bridge (ish-arm64) -// and the host DNS reader (which keeps the guest's /etc/resolv.conf correct). +// Exposes the C shim to Swift: the native LLVM/Clang/LLD compiler bridge. +// The embedded Linux engine (ISHBridge.h) and the guest DNS reader +// (HostDNS.h) are gone with the guest. // -#import "ISHBridge.h" -#import "HostDNS.h" #import "NativeToolchainBridge.h" diff --git a/Support/XForge.entitlements b/Support/XForge.entitlements index 58a2b85..a451b51 100644 --- a/Support/XForge.entitlements +++ b/Support/XForge.entitlements @@ -5,21 +5,22 @@ These are the only ones the app asks for, and they are here on purpose: - * com.apple.developer.kernel.extended-virtual-addressing — the embedded - ish-arm64 guest is a whole 64-bit Linux userland living in this process. It - maps a large sparse guest address space, and without this entitlement the - mapping fails and the guest cannot boot. ish-arm64's own iOS build declares - the same key. + * com.apple.developer.kernel.extended-virtual-addressing — the compiler runs in + this process, and LLVM maps large regions: the SDK's symbol tables, the object + files the linker reads back, and the JIT's reserved code space. The entitlement + was originally needed by the embedded Linux guest that used to live in this + process; the guest is gone, and no measurement has been taken since, so it is + kept for the compiler's maps rather than removed on a guess. - * com.apple.developer.kernel.increased-memory-limit — the guest holds a Linux - root filesystem, a Swift toolchain and the compiler's working set. The - default jetsam budget is far below what a compile needs, so this asks for the - larger limit. + * com.apple.developer.kernel.increased-memory-limit — a compiler's working set is + large (LLVM and Swift's frontend are linked into the app, and a build holds a + module graph in memory). The default jetsam budget is below what a real compile + needs, so this asks for the larger limit. Both are also the keys a signer may refuse: a wildcard or free-account profile - cannot always grant them. If a sideloader strips them, XForge still installs - and runs — the guest just gets the default address space and memory budget, - which is enough to boot and use the shell but not to compile large packages. + cannot always grant them. If a sideloader strips them, XForge still installs and + runs — it gets the default address space and memory budget, which is enough for + small projects and not for large ones. * get-task-allow — true, which is what a development-signed app has; a sideloader re-signs with its own certificate and sets this itself. diff --git a/Tools/engine-smoke/engine-smoke.c b/Tools/engine-smoke/engine-smoke.c deleted file mode 100644 index d7a0767..0000000 --- a/Tools/engine-smoke/engine-smoke.c +++ /dev/null @@ -1,444 +0,0 @@ -// -// engine-smoke.c -// XForge -// -// Host harness for the embedded Linux bridge. -// -// The app's install paths (the bundled Alpine rootfs, and every guest-side -// tool) all funnel through the same three bridge calls: import a rootfs into -// the engine's fakefs, boot the guest, run a command. When that goes wrong on a -// device the only symptom is "the app crashed", and the device cannot be -// attached to a debugger from here — so this harness drives *the same* -// ISHBridge.c, in *the same* order, on a real machine where the failure is -// visible. -// -// Build: see .github/workflows/engine-smoke.yml (adds an `engine_smoke` -// target to the engine's meson build, so the bridge is compiled with exactly the -// engine's own flags and guest-arch defines). -// -// Usage: engine-smoke [workdir] -// -// The root is an already-converted fakefs (a data/ tree plus meta.db), which -// is what the app has after unzipping the bundled alpine-rootfs.zip. The -// conversion and the unzip are both host-side, so neither goes through the -// engine and neither is what this harness needs to prove — it starts at the -// first call that actually touches the kernel. -// -// Every step prints a flushed breadcrumb *before* it runs, so a crash leaves -// the last step it reached on stdout with nothing after it. -// -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ISHBridge.h" - -static int failures = 0; - -static void say(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - vfprintf(stdout, fmt, ap); - va_end(ap); - fputc('\n', stdout); - fflush(stdout); -} - -// Announce a step before running it: if the process dies inside the step, the -// breadcrumb is already on the wire and nothing follows it. -static void step(const char *what) { say("[smoke] >>> %s", what); -} - -static void result(const char *what, int ok) { - say("[smoke] %s %s", ok ? "ok " : "FAIL", what); - if (!ok) failures++; -} - -static int rm_rf(const char *path) { - char cmd[4096]; - snprintf(cmd, sizeof cmd, "rm -rf '%s'", path); - return system(cmd); -} - -static int mkdirs(const char *path) { - return mkdir(path, 0777); -} - -// One guest command, with its outcome printed in full. -// -// `expect_exit` is the exit status the guest should end with -- 0 for the -// normal cases, 127 for the probes of programs a bare rootfs does not have yet, -// 7 for the deliberate failure below. The struct is read *before* it is freed: -// xf_guest_result_free() zeroes it, which is a mistake this harness made once -// and reported as every command failing. -static void run_guest(const char *label, const char *command, int timeout_ms, int expect_exit) { - step(label); - struct xf_guest_result r; - int rc = xf_ish_run(command, NULL, timeout_ms, 1 << 20, &r); - if (rc != 0) { - say("[smoke] FAIL %s: could not start (%d): %s", label, rc, xf_ish_last_error()); - failures++; - return; - } - say("[smoke] --- %s ---", label); - say("launched=%d exited=%d exit_code=%d signal=%d timed_out=%d truncated=%d bytes=%zu", - r.launched, r.exited, r.exit_code, r.term_signal, r.timed_out, r.truncated, r.output_len); - if (r.output != NULL && r.output[0] != '\0') { - fputs(r.output, stdout); - if (r.output[r.output_len > 0 ? r.output_len - 1 : 0] != '\n') - fputc('\n', stdout); - } - fflush(stdout); - - int ok = r.launched && r.exited && r.term_signal == 0 && r.exit_code == expect_exit; - if (!ok) - say("[smoke] expected exit %d", expect_exit); - xf_guest_result_free(&r); - result(label, ok); -} - -// The host's own DNS servers, the way the app gets them from iOS and writes -// them into the guest's /etc/resolv.conf (App/Services/GuestNetwork.swift). -// Without that file the guest cannot resolve anything: the bundled minirootfs -// ships no nameservers, and resolution happens inside the guest. -static int host_dns_servers(char out[][64], int max) { - FILE *f = popen("scutil --dns 2>/dev/null | awk '/nameserver\\[[0-9]+\\]/{print $3}' | " - "sort -u | head -4", "r"); - if (f == NULL) - return 0; - int count = 0; - char line[256]; - while (count < max && fgets(line, sizeof(line), f) != NULL) { - char *nl = strchr(line, '\n'); - if (nl != NULL) *nl = '\0'; - if (line[0] == '\0') continue; - snprintf(out[count], 64, "%s", line); - count++; - } - pclose(f); - return count; -} - -// Run a command and print everything about it, but do not judge it: used for -// probes whose failure is a fact about the environment (network reachability, -// package repositories) rather than a regression in the bridge. -static void probe_guest(const char *label, const char *command, int timeout_ms) { - step(label); - struct xf_guest_result r; - int rc = xf_ish_run(command, NULL, timeout_ms, 1 << 20, &r); - if (rc != 0) { - say("[smoke] probe could not start (%d): %s", rc, xf_ish_last_error()); - return; - } - say("[smoke] --- %s ---", label); - say("launched=%d exited=%d exit_code=%d signal=%d timed_out=%d truncated=%d bytes=%zu", - r.launched, r.exited, r.exit_code, r.term_signal, r.timed_out, r.truncated, r.output_len); - if (r.output != NULL && r.output[0] != '\0') { - fputs(r.output, stdout); - if (r.output[r.output_len > 0 ? r.output_len - 1 : 0] != '\n') - fputc('\n', stdout); - } - fflush(stdout); - xf_guest_result_free(&r); -} - -// Like probe_guest, but the result is asserted: these checks exist to prove a -// command *returns*, and probe_guest only reports. A hung lock burned a probe's -// whole 15-minute timeout while the harness still printed no failed steps. -static void probe_assert(const char *label, const char *command, int timeout_ms) { - step(label); - struct xf_guest_result r; - int rc = xf_ish_run(command, NULL, timeout_ms, 1 << 20, &r); - if (rc != 0) { - say("[smoke] probe could not start (%d): %s", rc, xf_ish_last_error()); - result(label, 0); - return; - } - say("[smoke] --- %s ---", label); - say("launched=%d exited=%d exit_code=%d signal=%d timed_out=%d truncated=%d bytes=%zu", - r.launched, r.exited, r.exit_code, r.term_signal, r.timed_out, r.truncated, r.output_len); - if (r.output != NULL && r.output[0] != '\0') { - fputs(r.output, stdout); - if (r.output[r.output_len > 0 ? r.output_len - 1 : 0] != '\n') - fputc('\n', stdout); - } - fflush(stdout); - result(label, r.launched && r.exited && !r.timed_out && r.term_signal == 0 && r.exit_code == 0); - xf_guest_result_free(&r); -} - -// Copy a host file into the shared folder, so the guest can see it at /host. -static int stage_into_share(const char *source, const char *destination) { - FILE *in = fopen(source, "rb"); - if (in == NULL) - return -1; - FILE *out = fopen(destination, "wb"); - if (out == NULL) { - fclose(in); - return -1; - } - char buffer[65536]; - size_t n; - while ((n = fread(buffer, 1, sizeof buffer, in)) > 0) { - if (fwrite(buffer, 1, n, out) != n) { - fclose(in); - fclose(out); - return -1; - } - } - fclose(in); - fclose(out); - return 0; -} - -int main(int argc, char **argv) { - if (argc < 2) { - fprintf(stderr, "usage: engine-smoke [workdir]\n"); - return 2; - } - const char *src_root = argv[1]; - const char *work = argc > 2 ? argv[2] : "/tmp/xforge-smoke"; - - char root[4096], host[4096]; - snprintf(root, sizeof root, "%s", src_root); - snprintf(host, sizeof host, "%s/host", work); - - say("[smoke] === XForge engine smoke test ==="); - say("[smoke] rootfs : %s (unpacked fakefs)", src_root); - say("[smoke] hostdir : %s", host); - - if (access(src_root, R_OK) != 0) { - say("[smoke] FAIL the rootfs directory is not readable: %s", strerror(errno)); - return 2; - } - - step("prepare work directory"); - rm_rf(work); - if (mkdirs(work) != 0 || mkdirs(host) != 0) { - say("[smoke] FAIL mkdir %s: %s", work, strerror(errno)); - return 1; - } - result("prepare work directory", 1); - - // --- 1. the root ------------------------------------------------------ - // The fakefs root is mounted in place, so point the harness at the caller's - // directory rather than copying it: in the app this is `/ - // embedded-linux/roots/alpine-3.21`, the result of unzipping the bundled - // alpine-rootfs.zip. Unzipping needs no engine, so this harness only checks - // that what it was given is a usable fakefs. - step("check the unpacked rootfs (data/ + meta.db)"); - char data_dir[4096]; - snprintf(data_dir, sizeof data_dir, "%s/data", root); - char meta_db[4096]; - snprintf(meta_db, sizeof meta_db, "%s/meta.db", root); - struct stat st; - int root_ok = 1; - if (stat(data_dir, &st) != 0 || !S_ISDIR(st.st_mode)) { - say("[smoke] FAIL %s is not a directory", data_dir); - root_ok = 0; - } else if (stat(meta_db, &st) != 0 || st.st_size == 0) { - say("[smoke] FAIL %s is missing or empty", meta_db); - root_ok = 0; - } - if (!root_ok) { - result("check the unpacked rootfs", 0); - say("[smoke] === stopped: not a fakefs root, so boot was never attempted ==="); - return 1; - } - say("[smoke] rootfs : %s", root); - result("check the unpacked rootfs", 1); - - // --- 2. boot ----------------------------------------------------------- - step("boot the guest (mount_root + become_first_process)"); - int rc = xf_ish_boot(root, host); - { - const char *err = xf_ish_last_error(); - say("[smoke] boot returned %d%s%s", rc, - (err != NULL && err[0] != '\0') ? " : " : "", - (err != NULL && err[0] != '\0') ? err : ""); - } - if (rc != 0) { - result("boot the guest", 0); - say("[smoke] === stopped: the guest did not boot ==="); - return 1; - } - result("boot the guest", 1); - - // --- 3. run commands --------------------------------------------------- - // The probes ToolchainManager.guestHas() runs, then the kind of work the - // install paths do. `swift`/`xtool` are *meant* to be missing here: this is - // a bare Alpine minirootfs, before any provisioning. - run_guest("uname", "uname -a", 120000, 0); - run_guest("alpine-release", "cat /etc/alpine-release; command -v sh; echo home=$HOME path=$PATH", 120000, 0); - run_guest("probe: swift (absent in a bare rootfs)", "command -v swift >/dev/null 2>&1", 120000, 127); - run_guest("probe: xtool (absent in a bare rootfs)", "command -v xtool >/dev/null 2>&1", 120000, 127); - run_guest("a failing command reports its status", "echo one; echo two >&2; exit 7", 120000, 7); - - // --- 4. the /host share ------------------------------------------------ - // The SDK install stages hundreds of MB here and installs from it, so a - // share that does not actually reach the guest breaks every install. - step("write a file into the host share"); - { - char path[4096]; - snprintf(path, sizeof path, "%s/share-probe.txt", host); - FILE *f = fopen(path, "w"); - if (f == NULL) { - say("[smoke] FAIL could not write %s: %s", path, strerror(errno)); - failures++; - } else { - fputs("host-share-ok\n", f); - fclose(f); - } - } - result("write a file into the host share", 1); - run_guest("read it from the guest at /host", "cat /host/share-probe.txt", 120000, 0); - run_guest("guest can write into /host", "echo from-guest > /host/written-by-guest.txt && cat /host/written-by-guest.txt", 120000, 0); - step("read the guest's file back on the host"); - { - char path[4096]; - snprintf(path, sizeof path, "%s/written-by-guest.txt", host); - FILE *f = fopen(path, "r"); - char buf[256] = {0}; - int ok = f != NULL && fgets(buf, sizeof buf, f) != NULL - && strncmp(buf, "from-guest", 10) == 0; - if (f != NULL) fclose(f); - say("[smoke] host sees: %s", ok ? buf : "(missing)"); - result("host share is bidirectional", ok); - } - - // --- 5. a long-running command, as a build would be -------------------- - run_guest("a slower command (shell loop)", "i=0; while [ $i -lt 200 ]; do i=$((i+1)); done; echo counted=$i", 120000, 0); - - // --- 6. what provisioning will need -------------------------------- - // Installing the Swift toolchain runs `apk add` and then downloads a - // several-hundred-megabyte tarball *inside* the guest, so the guest needs - // the host's network. These are probes, not assertions: a failure here is - // a fact about the environment, and it is far better to see it here than - // as a silent spinner on a phone. - step("write the guest's /etc/resolv.conf from the host's DNS"); - { - char servers[4][64]; - int count = host_dns_servers(servers, 4); - if (count == 0) { - say("[smoke] host published no DNS servers; leaving the guest alone"); - } else { - // One `nameserver` line per server: musl's resolver reads - // /etc/resolv.conf line by line and anything else on the line (a - // separator, a comment) becomes part of the address. - char cmd[1024]; - int n = snprintf(cmd, sizeof cmd, "rm -f /etc/resolv.conf"); - for (int i = 0; i < count && n > 0 && (size_t) n < sizeof(cmd); i++) - n += snprintf(cmd + n, sizeof(cmd) - (size_t) n, - "; echo 'nameserver %s' >> /etc/resolv.conf", servers[i]); - snprintf(cmd + n, sizeof(cmd) - (size_t) n, "; cat /etc/resolv.conf"); - struct xf_guest_result r; - int rc = xf_ish_run(cmd, NULL, 60000, 1 << 16, &r); - if (rc != 0) { - say("[smoke] could not write resolv.conf: %s", xf_ish_last_error()); - } else { - say("[smoke] guest resolv.conf now:\n%s", r.output != NULL ? r.output : ""); - } - xf_guest_result_free(&r); - } - } - - probe_guest("network: DNS + HTTP from the guest", - "wget -q -T 30 -O /dev/null https://dl-cdn.alpinelinux.org/alpine/ && echo net-ok", 180000); - probe_guest("network: apk update (the first thing provisioning runs)", - "apk update 2>&1 | tail -3", 300000); - - // --- 7. the provisioning steps themselves (opt-in) ---------------- - // With XFORGE_SMOKE_TOOLCHAIN pointing at the app's install script, run the - // same steps the Toolchain screen runs -- base packages, the glibc layer, - // xtool -- and then try to *run* what they installed. This is the difference - // between "the install command succeeded" and "the tool works in a guest". - { - const char *script = getenv("XFORGE_SMOKE_TOOLCHAIN"); - if (script != NULL && script[0] != '\0') { - char staged[4096]; - snprintf(staged, sizeof staged, "%s/install-toolchain.sh", host); - step("stage the provisioning script into the guest"); - if (stage_into_share(script, staged) != 0) { - say("[smoke] could not copy %s into the share", script); - } else { - result("stage the provisioning script", 1); - probe_guest("provision: base packages", - "cp -f /host/install-toolchain.sh /root/ && sh /root/install-toolchain.sh deps", 900000); - probe_guest("provision: glibc layer", - "sh /root/install-toolchain.sh glibc", 1800000); - probe_guest("provision: xtool", - "sh /root/install-toolchain.sh xtool", 900000); - probe_guest("provision: swiftly (the Swift installer)", - "sh /root/install-toolchain.sh swiftly", 900000); - probe_guest("provision: Swift toolchain (this downloads a lot)", - "sh /root/install-toolchain.sh swift", 3600000); - probe_guest("provision: verify (the script's own verdict)", - "sh /root/install-toolchain.sh verify", 600000); - probe_guest("does xtool actually run in the guest?", - "xtool --version", 300000); - probe_guest("does swift actually run in the guest?", - "swift --version", 600000); - - // --- PI futexes, which is what Swift's Mutex is built on ------- - // The engine answered ENOSYS for every PI futex op, and Swift 6's - // Synchronization.Mutex turns that unexpected errno into a - // fatalError -- a SIGTRAP, "Trace/breakpoint trap", exit 133. So - // any Swift binary doing concurrent work died on its first Mutex - // while single-threaded commands kept working. The C probe checks - // the syscall itself, under contention; the Swift probe is the - // end-to-end check that the fix reaches the runtime the app uses. - { - const char *sources[] = { - "Tools/engine-smoke/futex-pi-probe.c", - "Tools/engine-smoke/swift-mutex-probe.swift", - }; - int staged = 1; - step("stage the PI futex and Swift Mutex probes"); - for (unsigned i = 0; i < sizeof sources / sizeof sources[0]; i++) { - const char *slash = strrchr(sources[i], '/'); - char destination[4096]; - snprintf(destination, sizeof destination, "%s/%s", host, - slash != NULL ? slash + 1 : sources[i]); - if (stage_into_share(sources[i], destination) != 0) { - say("[smoke] could not stage %s (run this from the repository root)", - sources[i]); - staged = 0; - } - } - result("stage the PI futex and Swift Mutex probes", staged); - if (staged) { - probe_assert("PI futexes: FUTEX_LOCK_PI / TRYLOCK_PI / UNLOCK_PI", - "cd /root && cp -f /host/futex-pi-probe.c . && " - "clang -O1 -pthread -o futex-pi-probe futex-pi-probe.c && " - "./futex-pi-probe", - 900000); - // Informational, not a gate: compiling Swift in the emulated - // guest did not finish in 30 minutes on the CI host (no output - // at all), while the payload build compiles a Swift program - // natively in seconds — so *that* is where compile capability - // is asserted. What this step still shows, when it returns, is - // the end-to-end Mutex path under libdispatch. - probe_guest("Swift Synchronization.Mutex under libdispatch workers (informational)", - "swiftc -O -o /root/swift-mutex-probe /host/swift-mutex-probe.swift " - "&& /root/swift-mutex-probe", - 600000); - } - } - } - } - } - - step("shut down"); - xf_ish_shutdown(); - result("shut down", 1); - - say("[smoke] === %s (%d failed step%s) ===", - failures == 0 ? "PASSED" : "FAILED", failures, failures == 1 ? "" : "s"); - return failures == 0 ? 0 : 1; -} diff --git a/Tools/engine-smoke/futex-pi-probe.c b/Tools/engine-smoke/futex-pi-probe.c deleted file mode 100644 index f322da8..0000000 --- a/Tools/engine-smoke/futex-pi-probe.c +++ /dev/null @@ -1,174 +0,0 @@ -/* - * futex-pi-probe.c — run *inside* the XForge guest: do PI futexes work? - * - * Swift 6's Synchronization.Mutex is built on FUTEX_LOCK_PI, and the engine - * used to answer ENOSYS for it. Swift's Linux implementation turns an - * unexpected futex errno into - * - * Synchronization/LinuxImpl.swift:194: Fatal error: - * Unknown error occurred while attempting to acquire a Mutex - * - * which is a trap (exit 133, "Trace/breakpoint trap"), so any Swift binary - * doing concurrent work — xtool, libdispatch's DispatchWorker threads — died on - * its very first Mutex while single-threaded commands kept working. - * - * This checks the syscall directly (fast, deterministic) and then through the - * C library's own PI mutexes, including under contention with real threads, - * which is the part a wrong implementation would deadlock or double-take on. - * - * Built and run by Tools/engine-smoke/engine-smoke.c in the guest. - */ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(__has_include) -# if __has_include() -# include -# endif -#endif -#ifndef FUTEX_LOCK_PI -#define FUTEX_LOCK_PI 6 -#define FUTEX_UNLOCK_PI 7 -#define FUTEX_TRYLOCK_PI 8 -#endif -#ifndef FUTEX_PRIVATE_FLAG -#define FUTEX_PRIVATE_FLAG 128 -#endif - -static int failures; - -static void check(const char *what, int ok, const char *detail) { - printf(" %-46s %s%s%s\n", what, ok ? "ok" : "FAIL", - detail != NULL && detail[0] != '\0' ? " — " : "", - detail != NULL ? detail : ""); - fflush(stdout); - if (!ok) - failures++; -} - -/* A guest thread blocked on a lock that never comes free would otherwise burn - * the harness's whole timeout with no output; this turns that into a result. - * The correct probe finishes in well under a second of guest time (the emulator - * makes it seconds, not minutes). */ -static void watchdog(int sig) { - (void) sig; - printf("PI-FUTEX TIMED OUT — a lock or a wait never returned\n"); - fflush(stdout); - _exit(2); -} - -/* FUTEX_LOCK_PI/FUTEX_UNLOCK_PI take the value argument as unused, so this - * passes 0 like glibc does; errno is the useful result. */ -static int pi_op(int *word, int op) { - return (int) syscall(SYS_futex, word, op | FUTEX_PRIVATE_FLAG, 0, NULL, NULL, 0); -} - -/* Deliberately small: these ops are microseconds natively but the guest is an - * emulated aarch64, and the point is whether lock/unlock/wake work under - * contention at all — not throughput. 4 x 250 contended pairs is ample. */ -#define THREADS 4 -#define ROUNDS 250 - -static int counter; /* guarded by the raw PI word */ - -static void *raw_worker(void *arg) { - (void) arg; - for (int i = 0; i < ROUNDS; i++) { - while (pi_op(&counter, FUTEX_LOCK_PI) != 0) { - if (errno != EAGAIN && errno != EINTR) { - printf(" raw FUTEX_LOCK_PI failed: %s\n", strerror(errno)); - return NULL; - } - } - /* The word must name us while we hold it. */ - if (counter != (int) gettid()) { - printf(" owner tid wrong while held: %d != %d\n", counter, (int) gettid()); - return NULL; - } - /* The word must NOT be written here: for a PI futex the kernel owns it, - * and FUTEX_UNLOCK_PI is what clears it and wakes a waiter. Zeroing it - * first made the unlock an EPERM no-op, so no wake was ever issued and - * the waiters slept for good — which is how this probe first timed out. */ - pi_op(&counter, FUTEX_UNLOCK_PI); - } - return NULL; -} - -int main(void) { - setvbuf(stdout, NULL, _IONBF, 0); - signal(SIGALRM, watchdog); - alarm(120); - printf("futex-pi-probe: tid=%d\n", (int) gettid()); - - /* 1. The syscall on its own: free word, lock, unlock, and the owner check. */ - int word = 0; - int rc = pi_op(&word, FUTEX_LOCK_PI); - check("FUTEX_LOCK_PI on a free word", rc == 0, rc == 0 ? "" : strerror(errno)); - check("the word records the owner", word == (int) gettid(), ""); - - rc = pi_op(&word, FUTEX_TRYLOCK_PI); - check("FUTEX_TRYLOCK_PI on our own lock is EDEADLK", - rc == -1 && errno == EDEADLK, rc == -1 ? strerror(errno) : "succeeded"); - - rc = pi_op(&word, FUTEX_UNLOCK_PI); - check("FUTEX_UNLOCK_PI by the owner", rc == 0 && word == 0, - rc == 0 ? "" : strerror(errno)); - - errno = 0; - rc = pi_op(&word, FUTEX_UNLOCK_PI); - check("FUTEX_UNLOCK_PI by a non-owner is EPERM", - rc == -1 && errno == EPERM, rc == -1 ? strerror(errno) : "succeeded"); - - /* 2. Contended: take the lock in one thread, TRYLOCK_PI from another. */ - { - word = 0; - pi_op(&word, FUTEX_LOCK_PI); - pid_t child = fork(); - if (child == 0) { - errno = 0; - int r = pi_op(&word, FUTEX_TRYLOCK_PI); - _exit((r == -1 && errno == EAGAIN) ? 0 : 1); - } - int status = 0; - waitpid(child, &status, 0); - check("FUTEX_TRYLOCK_PI from another task is EAGAIN", - WIFEXITED(status) && WEXITSTATUS(status) == 0, ""); - pi_op(&word, FUTEX_UNLOCK_PI); - } - - /* 3. Real threads through the raw syscall, one word, contended. */ - { - counter = 0; - pthread_t t[THREADS]; - for (int i = 0; i < THREADS; i++) - pthread_create(&t[i], NULL, raw_worker, NULL); - for (int i = 0; i < THREADS; i++) - pthread_join(t[i], NULL); - check("raw PI futex, 4 threads x 250 contended lock/unlock", - counter == 0, "no corruption, no deadlock"); - } - - /* The C library's own PTHREAD_PRIO_INHERIT mutexes are deliberately NOT - * tested here. musl drives those through a different pattern — an absolute - * deadline passed as FUTEX_LOCK_PI's timeout, a userspace CAS of the word - * before a plain FUTEX_WAKE on unlock — and that path hung in the emulated - * guest, which says something about musl's PI mutexes in this environment - * rather than about the PI futexes Swift uses. Swift 6's Synchronization.Mutex - * calls FUTEX_LOCK_PI/FUTEX_UNLOCK_PI directly, which is what the tests above - * cover, so that is what this probe asserts. */ - - alarm(0); - if (failures == 0) - printf("PI-FUTEX OK\n"); - else - printf("PI-FUTEX FAILED (%d)\n", failures); - return failures == 0 ? 0 : 1; -} diff --git a/Tools/engine-smoke/swift-mutex-probe.swift b/Tools/engine-smoke/swift-mutex-probe.swift deleted file mode 100644 index 4e0eb58..0000000 --- a/Tools/engine-smoke/swift-mutex-probe.swift +++ /dev/null @@ -1,37 +0,0 @@ -// swift-mutex-probe.swift — run *inside* the XForge guest: does a Swift 6 -// Synchronization.Mutex work, under real concurrency? -// -// This is the exact shape of the failure the app hit: -// -// Synchronization/LinuxImpl.swift:194: Fatal error: -// Unknown error occurred while attempting to acquire a Mutex -// ... Trace/breakpoint trap (exit 133) -// -// Swift's Mutex is built on FUTEX_LOCK_PI, and the engine answered ENOSYS for -// it, so any Swift binary doing concurrent work died on its first Mutex — -// xtool's `xtool sdk build`, which drives libdispatch worker threads, was the -// one that surfaced it. `DispatchQueue.concurrentPerform` reproduces the same -// thread pattern without needing xtool's download, so this is the fast check -// that the engine fix actually reaches Swift. -// -// Built and run by Tools/engine-smoke/engine-smoke.c in the guest. - -import Dispatch -import Synchronization - -let counter = Mutex(0) -let iterations = 64 * 1000 -let workers = 8 - -DispatchQueue.concurrentPerform(iterations: workers) { _ in - for _ in 0..<(iterations / workers) { - counter.withLock { $0 += 1 } - } -} - -let total = counter.withLock { $0 } -if total == iterations { - print("swift-mutex-ok \(total)") -} else { - print("swift-mutex-wrong \(total) of \(iterations)") -} diff --git a/Tools/test-verify-toolchain.sh b/Tools/test-verify-toolchain.sh deleted file mode 100644 index 580a683..0000000 --- a/Tools/test-verify-toolchain.sh +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env bash -# -# test-verify-toolchain.sh — check that verify-rootfs.sh holds a root to what its -# manifest claims about the build toolchain. -# -# This is the half of the rootfs verification that a *packed* root gets: a fakefs -# ZIP cannot be chrooted, so "the console starts" is checked on the tree during -# the build and "the toolchain is really in here" is checked structurally, on the -# artifact. The structural half is easy to get wrong in a way that passes -# everything — a check that looks for the wrong path, or that trusts the manifest -# instead of the root. -# -# So each case below builds a root that is complete EXCEPT for one thing, and -# asserts that verify-rootfs.sh fails, naming that thing. -# -# Usage: Tools/test-verify-toolchain.sh -# -set -euo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -# Overridable so this can be pointed at a copy of the checker; the default is the -# one that ships. -VERIFY="${XFORGE_VERIFY:-$HERE/../EmbeddedLinux/verify-rootfs.sh}" -[ -f "$VERIFY" ] || { echo "cannot find $VERIFY" >&2; exit 1; } -WORK="$(mktemp -d)" -trap 'rm -rf "$WORK"' EXIT - -pass=0 -fail=0 - -note() { printf ' %s\n' "$*"; } -ok() { pass=$((pass + 1)); } -bad() { fail=$((fail + 1)); printf 'FAIL: %s\n' "$*" >&2; } - -# A root with everything the console checks want, so the toolchain checks are the -# only thing under test. -make_root() { - local root="$1" - mkdir -p "$root/bin" "$root/sbin" "$root/etc/init.d" "$root/root" \ - "$root/usr/share/terminfo/x" "$root/usr/local/share/xforge" - printf '#!/bin/sh\n' > "$root/bin/busybox" - chmod +x "$root/bin/busybox" - printf '#!/bin/sh\n' > "$root/sbin/init" - chmod +x "$root/sbin/init" - ln -sf /bin/busybox "$root/bin/sh" - cp "$HERE/../EmbeddedLinux/xforge-login" "$root/sbin/xforge-login" - chmod +x "$root/sbin/xforge-login" - printf '::sysinit:/etc/init.d/rcS\ntty1::respawn:/sbin/xforge-login root\n' > "$root/etc/inittab" - printf '#!/bin/sh\nexit 0\n' > "$root/etc/init.d/rcS" - printf 'root:x:0:0:root:/root:/bin/sh\n' > "$root/etc/passwd" - printf 'root:x:0:0:root:/root:/bin/sh\n' > "$root/etc/shadow" - printf 'export TERM=xterm-256color\nexport PATH=/bin\n' > "$root/etc/profile" - mkdir -p "$root/etc/apk" - printf 'https://example.invalid/main\n' > "$root/etc/apk/repositories" - : > "$root/usr/share/terminfo/x/xterm-256color" - printf '#!/bin/sh\n' > "$root/root/install-toolchain.sh" - cat > "$root/usr/local/share/xforge/rootfs-manifest.txt" <<'EOF' -base: alpine-minirootfs-3.21.0-aarch64.tar.gz -rootfs: 3.21.0 -engine: ish-arm64 -format: fakefs-zip -shell: /bin/sh -console: /sbin/xforge-login root (tty1, respawned by init) -toolchain: not provisioned (the guest installs it on demand) -stamp: rootfs-v5 -EOF -} - -# Add a complete, working toolchain to a root. -add_toolchain() { - local root="$1" - mkdir -p "$root/usr/local/bin" "$root/opt/xtool/usr/bin" \ - "$root/usr/local/share/xforge" \ - "$root/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/bin" \ - "$root/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/lib/swift/linux/aarch64" \ - "$root/root/.swiftpm/swift-sdks/darwin.artifactbundle" - for tool in xtool swift swiftc swiftly; do - printf '#!/bin/sh\n' > "$root/usr/local/bin/$tool" - chmod +x "$root/usr/local/bin/$tool" - done - printf 'XFORGE_GLIBC_LD=/opt/glibc/usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1\n' \ - > "$root/usr/local/share/xforge/glibc.env" - printf '#!/bin/sh\n' > "$root/opt/xtool/usr/bin/xtool" - chmod +x "$root/opt/xtool/usr/bin/xtool" - : > "$root/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/bin/swift-frontend" - : > "$root/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/lib/swift/linux/aarch64/Swift.swiftmodule" - : > "$root/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/lib/swift/linux/libswiftCore.so" - printf '{"name": "darwin"}\n' > "$root/root/.swiftpm/swift-sdks/darwin.artifactbundle/info.json" - cat > "$root/usr/local/share/xforge/darwin-sdk.txt" <<'EOF' -tag: darwin-sdk-7 -asset: darwin.artifactbundle.zip -sha256: 0000000000000000000000000000000000000000000000000000000000000000 -path: /root/.swiftpm/swift-sdks/darwin.artifactbundle -EOF - cat > "$root/usr/local/share/xforge/rootfs-manifest.txt" <<'EOF' -base: alpine-minirootfs-3.21.0-aarch64.tar.gz -rootfs: 3.21.0 -engine: ish-arm64 -format: fakefs-zip -shell: /bin/sh -console: /sbin/xforge-login root (tty1, respawned by init) -toolchain: xtool, swiftly, swift -darwin-sdk: darwin-sdk-7 (darwin.artifactbundle.zip) -sdk-sha256: 0000000000000000000000000000000000000000000000000000000000000000 -darwin-sdk-path: /root/.swiftpm/swift-sdks/darwin.artifactbundle -stamp: rootfs-v5 -EOF -} - -# run_verify — with the console check skipped: this is about the files, and -# running the console needs the *root's own* shell, which a synthetic root has not -# got. -run_verify() { - XFORGE_VERIFY_SKIP_CONSOLE=1 bash "$VERIFY" "$1" > "$WORK/out.txt" 2>&1 -} - -check_passes() { - local name="$1" root="$2" - if run_verify "$root"; then ok; note "$name: passes"; else - bad "$name should have passed"; sed 's/^/ /' "$WORK/out.txt" | tail -5 - fi -} - -check_fails() { - local name="$1" root="$2" expect="$3" - if run_verify "$root"; then - bad "$name should have failed" - elif grep -q "$expect" "$WORK/out.txt"; then - ok; note "$name: fails, naming '$expect'" - else - bad "$name failed without naming '$expect'"; sed 's/^/ /' "$WORK/out.txt" | tail -5 - fi -} - -printf '\n==> A root with no toolchain claim\n' -make_root "$WORK/plain" -check_passes "plain root" "$WORK/plain" - -printf '\n==> A root that claims a toolchain and has one\n' -make_root "$WORK/full" -add_toolchain "$WORK/full" -check_passes "provisioned root" "$WORK/full" - -printf '\n==> A root that claims a toolchain and does not have one\n' -make_root "$WORK/liar" -sed -i 's|^toolchain:.*|toolchain: xtool, swiftly, swift|' \ - "$WORK/liar/usr/local/share/xforge/rootfs-manifest.txt" -check_fails "manifest without tools" "$WORK/liar" 'usr/local/bin/xtool is missing' - -printf '\n==> The Swift toolchain, one piece at a time\n' -for piece in \ - 'usr/local/bin/xtool' \ - 'usr/local/bin/swift' \ - 'usr/local/bin/swiftc' \ - 'usr/local/bin/swiftly' \ - 'opt/xtool/usr/bin/xtool' \ - 'usr/local/share/xforge/glibc.env' ; do - rm -rf "$WORK/missing" - cp -a "$WORK/full" "$WORK/missing" - rm -rf "$WORK/missing/$piece" - check_fails "no $piece" "$WORK/missing" '/'"$piece"' is missing' -done - -rm -rf "$WORK/nofrontend" -cp -a "$WORK/full" "$WORK/nofrontend" -rm -f "$WORK/nofrontend/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/bin/swift-frontend" -check_fails "no swift-frontend" "$WORK/nofrontend" 'no Swift toolchain under' - -rm -rf "$WORK/nostdlib" -cp -a "$WORK/full" "$WORK/nostdlib" -rm -f "$WORK/nostdlib/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/lib/swift/linux/aarch64/Swift.swiftmodule" -check_fails "no stdlib interface" "$WORK/nostdlib" 'no Linux stdlib interface' - -rm -rf "$WORK/nostdlibso" -cp -a "$WORK/full" "$WORK/nostdlibso" -rm -f "$WORK/nostdlibso/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/lib/swift/linux/libswiftCore.so" -check_fails "no stdlib library" "$WORK/nostdlibso" 'no Linux stdlib under' - -rm -rf "$WORK/nosdkpath" -cp -a "$WORK/full" "$WORK/nosdkpath" -sed -i '/^path:/d' "$WORK/nosdkpath/usr/local/share/xforge/darwin-sdk.txt" -check_fails "SDK record without a path" "$WORK/nosdkpath" 'does not name where the SDK went' - -rm -rf "$WORK/nosdk" -cp -a "$WORK/full" "$WORK/nosdk" -rm -rf "$WORK/nosdk/root/.swiftpm/swift-sdks/darwin.artifactbundle" -check_fails "SDK missing" "$WORK/nosdk" 'which is not in this root' - -printf '\n==> A root with xtool but no Darwin SDK is fine (XFORGE_PROVISION_SDK=0)\n' -rm -rf "$WORK/nosdkclaim" -cp -a "$WORK/full" "$WORK/nosdkclaim" -rm -rf "$WORK/nosdkclaim/usr/local/share/xforge/darwin-sdk.txt" \ - "$WORK/nosdkclaim/root/.swiftpm" -sed -i -e '/^darwin-sdk:/d' -e '/^sdk-sha256:/d' -e '/^darwin-sdk-path:/d' \ - "$WORK/nosdkclaim/usr/local/share/xforge/rootfs-manifest.txt" -check_passes "toolchain without the SDK" "$WORK/nosdkclaim" - -printf '\n%s checks passed, %s failed\n' "$pass" "$fail" -[ "$fail" -eq 0 ] diff --git a/Vendor/ish-arm64 b/Vendor/ish-arm64 deleted file mode 160000 index e1d5794..0000000 --- a/Vendor/ish-arm64 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e1d579480fba88e8f0428e3cf23811bcdd05421f diff --git a/builder.json b/builder.json deleted file mode 100644 index 58b21a7..0000000 --- a/builder.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "project": "XForge", - "platform": "ios", - "github": { - "owner": "R0GUEEE", - "repo": "XForge" - }, - "provider": "github", - "codemagic": {}, - "bitrise": {}, - "ios": {}, - "flutter": { - "watch": {} - }, - "reactNative": {}, - "kmp": {}, - "mobai": {} -} \ No newline at end of file diff --git a/project.yml b/project.yml index 1dd6750..c728e1b 100644 --- a/project.yml +++ b/project.yml @@ -28,9 +28,10 @@ packages: # `Support/Info.plist` is generated by XcodeGen from the `info.properties` of # the XForge target — do not hand-edit it (it is gitignored). # -# ish-arm64 (Vendor/ish-arm64, a git submodule) provides the embedded Linux engine. -# Its core static libraries are built by EmbeddedLinux/build-ish-core.sh into -# Vendor/ish-arm64-build/lib, where the linker finds them. +# The on-device toolchain is linked in from Vendor/NativeToolchain (written by +# NativeToolchain/prepare-xcode.sh); there is no embedded Linux and no engine to +# build. A clone without the toolchain bundle still builds: the bridge compiles +# to a "not available" stub and the app reports that on the Toolchain screen. # --------------------------------------------------------------------------- settings: base: @@ -39,9 +40,6 @@ settings: XFORGE_DEVELOPMENT_TEAM: V5GXMU3HC2 XFORGE_MARKETING_VERSION: "0.5.0" XFORGE_BUILD_NUMBER: "8" - ISH_ROOT: $(SRCROOT)/Vendor/ish-arm64 - ISH_BUILD_DIR: $(SRCROOT)/Vendor/ish-arm64-build - ISH_LIB_DIR: $(SRCROOT)/Vendor/ish-arm64-build/lib targets: XForge: @@ -53,18 +51,6 @@ targets: sources: - path: App - path: Support - - path: Support/Resources - buildPhase: resources - optional: true - # In-guest provisioning scripts. The terminal can run the toolchain - # installer and the Build tab can install zsign into the guest before - # configuring/signing an IPA. - - path: EmbeddedLinux/install-toolchain.sh - buildPhase: resources - - path: EmbeddedLinux/install-zsign.sh - buildPhase: resources - - path: EmbeddedLinux/patch-zsign-password-file.py - buildPhase: resources dependencies: - package: xtool product: XKit @@ -72,42 +58,6 @@ targets: product: ZIPFoundation - sdk: WebKit.framework - sdk: SystemConfiguration.framework - preBuildScripts: - # The engine is a prerequisite of *compiling* this target, not just of - # linking it: ISHBridge.c includes the engine's headers, and a device build - # links libish.a / libish_emu.a / libfakefs.a out of $(ISH_LIB_DIR). So this - # runs before Sources rather than after them. - # - # It builds the engine when it is missing or out of date and does nothing - # otherwise (the engine build records a stamp; see build-ish-core.sh), and it - # skips itself entirely for simulator builds, which compile the stub branch - # of the bridge instead. Without it, a plain `xcodebuild` — or any pipeline - # that does not know to run EmbeddedLinux/build-ish-core.sh first — fails at - # ISHBridge.c with "'kernel/init.h' file not found". - - name: Build the embedded Linux engine - # `bash ` rather than the bare path: the phase must not depend on the - # file's executable bit surviving a checkout. - script: 'bash "$SRCROOT/EmbeddedLinux/build-engine-for-xcode.sh"' - basedOnDependencyAnalysis: false - # The other half of a working app is the guest rootfs, and it cannot be - # fetched here: it is a resource, so XcodeGen had to see it when it walked the - # directory (that is why EmbeddedLinux/fetch-rootfs.sh exists and why CI runs - # it before generating the project). A device build without it produces an app - # that cannot boot Linux, so this says so instead of shipping it quietly. - - name: Check the bundled guest rootfs - script: | - if [ "${PLATFORM_NAME:-}" != "iphoneos" ] || [ "${XFORGE_SKIP_ROOTFS:-0}" = "1" ]; then - exit 0 - fi - if [ -f "${SRCROOT}/Support/Resources/alpine-rootfs.zip" ]; then - exit 0 - fi - echo "error: no guest rootfs is staged at Support/Resources/alpine-rootfs.zip, so this app would boot no Linux." >&2 - echo " Stage the pinned one (about 165 MB) and build again:" >&2 - echo " bash EmbeddedLinux/fetch-rootfs.sh" >&2 - echo " Set XFORGE_SKIP_ROOTFS=1 to build without it anyway." >&2 - exit 1 - basedOnDependencyAnalysis: false settings: base: PRODUCT_BUNDLE_IDENTIFIER: $(XFORGE_BUNDLE_ID) @@ -122,7 +72,7 @@ targets: ASSETCATALOG_COMPILER_GENERATE_ASSET_SYMBOLS: YES # --- signing ------------------------------------------------------ # Automatic signing with the team above. CI builds pass - # CODE_SIGNING_ALLOWED=NO (see .github/workflows/unsigned-ipa.yml), so + # CODE_SIGNING_ALLOWED=NO (see .github/workflows/build-ipa.yml), so # an unsigned IPA can be produced without any certificate. DEVELOPMENT_TEAM: $(XFORGE_DEVELOPMENT_TEAM) CODE_SIGN_STYLE: Automatic @@ -136,49 +86,34 @@ targets: # generates that file from `properties`, and with none declared it wrote # an empty plist over it, which silently dropped every entitlement. CODE_SIGN_ENTITLEMENTS: $(SRCROOT)/Support/XForge.entitlements - # The engine's build phase above runs git, Homebrew and meson/ninja, and - # writes into Vendor/ish-arm64-build — all outside what a sandboxed script # phase may touch. Xcode 15+ defaults this to YES for new projects, which # would turn the phase into "Operation not permitted". ENABLE_USER_SCRIPT_SANDBOXING: NO ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME: AccentColor - # --- embedded ish-arm64 engine (device builds only) ----------------- - # The engine is built for the iOS device (arm64). Simulator builds - # compile the bridge's stub instead and link nothing extra. + # --- native compiler bridge ------------------------------------------- + # Hidden for the *guest* headers that used to be here; what remains is the + # bridge header and whatever toolchain bundle the xcconfig points at. SWIFT_OBJC_BRIDGING_HEADER: Support/XForge-Bridging-Header.h HEADER_SEARCH_PATHS: - $(inherited) - - $(SRCROOT)/App/EmbeddedVM - $(SRCROOT)/App/Services - $(SRCROOT)/App/NativeToolchain - $(XFORGE_NATIVE_HEADER_SEARCH_PATHS) "HEADER_SEARCH_PATHS[sdk=iphoneos*]": - $(inherited) - - $(SRCROOT)/App/EmbeddedVM - $(SRCROOT)/App/Services - $(SRCROOT)/App/NativeToolchain - - $(ISH_ROOT) - $(XFORGE_NATIVE_HEADER_SEARCH_PATHS) "LIBRARY_SEARCH_PATHS[sdk=iphoneos*]": - $(inherited) - - $(ISH_LIB_DIR) - $(XFORGE_NATIVE_LIBRARY_SEARCH_PATHS) - # Must match the guest-arch configuration the core was built with - # (-Dguest_arch=arm64); struct layouts differ per enabled arch. "OTHER_CFLAGS[sdk=iphoneos*]": - $(inherited) - - -DGUEST_ARM64=1 - $(XFORGE_NATIVE_CFLAGS) "OTHER_LDFLAGS[sdk=iphoneos*]": - $(inherited) - $(XFORGE_NATIVE_LDFLAGS) - - -lish - - -lish_emu - - -lfakefs - - -lz - - -liconv - - -lsqlite3 info: path: Support/Info.plist properties: @@ -193,15 +128,6 @@ targets: - arm64 UIApplicationSupportsIndirectInputEvents: true ITSAppUsesNonExemptEncryption: false - # The guest makes its own DNS and socket calls in this process, and a - # home router's resolver is a *local* address. iOS refuses connections - # to those unless the app declares why -- without this key the user - # never even gets the prompt, and the failure shows up inside the guest - # as "DNS: transient error" with a perfectly correct resolv.conf. - NSLocalNetworkUsageDescription: >- - XForge runs a Linux guest. Programs in it (apk, wget, git) make their - own network connections, including to servers on your local network such - as your router's DNS resolver. # The app builds, stores and exports .ipa artifacts in its Documents # directory; make that visible in the Files app. UIFileSharingEnabled: true